Adding/fixing aggregation rules

master
Luca Frosini 4 years ago
parent 97d3b55322
commit 5f82f611c3

1
.gitignore vendored

@ -1,3 +1,4 @@
target
.classpath
.project
/.settings/

@ -49,11 +49,11 @@ public class Harmonizer implements FieldAction {
List<MatcherReplace> matcherReplaceList = RegexRulesAggregator.getInstance().getMatcherReplaceList();
for(MatcherReplace matcherReplace : matcherReplaceList) {
boolean matched = matcherReplace.check(serviceClass, serviceName, (String) value);
if(matched) {
serviceUsageRecord.setServiceClass(matcherReplace.getReplacer().getServiceClass());
serviceUsageRecord.setServiceName(matcherReplace.getReplacer().getServiceName());
return matcherReplace.getReplacer().getCalledMethod();
Replace replace = matcherReplace.check(serviceClass, serviceName, (String) value);
if(replace != null) {
serviceUsageRecord.setServiceClass(replace.getServiceClass());
serviceUsageRecord.setServiceName(replace.getServiceName());
return replace.getCalledMethod();
}
}

@ -1,5 +1,7 @@
package org.gcube.accounting.datamodel.validations.validators;
import java.util.regex.Matcher;
import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.records.DSMapper;
@ -8,7 +10,7 @@ import com.fasterxml.jackson.annotation.JsonSetter;
public class MatcherReplace {
protected MultiMatcher multiMatcher;
protected Replacer replacer;
protected Replace replacementRegex;
public MatcherReplace() {}
@ -21,17 +23,31 @@ public class MatcherReplace {
this.multiMatcher = multiMatcher;
}
public Replacer getReplacer() {
return replacer;
protected Replace getReplacementRegex() {
return replacementRegex;
}
@JsonSetter(value="replace")
public void setReplacer(Replacer replacer) {
this.replacer = replacer;
public void setReplacementRegex(Replace replacementRegex) {
this.replacementRegex = replacementRegex;
}
public boolean check(String serviceClass, String serviceName, String calledMethod) throws InvalidValueException {
return multiMatcher.match(serviceClass, serviceName, calledMethod);
public Replace check(String serviceClass, String serviceName, String calledMethod) throws InvalidValueException {
boolean matched = multiMatcher.match(serviceClass, serviceName, calledMethod);
Replace replace;
if(matched) {
replace = new Replace();
Matcher serviceClassMatcher = multiMatcher.getServiceClassPattern().matcher(serviceClass);
replace.setServiceClass(serviceClassMatcher.replaceFirst(replacementRegex.getServiceClass()));
Matcher serviceNameMatcher = multiMatcher.getServiceClassPattern().matcher(serviceName);
replace.setServiceName(serviceNameMatcher.replaceFirst(replacementRegex.getServiceName()));
Matcher calledMethodMatcher = multiMatcher.getCalledMethodPattern().matcher(calledMethod);
String cm = calledMethodMatcher.replaceFirst(replacementRegex.getCalledMethod());
replace.setCalledMethod(cm);
}else {
replace = null;
}
return replace;
}
@Override
@ -39,7 +55,7 @@ public class MatcherReplace {
try {
return DSMapper.getObjectMapper().writeValueAsString(this);
} catch(Exception e) {
return "MatcherReplace [multiMatcher=" + multiMatcher + ", replacer=" + replacer + "]";
return "MatcherReplace [multiMatcher=" + multiMatcher + ", replacer=" + replacementRegex + "]";
}
}

@ -70,10 +70,16 @@ public class MultiMatcher {
+ ", calledMethodRegex=" + calledMethodRegex + "]";
}
public Pattern getServiceClassPattern() {
return serviceClassPattern;
}
public Pattern getServiceNamePattern() {
return serviceNamePattern;
}
public Pattern getCalledMethodPattern() {
return calledMethodPattern;
}
}

@ -1,16 +1,16 @@
package org.gcube.accounting.datamodel.validations.validators;
public class Replacer {
public class Replace {
protected String serviceClass;
protected String serviceName;
protected String calledMethod;
public Replacer() {
public Replace() {
}
public Replacer(String serviceClass, String serviceName, String calledMethod) {
public Replace(String serviceClass, String serviceName, String calledMethod) {
this.serviceClass = serviceClass;
this.serviceName = serviceName;
this.calledMethod = calledMethod;

@ -5,6 +5,11 @@ import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.gcube.accounting.aggregator.RegexRulesAggregator;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
@ -20,34 +25,10 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
public class TestRules {
public class TestRules extends ContextTest {
private static final Logger logger = LoggerFactory.getLogger(TestRules.class);
@Test
public void testAllRules() throws Exception {
File rulesDirectory = getRulesDirectory();
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".json");
}
};
File[] rulesFiles = rulesDirectory.listFiles(filenameFilter);
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
for(File rulesFile : rulesFiles) {
ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
regexRulesAggregator.addMatcherReplace(matcherReplace);
}
for(File rulesFile : rulesFiles) {
testRule(rulesFile);
}
}
public File getRulesDirectory() throws Exception {
URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml");
File logbackFile = new File(logbackFileURL.toURI());
@ -55,13 +36,11 @@ public class TestRules {
return new File(resourcesDirectory, "rules");
}
protected long durationWeightedAverage(int numberA, long durationA, int numberB, long durationB){
protected long durationWeightedAverage(int numberA, long durationA, int numberB, long durationB) {
return ((numberA * durationA) + (numberB * durationB)) / (numberA + numberB);
}
public void initAllRules() throws Exception {
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
public List<File> allRules() throws Exception {
File rulesDirectory = getRulesDirectory();
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
@ -69,53 +48,76 @@ public class TestRules {
return name.endsWith(".json");
}
};
File[] rulesFiles = rulesDirectory.listFiles(filenameFilter);
for(File rulesFile : rulesFiles) {
ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
regexRulesAggregator.addMatcherReplace(matcherReplace);
List<File> rulesFiles = new ArrayList<>();
for(File dir : rulesDirectory.listFiles()) {
if(!dir.isDirectory()) {
continue;
}
rulesFiles.addAll(Arrays.asList(dir.listFiles(filenameFilter)));
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
for(File rulesFile : rulesFiles) {
ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
regexRulesAggregator.addMatcherReplace(matcherReplace);
}
}
return rulesFiles;
}
public void testRule(File rulesFile) throws Exception {
logger.info("-----------------------------------------------------------------------------------------------------");
logger.info(
"-----------------------------------------------------------------------------------------------------");
logger.info("Analisyng rule from file {}\n", rulesFile.getAbsolutePath());
File rulesDirectory = rulesFile.getParentFile();
ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
Replacer replacer = matcherReplace.getReplacer();
Replace replacementRegex = matcherReplace.getReplacementRegex();
ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord();
serviceUsageRecord.setServiceClass(replacer.getServiceClass());
serviceUsageRecord.setServiceName(replacer.getServiceName());
serviceUsageRecord.setCalledMethod(replacer.getCalledMethod());
serviceUsageRecord.validate();
Map<String,AggregatedServiceUsageRecord> aggregatedMap = new HashMap<>();
AggregatedServiceUsageRecord aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord);
AggregatedServiceUsageRecord aggregated;
// it indicates that the rules can create multiple calledMethod because the replacement is based on
// Named Capture Group
final String requiredMatchesFileName = rulesFile.getName().replaceAll(".json", ".csv");
File elaborationFile = new File(rulesDirectory,requiredMatchesFileName);
int count = 1;
File elaborationFile = new File(rulesDirectory, requiredMatchesFileName);
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) {
for(String line; (line = br.readLine()) != null;) {
String[] splittedLine = line.split(",");
++count;
String serviceClass = splittedLine[0];
String serviceName = splittedLine[1];
String calledMethod = splittedLine[2];
boolean matched = matcherReplace.check(serviceClass,serviceName,calledMethod);
if(matched) {
logger.info("{} --> {},{},{}", line, replacer.getServiceClass(), replacer.getServiceName(), replacer.getCalledMethod());
Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
if(replace != null) {
logger.info("{} --> {},{},{}", line, replace.getServiceClass(), replace.getServiceName(),
replace.getCalledMethod());
} else {
logger.error("{} does not match {}. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString());
logger.error("{} does not match {}. This MUST not occur.", line,
matcherReplace.getMultiMatcher().toString());
throw new Exception();
}
String obtainedTriple = replace.getServiceClass() + "+++" + replace.getServiceName() + "+++"
+ replace.getCalledMethod();
if(aggregatedMap.containsKey(obtainedTriple)) {
aggregated = aggregatedMap.get(obtainedTriple);
} else {
ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord();
serviceUsageRecord.setServiceClass(replace.getServiceClass());
serviceUsageRecord.setServiceName(replace.getServiceName());
serviceUsageRecord.setCalledMethod(replace.getCalledMethod());
serviceUsageRecord.validate();
aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord);
aggregatedMap.put(obtainedTriple, aggregated);
}
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
sur.setServiceClass(serviceClass);
sur.setServiceName(serviceName);
@ -123,27 +125,29 @@ public class TestRules {
sur.validate();
// logger.trace("To Be aggregated ServiceUsageRecord {}", sur);
long minInvocationTime = aggregated.getMinInvocationTime();
long maxInvocationTime = aggregated.getMaxInvocationTime();
long oldMinInvocationTime = aggregated.getMinInvocationTime();
long oldMaxInvocationTime = aggregated.getMaxInvocationTime();
int oldOperationCount = aggregated.getOperationCount();
long oldDuration = aggregated.getDuration();
long surDuration = sur.getDuration();
aggregated.aggregate(sur);
//logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated);
long avgDuration = durationWeightedAverage(count-1, oldDuration, 1, surDuration);
long avgDuration = durationWeightedAverage(oldOperationCount, oldDuration, 1, surDuration);
Assert.assertTrue(aggregated.getDuration() == (avgDuration));
Assert.assertTrue(aggregated.getOperationCount() == count);
Assert.assertTrue(aggregated.getOperationCount() == ++oldOperationCount);
if(minInvocationTime >= surDuration){
if(oldMinInvocationTime >= surDuration) {
Assert.assertTrue(aggregated.getMinInvocationTime() == surDuration);
}else{
Assert.assertTrue(aggregated.getMinInvocationTime() == minInvocationTime);
} else {
Assert.assertTrue(aggregated.getMinInvocationTime() == oldMinInvocationTime);
}
if(maxInvocationTime >= surDuration){
Assert.assertTrue(aggregated.getMaxInvocationTime() == maxInvocationTime);
}else{
if(oldMaxInvocationTime >= surDuration) {
Assert.assertTrue(aggregated.getMaxInvocationTime() == oldMaxInvocationTime);
} else {
Assert.assertTrue(aggregated.getMaxInvocationTime() == surDuration);
}
@ -152,61 +156,79 @@ public class TestRules {
throw e;
}
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
boolean accept = name.endsWith(".csv");
return name.compareTo(requiredMatchesFileName)!=0 && accept;
return name.compareTo(requiredMatchesFileName) != 0 && accept;
}
};
File[] elaborationFilesNoMatch = rulesDirectory.listFiles(filenameFilter);
for(File elaborationFileNoMatch : elaborationFilesNoMatch) {
logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName());
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFileNoMatch))) {
for(String line; (line = br.readLine()) != null;) {
String[] splittedLine = line.split(",");
String serviceClass = splittedLine[0];
String serviceName = splittedLine[1];
String calledMethod = splittedLine[2];
boolean matched = matcherReplace.check(serviceClass,serviceName,calledMethod);
if(matched) {
logger.error("{} match {} but it should NOT. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString());
throw new Exception();
} else {
logger.trace("{} does NOT match as requested", line, replacer.getServiceClass(), replacer.getServiceName(), replacer.getCalledMethod());
}
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
sur.setServiceClass(serviceClass);
sur.setServiceName(serviceName);
sur.setCalledMethod(calledMethod);
sur.validate();
//logger.trace("Should not be aggregated ServiceUsageRecord {}", sur);
try {
aggregated.aggregate(sur);
throw new Exception("The record has been aggregated and it should NOT");
}catch (NotAggregatableRecordsExceptions e) {
//logger.trace("{} is not aggragable as expected", sur);
for(File dir : rulesDirectory.listFiles()) {
if(!dir.isDirectory()) {
continue;
}
File[] elaborationFilesNoMatch = dir.listFiles(filenameFilter);
for(File elaborationFileNoMatch : elaborationFilesNoMatch) {
logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName());
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFileNoMatch))) {
for(String line; (line = br.readLine()) != null;) {
String[] splittedLine = line.split(",");
String serviceClass = splittedLine[0];
String serviceName = splittedLine[1];
String calledMethod = splittedLine[2];
Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
if(replace != null) {
logger.error("{} match {} but it should NOT. This MUST not occur.", line,
matcherReplace.getMultiMatcher().toString());
throw new Exception();
} else {
logger.trace("{} does NOT match as requested", line, replacementRegex.getServiceClass(),
replacementRegex.getServiceName(), replacementRegex.getCalledMethod());
}
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
sur.setServiceClass(serviceClass);
sur.setServiceName(serviceName);
sur.setCalledMethod(calledMethod);
sur.validate();
//logger.trace("Should not be aggregated ServiceUsageRecord {}", sur);
for(AggregatedServiceUsageRecord asur : aggregatedMap.values()) {
try {
asur.aggregate(sur);
throw new Exception("The record has been aggregated and it should NOT");
} catch(NotAggregatableRecordsExceptions e) {
//logger.trace("{} is not aggragable as expected", sur);
}
}
}
} catch(Exception e) {
throw e;
}
} catch(Exception e) {
throw e;
}
}
logger.info("-----------------------------------------------------------------------------------------------------\n\n\n");
logger.info(
"-----------------------------------------------------------------------------------------------------\n\n\n");
}
@Test
public void testSingleRule() throws Exception {
ContextTest.setContextByName(ContextTest.DEFAULT_TEST_SCOPE);
initAllRules();
allRules();
File rulesDirectory = getRulesDirectory();
File rulesFile = new File(rulesDirectory, "geoanalytics-gos-ShapeManagement.json");
File rulesDirFile = new File(rulesDirectory, "Thredds");
File rulesFile = new File(rulesDirFile, "Thredds_METADATA::OTHER.json");
testRule(rulesFile);
}
@Test
public void testAllRules() throws Exception {
List<File> rulesFiles = allRules();
for(File rulesFile : rulesFiles) {
testRule(rulesFile);
}
}
}

@ -0,0 +1,12 @@
{
"match": {
"serviceClassRegex": "SDI",
"serviceNameRegex": "Thredds",
"calledMethodRegex": "^(\/){1}(?<Protocol>[a-zA-Z0-9]*){1}(\/public\/netcdf\/){1}(?<Collection>[a-zA-Z0-9_-]*)(\/){1}(.*\\.(nc|asc|tiff)).{1,}"
},
"replace": {
"serviceClass": "SDI",
"serviceName": "Thredds",
"calledMethod": "${Protocol}::${Collection}"
}
}

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save