Adding/fixing aggregation rules

This commit is contained in:
Luca Frosini 2020-01-07 11:07:06 +01:00
parent 97d3b55322
commit 5f82f611c3
117 changed files with 59982 additions and 118 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
target target
.classpath .classpath
.project .project
/.settings/

View File

@ -49,11 +49,11 @@ public class Harmonizer implements FieldAction {
List<MatcherReplace> matcherReplaceList = RegexRulesAggregator.getInstance().getMatcherReplaceList(); List<MatcherReplace> matcherReplaceList = RegexRulesAggregator.getInstance().getMatcherReplaceList();
for(MatcherReplace matcherReplace : matcherReplaceList) { for(MatcherReplace matcherReplace : matcherReplaceList) {
boolean matched = matcherReplace.check(serviceClass, serviceName, (String) value); Replace replace = matcherReplace.check(serviceClass, serviceName, (String) value);
if(matched) { if(replace != null) {
serviceUsageRecord.setServiceClass(matcherReplace.getReplacer().getServiceClass()); serviceUsageRecord.setServiceClass(replace.getServiceClass());
serviceUsageRecord.setServiceName(matcherReplace.getReplacer().getServiceName()); serviceUsageRecord.setServiceName(replace.getServiceName());
return matcherReplace.getReplacer().getCalledMethod(); return replace.getCalledMethod();
} }
} }

View File

@ -1,5 +1,7 @@
package org.gcube.accounting.datamodel.validations.validators; package org.gcube.accounting.datamodel.validations.validators;
import java.util.regex.Matcher;
import org.gcube.documentstore.exception.InvalidValueException; import org.gcube.documentstore.exception.InvalidValueException;
import org.gcube.documentstore.records.DSMapper; import org.gcube.documentstore.records.DSMapper;
@ -8,7 +10,7 @@ import com.fasterxml.jackson.annotation.JsonSetter;
public class MatcherReplace { public class MatcherReplace {
protected MultiMatcher multiMatcher; protected MultiMatcher multiMatcher;
protected Replacer replacer; protected Replace replacementRegex;
public MatcherReplace() {} public MatcherReplace() {}
@ -21,17 +23,31 @@ public class MatcherReplace {
this.multiMatcher = multiMatcher; this.multiMatcher = multiMatcher;
} }
public Replacer getReplacer() { protected Replace getReplacementRegex() {
return replacer; return replacementRegex;
} }
@JsonSetter(value="replace") @JsonSetter(value="replace")
public void setReplacer(Replacer replacer) { public void setReplacementRegex(Replace replacementRegex) {
this.replacer = replacer; this.replacementRegex = replacementRegex;
} }
public boolean check(String serviceClass, String serviceName, String calledMethod) throws InvalidValueException { public Replace check(String serviceClass, String serviceName, String calledMethod) throws InvalidValueException {
return multiMatcher.match(serviceClass, serviceName, calledMethod); boolean matched = multiMatcher.match(serviceClass, serviceName, calledMethod);
Replace replace;
if(matched) {
replace = new Replace();
Matcher serviceClassMatcher = multiMatcher.getServiceClassPattern().matcher(serviceClass);
replace.setServiceClass(serviceClassMatcher.replaceFirst(replacementRegex.getServiceClass()));
Matcher serviceNameMatcher = multiMatcher.getServiceClassPattern().matcher(serviceName);
replace.setServiceName(serviceNameMatcher.replaceFirst(replacementRegex.getServiceName()));
Matcher calledMethodMatcher = multiMatcher.getCalledMethodPattern().matcher(calledMethod);
String cm = calledMethodMatcher.replaceFirst(replacementRegex.getCalledMethod());
replace.setCalledMethod(cm);
}else {
replace = null;
}
return replace;
} }
@Override @Override
@ -39,7 +55,7 @@ public class MatcherReplace {
try { try {
return DSMapper.getObjectMapper().writeValueAsString(this); return DSMapper.getObjectMapper().writeValueAsString(this);
} catch(Exception e) { } catch(Exception e) {
return "MatcherReplace [multiMatcher=" + multiMatcher + ", replacer=" + replacer + "]"; return "MatcherReplace [multiMatcher=" + multiMatcher + ", replacer=" + replacementRegex + "]";
} }
} }

View File

@ -70,10 +70,16 @@ public class MultiMatcher {
+ ", calledMethodRegex=" + calledMethodRegex + "]"; + ", calledMethodRegex=" + calledMethodRegex + "]";
} }
public Pattern getServiceClassPattern() {
return serviceClassPattern;
}
public Pattern getServiceNamePattern() {
return serviceNamePattern;
}
public Pattern getCalledMethodPattern() {
return calledMethodPattern;
}
} }

View File

@ -1,16 +1,16 @@
package org.gcube.accounting.datamodel.validations.validators; package org.gcube.accounting.datamodel.validations.validators;
public class Replacer { public class Replace {
protected String serviceClass; protected String serviceClass;
protected String serviceName; protected String serviceName;
protected String calledMethod; protected String calledMethod;
public Replacer() { public Replace() {
} }
public Replacer(String serviceClass, String serviceName, String calledMethod) { public Replace(String serviceClass, String serviceName, String calledMethod) {
this.serviceClass = serviceClass; this.serviceClass = serviceClass;
this.serviceName = serviceName; this.serviceName = serviceName;
this.calledMethod = calledMethod; this.calledMethod = calledMethod;

View File

@ -5,6 +5,11 @@ import java.io.File;
import java.io.FileReader; import java.io.FileReader;
import java.io.FilenameFilter; import java.io.FilenameFilter;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.gcube.accounting.aggregator.RegexRulesAggregator; import org.gcube.accounting.aggregator.RegexRulesAggregator;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
@ -20,34 +25,10 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
public class TestRules { public class TestRules extends ContextTest {
private static final Logger logger = LoggerFactory.getLogger(TestRules.class); private static final Logger logger = LoggerFactory.getLogger(TestRules.class);
@Test
public void testAllRules() throws Exception {
File rulesDirectory = getRulesDirectory();
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".json");
}
};
File[] rulesFiles = rulesDirectory.listFiles(filenameFilter);
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
for(File rulesFile : rulesFiles) {
ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
regexRulesAggregator.addMatcherReplace(matcherReplace);
}
for(File rulesFile : rulesFiles) {
testRule(rulesFile);
}
}
public File getRulesDirectory() throws Exception { public File getRulesDirectory() throws Exception {
URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml"); URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml");
File logbackFile = new File(logbackFileURL.toURI()); File logbackFile = new File(logbackFileURL.toURI());
@ -59,9 +40,7 @@ public class TestRules {
return ((numberA * durationA) + (numberB * durationB)) / (numberA + numberB); return ((numberA * durationA) + (numberB * durationB)) / (numberA + numberB);
} }
public List<File> allRules() throws Exception {
public void initAllRules() throws Exception {
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
File rulesDirectory = getRulesDirectory(); File rulesDirectory = getRulesDirectory();
FilenameFilter filenameFilter = new FilenameFilter() { FilenameFilter filenameFilter = new FilenameFilter() {
@Override @Override
@ -69,53 +48,76 @@ public class TestRules {
return name.endsWith(".json"); return name.endsWith(".json");
} }
}; };
File[] rulesFiles = rulesDirectory.listFiles(filenameFilter);
List<File> rulesFiles = new ArrayList<>();
for(File dir : rulesDirectory.listFiles()) {
if(!dir.isDirectory()) {
continue;
}
rulesFiles.addAll(Arrays.asList(dir.listFiles(filenameFilter)));
RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
for(File rulesFile : rulesFiles) { for(File rulesFile : rulesFiles) {
ObjectMapper mapper = DSMapper.getObjectMapper(); ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class); MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
regexRulesAggregator.addMatcherReplace(matcherReplace); regexRulesAggregator.addMatcherReplace(matcherReplace);
} }
} }
return rulesFiles;
}
public void testRule(File rulesFile) throws Exception { public void testRule(File rulesFile) throws Exception {
logger.info("-----------------------------------------------------------------------------------------------------"); logger.info(
"-----------------------------------------------------------------------------------------------------");
logger.info("Analisyng rule from file {}\n", rulesFile.getAbsolutePath()); logger.info("Analisyng rule from file {}\n", rulesFile.getAbsolutePath());
File rulesDirectory = rulesFile.getParentFile(); File rulesDirectory = rulesFile.getParentFile();
ObjectMapper mapper = DSMapper.getObjectMapper(); ObjectMapper mapper = DSMapper.getObjectMapper();
MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class); MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
Replacer replacer = matcherReplace.getReplacer(); Replace replacementRegex = matcherReplace.getReplacementRegex();
ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord(); Map<String,AggregatedServiceUsageRecord> aggregatedMap = new HashMap<>();
serviceUsageRecord.setServiceClass(replacer.getServiceClass());
serviceUsageRecord.setServiceName(replacer.getServiceName());
serviceUsageRecord.setCalledMethod(replacer.getCalledMethod());
serviceUsageRecord.validate();
AggregatedServiceUsageRecord aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord); AggregatedServiceUsageRecord aggregated;
// it indicates that the rules can create multiple calledMethod because the replacement is based on
// Named Capture Group
final String requiredMatchesFileName = rulesFile.getName().replaceAll(".json", ".csv"); final String requiredMatchesFileName = rulesFile.getName().replaceAll(".json", ".csv");
File elaborationFile = new File(rulesDirectory, requiredMatchesFileName); File elaborationFile = new File(rulesDirectory, requiredMatchesFileName);
int count = 1;
try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) { try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) {
for(String line; (line = br.readLine()) != null;) { for(String line; (line = br.readLine()) != null;) {
String[] splittedLine = line.split(","); String[] splittedLine = line.split(",");
++count;
String serviceClass = splittedLine[0]; String serviceClass = splittedLine[0];
String serviceName = splittedLine[1]; String serviceName = splittedLine[1];
String calledMethod = splittedLine[2]; String calledMethod = splittedLine[2];
boolean matched = matcherReplace.check(serviceClass,serviceName,calledMethod); Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
if(matched) { if(replace != null) {
logger.info("{} --> {},{},{}", line, replacer.getServiceClass(), replacer.getServiceName(), replacer.getCalledMethod()); logger.info("{} --> {},{},{}", line, replace.getServiceClass(), replace.getServiceName(),
replace.getCalledMethod());
} else { } else {
logger.error("{} does not match {}. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString()); logger.error("{} does not match {}. This MUST not occur.", line,
matcherReplace.getMultiMatcher().toString());
throw new Exception(); throw new Exception();
} }
String obtainedTriple = replace.getServiceClass() + "+++" + replace.getServiceName() + "+++"
+ replace.getCalledMethod();
if(aggregatedMap.containsKey(obtainedTriple)) {
aggregated = aggregatedMap.get(obtainedTriple);
} else {
ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord();
serviceUsageRecord.setServiceClass(replace.getServiceClass());
serviceUsageRecord.setServiceName(replace.getServiceName());
serviceUsageRecord.setCalledMethod(replace.getCalledMethod());
serviceUsageRecord.validate();
aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord);
aggregatedMap.put(obtainedTriple, aggregated);
}
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord(); ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
sur.setServiceClass(serviceClass); sur.setServiceClass(serviceClass);
sur.setServiceName(serviceName); sur.setServiceName(serviceName);
@ -123,26 +125,28 @@ public class TestRules {
sur.validate(); sur.validate();
// logger.trace("To Be aggregated ServiceUsageRecord {}", sur); // logger.trace("To Be aggregated ServiceUsageRecord {}", sur);
long minInvocationTime = aggregated.getMinInvocationTime(); long oldMinInvocationTime = aggregated.getMinInvocationTime();
long maxInvocationTime = aggregated.getMaxInvocationTime(); long oldMaxInvocationTime = aggregated.getMaxInvocationTime();
int oldOperationCount = aggregated.getOperationCount();
long oldDuration = aggregated.getDuration(); long oldDuration = aggregated.getDuration();
long surDuration = sur.getDuration(); long surDuration = sur.getDuration();
aggregated.aggregate(sur); aggregated.aggregate(sur);
//logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated); //logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated);
long avgDuration = durationWeightedAverage(count-1, oldDuration, 1, surDuration); long avgDuration = durationWeightedAverage(oldOperationCount, oldDuration, 1, surDuration);
Assert.assertTrue(aggregated.getDuration() == (avgDuration)); Assert.assertTrue(aggregated.getDuration() == (avgDuration));
Assert.assertTrue(aggregated.getOperationCount() == count); Assert.assertTrue(aggregated.getOperationCount() == ++oldOperationCount);
if(minInvocationTime >= surDuration){ if(oldMinInvocationTime >= surDuration) {
Assert.assertTrue(aggregated.getMinInvocationTime() == surDuration); Assert.assertTrue(aggregated.getMinInvocationTime() == surDuration);
} else { } else {
Assert.assertTrue(aggregated.getMinInvocationTime() == minInvocationTime); Assert.assertTrue(aggregated.getMinInvocationTime() == oldMinInvocationTime);
} }
if(maxInvocationTime >= surDuration){ if(oldMaxInvocationTime >= surDuration) {
Assert.assertTrue(aggregated.getMaxInvocationTime() == maxInvocationTime); Assert.assertTrue(aggregated.getMaxInvocationTime() == oldMaxInvocationTime);
} else { } else {
Assert.assertTrue(aggregated.getMaxInvocationTime() == surDuration); Assert.assertTrue(aggregated.getMaxInvocationTime() == surDuration);
} }
@ -152,7 +156,6 @@ public class TestRules {
throw e; throw e;
} }
FilenameFilter filenameFilter = new FilenameFilter() { FilenameFilter filenameFilter = new FilenameFilter() {
@Override @Override
public boolean accept(File dir, String name) { public boolean accept(File dir, String name) {
@ -160,7 +163,12 @@ public class TestRules {
return name.compareTo(requiredMatchesFileName) != 0 && accept; return name.compareTo(requiredMatchesFileName) != 0 && accept;
} }
}; };
File[] elaborationFilesNoMatch = rulesDirectory.listFiles(filenameFilter);
for(File dir : rulesDirectory.listFiles()) {
if(!dir.isDirectory()) {
continue;
}
File[] elaborationFilesNoMatch = dir.listFiles(filenameFilter);
for(File elaborationFileNoMatch : elaborationFilesNoMatch) { for(File elaborationFileNoMatch : elaborationFilesNoMatch) {
logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName()); logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName());
@ -170,43 +178,57 @@ public class TestRules {
String serviceClass = splittedLine[0]; String serviceClass = splittedLine[0];
String serviceName = splittedLine[1]; String serviceName = splittedLine[1];
String calledMethod = splittedLine[2]; String calledMethod = splittedLine[2];
boolean matched = matcherReplace.check(serviceClass,serviceName,calledMethod); Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
if(matched) { if(replace != null) {
logger.error("{} match {} but it should NOT. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString()); logger.error("{} match {} but it should NOT. This MUST not occur.", line,
matcherReplace.getMultiMatcher().toString());
throw new Exception(); throw new Exception();
} else { } else {
logger.trace("{} does NOT match as requested", line, replacer.getServiceClass(), replacer.getServiceName(), replacer.getCalledMethod()); logger.trace("{} does NOT match as requested", line, replacementRegex.getServiceClass(),
replacementRegex.getServiceName(), replacementRegex.getCalledMethod());
} }
ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord(); ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
sur.setServiceClass(serviceClass); sur.setServiceClass(serviceClass);
sur.setServiceName(serviceName); sur.setServiceName(serviceName);
sur.setCalledMethod(calledMethod); sur.setCalledMethod(calledMethod);
sur.validate(); sur.validate();
//logger.trace("Should not be aggregated ServiceUsageRecord {}", sur); //logger.trace("Should not be aggregated ServiceUsageRecord {}", sur);
for(AggregatedServiceUsageRecord asur : aggregatedMap.values()) {
try { try {
aggregated.aggregate(sur); asur.aggregate(sur);
throw new Exception("The record has been aggregated and it should NOT"); throw new Exception("The record has been aggregated and it should NOT");
} catch(NotAggregatableRecordsExceptions e) { } catch(NotAggregatableRecordsExceptions e) {
//logger.trace("{} is not aggragable as expected", sur); //logger.trace("{} is not aggragable as expected", sur);
} }
} }
}
} catch(Exception e) { } catch(Exception e) {
throw e; throw e;
} }
} }
logger.info("-----------------------------------------------------------------------------------------------------\n\n\n"); }
logger.info(
"-----------------------------------------------------------------------------------------------------\n\n\n");
} }
@Test @Test
public void testSingleRule() throws Exception { public void testSingleRule() throws Exception {
ContextTest.setContextByName(ContextTest.DEFAULT_TEST_SCOPE); allRules();
initAllRules();
File rulesDirectory = getRulesDirectory(); File rulesDirectory = getRulesDirectory();
File rulesFile = new File(rulesDirectory, "geoanalytics-gos-ShapeManagement.json"); File rulesDirFile = new File(rulesDirectory, "Thredds");
File rulesFile = new File(rulesDirFile, "Thredds_METADATA::OTHER.json");
testRule(rulesFile); testRule(rulesFile);
} }
@Test
public void testAllRules() throws Exception {
List<File> rulesFiles = allRules();
for(File rulesFile : rulesFiles) {
testRule(rulesFile);
}
}
} }

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,12 @@
{
"match": {
"serviceClassRegex": "SDI",
"serviceNameRegex": "Thredds",
"calledMethodRegex": "^(\/){1}(?<Protocol>[a-zA-Z0-9]*){1}(\/public\/netcdf\/){1}(?<Collection>[a-zA-Z0-9_-]*)(\/){1}(.*\\.(nc|asc|tiff)).{1,}"
},
"replace": {
"serviceClass": "SDI",
"serviceName": "Thredds",
"calledMethod": "${Protocol}::${Collection}"
}
}

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More