Adding/fixing aggregation rules

.gitignore

@@ -1,3 +1,4 @@
 target
 .classpath
 .project
+/.settings/

src/main/java/org/gcube/accounting/datamodel/validations/validators/Harmonizer.java

@@ -49,11 +49,11 @@ public class Harmonizer implements FieldAction {
 		List<MatcherReplace> matcherReplaceList = RegexRulesAggregator.getInstance().getMatcherReplaceList();
 		
 		for(MatcherReplace matcherReplace : matcherReplaceList) {
-			boolean matched =  matcherReplace.check(serviceClass, serviceName, (String) value);
+			Replace replace = matcherReplace.check(serviceClass, serviceName, (String) value);
-			if(matched) {
+			if(replace != null) {
-				serviceUsageRecord.setServiceClass(matcherReplace.getReplacer().getServiceClass());
+				serviceUsageRecord.setServiceClass(replace.getServiceClass());
-				serviceUsageRecord.setServiceName(matcherReplace.getReplacer().getServiceName());
+				serviceUsageRecord.setServiceName(replace.getServiceName());
-				return matcherReplace.getReplacer().getCalledMethod();
+				return replace.getCalledMethod();
 			}
 		}
 		

src/main/java/org/gcube/accounting/datamodel/validations/validators/MatcherReplace.java

@@ -1,5 +1,7 @@
 package org.gcube.accounting.datamodel.validations.validators;
 
+import java.util.regex.Matcher;
+
 import org.gcube.documentstore.exception.InvalidValueException;
 import org.gcube.documentstore.records.DSMapper;
 
@@ -8,7 +10,7 @@ import com.fasterxml.jackson.annotation.JsonSetter;
 public class MatcherReplace {
 
 	protected MultiMatcher multiMatcher;
-	protected Replacer replacer;
+	protected Replace replacementRegex;
 	
 	public MatcherReplace() {}
 	
@@ -21,17 +23,31 @@ public class MatcherReplace {
 		this.multiMatcher = multiMatcher;
 	}
 
-	public Replacer getReplacer() {
+	protected Replace getReplacementRegex() {
-		return replacer;
+		return replacementRegex;
 	}
 
 	@JsonSetter(value="replace")
-	public void setReplacer(Replacer replacer) {
+	public void setReplacementRegex(Replace replacementRegex) {
-		this.replacer = replacer;
+		this.replacementRegex = replacementRegex;
 	}
 
-	public boolean check(String serviceClass, String serviceName, String calledMethod) throws InvalidValueException {
+	public Replace check(String serviceClass, String serviceName, String calledMethod) throws InvalidValueException {
-		return multiMatcher.match(serviceClass, serviceName, calledMethod);
+		boolean matched = multiMatcher.match(serviceClass, serviceName, calledMethod);
+		Replace replace;
+		if(matched) {
+			replace = new Replace();
+			Matcher serviceClassMatcher = multiMatcher.getServiceClassPattern().matcher(serviceClass);
+			replace.setServiceClass(serviceClassMatcher.replaceFirst(replacementRegex.getServiceClass()));
+			Matcher serviceNameMatcher = multiMatcher.getServiceClassPattern().matcher(serviceName);
+			replace.setServiceName(serviceNameMatcher.replaceFirst(replacementRegex.getServiceName()));
+			Matcher calledMethodMatcher = multiMatcher.getCalledMethodPattern().matcher(calledMethod);
+			String cm = calledMethodMatcher.replaceFirst(replacementRegex.getCalledMethod());
+			replace.setCalledMethod(cm);
+		}else {
+			replace = null;
+		}
+		return replace;
 	}
 
 	@Override
@@ -39,7 +55,7 @@ public class MatcherReplace {
 		try {
 			return DSMapper.getObjectMapper().writeValueAsString(this);
 		} catch(Exception e) {
-			return "MatcherReplace [multiMatcher=" + multiMatcher + ", replacer=" + replacer + "]";
+			return "MatcherReplace [multiMatcher=" + multiMatcher + ", replacer=" + replacementRegex + "]";
 		}
 	}
 	

src/main/java/org/gcube/accounting/datamodel/validations/validators/MultiMatcher.java

@@ -70,10 +70,16 @@ public class MultiMatcher {
 				+ ", calledMethodRegex=" + calledMethodRegex + "]";
 	}
 
-	
+	public Pattern getServiceClassPattern() {
+		return serviceClassPattern;
+	}
 
-	
+	public Pattern getServiceNamePattern() {
+		return serviceNamePattern;
+	}
 
-	
+	public Pattern getCalledMethodPattern() {
+		return calledMethodPattern;
+	}
 	
 }

src/main/java/org/gcube/accounting/datamodel/validations/validators/Replace.java

@@ -1,16 +1,16 @@
 package org.gcube.accounting.datamodel.validations.validators;
 
-public class Replacer {
+public class Replace {
 	
 	protected String serviceClass;
 	protected String serviceName; 
 	protected String calledMethod;
-
+	
-	public Replacer() {
+	public Replace() {
 		
 	}
 	
-	public Replacer(String serviceClass, String serviceName, String calledMethod) {
+	public Replace(String serviceClass, String serviceName, String calledMethod) {
 		this.serviceClass = serviceClass;
 		this.serviceName = serviceName;
 		this.calledMethod = calledMethod;

src/test/java/org/gcube/accounting/datamodel/validations/validators/TestRules.java

@@ -5,6 +5,11 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.FilenameFilter;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import org.gcube.accounting.aggregator.RegexRulesAggregator;
 import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
@@ -20,34 +25,10 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-public class TestRules {
+public class TestRules extends ContextTest {
 	
 	private static final Logger logger = LoggerFactory.getLogger(TestRules.class);
 	
-	@Test
-	public void testAllRules() throws Exception {
-
-		File rulesDirectory = getRulesDirectory();
-		FilenameFilter filenameFilter = new FilenameFilter() {
-			@Override
-			public boolean accept(File dir, String name) {
-				return name.endsWith(".json");
-			}
-		};
-		File[] rulesFiles = rulesDirectory.listFiles(filenameFilter);
-		RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
-		
-		for(File rulesFile : rulesFiles) {
-			ObjectMapper mapper = DSMapper.getObjectMapper();
-			MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
-			regexRulesAggregator.addMatcherReplace(matcherReplace);
-		}
-		
-		for(File rulesFile : rulesFiles) {
-			testRule(rulesFile);
-		}
-	}
-	
 	public File getRulesDirectory() throws Exception {
 		URL logbackFileURL = TestRules.class.getClassLoader().getResource("logback-test.xml");
 		File logbackFile = new File(logbackFileURL.toURI());
@@ -55,13 +36,11 @@ public class TestRules {
 		return new File(resourcesDirectory, "rules");
 	}
 	
-	protected long durationWeightedAverage(int numberA, long durationA, int numberB, long durationB){
+	protected long durationWeightedAverage(int numberA, long durationA, int numberB, long durationB) {
 		return ((numberA * durationA) + (numberB * durationB)) / (numberA + numberB);
 	}
 	
-	
+	public List<File> allRules() throws Exception {
-	public void initAllRules() throws Exception {
-		RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
 		File rulesDirectory = getRulesDirectory();
 		FilenameFilter filenameFilter = new FilenameFilter() {
 			@Override
@@ -69,53 +48,76 @@ public class TestRules {
 				return name.endsWith(".json");
 			}
 		};
-		File[] rulesFiles = rulesDirectory.listFiles(filenameFilter);
 		
-		for(File rulesFile : rulesFiles) {
+		List<File> rulesFiles = new ArrayList<>();
-			ObjectMapper mapper = DSMapper.getObjectMapper();
+		for(File dir : rulesDirectory.listFiles()) {
-			MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
+			if(!dir.isDirectory()) {
-			regexRulesAggregator.addMatcherReplace(matcherReplace);
+				continue;
+			}
+			rulesFiles.addAll(Arrays.asList(dir.listFiles(filenameFilter)));
+			RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance();
+			for(File rulesFile : rulesFiles) {
+				ObjectMapper mapper = DSMapper.getObjectMapper();
+				MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
+				regexRulesAggregator.addMatcherReplace(matcherReplace);
+			}
+			
 		}
+		
+		return rulesFiles;
 	}
 	
-	
 	public void testRule(File rulesFile) throws Exception {
-		logger.info("-----------------------------------------------------------------------------------------------------");
+		logger.info(
+				"-----------------------------------------------------------------------------------------------------");
 		logger.info("Analisyng rule from file {}\n", rulesFile.getAbsolutePath());
 		File rulesDirectory = rulesFile.getParentFile();
 		
 		ObjectMapper mapper = DSMapper.getObjectMapper();
 		MatcherReplace matcherReplace = mapper.readValue(rulesFile, MatcherReplace.class);
-		Replacer replacer = matcherReplace.getReplacer();
+		Replace replacementRegex = matcherReplace.getReplacementRegex();
 		
-		ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord();
+		Map<String,AggregatedServiceUsageRecord> aggregatedMap = new HashMap<>();
-		serviceUsageRecord.setServiceClass(replacer.getServiceClass());
-		serviceUsageRecord.setServiceName(replacer.getServiceName());
-		serviceUsageRecord.setCalledMethod(replacer.getCalledMethod());
-		serviceUsageRecord.validate();
 		
-		AggregatedServiceUsageRecord aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord);
+		AggregatedServiceUsageRecord aggregated;
+		
+		// it indicates that the rules can create multiple calledMethod because the replacement is based on
+		// Named Capture Group  
 		
 		final String requiredMatchesFileName = rulesFile.getName().replaceAll(".json", ".csv");
-		File elaborationFile = new File(rulesDirectory,requiredMatchesFileName);
+		File elaborationFile = new File(rulesDirectory, requiredMatchesFileName);
-		int count = 1;
 		try(BufferedReader br = new BufferedReader(new FileReader(elaborationFile))) {
 			for(String line; (line = br.readLine()) != null;) {
 				String[] splittedLine = line.split(",");
-				++count;
 				
 				String serviceClass = splittedLine[0];
 				String serviceName = splittedLine[1];
 				String calledMethod = splittedLine[2];
 				
-				boolean matched = matcherReplace.check(serviceClass,serviceName,calledMethod);
+				Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
-				if(matched) {
+				if(replace != null) {
-					logger.info("{} --> {},{},{}", line, replacer.getServiceClass(), replacer.getServiceName(), replacer.getCalledMethod());
+					logger.info("{} --> {},{},{}", line, replace.getServiceClass(), replace.getServiceName(),
+							replace.getCalledMethod());
 				} else {
-					logger.error("{} does not match {}. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString());
+					logger.error("{} does not match {}. This MUST not occur.", line,
+							matcherReplace.getMultiMatcher().toString());
 					throw new Exception();
 				}
 				
+				String obtainedTriple = replace.getServiceClass() + "+++" + replace.getServiceName() + "+++"
+						+ replace.getCalledMethod();
+				if(aggregatedMap.containsKey(obtainedTriple)) {
+					aggregated = aggregatedMap.get(obtainedTriple);
+				} else {
+					ServiceUsageRecord serviceUsageRecord = TestUsageRecord.createTestServiceUsageRecord();
+					serviceUsageRecord.setServiceClass(replace.getServiceClass());
+					serviceUsageRecord.setServiceName(replace.getServiceName());
+					serviceUsageRecord.setCalledMethod(replace.getCalledMethod());
+					serviceUsageRecord.validate();
+					aggregated = new AggregatedServiceUsageRecord(serviceUsageRecord);
+					aggregatedMap.put(obtainedTriple, aggregated);
+				}
+				
 				ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
 				sur.setServiceClass(serviceClass);
 				sur.setServiceName(serviceName);
@@ -123,27 +125,29 @@ public class TestRules {
 				sur.validate();
 				// logger.trace("To Be aggregated ServiceUsageRecord {}", sur);
 				
-				long minInvocationTime = aggregated.getMinInvocationTime();
+				long oldMinInvocationTime = aggregated.getMinInvocationTime();
-				long maxInvocationTime = aggregated.getMaxInvocationTime();
+				long oldMaxInvocationTime = aggregated.getMaxInvocationTime();
+				int oldOperationCount = aggregated.getOperationCount();
 				long oldDuration = aggregated.getDuration();
+				
 				long surDuration = sur.getDuration();
 				
 				aggregated.aggregate(sur);
 				//logger.debug("Resulting Aggregated ServiceUsageRecord: {}", aggregated);
 				
-				long avgDuration = durationWeightedAverage(count-1, oldDuration, 1, surDuration);
+				long avgDuration = durationWeightedAverage(oldOperationCount, oldDuration, 1, surDuration);
 				Assert.assertTrue(aggregated.getDuration() == (avgDuration));
-				Assert.assertTrue(aggregated.getOperationCount() == count);
+				Assert.assertTrue(aggregated.getOperationCount() == ++oldOperationCount);
 				
-				if(minInvocationTime >= surDuration){
+				if(oldMinInvocationTime >= surDuration) {
 					Assert.assertTrue(aggregated.getMinInvocationTime() == surDuration);
-				}else{
+				} else {
-					Assert.assertTrue(aggregated.getMinInvocationTime() == minInvocationTime);
+					Assert.assertTrue(aggregated.getMinInvocationTime() == oldMinInvocationTime);
 				}
 				
-				if(maxInvocationTime >= surDuration){
+				if(oldMaxInvocationTime >= surDuration) {
-					Assert.assertTrue(aggregated.getMaxInvocationTime() == maxInvocationTime);
+					Assert.assertTrue(aggregated.getMaxInvocationTime() == oldMaxInvocationTime);
-				}else{
+				} else {
 					Assert.assertTrue(aggregated.getMaxInvocationTime() == surDuration);
 				}
 				
@@ -152,61 +156,79 @@ public class TestRules {
 			throw e;
 		}
 		
-		
 		FilenameFilter filenameFilter = new FilenameFilter() {
 			@Override
 			public boolean accept(File dir, String name) {
 				boolean accept = name.endsWith(".csv");
-				return name.compareTo(requiredMatchesFileName)!=0 && accept;
+				return name.compareTo(requiredMatchesFileName) != 0 && accept;
 			}
 		};
-		File[] elaborationFilesNoMatch = rulesDirectory.listFiles(filenameFilter);
 		
-		for(File elaborationFileNoMatch : elaborationFilesNoMatch) {
+		for(File dir : rulesDirectory.listFiles()) {
-			logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName());
+			if(!dir.isDirectory()) {
-			try(BufferedReader br = new BufferedReader(new FileReader(elaborationFileNoMatch))) {
+				continue;
-				for(String line; (line = br.readLine()) != null;) {
+			}
-					String[] splittedLine = line.split(",");
+			File[] elaborationFilesNoMatch = dir.listFiles(filenameFilter);
-					String serviceClass = splittedLine[0];
+			
-					String serviceName = splittedLine[1];
+			for(File elaborationFileNoMatch : elaborationFilesNoMatch) {
-					String calledMethod = splittedLine[2];
+				logger.info("Comparing examples which must not match from file {}", elaborationFileNoMatch.getName());
-					boolean matched = matcherReplace.check(serviceClass,serviceName,calledMethod);
+				try(BufferedReader br = new BufferedReader(new FileReader(elaborationFileNoMatch))) {
-					if(matched) {
+					for(String line; (line = br.readLine()) != null;) {
-						logger.error("{} match {} but it should NOT. This MUST not occur.", line, matcherReplace.getMultiMatcher().toString());
+						String[] splittedLine = line.split(",");
-						throw new Exception();
+						String serviceClass = splittedLine[0];
-					} else {
+						String serviceName = splittedLine[1];
-						logger.trace("{} does NOT match as requested", line, replacer.getServiceClass(), replacer.getServiceName(), replacer.getCalledMethod());
+						String calledMethod = splittedLine[2];
-					}
+						Replace replace = matcherReplace.check(serviceClass, serviceName, calledMethod);
-					
+						if(replace != null) {
-					
+							logger.error("{} match {} but it should NOT. This MUST not occur.", line,
-					ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
+									matcherReplace.getMultiMatcher().toString());
-					sur.setServiceClass(serviceClass);
+							throw new Exception();
-					sur.setServiceName(serviceName);
+						} else {
-					sur.setCalledMethod(calledMethod);
+							logger.trace("{} does NOT match as requested", line, replacementRegex.getServiceClass(),
-					sur.validate();
+									replacementRegex.getServiceName(), replacementRegex.getCalledMethod());
-					//logger.trace("Should not be aggregated ServiceUsageRecord {}", sur);
+						}
-					
+						
-					try {
+						ServiceUsageRecord sur = TestUsageRecord.createTestServiceUsageRecord();
-						aggregated.aggregate(sur);
+						sur.setServiceClass(serviceClass);
-						throw new Exception("The record has been aggregated and it should NOT");
+						sur.setServiceName(serviceName);
-					}catch (NotAggregatableRecordsExceptions e) {
+						sur.setCalledMethod(calledMethod);
-						//logger.trace("{} is not aggragable as expected", sur);
+						sur.validate();
+						//logger.trace("Should not be aggregated ServiceUsageRecord {}", sur);
+						for(AggregatedServiceUsageRecord asur : aggregatedMap.values()) {
+							try {
+								asur.aggregate(sur);
+								throw new Exception("The record has been aggregated and it should NOT");
+								
+							} catch(NotAggregatableRecordsExceptions e) {
+								//logger.trace("{} is not aggragable as expected", sur);
+							}
+						}
 					}
+				} catch(Exception e) {
+					throw e;
 				}
-			} catch(Exception e) {
-				throw e;
 			}
 		}
-		logger.info("-----------------------------------------------------------------------------------------------------\n\n\n");
+		logger.info(
+				"-----------------------------------------------------------------------------------------------------\n\n\n");
 	}
 	
 	@Test
 	public void testSingleRule() throws Exception {
-		ContextTest.setContextByName(ContextTest.DEFAULT_TEST_SCOPE);
+		allRules();
-		initAllRules();
 		File rulesDirectory = getRulesDirectory();
-		File rulesFile = new File(rulesDirectory, "geoanalytics-gos-ShapeManagement.json");
+		File rulesDirFile = new File(rulesDirectory, "Thredds");
+		File rulesFile = new File(rulesDirFile, "Thredds_METADATA::OTHER.json");
 		testRule(rulesFile);
 	}
 	
+	@Test
+	public void testAllRules() throws Exception {
+		List<File> rulesFiles = allRules();
+		
+		for(File rulesFile : rulesFiles) {
+			testRule(rulesFile);
+		}
+	}
+	
 }

src/test/resources/rules/Thredds/Thredds_METADATA::COLLECTION.json

@@ -0,0 +1,12 @@
+{
+	"match": {
+		"serviceClassRegex": "SDI",
+		"serviceNameRegex": "Thredds",
+		"calledMethodRegex": "^(\/){1}(?<Protocol>[a-zA-Z0-9]*){1}(\/public\/netcdf\/){1}(?<Collection>[a-zA-Z0-9_-]*)(\/){1}(.*\\.(nc|asc|tiff)).{1,}"
+	},
+	"replace": {
+		"serviceClass": "SDI",
+		"serviceName": "Thredds",
+		"calledMethod": "${Protocol}::${Collection}"
+	}
+}