From 12f3c4eca8deefb9642acb94f1b11179e2eb6b8a Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 20 Feb 2018 17:24:14 +0000 Subject: [PATCH] Refs #11230: Retrieve Accounting Regex rules through Service Endpoint Task-Url: https://support.d4science.org/issues/11230 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-lib@164461 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../aggregator/RegexRulesAggregator.java | 120 ++++++++++++++++++ .../CalledMethodRegexReplaceValidator.java | 20 +-- .../validations/validators/RegexReplace.java | 20 ++- .../AccountingPersistenceConfiguration.java | 9 +- .../accounting/datamodel/UsageRecordTest.java | 1 + .../usagerecords/ServiceUsageRecordTest.java | 29 ++++- 6 files changed, 167 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/gcube/accounting/aggregator/RegexRulesAggregator.java diff --git a/src/main/java/org/gcube/accounting/aggregator/RegexRulesAggregator.java b/src/main/java/org/gcube/accounting/aggregator/RegexRulesAggregator.java new file mode 100644 index 0000000..d4ae8b6 --- /dev/null +++ b/src/main/java/org/gcube/accounting/aggregator/RegexRulesAggregator.java @@ -0,0 +1,120 @@ +package org.gcube.accounting.aggregator; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import org.gcube.accounting.datamodel.validations.validators.RegexReplace; +import org.gcube.accounting.persistence.AccountingPersistenceConfiguration; +import org.gcube.documentstore.persistence.ExecutorUtils; +import org.gcube.documentstore.records.DSMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class RegexRulesAggregator implements Runnable { + + private static final Logger logger = LoggerFactory.getLogger(RegexRulesAggregator.class); + + protected TimeUnit timeUnit = TimeUnit.MINUTES; + protected long delay = TimeUnit.MINUTES.toMinutes(15); + + protected static final String DELAY = "delay"; + protected static final String TIME_UNIT = "timeUnit"; + protected static final String JSON_ARRAY_CALLED_METHOD_RULES = "jsonArrayCalledMethodRules"; + + protected ScheduledFuture rulesReloader; + + protected List regexReplaceList; + + protected AccountingPersistenceConfiguration accountingPersistenceConfiguration; + + protected static RegexRulesAggregator instance; + + public synchronized static RegexRulesAggregator getInstance() { + if(instance==null) { + instance = new RegexRulesAggregator(); + } + return instance; + } + + protected RegexRulesAggregator() { + regexReplaceList = new ArrayList<>(); + readConfiguration(); + } + + public List getRegexReplacelist() { + return regexReplaceList; + } + + public RegexReplace addRegexReplace(String serviceClass, String serviceName, String regex, String replace) { + RegexReplace regexReplace = new RegexReplace(serviceClass, serviceName, regex, replace); + return addRegexReplace(regexReplace); + } + + public RegexReplace addRegexReplace(RegexReplace regexReplace) { + synchronized(regexReplaceList) { + regexReplaceList.add(regexReplace); + } + return regexReplace; + } + + + protected ScheduledFuture reloadAggregatorRules; + + public void readConfiguration() { + try { + accountingPersistenceConfiguration = new AccountingPersistenceConfiguration(this.getClass()); + + try { + String delayString = accountingPersistenceConfiguration.getProperty(DELAY); + delay = Long.parseLong(delayString); + + String timeUnitString = accountingPersistenceConfiguration.getProperty(TIME_UNIT); + timeUnit = TimeUnit.valueOf(timeUnitString.toUpperCase()); + }catch (Exception e) { + logger.warn("Unable to retrieve regex reload delay. Goign to use last known delay {} {}", delay, timeUnit.name().toLowerCase()); + } + + String rulesString = accountingPersistenceConfiguration.getProperty(JSON_ARRAY_CALLED_METHOD_RULES); + ObjectMapper mapper = DSMapper.getObjectMapper(); + JavaType type = mapper.getTypeFactory().constructCollectionType(List.class, RegexReplace.class); + List rules = mapper.readValue(rulesString, type); + synchronized(regexReplaceList) { + regexReplaceList = rules; + } + + } catch(Exception e) { + logger.error("Unable to properly load RegexRules", e); + } + } + + @Override + public void run() { + readConfiguration(); + } + + public void start() { + if(reloadAggregatorRules == null) { + reloadAggregatorRules = ExecutorUtils.scheduler.scheduleAtFixedRate(this, delay, delay, timeUnit); + } + } + + /** + * Stop rule reloader. Use only if you really know what you do. + */ + public void stop() { + if(reloadAggregatorRules != null) { + try { + reloadAggregatorRules.cancel(true); + reloadAggregatorRules = null; + }catch (Throwable t) { + logger.error("Unable to properly stop {} reloader", this.getClass().getSimpleName(), t); + } + } + } + +} diff --git a/src/main/java/org/gcube/accounting/datamodel/validations/validators/CalledMethodRegexReplaceValidator.java b/src/main/java/org/gcube/accounting/datamodel/validations/validators/CalledMethodRegexReplaceValidator.java index 63a10ce..9633adc 100644 --- a/src/main/java/org/gcube/accounting/datamodel/validations/validators/CalledMethodRegexReplaceValidator.java +++ b/src/main/java/org/gcube/accounting/datamodel/validations/validators/CalledMethodRegexReplaceValidator.java @@ -1,10 +1,10 @@ package org.gcube.accounting.datamodel.validations.validators; import java.io.Serializable; -import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; +import org.gcube.accounting.aggregator.RegexRulesAggregator; import org.gcube.accounting.datamodel.basetypes.AbstractServiceUsageRecord; import org.gcube.documentstore.exception.InvalidValueException; import org.gcube.documentstore.records.Record; @@ -19,25 +19,19 @@ public class CalledMethodRegexReplaceValidator implements FieldAction { private static Logger logger = LoggerFactory.getLogger(CalledMethodRegexReplaceValidator.class); - protected static final List regexReplaceList; - + @Deprecated public static List getRegexReplacelist() { - return regexReplaceList; + return RegexRulesAggregator.getInstance().getRegexReplacelist(); } + @Deprecated public static RegexReplace addRegexReplace(String serviceClass, String serviceName, String regex, String replace) { - RegexReplace regexReplace = new RegexReplace(serviceClass, serviceName, regex, replace); - regexReplaceList.add(regexReplace); - return regexReplace; + return RegexRulesAggregator.getInstance().addRegexReplace(serviceClass, serviceName, regex, replace); } + @Deprecated public static RegexReplace addRegexReplace(RegexReplace regexReplace) { - regexReplaceList.add(regexReplace); - return regexReplace; - } - - static { - regexReplaceList = new ArrayList<>(); + return RegexRulesAggregator.getInstance().addRegexReplace(regexReplace); } /** diff --git a/src/main/java/org/gcube/accounting/datamodel/validations/validators/RegexReplace.java b/src/main/java/org/gcube/accounting/datamodel/validations/validators/RegexReplace.java index 698cdc3..847ed10 100644 --- a/src/main/java/org/gcube/accounting/datamodel/validations/validators/RegexReplace.java +++ b/src/main/java/org/gcube/accounting/datamodel/validations/validators/RegexReplace.java @@ -4,21 +4,22 @@ import java.util.regex.Pattern; public class RegexReplace { - protected final String serviceClass; - protected final String serviceName; + protected String serviceClass; + protected String serviceName; - protected final String regex; - protected final Pattern regexPattern; + protected String regex; + protected Pattern regexPattern; - protected final String replace; + protected String replace; + + protected RegexReplace() {} public RegexReplace(String serviceClass, String serviceName, String regex, String replace) { super(); this.serviceClass = serviceClass; this.serviceName = serviceName; - this.regex = regex; - this.regexPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); this.replace = replace; + setRegex(regex); } public String getServiceClass() { @@ -29,6 +30,11 @@ public class RegexReplace { return serviceName; } + protected void setRegex(String regex) { + this.regex = regex; + this.regexPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + } + public String getRegex() { return regex; } diff --git a/src/main/java/org/gcube/accounting/persistence/AccountingPersistenceConfiguration.java b/src/main/java/org/gcube/accounting/persistence/AccountingPersistenceConfiguration.java index 9b95dd4..6067928 100644 --- a/src/main/java/org/gcube/accounting/persistence/AccountingPersistenceConfiguration.java +++ b/src/main/java/org/gcube/accounting/persistence/AccountingPersistenceConfiguration.java @@ -13,7 +13,6 @@ import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.utils.Group; -import org.gcube.documentstore.persistence.PersistenceBackend; import org.gcube.documentstore.persistence.PersistenceBackendConfiguration; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; @@ -37,13 +36,13 @@ public class AccountingPersistenceConfiguration extends PersistenceBackendConfig super(); } - public AccountingPersistenceConfiguration(Class clz) throws Exception { - super(clz); + public AccountingPersistenceConfiguration(Class clz) throws Exception { + super(); ServiceEndpoint serviceEndpoint = getServiceEndpoint(SERVICE_ENDPOINT_CATEGORY, SERVICE_ENDPOINT_NAME, clz); setValues(serviceEndpoint, clz); } - protected ServiceEndpoint getServiceEndpoint(String serviceEndpointCategory, String serviceEndpointName, Class clz){ + protected ServiceEndpoint getServiceEndpoint(String serviceEndpointCategory, String serviceEndpointName, Class clz){ SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class); query.addCondition(String.format("$resource/Profile/Category/text() eq '%s'", serviceEndpointCategory)); query.addCondition(String.format("$resource/Profile/Name/text() eq '%s'", serviceEndpointName)); @@ -71,7 +70,7 @@ public class AccountingPersistenceConfiguration extends PersistenceBackendConfig return StringEncrypter.getEncrypter().decrypt(encrypted); } - protected void setValues(ServiceEndpoint serviceEndpoint, Class clz) throws Exception{ + protected void setValues(ServiceEndpoint serviceEndpoint, Class clz) throws Exception{ Group accessPoints = serviceEndpoint.profile().accessPoints(); for(AccessPoint accessPoint : accessPoints){ if(accessPoint.name().compareTo(clz.getSimpleName())==0){ diff --git a/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java b/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java index c02425a..227c519 100644 --- a/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java +++ b/src/test/java/org/gcube/accounting/datamodel/UsageRecordTest.java @@ -20,6 +20,7 @@ public class UsageRecordTest { Assert.assertEquals(0, usageRecord.compareTo(ur)); Assert.assertEquals(0, ur.compareTo(usageRecord)); } + /* @Test public void testCompareToEqualsObject() throws Exception { diff --git a/src/test/java/org/gcube/accounting/datamodel/usagerecords/ServiceUsageRecordTest.java b/src/test/java/org/gcube/accounting/datamodel/usagerecords/ServiceUsageRecordTest.java index bd832d6..05d7b3f 100644 --- a/src/test/java/org/gcube/accounting/datamodel/usagerecords/ServiceUsageRecordTest.java +++ b/src/test/java/org/gcube/accounting/datamodel/usagerecords/ServiceUsageRecordTest.java @@ -8,10 +8,10 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import org.gcube.accounting.aggregator.RegexRulesAggregator; import org.gcube.accounting.datamodel.UsageRecord; import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.basetypes.AbstractServiceUsageRecord; -import org.gcube.accounting.datamodel.validations.validators.CalledMethodRegexReplaceValidator; import org.gcube.accounting.datamodel.validations.validators.RegexReplace; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; @@ -152,8 +152,9 @@ public class ServiceUsageRecordTest extends ScopedTest { + "\"operationResult\":\"SUCCESS\"" +"}"; - String goodReplace = "/delete/{collection-id}/{item-id}"; + + /* RegexReplace regexReplace = new RegexReplace("Index", "FullTextIndexNode", "/delete/[^\\t\\n\\r\\f\\v]+/[^\\t\\n\\r\\f\\v]+", goodReplace); CalledMethodRegexReplaceValidator.addRegexReplace(regexReplace); @@ -167,9 +168,18 @@ public class ServiceUsageRecordTest extends ScopedTest { regexReplace = new RegexReplace("InformationSystem", "resource-registry", "/access/instance/EService/[^\\t\\n\\r\\f\\v]+", replace); CalledMethodRegexReplaceValidator.addRegexReplace(regexReplace); - + */ + UsageRecord usageRecord = DSMapper.unmarshal(AggregatedServiceUsageRecord.class, usageRecordString); logger.debug("{}", usageRecord); + + RegexRulesAggregator regexRulesAggregator = RegexRulesAggregator.getInstance(); + List list = regexRulesAggregator.getRegexReplacelist(); + for(RegexReplace regexReplace : list) { + logger.debug("{} {} {} {}", regexReplace.getServiceClass(), regexReplace.getServiceName(), regexReplace.getRegex(), regexReplace.getReplace()); + } + + AggregatedServiceUsageRecord aggregatedServiceUsageRecord = new AggregatedServiceUsageRecord(usageRecord.getResourceProperties()); logger.debug("{}", aggregatedServiceUsageRecord); @@ -202,12 +212,13 @@ public class ServiceUsageRecordTest extends ScopedTest { + "\"operationResult\":\"SUCCESS\"" + "}"; - String replace = "/delete/{collection-id}/{item-id}"; + /* RegexReplace regexReplace = new RegexReplace("Index", "FullTextIndexNode", "/delete/[^\\t\\n\\r\\f\\v]+/[^\\t\\n\\r\\f\\v]+", replace); CalledMethodRegexReplaceValidator.addRegexReplace(regexReplace); - + */ + RegexRulesAggregator.getInstance(); UsageRecord usageRecord = DSMapper.unmarshal(AggregatedServiceUsageRecord.class, usageRecordString); logger.debug("{}", usageRecord); @@ -245,10 +256,12 @@ public class ServiceUsageRecordTest extends ScopedTest { + "}"; String replace = "GET /access/instance/Configuration/{id}"; + /* RegexReplace regexReplace = new RegexReplace("InformationSystem", "resource-registry", "/access/instance/Configuration/[^\\t\\n\\r\\f\\v]+", replace); CalledMethodRegexReplaceValidator.addRegexReplace(regexReplace); - + */ + RegexRulesAggregator.getInstance(); UsageRecord usageRecord = DSMapper.unmarshal(AggregatedServiceUsageRecord.class, usageRecordString); logger.debug("{}", usageRecord); @@ -287,10 +300,12 @@ public class ServiceUsageRecordTest extends ScopedTest { + "}"; String replace = "GET /access/instance/EService/{id}"; + /* RegexReplace regexReplace = new RegexReplace("InformationSystem", "resource-registry", "/access/instance/EService/[^\\t\\n\\r\\f\\v]+", replace); CalledMethodRegexReplaceValidator.addRegexReplace(regexReplace); - + */ + RegexRulesAggregator.getInstance(); UsageRecord usageRecord = DSMapper.unmarshal(AggregatedServiceUsageRecord.class, usageRecordString); logger.debug("{}", usageRecord);