From fbf99786bb8fb00c52c01691af716eebb27c1f5c Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 6 Jul 2015 13:33:54 +0000 Subject: [PATCH] refs #200: Create accouting-lib library https://support.d4science.org/issues/200 Fixing tests git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-lib@117029 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../messaging/ResourceAccounting.java | 3 +- .../persistence/FallbackPersistence.java | 3 +- .../accounting/persistence/Persistence.java | 113 +++++------------- .../persistence/PersistenceConfiguration.java | 10 +- .../persistence/PersistenceFactory.java | 108 +++++++++++++++++ .../PersistenceConfigurationTest.java | 7 +- .../persistence/PersistenceTest.java | 19 ++- .../accounting/persistence/SymmetricKey.java | 69 ----------- .../testutility/StressTestUtility.java | 11 +- src/test/resources/devsec.gcubekey | 1 + src/test/resources/gcube.gcubekey | 1 + 11 files changed, 168 insertions(+), 177 deletions(-) create mode 100644 src/main/java/org/gcube/accounting/persistence/PersistenceFactory.java delete mode 100644 src/test/java/org/gcube/accounting/persistence/SymmetricKey.java create mode 100644 src/test/resources/devsec.gcubekey create mode 100644 src/test/resources/gcube.gcubekey diff --git a/src/main/java/org/gcube/accounting/messaging/ResourceAccounting.java b/src/main/java/org/gcube/accounting/messaging/ResourceAccounting.java index 3a79bfd..c33d3c1 100644 --- a/src/main/java/org/gcube/accounting/messaging/ResourceAccounting.java +++ b/src/main/java/org/gcube/accounting/messaging/ResourceAccounting.java @@ -3,6 +3,7 @@ package org.gcube.accounting.messaging; import org.gcube.accounting.datamodel.RawUsageRecord; import org.gcube.accounting.exception.InvalidValueException; import org.gcube.accounting.persistence.Persistence; +import org.gcube.accounting.persistence.PersistenceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +23,7 @@ public class ResourceAccounting { @Deprecated public ResourceAccounting() { - persistence = Persistence.getInstance(); + persistence = PersistenceFactory.getPersistence(); } @Deprecated diff --git a/src/main/java/org/gcube/accounting/persistence/FallbackPersistence.java b/src/main/java/org/gcube/accounting/persistence/FallbackPersistence.java index c9e2443..3a8b21d 100644 --- a/src/main/java/org/gcube/accounting/persistence/FallbackPersistence.java +++ b/src/main/java/org/gcube/accounting/persistence/FallbackPersistence.java @@ -9,6 +9,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import org.gcube.accounting.aggregation.scheduler.AggregationScheduler; import org.gcube.accounting.datamodel.UsageRecord; /** @@ -19,7 +20,7 @@ public class FallbackPersistence extends Persistence { private File accountingFallbackFile; protected FallbackPersistence(File accountingFallbackFile) { - super(); + super(null, AggregationScheduler.getInstance()); this.accountingFallbackFile = accountingFallbackFile; } diff --git a/src/main/java/org/gcube/accounting/persistence/Persistence.java b/src/main/java/org/gcube/accounting/persistence/Persistence.java index a584905..f1d29fc 100644 --- a/src/main/java/org/gcube/accounting/persistence/Persistence.java +++ b/src/main/java/org/gcube/accounting/persistence/Persistence.java @@ -3,8 +3,6 @@ */ package org.gcube.accounting.persistence; -import java.io.File; -import java.util.ServiceLoader; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -22,97 +20,38 @@ public abstract class Persistence { private static final Logger logger = LoggerFactory.getLogger(Persistence.class); - private static final String ACCOUTING_FALLBACK_FILENAME = "accountingFallback.log"; - - /** - * The singleton instance of persistence - */ - protected static Persistence persistence; - protected static FallbackPersistence fallback; - protected static AggregationScheduler aggregationScheduler; - - private static File file(File file) throws IllegalArgumentException { - - if (file.isDirectory()) - throw new IllegalArgumentException(file.getAbsolutePath() + " cannot be used in write mode because it's folder"); - - //create folder structure it does not exist - if (!file.getParentFile().exists()) - file.getParentFile().mkdirs(); - - return file; - - } - - public synchronized static void setFallbackLocation(String path){ - if(fallback == null){ - if(path==null){ - path = "."; - } - File file = file(new File(path, ACCOUTING_FALLBACK_FILENAME)); - fallback = new FallbackPersistence(file); - } - } - - protected static void init() { - setFallbackLocation(null); - try { - ServiceLoader serviceLoader = ServiceLoader.load(Persistence.class); - for (Persistence foundPersistence : serviceLoader) { - if(foundPersistence.getClass().isInstance(FallbackPersistence.class)){ - continue; - } - try { - String foundPersistenceClassName = foundPersistence.getClass().getSimpleName(); - logger.debug("Testing {}", foundPersistenceClassName); - String scope = null; // TODO - PersistenceConfiguration configuration = PersistenceConfiguration.getPersistenceConfiguration(scope, foundPersistenceClassName); - foundPersistence.prepareConnection(configuration); - /* - * Uncomment the following line of code if you want to try - * to create a test UsageRecord before setting the - * foundPersistence as default - * - * foundPersistence.accountWithFallback(TestUsageRecord.createTestServiceUsageRecord()); - */ - persistence = foundPersistence; - break; - } catch (Exception e) { - logger.debug(String.format("%s not initialized correctly. It will not be used", foundPersistence.getClass().getSimpleName())); - - } - } - if(persistence==null){ - persistence = fallback; - } - } catch(Exception e){ - logger.error("Unable to instance a Persistence Implementation. Using fallback as default", - e.getCause()); - persistence = fallback; - } - aggregationScheduler = AggregationScheduler.getInstance(); - } + protected FallbackPersistence fallback; + protected AggregationScheduler aggregationScheduler; /** * Pool for thread execution */ private ExecutorService pool; + protected Persistence(){ + this.pool = Executors.newCachedThreadPool(); + } + + protected Persistence(FallbackPersistence fallback, AggregationScheduler aggregationScheduler){ + this.fallback = fallback; + this.aggregationScheduler = aggregationScheduler; + this.pool = Executors.newCachedThreadPool(); + } + /** - * @return the singleton instance of persistence - * @throws Exception if fails + * @param fallback the fallback to set */ - public static Persistence getInstance() { - if(persistence==null){ - init(); - } - return persistence; + protected void setFallback(FallbackPersistence fallback) { + this.fallback = fallback; } - - protected Persistence() { - pool = Executors.newCachedThreadPool(); + + /** + * @param aggregationScheduler the aggregationScheduler to set + */ + protected void setAggregationScheduler(AggregationScheduler aggregationScheduler) { + this.aggregationScheduler = aggregationScheduler; } - + /** * Prepare the connection to persistence. * This method must be used by implementation class to open @@ -141,11 +80,11 @@ public abstract class Persistence { protected abstract void reallyAccount(UsageRecord usageRecords) throws Exception; private void accountWithFallback(UsageRecord... usageRecords) { - String persistenceName = getInstance().getClass().getSimpleName(); + String persistenceName = this.getClass().getSimpleName(); for(UsageRecord usageRecord : usageRecords){ try { //logger.debug("Going to account {} using {}", usageRecord, persistenceName); - persistence.reallyAccount(usageRecord); + this.reallyAccount(usageRecord); logger.debug("{} accounted succesfully from {}.", usageRecord.toString(), persistenceName); } catch (Exception e) { String fallabackPersistenceName = fallback.getClass().getSimpleName(); @@ -169,6 +108,7 @@ public abstract class Persistence { usageRecord.validate(); } if(aggregate){ + final Persistence persistence = this; aggregationScheduler.aggregate(usageRecord, new PersistenceExecutor(){ @Override @@ -178,7 +118,7 @@ public abstract class Persistence { }); }else{ - persistence.accountWithFallback(usageRecord); + this.accountWithFallback(usageRecord); } } catch (InvalidValueException e) { @@ -209,6 +149,7 @@ public abstract class Persistence { } public void flush() throws Exception { + final Persistence persistence = this; aggregationScheduler.flush(new PersistenceExecutor(){ @Override diff --git a/src/main/java/org/gcube/accounting/persistence/PersistenceConfiguration.java b/src/main/java/org/gcube/accounting/persistence/PersistenceConfiguration.java index 5563aac..3428eaa 100644 --- a/src/main/java/org/gcube/accounting/persistence/PersistenceConfiguration.java +++ b/src/main/java/org/gcube/accounting/persistence/PersistenceConfiguration.java @@ -116,10 +116,10 @@ public class PersistenceConfiguration { } private static String decrypt(String encrypted, Key... key) throws Exception { - return StringEncrypter.getEncrypter().decrypt(encrypted, key); + return StringEncrypter.getEncrypter().decrypt(encrypted); } - private static PersistenceConfiguration createPersistenceConfiguration(ServiceEndpoint serviceEndpoint, Key... key) throws Exception{ + protected static PersistenceConfiguration createPersistenceConfiguration(ServiceEndpoint serviceEndpoint) throws Exception{ PersistenceConfiguration persistenceConfiguration = new PersistenceConfiguration(); Group accessPoints = serviceEndpoint.profile().accessPoints(); for(AccessPoint accessPoint : accessPoints){ @@ -127,7 +127,7 @@ public class PersistenceConfiguration { persistenceConfiguration.username = accessPoint.username(); String encryptedPassword = accessPoint.password(); - String password = decrypt(encryptedPassword, key); + String password = decrypt(encryptedPassword); persistenceConfiguration.password = password; persistenceConfiguration.propertyMap = accessPoint.propertyMap(); @@ -142,9 +142,9 @@ public class PersistenceConfiguration { * @return * @throws Exception */ - protected static PersistenceConfiguration getPersistenceConfiguration(String scope, String persistenceClassName, Key... key) throws Exception { + public static PersistenceConfiguration getPersistenceConfiguration(String scope, String persistenceClassName) throws Exception { ServiceEndpoint serviceEndpoint = getServiceEndpoint(scope, persistenceClassName); - return createPersistenceConfiguration(serviceEndpoint, key); + return createPersistenceConfiguration(serviceEndpoint); } } diff --git a/src/main/java/org/gcube/accounting/persistence/PersistenceFactory.java b/src/main/java/org/gcube/accounting/persistence/PersistenceFactory.java new file mode 100644 index 0000000..e9e2802 --- /dev/null +++ b/src/main/java/org/gcube/accounting/persistence/PersistenceFactory.java @@ -0,0 +1,108 @@ +/** + * + */ +package org.gcube.accounting.persistence; + +import java.io.File; +import java.util.ServiceLoader; + +import org.gcube.accounting.aggregation.scheduler.AggregationScheduler; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.common.scope.impl.ScopeBean.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ + * + */ +public abstract class PersistenceFactory { + + private static final Logger logger = LoggerFactory.getLogger(PersistenceFactory.class); + + public final static String HOME_SYSTEM_PROPERTY = "user.home"; + + private static final String ACCOUTING_FALLBACK_FILENAME = "accountingFallback.log"; + + private static String fallbackLocation; + + private static File file(File file) throws IllegalArgumentException { + + if(!file.isDirectory()){ + file = file.getParentFile(); + } + + //create folder structure if not exist + if (!file.exists()) + file.mkdirs(); + + return file; + + } + + public synchronized static void setFallbackLocation(String path){ + if(fallbackLocation == null){ + if(path==null){ + path = System.getProperty(HOME_SYSTEM_PROPERTY); + } + file(new File(path)); + fallbackLocation = path; + } + } + + public synchronized static Persistence getPersistence(){ + Persistence persistence = null; + + String scope = ScopeProvider.instance.get(); + String name = ""; + if(scope!=null){ + ScopeBean bean = new ScopeBean(scope); + if(bean.is(Type.VRE)){ + bean = bean.enclosingScope(); + } + name = bean.name(); + } + String separator = name.compareTo("")==0 ? "" : "."; + + File fallbackFile = new File(fallbackLocation, String.format("%s%s%s", name, separator ,ACCOUTING_FALLBACK_FILENAME)); + FallbackPersistence fallbackPersistence = new FallbackPersistence(fallbackFile); + try { + ServiceLoader serviceLoader = ServiceLoader.load(Persistence.class); + for (Persistence foundPersistence : serviceLoader) { + if(foundPersistence.getClass().isInstance(FallbackPersistence.class)){ + continue; + } + try { + String foundPersistenceClassName = foundPersistence.getClass().getSimpleName(); + logger.debug("Testing {}", foundPersistenceClassName); + PersistenceConfiguration configuration = PersistenceConfiguration.getPersistenceConfiguration(scope, foundPersistenceClassName); + foundPersistence.prepareConnection(configuration); + /* + * Uncomment the following line of code if you want to try + * to create a test UsageRecord before setting the + * foundPersistence as default + * + * foundPersistence.accountWithFallback(TestUsageRecord.createTestServiceUsageRecord()); + */ + persistence = foundPersistence; + break; + } catch (Exception e) { + logger.debug(String.format("%s not initialized correctly. It will not be used", foundPersistence.getClass().getSimpleName())); + + } + } + if(persistence==null){ + persistence = fallbackPersistence; + } + } catch(Exception e){ + logger.error("Unable to instance a Persistence Implementation. Using fallback as default", + e.getCause()); + persistence = fallbackPersistence; + } + persistence.setAggregationScheduler(AggregationScheduler.getInstance()); + persistence.setFallback(fallbackPersistence); + return persistence; + } + +} diff --git a/src/test/java/org/gcube/accounting/persistence/PersistenceConfigurationTest.java b/src/test/java/org/gcube/accounting/persistence/PersistenceConfigurationTest.java index 92b3fcf..d8aa7a2 100644 --- a/src/test/java/org/gcube/accounting/persistence/PersistenceConfigurationTest.java +++ b/src/test/java/org/gcube/accounting/persistence/PersistenceConfigurationTest.java @@ -5,7 +5,6 @@ package org.gcube.accounting.persistence; import java.io.StringWriter; import java.net.URL; -import java.security.Key; import java.util.Arrays; import java.util.List; @@ -56,7 +55,7 @@ public class PersistenceConfigurationTest { public static final String FAKE_PASSWORD = "fakepassword"; public static final String[] SCOPES = new String[]{"/gcube", "/gcube/devsec"}; - public static final String GCUBE_SCOPE = SCOPES[1]; + public static final String GCUBE_SCOPE = SCOPES[0]; public static final String GCUBE_DEVSEC_SCOPE = SCOPES[1]; public static final String DB_NAME_PROPERTY_NAME = "dbName"; @@ -198,9 +197,7 @@ public class PersistenceConfigurationTest { } try { - String keyName = "devsec.gcubekey"; - Key key = SymmetricKey.getKey(System.getProperty(PersistenceTest.HOME_SYSTEM_PROPERTY)+ "/" + keyName); - PersistenceConfiguration persitenceConfiguration = PersistenceConfiguration.getPersistenceConfiguration(GCUBE_DEVSEC_SCOPE, COUCHDB_CLASS_NAME, key); + PersistenceConfiguration persitenceConfiguration = PersistenceConfiguration.getPersistenceConfiguration(GCUBE_DEVSEC_SCOPE, COUCHDB_CLASS_NAME); if(createResource){ Assert.assertTrue(persitenceConfiguration.getUri().toURL().equals(new URL(RUNNING_ON))); Assert.assertTrue(persitenceConfiguration.getUsername().compareTo(FAKE_USERNAME)==0); diff --git a/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java b/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java index 5f871c5..d8da505 100644 --- a/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java +++ b/src/test/java/org/gcube/accounting/persistence/PersistenceTest.java @@ -15,16 +15,23 @@ import org.junit.Test; */ public class PersistenceTest { - protected final static String HOME_SYSTEM_PROPERTY = "user.home"; - public static Persistence getPersistence(){ - Persistence.setFallbackLocation(System.getProperty(HOME_SYSTEM_PROPERTY)); - return Persistence.getInstance(); + PersistenceFactory.setFallbackLocation(null); + return PersistenceFactory.getPersistence(); } @Test - public void test() throws Exception { - getPersistence(); + public void singleTest() throws Exception { + final Persistence persistence = getPersistence(); + StressTestUtility.stressTest(new TestOperation() { + @Override + public void operate(int i) { + SingleUsageRecord usageRecord = TestUsageRecord.createTestServiceUsageRecord(); + persistence.validateAccountAggregate(usageRecord, true, false); + } + }, 1); + + persistence.flush(); } @Test diff --git a/src/test/java/org/gcube/accounting/persistence/SymmetricKey.java b/src/test/java/org/gcube/accounting/persistence/SymmetricKey.java deleted file mode 100644 index 5fb5d83..0000000 --- a/src/test/java/org/gcube/accounting/persistence/SymmetricKey.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.gcube.accounting.persistence; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.security.InvalidKeyException; -import java.security.Key; - -import javax.crypto.spec.SecretKeySpec; - -final class SymmetricKey { - - private static Key key; - - private static String keyAlgorithm = "AES"; - - //private constructor - private SymmetricKey() {} - - /** - * Gets the key for encryption/decryption - * @return the key - * @throws InvalidKeyException if the key is not available or is invalid - */ - public static Key getKey(String keyPath) throws InvalidKeyException { - if (key == null) load(keyPath); - return key; - } - - /** - * Loads the key from the classpaht - * @throws InvalidKeyException if the key is not available or is invalid - */ - private static void load(String keyPath) throws InvalidKeyException { - byte[] rawKey; - String keyFileName=null; - try { - InputStream is = new FileInputStream(new File(keyPath)); - rawKey = getBytesFromStream(is); - } catch (Exception e) { - System.out.println("Unable to load the Key "+keyFileName+" from the classpath"); - e.printStackTrace(); - throw new InvalidKeyException("Unable to load the Key "+keyFileName+" from the classpath"); - } - try { - key = new SecretKeySpec(rawKey, 0, rawKey.length, keyAlgorithm); - }catch (Exception e) { - e.printStackTrace(); - throw new InvalidKeyException(); - } - } - - private static byte[] getBytesFromStream(InputStream is) throws IOException { - byte[] rawKey; - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - int nRead; - byte[] data = new byte[16384]; - while ((nRead = is.read(data, 0, data.length)) != -1) { - buffer.write(data, 0, nRead); - } - buffer.flush(); - rawKey= buffer.toByteArray(); - return rawKey; - } - - -} diff --git a/src/test/java/org/gcube/accounting/testutility/StressTestUtility.java b/src/test/java/org/gcube/accounting/testutility/StressTestUtility.java index 90767ec..7eddd95 100644 --- a/src/test/java/org/gcube/accounting/testutility/StressTestUtility.java +++ b/src/test/java/org/gcube/accounting/testutility/StressTestUtility.java @@ -18,19 +18,22 @@ public class StressTestUtility { private static final Logger logger = LoggerFactory.getLogger(StressTestUtility.class); - - protected final static int NUMBER_OF_RECORDS = 3000; + protected final static int DEFAULT_NUMBER_OF_RECORDS = 3000; public static void stressTest(TestOperation operation) throws Exception { + stressTest(operation, DEFAULT_NUMBER_OF_RECORDS); + } + + public static void stressTest(TestOperation operation, int runs) throws Exception { Calendar startTestTime = new GregorianCalendar(); - for(int i=0; i< NUMBER_OF_RECORDS; i++){ + for(int i=0; i< runs; i++){ operation.operate(i); } Calendar stopTestTime = new GregorianCalendar(); double startMillis = startTestTime.getTimeInMillis(); double stopMillis = stopTestTime.getTimeInMillis(); double duration = stopMillis - startMillis; - double average = (duration/NUMBER_OF_RECORDS); + double average = (duration/runs); logger.debug("Duration (in millisec) : " + duration); logger.debug("Average (in millisec) : " + average); } diff --git a/src/test/resources/devsec.gcubekey b/src/test/resources/devsec.gcubekey new file mode 100644 index 0000000..260f269 --- /dev/null +++ b/src/test/resources/devsec.gcubekey @@ -0,0 +1 @@ +6 4Zð/Uä‰ Cå±ß˜ \ No newline at end of file diff --git a/src/test/resources/gcube.gcubekey b/src/test/resources/gcube.gcubekey new file mode 100644 index 0000000..260f269 --- /dev/null +++ b/src/test/resources/gcube.gcubekey @@ -0,0 +1 @@ +6 4Zð/Uä‰ Cå±ß˜ \ No newline at end of file