commit f5a93f0850d010cd59830dbef19d49a0b52e7f3b Author: Luca Frosini Date: Fri Dec 18 16:03:05 2015 +0000 refs #1746: Separate Accounting Model and generalize solution https://support.d4science.org/issues/1746 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/document-store-lib-couchdb@121988 82a268e6-3cf1-43bd-a215-b396298e98cf diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..dcb2dc5 --- /dev/null +++ b/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..6deb1fd --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + document-store-lib-couchdb + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..ec4300d --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/distro/INSTALL b/distro/INSTALL new file mode 100644 index 0000000..049006c --- /dev/null +++ b/distro/INSTALL @@ -0,0 +1 @@ +Used as library in the gCube Framework \ No newline at end of file diff --git a/distro/LICENSE b/distro/LICENSE new file mode 100644 index 0000000..07f22ed --- /dev/null +++ b/distro/LICENSE @@ -0,0 +1,8 @@ +gCube System - License +------------------------------------------------------------ + +The gCube/gCore software is licensed as Free Open Source software conveying to +the EUPL (http://ec.europa.eu/idabc/eupl). +The software and documentation is provided by its authors/distributors "as is" +and no expressed or implied warranty is given for its use, quality or fitness +for a particular case. \ No newline at end of file diff --git a/distro/MAINTAINERS b/distro/MAINTAINERS new file mode 100644 index 0000000..2cdfd00 --- /dev/null +++ b/distro/MAINTAINERS @@ -0,0 +1,2 @@ +Luca Frosini (luca.frosini@isti.cnr.it), CNR Pisa, +Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo". diff --git a/distro/README b/distro/README new file mode 100644 index 0000000..86fd591 --- /dev/null +++ b/distro/README @@ -0,0 +1,28 @@ +The gCube System - Accounting Common Library +------------------------------------------------------------ + +This work has been partially supported by the following European projects: DILIGENT (FP6-2003-IST-2), +D4Science (FP7-INFRA-2007-1.2.2), D4Science-II (FP7-INFRA-2008-1.2.2), iMarine (FP7-INFRASTRUCTURES-2011-2), +and EUBrazilOpenBio (FP7-ICT-2011-EU-Brazil), Parthenos (H2020-INFRADEV-1-2014-1), BlueBridge (H2020-EINFRA-2015-1). + +Authors +------- + +* Luca Frosini (luca.frosini@isti.cnr.it), CNR Pisa, Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo". + + +Version and Release Date +------------------------ + +v 1.0, 30/09/2015 + + +Description +----------- + +${description} + +Licensing +--------- + +This software is licensed under the terms you may find in the file named "LICENSE" in this directory. diff --git a/distro/changelog.xml b/distro/changelog.xml new file mode 100644 index 0000000..35d6f7d --- /dev/null +++ b/distro/changelog.xml @@ -0,0 +1,5 @@ + + + First Release + + \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml new file mode 100644 index 0000000..21d8c88 --- /dev/null +++ b/distro/descriptor.xml @@ -0,0 +1,42 @@ + + servicearchive + + tar.gz + + / + + + ${distroDirectory} + / + true + + README + LICENSE + INSTALL + MAINTAINERS + changelog.xml + + 755 + true + + + + + ${distroDirectory}/profile.xml + / + true + + + target/${build.finalName}.jar + /${artifactId} + + + ${distroDirectory}/svnpath.txt + /${artifactId} + true + + + \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml new file mode 100644 index 0000000..70d3ede --- /dev/null +++ b/distro/profile.xml @@ -0,0 +1,28 @@ + + + + Service + + Accounting + ${artifactId} + ${description} + 1.0.0 + + + ${description} + ${artifactId} + ${version} + + ${groupId} + ${artifactId} + ${version} + + library + + ${build.finalName}.jar + + + + + + diff --git a/distro/svnpath.txt b/distro/svnpath.txt new file mode 100644 index 0000000..dcd0d22 --- /dev/null +++ b/distro/svnpath.txt @@ -0,0 +1 @@ +${scm.url} \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..336f57f --- /dev/null +++ b/pom.xml @@ -0,0 +1,98 @@ + + 4.0.0 + + org.gcube.tools + maven-parent + 1.0.0 + + org.gcube.data.publishing + document-store-lib-couchdb + 1.0.0-SNAPSHOT + Document Store CouchDB + Document Store CouchDB Implementation + + + UTF-8 + ${project.basedir}/distro + + + + scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId} + scm:https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId} + https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-publishing/${project.artifactId} + + + + + org.gcube.data.publishing + document-store-lib + [1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT) + + + org.slf4j + slf4j-api + 1.7.5 + + + org.gcube.common + couchdb-connector + [1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT) + + + + + org.gcube.accounting + accounting-lib + [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) + test + + + org.gcube.core + common-encryption + [1.0.2-SNAPSHOT, 2.0.0-SNAPSHOT) + test + + + org.gcube.resources + registry-publisher + [1.0.0-SNAPSHOT,) + test + + + junit + junit + 4.11 + test + + + ch.qos.logback + logback-classic + 1.0.13 + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + + package + + single + + + + + + + + \ No newline at end of file diff --git a/src/main/java/org/gcube/documentstore/persistence/PersistenceCouchDB.java b/src/main/java/org/gcube/documentstore/persistence/PersistenceCouchDB.java new file mode 100644 index 0000000..e2785a2 --- /dev/null +++ b/src/main/java/org/gcube/documentstore/persistence/PersistenceCouchDB.java @@ -0,0 +1,82 @@ +/** + * + */ +package org.gcube.documentstore.persistence; + +import java.io.Serializable; +import java.util.Map; + +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.gcube.common.couchdb.connector.HttpCouchClient; +import org.gcube.documentstore.persistence.PersistenceBackend; +import org.gcube.documentstore.persistence.PersistenceBackendConfiguration; +import org.gcube.documentstore.records.Record; +import org.gcube.documentstore.records.RecordUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ + */ +public class PersistenceCouchDB extends PersistenceBackend { + + private static final Logger logger = LoggerFactory.getLogger(PersistenceCouchDB.class); + + protected HttpCouchClient httpCouchClient; + + public static final String URL_PROPERTY_KEY = "URL"; + public static final String USERNAME_PROPERTY_KEY = "username"; + public static final String PASSWORD_PROPERTY_KEY = "password"; + public static final String DB_NAME = "dbName"; + + public PersistenceCouchDB() throws Exception { + super(); + } + + @Override + public void close() throws Exception {} + + /** + * {@inheritDoc} + */ + @Override + protected void prepareConnection(PersistenceBackendConfiguration configuration) throws Exception { + logger.debug("Preparing Connection for {}", this.getClass().getSimpleName()); + + String url = configuration.getProperty(URL_PROPERTY_KEY); + String username = configuration.getProperty(USERNAME_PROPERTY_KEY); + String password = configuration.getProperty(PASSWORD_PROPERTY_KEY); + String dbName = configuration.getProperty(DB_NAME); + + httpCouchClient = new HttpCouchClient(url, dbName, username, password); + } + + protected void createItem(JsonNode node, String id) throws Exception { + httpCouchClient.put(node.toString(), id); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reallyAccount(Record record) throws Exception { + JsonNode node = usageRecordToJsonNode(record); + createItem(node, record.getId()); + } + + public static JsonNode usageRecordToJsonNode(Record record) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + JsonNode node = mapper.valueToTree(record.getResourceProperties()); + return node; + } + + protected static Record jsonNodeToUsageRecord(JsonNode jsonNode) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + @SuppressWarnings("unchecked") + Map> result = mapper.convertValue(jsonNode, Map.class); + Record record =RecordUtility.getRecord(result); + return record; + } + +} diff --git a/src/main/resources/META-INF/services/org.gcube.documentstore.persistence.PersistenceBackend b/src/main/resources/META-INF/services/org.gcube.documentstore.persistence.PersistenceBackend new file mode 100644 index 0000000..a2d33e5 --- /dev/null +++ b/src/main/resources/META-INF/services/org.gcube.documentstore.persistence.PersistenceBackend @@ -0,0 +1 @@ +org.gcube.documentstore.persistence.PersistenceCouchDB \ No newline at end of file diff --git a/src/test/java/org/gcube/accounting/persistence/PersistenceCouchDBTest.java b/src/test/java/org/gcube/accounting/persistence/PersistenceCouchDBTest.java new file mode 100644 index 0000000..4a26213 --- /dev/null +++ b/src/test/java/org/gcube/accounting/persistence/PersistenceCouchDBTest.java @@ -0,0 +1,124 @@ +/** + * + */ +package org.gcube.accounting.persistence; + +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; + +import org.gcube.common.resources.gcore.Resource; +import org.gcube.common.resources.gcore.Resources; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.documentstore.persistence.PersistenceBackend; +import org.gcube.documentstore.persistence.PersistenceBackendFactory; +import org.gcube.documentstore.persistence.PersistenceCouchDB; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.informationsystem.publisher.ScopedPublisher; +import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ + * + */ +public class PersistenceCouchDBTest { + + private static final Logger logger = LoggerFactory.getLogger(PersistenceCouchDBTest.class); + + public static PersistenceBackend getPersistence(){ + ScopeProvider.instance.set("/gcube/devNext"); + PersistenceBackendFactory.setFallbackLocation(null); + return PersistenceBackendFactory.getPersistenceBackend(ScopeProvider.instance.get()); + } + + @Test + public void persistenceIsCouchDB() { + PersistenceBackend persistence = getPersistence(); + Assert.assertTrue(persistence instanceof PersistenceCouchDB); + } + + private static void publishScopedResource(Resource resource, List scopes) throws Exception { + StringWriter stringWriter = new StringWriter(); + Resources.marshal(resource, stringWriter); + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + try { + logger.debug("Trying to publish to {}:\n{}", scopes, stringWriter); + scopedPublisher.create(resource, scopes); + } catch (Exception e) { + logger.error("The resource was not published", e); + throw e; + } + } + + private static void unPublishScopedResource(Resource resource, List scopes) throws RegistryNotFoundException, Exception { + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + String id = resource.id(); + logger.debug("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id, scopes); + scopedPublisher.remove(resource, scopes); + logger.debug("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id); + } + + public void testScopeRecheck() throws Exception { + final String vreScopeToUse = "/gcube/devsec/LucioVRE"; + final String parentVREScopeToUse = "/gcube/devsec"; + + ScopeProvider.instance.set(vreScopeToUse); + + ServiceEndpoint serviceEndpoint = null; + try { + AccountingPersistenceConfiguration persitenceConfiguration = + new AccountingPersistenceConfiguration(PersistenceCouchDB.class); + serviceEndpoint = persitenceConfiguration.getServiceEndpoint( + AccountingPersistenceConfiguration.SERVICE_ENDPOINT_CATEGORY, + AccountingPersistenceConfiguration.SERVICE_ENDPOINT_NAME, + PersistenceCouchDB.class); + List scopes = new ArrayList(); + scopes.add(ScopeProvider.instance.get()); + unPublishScopedResource(serviceEndpoint, scopes); + } catch(IndexOutOfBoundsException e){ + ScopeProvider.instance.set(parentVREScopeToUse); + AccountingPersistenceConfiguration persitenceConfiguration = + new AccountingPersistenceConfiguration(PersistenceCouchDB.class); + serviceEndpoint = persitenceConfiguration.getServiceEndpoint( + AccountingPersistenceConfiguration.SERVICE_ENDPOINT_CATEGORY, + AccountingPersistenceConfiguration.SERVICE_ENDPOINT_NAME, + PersistenceCouchDB.class); + ScopeProvider.instance.set(vreScopeToUse); + } + + + long startTime = Calendar.getInstance().getTimeInMillis(); + long endTime = startTime; + while(endTime <= (startTime + 10*1000)){ // 10 sec + endTime = Calendar.getInstance().getTimeInMillis(); + } + + logger.debug("Going to check First Time"); + PersistenceBackend first = PersistenceBackendFactory.getPersistenceBackend(ScopeProvider.instance.get()); + logger.debug("First {} : {}", PersistenceBackend.class.getSimpleName(), first); + + List scopes = new ArrayList(); + scopes.add(ScopeProvider.instance.get()); + publishScopedResource(serviceEndpoint, scopes); + + startTime = Calendar.getInstance().getTimeInMillis(); + endTime = startTime; + while(endTime <= (startTime + (PersistenceBackendFactory.FALLBACK_RETRY_TIME + 100))){ + endTime = Calendar.getInstance().getTimeInMillis(); + } + + logger.debug("Going to check Second Time"); + PersistenceBackend second = PersistenceBackendFactory.getPersistenceBackend(ScopeProvider.instance.get()); + logger.debug("Second {} : {}", PersistenceBackend.class.getSimpleName(), second); + + Assert.assertNotEquals(first, second); + + } +} diff --git a/src/test/java/org/gcube/documentstore/persistence/PersistenceCouchBaseTest.java b/src/test/java/org/gcube/documentstore/persistence/PersistenceCouchBaseTest.java new file mode 100644 index 0000000..ed7b56b --- /dev/null +++ b/src/test/java/org/gcube/documentstore/persistence/PersistenceCouchBaseTest.java @@ -0,0 +1,61 @@ +/** + * + */ +package org.gcube.documentstore.persistence; + +import java.net.URL; + +import org.codehaus.jackson.JsonNode; +import org.gcube.accounting.datamodel.basetypes.TestUsageRecord; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.documentstore.records.Record; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ + * + */ +public class PersistenceCouchBaseTest { + + private static final Logger logger = LoggerFactory.getLogger(PersistenceCouchBaseTest.class); + + @Test + public void testJsonNodeUsageRecordConversions() throws Exception { + Record record = TestUsageRecord.createTestServiceUsageRecordAutomaticScope(); + logger.debug("UsageRecord : {}", record.toString()); + JsonNode node = PersistenceCouchDB.usageRecordToJsonNode(record); + logger.debug("Node : {}", node.toString()); + Record r = PersistenceCouchDB.jsonNodeToUsageRecord(node); + Assert.assertEquals(0, record.compareTo(r)); + Assert.assertEquals(0, r.compareTo(record)); + } + + public void testProxyWithTestConfiguration() throws Exception{ + // Production-Preproduction Nodes + //URL url = new URL("http://accounting-d4s.d4science.org"); + //URL url = new URL("http://couchdb02-d4s.d4science.org:5984"); + //URL url = new URL("http://couchdb01-d4s.d4science.org:5984"); + + URL url = new URL("http://accounting-d-d4s.d4science.org/_utils/"); + //URL url = new URL("http://couchdb02-d-d4s.d4science.org:5984"); + //URL url = new URL("http://couchdb01-d-d4s.d4science.org:5984"); + + PersistenceBackendConfiguration persitenceConfiguration = PersistenceBackendConfiguration.getUnconfiguredInstance(); + persitenceConfiguration.addProperty(PersistenceCouchDB.URL_PROPERTY_KEY, url.toString()); + persitenceConfiguration.addProperty(PersistenceCouchDB.USERNAME_PROPERTY_KEY, ""); + persitenceConfiguration.addProperty(PersistenceCouchDB.PASSWORD_PROPERTY_KEY, ""); + persitenceConfiguration.addProperty(PersistenceCouchDB.DB_NAME,""); + + PersistenceCouchDB couch = new PersistenceCouchDB(); + couch.prepareConnection(persitenceConfiguration); + + ScopeProvider.instance.set("/gcube/devNext"); + Record record = TestUsageRecord.createTestServiceUsageRecordAutomaticScope(); + couch.reallyAccount(record); + + } + +} \ No newline at end of file diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 0000000..de895c0 --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{0}: %msg%n + + + + + + + + + + + \ No newline at end of file