From 3fc8eda0a5a866d1f632be8e1879ea23b96e1495 Mon Sep 17 00:00:00 2001 From: roberto cirillo Date: Tue, 22 Dec 2020 17:59:21 +0100 Subject: [PATCH] retrieve new parameters from ServiceEndpoint: backendToken, region --- .../storageclient/wrapper/Configuration.java | 93 ++++++++++++++++--- .../wrapper/ISClientConnector.java | 71 ++++++++++---- .../storageclient/wrapper/StorageClient.java | 16 +++- .../storageclient/test/GetMetaFileTest.java | 8 +- .../storageclient/test/PropertiesTest.java | 14 +-- 5 files changed, 158 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/gcube/contentmanager/storageclient/wrapper/Configuration.java b/src/main/java/org/gcube/contentmanager/storageclient/wrapper/Configuration.java index 123e4c9..1624080 100644 --- a/src/main/java/org/gcube/contentmanager/storageclient/wrapper/Configuration.java +++ b/src/main/java/org/gcube/contentmanager/storageclient/wrapper/Configuration.java @@ -3,6 +3,7 @@ package org.gcube.contentmanager.storageclient.wrapper; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.TreeSet; import org.gcube.common.resources.gcore.ServiceEndpoint; @@ -28,6 +29,7 @@ public class Configuration { private String sn; private String user; private String password; + // if the backend is mongodb, this field is used for crypt/decrypt. If the backend is S3, this field is a token. private String passPhrase; private ISClientConnector isclient; // the scope used for discovering the runtimeResource @@ -36,12 +38,17 @@ public class Configuration { private String[] volatileHost; private String[] persistentHosts; private String resolverHost; + private String region; + + private static final Logger logger = LoggerFactory.getLogger(Configuration.class); private static final String DEFAULT_BACKEND_TYPE="MongoDB"; private static final String WRITE_CONCERN_PROPERTY_NAME="write_concern"; private static final String READ_PREFERENCE_PROPERTY_NAME="read_preference"; private static final String DB_PROPERTY_NAME="collection"; private static final String DB_STRING_SEPARATOR=";"; + private static final String PASS_PHRASE_FIELD="PassPhrase"; + private static final String TOKEN_FIELD="token"; /** * home library need a special configuration */ @@ -85,7 +92,7 @@ public class Configuration { * @param memory indicates the type of memory used by the storage: Persistent or Volatile * */ - public Configuration(String sc, String sn, String scopeString, String owner, String clientID, String accessType, String memory, BackendType backend){ + public Configuration(String sc, String sn, String scopeString, String owner, String clientID, String accessType, String memory, String backend){ this.sc=sc; this.sn=sn; this.owner=owner; @@ -93,7 +100,7 @@ public class Configuration { this.typeAccess=accessType; this.memoryType=memory; setScopeString(scopeString); - setBackendType(backend.toString()); + setBackendType(backend); } /** @@ -108,6 +115,10 @@ public class Configuration { } logger.debug("RuntimeResource scope "+RRScope); ServiceEndpoint resource=getISClient().getStorageEndpoint(RRScope); + + //in this case a specific backend type is passed as input parameter + if (!Objects.isNull(getBackendType())) + getISClient().setBackendType(getBackendType()); if(resource ==null ) throw new IllegalStateException("the storage resource is not present on IS in scope: "+RRScope); List resolverResource =getISClient().getServiceEndpoint(Utils.URI_RESOLVER_RESOURCE_CATEGORY, Utils.URI_RESOLVER_RESOURCE_NAME); @@ -123,10 +134,14 @@ public class Configuration { serverFound=getISClient().retrieveConnectionInfo(resource); setUser(getISClient().getUsername()); setPassword(getISClient().password); - setBackendType(getISClient().getBackendType(resource)); + setRegion(getISClient().getRegion()); + if (!Objects.isNull(getBackendType())) + setBackendType(getISClient().getBackendType(resource)); }else{ - setBackendType(checkVarEnvBackendType()); - if(getBackendType() == null) setBackendType(DEFAULT_BACKEND_TYPE); + if (!Objects.isNull(getBackendType())) + setBackendType(checkVarEnvBackendType()); + else + setBackendType(DEFAULT_BACKEND_TYPE); setUser(checkVarEnvUser()); setPassword(checkVarEnvPassword()); } @@ -140,15 +155,46 @@ public class Configuration { if(newServer==null){ throw new IllegalStateException("Resource not found on Information System"); }else{ - setEnvironment(setAreaStorage(getSc(), getSn())); - setServerHosts(newServer, isclient, resource); - try { - setPassPhrase(retrieveEncryptionPhrase()); - } catch (Exception e) { - e.printStackTrace(); + setEnvironment(setAreaStorage(getSc(), getSn())); +// if the backend is mongodb we should set a separated server for volatile area + if(getBackendType().equals(BackendType.MongoDB)) { + setServerHosts(newServer, isclient, resource); + try { + setPassPhrase(retrieveEncryptionField(PASS_PHRASE_FIELD)); + } catch (Exception e) { + e.printStackTrace(); + } + }else { +// if the backend isn't mongodb, the passPhrase field will be filled with a token if it is present on ServiceEndpoint + try { + setPassPhrase(retrieveEncryptionField(TOKEN_FIELD)); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + setPersistentHosts(newServer); + } } + + logger.debug("Configuration parameters retrieved: host: "+getPersistentHosts()+" user: "+getUser()); } + + +// /** +// * Retrieve a valid configuration from IS for instantiating the engine +// */ +// public void getConfiguration2(){ +// String currentScope=ScopeProvider.instance.get(); +// ServiceEndpoint resource=getISClient().getStorageEndpoint(currentScope); +// if((getMemoryType() != null) && ((getMemoryType().equals(MemoryType.VOLATILE.toString()) || (getMemoryType().equals(MemoryType.BOTH.toString()))))){ +// setVolatileHost(isclient.getVolatileHost(resource)); +// } +// } +// + + private void setServerHosts(String[] newServer, ISClientConnector isclient, ServiceEndpoint resource) { if((getMemoryType() != null) && ((getMemoryType().equals(MemoryType.VOLATILE.toString()) || (getMemoryType().equals(MemoryType.BOTH.toString()))))){ @@ -222,7 +268,8 @@ public class Configuration { } if (dbNames.length > 1) logger.debug("multiple collection discovered"); - engine.setDbNames(dbNames); + if (Objects.isNull(engine.getDbNames())) + engine.setDbNames(dbNames); } // added db check also on volatile are }else if((getMemoryType().equals(MemoryType.VOLATILE.toString()))){ @@ -338,7 +385,7 @@ public class Configuration { return null; } - + @Deprecated public String retrieveEncryptionPhrase() throws Exception { String currentScope=ScopeProvider.instance.get(); logger.debug("retrieve encryption prhase on scope: "+currentScope); @@ -351,6 +398,18 @@ public class Configuration { return decryptString; } + public String retrieveEncryptionField(String fieldName) throws Exception { + String currentScope=ScopeProvider.instance.get(); + logger.debug("retrieve encryption prhase on scope: "+currentScope); + String encryptedKey=null; +// ISClientConnector isclient=getISClient(); + logger.info("retrieve encryption phrase from scope "+currentScope); + encryptedKey=getISClient().retrievePropertyValue(fieldName, currentScope); + logger.info("encrypted prhase is "+encryptedKey); + String decryptString=(!Objects.isNull(encryptedKey))?org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(encryptedKey):null; + return decryptString; + } + public String getClientID() { return clientID; @@ -493,5 +552,13 @@ public class Configuration { public void setSn(String sn) { this.sn = sn; } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } } \ No newline at end of file diff --git a/src/main/java/org/gcube/contentmanager/storageclient/wrapper/ISClientConnector.java b/src/main/java/org/gcube/contentmanager/storageclient/wrapper/ISClientConnector.java index 30e1831..3915c34 100644 --- a/src/main/java/org/gcube/contentmanager/storageclient/wrapper/ISClientConnector.java +++ b/src/main/java/org/gcube/contentmanager/storageclient/wrapper/ISClientConnector.java @@ -8,6 +8,8 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; + import org.gcube.common.encryption.StringEncrypter; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.contentmanager.storageclient.wrapper.BackendType; @@ -35,6 +37,7 @@ public class ISClientConnector { protected String username; protected String password; protected ServiceEndpoint storageResource; + public String region; private static HashMap isCache; public ISClientConnector(){ @@ -85,26 +88,38 @@ public class ISClientConnector { private String[] fillConnectionFields(ServiceEndpoint resource) { + logger.debug("fillConnectionsFields method "); if(resource!=null){ - String [] server=new String[resource.profile().accessPoints().size()]; - int i=0; - for (AccessPoint ap:resource.profile().accessPoints()) { - if (ap.name().equals("server"+(i+1))) { - server[i] = ap.address(); - // if presents, try to get user and password - setUsername(ap.username()); - decryptPwd(ap); - i++; - }else if((!getBackendType().equals(BackendType.MongoDB) && (ap.name().equals(getBackendType())))){ - // in this case, the backend type has been set in the constructor hence we want to use this one - // if presents, try to get user and password - setUsername(ap.username()); - decryptPwd(ap); - //in v1.6.0 we are considering only one accessPoint for a backend except for mongodb. - break; + logger.debug("checking resource "+resource.id()); + String [] server=new String[resource.profile().accessPoints().size()]; + if(!Objects.isNull(getBackendType())) { + logger.info("searching backend: "+getBackendType()); + for (AccessPoint ap:resource.profile().accessPoints()) { + if((!getBackendType().equals(BackendType.MongoDB) && (ap.name().equals(getBackendType())))){ + //in v1.6.0 we are considering only one accessPoint for a backend except for mongodb. + server[0] = ap.address(); + // if presents, try to get user and password + setUsername(ap.username()); + decryptPwd(ap); + setRegion(retrievePropertyValue(ap, "region")); + break; + } } + }else { + int i=0; + + for (AccessPoint ap:resource.profile().accessPoints()) { + if (ap.name().equals("server"+(i+1))) { + // in this case, the backend type has been set in the constructor hence we want to use this one + // if presents, try to get user and password + setUsername(ap.username()); + decryptPwd(ap); + + } + } + setBackendType(retrievePropertyValue(resource, "type")); } - setBackendType(retrievePropertyValue(resource, "type")); + String [] volatileHost= new String [1]; volatileHost[0]=retrievePropertyValue(resource, "volatile"); setVolatileHost(volatileHost); @@ -116,6 +131,9 @@ public class ISClientConnector { } } + + + private void decryptPwd(AccessPoint ap) { // set password default value to empty string setPassword(""); @@ -176,6 +194,17 @@ public class ISClientConnector { } return value; } + + public String retrievePropertyValue(AccessPoint ap, String name) { + String value=null; + Mapmap= ap.propertyMap(); + Property type=map.get(name); + if(type!=null){ + value=type.value(); + + } + return value; + } private String retrievePropertyValue(ServiceEndpoint res, String name) { Iterator it= res.profile().accessPoints().iterator(); @@ -241,6 +270,14 @@ public class ISClientConnector { this.storageResource = storageResource; } + private void setRegion(String region) { + this.region=region; + + } + + public String getRegion() { + return region; + } } \ No newline at end of file diff --git a/src/main/java/org/gcube/contentmanager/storageclient/wrapper/StorageClient.java b/src/main/java/org/gcube/contentmanager/storageclient/wrapper/StorageClient.java index 60a9b96..b375f1f 100644 --- a/src/main/java/org/gcube/contentmanager/storageclient/wrapper/StorageClient.java +++ b/src/main/java/org/gcube/contentmanager/storageclient/wrapper/StorageClient.java @@ -1,9 +1,12 @@ -package org.gcube.contentmanager.storageclient.wrapper; + package org.gcube.contentmanager.storageclient.wrapper; import org.gcube.contentmanager.storageclient.wrapper.BackendType; import org.gcube.contentmanager.storageclient.wrapper.AccessType; //import org.gcube.contentmanagement.blobstorage.resource.MemoryType; import org.gcube.contentmanager.storageclient.wrapper.MemoryType; + +import java.util.Objects; + import org.gcube.common.scope.api.ScopeProvider; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanagement.blobstorage.service.impl.ServiceEngine; @@ -264,7 +267,11 @@ public class StorageClient { * @throws IllegalStateException if the resource is not found on the IS */ public IClient getClient(){ - Configuration cfg= new Configuration(serviceClass, serviceName, scopeString, owner, clientID, typeAccess.toString(), memoryType.toString()); + Configuration cfg=null; + if(Objects.isNull(getBackendType())) + cfg= new Configuration(serviceClass, serviceName, scopeString, owner, clientID, typeAccess.toString(), memoryType.toString()); + else + cfg= new Configuration(serviceClass, serviceName, scopeString, owner, clientID, typeAccess.toString(), memoryType.toString(), getBackendType()); cfg.getConfiguration(); ServiceEngine engine= new ServiceEngine(cfg.getPersistentHosts(), cfg.getVolatileHost(), cfg.getEnvironment(), cfg.getTypeAccess(), cfg.getOwner(), cfg.getMemoryType()); // set additional fields for the new engine object @@ -275,8 +282,11 @@ public class StorageClient { engine.setBackendUser(cfg.getUser()); engine.setBackendPassword(cfg.getPassword()); engine.setResolverHost(cfg.getResolverHost()); - if(cfg.getPassPhrase()!=null) + engine.setRegion(cfg.getRegion()); + if(cfg.getPassPhrase()!=null) { engine.setPassPhrase(cfg.getPassPhrase()); + engine.setToken(cfg.getPassPhrase()); + } if(getMemoryType() !=null) engine.setGcubeMemoryType(getMemoryType().toString()); engine.setGcubeScope(ScopeProvider.instance.get()); diff --git a/src/test/java/org/gcube/contentmanager/storageclient/test/GetMetaFileTest.java b/src/test/java/org/gcube/contentmanager/storageclient/test/GetMetaFileTest.java index 79533e6..9c76161 100644 --- a/src/test/java/org/gcube/contentmanager/storageclient/test/GetMetaFileTest.java +++ b/src/test/java/org/gcube/contentmanager/storageclient/test/GetMetaFileTest.java @@ -6,7 +6,7 @@ import java.io.File; import java.util.List; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.contentmanagement.blobstorage.resource.MyFile; +import org.gcube.contentmanagement.blobstorage.resource.RequestObject; import org.gcube.contentmanagement.blobstorage.resource.StorageObject; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; @@ -53,7 +53,7 @@ public class GetMetaFileTest { @Test public void getMetaFileByPath() throws RemoteBackendException { - MyFile f= client.getMetaFile().RFile(remotePath); + RequestObject f= client.getMetaFile().RFile(remotePath); // System.out.println("mime is: "+f.getMimeType()); client.put(true, "image/png").LFile("src/test/resources/dog.jpg").RFile(remotePath); f= client.getMetaFile().RFile(remotePath); @@ -70,7 +70,7 @@ public class GetMetaFileTest { @Test public void getMetaFileById() throws RemoteBackendException { - MyFile f= client.getMetaFile().RFile(id); + RequestObject f= client.getMetaFile().RFile(id); assertNotNull(f); assertEquals(id, f.getId()); print(f); @@ -89,7 +89,7 @@ public class GetMetaFileTest { assertTrue(list.isEmpty()); } - private void print(MyFile f) { + private void print(RequestObject f) { System.out.println("\t name "+f.getName()); System.out.println("\t size "+f.getSize()); System.out.println("\t owner "+f.getOwner()); diff --git a/src/test/java/org/gcube/contentmanager/storageclient/test/PropertiesTest.java b/src/test/java/org/gcube/contentmanager/storageclient/test/PropertiesTest.java index b4557f7..3d70e6b 100644 --- a/src/test/java/org/gcube/contentmanager/storageclient/test/PropertiesTest.java +++ b/src/test/java/org/gcube/contentmanager/storageclient/test/PropertiesTest.java @@ -6,7 +6,7 @@ import java.io.File; import java.util.List; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.contentmanagement.blobstorage.resource.MyFile; +import org.gcube.contentmanagement.blobstorage.resource.RequestObject; import org.gcube.contentmanagement.blobstorage.resource.StorageObject; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; @@ -53,12 +53,12 @@ public class PropertiesTest { public void getPropertyByPath() throws RemoteBackendException { String valueFound= client.getMetaInfo(field).RFile(remotePath); System.out.println("value found for property: "+field+" is "+valueFound); - MyFile f= client.getMetaFile().RFile(remotePath); + RequestObject f= client.getMetaFile().RFile(remotePath); assertNotNull(f); print(f); String result= client.setMetaInfo(field, value).RFile(remotePath); System.out.println("new property set "); - MyFile f1= client.getMetaFile().RFile(remotePath); + RequestObject f1= client.getMetaFile().RFile(remotePath); print(f1); assertNotNull(f1); assertEquals(f1.getMimeType(), value); @@ -70,12 +70,12 @@ public class PropertiesTest { public void getPropertyById() throws RemoteBackendException { String valueFound= client.getMetaInfo(field).RFile(id); System.out.println("value found for property: "+field+" is "+valueFound); - MyFile f= client.getMetaFile().RFile(id); + RequestObject f= client.getMetaFile().RFile(id); assertNotNull(f); print(f); String result= client.setMetaInfo(field, value).RFile(id); System.out.println("new property set "); - MyFile f1= client.getMetaFile().RFile(id); + RequestObject f1= client.getMetaFile().RFile(id); print(f1); assertNotNull(f1); assertEquals(f1.getMimeType(), value); @@ -85,7 +85,7 @@ public class PropertiesTest { // @Test public void setProperty() throws RemoteBackendException { String result= client.setMetaInfo(field, value).RFile(id); - MyFile f= client.getMetaFile().RFile(remotePath); + RequestObject f= client.getMetaFile().RFile(remotePath); assertNotNull(f); print(f); @@ -99,7 +99,7 @@ public class PropertiesTest { removeLocalFile(); } - private void print(MyFile f) { + private void print(RequestObject f) { System.out.println("\t name "+f.getName()); System.out.println("\t size "+f.getSize()); System.out.println("\t owner "+f.getOwner());