gCubeSystem
/
storage-manager-wrapper
16
0
Fork 0
Code Issues Pull Requests 1 Releases 2 Wiki Activity

retrieve new parameters from ServiceEndpoint: backendToken, region

This commit is contained in:
roberto cirillo 2020-12-22 17:59:21 +01:00
parent bd082de4f2
commit 3fc8eda0a5
5 changed files with 158 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
src/main/java/org/gcube/contentmanager/storageclient/wrapper/Configuration.java
View File

@ -3,6 +3,7 @@ package org.gcube.contentmanager.storageclient.wrapper;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeSet;
import org.gcube.common.resources.gcore.ServiceEndpoint;
@ -28,6 +29,7 @@ public class Configuration {
private String sn;
private String user;
private String password;
// if the backend is mongodb, this field is used for crypt/decrypt. If the backend is S3, this field is a token.
private String passPhrase;
private ISClientConnector isclient;
// the scope used for discovering the runtimeResource
@ -36,12 +38,17 @@ public class Configuration {
private String[] volatileHost;
private String[] persistentHosts;
private String resolverHost;
private String region;
private static final Logger logger = LoggerFactory.getLogger(Configuration.class);
private static final String DEFAULT_BACKEND_TYPE="MongoDB";
private static final String WRITE_CONCERN_PROPERTY_NAME="write_concern";
private static final String READ_PREFERENCE_PROPERTY_NAME="read_preference";
private static final String DB_PROPERTY_NAME="collection";
private static final String DB_STRING_SEPARATOR=";";
private static final String PASS_PHRASE_FIELD="PassPhrase";
private static final String TOKEN_FIELD="token";
/**
* home library need a special configuration
*/
@ -85,7 +92,7 @@ public class Configuration {
* @param memory indicates the type of memory used by the storage: Persistent or Volatile
*
*/
public Configuration(String sc, String sn, String scopeString, String owner, String clientID, String accessType, String memory, BackendType backend){
public Configuration(String sc, String sn, String scopeString, String owner, String clientID, String accessType, String memory, String backend){
this.sc=sc;
this.sn=sn;
this.owner=owner;
@ -93,7 +100,7 @@ public class Configuration {
this.typeAccess=accessType;
this.memoryType=memory;
setScopeString(scopeString);
setBackendType(backend.toString());
setBackendType(backend);
}
/**
@ -108,6 +115,10 @@ public class Configuration {
}
logger.debug("RuntimeResource scope "+RRScope);
ServiceEndpoint resource=getISClient().getStorageEndpoint(RRScope);
//in this case a specific backend type is passed as input parameter
if (!Objects.isNull(getBackendType()))
getISClient().setBackendType(getBackendType());
if(resource ==null )
throw new IllegalStateException("the storage resource is not present on IS in scope: "+RRScope);
List<ServiceEndpoint> resolverResource =getISClient().getServiceEndpoint(Utils.URI_RESOLVER_RESOURCE_CATEGORY, Utils.URI_RESOLVER_RESOURCE_NAME);
@ -123,10 +134,14 @@ public class Configuration {
serverFound=getISClient().retrieveConnectionInfo(resource);
setUser(getISClient().getUsername());
setPassword(getISClient().password);
setBackendType(getISClient().getBackendType(resource));
setRegion(getISClient().getRegion());
if (!Objects.isNull(getBackendType()))
setBackendType(getISClient().getBackendType(resource));
}else{
setBackendType(checkVarEnvBackendType());
if(getBackendType() == null) setBackendType(DEFAULT_BACKEND_TYPE);
if (!Objects.isNull(getBackendType()))
setBackendType(checkVarEnvBackendType());
else
setBackendType(DEFAULT_BACKEND_TYPE);
setUser(checkVarEnvUser());
setPassword(checkVarEnvPassword());
}
@ -140,15 +155,46 @@ public class Configuration {
if(newServer==null){
throw new IllegalStateException("Resource not found on Information System");
}else{
setEnvironment(setAreaStorage(getSc(), getSn()));
setServerHosts(newServer, isclient, resource);
try {
setPassPhrase(retrieveEncryptionPhrase());
} catch (Exception e) {
e.printStackTrace();
setEnvironment(setAreaStorage(getSc(), getSn()));
// if the backend is mongodb we should set a separated server for volatile area
if(getBackendType().equals(BackendType.MongoDB)) {
setServerHosts(newServer, isclient, resource);
try {
setPassPhrase(retrieveEncryptionField(PASS_PHRASE_FIELD));
} catch (Exception e) {
e.printStackTrace();
}
}else {
// if the backend isn't mongodb, the passPhrase field will be filled with a token if it is present on ServiceEndpoint
try {
setPassPhrase(retrieveEncryptionField(TOKEN_FIELD));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
setPersistentHosts(newServer);
}
}
logger.debug("Configuration parameters retrieved: host: "+getPersistentHosts()+" user: "+getUser());
}
// /**
// * Retrieve a valid configuration from IS for instantiating the engine
// */
// public void getConfiguration2(){
// String currentScope=ScopeProvider.instance.get();
// ServiceEndpoint resource=getISClient().getStorageEndpoint(currentScope);
// if((getMemoryType() != null) && ((getMemoryType().equals(MemoryType.VOLATILE.toString()) || (getMemoryType().equals(MemoryType.BOTH.toString()))))){
// setVolatileHost(isclient.getVolatileHost(resource));
// }
// }
//
private void setServerHosts(String[] newServer, ISClientConnector isclient, ServiceEndpoint resource) {
if((getMemoryType() != null) && ((getMemoryType().equals(MemoryType.VOLATILE.toString()) || (getMemoryType().equals(MemoryType.BOTH.toString()))))){
@ -222,7 +268,8 @@ public class Configuration {
}
if (dbNames.length > 1)
logger.debug("multiple collection discovered");
engine.setDbNames(dbNames);
if (Objects.isNull(engine.getDbNames()))
engine.setDbNames(dbNames);
}
// added db check also on volatile are
}else if((getMemoryType().equals(MemoryType.VOLATILE.toString()))){
@ -338,7 +385,7 @@ public class Configuration {
return null;
}
@Deprecated
public String retrieveEncryptionPhrase() throws Exception {
String currentScope=ScopeProvider.instance.get();
logger.debug("retrieve encryption prhase on scope: "+currentScope);
@ -351,6 +398,18 @@ public class Configuration {
return decryptString;
}
public String retrieveEncryptionField(String fieldName) throws Exception {
String currentScope=ScopeProvider.instance.get();
logger.debug("retrieve encryption prhase on scope: "+currentScope);
String encryptedKey=null;
// ISClientConnector isclient=getISClient();
logger.info("retrieve encryption phrase from scope "+currentScope);
encryptedKey=getISClient().retrievePropertyValue(fieldName, currentScope);
logger.info("encrypted prhase is "+encryptedKey);
String decryptString=(!Objects.isNull(encryptedKey))?org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(encryptedKey):null;
return decryptString;
}
public String getClientID() {
return clientID;
@ -493,5 +552,13 @@ public class Configuration {
public void setSn(String sn) {
this.sn = sn;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
}

71
src/main/java/org/gcube/contentmanager/storageclient/wrapper/ISClientConnector.java
View File

@ -8,6 +8,8 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.contentmanager.storageclient.wrapper.BackendType;
@ -35,6 +37,7 @@ public class ISClientConnector {
protected String username;
protected String password;
protected ServiceEndpoint storageResource;
public String region;
private static HashMap<String, Object> isCache;
public ISClientConnector(){
@ -85,26 +88,38 @@ public class ISClientConnector {
private String[] fillConnectionFields(ServiceEndpoint resource) {
logger.debug("fillConnectionsFields method ");
if(resource!=null){
String [] server=new String[resource.profile().accessPoints().size()];
int i=0;
for (AccessPoint ap:resource.profile().accessPoints()) {
if (ap.name().equals("server"+(i+1))) {
server[i] = ap.address();
// if presents, try to get user and password
setUsername(ap.username());
decryptPwd(ap);
i++;
}else if((!getBackendType().equals(BackendType.MongoDB) && (ap.name().equals(getBackendType())))){
// in this case, the backend type has been set in the constructor hence we want to use this one
// if presents, try to get user and password
setUsername(ap.username());
decryptPwd(ap);
//in v1.6.0 we are considering only one accessPoint for a backend except for mongodb.
break;
logger.debug("checking resource "+resource.id());
String [] server=new String[resource.profile().accessPoints().size()];
if(!Objects.isNull(getBackendType())) {
logger.info("searching backend: "+getBackendType());
for (AccessPoint ap:resource.profile().accessPoints()) {
if((!getBackendType().equals(BackendType.MongoDB) && (ap.name().equals(getBackendType())))){
//in v1.6.0 we are considering only one accessPoint for a backend except for mongodb.
server[0] = ap.address();
// if presents, try to get user and password
setUsername(ap.username());
decryptPwd(ap);
setRegion(retrievePropertyValue(ap, "region"));
break;
}
}
}else {
int i=0;
for (AccessPoint ap:resource.profile().accessPoints()) {
if (ap.name().equals("server"+(i+1))) {
// in this case, the backend type has been set in the constructor hence we want to use this one
// if presents, try to get user and password
setUsername(ap.username());
decryptPwd(ap);
}
}
setBackendType(retrievePropertyValue(resource, "type"));
}
setBackendType(retrievePropertyValue(resource, "type"));
String [] volatileHost= new String [1];
volatileHost[0]=retrievePropertyValue(resource, "volatile");
setVolatileHost(volatileHost);
@ -116,6 +131,9 @@ public class ISClientConnector {
}
}
private void decryptPwd(AccessPoint ap) {
// set password default value to empty string
setPassword("");
@ -176,6 +194,17 @@ public class ISClientConnector {
}
return value;
}
public String retrievePropertyValue(AccessPoint ap, String name) {
String value=null;
Map<String, Property>map= ap.propertyMap();
Property type=map.get(name);
if(type!=null){
value=type.value();
}
return value;
}
private String retrievePropertyValue(ServiceEndpoint res, String name) {
Iterator<AccessPoint> it= res.profile().accessPoints().iterator();
@ -241,6 +270,14 @@ public class ISClientConnector {
this.storageResource = storageResource;
}
private void setRegion(String region) {
this.region=region;
}
public String getRegion() {
return region;
}
}

16
src/main/java/org/gcube/contentmanager/storageclient/wrapper/StorageClient.java
View File

@ -1,9 +1,12 @@
package org.gcube.contentmanager.storageclient.wrapper;
package org.gcube.contentmanager.storageclient.wrapper;
import org.gcube.contentmanager.storageclient.wrapper.BackendType;
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
//import org.gcube.contentmanagement.blobstorage.resource.MemoryType;
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
import java.util.Objects;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.blobstorage.service.impl.ServiceEngine;
@ -264,7 +267,11 @@ public class StorageClient {
* @throws IllegalStateException if the resource is not found on the IS
*/
public IClient getClient(){
Configuration cfg= new Configuration(serviceClass, serviceName, scopeString, owner, clientID, typeAccess.toString(), memoryType.toString());
Configuration cfg=null;
if(Objects.isNull(getBackendType()))
cfg= new Configuration(serviceClass, serviceName, scopeString, owner, clientID, typeAccess.toString(), memoryType.toString());
else
cfg= new Configuration(serviceClass, serviceName, scopeString, owner, clientID, typeAccess.toString(), memoryType.toString(), getBackendType());
cfg.getConfiguration();
ServiceEngine engine= new ServiceEngine(cfg.getPersistentHosts(), cfg.getVolatileHost(), cfg.getEnvironment(), cfg.getTypeAccess(), cfg.getOwner(), cfg.getMemoryType());
// set additional fields for the new engine object
@ -275,8 +282,11 @@ public class StorageClient {
engine.setBackendUser(cfg.getUser());
engine.setBackendPassword(cfg.getPassword());
engine.setResolverHost(cfg.getResolverHost());
if(cfg.getPassPhrase()!=null)
engine.setRegion(cfg.getRegion());
if(cfg.getPassPhrase()!=null) {
engine.setPassPhrase(cfg.getPassPhrase());
engine.setToken(cfg.getPassPhrase());
}
if(getMemoryType() !=null)
engine.setGcubeMemoryType(getMemoryType().toString());
engine.setGcubeScope(ScopeProvider.instance.get());

8
src/test/java/org/gcube/contentmanager/storageclient/test/GetMetaFileTest.java
View File

@ -6,7 +6,7 @@ import java.io.File;
import java.util.List;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.resource.MyFile;
import org.gcube.contentmanagement.blobstorage.resource.RequestObject;
import org.gcube.contentmanagement.blobstorage.resource.StorageObject;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
@ -53,7 +53,7 @@ public class GetMetaFileTest {
@Test
public void getMetaFileByPath() throws RemoteBackendException {
MyFile f= client.getMetaFile().RFile(remotePath);
RequestObject f= client.getMetaFile().RFile(remotePath);
// System.out.println("mime is: "+f.getMimeType());
client.put(true, "image/png").LFile("src/test/resources/dog.jpg").RFile(remotePath);
f= client.getMetaFile().RFile(remotePath);
@ -70,7 +70,7 @@ public class GetMetaFileTest {
@Test
public void getMetaFileById() throws RemoteBackendException {
MyFile f= client.getMetaFile().RFile(id);
RequestObject f= client.getMetaFile().RFile(id);
assertNotNull(f);
assertEquals(id, f.getId());
print(f);
@ -89,7 +89,7 @@ public class GetMetaFileTest {
assertTrue(list.isEmpty());
}
private void print(MyFile f) {
private void print(RequestObject f) {
System.out.println("\t name "+f.getName());
System.out.println("\t size "+f.getSize());
System.out.println("\t owner "+f.getOwner());

14
src/test/java/org/gcube/contentmanager/storageclient/test/PropertiesTest.java
View File

@ -6,7 +6,7 @@ import java.io.File;
import java.util.List;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.resource.MyFile;
import org.gcube.contentmanagement.blobstorage.resource.RequestObject;
import org.gcube.contentmanagement.blobstorage.resource.StorageObject;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException;
@ -53,12 +53,12 @@ public class PropertiesTest {
public void getPropertyByPath() throws RemoteBackendException {
String valueFound= client.getMetaInfo(field).RFile(remotePath);
System.out.println("value found for property: "+field+" is "+valueFound);
MyFile f= client.getMetaFile().RFile(remotePath);
RequestObject f= client.getMetaFile().RFile(remotePath);
assertNotNull(f);
print(f);
String result= client.setMetaInfo(field, value).RFile(remotePath);
System.out.println("new property set ");
MyFile f1= client.getMetaFile().RFile(remotePath);
RequestObject f1= client.getMetaFile().RFile(remotePath);
print(f1);
assertNotNull(f1);
assertEquals(f1.getMimeType(), value);
@ -70,12 +70,12 @@ public class PropertiesTest {
public void getPropertyById() throws RemoteBackendException {
String valueFound= client.getMetaInfo(field).RFile(id);
System.out.println("value found for property: "+field+" is "+valueFound);
MyFile f= client.getMetaFile().RFile(id);
RequestObject f= client.getMetaFile().RFile(id);
assertNotNull(f);
print(f);
String result= client.setMetaInfo(field, value).RFile(id);
System.out.println("new property set ");
MyFile f1= client.getMetaFile().RFile(id);
RequestObject f1= client.getMetaFile().RFile(id);
print(f1);
assertNotNull(f1);
assertEquals(f1.getMimeType(), value);
@ -85,7 +85,7 @@ public class PropertiesTest {
// @Test
public void setProperty() throws RemoteBackendException {
String result= client.setMetaInfo(field, value).RFile(id);
MyFile f= client.getMetaFile().RFile(remotePath);
RequestObject f= client.getMetaFile().RFile(remotePath);
assertNotNull(f);
print(f);
@ -99,7 +99,7 @@ public class PropertiesTest {
removeLocalFile();
}
private void print(MyFile f) {
private void print(RequestObject f) {
System.out.println("\t name "+f.getName());
System.out.println("\t size "+f.getSize());
System.out.println("\t owner "+f.getOwner());