Compare commits
1 Commits
master
...
migrating_
Author | SHA1 | Date |
---|---|---|
Luca Frosini | 8da630591d |
|
@ -2,5 +2,3 @@ target
|
||||||
.classpath
|
.classpath
|
||||||
.project
|
.project
|
||||||
.settings
|
.settings
|
||||||
/.DS_Store
|
|
||||||
/bin/
|
|
||||||
|
|
19
CHANGELOG.md
19
CHANGELOG.md
|
@ -2,27 +2,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for StorageHub Application Persistence
|
# Changelog for StorageHub Application Persistence
|
||||||
|
|
||||||
## [v3.5.0]
|
## [v4.0.0-SNAPSHOT]
|
||||||
|
|
||||||
- Enhanced gcube-bom version to 3.4.1 [#27999]
|
|
||||||
- Updated maven-parent version to 1.2.0
|
|
||||||
|
|
||||||
|
|
||||||
## [v3.4.0]
|
|
||||||
|
|
||||||
- The persistence of file already stored in the workspace can be ensured by using the copy provided by storage-hub #27736
|
|
||||||
- The removal of a file persited is resolved using the public link #27952
|
|
||||||
|
|
||||||
## [v3.3.0]
|
|
||||||
|
|
||||||
- Added GENERATING_APPLICATION_NAME and GENERATING_APPLICATION_METADATA_VERSION to be used in Metadata
|
|
||||||
|
|
||||||
|
- Library migrated to Smartgears 4
|
||||||
|
|
||||||
## [v3.2.0]
|
## [v3.2.0]
|
||||||
|
|
||||||
- Added dependency to be able to compile with JDK 11
|
- Added dependency to be able to compile with JDK 11
|
||||||
|
|
||||||
|
|
||||||
## [v3.1.0]
|
## [v3.1.0]
|
||||||
|
|
||||||
- Enhanced range of storagehub-client-library to 2.0.0,3.0.0-SNAPSHOT [#22777]
|
- Enhanced range of storagehub-client-library to 2.0.0,3.0.0-SNAPSHOT [#22777]
|
||||||
|
@ -37,7 +24,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
- Removed home library [#21435]
|
- Removed home library [#21435]
|
||||||
- Switched HTTP requests gxHTTP 2.0.0 [#19283]
|
- Switched HTTP requests gxHTTP 2.0.0 [#19283]
|
||||||
- Switched gcube-bom to 2.0.0 [#19283]
|
- Switched gCube BOM to 2.0.0 [#19283]
|
||||||
- Migrating to the IAM [#22472]
|
- Migrating to the IAM [#22472]
|
||||||
|
|
||||||
|
|
||||||
|
|
23
pom.xml
23
pom.xml
|
@ -5,12 +5,12 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.tools</groupId>
|
<groupId>org.gcube.tools</groupId>
|
||||||
<artifactId>maven-parent</artifactId>
|
<artifactId>maven-parent</artifactId>
|
||||||
<version>1.2.0</version>
|
<version>1.1.0</version>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<groupId>org.gcube.data-publishing</groupId>
|
<groupId>org.gcube.data-publishing</groupId>
|
||||||
<artifactId>storagehub-application-persistence</artifactId>
|
<artifactId>storagehub-application-persistence</artifactId>
|
||||||
<version>3.5.0</version>
|
<version>4.0.0-SNAPSHOT</version>
|
||||||
<name>StorageHub Application Persistence</name>
|
<name>StorageHub Application Persistence</name>
|
||||||
<description>
|
<description>
|
||||||
This library allows any application to persist in its workspace home any
|
This library allows any application to persist in its workspace home any
|
||||||
|
@ -37,7 +37,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.distribution</groupId>
|
<groupId>org.gcube.distribution</groupId>
|
||||||
<artifactId>gcube-bom</artifactId>
|
<artifactId>gcube-bom</artifactId>
|
||||||
<version>2.4.1</version>
|
<version>3.0.0-SNAPSHOT</version>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
@ -54,12 +54,7 @@
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.common</groupId>
|
<groupId>org.gcube.common</groupId>
|
||||||
<artifactId>authorization-utils</artifactId>
|
<artifactId>common-security</artifactId>
|
||||||
<version>[2.2.0, 3.0.0-SNAPSHOT)</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.gcube.core</groupId>
|
|
||||||
<artifactId>common-encryption</artifactId>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.projectlombok</groupId>
|
<groupId>org.projectlombok</groupId>
|
||||||
|
@ -79,5 +74,15 @@
|
||||||
<artifactId>logback-classic</artifactId>
|
<artifactId>logback-classic</artifactId>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.common</groupId>
|
||||||
|
<artifactId>keycloak-client</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.common.security</groupId>
|
||||||
|
<artifactId>gcube-secrets</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -1,44 +1,9 @@
|
||||||
package org.gcube.storagehub;
|
package org.gcube.storagehub;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import org.gcube.common.storagehub.model.Metadata;
|
import org.gcube.common.storagehub.model.Metadata;
|
||||||
|
|
||||||
/**
|
public interface MetadataMatcher {
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
|
||||||
*/
|
|
||||||
public abstract class MetadataMatcher {
|
|
||||||
|
|
||||||
public static final String GENERATING_METADATA_NAME = "Generating Metadata Name";
|
public boolean check(Metadata metadata);
|
||||||
public static final String GENERATING_METADATA_VERSION = "Generating Metadata Version";
|
|
||||||
|
|
||||||
protected final String metadataName;
|
|
||||||
protected final String metadataVersion;
|
|
||||||
protected final String id;
|
|
||||||
|
|
||||||
protected MetadataMatcher(String metadataName, String metadataVersion, String id) {
|
|
||||||
this.metadataName = metadataName;
|
|
||||||
this.metadataVersion = metadataVersion;
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public abstract boolean check(Metadata metadata);
|
|
||||||
|
|
||||||
protected Map<String,Object> getBaseMetadataMap() {
|
|
||||||
Map<String,Object> map = new HashMap<>();
|
|
||||||
map.put(MetadataMatcher.GENERATING_METADATA_NAME, metadataName);
|
|
||||||
map.put(MetadataMatcher.GENERATING_METADATA_VERSION, metadataVersion);
|
|
||||||
return map;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected abstract Map<String,Object> getSpecificMetadataMap();
|
|
||||||
|
|
||||||
public Metadata getMetadata() {
|
|
||||||
Map<String, Object> map = getBaseMetadataMap();
|
|
||||||
map.putAll(getSpecificMetadataMap());
|
|
||||||
Metadata metadata = new Metadata(map);
|
|
||||||
return metadata;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,21 +3,18 @@ package org.gcube.storagehub;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.Base64;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManager;
|
import org.gcube.common.security.ContextBean;
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
import org.gcube.common.security.ContextBean.Type;
|
||||||
import org.gcube.common.encryption.encrypter.StringEncrypter;
|
import org.gcube.common.security.providers.SecretManagerProvider;
|
||||||
import org.gcube.common.scope.impl.ScopeBean;
|
import org.gcube.common.security.secrets.Secret;
|
||||||
import org.gcube.common.scope.impl.ScopeBean.Type;
|
|
||||||
import org.gcube.common.storagehub.client.dsl.ContainerType;
|
import org.gcube.common.storagehub.client.dsl.ContainerType;
|
||||||
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
||||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||||
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
||||||
import org.gcube.common.storagehub.client.dsl.ListResolver;
|
import org.gcube.common.storagehub.client.dsl.ListResolver;
|
||||||
import org.gcube.common.storagehub.client.dsl.ListResolverTyped;
|
import org.gcube.common.storagehub.client.dsl.ListResolverTyped;
|
||||||
import org.gcube.common.storagehub.client.dsl.OpenResolver;
|
|
||||||
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
||||||
import org.gcube.common.storagehub.model.Metadata;
|
import org.gcube.common.storagehub.model.Metadata;
|
||||||
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
||||||
|
@ -33,10 +30,6 @@ public class StorageHubManagement {
|
||||||
|
|
||||||
protected final StorageHubClient storageHubClient;
|
protected final StorageHubClient storageHubClient;
|
||||||
|
|
||||||
public StorageHubClient getStorageHubClient() {
|
|
||||||
return storageHubClient;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected FileContainer persitedFile;
|
protected FileContainer persitedFile;
|
||||||
protected String mimeType;
|
protected String mimeType;
|
||||||
|
|
||||||
|
@ -44,7 +37,7 @@ public class StorageHubManagement {
|
||||||
storageHubClient = new StorageHubClient();
|
storageHubClient = new StorageHubClient();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setMetadataMatcher(MetadataMatcher checkMetadata) {
|
public void setCheckMetadata(MetadataMatcher checkMetadata) {
|
||||||
this.metadataMatcher = checkMetadata;
|
this.metadataMatcher = checkMetadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -60,8 +53,14 @@ public class StorageHubManagement {
|
||||||
ListResolverTyped listResolverTyped = folder.list();
|
ListResolverTyped listResolverTyped = folder.list();
|
||||||
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
|
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
|
||||||
for(ItemContainer<? extends Item> itemContainer : containers) {
|
for(ItemContainer<? extends Item> itemContainer : containers) {
|
||||||
|
Item item = itemContainer.get();
|
||||||
|
String name = item.getName();
|
||||||
ContainerType containerType = itemContainer.getType();
|
ContainerType containerType = itemContainer.getType();
|
||||||
logItem(itemContainer, level);
|
StringWriter indent = new StringWriter(level + 1);
|
||||||
|
for(int i = 0; i < level + 1; i++) {
|
||||||
|
indent.append('-');
|
||||||
|
}
|
||||||
|
logger.debug("{} {} {} (ID:{}) {}", indent.toString(), containerType, name, itemContainer.getId(), item.isHidden() ? " (hidden)" : "");
|
||||||
switch(containerType) {
|
switch(containerType) {
|
||||||
case FOLDER:
|
case FOLDER:
|
||||||
FolderContainer folderContainer = (FolderContainer) itemContainer;
|
FolderContainer folderContainer = (FolderContainer) itemContainer;
|
||||||
|
@ -109,8 +108,8 @@ public class StorageHubManagement {
|
||||||
protected FolderContainer getContextFolder() throws Exception {
|
protected FolderContainer getContextFolder() throws Exception {
|
||||||
FolderContainer destinationFolder = getWorkspaceRoot();
|
FolderContainer destinationFolder = getWorkspaceRoot();
|
||||||
String currentContext = SecretManagerProvider.instance.get().getContext();
|
String currentContext = SecretManagerProvider.instance.get().getContext();
|
||||||
ScopeBean scopeBean = new ScopeBean(currentContext);
|
ContextBean contextBean = new ContextBean(currentContext);
|
||||||
switch(scopeBean.type()) {
|
switch(contextBean.type()) {
|
||||||
case INFRASTRUCTURE:
|
case INFRASTRUCTURE:
|
||||||
case VO:
|
case VO:
|
||||||
String folderName = currentContext.replaceFirst("/", "").replace("/", "_");
|
String folderName = currentContext.replaceFirst("/", "").replace("/", "_");
|
||||||
|
@ -130,11 +129,11 @@ public class StorageHubManagement {
|
||||||
|
|
||||||
public FolderContainer getApplicationFolder() throws Exception {
|
public FolderContainer getApplicationFolder() throws Exception {
|
||||||
FolderContainer destinationFolder = getContextFolder();
|
FolderContainer destinationFolder = getContextFolder();
|
||||||
SecretManager secretManager = SecretManagerProvider.instance.get();
|
Secret secret = SecretManagerProvider.instance.get();
|
||||||
String currentContext = secretManager.getContext();
|
String currentContext = secret.getContext();
|
||||||
ScopeBean scopeBean = new ScopeBean(currentContext);
|
ContextBean contextBean = new ContextBean(currentContext);
|
||||||
if(scopeBean.is(Type.VRE)) {
|
if(contextBean.is(Type.VRE)) {
|
||||||
String username = secretManager.getUser().getUsername();
|
String username = secret.getOwner().getId();
|
||||||
destinationFolder = getOrCreateFolder(destinationFolder, username, "Folder Created for user/application", true);
|
destinationFolder = getOrCreateFolder(destinationFolder, username, "Folder Created for user/application", true);
|
||||||
}
|
}
|
||||||
return destinationFolder;
|
return destinationFolder;
|
||||||
|
@ -162,92 +161,24 @@ public class StorageHubManagement {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void logItem(ItemContainer<? extends Item> itemContainer) {
|
|
||||||
logItem(itemContainer, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void logItem(ItemContainer<? extends Item> itemContainer, int level) {
|
|
||||||
StringWriter indent = new StringWriter(level + 1);
|
|
||||||
for(int i = 0; i < level+1; i++) {
|
|
||||||
indent.append('-');
|
|
||||||
}
|
|
||||||
indent.append(" ");
|
|
||||||
Item item = itemContainer.get();
|
|
||||||
logger.debug("{}{} {} (ID:{}){}", indent.toString(), itemContainer.getType(), item.getName(), itemContainer.getId(), item.isHidden() ? " (hidden)" : "");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void tree(FolderContainer folderContainer) throws Exception {
|
protected void tree(FolderContainer folderContainer) throws Exception {
|
||||||
logItem(folderContainer,0);
|
logger.debug("{} (ID:{})", folderContainer.get().getName(), folderContainer.getId());
|
||||||
recursiveList(folderContainer, 1);
|
recursiveList(folderContainer, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getPublicLinkID(URL url) {
|
public URL persistFile(InputStream inputStream, String fileName, String mimeType, Metadata metadata)
|
||||||
String path = url.getPath().toString();
|
|
||||||
if(path.endsWith("/")) {
|
|
||||||
path = path.substring(0,path.length()-1);
|
|
||||||
}
|
|
||||||
String[] parts = path.split("/");
|
|
||||||
String id = parts[parts.length-1];
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static final String enchriptedPrefix = "E_";
|
|
||||||
|
|
||||||
public static String getFileId(URL url) throws Exception {
|
|
||||||
String encId = getPublicLinkID(url);
|
|
||||||
encId = encId.replace(enchriptedPrefix, "");
|
|
||||||
String decodeURL = new String(Base64.getUrlDecoder().decode(encId));
|
|
||||||
String decodedId = StringEncrypter.getEncrypter().decrypt(decodeURL);
|
|
||||||
logger.trace("From public link URL {} has been extraced the file id={}", url.toString(), decodedId);
|
|
||||||
return decodedId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FileContainer getFileContainer(URL shubPublicLink) throws Exception {
|
|
||||||
String id = getFileId(shubPublicLink);
|
|
||||||
OpenResolver openResolver = storageHubClient.open(id);
|
|
||||||
FileContainer fileContainer = openResolver.asFile();
|
|
||||||
return fileContainer;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This method can be used only if the caller has the right to access
|
|
||||||
* the file already stored in the workspace
|
|
||||||
* @param shubPublicLink
|
|
||||||
* @param fileName
|
|
||||||
* @param mimeType
|
|
||||||
* @return the public link of the persisted file
|
|
||||||
* @throws Exception
|
|
||||||
*/
|
|
||||||
public URL persistFile(URL shubPublicLink, String fileName, String mimeType)
|
|
||||||
throws Exception {
|
|
||||||
this.mimeType = mimeType;
|
|
||||||
FolderContainer destinationFolder = getDestinationFolder(mimeType);
|
|
||||||
|
|
||||||
FileContainer fileContainer = getFileContainer(shubPublicLink);
|
|
||||||
persitedFile = fileContainer.copy(destinationFolder, fileName);
|
|
||||||
|
|
||||||
if(metadataMatcher != null) {
|
|
||||||
persitedFile.setMetadata(metadataMatcher.getMetadata());
|
|
||||||
}
|
|
||||||
|
|
||||||
URL finalURL = persitedFile.getPublicLink();
|
|
||||||
logger.debug("File persistence has been ensured. The file is available at {}", finalURL);
|
|
||||||
return finalURL;
|
|
||||||
}
|
|
||||||
|
|
||||||
public URL persistFile(InputStream inputStream, String fileName, String mimeType)
|
|
||||||
throws Exception {
|
throws Exception {
|
||||||
this.mimeType = mimeType;
|
this.mimeType = mimeType;
|
||||||
FolderContainer destinationFolder = getDestinationFolder(mimeType);
|
FolderContainer destinationFolder = getDestinationFolder(mimeType);
|
||||||
persitedFile = destinationFolder.uploadFile(inputStream, fileName,
|
persitedFile = destinationFolder.uploadFile(inputStream, fileName,
|
||||||
"This file has been created to ensure persistence");
|
"This file has been created to ensure persistence");
|
||||||
|
|
||||||
if(metadataMatcher != null) {
|
if(metadata != null) {
|
||||||
persitedFile.setMetadata(metadataMatcher.getMetadata());
|
persitedFile.setMetadata(metadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
URL finalURL = persitedFile.getPublicLink();
|
URL finalURL = persitedFile.getPublicLink();
|
||||||
logger.debug("File persistence has been ensured (id={}. The file is available at {}", persitedFile.getId(), finalURL);
|
logger.debug("File persistence has been ensured. The file is available at {}", finalURL);
|
||||||
return finalURL;
|
return finalURL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -258,46 +189,19 @@ public class StorageHubManagement {
|
||||||
for(ItemContainer<? extends Item> itemContainer : itemContainers) {
|
for(ItemContainer<? extends Item> itemContainer : itemContainers) {
|
||||||
if(itemContainer.getType()==ContainerType.FILE) {
|
if(itemContainer.getType()==ContainerType.FILE) {
|
||||||
if(isPersistedFile((FileContainer) itemContainer, filename)) {
|
if(isPersistedFile((FileContainer) itemContainer, filename)) {
|
||||||
logger.debug("The file with mimetype {} and name {} was found in the expected folder (i.e. id:{}, path:{}) and the check on metadata succeded. The file is the one expected.",
|
|
||||||
mimeType, filename, destinationFolder.getId(), destinationFolder.get().getPath());
|
|
||||||
this.persitedFile = (FileContainer) itemContainer;
|
this.persitedFile = (FileContainer) itemContainer;
|
||||||
return this.persitedFile;
|
return this.persitedFile;
|
||||||
}else {
|
|
||||||
logger.warn("The file with mimetype {} and name {} was found in the expected folder (i.e. id:{}, path:{}) but the check on metadata failed. The file is not the one expected.",
|
|
||||||
mimeType, filename, destinationFolder.getId(), destinationFolder.get().getPath());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.warn("Unable to find file with mimetype {} and name {} in the expected folder (i.e. id:{}, path:{})",
|
return this.persitedFile;
|
||||||
mimeType, filename, destinationFolder.getId(), destinationFolder.get().getPath());
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* The caller must have the rights to access the file
|
|
||||||
* @param filename
|
|
||||||
* @param mimeType
|
|
||||||
* @throws Exception
|
|
||||||
*/
|
|
||||||
@Deprecated
|
|
||||||
public void removePersistedFile(String filename, String mimeType) throws Exception {
|
public void removePersistedFile(String filename, String mimeType) throws Exception {
|
||||||
persitedFile = getPersistedFile(filename, mimeType);
|
getPersistedFile(filename, mimeType);
|
||||||
if(persitedFile !=null) {
|
if(this.persitedFile !=null) {
|
||||||
logger.info("Persited file with mimetype {} and name {} was found (id={}). Goign to remove it.", mimeType, filename, persitedFile.getId());
|
this.persitedFile.delete();
|
||||||
persitedFile.forceDelete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The caller must have the rights to access the file pointed to the public link
|
|
||||||
* @param shubPublicLink
|
|
||||||
* @throws Exception
|
|
||||||
*/
|
|
||||||
public void removePersistedFile(URL shubPublicLink) throws Exception {
|
|
||||||
persitedFile = getFileContainer(shubPublicLink);
|
|
||||||
if(persitedFile !=null) {
|
|
||||||
logger.info("Persited file available at {} was found (id={}). Goign to remove it.", shubPublicLink, persitedFile.getId());
|
|
||||||
persitedFile.forceDelete();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,165 +7,67 @@ import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManager;
|
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
|
||||||
import org.gcube.common.authorization.utils.secret.JWTSecret;
|
|
||||||
import org.gcube.common.authorization.utils.secret.Secret;
|
|
||||||
import org.gcube.common.authorization.utils.secret.SecretUtility;
|
|
||||||
import org.gcube.common.keycloak.KeycloakClientFactory;
|
import org.gcube.common.keycloak.KeycloakClientFactory;
|
||||||
import org.gcube.common.keycloak.KeycloakClientHelper;
|
|
||||||
import org.gcube.common.keycloak.model.TokenResponse;
|
import org.gcube.common.keycloak.model.TokenResponse;
|
||||||
|
import org.gcube.common.security.providers.SecretManagerProvider;
|
||||||
|
import org.gcube.common.security.secrets.AccessTokenSecret;
|
||||||
|
import org.gcube.common.security.secrets.Secret;
|
||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
*/
|
*/
|
||||||
public class ContextTest {
|
public class ContextTest {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(ContextTest.class);
|
protected static Properties properties;
|
||||||
|
protected static final String PROPERTIES_FILENAME = "token.properties";
|
||||||
|
|
||||||
protected static final String CONFIG_INI_FILENAME = "config.ini";
|
public static final String ROOT;
|
||||||
|
public static final String VO;
|
||||||
public static final String DEFAULT_TEST_SCOPE;
|
public static final String VRE;
|
||||||
|
|
||||||
public static final String GCUBE;
|
|
||||||
public static final String DEVNEXT;
|
|
||||||
public static final String NEXTNEXT;
|
|
||||||
public static final String DEVSEC;
|
|
||||||
public static final String DEVVRE;
|
|
||||||
|
|
||||||
public static final String ROOT_PRE;
|
|
||||||
public static final String ROOT_PROD;
|
|
||||||
|
|
||||||
protected static final Properties properties;
|
|
||||||
|
|
||||||
public static final String TYPE_PROPERTY_KEY = "type";
|
|
||||||
public static final String USERNAME_PROPERTY_KEY = "username";
|
|
||||||
public static final String PASSWORD_PROPERTY_KEY = "password";
|
|
||||||
public static final String CLIENT_ID_PROPERTY_KEY = "clientId";
|
|
||||||
|
|
||||||
static {
|
static {
|
||||||
GCUBE = "/gcube";
|
|
||||||
DEVNEXT = GCUBE + "/devNext";
|
|
||||||
NEXTNEXT = DEVNEXT + "/NextNext";
|
|
||||||
DEVSEC = GCUBE + "/devsec";
|
|
||||||
DEVVRE = DEVSEC + "/devVRE";
|
|
||||||
|
|
||||||
ROOT_PRE = "/pred4s";
|
|
||||||
ROOT_PROD = "/d4science.research-infrastructures.eu";
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_TEST_SCOPE = DEVVRE;
|
|
||||||
|
|
||||||
|
|
||||||
properties = new Properties();
|
properties = new Properties();
|
||||||
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(CONFIG_INI_FILENAME);
|
InputStream input = ContextTest.class.getClassLoader().getResourceAsStream(PROPERTIES_FILENAME);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// load the properties file
|
// load the properties file
|
||||||
properties.load(input);
|
properties.load(input);
|
||||||
} catch (IOException e) {
|
} catch(IOException e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE";
|
||||||
|
// DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE";
|
||||||
|
|
||||||
|
ROOT = "/gcube";
|
||||||
|
VO = ROOT + "/devsec";
|
||||||
|
VRE = VO + "/devVRE";
|
||||||
|
// VO = ROOT + "/devNext";
|
||||||
|
// VRE = VO + "/NextNext";
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private enum Type{
|
public static void setContextBySecret(Secret secret) throws Exception {
|
||||||
USER, CLIENT_ID
|
SecretManagerProvider.instance.set(secret);
|
||||||
};
|
|
||||||
|
|
||||||
public static void set(Secret secret) throws Exception {
|
|
||||||
SecretManagerProvider.instance.reset();
|
|
||||||
SecretManager secretManager = new SecretManager();
|
|
||||||
secretManager.addSecret(secret);
|
|
||||||
SecretManagerProvider.instance.set(secretManager);
|
|
||||||
SecretManagerProvider.instance.get().set();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void setContextByName(String fullContextName) throws Exception {
|
public static void setContextByName(String fullContextName) throws Exception {
|
||||||
logger.debug("Going to set credentials for context {}", fullContextName);
|
|
||||||
Secret secret = getSecretByContextName(fullContextName);
|
Secret secret = getSecretByContextName(fullContextName);
|
||||||
set(secret);
|
setContextBySecret(secret);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static Secret getSecretByContextName(String fullContextName) throws Exception {
|
||||||
private static TokenResponse getJWTAccessToken(String context) throws Exception {
|
String clientID = "";
|
||||||
Type type = Type.valueOf(properties.get(TYPE_PROPERTY_KEY).toString());
|
String clientSecret = ContextTest.properties.getProperty(fullContextName);
|
||||||
|
TokenResponse tokenResponse = KeycloakClientFactory.newInstance().queryUMAToken(fullContextName, clientID, clientSecret, fullContextName, null);
|
||||||
TokenResponse tr = null;
|
return new AccessTokenSecret(tokenResponse.getAccessToken());
|
||||||
|
|
||||||
int index = context.indexOf('/', 1);
|
|
||||||
String root = context.substring(0, index == -1 ? context.length() : index);
|
|
||||||
|
|
||||||
switch (type) {
|
|
||||||
case CLIENT_ID:
|
|
||||||
String clientId = properties.getProperty(CLIENT_ID_PROPERTY_KEY);
|
|
||||||
String clientSecret = properties.getProperty(root);
|
|
||||||
|
|
||||||
tr = KeycloakClientFactory.newInstance().queryUMAToken(context, clientId, clientSecret, context, null);
|
|
||||||
break;
|
|
||||||
|
|
||||||
case USER:
|
|
||||||
default:
|
|
||||||
String username = properties.getProperty(USERNAME_PROPERTY_KEY);
|
|
||||||
String password = properties.getProperty(PASSWORD_PROPERTY_KEY);
|
|
||||||
|
|
||||||
switch (root) {
|
|
||||||
case "/gcube":
|
|
||||||
default:
|
|
||||||
clientId = "next.d4science.org";
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "/pred4s":
|
|
||||||
clientId = "pre.d4science.org";
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "/d4science.research-infrastructures.eu":
|
|
||||||
clientId = "services.d4science.org";
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
clientSecret = null;
|
|
||||||
|
|
||||||
tr = KeycloakClientHelper.getTokenForUser(context, username, password);
|
|
||||||
break;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return tr;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Secret getSecretByContextName(String context) throws Exception {
|
|
||||||
TokenResponse tr = getJWTAccessToken(context);
|
|
||||||
Secret secret = new JWTSecret(tr.getAccessToken());
|
|
||||||
return secret;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void setContext(String token) throws Exception {
|
|
||||||
Secret secret = getSecret(token);
|
|
||||||
set(secret);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Secret getSecret(String token) throws Exception {
|
|
||||||
Secret secret = SecretUtility.getSecretByTokenString(token);
|
|
||||||
return secret;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getUser() {
|
|
||||||
String user = "UNKNOWN";
|
|
||||||
try {
|
|
||||||
user = SecretManagerProvider.instance.get().getUser().getUsername();
|
|
||||||
} catch(Exception e) {
|
|
||||||
logger.error("Unable to retrieve user. {} will be used", user);
|
|
||||||
}
|
|
||||||
return user;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() throws Exception {
|
public static void beforeClass() throws Exception {
|
||||||
setContextByName(DEFAULT_TEST_SCOPE);
|
setContextByName(VRE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package org.gcube.storagehub;
|
package org.gcube.storagehub;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
@ -14,7 +13,6 @@ import org.gcube.common.storagehub.client.dsl.OpenResolver;
|
||||||
import org.gcube.common.storagehub.model.Metadata;
|
import org.gcube.common.storagehub.model.Metadata;
|
||||||
import org.gcube.common.storagehub.model.items.Item;
|
import org.gcube.common.storagehub.model.items.Item;
|
||||||
import org.gcube.common.storagehub.model.service.Version;
|
import org.gcube.common.storagehub.model.service.Version;
|
||||||
import org.junit.Ignore;
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -30,104 +28,49 @@ public class StorageHubManagementTest extends ContextTest {
|
||||||
logger.debug("Context Folder ID : {} - Name : {}", contextFolder.getId(), contextFolder.get().getName());
|
logger.debug("Context Folder ID : {} - Name : {}", contextFolder.getId(), contextFolder.get().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
@Test
|
||||||
public void test() throws Exception {
|
public void test() throws Exception {
|
||||||
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
OpenResolver openResolver = storageHubManagement.storageHubClient.open("");
|
OpenResolver openResolver = storageHubManagement.storageHubClient.open("");
|
||||||
|
// openResolver.asItem().delete();
|
||||||
/*
|
/*
|
||||||
|
openResolver = storageHubManagement.storageHubClient.open("656cd713-bd79-4659-abd6-9f1baaedb5bc");
|
||||||
openResolver.asItem().delete();
|
openResolver.asItem().delete();
|
||||||
openResolver = storageHubManagement.storageHubClient.open("");
|
openResolver = storageHubManagement.storageHubClient.open("bd44d81e-0e2f-4527-b634-2e26e8908f36");
|
||||||
openResolver.asItem().delete();
|
|
||||||
openResolver = storageHubManagement.storageHubClient.open("");
|
|
||||||
openResolver.asItem().delete();
|
|
||||||
openResolver = storageHubManagement.storageHubClient.open("");
|
|
||||||
openResolver.asItem().delete();
|
openResolver.asItem().delete();
|
||||||
*/
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
public void testPublicLink() throws Exception {
|
|
||||||
@SuppressWarnings("unused")
|
|
||||||
String urlString = "https://data.dev.d4science.org/shub/E_WE9XM3Vad1orRGNhZVJJY0NybkFQR1BzRzZKUndPeVdLYTk4b1Y2MmVhVE9oZFYvM2hBb1JKRkluT1JQR1RZdQ==";
|
|
||||||
}
|
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
@Test
|
||||||
public void listFolders() throws Exception {
|
public void listFolders() throws Exception {
|
||||||
String c = "/gcube";
|
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
||||||
// c = "/pred4s/preprod/GRSF_Pre";
|
@SuppressWarnings("unused")
|
||||||
// c = "/d4science.research-infrastructures.eu/FARM/GRSF_Pre";
|
FolderContainer root = storageHubManagement.getWorkspaceRoot();
|
||||||
// c = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin";
|
FolderContainer contextFolder = storageHubManagement.getContextFolder();
|
||||||
// c = "/d4science.research-infrastructures.eu/FARM/GRSF";
|
ListResolverTyped listResolverTyped = contextFolder.list();
|
||||||
List<String> contexts = new ArrayList<>();
|
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
|
||||||
contexts.add(c);
|
for(ItemContainer<? extends Item> itemContainer : containers) {
|
||||||
|
Item item = itemContainer.get();
|
||||||
for(String context : contexts) {
|
String name = item.getName();
|
||||||
ContextTest.setContextByName(context);
|
ContainerType containerType = itemContainer.getType();
|
||||||
|
if(containerType==ContainerType.FOLDER && name.compareTo(".catalogue")==0) {
|
||||||
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
itemContainer.delete();
|
||||||
|
|
||||||
// FolderContainer root = storageHubManagement.getWorkspaceRoot();
|
|
||||||
// storageHubManagement.tree(root);
|
|
||||||
|
|
||||||
FolderContainer contextFolder = storageHubManagement.getContextFolder();
|
|
||||||
// storageHubManagement.tree(contextFolder);
|
|
||||||
|
|
||||||
ListResolverTyped listResolverTyped = contextFolder.list();
|
|
||||||
List<ItemContainer<? extends Item>> containers = listResolverTyped.includeHidden().getContainers();
|
|
||||||
for(ItemContainer<? extends Item> itemContainer : containers) {
|
|
||||||
Item item = itemContainer.get();
|
|
||||||
String name = item.getName();
|
|
||||||
ContainerType containerType = itemContainer.getType();
|
|
||||||
if(containerType==ContainerType.FOLDER) {
|
|
||||||
if(item.isHidden()) {
|
|
||||||
storageHubManagement.logItem(itemContainer);
|
|
||||||
}
|
|
||||||
if(name.compareTo(".catalogue")==0 || name.compareTo("service-account-gcat")==0 || name.compareTo("service-account-grsf-publisher")==0) {
|
|
||||||
logger.info("Catalogue folder found");
|
|
||||||
storageHubManagement.logItem(itemContainer);
|
|
||||||
// storageHubManagement.tree((FolderContainer) itemContainer);
|
|
||||||
if(name.compareTo(".catalogue")==0) {
|
|
||||||
// storageHubManagement.tree((FolderContainer) itemContainer);
|
|
||||||
// itemContainer.forceDelete();
|
|
||||||
}
|
|
||||||
if(name.compareTo("service-account-grsf-publisher")==0) {
|
|
||||||
// storageHubManagement.tree((FolderContainer) itemContainer);
|
|
||||||
// itemContainer.forceDelete();
|
|
||||||
}
|
|
||||||
if(name.compareTo("service-account-gcat")==0) {
|
|
||||||
// storageHubManagement.tree((FolderContainer) itemContainer);
|
|
||||||
// itemContainer.forceDelete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// FolderContainer dstFolder = storageHubManagement.getDestinationFolder("application/pdf");
|
||||||
|
// storageHubManagement.tree(root);
|
||||||
|
// storageHubManagement.tree(contextFolder);
|
||||||
|
// storageHubManagement.tree(dstFolder);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
public void emptyTrash() throws Exception {
|
|
||||||
String c = GCUBE;
|
|
||||||
// c = ContextTest.ROOT_PRE;
|
|
||||||
// c = ContextTest.ROOT_PROD;
|
|
||||||
ContextTest.setContextByName(c);
|
|
||||||
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
|
||||||
storageHubManagement.getStorageHubClient().emptyTrash();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
@Test
|
||||||
public void getFileInfo() throws Exception {
|
public void getFileInfo() throws Exception {
|
||||||
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
||||||
String id = "";
|
String id = "3daf465b-b84e-4d1c-9786-a388a267382c";
|
||||||
OpenResolver openResolver = storageHubManagement.storageHubClient.open(id);
|
OpenResolver openResolver = storageHubManagement.storageHubClient.open(id);
|
||||||
FileContainer fileContainer = (FileContainer) openResolver.asFile();
|
FileContainer fileContainer = (FileContainer) openResolver.asFile();
|
||||||
storageHubManagement.logItem(fileContainer);
|
logger.debug("StorageHub ID {} - File Name {}", id, fileContainer.get().getName());
|
||||||
|
|
||||||
/*
|
/*
|
||||||
ListResolver listResolver = fileContainer.getAnchestors();
|
ListResolver listResolver = fileContainer.getAnchestors();
|
||||||
|
@ -147,20 +90,21 @@ public class StorageHubManagementTest extends ContextTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
@Test
|
||||||
public void getFileInfoViaDirectoryListing() throws Exception {
|
public void getFileInfoViaDirectoryListing() throws Exception {
|
||||||
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
StorageHubManagement storageHubManagement = new StorageHubManagement();
|
||||||
String id = "";
|
String id = "22bd9034-1da0-45ac-868f-91d1e5438344";
|
||||||
OpenResolver openResolver = storageHubManagement.storageHubClient.open(id);
|
OpenResolver openResolver = storageHubManagement.storageHubClient.open(id);
|
||||||
FolderContainer folderContainer = (FolderContainer) openResolver.asFolder();
|
FolderContainer folderContainer = (FolderContainer) openResolver.asFolder();
|
||||||
String filename = "";
|
String filename = "56a55300-9272-4c12-9ac8-ff61411a5976";
|
||||||
ListResolver listResolver = folderContainer.findByName(filename);
|
ListResolver listResolver = folderContainer.findByName(filename);
|
||||||
List<ItemContainer<? extends Item>> itemContainers = listResolver.withMetadata().getContainers();
|
List<ItemContainer<? extends Item>> itemContainers = listResolver.withMetadata().getContainers();
|
||||||
for(ItemContainer<? extends Item> itemContainer : itemContainers) {
|
for(ItemContainer<? extends Item> itemContainer : itemContainers) {
|
||||||
|
|
||||||
FileContainer fileContainer = (FileContainer) itemContainer;
|
FileContainer fileContainer = (FileContainer) itemContainer;
|
||||||
storageHubManagement.logItem(fileContainer);
|
|
||||||
|
|
||||||
|
logger.debug("StorageHub ID {} - File Name {}", id, fileContainer.get().getName());
|
||||||
|
|
||||||
/*
|
/*
|
||||||
ListResolver listResolver = fileContainer.getAnchestors();
|
ListResolver listResolver = fileContainer.getAnchestors();
|
||||||
|
@ -179,7 +123,6 @@ public class StorageHubManagementTest extends ContextTest {
|
||||||
logger.debug("Version {} {}", version.getId(), version.getName());
|
logger.debug("Version {} {}", version.getId(), version.getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,2 @@
|
||||||
/*.gcubekey
|
/*.gcubekey
|
||||||
/*.properties
|
/*.properties
|
||||||
/config.ini
|
|
||||||
|
|
Loading…
Reference in New Issue