gCubeSystem
/
storagehub
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

a part of s3StorageIntegration

This commit is contained in:
Lucio Lelii 2021-11-26 17:49:35 +01:00
parent af9290cbca
commit e11bb536a7
19 changed files with 378 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.classpath
View File

@ -27,7 +27,6 @@
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="org.eclipse.jst.component.dependency" value="/WEB-INF/lib"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">

33
pom.xml
View File

@ -296,19 +296,7 @@
<scope>runtime</scope>
</dependency>
<!-- Storage dependencies -->
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-core</artifactId>
<version>[3.0.0-SNAPSHOT,4.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-wrapper</artifactId>
<version>[3.0.0-SNAPSHOT,4.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
@ -418,6 +406,25 @@
<artifactId>jersey-test-framework-provider-grizzly2</artifactId>
<scope>test</scope>
</dependency>
<!-- Storage dependencies -->
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-core</artifactId>
<version>[3.0.0-SNAPSHOT,4.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-wrapper</artifactId>
<version>[3.0.0-SNAPSHOT,4.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.163</version>
</dependency>
</dependencies>
<build>

17
src/main/java/org/gcube/data/access/storagehub/Utils.java
View File

@ -30,12 +30,14 @@ import org.gcube.common.storagehub.model.exceptions.IdNotFoundException;
import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.exceptions.UserNotAuthorizedException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.ExternalLink;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.GCubeItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.RootItem;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.types.ItemAction;
import org.gcube.common.storagehub.model.types.NodeProperty;
import org.gcube.data.access.storagehub.accounting.AccountingHandler;
@ -188,7 +190,7 @@ public class Utils {
}
public static void copyStream(InputStream in, OutputStream out) throws IOException {
byte[] buffer = new byte[2048];
@ -211,7 +213,7 @@ public class Utils {
return false;
}
public static String checkExistanceAndGetUniqueName(Session ses, Node destination, String name) throws BackendGenericError{
try {
destination.getNode(name);
@ -249,7 +251,7 @@ public class Utils {
public static Node createFolderInternally(FolderCreationParameters params, AccountingHandler accountingHandler) throws StorageHubException {
logger.debug("creating folder {} in {}", params.getName(), params.getParentId());
Node destinationNode;
try {
destinationNode = params.getSession().getNodeByIdentifier(params.getParentId());
@ -346,5 +348,12 @@ public class Utils {
node.setProperty(NodeProperty.LAST_ACTION.toString(), action.name());
}
public static void setContentFromMetaInfo(AbstractFileItem item, MetaInfo contentInfo) {
item.getContent().setManagedBy(contentInfo.getManagedBy());
item.getContent().setSize(contentInfo.getSize());
item.getContent().setRemotePath(contentInfo.getRemotePath());
item.getContent().setSize(contentInfo.getSize());
item.getContent().setStorageId(contentInfo.getStorageId());
}
}

16
src/main/java/org/gcube/data/access/storagehub/handlers/CompressHandler.java
View File

@ -19,10 +19,11 @@ import org.gcube.common.storagehub.model.exceptions.BackendGenericError;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.data.access.storagehub.Utils;
import org.gcube.data.access.storagehub.accounting.AccountingHandler;
import org.gcube.data.access.storagehub.handlers.plugins.FolderPluginHandler;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -31,7 +32,8 @@ public class CompressHandler {
private Logger logger = LoggerFactory.getLogger(CompressHandler.class);
@Inject
FolderPluginHandler pluginHandler;
StorageBackendHandler storageBackendHandler;
public Deque<Item> getAllNodesForZip(FolderItem directory, Session session, String login, AccountingHandler accountingHandler, List<String> excludes) throws RepositoryException, BackendGenericError{
Deque<Item> queue = new LinkedList<Item>();
@ -74,8 +76,12 @@ public class CompressHandler {
} else if (item instanceof AbstractFileItem){
try {
AbstractFileItem fileItem = (AbstractFileItem)item;
FolderManager manager = pluginHandler.getFolderManager(fileItem);
InputStream streamToWrite = manager.getStorageBackend().download(fileItem.getContent());
StorageBackendFactory sbf = storageBackendHandler.get(fileItem.getContent().getManagedBy());
StorageBackend sb = sbf.create(null);
InputStream streamToWrite = sb.download(fileItem.getContent());
if (streamToWrite == null){
logger.warn("discarding item {} ",item.getName());
continue;

19
src/main/java/org/gcube/data/access/storagehub/handlers/TrashHandler.java
View File

@ -30,7 +30,8 @@ import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.TrashItem;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.common.storagehub.model.types.ItemAction;
import org.gcube.data.access.storagehub.AuthorizationChecker;
import org.gcube.data.access.storagehub.PathUtil;
@ -38,7 +39,7 @@ import org.gcube.data.access.storagehub.Utils;
import org.gcube.data.access.storagehub.accounting.AccountingHandler;
import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter;
import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter;
import org.gcube.data.access.storagehub.handlers.plugins.FolderPluginHandler;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.gcube.data.access.storagehub.types.ContentPair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -68,10 +69,9 @@ public class TrashHandler {
@Inject
PathUtil pathUtil;
@Inject FolderPluginHandler managerHandler;
@Inject
FolderPluginHandler folderHandler;
StorageBackendHandler storageBackendHandler;
public void removeNodes(Session ses, List<Item> itemsToDelete) throws RepositoryException, StorageHubException{
log.debug("defnitively removing nodes with ids {}",itemsToDelete);
@ -109,8 +109,13 @@ public class TrashHandler {
}
try {
FolderManager manager = folderHandler.getFolderManager(item);
contentSet.add(new ContentPair(item.getContent(), manager.getStorageBackend()));
StorageBackendFactory sbf = storageBackendHandler.get(item.getContent().getManagedBy());
StorageBackend sb = sbf.create(null);
contentSet.add(new ContentPair(item.getContent(), sb));
List<Version> versions = versionHandler.getContentVersionHistory((Node)item.getRelatedNode());
@ -118,7 +123,7 @@ public class TrashHandler {
try {
Content content = node2Item.getContentFromVersion(version);
if (content!= null && content.getStorageId()!=null)
contentSet.add(new ContentPair(content, manager.getStorageBackend()));
contentSet.add(new ContentPair(content, sb));
else log.warn("invalid version {}",version.getName());
}catch (Throwable t) {
log.warn("error retrieving version content for {}",version.getName(),t);

23
src/main/java/org/gcube/data/access/storagehub/handlers/items/ItemHandler.java
View File

@ -38,9 +38,9 @@ import org.gcube.common.storagehub.model.exceptions.ItemLockedException;
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.common.storagehub.model.types.ItemAction;
import org.gcube.data.access.storagehub.AuthorizationChecker;
import org.gcube.data.access.storagehub.MultipleOutputStream;
@ -55,7 +55,7 @@ import org.gcube.data.access.storagehub.handlers.items.builders.FileCreationPara
import org.gcube.data.access.storagehub.handlers.items.builders.FolderCreationParameters;
import org.gcube.data.access.storagehub.handlers.items.builders.GCubeItemCreationParameters;
import org.gcube.data.access.storagehub.handlers.items.builders.URLCreationParameters;
import org.gcube.data.access.storagehub.handlers.plugins.FolderPluginHandler;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -76,7 +76,7 @@ public class ItemHandler {
VersionHandler versionHandler;
@Inject
FolderPluginHandler pluginHandler;
StorageBackendHandler storageBackendHandler;
private static ExecutorService executor = Executors.newFixedThreadPool(100);
@ -252,24 +252,23 @@ public class ItemHandler {
Node newNode;
FolderItem destinationItem = node2Item.getItem(destinationNode, Excludes.ALL);
FolderManager folderManager = pluginHandler.getFolderManager(destinationItem);
StorageBackend storageBackend = folderManager.getStorageBackend();
String relativePath = destinationNode.getPath();
if (destinationItem.isExternalManaged())
relativePath = relativePath.replace(folderManager.getRootFolder().getPath(), "");
StorageBackendFactory sbf = storageBackendHandler.get(destinationItem.getBackend().getStorageName());
//TODO: add metadata taken from content node
StorageBackend sb = sbf.create(destinationItem.getBackend().getParameters());
String relativePath = destinationNode.getPath();
String newNodePath = Paths.append(Paths.getPath(destinationNode.getPath()), name).toPath();
if (ses.nodeExists(newNodePath)) {
if (!folderManager.manageVersion())
throw new InvalidCallParameters("storage for plugin "+folderManager.getClass().getName()+" doesn't support versioning");
newNode = ses.getNode(newNodePath);
authChecker.checkWriteAuthorizationControl(ses, login, newNode.getIdentifier(), false);
AbstractFileItem item = fillItemWithContent(stream, storageBackend, name, description, relativePath,login);
AbstractFileItem item = fillItemWithContent(stream, sb, name, description, relativePath,login);
if (withLock) {
try {
@ -290,7 +289,7 @@ public class ItemHandler {
}
else {
authChecker.checkWriteAuthorizationControl(ses, login, destinationNode.getIdentifier(), true);
AbstractFileItem item = fillItemWithContent(stream, storageBackend, name, description, relativePath, login);
AbstractFileItem item = fillItemWithContent(stream, sb, name, description, relativePath, login);
if (withLock) {
try {
log.debug("trying to acquire lock");

86
src/main/java/org/gcube/data/access/storagehub/handlers/plugins/FolderPluginHandler.java
View File

@ -1,86 +0,0 @@
package org.gcube.data.access.storagehub.handlers.plugins;
import java.util.Collections;
import java.util.Map;
import javax.enterprise.inject.Instance;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.jcr.Node;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import org.gcube.common.storagehub.model.Excludes;
import org.gcube.common.storagehub.model.exceptions.BackendGenericError;
import org.gcube.common.storagehub.model.exceptions.PluginInitializationException;
import org.gcube.common.storagehub.model.exceptions.PluginNotFoundException;
import org.gcube.common.storagehub.model.items.ExternalFolder;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.plugins.FolderManagerConnector;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter;
import org.gcube.data.access.storagehub.storage.backend.impl.GcubeFolderManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class FolderPluginHandler {
private static Logger log = LoggerFactory.getLogger(FolderPluginHandler.class);
@Inject
Node2ItemConverter node2Item;
private GcubeFolderManager defaultManager = new GcubeFolderManager();
public FolderManager getDefault() {
return defaultManager;
}
@Inject
Instance<FolderManagerConnector> connectors;
private Map<String, FolderManagerConnector> connectorsMap;
private Map<String, StorageBackend> storagesMap;
FolderPluginHandler(){
if (connectors !=null)
for (FolderManagerConnector connector : connectors) {
if (connectorsMap.containsKey(connector.getName()))
throw new RuntimeException("multiple connectors with the same name found");
connectorsMap.put(connector.getName(), connector);
}
else {
log.info("connectors are null");
connectorsMap = Collections.emptyMap();
}
}
public FolderManagerConnector getConnector(String name) throws PluginNotFoundException {
if (!connectorsMap.containsKey(name)) throw new PluginNotFoundException("plugin "+name+" not found");
return connectorsMap.get(name);
}
public FolderManager getFolderManager(Item item) throws PluginInitializationException, PluginNotFoundException, RepositoryException, BackendGenericError{
if (!item.isExternalManaged())
return defaultManager;
Session session = ((Node)item.getRelatedNode()).getSession();
Item parent = null;
do {
String parentId = item.getParentId();
Node node = session.getNodeByIdentifier(parentId);
parent = node2Item.getItem(node, Excludes.ALL);
if (parent !=null && parent instanceof ExternalFolder) {
ExternalFolder extParent = (ExternalFolder) parent;
String plugin = extParent.getManagedBy();
Map<String, Object> parameters = extParent.getConnectionParameters().getMap();
return getConnector(plugin).connect(extParent, parameters);
}
} while (parent!=null);
throw new BackendGenericError("selected external managed item doesn't have a parent external folder");
}
}

20
src/main/java/org/gcube/data/access/storagehub/handlers/plugins/OperationMediator.java
View File

@ -1,28 +1,10 @@
package org.gcube.data.access.storagehub.handlers.plugins;
import javax.inject.Inject;
import javax.inject.Singleton;
@Singleton
public class OperationMediator {
@Inject
FolderPluginHandler folderHandler;
/*
boolean onMove(Item source, Item destination, Session session) throws PluginInitializationException, PluginNotFoundException, BackendGenericError, RepositoryException{
FolderManager sourceFolderManager = folderHandler.getFolderManager(source);
FolderManager destinationFolderManager = folderHandler.getFolderManager(destination);
if (source instanceof FolderItem) {
destinationFolderManager.onCreatedFolder((FolderItem) source);
session.move(source.getPath(), destination.getPath());
sourceFolderManager.onDeletingFolder((FolderItem) source);
} else if (source instanceof AbstractFileItem){
}
}
*/
}

47
src/main/java/org/gcube/data/access/storagehub/handlers/plugins/StorageBackendHandler.java Normal file
View File

@ -0,0 +1,47 @@
package org.gcube.data.access.storagehub.handlers.plugins;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.enterprise.inject.Instance;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.gcube.common.storagehub.model.exceptions.PluginNotFoundException;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class StorageBackendHandler {
private static Logger log = LoggerFactory.getLogger(StorageBackendHandler.class);
@Inject
Instance<StorageBackendFactory> factories;
Map<String, StorageBackendFactory> storagebackendMap= new HashMap<String, StorageBackendFactory>();
@PostConstruct
void init(){
if (factories !=null)
for (StorageBackendFactory connector : factories) {
if (storagebackendMap.containsKey(connector.getName())) {
log.error("multiple storage backend with the same name");
throw new RuntimeException("multiple storage backend with the same name");
}
storagebackendMap.put(connector.getName(), connector);
}
else
throw new RuntimeException("storage backend implementation not found");
}
public StorageBackendFactory get(String storageName) throws PluginNotFoundException {
if (!storagebackendMap.containsKey(storageName))
throw new PluginNotFoundException(String.format("implementation for storage %s not found", storageName));
return storagebackendMap.get(storageName);
}
}

66
src/main/java/org/gcube/data/access/storagehub/services/ItemsManager.java
View File

@ -66,10 +66,12 @@ import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.gcube.common.storagehub.model.items.VreFolder;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.service.ItemList;
import org.gcube.common.storagehub.model.service.ItemWrapper;
import org.gcube.common.storagehub.model.service.VersionList;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.common.storagehub.model.types.ItemAction;
import org.gcube.common.storagehub.model.types.NodeProperty;
import org.gcube.data.access.storagehub.AuthorizationChecker;
@ -88,7 +90,8 @@ import org.gcube.data.access.storagehub.handlers.TrashHandler;
import org.gcube.data.access.storagehub.handlers.VersionHandler;
import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter;
import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter;
import org.gcube.data.access.storagehub.handlers.plugins.FolderPluginHandler;
import org.gcube.data.access.storagehub.handlers.plugins.OperationMediator;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.gcube.smartgears.annotations.ManagedBy;
import org.gcube.smartgears.utils.InnerMethodName;
import org.slf4j.Logger;
@ -127,13 +130,16 @@ public class ItemsManager extends Impersonable{
@Inject Node2ItemConverter node2Item;
@Inject Item2NodeConverter item2Node;
@Inject
FolderPluginHandler folderPluginHandler;
@Inject
CompressHandler compressHandler;
@Inject
OperationMediator opMediator;
@Inject
StorageBackendHandler storageBackendHandler;
@GET
@Path("{id}")
@ -703,11 +709,15 @@ public class ItemsManager extends Impersonable{
log.debug("retrieved version id {}, name {}", version.getIdentifier(), version.getName());
if (version.getName().equals(versionName)) {
Content content = node2Item.getContentFromVersion(version);
FolderManager folderManager = folderPluginHandler.getFolderManager((AbstractFileItem) currentItem);
final InputStream streamToWrite = folderManager.getStorageBackend().download(content);
log.debug("retrieved storage id is {} with storageBackend {} (stream is null? {})",content.getStorageId(), folderManager.getStorageBackend().getClass().getSimpleName(), streamToWrite==null );
StorageBackendFactory sbf = storageBackendHandler.get(content.getManagedBy());
//TODO: add metadata taken from content node
StorageBackend sb = sbf.create(null);
final InputStream streamToWrite = sb.download(content);
log.debug("retrieved storage id is {} with storageBackend {} (stream is null? {})",content.getStorageId(), sb.getName(), streamToWrite==null );
String oldfilename = FilenameUtils.getBaseName(currentItem.getTitle());
String ext = FilenameUtils.getExtension(currentItem.getTitle());
@ -849,11 +859,15 @@ public class ItemsManager extends Impersonable{
return response;
}
private Response downloadFileInternal(Session ses, AbstractFileItem fileItem, String login, boolean withAccounting) throws RepositoryException, PluginInitializationException, PluginNotFoundException, BackendGenericError {
private Response downloadFileInternal(Session ses, AbstractFileItem fileItem, String login, boolean withAccounting) throws RepositoryException, PluginInitializationException, PluginNotFoundException, StorageHubException {
FolderManager folderManager = folderPluginHandler.getFolderManager(fileItem);
Content content = fileItem.getContent();
final InputStream streamToWrite = folderManager.getStorageBackend().download(fileItem.getContent());
StorageBackendFactory sbf = storageBackendHandler.get(content.getManagedBy());
StorageBackend sb = sbf.create(null);
final InputStream streamToWrite = sb.download(content);
if (withAccounting)
accountingHandler.createReadObj(fileItem.getTitle(), ses, (Node) fileItem.getRelatedNode(), login, true);
@ -923,10 +937,11 @@ public class ItemsManager extends Impersonable{
if (movingSharedItemOutside)
item2Node.updateOwnerOnSubTree(nodeToMove, currentUser);
//folderHandler.onMove(source, destination);
//add onMove (if it changes the remotePath) and in case of different backend
accountingHandler.createFolderAddObj(uniqueName, item.getClass().getSimpleName(), mimeTypeForAccounting, ses, currentUser, destination, false);
accountingHandler.createFolderRemoveObj(item.getTitle(), item.getClass().getSimpleName(), mimeTypeForAccounting, ses, currentUser, originalParent, false);
ses.save();
}finally {
ses.getWorkspace().getLockManager().unlock(nodeToMove.getPath());
@ -976,23 +991,27 @@ public class ItemsManager extends Impersonable{
}catch (LockException e) {
throw new ItemLockedException(e);
}
try {
String uniqueName = Utils.checkExistanceAndGetUniqueName(ses, destination, newFileName);
String newPath= String.format("%s/%s", destination.getPath(), uniqueName);
ses.getWorkspace().copy(nodeToCopy.getPath(), newPath);
Node newNode = ses.getNode(newPath);
newFileIdentifier = newNode.getIdentifier();
//TODO: folderHandler.onCopy(source, destination);
if (item instanceof AbstractFileItem) {
FolderManager manager = folderPluginHandler.getFolderManager(item);
((AbstractFileItem) item).getContent().setRemotePath(newPath);
String newStorageID = manager.getStorageBackend().onCopy((AbstractFileItem) item);
((AbstractFileItem) item).getContent().setStorageId(newStorageID);
item2Node.replaceContent(newNode, (AbstractFileItem) item, ItemAction.CLONED);
}
Content contentToCopy = ((AbstractFileItem) item).getContent();
//TODO : understand if it is needed
//opMediator.copy(((AbstractFileItem) item).getContent(),destinationItem, uniqueName);
StorageBackendFactory sbf = storageBackendHandler.get(contentToCopy.getManagedBy());
//TODO: add metadata taken from content node
StorageBackend sb = sbf.create(null);
MetaInfo contentInfo = sb.onCopy(contentToCopy, destination.getPath(), uniqueName);
Utils.setContentFromMetaInfo((AbstractFileItem) item, contentInfo);
item2Node.replaceContent(newNode, (AbstractFileItem) item, ItemAction.CLONED);
Utils.setPropertyOnChangeNode(newNode, currentUser, ItemAction.CLONED);
newNode.setProperty(NodeProperty.PORTAL_LOGIN.toString(), currentUser);
newNode.setProperty(NodeProperty.IS_PUBLIC.toString(), false);
@ -1061,6 +1080,7 @@ public class ItemsManager extends Impersonable{
nodeToMove.setProperty(NodeProperty.TITLE.toString(), uniqueName);
Utils.setPropertyOnChangeNode(nodeToMove, currentUser, ItemAction.RENAMED);
ses.move(nodeToMove.getPath(), newPath);
accountingHandler.createRename(item.getTitle(), uniqueName, ses.getNode(newPath), currentUser, ses, false);
ses.save();
}finally {

33
src/main/java/org/gcube/data/access/storagehub/services/MessageManager.java
View File

@ -40,9 +40,13 @@ import org.gcube.common.storagehub.model.exceptions.StorageHubException;
import org.gcube.common.storagehub.model.exceptions.UserNotAuthorizedException;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.items.nodes.Owner;
import org.gcube.common.storagehub.model.messages.Message;
import org.gcube.common.storagehub.model.service.ItemList;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
import org.gcube.common.storagehub.model.types.ItemAction;
import org.gcube.common.storagehub.model.types.MessageList;
import org.gcube.common.storagehub.model.types.NodeProperty;
@ -56,7 +60,7 @@ import org.gcube.data.access.storagehub.handlers.TrashHandler;
import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter;
import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter.Values;
import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter;
import org.gcube.data.access.storagehub.handlers.plugins.FolderPluginHandler;
import org.gcube.data.access.storagehub.handlers.plugins.StorageBackendHandler;
import org.gcube.data.access.storagehub.types.MessageSharable;
import org.gcube.smartgears.annotations.ManagedBy;
import org.gcube.smartgears.utils.InnerMethodName;
@ -87,7 +91,10 @@ public class MessageManager extends Impersonable{
@Inject Item2NodeConverter item2Node;
@Inject TrashHandler trashHandler;
@Inject
StorageBackendHandler storageBackendHandler;
@GET
@Path("{id}")
@Produces(MediaType.APPLICATION_JSON)
@ -327,7 +334,7 @@ public class MessageManager extends Impersonable{
private Node saveAttachments(Session ses, Node messageNode , List<String> attachments) throws RepositoryException, BackendGenericError{
private Node saveAttachments(Session ses, Node messageNode , List<String> attachments) throws RepositoryException, StorageHubException{
Node attachmentNode = messageNode.getNode(Constants.ATTACHMENTNODE_NAME);
for (String itemId: attachments) {
@ -392,10 +399,8 @@ public class MessageManager extends Impersonable{
return personalNode== null ? messageNode : personalNode;
}
//TODO: move in a common place
@Inject FolderPluginHandler folderPluginHandler;
private Node copyNode(Session session, Node destination, Item itemToCopy) throws RepositoryException, BackendGenericError{
private Node copyNode(Session session, Node destination, Item itemToCopy) throws RepositoryException, StorageHubException{
//it needs to be locked ??
Node nodeToCopy = ((Node)itemToCopy.getRelatedNode());
String uniqueName = Utils.checkExistanceAndGetUniqueName(session, destination,itemToCopy.getName() );
@ -405,9 +410,17 @@ public class MessageManager extends Impersonable{
if (itemToCopy instanceof AbstractFileItem) {
AbstractFileItem newNodeItem = node2Item.getItem(newNode, Excludes.EXCLUDE_ACCOUNTING);
newNodeItem.getContent().setRemotePath(newPath);
String newStorageID = folderPluginHandler.getDefault().getStorageBackend().onCopy(newNodeItem);
newNodeItem.getContent().setStorageId(newStorageID);
Content contentToCopy = newNodeItem.getContent();
StorageBackendFactory sbf = storageBackendHandler.get(contentToCopy.getManagedBy());
//TODO: add metadata taken from content node
StorageBackend sb = sbf.create(null);
MetaInfo contentInfo = sb.onCopy(contentToCopy, destination.getPath(), uniqueName);
Utils.setContentFromMetaInfo(newNodeItem, contentInfo);
item2Node.replaceContent(newNode, newNodeItem, ItemAction.CLONED);
}

8
src/main/java/org/gcube/data/access/storagehub/services/WorkspaceManager.java
View File

@ -46,7 +46,6 @@ import org.gcube.data.access.storagehub.handlers.TrashHandler;
import org.gcube.data.access.storagehub.handlers.items.Item2NodeConverter;
import org.gcube.data.access.storagehub.handlers.items.Node2ItemConverter;
import org.gcube.data.access.storagehub.handlers.items.builders.FolderCreationParameters;
import org.gcube.data.access.storagehub.handlers.plugins.FolderPluginHandler;
import org.gcube.data.access.storagehub.handlers.vres.VRE;
import org.gcube.data.access.storagehub.handlers.vres.VREManager;
import org.gcube.data.access.storagehub.query.sql2.evaluators.Evaluators;
@ -89,9 +88,6 @@ public class WorkspaceManager extends Impersonable{
@Inject Node2ItemConverter node2Item;
@Inject Item2NodeConverter item2Node;
@Inject
FolderPluginHandler folderHandler;
@Path("/")
@ -357,7 +353,7 @@ public class WorkspaceManager extends Impersonable{
@GET
public String getTotalItemsCount(){
InnerMethodName.instance.set("getTotalItemsCount");
return folderHandler.getDefault().getStorageBackend().getTotalItemsCount();
return "1203";
}
@ -365,7 +361,7 @@ public class WorkspaceManager extends Impersonable{
@GET
public String getTotalVolume(){
InnerMethodName.instance.set("getTotalSize");
return folderHandler.getDefault().getStorageBackend().getTotalSizeStored();
return "120300000";
}

22
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeFolderManagerConnector.java
View File

@ -1,22 +0,0 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import java.util.Map;
import javax.inject.Singleton;
import org.gcube.common.storagehub.model.exceptions.PluginInitializationException;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.plugins.FolderManagerConnector;
@Singleton
public class GCubeFolderManagerConnector implements FolderManagerConnector {
@Override
public FolderManager connect(FolderItem item, Map<String, Object> parameters) throws PluginInitializationException {
return new GcubeFolderManager();
}
}

42
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeStorageBackend.java → src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeMongoStorageBackend.java
View File

@ -1,11 +1,11 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.UUID;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.storagehub.model.Constants;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
@ -16,38 +16,42 @@ import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GCubeStorageBackend implements StorageBackend {
public class GCubeMongoStorageBackend implements StorageBackend {
private static final Logger log = LoggerFactory.getLogger(GCubeStorageBackend.class);
private static final Logger log = LoggerFactory.getLogger(GCubeMongoStorageBackend.class);
private final static String SERVICE_NAME = "home-library";
private final static String SERVICE_CLASS = "org.gcube.portlets.user";
protected GCubeStorageBackend() {}
@Override
public InputStream download(Content content) {
return getStorageClient(AuthorizationProvider.instance.get().getClient().getId()).getClient().get().RFileAsInputStream(content.getStorageId());
}
@Override
public String getName() {
return Constants.mongoStorageConstant;
}
protected StorageClient getStorageClient(String login){
return new StorageClient(SERVICE_CLASS, SERVICE_NAME, login, AccessType.SHARED, MemoryType.PERSISTENT);
}
@Override
public String onCopy(AbstractFileItem item) {
log.info("copying storage Id {} to newPath {}", item.getContent().getStorageId(), item.getPath());
String newStorageID = getStorageClient(AuthorizationProvider.instance.get().getClient().getId()).getClient().copyFile(true).from(item.getContent().getStorageId()).to(item.getPath());
public MetaInfo onCopy(Content content, String newParentPath, String newName) {
log.info("copying storage Id {} to newPath {}", content.getStorageId(), newParentPath);
String newRemotePath = Paths.get(newParentPath, newName).toString();
String newStorageID = getStorageClient(AuthorizationProvider.instance.get().getClient().getId()).getClient().copyFile(true).from(content.getStorageId()).to(newRemotePath);
log.info("The id returned by storage is {}", newStorageID);
return newStorageID;
return new MetaInfo(content.getSize(),newStorageID, newRemotePath, getName());
}
@Override
public String onMove(AbstractFileItem item) {
return item.getContent().getStorageId();
public MetaInfo onMove(Content content, String newParentPath) {
//new contentPath can be set as remotePath to the storage backend ?
return new MetaInfo(content.getSize(),content.getStorageId(), content.getRemotePath(), getName());
}
@Override
@ -58,11 +62,7 @@ public class GCubeStorageBackend implements StorageBackend {
String remotePath= String.format("%s/%s-%s",relPath,uid,name);
String storageId =storageClient.put(true).LFile(stream).RFile(remotePath);
long size = storageClient.getSize().RFileById(storageId);
MetaInfo info = new MetaInfo();
info.setSize(size);
info.setStorageId(storageId);
info.setRemotePath(remotePath);
info.setManagedBy(Constants.mongoStorageConstant);
MetaInfo info = new MetaInfo(size, storageId, remotePath, getName());
return info;
}
@ -72,11 +72,7 @@ public class GCubeStorageBackend implements StorageBackend {
IClient storageClient = getStorageClient(AuthorizationProvider.instance.get().getClient().getId()).getClient();
storageClient.remove().RFileById(content.getStorageId());
}
private static StorageClient getStorageClient(String login){
return new StorageClient(SERVICE_CLASS, SERVICE_NAME, login, AccessType.SHARED, MemoryType.PERSISTENT);
}
@Override
public String getTotalSizeStored() {
@ -91,4 +87,4 @@ public class GCubeStorageBackend implements StorageBackend {
}
}
}

23
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GCubeMongoStorageBackendFactory.java Normal file
View File

@ -0,0 +1,23 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import org.gcube.common.storagehub.model.Constants;
import org.gcube.common.storagehub.model.Metadata;
import org.gcube.common.storagehub.model.exceptions.InvalidCallParameters;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import org.gcube.common.storagehub.model.storages.StorageBackendFactory;
public class GCubeMongoStorageBackendFactory implements StorageBackendFactory {
@Override
public String getName() {
return Constants.mongoStorageConstant;
}
@Override
public StorageBackend create(Metadata parameter) throws InvalidCallParameters {
return new GCubeMongoStorageBackend();
}
}

48
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/GcubeFolderManager.java
View File

@ -1,48 +0,0 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import javax.inject.Singleton;
import org.gcube.common.storagehub.model.items.FolderItem;
import org.gcube.common.storagehub.model.plugins.FolderManager;
import org.gcube.common.storagehub.model.storages.StorageBackend;
@Singleton
public class GcubeFolderManager implements FolderManager {
@Override
public StorageBackend getStorageBackend() {
return new GCubeStorageBackend();
}
@Override
public boolean manageVersion() {
return true;
}
@Override
public void onCreatedFolder(FolderItem folder) {
}
@Override
public void onDeletingFolder(FolderItem folder) {
}
@Override
public void onMovedFolder(FolderItem movedFolder) {
// TODO Auto-generated method stub
}
@Override
public void onCopiedFolder(FolderItem copiedFolder) {
// TODO Auto-generated method stub
}
@Override
public FolderItem getRootFolder() {
return null;
}
}

139
src/main/java/org/gcube/data/access/storagehub/storage/backend/impl/S3Backend.java Normal file
View File

@ -0,0 +1,139 @@
package org.gcube.data.access.storagehub.storage.backend.impl;
import java.io.InputStream;
import java.util.Map;
import java.util.function.Function;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.storagehub.model.items.nodes.Content;
import org.gcube.common.storagehub.model.storages.MetaInfo;
import org.gcube.common.storagehub.model.storages.StorageBackend;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
public class S3Backend implements StorageBackend{
Map<String, Object> parameters;
Function<Void, String> keyGenerator;
AWSCredentials credentials = new BasicAWSCredentials("user", "password");
final static Regions clientRegion = Regions.DEFAULT_REGION;
String bucketName;
public S3Backend(Map<String, Object> parameters, Function<Void, String> keyGenerator) {
super();
this.parameters = parameters;
this.keyGenerator = keyGenerator;
this.bucketName = (String)parameters.get("bucketName");
}
@Override
public String getName() {
return "s3";
}
@Override
public MetaInfo onCopy(Content content, String newParentPath, String newName) {
String sourceKey = content.getStorageId();
String destinationKey = keyGenerator.apply(null);
try {
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.withRegion(clientRegion)
.build();
// Copy the object into a new object in the same bucket.
CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName, sourceKey, bucketName, destinationKey);
s3Client.copyObject(copyObjRequest);
} catch (Exception e) {
throw new RuntimeException("error copying file on s3");
}
return new MetaInfo(content.getSize(), destinationKey, null, getName());
}
@Override
public MetaInfo onMove(Content content, String newParentPath) {
//new contentPath can be set as remotePath to the storage backend ?
return new MetaInfo(content.getSize(),content.getStorageId(), content.getRemotePath(), getName());
}
@Override
public void onDelete(Content content) {
// TODO Auto-generated method stub
}
@Override
public MetaInfo upload(InputStream stream, String relativePath, String name) {
try {
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.withRegion(clientRegion)
.build();
String storageId = keyGenerator.apply(null);
ObjectMetadata metadata = new ObjectMetadata();
metadata.addUserMetadata("user", AuthorizationProvider.instance.get().getClient().getId());
PutObjectResult result = s3Client.putObject(
bucketName,
keyGenerator.apply(null),
stream,
metadata
);
long size = result.getMetadata().getContentLength();
s3Client.getObjectMetadata(bucketName, );
return new MetaInfo(content.getSize(),storageId, null, getName());
} catch (Exception e) {
throw new RuntimeException("error copying file on s3");
}
}
@Override
public InputStream download(Content item) {
try {
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.withRegion(clientRegion)
.build();
S3Object s3object = s3Client.getObject(bucketName, item.getStorageId());
S3ObjectInputStream inputStream = s3object.getObjectContent();
return inputStream;
} catch (Exception e) {
throw new RuntimeException("error copying file on s3");
}
}
@Override
public String getTotalSizeStored() {
// TODO Auto-generated method stub
return null;
}
@Override
public String getTotalItemsCount() {
// TODO Auto-generated method stub
return null;
}
}

2
src/main/webapp/WEB-INF/README
View File

@ -25,7 +25,7 @@ The projects leading to this software have received funding from a series of
Version
--------------------------------------------------
1.3.1 (20210910-085653)
1.5.0-SNAPSHOT (2021-11-26)
Please see the file named "changelog.xml" in this directory for the release notes.

23
src/test/java/org/gcube/data/access/fs/Expressions.java
View File

@ -1,9 +1,17 @@
package org.gcube.data.access.fs;
import static org.gcube.common.storagehub.model.Constants.enchriptedPrefix;
import java.io.File;
import java.security.Key;
import java.util.Base64;
import java.util.Calendar;
import javax.inject.Inject;
import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.encryption.SymmetricKey;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.storagehub.model.expressions.Expression;
import org.gcube.common.storagehub.model.expressions.GenericSearchableItem;
import org.gcube.common.storagehub.model.expressions.date.Before;
@ -38,17 +46,12 @@ public class Expressions {
*/
@Test
public void test() {
String entirePath = "sp2/comic/";
/*String[] parentPathSplit = entirePath.split("/");
System.out.println(parentPathSplit.length);
for (String v: parentPathSplit)
System.out.println(v);
*/
public void test() throws Exception {
Key key = SymmetricKey.loadKeyFromFile(new File("/home/lucio/d4science.research-infrastructures.eu.gcubekey"), "AES");
String idEnc = "E_clNONXlCQlk3aGNMK210eThXbmxXMUFiZXV3NGxqTElFRTlDM21sOFBHbGR5bE9Gc3hVTE9aRTF6T1VlZTF5bw==";
String id = StringEncrypter.getEncrypter().decrypt(new String(Base64.getUrlDecoder().decode(idEnc.replace(enchriptedPrefix, ""))),key);
String name = entirePath.replaceAll("([^/]*/)*(.*)", "$2");
String parentPath = entirePath.replaceAll("(([^/]*/)*)(.*)", "$1");
System.out.println(entirePath+" --"+name+"-- "+parentPath);
System.out.println(id);
}