From 309ea0945337607b6ac0171c1a966ea83400b0e4 Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Fri, 29 Jun 2018 16:04:59 +0000 Subject: [PATCH] ref #11879: TDM - Update to StorageHUB https://support.d4science.org/issues/#11879 Updated to StorageHub git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/portlets/user/tabular-data-gwt-service@169564 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 6 +- distro/changelog.xml | 4 + pom.xml | 9 +- .../server/storage/FilesStorage.java | 170 ++++++++---------- 4 files changed, 86 insertions(+), 103 deletions(-) diff --git a/.classpath b/.classpath index d83b25c..1062553 100644 --- a/.classpath +++ b/.classpath @@ -1,12 +1,12 @@ - + - + @@ -33,5 +33,5 @@ - + diff --git a/distro/changelog.xml b/distro/changelog.xml index 47673c7..1dda551 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,4 +1,8 @@ + + Updated to StorageHub [task #11879] + Added Excel Dataset Export support [ticket #9944] diff --git a/pom.xml b/pom.xml index 2592888..2fb0c9c 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ 4.0.0 org.gcube.portlets.user tabular-data-gwt-service - 2.20.0-SNAPSHOT + 2.21.0-SNAPSHOT tabular-data-gwt-service tabular-data-gwt-service allows communication between the GUI and services @@ -202,6 +202,13 @@ storage-manager-wrapper provided + + + + org.gcube.common + storagehub-client-library + provided + diff --git a/src/main/java/org/gcube/portlets/user/td/gwtservice/server/storage/FilesStorage.java b/src/main/java/org/gcube/portlets/user/td/gwtservice/server/storage/FilesStorage.java index 9b686be..fc652a0 100644 --- a/src/main/java/org/gcube/portlets/user/td/gwtservice/server/storage/FilesStorage.java +++ b/src/main/java/org/gcube/portlets/user/td/gwtservice/server/storage/FilesStorage.java @@ -4,7 +4,6 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; -import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; @@ -15,6 +14,11 @@ import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder; import org.gcube.common.homelibrary.util.WorkspaceUtil; +import org.gcube.common.storagehub.client.StreamDescriptor; +import org.gcube.common.storagehub.client.dsl.FileContainer; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.items.AbstractFileItem; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanager.storageclient.wrapper.AccessType; import org.gcube.contentmanager.storageclient.wrapper.MemoryType; @@ -52,7 +56,6 @@ public class FilesStorage { } catch (Throwable e) { logger.error("Error no csv file loaded on storage" + e.getLocalizedMessage()); - e.printStackTrace(); throw e; } @@ -75,7 +78,6 @@ public class FilesStorage { return id; } catch (Throwable e) { logger.error("Error no codelist mapping file loaded on storage" + e.getLocalizedMessage()); - e.printStackTrace(); throw e; } @@ -92,29 +94,24 @@ public class FilesStorage { * Exception */ public String storageCodelistMappingTempFile(String user, String url) throws TDGWTServiceException { - InputStream is = null; try { - URL address = new URL(url); - is = address.openStream(); + String id = null; - IClient client = new StorageClient(user, AccessType.PUBLIC, MemoryType.VOLATILE).getClient(); - String remotePath = "/CodelistMappingImport/" + address.getFile(); - logger.debug("remotePath: " + remotePath); - String id = client.put(true).LFile(is).RFile(remotePath); - is.close(); + URL address = new URL(url); + try (InputStream is = address.openStream()) { + + IClient client = new StorageClient(user, AccessType.PUBLIC, MemoryType.VOLATILE).getClient(); + String remotePath = "/CodelistMappingImport/" + address.getFile(); + logger.debug("remotePath: " + remotePath); + id = client.put(true).LFile(is).RFile(remotePath); + } return id; - } catch (IOException e) { - logger.error("Error no codelist mapping file loaded on storage" + e.getLocalizedMessage()); - e.printStackTrace(); - throw new TDGWTServiceException( - "Error no codelist mapping file loaded on storage" + e.getLocalizedMessage()); } catch (Throwable e) { logger.error("Error no codelist mapping file loaded on storage" + e.getLocalizedMessage()); - e.printStackTrace(); - throw e; + throw new TDGWTServiceException( + "Error no codelist mapping file loaded on storage" + e.getLocalizedMessage()); } - } /** @@ -130,41 +127,31 @@ public class FilesStorage { * Exception */ public void retrieveFile(String user, WorkspaceItem wi, File file) throws TDGWTServiceException { - InputStream is = null; try { - - org.gcube.common.homelibrary.home.workspace.folder.items.File gcubeItem = ((org.gcube.common.homelibrary.home.workspace.folder.items.File) wi); - // SMPUrl smsHome = new SMPUrl(gcubeItem.getPublicLink()); - URL url = new URL(gcubeItem.getPublicLink()); - - logger.debug("smsHome: [host:" + url.getHost() + " path:" + url.getPath() + " ref:" + url.getRef() - + " userinfo:" + url.getUserInfo() + " ]"); - URLConnection uc = null; - uc = (URLConnection) url.openConnection(); - is = uc.getInputStream(); - - } catch (Throwable e) { - logger.error("Error retrieving file from storage", e); - e.printStackTrace(); - throw new TDGWTServiceException("Error retrieving file from storage: " + e.getLocalizedMessage(), e); - } - - try { - BufferedInputStream bis = new BufferedInputStream(is); - FileOutputStream os = new FileOutputStream(file); - BufferedOutputStream bos = new BufferedOutputStream(os); - byte[] buffer = new byte[1024]; - int readCount; - while ((readCount = bis.read(buffer)) > 0) { - bos.write(buffer, 0, readCount); + logger.debug("retrieveFile: [user=" + user + ", item=" + wi + "]"); + StorageHubClient shc = new StorageHubClient(); + FileContainer fileContainer = shc.open(wi.getId()).asFile(); + StreamDescriptor streamDescriptor = fileContainer.download(); + try (BufferedInputStream bis = new BufferedInputStream(streamDescriptor.getStream())) { + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) { + + byte[] buffer = new byte[1024]; + int readCount; + while ((readCount = bis.read(buffer)) > 0) { + bos.write(buffer, 0, readCount); + } + } catch (Throwable e) { + logger.error("Error trasferring file from storage: " + e.getLocalizedMessage(), e); + throw new TDGWTServiceException("Error trasferring file from storage: " + e.getLocalizedMessage(), + e); + } } - bos.close(); - } catch (Throwable e) { - logger.error("Error trasferring file from storage: " + e.getLocalizedMessage(), e); - e.printStackTrace(); - throw new TDGWTServiceException("Error trasferring file from storage: " + e.getLocalizedMessage(), e); + } catch (Throwable e) { + logger.error("Error retrieving file from StorageHub", e); + throw new TDGWTServiceException("Error retrieving file from StorageHub: " + e.getLocalizedMessage(), e); } + } /** @@ -181,21 +168,15 @@ public class FilesStorage { public InputStream retrieveInputStream(String user, WorkspaceItem wi) throws TDGWTServiceException { InputStream is = null; try { - - org.gcube.common.homelibrary.home.workspace.folder.items.File gcubeItem = ((org.gcube.common.homelibrary.home.workspace.folder.items.File) wi); - // SMPUrl smsHome = new SMPUrl(gcubeItem.getPublicLink()); - URL url = new URL(gcubeItem.getPublicLink()); - - logger.debug("smsHome: [host:" + url.getHost() + " path:" + url.getPath() + " ref:" + url.getRef() - + " userinfo:" + url.getUserInfo() + " ]"); - URLConnection uc = null; - uc = (URLConnection) url.openConnection(); - is = uc.getInputStream(); + logger.debug("retrieveFile: [user=" + user + ", item=" + wi + "]"); + StorageHubClient shc = new StorageHubClient(); + FileContainer fileContainer = shc.open(wi.getId()).asFile(); + StreamDescriptor streamDescriptor = fileContainer.download(); + is = streamDescriptor.getStream(); } catch (Throwable e) { - logger.error("Error retrieving file from storage", e); - e.printStackTrace(); - throw new TDGWTServiceException("Error retrieving file from storage: " + e.getLocalizedMessage(), e); + logger.error("Error retrieving file from StorageHub", e); + throw new TDGWTServiceException("Error retrieving file from StorageHub: " + e.getLocalizedMessage(), e); } return is; @@ -225,7 +206,6 @@ public class FilesStorage { } catch (Throwable e) { logger.error("Error retrieving file from storage", e); - e.printStackTrace(); throw new TDGWTServiceException("Error retrieving file from storage: " + e.getLocalizedMessage(), e); } @@ -239,23 +219,22 @@ public class FilesStorage { * Uri * @param user * User - * @param item_name + * @param itemName * Item name - * @param item_description + * @param itemDescription * Item description - * @param item_mimetype + * @param itemMimetype * Item mime type - * @param item_folder + * @param folderId * Destination folder * @throws TDGWTServiceException * Exception */ - public void createItemOnWorkspace(String uri, String user, String item_name, String item_description, - String item_mimetype, String item_folder) throws TDGWTServiceException { - InputStream is = null; + public void createItemOnWorkspace(String uri, String user, String itemName, String itemDescription, + String itemMimetype, String folderId) throws TDGWTServiceException { + try { - // SMPUrl smsHome = new SMPUrl(uri); URL url = new URL(uri); logger.debug("smsHome: [host:" + url.getHost() + " path:" + url.getPath() + " ref:" + url.getRef() @@ -263,22 +242,17 @@ public class FilesStorage { URLConnection uc = null; uc = (URLConnection) url.openConnection(); - is = uc.getInputStream(); + try (InputStream is = uc.getInputStream()) { - Workspace ws = HomeLibrary.getUserWorkspace(user); - - WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(item_folder); - String uniqueName = WorkspaceUtil.getUniqueName(item_name, folder); - - logger.debug("ws.createExternalFile [folder: " + folder + ", uniqueName: " + uniqueName + ", description: " - + item_description + ", mimetype: " + item_mimetype + ", InputStream: " + is + "]"); - WorkspaceUtil.createExternalFile(folder, uniqueName, item_description, item_mimetype, is); - - is.close(); + StorageHubClient shc = new StorageHubClient(); + FolderContainer folderContainer = shc.open(folderId).asFolder(); + FileContainer fileContainer = folderContainer.uploadFile(is, itemName, itemDescription); + AbstractFileItem afi = fileContainer.get(); + logger.debug("Created file on workspace: " + afi.getId()); + } } catch (Throwable e) { logger.error("Error creating item on workspace", e); - e.printStackTrace(); throw new TDGWTServiceException("Error creating item on workspace: " + e.getLocalizedMessage(), e); } @@ -290,34 +264,33 @@ public class FilesStorage { * Storage id * @param user * User - * @param item_name + * @param itemName * Item name - * @param item_description + * @param itemDescription * Item description - * @param item_mimetype + * @param itemMimetype * Item mime type - * @param item_folder + * @param itemFolder * Destination Folder * @throws TDGWTServiceException * Exception */ - public void createItemOnWorkspaceByStorageId(String storageId, String user, String item_name, - String item_description, String item_mimetype, String item_folder) throws TDGWTServiceException { + public void createItemOnWorkspaceByStorageId(String storageId, String user, String itemName, + String itemDescription, String itemMimetype, String itemFolder) throws TDGWTServiceException { try { Workspace ws = HomeLibrary.getUserWorkspace(user); - - WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(item_folder); - String uniqueName = WorkspaceUtil.getUniqueName(item_name, folder); + + WorkspaceFolder folder = (WorkspaceFolder) ws.getItem(itemFolder); + String uniqueName = WorkspaceUtil.getUniqueName(itemName, folder); logger.debug("ws.createExternalFile [folder: " + folder + ", uniqueName: " + uniqueName + ", description: " - + item_description + ", mimetype: " + item_mimetype + ", StorageId: " + storageId + "]"); - WorkspaceUtil.createExternalFile(folder, uniqueName, item_description, item_mimetype, storageId); + + itemDescription + ", mimetype: " + itemMimetype + ", StorageId: " + storageId + "]"); + WorkspaceUtil.createExternalFile(folder, uniqueName, itemDescription, itemMimetype, storageId); } catch (Throwable e) { - logger.error("Error creating item on workspace", e); - e.printStackTrace(); + logger.error("Error creating item on workspace: " + e.getLocalizedMessage(), e); throw new TDGWTServiceException("Error creating item on workspace: " + e.getLocalizedMessage(), e); } @@ -346,8 +319,7 @@ public class FilesStorage { } } catch (Throwable e) { - logger.error("Error retrieving VRE folder by scope!", e); - e.printStackTrace(); + logger.error("Error retrieving VRE folder by scope! " + e.getLocalizedMessage(), e); throw new TDGWTServiceException("Error retrieving VRE folder by scope: " + e.getLocalizedMessage(), e); }