From e0ae4727626f1f4854ca671001f2f554dc673b1e Mon Sep 17 00:00:00 2001 From: Giancarlo Panichi Date: Tue, 31 Jul 2018 14:48:35 +0000 Subject: [PATCH] ref 11724: SAI - Update to StorageHUB https://support.d4science.org/issues/11724 Updated to StorageHub git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/portlets/user/statistical-algorithms-importer@169997 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../server/generator/ProjectBuilder.java | 26 ++- .../server/storage/FilesStorage.java | 188 +++++++++--------- 2 files changed, 120 insertions(+), 94 deletions(-) diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectBuilder.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectBuilder.java index 43ccfa9..37cbc0f 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectBuilder.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectBuilder.java @@ -189,6 +189,19 @@ public class ProjectBuilder { project.getProjectTarget().getProjectDeploy().setPackageProject(packageProjectItemDescription); logger.debug("ProjectDeploy: " + project.getProjectTarget().getProjectDeploy()); + + try { + inputStream.close(); + } catch (IOException e) { + logger.error("Error in input stream close: " + e.getLocalizedMessage(), e); + } + + + if(projectPackageFile!=null&&projectPackageFile.exists()){ + projectPackageFile.delete(); + } + + } @@ -440,8 +453,19 @@ public class ProjectBuilder { project.getProjectTarget().getProjectDeploy().getFolder().getId()); logger.debug("ProjectPackageItem:" + packageUrl); - project.getProjectTarget().getProjectDeploy().setPackageProject(packageUrl); + + try { + inputStream.close(); + } catch (IOException e) { + logger.error("Error in input stream close: " + e.getLocalizedMessage(), e); + } + + + if(projectPackageFile!=null&&projectPackageFile.exists()){ + projectPackageFile.delete(); + } + } private void createTargetFolder() throws StatAlgoImporterServiceException { diff --git a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/FilesStorage.java b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/FilesStorage.java index 6cd6237..2fe729b 100644 --- a/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/FilesStorage.java +++ b/src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/FilesStorage.java @@ -2,12 +2,11 @@ package org.gcube.portlets.user.statisticalalgorithmsimporter.server.storage; import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -18,7 +17,7 @@ import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceSharedFolder; -import org.gcube.common.homelibrary.util.zip.ZipUtil; +import org.gcube.common.storagehub.client.StreamDescriptor; import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.ItemContainer; @@ -73,6 +72,7 @@ public class FilesStorage { } /** + * TODO * * @param user * User @@ -112,6 +112,8 @@ public class FilesStorage { } /** + * + * TODO * * @param user * User @@ -180,6 +182,8 @@ public class FilesStorage { } /** + * TODO + * * * @param user * user @@ -217,6 +221,8 @@ public class FilesStorage { } /** + * TODO + * * * @param user * user @@ -346,6 +352,8 @@ public class FilesStorage { } /** + * + * TODO * * @param user * User @@ -393,6 +401,8 @@ public class FilesStorage { } /** + * + * TODO * * @param user * User @@ -440,6 +450,8 @@ public class FilesStorage { } /** + * + * TODO * * @param user * User @@ -502,6 +514,7 @@ public class FilesStorage { } /** + * * * @param user * User @@ -514,24 +527,29 @@ public class FilesStorage { */ public InputStream retrieveProjectItemOnWorkspace(String user, String folderId) throws StatAlgoImporterServiceException { - Workspace ws; try { logger.info("Retrieve project item on workspace: [user=" + user + ", folderId=" + folderId + "]"); + StorageHubClient shc = new StorageHubClient(); + OpenResolver openResolver = shc.open(folderId); - ws = HomeLibrary.getUserWorkspace(user); - - WorkspaceItem workSpaceItem = ws.getItem(folderId); - if (!workSpaceItem.isFolder()) { - throw new StatAlgoImporterServiceException("Item is not valid folder!"); - } - - WorkspaceItem projectItem = ws.find(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME, folderId); - - if (projectItem == null) { + FolderContainer folderContainer = openResolver.asFolder(); + ListResolver listResolver = folderContainer.findByName(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME); + List items = listResolver.getItems(); + if (items == null || items.isEmpty()) { + logger.debug("No project found in this folder!"); throw new StatAlgoImporterServiceException("No project found in this folder!"); - } + } else { + Item item = items.get(0); + logger.debug("Item: " + item); + if (item != null) { + StreamDescriptor streamDescr = shc.open(item.getId()).asFile().download(); + return streamDescr.getStream(); - return retrieveInputStream(user, projectItem); + } else { + logger.debug("No project found in this folder!"); + throw new StatAlgoImporterServiceException("No project found in this folder!"); + } + } } catch (Throwable e) { logger.error("Retrieve project item on workspace: " + e.getLocalizedMessage(), e); @@ -550,23 +568,25 @@ public class FilesStorage { * Exception */ public boolean existProjectItemOnWorkspace(String user, String folderId) throws StatAlgoImporterServiceException { - Workspace ws; try { logger.info("Exist project item on workspace: [user=" + user + ", folderId=" + folderId + "]"); + StorageHubClient shc = new StorageHubClient(); + OpenResolver openResolver = shc.open(folderId); - ws = HomeLibrary.getUserWorkspace(user); - - WorkspaceItem workSpaceItem = ws.getItem(folderId); - if (!workSpaceItem.isFolder()) { - throw new StatAlgoImporterServiceException("Item is not valid folder!"); - } - - WorkspaceItem projectItem = ws.find(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME, folderId); - - if (projectItem == null) { + FolderContainer folderContainer = openResolver.asFolder(); + ListResolver listResolver = folderContainer.findByName(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME); + List items = listResolver.getItems(); + if (items == null || items.isEmpty()) { return false; } else { - return true; + Item item = items.get(0); + logger.debug("Item: " + item); + if (item != null) { + return true; + + } else { + return false; + } } } catch (Throwable e) { @@ -576,6 +596,7 @@ public class FilesStorage { } /** + * TODO * * @param user * User @@ -608,6 +629,7 @@ public class FilesStorage { } /** + * TODO * * @param user * User @@ -649,18 +671,16 @@ public class FilesStorage { * Excetpion */ public InputStream retrieveItemOnWorkspace(String user, String itemId) throws StatAlgoImporterServiceException { - Workspace ws; + try { logger.info("Retrieve item on workspace: [user=" + user + ", itemId=" + itemId + "]"); - ws = HomeLibrary.getUserWorkspace(user); + StorageHubClient shc = new StorageHubClient(); - WorkspaceItem workSpaceItem = ws.getItem(itemId); - if (workSpaceItem.isFolder()) { - throw new StatAlgoImporterServiceException("Folder Item is not valid!"); - } + StreamDescriptor streamDescr = shc.open(itemId).asFile().download(); - return retrieveInputStream(user, workSpaceItem); + InputStream is = streamDescr.getStream(); + return is; } catch (Throwable e) { logger.error("Retieve item on workspace: " + e.getLocalizedMessage(), e); @@ -670,33 +690,7 @@ public class FilesStorage { } /** - * - * @param user - * User - * @param wi - * Workspace item - * @return InputStream - * @throws StatAlgoImporterServiceException - * Exception - */ - private InputStream retrieveInputStream(String user, WorkspaceItem wi) throws StatAlgoImporterServiceException { - InputStream is = null; - try { - logger.info("Retrieve input stream: [user=" + user + ", workspaceItem=" + wi + "]"); - - org.gcube.common.homelibrary.home.workspace.folder.items.File gcubeItem = ((org.gcube.common.homelibrary.home.workspace.folder.items.File) wi); - is = gcubeItem.getData(); - return is; - - } catch (Throwable e) { - logger.error("Error retrieving file from storage: " + e.getLocalizedMessage(), e); - throw new StatAlgoImporterServiceException("Error retrieving file from storage: " + e.getLocalizedMessage(), - e); - } - - } - - /** + * TODO * * @param user * User @@ -738,6 +732,8 @@ public class FilesStorage { } /** + * TODO + * * * @param user * User @@ -766,7 +762,7 @@ public class FilesStorage { } } - // convert String into InputStream + ws.updateItem(itemId, is); return; @@ -788,21 +784,26 @@ public class FilesStorage { * Exception */ public File zipFolder(String user, String folderId) throws StatAlgoImporterServiceException { - Workspace ws; try { logger.info("Zip folder: [user=" + user + ", folderId=" + folderId + "]"); - ws = HomeLibrary.getUserWorkspace(user); + StorageHubClient shc = new StorageHubClient(); + + StreamDescriptor streamDescr = shc.open(folderId).asFolder().download(); + + File fileZip = Files.createTempFile(streamDescr.getFileName(), "").toFile(); + logger.debug("File zip: " + fileZip.getAbsolutePath()); + + try (FileOutputStream fos = new FileOutputStream(fileZip)) { + InputStream is = streamDescr.getStream(); + byte[] buf = new byte[2048]; + int read = -1; + while ((read = is.read(buf)) != -1) { + fos.write(buf, 0, read); + } - WorkspaceItem workSpaceItem = ws.getItem(folderId); - if (!workSpaceItem.isFolder()) { - throw new StatAlgoImporterServiceException("Item is not valid folder!"); } - WorkspaceFolder folder = (WorkspaceFolder) workSpaceItem; - - File fileZip = ZipUtil.zipFolder(folder); - return fileZip; } catch (Throwable e) { @@ -826,21 +827,25 @@ public class FilesStorage { */ public File zipFolder(String user, String folderId, List idsToExclude) throws StatAlgoImporterServiceException { - Workspace ws; try { logger.info("Zip folder with exclude: [user=" + user + ", folderId=" + folderId + "]"); + String[] idsArray = new String[idsToExclude.size()]; + idsArray = idsToExclude.toArray(idsArray); - ws = HomeLibrary.getUserWorkspace(user); + StorageHubClient shc = new StorageHubClient(); + StreamDescriptor streamDescr = shc.open(folderId).asFolder().download(idsArray); - WorkspaceItem workSpaceItem = ws.getItem(folderId); - if (!workSpaceItem.isFolder()) { - throw new StatAlgoImporterServiceException("Item is not valid folder!"); + File fileZip = Files.createTempFile(streamDescr.getFileName(), "").toFile(); + logger.debug("File zip: " + fileZip.getAbsolutePath()); + + try (FileOutputStream fos = new FileOutputStream(fileZip); InputStream is = streamDescr.getStream()) { + byte[] buf = new byte[1024]; + int read = -1; + while ((read = is.read(buf)) != -1) { + fos.write(buf, 0, read); + } } - WorkspaceFolder folder = (WorkspaceFolder) workSpaceItem; - - File fileZip = ZipUtil.zipFolder(folder, false, idsToExclude); - return fileZip; } catch (Throwable e) { @@ -866,9 +871,14 @@ public class FilesStorage { URL smpFile = new URL(fileUrl); URLConnection uc = (URLConnection) smpFile.openConnection(); - InputStream is = uc.getInputStream(); - inputStreamToFile(is, destination); - is.close(); + + try (FileOutputStream out = new FileOutputStream(destination.toFile()); + InputStream is = uc.getInputStream();) { + byte buf[] = new byte[1024]; + int len = 0; + while ((len = is.read(buf)) > 0) + out.write(buf, 0, len); + } } catch (Throwable e) { logger.error("Download input file: " + e.getLocalizedMessage(), e); @@ -876,16 +886,8 @@ public class FilesStorage { } } - private void inputStreamToFile(InputStream is, Path destination) throws FileNotFoundException, IOException { - FileOutputStream out = new FileOutputStream(destination.toFile()); - byte buf[] = new byte[1024]; - int len = 0; - while ((len = is.read(buf)) > 0) - out.write(buf, 0, len); - out.close(); - } - /** + * TODO * * @param user * User