gCubeSystem
/
statistical-algorithms-importer
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ref 11724: SAI - Update to StorageHUB 

https://support.d4science.org/issues/11724

Updated to StorageHub 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/portlets/user/statistical-algorithms-importer@169997 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Giancarlo Panichi 2018-07-31 14:48:35 +00:00
parent 377d755e87
commit e0ae472762
2 changed files with 120 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/generator/ProjectBuilder.java
View File

@ -189,6 +189,19 @@ public class ProjectBuilder {
project.getProjectTarget().getProjectDeploy().setPackageProject(packageProjectItemDescription);
logger.debug("ProjectDeploy: " + project.getProjectTarget().getProjectDeploy());
try {
inputStream.close();
} catch (IOException e) {
logger.error("Error in input stream close: " + e.getLocalizedMessage(), e);
}
if(projectPackageFile!=null&&projectPackageFile.exists()){
projectPackageFile.delete();
}
}
@ -440,8 +453,19 @@ public class ProjectBuilder {
project.getProjectTarget().getProjectDeploy().getFolder().getId());
logger.debug("ProjectPackageItem:" + packageUrl);
project.getProjectTarget().getProjectDeploy().setPackageProject(packageUrl);
try {
inputStream.close();
} catch (IOException e) {
logger.error("Error in input stream close: " + e.getLocalizedMessage(), e);
}
if(projectPackageFile!=null&&projectPackageFile.exists()){
projectPackageFile.delete();
}
}
private void createTargetFolder() throws StatAlgoImporterServiceException {

188
src/main/java/org/gcube/portlets/user/statisticalalgorithmsimporter/server/storage/FilesStorage.java
View File

@ -2,12 +2,11 @@ package org.gcube.portlets.user.statisticalalgorithmsimporter.server.storage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@ -18,7 +17,7 @@ import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceSharedFolder;
import org.gcube.common.homelibrary.util.zip.ZipUtil;
import org.gcube.common.storagehub.client.StreamDescriptor;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
@ -73,6 +72,7 @@ public class FilesStorage {
}
/**
* TODO
*
* @param user
* User
@ -112,6 +112,8 @@ public class FilesStorage {
}
/**
*
* TODO
*
* @param user
* User
@ -180,6 +182,8 @@ public class FilesStorage {
}
/**
* TODO
*
*
* @param user
* user
@ -217,6 +221,8 @@ public class FilesStorage {
}
/**
* TODO
*
*
* @param user
* user
@ -346,6 +352,8 @@ public class FilesStorage {
}
/**
*
* TODO
*
* @param user
* User
@ -393,6 +401,8 @@ public class FilesStorage {
}
/**
*
* TODO
*
* @param user
* User
@ -440,6 +450,8 @@ public class FilesStorage {
}
/**
*
* TODO
*
* @param user
* User
@ -502,6 +514,7 @@ public class FilesStorage {
}
/**
*
*
* @param user
* User
@ -514,24 +527,29 @@ public class FilesStorage {
*/
public InputStream retrieveProjectItemOnWorkspace(String user, String folderId)
throws StatAlgoImporterServiceException {
Workspace ws;
try {
logger.info("Retrieve project item on workspace: [user=" + user + ", folderId=" + folderId + "]");
StorageHubClient shc = new StorageHubClient();
OpenResolver openResolver = shc.open(folderId);
ws = HomeLibrary.getUserWorkspace(user);
WorkspaceItem workSpaceItem = ws.getItem(folderId);
if (!workSpaceItem.isFolder()) {
throw new StatAlgoImporterServiceException("Item is not valid folder!");
}
WorkspaceItem projectItem = ws.find(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME, folderId);
if (projectItem == null) {
FolderContainer folderContainer = openResolver.asFolder();
ListResolver listResolver = folderContainer.findByName(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME);
List<? extends Item> items = listResolver.getItems();
if (items == null || items.isEmpty()) {
logger.debug("No project found in this folder!");
throw new StatAlgoImporterServiceException("No project found in this folder!");
}
} else {
Item item = items.get(0);
logger.debug("Item: " + item);
if (item != null) {
StreamDescriptor streamDescr = shc.open(item.getId()).asFile().download();
return streamDescr.getStream();
return retrieveInputStream(user, projectItem);
} else {
logger.debug("No project found in this folder!");
throw new StatAlgoImporterServiceException("No project found in this folder!");
}
}
} catch (Throwable e) {
logger.error("Retrieve project item on workspace: " + e.getLocalizedMessage(), e);
@ -550,23 +568,25 @@ public class FilesStorage {
* Exception
*/
public boolean existProjectItemOnWorkspace(String user, String folderId) throws StatAlgoImporterServiceException {
Workspace ws;
try {
logger.info("Exist project item on workspace: [user=" + user + ", folderId=" + folderId + "]");
StorageHubClient shc = new StorageHubClient();
OpenResolver openResolver = shc.open(folderId);
ws = HomeLibrary.getUserWorkspace(user);
WorkspaceItem workSpaceItem = ws.getItem(folderId);
if (!workSpaceItem.isFolder()) {
throw new StatAlgoImporterServiceException("Item is not valid folder!");
}
WorkspaceItem projectItem = ws.find(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME, folderId);
if (projectItem == null) {
FolderContainer folderContainer = openResolver.asFolder();
ListResolver listResolver = folderContainer.findByName(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME);
List<? extends Item> items = listResolver.getItems();
if (items == null || items.isEmpty()) {
return false;
} else {
return true;
Item item = items.get(0);
logger.debug("Item: " + item);
if (item != null) {
return true;
} else {
return false;
}
}
} catch (Throwable e) {
@ -576,6 +596,7 @@ public class FilesStorage {
}
/**
* TODO
*
* @param user
* User
@ -608,6 +629,7 @@ public class FilesStorage {
}
/**
* TODO
*
* @param user
* User
@ -649,18 +671,16 @@ public class FilesStorage {
* Excetpion
*/
public InputStream retrieveItemOnWorkspace(String user, String itemId) throws StatAlgoImporterServiceException {
Workspace ws;
try {
logger.info("Retrieve item on workspace: [user=" + user + ", itemId=" + itemId + "]");
ws = HomeLibrary.getUserWorkspace(user);
StorageHubClient shc = new StorageHubClient();
WorkspaceItem workSpaceItem = ws.getItem(itemId);
if (workSpaceItem.isFolder()) {
throw new StatAlgoImporterServiceException("Folder Item is not valid!");
}
StreamDescriptor streamDescr = shc.open(itemId).asFile().download();
return retrieveInputStream(user, workSpaceItem);
InputStream is = streamDescr.getStream();
return is;
} catch (Throwable e) {
logger.error("Retieve item on workspace: " + e.getLocalizedMessage(), e);
@ -670,33 +690,7 @@ public class FilesStorage {
}
/**
*
* @param user
* User
* @param wi
* Workspace item
* @return InputStream
* @throws StatAlgoImporterServiceException
* Exception
*/
private InputStream retrieveInputStream(String user, WorkspaceItem wi) throws StatAlgoImporterServiceException {
InputStream is = null;
try {
logger.info("Retrieve input stream: [user=" + user + ", workspaceItem=" + wi + "]");
org.gcube.common.homelibrary.home.workspace.folder.items.File gcubeItem = ((org.gcube.common.homelibrary.home.workspace.folder.items.File) wi);
is = gcubeItem.getData();
return is;
} catch (Throwable e) {
logger.error("Error retrieving file from storage: " + e.getLocalizedMessage(), e);
throw new StatAlgoImporterServiceException("Error retrieving file from storage: " + e.getLocalizedMessage(),
e);
}
}
/**
* TODO
*
* @param user
* User
@ -738,6 +732,8 @@ public class FilesStorage {
}
/**
* TODO
*
*
* @param user
* User
@ -766,7 +762,7 @@ public class FilesStorage {
}
}
// convert String into InputStream
ws.updateItem(itemId, is);
return;
@ -788,21 +784,26 @@ public class FilesStorage {
* Exception
*/
public File zipFolder(String user, String folderId) throws StatAlgoImporterServiceException {
Workspace ws;
try {
logger.info("Zip folder: [user=" + user + ", folderId=" + folderId + "]");
ws = HomeLibrary.getUserWorkspace(user);
StorageHubClient shc = new StorageHubClient();
StreamDescriptor streamDescr = shc.open(folderId).asFolder().download();
File fileZip = Files.createTempFile(streamDescr.getFileName(), "").toFile();
logger.debug("File zip: " + fileZip.getAbsolutePath());
try (FileOutputStream fos = new FileOutputStream(fileZip)) {
InputStream is = streamDescr.getStream();
byte[] buf = new byte[2048];
int read = -1;
while ((read = is.read(buf)) != -1) {
fos.write(buf, 0, read);
}
WorkspaceItem workSpaceItem = ws.getItem(folderId);
if (!workSpaceItem.isFolder()) {
throw new StatAlgoImporterServiceException("Item is not valid folder!");
}
WorkspaceFolder folder = (WorkspaceFolder) workSpaceItem;
File fileZip = ZipUtil.zipFolder(folder);
return fileZip;
} catch (Throwable e) {
@ -826,21 +827,25 @@ public class FilesStorage {
*/
public File zipFolder(String user, String folderId, List<String> idsToExclude)
throws StatAlgoImporterServiceException {
Workspace ws;
try {
logger.info("Zip folder with exclude: [user=" + user + ", folderId=" + folderId + "]");
String[] idsArray = new String[idsToExclude.size()];
idsArray = idsToExclude.toArray(idsArray);
ws = HomeLibrary.getUserWorkspace(user);
StorageHubClient shc = new StorageHubClient();
StreamDescriptor streamDescr = shc.open(folderId).asFolder().download(idsArray);
WorkspaceItem workSpaceItem = ws.getItem(folderId);
if (!workSpaceItem.isFolder()) {
throw new StatAlgoImporterServiceException("Item is not valid folder!");
File fileZip = Files.createTempFile(streamDescr.getFileName(), "").toFile();
logger.debug("File zip: " + fileZip.getAbsolutePath());
try (FileOutputStream fos = new FileOutputStream(fileZip); InputStream is = streamDescr.getStream()) {
byte[] buf = new byte[1024];
int read = -1;
while ((read = is.read(buf)) != -1) {
fos.write(buf, 0, read);
}
}
WorkspaceFolder folder = (WorkspaceFolder) workSpaceItem;
File fileZip = ZipUtil.zipFolder(folder, false, idsToExclude);
return fileZip;
} catch (Throwable e) {
@ -866,9 +871,14 @@ public class FilesStorage {
URL smpFile = new URL(fileUrl);
URLConnection uc = (URLConnection) smpFile.openConnection();
InputStream is = uc.getInputStream();
inputStreamToFile(is, destination);
is.close();
try (FileOutputStream out = new FileOutputStream(destination.toFile());
InputStream is = uc.getInputStream();) {
byte buf[] = new byte[1024];
int len = 0;
while ((len = is.read(buf)) > 0)
out.write(buf, 0, len);
}
} catch (Throwable e) {
logger.error("Download input file: " + e.getLocalizedMessage(), e);
@ -876,16 +886,8 @@ public class FilesStorage {
}
}
private void inputStreamToFile(InputStream is, Path destination) throws FileNotFoundException, IOException {
FileOutputStream out = new FileOutputStream(destination.toFile());
byte buf[] = new byte[1024];
int len = 0;
while ((len = is.read(buf)) > 0)
out.write(buf, 0, len);
out.close();
}
/**
* TODO
*
* @param user
* User