ref 11724: SAI - Update to StorageHUB
https://support.d4science.org/issues/11724 Updated to StorageHub git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/portlets/user/statistical-algorithms-importer@169997 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
377d755e87
commit
e0ae472762
|
@ -189,6 +189,19 @@ public class ProjectBuilder {
|
||||||
|
|
||||||
project.getProjectTarget().getProjectDeploy().setPackageProject(packageProjectItemDescription);
|
project.getProjectTarget().getProjectDeploy().setPackageProject(packageProjectItemDescription);
|
||||||
logger.debug("ProjectDeploy: " + project.getProjectTarget().getProjectDeploy());
|
logger.debug("ProjectDeploy: " + project.getProjectTarget().getProjectDeploy());
|
||||||
|
|
||||||
|
try {
|
||||||
|
inputStream.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error("Error in input stream close: " + e.getLocalizedMessage(), e);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(projectPackageFile!=null&&projectPackageFile.exists()){
|
||||||
|
projectPackageFile.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -440,8 +453,19 @@ public class ProjectBuilder {
|
||||||
project.getProjectTarget().getProjectDeploy().getFolder().getId());
|
project.getProjectTarget().getProjectDeploy().getFolder().getId());
|
||||||
|
|
||||||
logger.debug("ProjectPackageItem:" + packageUrl);
|
logger.debug("ProjectPackageItem:" + packageUrl);
|
||||||
|
|
||||||
project.getProjectTarget().getProjectDeploy().setPackageProject(packageUrl);
|
project.getProjectTarget().getProjectDeploy().setPackageProject(packageUrl);
|
||||||
|
|
||||||
|
try {
|
||||||
|
inputStream.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error("Error in input stream close: " + e.getLocalizedMessage(), e);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(projectPackageFile!=null&&projectPackageFile.exists()){
|
||||||
|
projectPackageFile.delete();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createTargetFolder() throws StatAlgoImporterServiceException {
|
private void createTargetFolder() throws StatAlgoImporterServiceException {
|
||||||
|
|
|
@ -2,12 +2,11 @@ package org.gcube.portlets.user.statisticalalgorithmsimporter.server.storage;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.net.URLConnection;
|
import java.net.URLConnection;
|
||||||
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -18,7 +17,7 @@ import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
|
||||||
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
|
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
|
||||||
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
|
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
|
||||||
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceSharedFolder;
|
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceSharedFolder;
|
||||||
import org.gcube.common.homelibrary.util.zip.ZipUtil;
|
import org.gcube.common.storagehub.client.StreamDescriptor;
|
||||||
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
||||||
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
||||||
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
||||||
|
@ -73,6 +72,7 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -112,6 +112,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
*
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -180,6 +182,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
|
*
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* user
|
* user
|
||||||
|
@ -217,6 +221,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
|
*
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* user
|
* user
|
||||||
|
@ -346,6 +352,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
*
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -393,6 +401,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
*
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -440,6 +450,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
*
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -502,6 +514,7 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
*
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -514,24 +527,29 @@ public class FilesStorage {
|
||||||
*/
|
*/
|
||||||
public InputStream retrieveProjectItemOnWorkspace(String user, String folderId)
|
public InputStream retrieveProjectItemOnWorkspace(String user, String folderId)
|
||||||
throws StatAlgoImporterServiceException {
|
throws StatAlgoImporterServiceException {
|
||||||
Workspace ws;
|
|
||||||
try {
|
try {
|
||||||
logger.info("Retrieve project item on workspace: [user=" + user + ", folderId=" + folderId + "]");
|
logger.info("Retrieve project item on workspace: [user=" + user + ", folderId=" + folderId + "]");
|
||||||
|
StorageHubClient shc = new StorageHubClient();
|
||||||
|
OpenResolver openResolver = shc.open(folderId);
|
||||||
|
|
||||||
ws = HomeLibrary.getUserWorkspace(user);
|
FolderContainer folderContainer = openResolver.asFolder();
|
||||||
|
ListResolver listResolver = folderContainer.findByName(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME);
|
||||||
WorkspaceItem workSpaceItem = ws.getItem(folderId);
|
List<? extends Item> items = listResolver.getItems();
|
||||||
if (!workSpaceItem.isFolder()) {
|
if (items == null || items.isEmpty()) {
|
||||||
throw new StatAlgoImporterServiceException("Item is not valid folder!");
|
logger.debug("No project found in this folder!");
|
||||||
}
|
|
||||||
|
|
||||||
WorkspaceItem projectItem = ws.find(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME, folderId);
|
|
||||||
|
|
||||||
if (projectItem == null) {
|
|
||||||
throw new StatAlgoImporterServiceException("No project found in this folder!");
|
throw new StatAlgoImporterServiceException("No project found in this folder!");
|
||||||
}
|
} else {
|
||||||
|
Item item = items.get(0);
|
||||||
|
logger.debug("Item: " + item);
|
||||||
|
if (item != null) {
|
||||||
|
StreamDescriptor streamDescr = shc.open(item.getId()).asFile().download();
|
||||||
|
return streamDescr.getStream();
|
||||||
|
|
||||||
return retrieveInputStream(user, projectItem);
|
} else {
|
||||||
|
logger.debug("No project found in this folder!");
|
||||||
|
throw new StatAlgoImporterServiceException("No project found in this folder!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
logger.error("Retrieve project item on workspace: " + e.getLocalizedMessage(), e);
|
logger.error("Retrieve project item on workspace: " + e.getLocalizedMessage(), e);
|
||||||
|
@ -550,23 +568,25 @@ public class FilesStorage {
|
||||||
* Exception
|
* Exception
|
||||||
*/
|
*/
|
||||||
public boolean existProjectItemOnWorkspace(String user, String folderId) throws StatAlgoImporterServiceException {
|
public boolean existProjectItemOnWorkspace(String user, String folderId) throws StatAlgoImporterServiceException {
|
||||||
Workspace ws;
|
|
||||||
try {
|
try {
|
||||||
logger.info("Exist project item on workspace: [user=" + user + ", folderId=" + folderId + "]");
|
logger.info("Exist project item on workspace: [user=" + user + ", folderId=" + folderId + "]");
|
||||||
|
StorageHubClient shc = new StorageHubClient();
|
||||||
|
OpenResolver openResolver = shc.open(folderId);
|
||||||
|
|
||||||
ws = HomeLibrary.getUserWorkspace(user);
|
FolderContainer folderContainer = openResolver.asFolder();
|
||||||
|
ListResolver listResolver = folderContainer.findByName(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME);
|
||||||
WorkspaceItem workSpaceItem = ws.getItem(folderId);
|
List<? extends Item> items = listResolver.getItems();
|
||||||
if (!workSpaceItem.isFolder()) {
|
if (items == null || items.isEmpty()) {
|
||||||
throw new StatAlgoImporterServiceException("Item is not valid folder!");
|
|
||||||
}
|
|
||||||
|
|
||||||
WorkspaceItem projectItem = ws.find(STATISTICAL_ALGORITHM_PROJECT_FILE_NAME, folderId);
|
|
||||||
|
|
||||||
if (projectItem == null) {
|
|
||||||
return false;
|
return false;
|
||||||
} else {
|
} else {
|
||||||
return true;
|
Item item = items.get(0);
|
||||||
|
logger.debug("Item: " + item);
|
||||||
|
if (item != null) {
|
||||||
|
return true;
|
||||||
|
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
|
@ -576,6 +596,7 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -608,6 +629,7 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -649,18 +671,16 @@ public class FilesStorage {
|
||||||
* Excetpion
|
* Excetpion
|
||||||
*/
|
*/
|
||||||
public InputStream retrieveItemOnWorkspace(String user, String itemId) throws StatAlgoImporterServiceException {
|
public InputStream retrieveItemOnWorkspace(String user, String itemId) throws StatAlgoImporterServiceException {
|
||||||
Workspace ws;
|
|
||||||
try {
|
try {
|
||||||
logger.info("Retrieve item on workspace: [user=" + user + ", itemId=" + itemId + "]");
|
logger.info("Retrieve item on workspace: [user=" + user + ", itemId=" + itemId + "]");
|
||||||
|
|
||||||
ws = HomeLibrary.getUserWorkspace(user);
|
StorageHubClient shc = new StorageHubClient();
|
||||||
|
|
||||||
WorkspaceItem workSpaceItem = ws.getItem(itemId);
|
StreamDescriptor streamDescr = shc.open(itemId).asFile().download();
|
||||||
if (workSpaceItem.isFolder()) {
|
|
||||||
throw new StatAlgoImporterServiceException("Folder Item is not valid!");
|
|
||||||
}
|
|
||||||
|
|
||||||
return retrieveInputStream(user, workSpaceItem);
|
InputStream is = streamDescr.getStream();
|
||||||
|
return is;
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
logger.error("Retieve item on workspace: " + e.getLocalizedMessage(), e);
|
logger.error("Retieve item on workspace: " + e.getLocalizedMessage(), e);
|
||||||
|
@ -670,33 +690,7 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
* TODO
|
||||||
* @param user
|
|
||||||
* User
|
|
||||||
* @param wi
|
|
||||||
* Workspace item
|
|
||||||
* @return InputStream
|
|
||||||
* @throws StatAlgoImporterServiceException
|
|
||||||
* Exception
|
|
||||||
*/
|
|
||||||
private InputStream retrieveInputStream(String user, WorkspaceItem wi) throws StatAlgoImporterServiceException {
|
|
||||||
InputStream is = null;
|
|
||||||
try {
|
|
||||||
logger.info("Retrieve input stream: [user=" + user + ", workspaceItem=" + wi + "]");
|
|
||||||
|
|
||||||
org.gcube.common.homelibrary.home.workspace.folder.items.File gcubeItem = ((org.gcube.common.homelibrary.home.workspace.folder.items.File) wi);
|
|
||||||
is = gcubeItem.getData();
|
|
||||||
return is;
|
|
||||||
|
|
||||||
} catch (Throwable e) {
|
|
||||||
logger.error("Error retrieving file from storage: " + e.getLocalizedMessage(), e);
|
|
||||||
throw new StatAlgoImporterServiceException("Error retrieving file from storage: " + e.getLocalizedMessage(),
|
|
||||||
e);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -738,6 +732,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
|
*
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
@ -766,7 +762,7 @@ public class FilesStorage {
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// convert String into InputStream
|
|
||||||
ws.updateItem(itemId, is);
|
ws.updateItem(itemId, is);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
|
@ -788,21 +784,26 @@ public class FilesStorage {
|
||||||
* Exception
|
* Exception
|
||||||
*/
|
*/
|
||||||
public File zipFolder(String user, String folderId) throws StatAlgoImporterServiceException {
|
public File zipFolder(String user, String folderId) throws StatAlgoImporterServiceException {
|
||||||
Workspace ws;
|
|
||||||
try {
|
try {
|
||||||
logger.info("Zip folder: [user=" + user + ", folderId=" + folderId + "]");
|
logger.info("Zip folder: [user=" + user + ", folderId=" + folderId + "]");
|
||||||
|
|
||||||
ws = HomeLibrary.getUserWorkspace(user);
|
StorageHubClient shc = new StorageHubClient();
|
||||||
|
|
||||||
|
StreamDescriptor streamDescr = shc.open(folderId).asFolder().download();
|
||||||
|
|
||||||
|
File fileZip = Files.createTempFile(streamDescr.getFileName(), "").toFile();
|
||||||
|
logger.debug("File zip: " + fileZip.getAbsolutePath());
|
||||||
|
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(fileZip)) {
|
||||||
|
InputStream is = streamDescr.getStream();
|
||||||
|
byte[] buf = new byte[2048];
|
||||||
|
int read = -1;
|
||||||
|
while ((read = is.read(buf)) != -1) {
|
||||||
|
fos.write(buf, 0, read);
|
||||||
|
}
|
||||||
|
|
||||||
WorkspaceItem workSpaceItem = ws.getItem(folderId);
|
|
||||||
if (!workSpaceItem.isFolder()) {
|
|
||||||
throw new StatAlgoImporterServiceException("Item is not valid folder!");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
WorkspaceFolder folder = (WorkspaceFolder) workSpaceItem;
|
|
||||||
|
|
||||||
File fileZip = ZipUtil.zipFolder(folder);
|
|
||||||
|
|
||||||
return fileZip;
|
return fileZip;
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
|
@ -826,21 +827,25 @@ public class FilesStorage {
|
||||||
*/
|
*/
|
||||||
public File zipFolder(String user, String folderId, List<String> idsToExclude)
|
public File zipFolder(String user, String folderId, List<String> idsToExclude)
|
||||||
throws StatAlgoImporterServiceException {
|
throws StatAlgoImporterServiceException {
|
||||||
Workspace ws;
|
|
||||||
try {
|
try {
|
||||||
logger.info("Zip folder with exclude: [user=" + user + ", folderId=" + folderId + "]");
|
logger.info("Zip folder with exclude: [user=" + user + ", folderId=" + folderId + "]");
|
||||||
|
String[] idsArray = new String[idsToExclude.size()];
|
||||||
|
idsArray = idsToExclude.toArray(idsArray);
|
||||||
|
|
||||||
ws = HomeLibrary.getUserWorkspace(user);
|
StorageHubClient shc = new StorageHubClient();
|
||||||
|
StreamDescriptor streamDescr = shc.open(folderId).asFolder().download(idsArray);
|
||||||
|
|
||||||
WorkspaceItem workSpaceItem = ws.getItem(folderId);
|
File fileZip = Files.createTempFile(streamDescr.getFileName(), "").toFile();
|
||||||
if (!workSpaceItem.isFolder()) {
|
logger.debug("File zip: " + fileZip.getAbsolutePath());
|
||||||
throw new StatAlgoImporterServiceException("Item is not valid folder!");
|
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(fileZip); InputStream is = streamDescr.getStream()) {
|
||||||
|
byte[] buf = new byte[1024];
|
||||||
|
int read = -1;
|
||||||
|
while ((read = is.read(buf)) != -1) {
|
||||||
|
fos.write(buf, 0, read);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
WorkspaceFolder folder = (WorkspaceFolder) workSpaceItem;
|
|
||||||
|
|
||||||
File fileZip = ZipUtil.zipFolder(folder, false, idsToExclude);
|
|
||||||
|
|
||||||
return fileZip;
|
return fileZip;
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
|
@ -866,9 +871,14 @@ public class FilesStorage {
|
||||||
|
|
||||||
URL smpFile = new URL(fileUrl);
|
URL smpFile = new URL(fileUrl);
|
||||||
URLConnection uc = (URLConnection) smpFile.openConnection();
|
URLConnection uc = (URLConnection) smpFile.openConnection();
|
||||||
InputStream is = uc.getInputStream();
|
|
||||||
inputStreamToFile(is, destination);
|
try (FileOutputStream out = new FileOutputStream(destination.toFile());
|
||||||
is.close();
|
InputStream is = uc.getInputStream();) {
|
||||||
|
byte buf[] = new byte[1024];
|
||||||
|
int len = 0;
|
||||||
|
while ((len = is.read(buf)) > 0)
|
||||||
|
out.write(buf, 0, len);
|
||||||
|
}
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
logger.error("Download input file: " + e.getLocalizedMessage(), e);
|
logger.error("Download input file: " + e.getLocalizedMessage(), e);
|
||||||
|
@ -876,16 +886,8 @@ public class FilesStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void inputStreamToFile(InputStream is, Path destination) throws FileNotFoundException, IOException {
|
|
||||||
FileOutputStream out = new FileOutputStream(destination.toFile());
|
|
||||||
byte buf[] = new byte[1024];
|
|
||||||
int len = 0;
|
|
||||||
while ((len = is.read(buf)) > 0)
|
|
||||||
out.write(buf, 0, len);
|
|
||||||
out.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* TODO
|
||||||
*
|
*
|
||||||
* @param user
|
* @param user
|
||||||
* User
|
* User
|
||||||
|
|
Loading…
Reference in New Issue