From 3519fc2888d19a2f9ae63f6e610ed3471d169d6c Mon Sep 17 00:00:00 2001 From: "lucio.lelii" Date: Fri, 10 Sep 2021 15:51:41 +0200 Subject: [PATCH] moved to Storagehub --- .settings/org.eclipse.wst.common.component | 15 +- CHANGELOG.md | 5 +- pom.xml | 21 ++- .../grsf_publish_ws/utils/HelperMethods.java | 76 ++++------ .../utils/csv/ManageTimeSeriesThread.java | 57 ++++---- .../grsf_publish_ws/JTests.java | 134 +----------------- 6 files changed, 79 insertions(+), 229 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index d6f218f..cf9d552 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -1,12 +1,23 @@ - + + + + + - + uses + + uses + + + + + diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ea46ce..9306095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,14 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [v1.12.1-SNAPSHOT] +## [v1.13.0-SNAPSHOT] ### Changed - Switched dependency management to gcube-bom 2.0.0 +- moved to storagehub -## [v1.12.0] [r4.24.0] - 2020-06-19 +## [v1.12.0] - 2020-06-19 ### Added diff --git a/pom.xml b/pom.xml index 44cfa75..35e1497 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.gcube.data-catalogue grsf-publisher-ws - 1.12.1-SNAPSHOT + 1.13.0-SNAPSHOT war grsf-publisher-ws Utility library to publish GRSF products on GRSF catalogue. @@ -52,6 +52,11 @@ grsf-common-library [1-0-0-SNAPSHOT, 2.0.0-SNAPSHOT) + + org.gcube.common + storagehub-client-library + [1.0.0, 2.0.0-SNAPSHOT) + org.jsoup @@ -100,12 +105,7 @@ authorization-client provided - - org.gcube.common - home-library-jcr - [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) - compile - + org.gcube.common common-authorization @@ -157,12 +157,7 @@ common-smartgears provided - - org.gcube.common - home-library - [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) - compile - + org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-jetty diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java index 6ad2ca4..78d03d4 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java @@ -5,7 +5,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.client; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; @@ -20,15 +19,11 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; import org.gcube.common.resources.gcore.utils.XPathHelper; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.storagehub.client.dsl.FileContainer; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory; import org.gcube.datacatalogue.common.caches.CacheImpl; @@ -74,7 +69,6 @@ public abstract class HelperMethods { public static final String MANAGE_CONTEX_KEY = "ManageVRE"; public static final String PREVALIDATE_CONTEX_KEY = "PreValidateVRE"; public static final String PUBLIC_CONTEX_KEY = "PublicVRE"; - private static final String CSV_MIME = "text/csv"; private static final String PATH_SEPARATOR = "/"; // caches @@ -258,29 +252,20 @@ public abstract class HelperMethods { * @param csvFile * @return */ - public static ExternalFile uploadExternalFile(WorkspaceFolder resourceFormatFolder, String resourceToAttachName, String description, File csvFile) { + public static FileContainer uploadExternalFile(FolderContainer resourceFormatFolder, String resourceToAttachName, String description, File csvFile) { try { - - WorkspaceItem existsFile = resourceFormatFolder.find(resourceToAttachName); - - if(existsFile == null) - return resourceFormatFolder.createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile); - else{ - InputStream targetStream = new FileInputStream(csvFile); - existsFile.updateItem(targetStream); - return (ExternalFile)existsFile; - } - - } catch (InsufficientPrivilegesException | ItemAlreadyExistException - | InternalErrorException e) { - logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e); - } catch (FileNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ItemNotFoundException e) { + + try (InputStream is= new FileInputStream(csvFile)) { + FileContainer fileContainer = resourceFormatFolder.uploadFile(is , resourceToAttachName, description); + return fileContainer; + } + + } catch (StorageHubException she) { + logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, she); + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); - } + } return null; } @@ -291,9 +276,10 @@ public abstract class HelperMethods { * @param subPath * @return null if an error occurred */ - public static WorkspaceFolder createOrGetSubFoldersByPath(WorkspaceFolder folder, String subPath){ - - WorkspaceFolder parentFolder = folder; + public static FolderContainer createOrGetSubFoldersByPath(FolderContainer folder, String subPath){ + + FolderContainer parentFolder = folder; + if(folder == null) throw new IllegalArgumentException("Root folder is null!"); @@ -312,8 +298,9 @@ public abstract class HelperMethods { String[] splittedPaths = subPath.split(PATH_SEPARATOR); for (String path : splittedPaths) { - WorkspaceFolder createdFolder = getFolderOrCreate(parentFolder, path, ""); - logger.debug("Created subfolder with path " + createdFolder.getPath()); + + FolderContainer createdFolder = getFolderOrCreate(parentFolder, path, ""); + logger.debug("Created subfolder with path " + createdFolder.get().getPath()); parentFolder = createdFolder; } @@ -329,23 +316,18 @@ public abstract class HelperMethods { * Get a folder within the catalogue folder or create it if it doesn't exist. * @return */ - public static WorkspaceFolder getFolderOrCreate(WorkspaceFolder folder, String relativePath, String descriptionFolder){ - WorkspaceFolder result = null; + public static FolderContainer getFolderOrCreate(FolderContainer folder, String relativePath, String descriptionFolder){ + + FolderContainer result = null; try { - WorkspaceItem foundFolder = folder.find(relativePath); - if(foundFolder != null && foundFolder.isFolder()) - result = (WorkspaceFolder)foundFolder; - - if(result != null) - logger.debug("Folder found with name " + result.getName() + ", it has id " + result.getId()); - else - throw new Exception("There is no folder with name " + relativePath + " under folder " + folder.getName()); + result = folder.openByRelativePath(relativePath).asFolder(); + } catch (Exception e) { logger.debug("Probably the folder doesn't exist"); try{ - result = folder.createFolder(relativePath, descriptionFolder); - } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e2) { - logger.error("Failed to get or generate this folder", e2); + result = folder.newFolder(relativePath, descriptionFolder); + } catch (StorageHubException se) { + logger.error("Failed to get or generate this folder", se); } } return result; diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/csv/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/csv/ManageTimeSeriesThread.java index 5d70937..b992ead 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/csv/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/csv/ManageTimeSeriesThread.java @@ -1,6 +1,5 @@ package org.gcube.data_catalogue.grsf_publish_ws.utils.csv; -import java.beans.IntrospectionException; import java.beans.PropertyDescriptor; import java.io.File; import java.lang.reflect.Field; @@ -8,18 +7,11 @@ import java.lang.reflect.InvocationTargetException; import java.util.List; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder; -import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; -import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.storagehub.client.dsl.FileContainer; +import org.gcube.common.storagehub.client.dsl.FolderContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; +import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.TimeSeries; import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common; @@ -42,13 +34,15 @@ public class ManageTimeSeriesThread extends Thread{ private static final String PATH_SEPARATOR = "/"; + private static final String CATALOGUE_FOLDER = ".catalogue"; + // Logger private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ManageTimeSeriesThread.class); // try to attach the source at most CHANCES times .. private static final int CHANCES = 10; - private static CacheInterface vreFolderCache = new CacheImpl(1000 * 60 * 60 * 24); + private static CacheInterface vreFolderCache = new CacheImpl(1000 * 60 * 60 * 24); private static final int MAX_NAME_CSV_FILE_WITHOUT_MEASURE = 50; @@ -96,17 +90,7 @@ public class ManageTimeSeriesThread extends Thread{ logger.error("Error was " + e.getMessage()); } catch (InvocationTargetException e) { logger.error("Error was " + e.getMessage()); - } catch (WorkspaceFolderNotFoundException e) { - logger.error("Error was " + e.getMessage()); - } catch (ItemNotFoundException e) { - logger.error("Error was " + e.getMessage()); - } catch (IntrospectionException e) { - logger.error("Error was " + e.getMessage()); - } catch (InternalErrorException e) { - logger.error("Error was " + e.getMessage()); - } catch (HomeNotFoundException e) { - logger.error("Error was " + e.getMessage()); - } catch (UserNotFoundException e) { + } catch (StorageHubException e) { logger.error("Error was " + e.getMessage()); } catch (Exception e) { logger.error("Error was " + e.getMessage()); @@ -130,17 +114,26 @@ public class ManageTimeSeriesThread extends Thread{ if(record == null) throw new IllegalArgumentException("The given record is null!!"); + StorageHubClient shClient = new StorageHubClient(); + String token = SecurityTokenProvider.instance.get(); - WorkspaceCatalogue catalogueFolder = null; + FolderContainer catalogueFolder = null; + + //WorkspaceCatalogue catalogueFolder = null; if((catalogueFolder = vreFolderCache.get(token)) == null){ - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(ScopeProvider.instance.get()); - catalogueFolder = vreFolder.getVRECatalogue(); + //Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + FolderContainer vreFolder = shClient.openVREFolder(); + try { + catalogueFolder = vreFolder.openByRelativePath(CATALOGUE_FOLDER).asFolder(); + }catch (StorageHubException e) { + catalogueFolder = vreFolder.newHiddenFolder(CATALOGUE_FOLDER, "catalogue folder"); + } + vreFolderCache.insert(token, catalogueFolder); } - logger.debug("Catalogue folder in vre has path " + catalogueFolder.getPath()); + logger.debug("Catalogue folder in vre has path " + catalogueFolder.get().getPath()); // the structure under the .catalogue will be as follows: // .catalogue: @@ -167,7 +160,7 @@ public class ManageTimeSeriesThread extends Thread{ // the whole path of the directory is going to be... String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB, "_") + PATH_SEPARATOR + CSVUtils.CSV_EXTENSION.replace(".", ""); logger.debug("The path under which the time series are going to be saved is " + csvDirectoryForThisProduct); - WorkspaceFolder csvFolder = HelperMethods.createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct); + FolderContainer csvFolder = HelperMethods.createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct); if(csvFolder == null) logger.error("Failed to create directory where csv files will be deployed in the workspace!!"); @@ -192,7 +185,7 @@ public class ManageTimeSeriesThread extends Thread{ String resourceToAttachOnCkanDescription = productName; CkanResourceBase ckanResource = null; - ExternalFile createdFileOnWorkspace = null; + FileContainer createdFileOnWorkspace = null; String[] relevantSources = new String[1]; File csvFile = CSVUtils.listToCSV(asList, relevantSources); if(csvFile != null){ @@ -212,7 +205,7 @@ public class ManageTimeSeriesThread extends Thread{ + customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); if(createdFileOnWorkspace != null){ - String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); + String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink().toString(); // wait for patching.. Thread.sleep(1500); diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index 00d21ca..4dd5011 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -13,17 +13,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.homelibrary.home.HomeLibrary; -import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.workspace.Workspace; -import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; -import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; -import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder; -import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.scope.api.ScopeProvider; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag; @@ -238,129 +227,8 @@ public class JTests { instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null); } - // @Test - public void sharedVREFolderWriteTest() throws Exception{ - - String token = ""; - String context = "/gcube/devNext/NextNext"; - - ScopeProvider.instance.set(context); - SecurityTokenProvider.instance.set(token); - - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - - // Get a VRE folder by scope - WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context); - - //Get the VRE Folder catalogue - WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue(); - - logger.debug("Catalogue folder retrieved " + catalogueFolder.getName()); - - // WorkspaceItem stockFolder = catalogueFolder.find("stock"); - - // vreFolder.removeChild(stockFolder); - - /** - * Test is - * .catalogue: - * -test - * - a - * -aproductwiththisname - * - csv - * - testfile.csv - */ - - String allSubPath = "/test/a/aproductwiththisname/"; - //WorkspaceFolder lastFolder = createGetSubFoldersByPath(catalogueFolder, allSubPath); - // WorkspaceFolder recordFolder = (WorkspaceFolder)getFolderOrCreate(catalogueFolder, "test", ""); - // String firstLetter = "a"; - // WorkspaceFolder firstLetterFolder = (WorkspaceFolder)getFolderOrCreate(recordFolder, firstLetter, ""); - // String folderPath = "aproductwiththisname"; - // WorkspaceFolder productFolder = (WorkspaceFolder)getFolderOrCreate(firstLetterFolder, folderPath, ""); - //logger.debug("Test folder created/get..its path is " + lastFolder.getPath()); - // String ccsvUnderProductFolderName = productFolderName + "/" + "csv"; - // WorkspaceFolder csvUnderProductFolder = (WorkspaceFolder)getFolderOrCreate(catalogueFolder, ccsvUnderProductFolderName, ""); - // - // logger.debug("FOLDERS created " + csvUnderProductFolder.getPath()); - // treeCheck(catalogueFolder); - - } - - public void treeCheck(WorkspaceFolder rootFolder) throws InternalErrorException{ - List children = rootFolder.getChildren(); - for (WorkspaceItem workspaceItem : children) { - if(workspaceItem.isFolder()){ - logger.debug("children folder is " + workspaceItem.getName()); - treeCheck((WorkspaceFolder)workspaceItem); - } - } - } - - /** - * Create subfolders in cascade, returning the last created ones - * It could be also used for getting them if they already exists - * @param folder - * @param subPath - * @return - */ - private static WorkspaceFolder createGetSubFoldersByPath(WorkspaceFolder folder, String subPath){ - - String pathSeparator = "/"; - WorkspaceFolder parentFolder = folder; - if(folder == null) - throw new IllegalArgumentException("Root folder is null!"); - - if(subPath == null || subPath.isEmpty()) - throw new IllegalArgumentException("subPath is null/empty!"); - - try{ - if(subPath.startsWith(pathSeparator)) - subPath = subPath.replaceFirst(pathSeparator, ""); - - if(subPath.endsWith(subPath)) - subPath = subPath.substring(0, subPath.length() - 1); - - logger.debug("Splitting path " + subPath); - - String[] splittedPaths = subPath.split(pathSeparator); - - for (String path : splittedPaths) { - WorkspaceFolder createdFolder = getFolderOrCreate(parentFolder, path, ""); - logger.debug("Created subfolder with path " + createdFolder.getPath()); - parentFolder = createdFolder; - } - - }catch(Exception e){ - logger.error("Failed to create the subfolders by path " + subPath); - } - - return parentFolder; - } + - /** - * Get a folder within the catalogue folder or create it if it doesn't exist. - * @return - */ - private static WorkspaceFolder getFolderOrCreate(WorkspaceFolder folder, String relativePath, String descriptionFolder){ - WorkspaceFolder result = null; - try { - if(folder.exists(relativePath) && folder.find(relativePath).isFolder()) - result = (WorkspaceFolder) folder.find(relativePath); - if(result != null) - logger.debug("Folder found with name " + result.getName() + ", it has id " + result.getId()); - else - throw new Exception("There is no folder with name " + relativePath + " under foler " + folder.getName()); - } catch (Exception e) { - logger.debug("Probably the folder doesn't exist", e); - try{ - result = folder.createFolder(relativePath, descriptionFolder); - } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e2) { - logger.error("Failed to get or generate this folder", e2); - } - } - return result; - } //@Test public void GRSFServiceUrl() throws Exception{