From f69d3261452b84395f4af210dd0c39d25d8d4f47 Mon Sep 17 00:00:00 2001 From: Costantino Perciante Date: Mon, 28 Nov 2016 11:20:43 +0000 Subject: [PATCH] minor fixes for files creation in shared folder git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@134945 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 2 + .../services/GrsfPublisherFisheryService.java | 2 +- .../services/GrsfPublisherStockService.java | 2 +- .../utils/threads/ManageTimeSeriesThread.java | 122 +++++++++++---- .../grsf_publish_ws/JTests.java | 148 ++++++++++++++++-- 5 files changed, 235 insertions(+), 41 deletions(-) diff --git a/pom.xml b/pom.xml index 514f520..fea36ff 100644 --- a/pom.xml +++ b/pom.xml @@ -86,10 +86,12 @@ org.gcube.common home-library-jcr + provided org.gcube.common home-library + provided org.gcube.common diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java index a13621f..4f4898c 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java @@ -249,7 +249,7 @@ public class GrsfPublisherFisheryService { // manage time series logger.info("Launching thread for time series handling"); - new ManageTimeSeriesThread(record, futureName, username, catalogue, ScopeProvider.instance.get()).start(); + new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start(); } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java index f979a9c..65a95f4 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java @@ -238,7 +238,7 @@ public class GrsfPublisherStockService { // manage time series logger.info("Launching thread for time series handling"); - new ManageTimeSeriesThread(record, futureName, username, catalogue, ScopeProvider.instance.get()).start(); + new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start(); } } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index 4595575..4861154 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -8,6 +8,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.Collection; import java.util.List; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; @@ -22,6 +23,7 @@ import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistEx import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; @@ -52,6 +54,7 @@ public class ManageTimeSeriesThread extends Thread{ private String username; private DataCatalogue catalogue; private String context; + private String token; /** * @param record @@ -61,21 +64,25 @@ public class ManageTimeSeriesThread extends Thread{ * @param context */ public ManageTimeSeriesThread(Common record, String packageName, - String username, DataCatalogue catalogue, String context) { + String username, DataCatalogue catalogue, String context, String token) { super(); this.record = record; this.packageName = packageName; this.username = username; this.catalogue = catalogue; this.context = context; + this.token = token; } @Override public void run() { logger.info("Time series manager thread started"); + ScopeProvider.instance.set(context); + SecurityTokenProvider.instance.set(token); + try { - manageTimeSeries(record, packageName, username, catalogue, context); + manageTimeSeries(record, packageName, username, catalogue); logger.info("The time series manager thread ended correctly"); return; } catch (IllegalAccessException e) { @@ -96,6 +103,9 @@ public class ManageTimeSeriesThread extends Thread{ logger.error("Error was " + e.getMessage()); } catch (UserNotFoundException e) { logger.error("Error was " + e.getMessage()); + }finally{ + ScopeProvider.instance.reset(); + SecurityTokenProvider.instance.reset(); } logger.error("Failed to attach csv files to the product..."); @@ -115,42 +125,44 @@ public class ManageTimeSeriesThread extends Thread{ * @throws WorkspaceFolderNotFoundException * @throws ItemNotFoundException */ - public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{ + public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{ if(record == null) throw new IllegalArgumentException("The given record is null!!"); - Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(username).getWorkspace(); + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); // Get a VRE folder by scope - WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context); + WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(ScopeProvider.instance.get()); //Get the VRE Folder catalogue WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue(); + logger.debug("Catalogue folder in vre has path " + catalogueFolder.getPath()); + // the structure under the .catalogue will be as follows: // .catalogue: // - stock: // - first_letter_of_the_product // - product_name // - type of files (e.g., csv) - // -files + // -files (csv) // - fishery // - first_letter_of_the_product // - product_name // - type of files (e.g., csv) - // -files + // -files (csv) + String recordTypeFolderName = record.getProductType().toLowerCase(); - WorkspaceItem recordFolder = getFolderOrCreate(catalogueFolder, recordTypeFolderName, "The folder related to " + recordTypeFolderName + " products"); String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName(); char firstLetter = productName.charAt(0); - String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName; - WorkspaceItem folderProduct = getFolderOrCreate(catalogueFolder, folderPath, "The folder for the product of type " + recordTypeFolderName + " and name " + productName); - String resourceFormatPath = folderProduct.getPath() + PATH_SEPARATOR + "csv"; - WorkspaceItem resourceFormatFolder = getFolderOrCreate(catalogueFolder, resourceFormatPath, "The folder for resources of type csv"); - if(folderProduct != null){ - logger.info("Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + ", exists"); + // the whole path of the directory is going to be... + String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName + PATH_SEPARATOR + "csv"; + logger.debug("The path under which the time series are going to be saved is " + csvDirectoryForThisProduct); + WorkspaceFolder csvFolder = createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct); + + if(csvFolder != null){ String apiKeyUser = catalogue.getApiKeyFromUsername(username); @@ -171,18 +183,18 @@ public class ManageTimeSeriesThread extends Thread{ CustomField customAnnotation = field.getAnnotation(CustomField.class); String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT; - logger.debug("A time series has been just found"); + logger.debug("A time series has been just found (from field " + customAnnotation.key() + ")"); File csvFile = CSVHelpers.listToCSV(asList); if(csvFile != null){ // upload this file on ckan - CkanResourceBase ckanResource = uploadFileOnCkan(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product"); + CkanResourceBase ckanResource = uploadFileOnCatalogue(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product", apiKeyUser); //upload this file on the folder of the vre (under .catalogue) and change the url of the resource if(ckanResource != null){ - ExternalFile createdFileOnWorkspace = uploadExternalFile(resourceFormatFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile); + ExternalFile createdFileOnWorkspace = uploadExternalFile(csvFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile); if(createdFileOnWorkspace != null){ String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); @@ -213,9 +225,9 @@ public class ManageTimeSeriesThread extends Thread{ * @param csvFile * @return */ - private static ExternalFile uploadExternalFile(WorkspaceItem resourceFormatFolder, String resourceToAttachName, String description, File csvFile) { + private static ExternalFile uploadExternalFile(WorkspaceFolder resourceFormatFolder, String resourceToAttachName, String description, File csvFile) { try { - return ((WorkspaceFolder)resourceFormatFolder).createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile); + return resourceFormatFolder.createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile); } catch (InsufficientPrivilegesException | ItemAlreadyExistException | InternalErrorException e) { logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e); @@ -233,32 +245,82 @@ public class ManageTimeSeriesThread extends Thread{ * @param description * @return a ckan resource on success, null otherwise */ - private static CkanResourceBase uploadFileOnCkan(File csvFile, + private static CkanResourceBase uploadFileOnCatalogue(File csvFile, String packageName, DataCatalogue catalogue, String username, - String resourceToAttachName, String description) { + String resourceToAttachName, String description, String apiKey) { return catalogue.uploadResourceFile( csvFile, packageName, - catalogue.getApiKeyFromUsername(username), + apiKey, resourceToAttachName, description); } + /** + * Create subfolders in cascade, returning the last created ones + * It could be also used for getting them if they already exists + * @param folder + * @param subPath + * @return null if an error occurred + */ + private static WorkspaceFolder createOrGetSubFoldersByPath(WorkspaceFolder folder, String subPath){ + + WorkspaceFolder parentFolder = folder; + if(folder == null) + throw new IllegalArgumentException("Root folder is null!"); + + if(subPath == null || subPath.isEmpty()) + throw new IllegalArgumentException("subPath is null/empty!"); + + try{ + if(subPath.startsWith(PATH_SEPARATOR)) + subPath = subPath.replaceFirst(PATH_SEPARATOR, ""); + + if(subPath.endsWith(subPath)) + subPath = subPath.substring(0, subPath.length() - 1); + + logger.debug("Splitting path " + subPath); + + String[] splittedPaths = subPath.split(PATH_SEPARATOR); + + for (String path : splittedPaths) { + WorkspaceFolder createdFolder = getFolderOrCreate(parentFolder, path, ""); + logger.debug("Created subfolder with path " + createdFolder.getPath()); + parentFolder = createdFolder; + } + + }catch(Exception e){ + logger.error("Failed to create the subfolders by path " + subPath); + return null; + } + + return parentFolder; + } + /** * Get a folder within the catalogue folder or create it if it doesn't exist. * @return */ - private static WorkspaceItem getFolderOrCreate(WorkspaceCatalogue catalogueFolder, String relativePath, String descriptionFolder){ - WorkspaceItem folder = null; + private static WorkspaceFolder getFolderOrCreate(WorkspaceFolder folder, String relativePath, String descriptionFolder){ + WorkspaceFolder result = null; try { - folder = catalogueFolder.getCatalogueItemByPath(relativePath); - if(folder == null){ - catalogueFolder.createFolder(relativePath, descriptionFolder); + WorkspaceItem foundFolder = folder.find(relativePath); + if(foundFolder != null && foundFolder.isFolder()) + result = (WorkspaceFolder)foundFolder; + + if(result != null) + logger.debug("Folder found with name " + result.getName() + ", it has id " + result.getId()); + else + throw new Exception("There is no folder with name " + relativePath + " under folder " + folder.getName()); + } catch (Exception e) { + logger.debug("Probably the folder doesn't exist", e); + try{ + result = folder.createFolder(relativePath, descriptionFolder); + } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e2) { + logger.error("Failed to get or generate this folder", e2); } - } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e) { - logger.error("Failed to get or generate this folder", e); } - return folder; + return result; } } diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index b25b0e5..0c3a0f1 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -16,6 +16,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder; +import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.scope.api.ScopeProvider; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag; @@ -31,7 +42,6 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Type_Fishery; import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.DataCatalogueFactory; -import org.junit.Test; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonProcessingException; @@ -267,7 +277,7 @@ public class JTests { } - @Test + //@Test public void testJSONResource() throws Exception{ DataCatalogueFactory factory = DataCatalogueFactory.getFactory(); DataCatalogue instance = factory.getUtilsPerScope("/gcube/devNext/NextNext"); @@ -355,13 +365,13 @@ public class JTests { if(f != null){ List asList = (List)f; if(!asList.isEmpty()) - if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ - logger.debug("Name is " + field.getName()); - CustomField customAnnotation = field.getAnnotation(CustomField.class); - if(customAnnotation != null) - logger.debug("Name is " + customAnnotation.key()); - - } + if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ + logger.debug("Name is " + field.getName()); + CustomField customAnnotation = field.getAnnotation(CustomField.class); + if(customAnnotation != null) + logger.debug("Name is " + customAnnotation.key()); + + } } } @@ -371,4 +381,124 @@ public class JTests { } + //@Test + public void sharedVREFolderWriteTest() throws Exception{ + + String token = ""; + String context = "/gcube/devNext/NextNext"; + + ScopeProvider.instance.set(context); + SecurityTokenProvider.instance.set(token); + + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); + + // Get a VRE folder by scope + WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context); + + //Get the VRE Folder catalogue + WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue(); + + logger.debug("Catalogue folder retrieved " + catalogueFolder.getName()); + + /** + * Test is + * .catalogue: + * -test + * - a + * -aproductwiththisname + * - csv + * - testfile.csv + */ + + String allSubPath = "/test/a/aproductwiththisname/"; + //WorkspaceFolder lastFolder = createGetSubFoldersByPath(catalogueFolder, allSubPath); + // WorkspaceFolder recordFolder = (WorkspaceFolder)getFolderOrCreate(catalogueFolder, "test", ""); + // String firstLetter = "a"; + // WorkspaceFolder firstLetterFolder = (WorkspaceFolder)getFolderOrCreate(recordFolder, firstLetter, ""); + // String folderPath = "aproductwiththisname"; + // WorkspaceFolder productFolder = (WorkspaceFolder)getFolderOrCreate(firstLetterFolder, folderPath, ""); + //logger.debug("Test folder created/get..its path is " + lastFolder.getPath()); + // String ccsvUnderProductFolderName = productFolderName + "/" + "csv"; + // WorkspaceFolder csvUnderProductFolder = (WorkspaceFolder)getFolderOrCreate(catalogueFolder, ccsvUnderProductFolderName, ""); + // + // logger.debug("FOLDERS created " + csvUnderProductFolder.getPath()); + // treeCheck(catalogueFolder); + + } + + public void treeCheck(WorkspaceFolder rootFolder) throws InternalErrorException{ + List children = rootFolder.getChildren(); + for (WorkspaceItem workspaceItem : children) { + if(workspaceItem.isFolder()){ + logger.debug("children folder is " + workspaceItem.getName()); + treeCheck((WorkspaceFolder)workspaceItem); + } + } + } + + /** + * Create subfolders in cascade, returning the last created ones + * It could be also used for getting them if they already exists + * @param folder + * @param subPath + * @return + */ + private static WorkspaceFolder createGetSubFoldersByPath(WorkspaceFolder folder, String subPath){ + + String pathSeparator = "/"; + WorkspaceFolder parentFolder = folder; + if(folder == null) + throw new IllegalArgumentException("Root folder is null!"); + + if(subPath == null || subPath.isEmpty()) + throw new IllegalArgumentException("subPath is null/empty!"); + + try{ + if(subPath.startsWith(pathSeparator)) + subPath = subPath.replaceFirst(pathSeparator, ""); + + if(subPath.endsWith(subPath)) + subPath = subPath.substring(0, subPath.length() - 1); + + logger.debug("Splitting path " + subPath); + + String[] splittedPaths = subPath.split(pathSeparator); + + for (String path : splittedPaths) { + WorkspaceFolder createdFolder = getFolderOrCreate(parentFolder, path, ""); + logger.debug("Created subfolder with path " + createdFolder.getPath()); + parentFolder = createdFolder; + } + + }catch(Exception e){ + logger.error("Failed to create the subfolders by path " + subPath); + } + + return parentFolder; + } + + /** + * Get a folder within the catalogue folder or create it if it doesn't exist. + * @return + */ + private static WorkspaceFolder getFolderOrCreate(WorkspaceFolder folder, String relativePath, String descriptionFolder){ + WorkspaceFolder result = null; + try { + if(folder.exists(relativePath) && folder.find(relativePath).isFolder()) + result = (WorkspaceFolder) folder.find(relativePath); + if(result != null) + logger.debug("Folder found with name " + result.getName() + ", it has id " + result.getId()); + else + throw new Exception("There is no folder with name " + relativePath + " under foler " + folder.getName()); + } catch (Exception e) { + logger.debug("Probably the folder doesn't exist", e); + try{ + result = folder.createFolder(relativePath, descriptionFolder); + } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e2) { + logger.error("Failed to get or generate this folder", e2); + } + } + return result; + } + }