From 64c9a8060b4262da0c4c30e2627a01761ee62a17 Mon Sep 17 00:00:00 2001 From: Costantino Perciante Date: Sun, 27 Nov 2016 11:33:41 +0000 Subject: [PATCH] added code to copy files also under the .catalogue area of the shared folder.. needs to be tested git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@134921 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 8 + .../grsf_publish_ws/utils/HelperMethods.java | 68 +----- .../utils/threads/ManageTimeSeriesThread.java | 220 +++++++++++++++++- 3 files changed, 223 insertions(+), 73 deletions(-) diff --git a/pom.xml b/pom.xml index 8dc65e4..514f520 100644 --- a/pom.xml +++ b/pom.xml @@ -83,6 +83,14 @@ + + org.gcube.common + home-library-jcr + + + org.gcube.common + home-library + org.gcube.common common-authorization diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java index c79ecd5..ee31472 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java @@ -1,14 +1,10 @@ package org.gcube.data_catalogue.grsf_publish_ws.utils; -import java.beans.IntrospectionException; import java.beans.PropertyDescriptor; import java.io.BufferedReader; -import java.io.File; import java.io.InputStreamReader; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,6 +36,7 @@ import eu.trentorise.opendata.jackan.model.CkanLicense; * Helper methods * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ +@SuppressWarnings({"unchecked","rawtypes"}) public abstract class HelperMethods { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HelperMethods.class); @@ -47,9 +44,9 @@ public abstract class HelperMethods { // to be retrieved from the web.xml private static final String PENDING_CONTEX_KEY = "PendingContext"; private static final String CONFIRMED_CONTEX_KEY = "ConfirmedContext"; - private static final String CSV_FILE_FORMAT = ".csv"; private static final int TIME_SERIES_TAKE_LAST_VALUES = 5; + /** * Convert a group name to its id on ckan * @param origName @@ -427,65 +424,4 @@ public abstract class HelperMethods { logger.info("Returning resources " + toReturn); return toReturn; } - - /** - * Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on). - * The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder. - * @param record - * @throws IntrospectionException - * @throws InvocationTargetException - * @throws IllegalArgumentException - * @throws IllegalAccessException - */ - public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException{ - - if(record == null) - throw new IllegalArgumentException("The given record is null!!"); - - Class current = record.getClass(); - do{ - Field[] fields = current.getDeclaredFields(); - for (Field field : fields) { - if (Collection.class.isAssignableFrom(field.getType())) { - // if the list is not null, get an element - Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - List asList = (List)f; - - if(!asList.isEmpty()){ - if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ - - CustomField customAnnotation = field.getAnnotation(CustomField.class); - String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT; - - logger.debug("A time series has been just found"); - File csvFile = CSVHelpers.listToCSV(asList); - - if(csvFile != null){ - - // upload this file on ckan - eu.trentorise.opendata.jackan.model.CkanResource res = catalogue.uploadResourceFile( - csvFile, - packageName, - catalogue.getApiKeyFromUsername(username), - resourceToAttachName, - customAnnotation.key() + " time series for this product"); - - logger.debug("The resource returned is " + res.getName() + " with package id " + res.getPackageId() + " " + res.getDescription() + " " + res.getState()); - - // upload this file on the folder of the vre (under .catalogue) and change the url of the resource TODO - // if(res != null){ - // - // - // } - } - } - } - } - } - } - } - while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class - - } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index 6e95388..4595575 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -1,13 +1,38 @@ package org.gcube.data_catalogue.grsf_publish_ws.utils.threads; import java.beans.IntrospectionException; +import java.beans.PropertyDescriptor; +import java.io.File; +import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.List; +import org.gcube.common.homelibrary.home.HomeLibrary; +import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; +import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; +import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; +import org.gcube.common.homelibrary.home.workspace.Workspace; +import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; +import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; +import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder; +import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue; +import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; +import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; +import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; +import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; +import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; -import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean; +import org.gcube.data_catalogue.grsf_publish_ws.utils.CSVHelpers; import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue; import org.slf4j.LoggerFactory; +import eu.trentorise.opendata.jackan.model.CkanResourceBase; + /** * Extract the time series present in the record, load them as resource on ckan and on the .catalogue * folder under the vre folder. @@ -15,6 +40,10 @@ import org.slf4j.LoggerFactory; */ public class ManageTimeSeriesThread extends Thread{ + private static final String CSV_FILE_FORMAT = ".csv"; + private static final String PATH_SEPARATOR = "/"; + private static final String CSV_MIME = "text/csv"; + // Logger private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ManageTimeSeriesThread.class); @@ -41,18 +70,195 @@ public class ManageTimeSeriesThread extends Thread{ this.context = context; } - - @Override public void run() { logger.info("Time series manager thread started"); + try { - HelperMethods.manageTimeSeries(record, packageName, username, catalogue, context); + manageTimeSeries(record, packageName, username, catalogue, context); logger.info("The time series manager thread ended correctly"); - } catch (IllegalAccessException | IllegalArgumentException - | InvocationTargetException | IntrospectionException e) { - logger.error("Failed to attach csv files to the product..."); + return; + } catch (IllegalAccessException e) { + logger.error("Error was " + e.getMessage()); + } catch (IllegalArgumentException e) { + logger.error("Error was " + e.getMessage()); + } catch (InvocationTargetException e) { + logger.error("Error was " + e.getMessage()); + } catch (WorkspaceFolderNotFoundException e) { + logger.error("Error was " + e.getMessage()); + } catch (ItemNotFoundException e) { + logger.error("Error was " + e.getMessage()); + } catch (IntrospectionException e) { + logger.error("Error was " + e.getMessage()); + } catch (InternalErrorException e) { + logger.error("Error was " + e.getMessage()); + } catch (HomeNotFoundException e) { + logger.error("Error was " + e.getMessage()); + } catch (UserNotFoundException e) { + logger.error("Error was " + e.getMessage()); + } + + logger.error("Failed to attach csv files to the product..."); + } + + /** + * Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on). + * The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder. + * @param record + * @throws IntrospectionException + * @throws InvocationTargetException + * @throws IllegalArgumentException + * @throws IllegalAccessException + * @throws UserNotFoundException + * @throws HomeNotFoundException + * @throws InternalErrorException + * @throws WorkspaceFolderNotFoundException + * @throws ItemNotFoundException + */ + public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{ + + if(record == null) + throw new IllegalArgumentException("The given record is null!!"); + + Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(username).getWorkspace(); + + // Get a VRE folder by scope + WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context); + + //Get the VRE Folder catalogue + WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue(); + + // the structure under the .catalogue will be as follows: + // .catalogue: + // - stock: + // - first_letter_of_the_product + // - product_name + // - type of files (e.g., csv) + // -files + // - fishery + // - first_letter_of_the_product + // - product_name + // - type of files (e.g., csv) + // -files + String recordTypeFolderName = record.getProductType().toLowerCase(); + WorkspaceItem recordFolder = getFolderOrCreate(catalogueFolder, recordTypeFolderName, "The folder related to " + recordTypeFolderName + " products"); + String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName(); + char firstLetter = productName.charAt(0); + String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName; + WorkspaceItem folderProduct = getFolderOrCreate(catalogueFolder, folderPath, "The folder for the product of type " + recordTypeFolderName + " and name " + productName); + String resourceFormatPath = folderProduct.getPath() + PATH_SEPARATOR + "csv"; + WorkspaceItem resourceFormatFolder = getFolderOrCreate(catalogueFolder, resourceFormatPath, "The folder for resources of type csv"); + + if(folderProduct != null){ + logger.info("Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + ", exists"); + + String apiKeyUser = catalogue.getApiKeyFromUsername(username); + + Class current = record.getClass(); + do{ + Field[] fields = current.getDeclaredFields(); + for (Field field : fields) { + if (Collection.class.isAssignableFrom(field.getType())) { + + Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); + if(f != null){ + + List asList = (List)f; + + if(!asList.isEmpty()){ + if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ + + CustomField customAnnotation = field.getAnnotation(CustomField.class); + String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT; + + logger.debug("A time series has been just found"); + File csvFile = CSVHelpers.listToCSV(asList); + + if(csvFile != null){ + + // upload this file on ckan + CkanResourceBase ckanResource = uploadFileOnCkan(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product"); + + //upload this file on the folder of the vre (under .catalogue) and change the url of the resource + if(ckanResource != null){ + + ExternalFile createdFileOnWorkspace = uploadExternalFile(resourceFormatFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile); + if(createdFileOnWorkspace != null){ + + String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); + logger.info("going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan); + boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachName, customAnnotation.key() + " time series for this product", "", apiKeyUser); + + if(updated) + logger.info("Resource has been updated with the new url"); + + } + } + } + } + } + } + } + } + } + while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class } } + /** + * Upload a file in the shared folder + * @param resourceFormatFolder + * @param resourceToAttachName + * @param description + * @param csvFile + * @return + */ + private static ExternalFile uploadExternalFile(WorkspaceItem resourceFormatFolder, String resourceToAttachName, String description, File csvFile) { + try { + return ((WorkspaceFolder)resourceFormatFolder).createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile); + } catch (InsufficientPrivilegesException | ItemAlreadyExistException + | InternalErrorException e) { + logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e); + } + return null; + } + + /** + * Upload a resource on ckan + * @param csvFile + * @param packageName + * @param catalogue + * @param username + * @param resourceToAttachName + * @param description + * @return a ckan resource on success, null otherwise + */ + private static CkanResourceBase uploadFileOnCkan(File csvFile, + String packageName, DataCatalogue catalogue, String username, + String resourceToAttachName, String description) { + return catalogue.uploadResourceFile( + csvFile, + packageName, + catalogue.getApiKeyFromUsername(username), + resourceToAttachName, + description); + } + + /** + * Get a folder within the catalogue folder or create it if it doesn't exist. + * @return + */ + private static WorkspaceItem getFolderOrCreate(WorkspaceCatalogue catalogueFolder, String relativePath, String descriptionFolder){ + WorkspaceItem folder = null; + try { + folder = catalogueFolder.getCatalogueItemByPath(relativePath); + if(folder == null){ + catalogueFolder.createFolder(relativePath, descriptionFolder); + } + } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e) { + logger.error("Failed to get or generate this folder", e); + } + return folder; + } + }