package org.gcube.data_catalogue.grsf_publish_ws.utils.threads; import java.beans.IntrospectionException; import java.beans.PropertyDescriptor; import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.util.Collection; import java.util.List; import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; import org.gcube.common.homelibrary.home.workspace.Workspace; import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder; import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder; import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue; import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean; import org.gcube.data_catalogue.grsf_publish_ws.utils.CSVHelpers; import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.model.CkanResourceBase; /** * Extract the time series present in the record, load them as resource on ckan and on the .catalogue * folder under the vre folder. * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ public class ManageTimeSeriesThread extends Thread{ private static final String CSV_FILE_FORMAT = ".csv"; private static final String PATH_SEPARATOR = "/"; private static final String CSV_MIME = "text/csv"; // Logger private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ManageTimeSeriesThread.class); private Common record; private String packageName; private String username; private DataCatalogue catalogue; private String context; /** * @param record * @param packageId * @param username * @param catalogue * @param context */ public ManageTimeSeriesThread(Common record, String packageName, String username, DataCatalogue catalogue, String context) { super(); this.record = record; this.packageName = packageName; this.username = username; this.catalogue = catalogue; this.context = context; } @Override public void run() { logger.info("Time series manager thread started"); try { manageTimeSeries(record, packageName, username, catalogue, context); logger.info("The time series manager thread ended correctly"); return; } catch (IllegalAccessException e) { logger.error("Error was " + e.getMessage()); } catch (IllegalArgumentException e) { logger.error("Error was " + e.getMessage()); } catch (InvocationTargetException e) { logger.error("Error was " + e.getMessage()); } catch (WorkspaceFolderNotFoundException e) { logger.error("Error was " + e.getMessage()); } catch (ItemNotFoundException e) { logger.error("Error was " + e.getMessage()); } catch (IntrospectionException e) { logger.error("Error was " + e.getMessage()); } catch (InternalErrorException e) { logger.error("Error was " + e.getMessage()); } catch (HomeNotFoundException e) { logger.error("Error was " + e.getMessage()); } catch (UserNotFoundException e) { logger.error("Error was " + e.getMessage()); } logger.error("Failed to attach csv files to the product..."); } /** * Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on). * The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder. * @param record * @throws IntrospectionException * @throws InvocationTargetException * @throws IllegalArgumentException * @throws IllegalAccessException * @throws UserNotFoundException * @throws HomeNotFoundException * @throws InternalErrorException * @throws WorkspaceFolderNotFoundException * @throws ItemNotFoundException */ public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{ if(record == null) throw new IllegalArgumentException("The given record is null!!"); Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(username).getWorkspace(); // Get a VRE folder by scope WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context); //Get the VRE Folder catalogue WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue(); // the structure under the .catalogue will be as follows: // .catalogue: // - stock: // - first_letter_of_the_product // - product_name // - type of files (e.g., csv) // -files // - fishery // - first_letter_of_the_product // - product_name // - type of files (e.g., csv) // -files String recordTypeFolderName = record.getProductType().toLowerCase(); WorkspaceItem recordFolder = getFolderOrCreate(catalogueFolder, recordTypeFolderName, "The folder related to " + recordTypeFolderName + " products"); String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName(); char firstLetter = productName.charAt(0); String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName; WorkspaceItem folderProduct = getFolderOrCreate(catalogueFolder, folderPath, "The folder for the product of type " + recordTypeFolderName + " and name " + productName); String resourceFormatPath = folderProduct.getPath() + PATH_SEPARATOR + "csv"; WorkspaceItem resourceFormatFolder = getFolderOrCreate(catalogueFolder, resourceFormatPath, "The folder for resources of type csv"); if(folderProduct != null){ logger.info("Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + ", exists"); String apiKeyUser = catalogue.getApiKeyFromUsername(username); Class current = record.getClass(); do{ Field[] fields = current.getDeclaredFields(); for (Field field : fields) { if (Collection.class.isAssignableFrom(field.getType())) { Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); if(f != null){ List asList = (List)f; if(!asList.isEmpty()){ if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ CustomField customAnnotation = field.getAnnotation(CustomField.class); String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT; logger.debug("A time series has been just found"); File csvFile = CSVHelpers.listToCSV(asList); if(csvFile != null){ // upload this file on ckan CkanResourceBase ckanResource = uploadFileOnCkan(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product"); //upload this file on the folder of the vre (under .catalogue) and change the url of the resource if(ckanResource != null){ ExternalFile createdFileOnWorkspace = uploadExternalFile(resourceFormatFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile); if(createdFileOnWorkspace != null){ String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); logger.info("going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan); boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachName, customAnnotation.key() + " time series for this product", "", apiKeyUser); if(updated) logger.info("Resource has been updated with the new url"); } } } } } } } } } while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class } } /** * Upload a file in the shared folder * @param resourceFormatFolder * @param resourceToAttachName * @param description * @param csvFile * @return */ private static ExternalFile uploadExternalFile(WorkspaceItem resourceFormatFolder, String resourceToAttachName, String description, File csvFile) { try { return ((WorkspaceFolder)resourceFormatFolder).createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile); } catch (InsufficientPrivilegesException | ItemAlreadyExistException | InternalErrorException e) { logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e); } return null; } /** * Upload a resource on ckan * @param csvFile * @param packageName * @param catalogue * @param username * @param resourceToAttachName * @param description * @return a ckan resource on success, null otherwise */ private static CkanResourceBase uploadFileOnCkan(File csvFile, String packageName, DataCatalogue catalogue, String username, String resourceToAttachName, String description) { return catalogue.uploadResourceFile( csvFile, packageName, catalogue.getApiKeyFromUsername(username), resourceToAttachName, description); } /** * Get a folder within the catalogue folder or create it if it doesn't exist. * @return */ private static WorkspaceItem getFolderOrCreate(WorkspaceCatalogue catalogueFolder, String relativePath, String descriptionFolder){ WorkspaceItem folder = null; try { folder = catalogueFolder.getCatalogueItemByPath(relativePath); if(folder == null){ catalogueFolder.createFolder(relativePath, descriptionFolder); } } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e) { logger.error("Failed to get or generate this folder", e); } return folder; } }