diff --git a/pom.xml b/pom.xml
index 8dc65e4..514f520 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,6 +83,14 @@
+
+ org.gcube.common
+ home-library-jcr
+
+
+ org.gcube.common
+ home-library
+
org.gcube.common
common-authorization
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
index c79ecd5..ee31472 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
@@ -1,14 +1,10 @@
package org.gcube.data_catalogue.grsf_publish_ws.utils;
-import java.beans.IntrospectionException;
import java.beans.PropertyDescriptor;
import java.io.BufferedReader;
-import java.io.File;
import java.io.InputStreamReader;
import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -40,6 +36,7 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
* Helper methods
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
*/
+@SuppressWarnings({"unchecked","rawtypes"})
public abstract class HelperMethods {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HelperMethods.class);
@@ -47,9 +44,9 @@ public abstract class HelperMethods {
// to be retrieved from the web.xml
private static final String PENDING_CONTEX_KEY = "PendingContext";
private static final String CONFIRMED_CONTEX_KEY = "ConfirmedContext";
- private static final String CSV_FILE_FORMAT = ".csv";
private static final int TIME_SERIES_TAKE_LAST_VALUES = 5;
+
/**
* Convert a group name to its id on ckan
* @param origName
@@ -427,65 +424,4 @@ public abstract class HelperMethods {
logger.info("Returning resources " + toReturn);
return toReturn;
}
-
- /**
- * Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
- * The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
- * @param record
- * @throws IntrospectionException
- * @throws InvocationTargetException
- * @throws IllegalArgumentException
- * @throws IllegalAccessException
- */
- public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException{
-
- if(record == null)
- throw new IllegalArgumentException("The given record is null!!");
-
- Class> current = record.getClass();
- do{
- Field[] fields = current.getDeclaredFields();
- for (Field field : fields) {
- if (Collection.class.isAssignableFrom(field.getType())) {
- // if the list is not null, get an element
- Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
- if(f != null){
- List asList = (List)f;
-
- if(!asList.isEmpty()){
- if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
-
- CustomField customAnnotation = field.getAnnotation(CustomField.class);
- String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT;
-
- logger.debug("A time series has been just found");
- File csvFile = CSVHelpers.listToCSV(asList);
-
- if(csvFile != null){
-
- // upload this file on ckan
- eu.trentorise.opendata.jackan.model.CkanResource res = catalogue.uploadResourceFile(
- csvFile,
- packageName,
- catalogue.getApiKeyFromUsername(username),
- resourceToAttachName,
- customAnnotation.key() + " time series for this product");
-
- logger.debug("The resource returned is " + res.getName() + " with package id " + res.getPackageId() + " " + res.getDescription() + " " + res.getState());
-
- // upload this file on the folder of the vre (under .catalogue) and change the url of the resource TODO
- // if(res != null){
- //
- //
- // }
- }
- }
- }
- }
- }
- }
- }
- while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class
-
- }
}
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
index 6e95388..4595575 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
@@ -1,13 +1,38 @@
package org.gcube.data_catalogue.grsf_publish_ws.utils.threads;
import java.beans.IntrospectionException;
+import java.beans.PropertyDescriptor;
+import java.io.File;
+import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
+import java.util.List;
+import org.gcube.common.homelibrary.home.HomeLibrary;
+import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
+import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
+import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
+import org.gcube.common.homelibrary.home.workspace.Workspace;
+import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
+import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
+import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder;
+import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue;
+import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
+import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
+import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
+import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
+import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
+import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common;
-import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods;
+import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord;
+import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord;
+import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean;
+import org.gcube.data_catalogue.grsf_publish_ws.utils.CSVHelpers;
import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue;
import org.slf4j.LoggerFactory;
+import eu.trentorise.opendata.jackan.model.CkanResourceBase;
+
/**
* Extract the time series present in the record, load them as resource on ckan and on the .catalogue
* folder under the vre folder.
@@ -15,6 +40,10 @@ import org.slf4j.LoggerFactory;
*/
public class ManageTimeSeriesThread extends Thread{
+ private static final String CSV_FILE_FORMAT = ".csv";
+ private static final String PATH_SEPARATOR = "/";
+ private static final String CSV_MIME = "text/csv";
+
// Logger
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ManageTimeSeriesThread.class);
@@ -41,18 +70,195 @@ public class ManageTimeSeriesThread extends Thread{
this.context = context;
}
-
-
@Override
public void run() {
logger.info("Time series manager thread started");
+
try {
- HelperMethods.manageTimeSeries(record, packageName, username, catalogue, context);
+ manageTimeSeries(record, packageName, username, catalogue, context);
logger.info("The time series manager thread ended correctly");
- } catch (IllegalAccessException | IllegalArgumentException
- | InvocationTargetException | IntrospectionException e) {
- logger.error("Failed to attach csv files to the product...");
+ return;
+ } catch (IllegalAccessException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (IllegalArgumentException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (InvocationTargetException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (WorkspaceFolderNotFoundException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (ItemNotFoundException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (IntrospectionException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (InternalErrorException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (HomeNotFoundException e) {
+ logger.error("Error was " + e.getMessage());
+ } catch (UserNotFoundException e) {
+ logger.error("Error was " + e.getMessage());
+ }
+
+ logger.error("Failed to attach csv files to the product...");
+ }
+
+ /**
+ * Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
+ * The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
+ * @param record
+ * @throws IntrospectionException
+ * @throws InvocationTargetException
+ * @throws IllegalArgumentException
+ * @throws IllegalAccessException
+ * @throws UserNotFoundException
+ * @throws HomeNotFoundException
+ * @throws InternalErrorException
+ * @throws WorkspaceFolderNotFoundException
+ * @throws ItemNotFoundException
+ */
+ public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{
+
+ if(record == null)
+ throw new IllegalArgumentException("The given record is null!!");
+
+ Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(username).getWorkspace();
+
+ // Get a VRE folder by scope
+ WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context);
+
+ //Get the VRE Folder catalogue
+ WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue();
+
+ // the structure under the .catalogue will be as follows:
+ // .catalogue:
+ // - stock:
+ // - first_letter_of_the_product
+ // - product_name
+ // - type of files (e.g., csv)
+ // -files
+ // - fishery
+ // - first_letter_of_the_product
+ // - product_name
+ // - type of files (e.g., csv)
+ // -files
+ String recordTypeFolderName = record.getProductType().toLowerCase();
+ WorkspaceItem recordFolder = getFolderOrCreate(catalogueFolder, recordTypeFolderName, "The folder related to " + recordTypeFolderName + " products");
+ String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName();
+ char firstLetter = productName.charAt(0);
+ String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName;
+ WorkspaceItem folderProduct = getFolderOrCreate(catalogueFolder, folderPath, "The folder for the product of type " + recordTypeFolderName + " and name " + productName);
+ String resourceFormatPath = folderProduct.getPath() + PATH_SEPARATOR + "csv";
+ WorkspaceItem resourceFormatFolder = getFolderOrCreate(catalogueFolder, resourceFormatPath, "The folder for resources of type csv");
+
+ if(folderProduct != null){
+ logger.info("Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + ", exists");
+
+ String apiKeyUser = catalogue.getApiKeyFromUsername(username);
+
+ Class> current = record.getClass();
+ do{
+ Field[] fields = current.getDeclaredFields();
+ for (Field field : fields) {
+ if (Collection.class.isAssignableFrom(field.getType())) {
+
+ Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
+ if(f != null){
+
+ List asList = (List)f;
+
+ if(!asList.isEmpty()){
+ if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
+
+ CustomField customAnnotation = field.getAnnotation(CustomField.class);
+ String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT;
+
+ logger.debug("A time series has been just found");
+ File csvFile = CSVHelpers.listToCSV(asList);
+
+ if(csvFile != null){
+
+ // upload this file on ckan
+ CkanResourceBase ckanResource = uploadFileOnCkan(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product");
+
+ //upload this file on the folder of the vre (under .catalogue) and change the url of the resource
+ if(ckanResource != null){
+
+ ExternalFile createdFileOnWorkspace = uploadExternalFile(resourceFormatFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile);
+ if(createdFileOnWorkspace != null){
+
+ String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
+ logger.info("going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan);
+ boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachName, customAnnotation.key() + " time series for this product", "", apiKeyUser);
+
+ if(updated)
+ logger.info("Resource has been updated with the new url");
+
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class
}
}
+ /**
+ * Upload a file in the shared folder
+ * @param resourceFormatFolder
+ * @param resourceToAttachName
+ * @param description
+ * @param csvFile
+ * @return
+ */
+ private static ExternalFile uploadExternalFile(WorkspaceItem resourceFormatFolder, String resourceToAttachName, String description, File csvFile) {
+ try {
+ return ((WorkspaceFolder)resourceFormatFolder).createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile);
+ } catch (InsufficientPrivilegesException | ItemAlreadyExistException
+ | InternalErrorException e) {
+ logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e);
+ }
+ return null;
+ }
+
+ /**
+ * Upload a resource on ckan
+ * @param csvFile
+ * @param packageName
+ * @param catalogue
+ * @param username
+ * @param resourceToAttachName
+ * @param description
+ * @return a ckan resource on success, null otherwise
+ */
+ private static CkanResourceBase uploadFileOnCkan(File csvFile,
+ String packageName, DataCatalogue catalogue, String username,
+ String resourceToAttachName, String description) {
+ return catalogue.uploadResourceFile(
+ csvFile,
+ packageName,
+ catalogue.getApiKeyFromUsername(username),
+ resourceToAttachName,
+ description);
+ }
+
+ /**
+ * Get a folder within the catalogue folder or create it if it doesn't exist.
+ * @return
+ */
+ private static WorkspaceItem getFolderOrCreate(WorkspaceCatalogue catalogueFolder, String relativePath, String descriptionFolder){
+ WorkspaceItem folder = null;
+ try {
+ folder = catalogueFolder.getCatalogueItemByPath(relativePath);
+ if(folder == null){
+ catalogueFolder.createFolder(relativePath, descriptionFolder);
+ }
+ } catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e) {
+ logger.error("Failed to get or generate this folder", e);
+ }
+ return folder;
+ }
+
}