gCubeSystem
/
grsf-publisher-ws
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added code to copy files also under the .catalogue area of the shared folder.. needs to be tested 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@134921 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Costantino Perciante 2016-11-27 11:33:41 +00:00
parent 2c394704a5
commit 64c9a8060b
3 changed files with 223 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pom.xml
View File

@ -83,6 +83,14 @@
<!-- <artifactId>accounting-lib</artifactId> -->
<!-- <scope>provided</scope> -->
<!-- </dependency> -->
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>common-authorization</artifactId>

68
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
View File

@ -1,14 +1,10 @@
package org.gcube.data_catalogue.grsf_publish_ws.utils;
import java.beans.IntrospectionException;
import java.beans.PropertyDescriptor;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -40,6 +36,7 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
* Helper methods
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
*/
@SuppressWarnings({"unchecked","rawtypes"})
public abstract class HelperMethods {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HelperMethods.class);
@ -47,9 +44,9 @@ public abstract class HelperMethods {
// to be retrieved from the web.xml
private static final String PENDING_CONTEX_KEY = "PendingContext";
private static final String CONFIRMED_CONTEX_KEY = "ConfirmedContext";
private static final String CSV_FILE_FORMAT = ".csv";
private static final int TIME_SERIES_TAKE_LAST_VALUES = 5;
/**
* Convert a group name to its id on ckan
* @param origName
@ -427,65 +424,4 @@ public abstract class HelperMethods {
logger.info("Returning resources " + toReturn);
return toReturn;
}
/**
* Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
* The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
* @param record
* @throws IntrospectionException
* @throws InvocationTargetException
* @throws IllegalArgumentException
* @throws IllegalAccessException
*/
public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException{
if(record == null)
throw new IllegalArgumentException("The given record is null!!");
Class<?> current = record.getClass();
do{
Field[] fields = current.getDeclaredFields();
for (Field field : fields) {
if (Collection.class.isAssignableFrom(field.getType())) {
// if the list is not null, get an element
Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
if(f != null){
List asList = (List)f;
if(!asList.isEmpty()){
if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
CustomField customAnnotation = field.getAnnotation(CustomField.class);
String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT;
logger.debug("A time series has been just found");
File csvFile = CSVHelpers.listToCSV(asList);
if(csvFile != null){
// upload this file on ckan
eu.trentorise.opendata.jackan.model.CkanResource res = catalogue.uploadResourceFile(
csvFile,
packageName,
catalogue.getApiKeyFromUsername(username),
resourceToAttachName,
customAnnotation.key() + " time series for this product");
logger.debug("The resource returned is " + res.getName() + " with package id " + res.getPackageId() + " " + res.getDescription() + " " + res.getState());
// upload this file on the folder of the vre (under .catalogue) and change the url of the resource TODO
// if(res != null){
//
//
// }
}
}
}
}
}
}
}
while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class
}
}

220
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
View File

@ -1,13 +1,38 @@
package org.gcube.data_catalogue.grsf_publish_ws.utils.threads;
import java.beans.IntrospectionException;
import java.beans.PropertyDescriptor;
import java.io.File;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.Collection;
import java.util.List;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder;
import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common;
import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean;
import org.gcube.data_catalogue.grsf_publish_ws.utils.CSVHelpers;
import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanResourceBase;
/**
* Extract the time series present in the record, load them as resource on ckan and on the .catalogue
* folder under the vre folder.
@ -15,6 +40,10 @@ import org.slf4j.LoggerFactory;
*/
public class ManageTimeSeriesThread extends Thread{
private static final String CSV_FILE_FORMAT = ".csv";
private static final String PATH_SEPARATOR = "/";
private static final String CSV_MIME = "text/csv";
// Logger
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ManageTimeSeriesThread.class);
@ -41,18 +70,195 @@ public class ManageTimeSeriesThread extends Thread{
this.context = context;
}
@Override
public void run() {
logger.info("Time series manager thread started");
try {
HelperMethods.manageTimeSeries(record, packageName, username, catalogue, context);
manageTimeSeries(record, packageName, username, catalogue, context);
logger.info("The time series manager thread ended correctly");
} catch (IllegalAccessException | IllegalArgumentException
| InvocationTargetException | IntrospectionException e) {
logger.error("Failed to attach csv files to the product...");
return;
} catch (IllegalAccessException e) {
logger.error("Error was " + e.getMessage());
} catch (IllegalArgumentException e) {
logger.error("Error was " + e.getMessage());
} catch (InvocationTargetException e) {
logger.error("Error was " + e.getMessage());
} catch (WorkspaceFolderNotFoundException e) {
logger.error("Error was " + e.getMessage());
} catch (ItemNotFoundException e) {
logger.error("Error was " + e.getMessage());
} catch (IntrospectionException e) {
logger.error("Error was " + e.getMessage());
} catch (InternalErrorException e) {
logger.error("Error was " + e.getMessage());
} catch (HomeNotFoundException e) {
logger.error("Error was " + e.getMessage());
} catch (UserNotFoundException e) {
logger.error("Error was " + e.getMessage());
}
logger.error("Failed to attach csv files to the product...");
}
/**
* Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
* The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
* @param record
* @throws IntrospectionException
* @throws InvocationTargetException
* @throws IllegalArgumentException
* @throws IllegalAccessException
* @throws UserNotFoundException
* @throws HomeNotFoundException
* @throws InternalErrorException
* @throws WorkspaceFolderNotFoundException
* @throws ItemNotFoundException
*/
public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{
if(record == null)
throw new IllegalArgumentException("The given record is null!!");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(username).getWorkspace();
// Get a VRE folder by scope
WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context);
//Get the VRE Folder catalogue
WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue();
// the structure under the .catalogue will be as follows:
// .catalogue:
// - stock:
// - first_letter_of_the_product
// - product_name
// - type of files (e.g., csv)
// -files
// - fishery
// - first_letter_of_the_product
// - product_name
// - type of files (e.g., csv)
// -files
String recordTypeFolderName = record.getProductType().toLowerCase();
WorkspaceItem recordFolder = getFolderOrCreate(catalogueFolder, recordTypeFolderName, "The folder related to " + recordTypeFolderName + " products");
String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName();
char firstLetter = productName.charAt(0);
String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName;
WorkspaceItem folderProduct = getFolderOrCreate(catalogueFolder, folderPath, "The folder for the product of type " + recordTypeFolderName + " and name " + productName);
String resourceFormatPath = folderProduct.getPath() + PATH_SEPARATOR + "csv";
WorkspaceItem resourceFormatFolder = getFolderOrCreate(catalogueFolder, resourceFormatPath, "The folder for resources of type csv");
if(folderProduct != null){
logger.info("Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + ", exists");
String apiKeyUser = catalogue.getApiKeyFromUsername(username);
Class<?> current = record.getClass();
do{
Field[] fields = current.getDeclaredFields();
for (Field field : fields) {
if (Collection.class.isAssignableFrom(field.getType())) {
Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
if(f != null){
List asList = (List)f;
if(!asList.isEmpty()){
if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
CustomField customAnnotation = field.getAnnotation(CustomField.class);
String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT;
logger.debug("A time series has been just found");
File csvFile = CSVHelpers.listToCSV(asList);
if(csvFile != null){
// upload this file on ckan
CkanResourceBase ckanResource = uploadFileOnCkan(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product");
//upload this file on the folder of the vre (under .catalogue) and change the url of the resource
if(ckanResource != null){
ExternalFile createdFileOnWorkspace = uploadExternalFile(resourceFormatFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile);
if(createdFileOnWorkspace != null){
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
logger.info("going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan);
boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachName, customAnnotation.key() + " time series for this product", "", apiKeyUser);
if(updated)
logger.info("Resource has been updated with the new url");
}
}
}
}
}
}
}
}
}
while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class
}
}
/**
* Upload a file in the shared folder
* @param resourceFormatFolder
* @param resourceToAttachName
* @param description
* @param csvFile
* @return
*/
private static ExternalFile uploadExternalFile(WorkspaceItem resourceFormatFolder, String resourceToAttachName, String description, File csvFile) {
try {
return ((WorkspaceFolder)resourceFormatFolder).createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile);
} catch (InsufficientPrivilegesException | ItemAlreadyExistException
| InternalErrorException e) {
logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e);
}
return null;
}
/**
* Upload a resource on ckan
* @param csvFile
* @param packageName
* @param catalogue
* @param username
* @param resourceToAttachName
* @param description
* @return a ckan resource on success, null otherwise
*/
private static CkanResourceBase uploadFileOnCkan(File csvFile,
String packageName, DataCatalogue catalogue, String username,
String resourceToAttachName, String description) {
return catalogue.uploadResourceFile(
csvFile,
packageName,
catalogue.getApiKeyFromUsername(username),
resourceToAttachName,
description);
}
/**
* Get a folder within the catalogue folder or create it if it doesn't exist.
* @return
*/
private static WorkspaceItem getFolderOrCreate(WorkspaceCatalogue catalogueFolder, String relativePath, String descriptionFolder){
WorkspaceItem folder = null;
try {
folder = catalogueFolder.getCatalogueItemByPath(relativePath);
if(folder == null){
catalogueFolder.createFolder(relativePath, descriptionFolder);
}
} catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e) {
logger.error("Failed to get or generate this folder", e);
}
return folder;
}
}