2016-11-24 17:53:50 +01:00
package org.gcube.data_catalogue.grsf_publish_ws.utils.threads ;
2016-11-25 13:56:27 +01:00
import java.beans.IntrospectionException ;
2016-11-27 12:33:41 +01:00
import java.beans.PropertyDescriptor ;
import java.io.File ;
import java.lang.reflect.Field ;
2016-11-25 13:56:27 +01:00
import java.lang.reflect.InvocationTargetException ;
2016-11-27 12:33:41 +01:00
import java.util.Collection ;
import java.util.List ;
2016-11-25 13:56:27 +01:00
2016-11-27 12:33:41 +01:00
import org.gcube.common.homelibrary.home.HomeLibrary ;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException ;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException ;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException ;
import org.gcube.common.homelibrary.home.workspace.Workspace ;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder ;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem ;
import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder ;
import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue ;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException ;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException ;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException ;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException ;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile ;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField ;
2016-11-24 17:53:50 +01:00
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common ;
2016-11-27 12:33:41 +01:00
import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord ;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord ;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean ;
import org.gcube.data_catalogue.grsf_publish_ws.utils.CSVHelpers ;
2016-11-24 17:53:50 +01:00
import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue ;
import org.slf4j.LoggerFactory ;
2016-11-27 12:33:41 +01:00
import eu.trentorise.opendata.jackan.model.CkanResourceBase ;
2016-11-24 17:53:50 +01:00
/ * *
* Extract the time series present in the record , load them as resource on ckan and on the . catalogue
* folder under the vre folder .
* @author Costantino Perciante at ISTI - CNR ( costantino . perciante @isti.cnr.it )
* /
public class ManageTimeSeriesThread extends Thread {
2016-11-27 12:33:41 +01:00
private static final String CSV_FILE_FORMAT = " .csv " ;
private static final String PATH_SEPARATOR = " / " ;
private static final String CSV_MIME = " text/csv " ;
2016-11-24 17:53:50 +01:00
// Logger
private static final org . slf4j . Logger logger = LoggerFactory . getLogger ( ManageTimeSeriesThread . class ) ;
2016-11-25 13:56:27 +01:00
private Common record ;
2016-11-25 14:35:00 +01:00
private String packageName ;
2016-11-25 13:56:27 +01:00
private String username ;
private DataCatalogue catalogue ;
private String context ;
/ * *
* @param record
* @param packageId
* @param username
* @param catalogue
* @param context
* /
2016-11-25 14:35:00 +01:00
public ManageTimeSeriesThread ( Common record , String packageName ,
2016-11-24 17:53:50 +01:00
String username , DataCatalogue catalogue , String context ) {
2016-11-25 13:56:27 +01:00
super ( ) ;
this . record = record ;
2016-11-25 14:35:00 +01:00
this . packageName = packageName ;
2016-11-25 13:56:27 +01:00
this . username = username ;
this . catalogue = catalogue ;
this . context = context ;
}
2016-11-24 17:53:50 +01:00
2016-11-25 13:56:27 +01:00
@Override
public void run ( ) {
2016-11-24 17:53:50 +01:00
logger . info ( " Time series manager thread started " ) ;
2016-11-27 12:33:41 +01:00
2016-11-24 17:53:50 +01:00
try {
2016-11-27 12:33:41 +01:00
manageTimeSeries ( record , packageName , username , catalogue , context ) ;
2016-11-25 13:56:27 +01:00
logger . info ( " The time series manager thread ended correctly " ) ;
2016-11-27 12:33:41 +01:00
return ;
} catch ( IllegalAccessException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( IllegalArgumentException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( InvocationTargetException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( WorkspaceFolderNotFoundException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( ItemNotFoundException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( IntrospectionException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( InternalErrorException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( HomeNotFoundException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
} catch ( UserNotFoundException e ) {
logger . error ( " Error was " + e . getMessage ( ) ) ;
}
logger . error ( " Failed to attach csv files to the product... " ) ;
}
/ * *
* Manage the time series bean within a resource ( e . g . , catches or landings , exploitation rate and so on ) .
* The method save the time series as csv on ckan , and also save the file in the . catalogue area of the shared vre folder .
* @param record
* @throws IntrospectionException
* @throws InvocationTargetException
* @throws IllegalArgumentException
* @throws IllegalAccessException
* @throws UserNotFoundException
* @throws HomeNotFoundException
* @throws InternalErrorException
* @throws WorkspaceFolderNotFoundException
* @throws ItemNotFoundException
* /
public static void manageTimeSeries ( Common record , String packageName , String username , DataCatalogue catalogue , String context ) throws IllegalAccessException , IllegalArgumentException , InvocationTargetException , IntrospectionException , WorkspaceFolderNotFoundException , InternalErrorException , HomeNotFoundException , UserNotFoundException , ItemNotFoundException {
if ( record = = null )
throw new IllegalArgumentException ( " The given record is null!! " ) ;
Workspace ws = HomeLibrary . getHomeManagerFactory ( ) . getHomeManager ( ) . getHome ( username ) . getWorkspace ( ) ;
// Get a VRE folder by scope
WorkspaceSharedFolder vreFolder = ws . getVREFolderByScope ( context ) ;
//Get the VRE Folder catalogue
WorkspaceCatalogue catalogueFolder = vreFolder . getVRECatalogue ( ) ;
// the structure under the .catalogue will be as follows:
// .catalogue:
// - stock:
// - first_letter_of_the_product
// - product_name
// - type of files (e.g., csv)
// -files
// - fishery
// - first_letter_of_the_product
// - product_name
// - type of files (e.g., csv)
// -files
String recordTypeFolderName = record . getProductType ( ) . toLowerCase ( ) ;
WorkspaceItem recordFolder = getFolderOrCreate ( catalogueFolder , recordTypeFolderName , " The folder related to " + recordTypeFolderName + " products " ) ;
String productName = record . getClass ( ) . equals ( StockRecord . class ) ? ( ( StockRecord ) record ) . getStockName ( ) : ( ( FisheryRecord ) record ) . getFisheryName ( ) ;
char firstLetter = productName . charAt ( 0 ) ;
String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName ;
WorkspaceItem folderProduct = getFolderOrCreate ( catalogueFolder , folderPath , " The folder for the product of type " + recordTypeFolderName + " and name " + productName ) ;
String resourceFormatPath = folderProduct . getPath ( ) + PATH_SEPARATOR + " csv " ;
WorkspaceItem resourceFormatFolder = getFolderOrCreate ( catalogueFolder , resourceFormatPath , " The folder for resources of type csv " ) ;
if ( folderProduct ! = null ) {
logger . info ( " Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + " , exists " ) ;
String apiKeyUser = catalogue . getApiKeyFromUsername ( username ) ;
Class < ? > current = record . getClass ( ) ;
do {
Field [ ] fields = current . getDeclaredFields ( ) ;
for ( Field field : fields ) {
if ( Collection . class . isAssignableFrom ( field . getType ( ) ) ) {
Object f = new PropertyDescriptor ( field . getName ( ) , current ) . getReadMethod ( ) . invoke ( record ) ;
if ( f ! = null ) {
List asList = ( List ) f ;
if ( ! asList . isEmpty ( ) ) {
if ( asList . get ( 0 ) . getClass ( ) . equals ( TimeSeriesBean . class ) ) {
CustomField customAnnotation = field . getAnnotation ( CustomField . class ) ;
String resourceToAttachName = customAnnotation . key ( ) . replaceAll ( " \\ s " , " _ " ) + CSV_FILE_FORMAT ;
logger . debug ( " A time series has been just found " ) ;
File csvFile = CSVHelpers . listToCSV ( asList ) ;
if ( csvFile ! = null ) {
// upload this file on ckan
CkanResourceBase ckanResource = uploadFileOnCkan ( csvFile , packageName , catalogue , username , resourceToAttachName , customAnnotation . key ( ) + " time series for this product " ) ;
//upload this file on the folder of the vre (under .catalogue) and change the url of the resource
if ( ckanResource ! = null ) {
ExternalFile createdFileOnWorkspace = uploadExternalFile ( resourceFormatFolder , resourceToAttachName , customAnnotation . key ( ) + " time series for this product " , csvFile ) ;
if ( createdFileOnWorkspace ! = null ) {
String publicUrlToSetOnCkan = createdFileOnWorkspace . getPublicLink ( true ) ;
logger . info ( " going to patch the created resource with id " + ckanResource . getId ( ) + " with url " + publicUrlToSetOnCkan ) ;
boolean updated = catalogue . patchResource ( ckanResource . getId ( ) , publicUrlToSetOnCkan , resourceToAttachName , customAnnotation . key ( ) + " time series for this product " , " " , apiKeyUser ) ;
if ( updated )
logger . info ( " Resource has been updated with the new url " ) ;
}
}
}
}
}
}
}
}
}
while ( ( current = current . getSuperclass ( ) ) ! = null ) ; // iterate from the inherited class up to the Object.class
}
}
/ * *
* Upload a file in the shared folder
* @param resourceFormatFolder
* @param resourceToAttachName
* @param description
* @param csvFile
* @return
* /
private static ExternalFile uploadExternalFile ( WorkspaceItem resourceFormatFolder , String resourceToAttachName , String description , File csvFile ) {
try {
return ( ( WorkspaceFolder ) resourceFormatFolder ) . createExternalFileItem ( resourceToAttachName , description , CSV_MIME , csvFile ) ;
} catch ( InsufficientPrivilegesException | ItemAlreadyExistException
| InternalErrorException e ) {
logger . error ( " Failed to upload the file into the workspace shared folder for " + resourceToAttachName , e ) ;
}
return null ;
}
/ * *
* Upload a resource on ckan
* @param csvFile
* @param packageName
* @param catalogue
* @param username
* @param resourceToAttachName
* @param description
* @return a ckan resource on success , null otherwise
* /
private static CkanResourceBase uploadFileOnCkan ( File csvFile ,
String packageName , DataCatalogue catalogue , String username ,
String resourceToAttachName , String description ) {
return catalogue . uploadResourceFile (
csvFile ,
packageName ,
catalogue . getApiKeyFromUsername ( username ) ,
resourceToAttachName ,
description ) ;
}
/ * *
* Get a folder within the catalogue folder or create it if it doesn ' t exist .
* @return
* /
private static WorkspaceItem getFolderOrCreate ( WorkspaceCatalogue catalogueFolder , String relativePath , String descriptionFolder ) {
WorkspaceItem folder = null ;
try {
folder = catalogueFolder . getCatalogueItemByPath ( relativePath ) ;
if ( folder = = null ) {
catalogueFolder . createFolder ( relativePath , descriptionFolder ) ;
}
} catch ( InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e ) {
logger . error ( " Failed to get or generate this folder " , e ) ;
2016-11-24 17:53:50 +01:00
}
2016-11-27 12:33:41 +01:00
return folder ;
2016-11-24 17:53:50 +01:00
}
}