minor fixes for files creation in shared folder

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@134945 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Costantino Perciante 2016-11-28 11:20:43 +00:00
parent 1894ed250c
commit f69d326145
5 changed files with 235 additions and 41 deletions

View File

@ -86,10 +86,12 @@
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>home-library</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>

View File

@ -249,7 +249,7 @@ public class GrsfPublisherFisheryService {
// manage time series
logger.info("Launching thread for time series handling");
new ManageTimeSeriesThread(record, futureName, username, catalogue, ScopeProvider.instance.get()).start();
new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start();
}
}

View File

@ -238,7 +238,7 @@ public class GrsfPublisherStockService {
// manage time series
logger.info("Launching thread for time series handling");
new ManageTimeSeriesThread(record, futureName, username, catalogue, ScopeProvider.instance.get()).start();
new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start();
}
}
}

View File

@ -8,6 +8,7 @@ import java.lang.reflect.InvocationTargetException;
import java.util.Collection;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
@ -22,6 +23,7 @@ import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistEx
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord;
@ -52,6 +54,7 @@ public class ManageTimeSeriesThread extends Thread{
private String username;
private DataCatalogue catalogue;
private String context;
private String token;
/**
* @param record
@ -61,21 +64,25 @@ public class ManageTimeSeriesThread extends Thread{
* @param context
*/
public ManageTimeSeriesThread(Common record, String packageName,
String username, DataCatalogue catalogue, String context) {
String username, DataCatalogue catalogue, String context, String token) {
super();
this.record = record;
this.packageName = packageName;
this.username = username;
this.catalogue = catalogue;
this.context = context;
this.token = token;
}
@Override
public void run() {
logger.info("Time series manager thread started");
ScopeProvider.instance.set(context);
SecurityTokenProvider.instance.set(token);
try {
manageTimeSeries(record, packageName, username, catalogue, context);
manageTimeSeries(record, packageName, username, catalogue);
logger.info("The time series manager thread ended correctly");
return;
} catch (IllegalAccessException e) {
@ -96,6 +103,9 @@ public class ManageTimeSeriesThread extends Thread{
logger.error("Error was " + e.getMessage());
} catch (UserNotFoundException e) {
logger.error("Error was " + e.getMessage());
}finally{
ScopeProvider.instance.reset();
SecurityTokenProvider.instance.reset();
}
logger.error("Failed to attach csv files to the product...");
@ -115,42 +125,44 @@ public class ManageTimeSeriesThread extends Thread{
* @throws WorkspaceFolderNotFoundException
* @throws ItemNotFoundException
*/
public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue, String context) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{
public static void manageTimeSeries(Common record, String packageName, String username, DataCatalogue catalogue) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{
if(record == null)
throw new IllegalArgumentException("The given record is null!!");
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome(username).getWorkspace();
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
// Get a VRE folder by scope
WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context);
WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(ScopeProvider.instance.get());
//Get the VRE Folder catalogue
WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue();
logger.debug("Catalogue folder in vre has path " + catalogueFolder.getPath());
// the structure under the .catalogue will be as follows:
// .catalogue:
// - stock:
// - first_letter_of_the_product
// - product_name
// - type of files (e.g., csv)
// -files
// -files (csv)
// - fishery
// - first_letter_of_the_product
// - product_name
// - type of files (e.g., csv)
// -files
// -files (csv)
String recordTypeFolderName = record.getProductType().toLowerCase();
WorkspaceItem recordFolder = getFolderOrCreate(catalogueFolder, recordTypeFolderName, "The folder related to " + recordTypeFolderName + " products");
String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName();
char firstLetter = productName.charAt(0);
String folderPath = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName;
WorkspaceItem folderProduct = getFolderOrCreate(catalogueFolder, folderPath, "The folder for the product of type " + recordTypeFolderName + " and name " + productName);
String resourceFormatPath = folderProduct.getPath() + PATH_SEPARATOR + "csv";
WorkspaceItem resourceFormatFolder = getFolderOrCreate(catalogueFolder, resourceFormatPath, "The folder for resources of type csv");
if(folderProduct != null){
logger.info("Folder under .catalogue area in shared folder, for type " + recordTypeFolderName + " and product " + productName + ", exists");
// the whole path of the directory is going to be...
String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + productName + PATH_SEPARATOR + "csv";
logger.debug("The path under which the time series are going to be saved is " + csvDirectoryForThisProduct);
WorkspaceFolder csvFolder = createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct);
if(csvFolder != null){
String apiKeyUser = catalogue.getApiKeyFromUsername(username);
@ -171,18 +183,18 @@ public class ManageTimeSeriesThread extends Thread{
CustomField customAnnotation = field.getAnnotation(CustomField.class);
String resourceToAttachName = customAnnotation.key().replaceAll("\\s", "_") + CSV_FILE_FORMAT;
logger.debug("A time series has been just found");
logger.debug("A time series has been just found (from field " + customAnnotation.key() + ")");
File csvFile = CSVHelpers.listToCSV(asList);
if(csvFile != null){
// upload this file on ckan
CkanResourceBase ckanResource = uploadFileOnCkan(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product");
CkanResourceBase ckanResource = uploadFileOnCatalogue(csvFile, packageName, catalogue, username, resourceToAttachName, customAnnotation.key() + " time series for this product", apiKeyUser);
//upload this file on the folder of the vre (under .catalogue) and change the url of the resource
if(ckanResource != null){
ExternalFile createdFileOnWorkspace = uploadExternalFile(resourceFormatFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile);
ExternalFile createdFileOnWorkspace = uploadExternalFile(csvFolder, resourceToAttachName, customAnnotation.key() + " time series for this product", csvFile);
if(createdFileOnWorkspace != null){
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
@ -213,9 +225,9 @@ public class ManageTimeSeriesThread extends Thread{
* @param csvFile
* @return
*/
private static ExternalFile uploadExternalFile(WorkspaceItem resourceFormatFolder, String resourceToAttachName, String description, File csvFile) {
private static ExternalFile uploadExternalFile(WorkspaceFolder resourceFormatFolder, String resourceToAttachName, String description, File csvFile) {
try {
return ((WorkspaceFolder)resourceFormatFolder).createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile);
return resourceFormatFolder.createExternalFileItem(resourceToAttachName, description, CSV_MIME, csvFile);
} catch (InsufficientPrivilegesException | ItemAlreadyExistException
| InternalErrorException e) {
logger.error("Failed to upload the file into the workspace shared folder for " + resourceToAttachName, e);
@ -233,32 +245,82 @@ public class ManageTimeSeriesThread extends Thread{
* @param description
* @return a ckan resource on success, null otherwise
*/
private static CkanResourceBase uploadFileOnCkan(File csvFile,
private static CkanResourceBase uploadFileOnCatalogue(File csvFile,
String packageName, DataCatalogue catalogue, String username,
String resourceToAttachName, String description) {
String resourceToAttachName, String description, String apiKey) {
return catalogue.uploadResourceFile(
csvFile,
packageName,
catalogue.getApiKeyFromUsername(username),
apiKey,
resourceToAttachName,
description);
}
/**
* Create subfolders in cascade, returning the last created ones
* It could be also used for getting them if they already exists
* @param folder
* @param subPath
* @return null if an error occurred
*/
private static WorkspaceFolder createOrGetSubFoldersByPath(WorkspaceFolder folder, String subPath){
WorkspaceFolder parentFolder = folder;
if(folder == null)
throw new IllegalArgumentException("Root folder is null!");
if(subPath == null || subPath.isEmpty())
throw new IllegalArgumentException("subPath is null/empty!");
try{
if(subPath.startsWith(PATH_SEPARATOR))
subPath = subPath.replaceFirst(PATH_SEPARATOR, "");
if(subPath.endsWith(subPath))
subPath = subPath.substring(0, subPath.length() - 1);
logger.debug("Splitting path " + subPath);
String[] splittedPaths = subPath.split(PATH_SEPARATOR);
for (String path : splittedPaths) {
WorkspaceFolder createdFolder = getFolderOrCreate(parentFolder, path, "");
logger.debug("Created subfolder with path " + createdFolder.getPath());
parentFolder = createdFolder;
}
}catch(Exception e){
logger.error("Failed to create the subfolders by path " + subPath);
return null;
}
return parentFolder;
}
/**
* Get a folder within the catalogue folder or create it if it doesn't exist.
* @return
*/
private static WorkspaceItem getFolderOrCreate(WorkspaceCatalogue catalogueFolder, String relativePath, String descriptionFolder){
WorkspaceItem folder = null;
private static WorkspaceFolder getFolderOrCreate(WorkspaceFolder folder, String relativePath, String descriptionFolder){
WorkspaceFolder result = null;
try {
folder = catalogueFolder.getCatalogueItemByPath(relativePath);
if(folder == null){
catalogueFolder.createFolder(relativePath, descriptionFolder);
WorkspaceItem foundFolder = folder.find(relativePath);
if(foundFolder != null && foundFolder.isFolder())
result = (WorkspaceFolder)foundFolder;
if(result != null)
logger.debug("Folder found with name " + result.getName() + ", it has id " + result.getId());
else
throw new Exception("There is no folder with name " + relativePath + " under folder " + folder.getName());
} catch (Exception e) {
logger.debug("Probably the folder doesn't exist", e);
try{
result = folder.createFolder(relativePath, descriptionFolder);
} catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e2) {
logger.error("Failed to get or generate this folder", e2);
}
} catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e) {
logger.error("Failed to get or generate this folder", e);
}
return folder;
return result;
}
}

View File

@ -16,6 +16,17 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.WorkspaceSharedFolder;
import org.gcube.common.homelibrary.home.workspace.catalogue.WorkspaceCatalogue;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag;
@ -31,7 +42,6 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Type_Fishery;
import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.DataCatalogueFactory;
import org.junit.Test;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
@ -267,7 +277,7 @@ public class JTests {
}
@Test
//@Test
public void testJSONResource() throws Exception{
DataCatalogueFactory factory = DataCatalogueFactory.getFactory();
DataCatalogue instance = factory.getUtilsPerScope("/gcube/devNext/NextNext");
@ -355,13 +365,13 @@ public class JTests {
if(f != null){
List asList = (List)f;
if(!asList.isEmpty())
if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
logger.debug("Name is " + field.getName());
CustomField customAnnotation = field.getAnnotation(CustomField.class);
if(customAnnotation != null)
logger.debug("Name is " + customAnnotation.key());
}
if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
logger.debug("Name is " + field.getName());
CustomField customAnnotation = field.getAnnotation(CustomField.class);
if(customAnnotation != null)
logger.debug("Name is " + customAnnotation.key());
}
}
}
@ -371,4 +381,124 @@ public class JTests {
}
//@Test
public void sharedVREFolderWriteTest() throws Exception{
String token = "";
String context = "/gcube/devNext/NextNext";
ScopeProvider.instance.set(context);
SecurityTokenProvider.instance.set(token);
Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace();
// Get a VRE folder by scope
WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(context);
//Get the VRE Folder catalogue
WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue();
logger.debug("Catalogue folder retrieved " + catalogueFolder.getName());
/**
* Test is
* .catalogue:
* -test
* - a
* -aproductwiththisname
* - csv
* - testfile.csv
*/
String allSubPath = "/test/a/aproductwiththisname/";
//WorkspaceFolder lastFolder = createGetSubFoldersByPath(catalogueFolder, allSubPath);
// WorkspaceFolder recordFolder = (WorkspaceFolder)getFolderOrCreate(catalogueFolder, "test", "");
// String firstLetter = "a";
// WorkspaceFolder firstLetterFolder = (WorkspaceFolder)getFolderOrCreate(recordFolder, firstLetter, "");
// String folderPath = "aproductwiththisname";
// WorkspaceFolder productFolder = (WorkspaceFolder)getFolderOrCreate(firstLetterFolder, folderPath, "");
//logger.debug("Test folder created/get..its path is " + lastFolder.getPath());
// String ccsvUnderProductFolderName = productFolderName + "/" + "csv";
// WorkspaceFolder csvUnderProductFolder = (WorkspaceFolder)getFolderOrCreate(catalogueFolder, ccsvUnderProductFolderName, "");
//
// logger.debug("FOLDERS created " + csvUnderProductFolder.getPath());
// treeCheck(catalogueFolder);
}
public void treeCheck(WorkspaceFolder rootFolder) throws InternalErrorException{
List<WorkspaceItem> children = rootFolder.getChildren();
for (WorkspaceItem workspaceItem : children) {
if(workspaceItem.isFolder()){
logger.debug("children folder is " + workspaceItem.getName());
treeCheck((WorkspaceFolder)workspaceItem);
}
}
}
/**
* Create subfolders in cascade, returning the last created ones
* It could be also used for getting them if they already exists
* @param folder
* @param subPath
* @return
*/
private static WorkspaceFolder createGetSubFoldersByPath(WorkspaceFolder folder, String subPath){
String pathSeparator = "/";
WorkspaceFolder parentFolder = folder;
if(folder == null)
throw new IllegalArgumentException("Root folder is null!");
if(subPath == null || subPath.isEmpty())
throw new IllegalArgumentException("subPath is null/empty!");
try{
if(subPath.startsWith(pathSeparator))
subPath = subPath.replaceFirst(pathSeparator, "");
if(subPath.endsWith(subPath))
subPath = subPath.substring(0, subPath.length() - 1);
logger.debug("Splitting path " + subPath);
String[] splittedPaths = subPath.split(pathSeparator);
for (String path : splittedPaths) {
WorkspaceFolder createdFolder = getFolderOrCreate(parentFolder, path, "");
logger.debug("Created subfolder with path " + createdFolder.getPath());
parentFolder = createdFolder;
}
}catch(Exception e){
logger.error("Failed to create the subfolders by path " + subPath);
}
return parentFolder;
}
/**
* Get a folder within the catalogue folder or create it if it doesn't exist.
* @return
*/
private static WorkspaceFolder getFolderOrCreate(WorkspaceFolder folder, String relativePath, String descriptionFolder){
WorkspaceFolder result = null;
try {
if(folder.exists(relativePath) && folder.find(relativePath).isFolder())
result = (WorkspaceFolder) folder.find(relativePath);
if(result != null)
logger.debug("Folder found with name " + result.getName() + ", it has id " + result.getId());
else
throw new Exception("There is no folder with name " + relativePath + " under foler " + folder.getName());
} catch (Exception e) {
logger.debug("Probably the folder doesn't exist", e);
try{
result = folder.createFolder(relativePath, descriptionFolder);
} catch (InsufficientPrivilegesException | InternalErrorException | ItemAlreadyExistException e2) {
logger.error("Failed to get or generate this folder", e2);
}
}
return result;
}
}