From 7fd2b3a4354871cb26978dbeb9f68a3f80558564 Mon Sep 17 00:00:00 2001 From: Costantino Perciante Date: Mon, 31 Jul 2017 16:29:48 +0000 Subject: [PATCH] minor fixes before branching git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@151510 82a268e6-3cf1-43bd-a215-b396298e98cf --- .settings/org.eclipse.wst.common.component | 3 + .../grsf_publish_ws/json/input/Base.java | 12 +++- .../services/CommonServiceUtils.java | 9 ++- .../utils/threads/ManageTimeSeriesThread.java | 65 ++++++++++++++----- src/main/webapp/WEB-INF/gcube-app.xml | 2 +- .../grsf_publish_ws/JTests.java | 3 +- 6 files changed, 71 insertions(+), 23 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 6caf699..0fa49c2 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -3,6 +3,9 @@ + + uses + diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java index 4ec3c47..3c651d0 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java @@ -1,6 +1,7 @@ package org.gcube.data_catalogue.grsf_publish_ws.json.input; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -15,16 +16,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; /** * The base class contains basic information needed to publish something in the data catalogue. * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ -@JsonIgnoreProperties(value = {"author", "author_contact"}) public class Base { + private static final List FIELDS_TO_IGNORE = Arrays.asList("author", "author_contact", "stock_uri", "fishery_uri", "reference_year", "reporting_year", "water_area", + "environment", "state_of_marine_resource"); + private static Logger logger = LoggerFactory.getLogger(Base.class); public static final String UUID_KB_KEY = "GRSF UUID"; @@ -198,6 +200,12 @@ public class Base { private void genericSetter(String key, Object value){ logger.info("Found extra property: [" + key + "," + value + "]"); + + if(FIELDS_TO_IGNORE.contains(key)){ + logger.debug("Ignoring it"); + return; + } + List values = new ArrayList(); if(extrasFields.containsKey(key)) values = extrasFields.get(key); diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java index c51cf45..d907477 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java @@ -38,6 +38,8 @@ import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.slf4j.LoggerFactory; +import eu.trentorise.opendata.jackan.model.CkanDataset; + /** * Services common utils. * @author Costantino Perciante at ISTI-CNR @@ -259,7 +261,7 @@ public class CommonServiceUtils { logger.error("Failed to read value for field " + field.getName() + " skipping", e); } } - + } /** @@ -475,7 +477,10 @@ public class CommonServiceUtils { public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{ DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context); - String systemTypeValue = catalog.getDataset(itemIdOrName, apiKey).getExtrasAsHashMap().get(SYSTEM_TYPE); + CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey); + if(dataset == null) + throw new Exception("Unable to find record with id or name " + itemIdOrName); + String systemTypeValue = dataset.getExtrasAsHashMap().get(SYSTEM_TYPE); if(systemTypeValue == null || systemTypeValue.isEmpty()) throw new Exception(SYSTEM_TYPE + " property not set in record " + itemIdOrName); else diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index 8cd0ad6..981a10e 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -51,6 +51,10 @@ public class ManageTimeSeriesThread extends Thread{ private static CacheInterface vreFolderCache = new CacheImpl(1000 * 60 * 60 * 24); + private static final int MAX_NAME_CSV_FILE_WITHOUT_MEASURE = 50; + + private static final int DELTA = 10; + private Common record; private String uuidKB; private String username; @@ -105,6 +109,8 @@ public class ManageTimeSeriesThread extends Thread{ logger.error("Error was " + e.getMessage()); } catch (UserNotFoundException e) { logger.error("Error was " + e.getMessage()); + } catch (Exception e) { + logger.error("Error was " + e.getMessage()); }finally{ ScopeProvider.instance.reset(); SecurityTokenProvider.instance.reset(); @@ -117,18 +123,10 @@ public class ManageTimeSeriesThread extends Thread{ * Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on). * The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder. * @param record - * @throws IntrospectionException - * @throws InvocationTargetException - * @throws IllegalArgumentException - * @throws IllegalAccessException - * @throws UserNotFoundException - * @throws HomeNotFoundException - * @throws InternalErrorException - * @throws WorkspaceFolderNotFoundException - * @throws ItemNotFoundException + * @throws Exception */ @SuppressWarnings("rawtypes") - public static void manageTimeSeries(Common record, String uuidKB, String username, DataCatalogue catalogue) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{ + public static void manageTimeSeries(Common record, String uuidKB, String username, DataCatalogue catalogue) throws Exception{ if(record == null) throw new IllegalArgumentException("The given record is null!!"); @@ -160,7 +158,11 @@ public class ManageTimeSeriesThread extends Thread{ String recordTypeFolderName = record.getGrsfType().toLowerCase(); String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName(); - String semanticIdentifier = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockId() : ((FisheryRecord)record).getFisheryId(); + String csvFileName = replaceIllegalChars(productName, " "); + csvFileName = generateCSVFileName(csvFileName); + logger.debug("CSV file name (without custom key) is going to be " + csvFileName); + + // find the first space in the name, if any char firstLetter = uuidKB.charAt(0); // the whole path of the directory is going to be... @@ -199,19 +201,19 @@ public class ManageTimeSeriesThread extends Thread{ // upload this file on ckan if(ckanResource == null) - ckanResource = uploadFileOnCatalogue(csvFile, uuidKB, catalogue, username, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, apiKeyUser); + ckanResource = uploadFileOnCatalogue(csvFile, uuidKB, catalogue, username, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, apiKeyUser, "text/csv", "text/csv"); //upload this file on the folder of the vre (under .catalogue) and change the url of the resource if(ckanResource != null){ if(createdFileOnWorkspace == null) - createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, replaceIllegalChars(semanticIdentifier, " ") + " " + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); + createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); if(createdFileOnWorkspace != null){ String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); - + // wait for patching.. - Thread.sleep(500); + Thread.sleep(1000); logger.debug("Going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan); boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, "", apiKeyUser); @@ -240,6 +242,32 @@ public class ManageTimeSeriesThread extends Thread{ } } + /** + * Filename + * @param productName + * @return + * @throws Exception + */ + private static String generateCSVFileName(String productName) throws Exception { + if(productName == null || productName.isEmpty()) + throw new Exception("Record name cannot be null"); + + String trimmedName = productName.trim(); + + if(trimmedName.length() <= MAX_NAME_CSV_FILE_WITHOUT_MEASURE) + return trimmedName; + else{ + + int spaceIndex = trimmedName.substring(MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA, MAX_NAME_CSV_FILE_WITHOUT_MEASURE).indexOf(' '); + logger.trace("Space is at " + spaceIndex + " and limits are " + "[" + (MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA) + ", " + (MAX_NAME_CSV_FILE_WITHOUT_MEASURE) + "]"); + if(spaceIndex != -1) + return trimmedName.substring(0, (MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA) + spaceIndex); + else + return trimmedName.substring(0, MAX_NAME_CSV_FILE_WITHOUT_MEASURE); + + } + } + /** * Replace chars * @param productName @@ -261,12 +289,15 @@ public class ManageTimeSeriesThread extends Thread{ */ private static CkanResourceBase uploadFileOnCatalogue(File csvFile, String uuidKB, DataCatalogue catalogue, String username, - String resourceToAttachName, String description, String apiKey) { + String resourceToAttachName, String description, String apiKey, String mimeType, String format) { return catalogue.uploadResourceFile( csvFile, uuidKB, apiKey, resourceToAttachName, - description); + description, + mimeType, + format + ); } } \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/gcube-app.xml b/src/main/webapp/WEB-INF/gcube-app.xml index c1fa43c..c855f67 100644 --- a/src/main/webapp/WEB-INF/gcube-app.xml +++ b/src/main/webapp/WEB-INF/gcube-app.xml @@ -1,7 +1,7 @@ GRSFPublisher Data-Catalogue - 1.2.0 + 1.3.0 Data Catalogue Service /rest/ diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index 7ba4f53..0a48795 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -236,7 +236,7 @@ public class JTests { File csvFile = CSVHelpers.listToCSV(timeSeries); // send file - instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null); + instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null); } // @Test @@ -455,4 +455,5 @@ public class JTests { } + }