gCubeSystem
/
grsf-publisher-ws
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

minor fixes before branching 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@151510 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Costantino Perciante 2017-07-31 16:29:48 +00:00
parent 94d0c3a1ed
commit 7fd2b3a435
6 changed files with 71 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.settings/org.eclipse.wst.common.component
View File

@ -3,6 +3,9 @@
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<dependent-module archiveName="ckan-util-library-2.3.2-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/ckan-util-library/ckan-util-library">
<dependency-type>uses</dependency-type>
</dependent-module>
<property name="context-root" value="grsf-publisher-ws"/>
<property name="java-output-path" value="/grsf-publisher-ws/target/classes"/>
</wb-module>

12
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java
View File

@ -1,6 +1,7 @@
package org.gcube.data_catalogue.grsf_publish_ws.json.input;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -15,16 +16,17 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* The base class contains basic information needed to publish something in the data catalogue.
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
*/
@JsonIgnoreProperties(value = {"author", "author_contact"})
public class Base {
private static final List<String> FIELDS_TO_IGNORE = Arrays.asList("author", "author_contact", "stock_uri", "fishery_uri", "reference_year", "reporting_year", "water_area",
"environment", "state_of_marine_resource");
private static Logger logger = LoggerFactory.getLogger(Base.class);
public static final String UUID_KB_KEY = "GRSF UUID";
@ -198,6 +200,12 @@ public class Base {
private void genericSetter(String key, Object value){
logger.info("Found extra property: [" + key + "," + value + "]");
if(FIELDS_TO_IGNORE.contains(key)){
logger.debug("Ignoring it");
return;
}
List<String> values = new ArrayList<String>();
if(extrasFields.containsKey(key))
values = extrasFields.get(key);

9
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java
View File

@ -38,6 +38,8 @@ import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
/**
* Services common utils.
* @author Costantino Perciante at ISTI-CNR
@ -259,7 +261,7 @@ public class CommonServiceUtils {
logger.error("Failed to read value for field " + field.getName() + " skipping", e);
}
}
}
/**
@ -475,7 +477,10 @@ public class CommonServiceUtils {
public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{
DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context);
String systemTypeValue = catalog.getDataset(itemIdOrName, apiKey).getExtrasAsHashMap().get(SYSTEM_TYPE);
CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey);
if(dataset == null)
throw new Exception("Unable to find record with id or name " + itemIdOrName);
String systemTypeValue = dataset.getExtrasAsHashMap().get(SYSTEM_TYPE);
if(systemTypeValue == null || systemTypeValue.isEmpty())
throw new Exception(SYSTEM_TYPE + " property not set in record " + itemIdOrName);
else

65
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
View File

@ -51,6 +51,10 @@ public class ManageTimeSeriesThread extends Thread{
private static CacheInterface<String, WorkspaceCatalogue> vreFolderCache = new CacheImpl<String, WorkspaceCatalogue>(1000 * 60 * 60 * 24);
private static final int MAX_NAME_CSV_FILE_WITHOUT_MEASURE = 50;
private static final int DELTA = 10;
private Common record;
private String uuidKB;
private String username;
@ -105,6 +109,8 @@ public class ManageTimeSeriesThread extends Thread{
logger.error("Error was " + e.getMessage());
} catch (UserNotFoundException e) {
logger.error("Error was " + e.getMessage());
} catch (Exception e) {
logger.error("Error was " + e.getMessage());
}finally{
ScopeProvider.instance.reset();
SecurityTokenProvider.instance.reset();
@ -117,18 +123,10 @@ public class ManageTimeSeriesThread extends Thread{
* Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
* The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
* @param record
* @throws IntrospectionException
* @throws InvocationTargetException
* @throws IllegalArgumentException
* @throws IllegalAccessException
* @throws UserNotFoundException
* @throws HomeNotFoundException
* @throws InternalErrorException
* @throws WorkspaceFolderNotFoundException
* @throws ItemNotFoundException
* @throws Exception
*/
@SuppressWarnings("rawtypes")
public static void manageTimeSeries(Common record, String uuidKB, String username, DataCatalogue catalogue) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{
public static void manageTimeSeries(Common record, String uuidKB, String username, DataCatalogue catalogue) throws Exception{
if(record == null)
throw new IllegalArgumentException("The given record is null!!");
@ -160,7 +158,11 @@ public class ManageTimeSeriesThread extends Thread{
String recordTypeFolderName = record.getGrsfType().toLowerCase();
String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName();
String semanticIdentifier = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockId() : ((FisheryRecord)record).getFisheryId();
String csvFileName = replaceIllegalChars(productName, " ");
csvFileName = generateCSVFileName(csvFileName);
logger.debug("CSV file name (without custom key) is going to be " + csvFileName);
// find the first space in the name, if any
char firstLetter = uuidKB.charAt(0);
// the whole path of the directory is going to be...
@ -199,19 +201,19 @@ public class ManageTimeSeriesThread extends Thread{
// upload this file on ckan
if(ckanResource == null)
ckanResource = uploadFileOnCatalogue(csvFile, uuidKB, catalogue, username, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, apiKeyUser);
ckanResource = uploadFileOnCatalogue(csvFile, uuidKB, catalogue, username, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, apiKeyUser, "text/csv", "text/csv");
//upload this file on the folder of the vre (under .catalogue) and change the url of the resource
if(ckanResource != null){
if(createdFileOnWorkspace == null)
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, replaceIllegalChars(semanticIdentifier, " ") + " " + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
if(createdFileOnWorkspace != null){
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
// wait for patching..
Thread.sleep(500);
Thread.sleep(1000);
logger.debug("Going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan);
boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, "", apiKeyUser);
@ -240,6 +242,32 @@ public class ManageTimeSeriesThread extends Thread{
}
}
/**
* Filename
* @param productName
* @return
* @throws Exception
*/
private static String generateCSVFileName(String productName) throws Exception {
if(productName == null || productName.isEmpty())
throw new Exception("Record name cannot be null");
String trimmedName = productName.trim();
if(trimmedName.length() <= MAX_NAME_CSV_FILE_WITHOUT_MEASURE)
return trimmedName;
else{
int spaceIndex = trimmedName.substring(MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA, MAX_NAME_CSV_FILE_WITHOUT_MEASURE).indexOf(' ');
logger.trace("Space is at " + spaceIndex + " and limits are " + "[" + (MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA) + ", " + (MAX_NAME_CSV_FILE_WITHOUT_MEASURE) + "]");
if(spaceIndex != -1)
return trimmedName.substring(0, (MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA) + spaceIndex);
else
return trimmedName.substring(0, MAX_NAME_CSV_FILE_WITHOUT_MEASURE);
}
}
/**
* Replace chars
* @param productName
@ -261,12 +289,15 @@ public class ManageTimeSeriesThread extends Thread{
*/
private static CkanResourceBase uploadFileOnCatalogue(File csvFile,
String uuidKB, DataCatalogue catalogue, String username,
String resourceToAttachName, String description, String apiKey) {
String resourceToAttachName, String description, String apiKey, String mimeType, String format) {
return catalogue.uploadResourceFile(
csvFile,
uuidKB,
apiKey,
resourceToAttachName,
description);
description,
mimeType,
format
);
}
}

2
src/main/webapp/WEB-INF/gcube-app.xml
View File

@ -1,7 +1,7 @@
<application mode='online'>
<name>GRSFPublisher</name>
<group>Data-Catalogue</group>
<version>1.2.0</version>
<version>1.3.0</version>
<description>Data Catalogue Service</description>
<local-persistence location='target' />
<exclude>/rest/</exclude>

3
src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java
View File

@ -236,7 +236,7 @@ public class JTests {
File csvFile = CSVHelpers.listToCSV(timeSeries);
// send file
instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null);
instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null);
}
// @Test
@ -455,4 +455,5 @@ public class JTests {
}
}