minor fixes before branching
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@151510 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
94d0c3a1ed
commit
7fd2b3a435
|
@ -3,6 +3,9 @@
|
||||||
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
|
<wb-resource deploy-path="/" source-path="/target/m2e-wtp/web-resources"/>
|
||||||
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
|
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
|
||||||
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
|
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
|
||||||
|
<dependent-module archiveName="ckan-util-library-2.3.2-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/ckan-util-library/ckan-util-library">
|
||||||
|
<dependency-type>uses</dependency-type>
|
||||||
|
</dependent-module>
|
||||||
<property name="context-root" value="grsf-publisher-ws"/>
|
<property name="context-root" value="grsf-publisher-ws"/>
|
||||||
<property name="java-output-path" value="/grsf-publisher-ws/target/classes"/>
|
<property name="java-output-path" value="/grsf-publisher-ws/target/classes"/>
|
||||||
</wb-module>
|
</wb-module>
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package org.gcube.data_catalogue.grsf_publish_ws.json.input;
|
package org.gcube.data_catalogue.grsf_publish_ws.json.input;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -15,16 +16,17 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonAnySetter;
|
import com.fasterxml.jackson.annotation.JsonAnySetter;
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The base class contains basic information needed to publish something in the data catalogue.
|
* The base class contains basic information needed to publish something in the data catalogue.
|
||||||
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
|
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
|
||||||
*/
|
*/
|
||||||
@JsonIgnoreProperties(value = {"author", "author_contact"})
|
|
||||||
public class Base {
|
public class Base {
|
||||||
|
|
||||||
|
private static final List<String> FIELDS_TO_IGNORE = Arrays.asList("author", "author_contact", "stock_uri", "fishery_uri", "reference_year", "reporting_year", "water_area",
|
||||||
|
"environment", "state_of_marine_resource");
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(Base.class);
|
private static Logger logger = LoggerFactory.getLogger(Base.class);
|
||||||
public static final String UUID_KB_KEY = "GRSF UUID";
|
public static final String UUID_KB_KEY = "GRSF UUID";
|
||||||
|
|
||||||
|
@ -198,6 +200,12 @@ public class Base {
|
||||||
private void genericSetter(String key, Object value){
|
private void genericSetter(String key, Object value){
|
||||||
|
|
||||||
logger.info("Found extra property: [" + key + "," + value + "]");
|
logger.info("Found extra property: [" + key + "," + value + "]");
|
||||||
|
|
||||||
|
if(FIELDS_TO_IGNORE.contains(key)){
|
||||||
|
logger.debug("Ignoring it");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
List<String> values = new ArrayList<String>();
|
List<String> values = new ArrayList<String>();
|
||||||
if(extrasFields.containsKey(key))
|
if(extrasFields.containsKey(key))
|
||||||
values = extrasFields.get(key);
|
values = extrasFields.get(key);
|
||||||
|
|
|
@ -38,6 +38,8 @@ import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
|
||||||
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
|
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import eu.trentorise.opendata.jackan.model.CkanDataset;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Services common utils.
|
* Services common utils.
|
||||||
* @author Costantino Perciante at ISTI-CNR
|
* @author Costantino Perciante at ISTI-CNR
|
||||||
|
@ -475,7 +477,10 @@ public class CommonServiceUtils {
|
||||||
public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{
|
public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{
|
||||||
|
|
||||||
DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context);
|
DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context);
|
||||||
String systemTypeValue = catalog.getDataset(itemIdOrName, apiKey).getExtrasAsHashMap().get(SYSTEM_TYPE);
|
CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey);
|
||||||
|
if(dataset == null)
|
||||||
|
throw new Exception("Unable to find record with id or name " + itemIdOrName);
|
||||||
|
String systemTypeValue = dataset.getExtrasAsHashMap().get(SYSTEM_TYPE);
|
||||||
if(systemTypeValue == null || systemTypeValue.isEmpty())
|
if(systemTypeValue == null || systemTypeValue.isEmpty())
|
||||||
throw new Exception(SYSTEM_TYPE + " property not set in record " + itemIdOrName);
|
throw new Exception(SYSTEM_TYPE + " property not set in record " + itemIdOrName);
|
||||||
else
|
else
|
||||||
|
|
|
@ -51,6 +51,10 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
|
|
||||||
private static CacheInterface<String, WorkspaceCatalogue> vreFolderCache = new CacheImpl<String, WorkspaceCatalogue>(1000 * 60 * 60 * 24);
|
private static CacheInterface<String, WorkspaceCatalogue> vreFolderCache = new CacheImpl<String, WorkspaceCatalogue>(1000 * 60 * 60 * 24);
|
||||||
|
|
||||||
|
private static final int MAX_NAME_CSV_FILE_WITHOUT_MEASURE = 50;
|
||||||
|
|
||||||
|
private static final int DELTA = 10;
|
||||||
|
|
||||||
private Common record;
|
private Common record;
|
||||||
private String uuidKB;
|
private String uuidKB;
|
||||||
private String username;
|
private String username;
|
||||||
|
@ -105,6 +109,8 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
logger.error("Error was " + e.getMessage());
|
logger.error("Error was " + e.getMessage());
|
||||||
} catch (UserNotFoundException e) {
|
} catch (UserNotFoundException e) {
|
||||||
logger.error("Error was " + e.getMessage());
|
logger.error("Error was " + e.getMessage());
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("Error was " + e.getMessage());
|
||||||
}finally{
|
}finally{
|
||||||
ScopeProvider.instance.reset();
|
ScopeProvider.instance.reset();
|
||||||
SecurityTokenProvider.instance.reset();
|
SecurityTokenProvider.instance.reset();
|
||||||
|
@ -117,18 +123,10 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
* Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
|
* Manage the time series bean within a resource (e.g., catches or landings, exploitation rate and so on).
|
||||||
* The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
|
* The method save the time series as csv on ckan, and also save the file in the .catalogue area of the shared vre folder.
|
||||||
* @param record
|
* @param record
|
||||||
* @throws IntrospectionException
|
* @throws Exception
|
||||||
* @throws InvocationTargetException
|
|
||||||
* @throws IllegalArgumentException
|
|
||||||
* @throws IllegalAccessException
|
|
||||||
* @throws UserNotFoundException
|
|
||||||
* @throws HomeNotFoundException
|
|
||||||
* @throws InternalErrorException
|
|
||||||
* @throws WorkspaceFolderNotFoundException
|
|
||||||
* @throws ItemNotFoundException
|
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("rawtypes")
|
@SuppressWarnings("rawtypes")
|
||||||
public static void manageTimeSeries(Common record, String uuidKB, String username, DataCatalogue catalogue) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, IntrospectionException, WorkspaceFolderNotFoundException, InternalErrorException, HomeNotFoundException, UserNotFoundException, ItemNotFoundException{
|
public static void manageTimeSeries(Common record, String uuidKB, String username, DataCatalogue catalogue) throws Exception{
|
||||||
|
|
||||||
if(record == null)
|
if(record == null)
|
||||||
throw new IllegalArgumentException("The given record is null!!");
|
throw new IllegalArgumentException("The given record is null!!");
|
||||||
|
@ -160,7 +158,11 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
|
|
||||||
String recordTypeFolderName = record.getGrsfType().toLowerCase();
|
String recordTypeFolderName = record.getGrsfType().toLowerCase();
|
||||||
String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName();
|
String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName();
|
||||||
String semanticIdentifier = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockId() : ((FisheryRecord)record).getFisheryId();
|
String csvFileName = replaceIllegalChars(productName, " ");
|
||||||
|
csvFileName = generateCSVFileName(csvFileName);
|
||||||
|
logger.debug("CSV file name (without custom key) is going to be " + csvFileName);
|
||||||
|
|
||||||
|
// find the first space in the name, if any
|
||||||
char firstLetter = uuidKB.charAt(0);
|
char firstLetter = uuidKB.charAt(0);
|
||||||
|
|
||||||
// the whole path of the directory is going to be...
|
// the whole path of the directory is going to be...
|
||||||
|
@ -199,19 +201,19 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
|
|
||||||
// upload this file on ckan
|
// upload this file on ckan
|
||||||
if(ckanResource == null)
|
if(ckanResource == null)
|
||||||
ckanResource = uploadFileOnCatalogue(csvFile, uuidKB, catalogue, username, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, apiKeyUser);
|
ckanResource = uploadFileOnCatalogue(csvFile, uuidKB, catalogue, username, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, apiKeyUser, "text/csv", "text/csv");
|
||||||
|
|
||||||
//upload this file on the folder of the vre (under .catalogue) and change the url of the resource
|
//upload this file on the folder of the vre (under .catalogue) and change the url of the resource
|
||||||
if(ckanResource != null){
|
if(ckanResource != null){
|
||||||
|
|
||||||
if(createdFileOnWorkspace == null)
|
if(createdFileOnWorkspace == null)
|
||||||
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, replaceIllegalChars(semanticIdentifier, " ") + " " + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
|
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
|
||||||
|
|
||||||
if(createdFileOnWorkspace != null){
|
if(createdFileOnWorkspace != null){
|
||||||
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
|
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
|
||||||
|
|
||||||
// wait for patching..
|
// wait for patching..
|
||||||
Thread.sleep(500);
|
Thread.sleep(1000);
|
||||||
logger.debug("Going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan);
|
logger.debug("Going to patch the created resource with id " + ckanResource.getId() + " with url " + publicUrlToSetOnCkan);
|
||||||
boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, "", apiKeyUser);
|
boolean updated = catalogue.patchResource(ckanResource.getId(), publicUrlToSetOnCkan, resourceToAttachOnCkanName, resourceToAttachOnCkanDescription, "", apiKeyUser);
|
||||||
|
|
||||||
|
@ -240,6 +242,32 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Filename
|
||||||
|
* @param productName
|
||||||
|
* @return
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
private static String generateCSVFileName(String productName) throws Exception {
|
||||||
|
if(productName == null || productName.isEmpty())
|
||||||
|
throw new Exception("Record name cannot be null");
|
||||||
|
|
||||||
|
String trimmedName = productName.trim();
|
||||||
|
|
||||||
|
if(trimmedName.length() <= MAX_NAME_CSV_FILE_WITHOUT_MEASURE)
|
||||||
|
return trimmedName;
|
||||||
|
else{
|
||||||
|
|
||||||
|
int spaceIndex = trimmedName.substring(MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA, MAX_NAME_CSV_FILE_WITHOUT_MEASURE).indexOf(' ');
|
||||||
|
logger.trace("Space is at " + spaceIndex + " and limits are " + "[" + (MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA) + ", " + (MAX_NAME_CSV_FILE_WITHOUT_MEASURE) + "]");
|
||||||
|
if(spaceIndex != -1)
|
||||||
|
return trimmedName.substring(0, (MAX_NAME_CSV_FILE_WITHOUT_MEASURE - DELTA) + spaceIndex);
|
||||||
|
else
|
||||||
|
return trimmedName.substring(0, MAX_NAME_CSV_FILE_WITHOUT_MEASURE);
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Replace chars
|
* Replace chars
|
||||||
* @param productName
|
* @param productName
|
||||||
|
@ -261,12 +289,15 @@ public class ManageTimeSeriesThread extends Thread{
|
||||||
*/
|
*/
|
||||||
private static CkanResourceBase uploadFileOnCatalogue(File csvFile,
|
private static CkanResourceBase uploadFileOnCatalogue(File csvFile,
|
||||||
String uuidKB, DataCatalogue catalogue, String username,
|
String uuidKB, DataCatalogue catalogue, String username,
|
||||||
String resourceToAttachName, String description, String apiKey) {
|
String resourceToAttachName, String description, String apiKey, String mimeType, String format) {
|
||||||
return catalogue.uploadResourceFile(
|
return catalogue.uploadResourceFile(
|
||||||
csvFile,
|
csvFile,
|
||||||
uuidKB,
|
uuidKB,
|
||||||
apiKey,
|
apiKey,
|
||||||
resourceToAttachName,
|
resourceToAttachName,
|
||||||
description);
|
description,
|
||||||
|
mimeType,
|
||||||
|
format
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,7 +1,7 @@
|
||||||
<application mode='online'>
|
<application mode='online'>
|
||||||
<name>GRSFPublisher</name>
|
<name>GRSFPublisher</name>
|
||||||
<group>Data-Catalogue</group>
|
<group>Data-Catalogue</group>
|
||||||
<version>1.2.0</version>
|
<version>1.3.0</version>
|
||||||
<description>Data Catalogue Service</description>
|
<description>Data Catalogue Service</description>
|
||||||
<local-persistence location='target' />
|
<local-persistence location='target' />
|
||||||
<exclude>/rest/</exclude>
|
<exclude>/rest/</exclude>
|
||||||
|
|
|
@ -236,7 +236,7 @@ public class JTests {
|
||||||
File csvFile = CSVHelpers.listToCSV(timeSeries);
|
File csvFile = CSVHelpers.listToCSV(timeSeries);
|
||||||
|
|
||||||
// send file
|
// send file
|
||||||
instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null);
|
instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
// @Test
|
// @Test
|
||||||
|
@ -455,4 +455,5 @@ public class JTests {
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue