semantic identifier is modified by the service, changed system:type to fishery and stocks. Other minor changes in resource names. Version is 1.4.0

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@153263 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Costantino Perciante 2017-09-19 14:42:15 +00:00
parent 1512ab942c
commit 0e63ab07bc
10 changed files with 137 additions and 32 deletions

View File

@ -11,7 +11,7 @@
<groupId>org.gcube.data-catalogue</groupId>
<artifactId>grsf-publisher-ws</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.4.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>grsf-publisher-ws</name>

View File

@ -115,7 +115,22 @@ public class FisheryRecord extends Common{
}
public void setFisheryId(String fisheryId) {
this.fisheryId = fisheryId;
String fisheryIdmodified = "";
if(fisheryId != null){
String[] splitCodesValues = fisheryId.split("+");
for (int i = 0; i < splitCodesValues.length; i++) {
String prefixAndCode = splitCodesValues[i];
String prefix = prefixAndCode.split(":")[0];
String code = prefixAndCode.split(":")[1];
fisheryIdmodified += prefix.toLowerCase() + ":" + code.toUpperCase();
if(splitCodesValues.length > 1 && i < (splitCodesValues.length - 1))
fisheryIdmodified += "+";
}
}
this.fisheryId = fisheryIdmodified;
}
public List<String> getFishingArea() {

View File

@ -180,7 +180,22 @@ public class StockRecord extends Common{
}
public void setStockId(String stockId) {
this.stockId = stockId;
String stockIdmodified = "";
if(stockId != null){
String[] splitCodesValues = stockId.split("+");
for (int i = 0; i < splitCodesValues.length; i++) {
String prefixAndCode = splitCodesValues[i];
String prefix = prefixAndCode.split(":")[0];
String code = prefixAndCode.split(":")[1];
stockIdmodified += prefix.toLowerCase() + ":" + code.toUpperCase();
if(splitCodesValues.length > 1 && i < (splitCodesValues.length - 1))
stockIdmodified += "+";
}
}
this.stockId = stockIdmodified;
}
public List<String> getArea() {

View File

@ -29,7 +29,6 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.utils.Caller;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.FisheryRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.DeleteProductBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean;
@ -40,6 +39,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
@ -109,13 +109,16 @@ public class GrsfPublisherFisheryService {
String apiKey = catalogue.getApiKeyFromUsername(username);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
// extend this role to the other organizations in this context
CommonServiceUtils.extendRoleToOtherOrganizations(username, catalogue, organization, RolesCkanGroupOrOrg.ADMIN);
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(authorMail == null || authorFullname == null){
throw new Exception("Sorry but there was not possible to retrieve your fullname/email!");
throw new Exception("Sorry but it was not possible to retrieve your fullname/email!");
}
// The name of the product will be the uuid of the kb. The title will be the fishery's fishery_name. Fishery has also the constraint that
@ -155,6 +158,8 @@ public class GrsfPublisherFisheryService {
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking create method..");
@ -163,7 +168,7 @@ public class GrsfPublisherFisheryService {
apiKey,
futureTitle,
futureName,
organization,
publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer() == null? authorFullname : record.getMaintainer(),
@ -241,10 +246,10 @@ public class GrsfPublisherFisheryService {
}
// check it is in the right source and it is a fishery
String grsfTypeValue = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
String systemTypeValue = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
String systemType = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.FISHERY.getOrigName().equals(grsfTypeValue)){
if(fisheryInCkan.getOrganization().getName().equalsIgnoreCase(source) &&
CommonServiceUtils.SYSTEM_TYPE_FISHERY.equals(systemType)){
logger.debug("Ok, this is a fishery of the right source, removing it");
boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), apiKey, true);
@ -300,13 +305,13 @@ public class GrsfPublisherFisheryService {
if(sourceInPath.equals(Sources.GRSF))
datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "fishery", catalogue);
else{
List<String> fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue);
List<String> fullGroupListIds = HelperMethods.getProductsInOrganization(source, catalogue);
String apiKey = catalogue.getApiKeyFromUsername(username);
for (String id : fullGroupListIds) {
CkanDataset dataset = catalogue.getDataset(id, apiKey);
if(dataset != null){
String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
if(grsfType.equals(Product_Type.FISHERY.getOrigName()))
String type = dataset.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
if(CommonServiceUtils.SYSTEM_TYPE_FISHERY.equals(type))
datasetsIds.add(id);
}
}
@ -476,6 +481,8 @@ public class GrsfPublisherFisheryService {
// replace fields
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking update method..");
@ -484,7 +491,7 @@ public class GrsfPublisherFisheryService {
apiKey,
catalogId,
title, name,
organization,
publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer(),

View File

@ -29,7 +29,6 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.utils.Caller;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.StockRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.DeleteProductBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean;
@ -40,6 +39,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
@ -111,6 +111,9 @@ public class GrsfPublisherStockService {
// check it has admin role or throw exception
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
// extend this role to the other organizations in this context
CommonServiceUtils.extendRoleToOtherOrganizations(username, catalogue, organization, RolesCkanGroupOrOrg.ADMIN);
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
@ -170,6 +173,8 @@ public class GrsfPublisherStockService {
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking create method..");
@ -178,7 +183,7 @@ public class GrsfPublisherStockService {
apiKey,
futureTitle,
futureName,
organization,
publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer() == null? authorFullname : record.getMaintainer(),
@ -256,10 +261,10 @@ public class GrsfPublisherStockService {
}
// check it is in the right source and it is a stock
String grsfTypeValue = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
String systemTypeValue = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
String systemType = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.STOCK.getOrigName().equals(grsfTypeValue)){
if(stockInCkan.getOrganization().getName().equalsIgnoreCase(source) &&
CommonServiceUtils.SYSTEM_TYPE_STOCK.equals(systemType)){
logger.debug("Ok, this is a stock of the right type, removing it");
boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), apiKey, true);
@ -314,12 +319,12 @@ public class GrsfPublisherStockService {
if(sourceInPath.equals(Sources.GRSF))
datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "stock", catalogue);
else{
List<String> fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue);
List<String> fullGroupListIds = HelperMethods.getProductsInOrganization(source, catalogue);
for (String id : fullGroupListIds) {
CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username));
if(dataset != null){
String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
if(grsfType.equals(Product_Type.STOCK.getOrigName()))
String type = dataset.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
if(CommonServiceUtils.SYSTEM_TYPE_STOCK.equals(type))
datasetsIds.add(id);
}
}
@ -484,6 +489,8 @@ public class GrsfPublisherStockService {
// replace fields
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking update method..");
@ -492,7 +499,7 @@ public class GrsfPublisherStockService {
apiKey,
catalogId,
title, name,
organization,
publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer(),

View File

@ -4,7 +4,9 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.TimeSeriesBean;
import org.slf4j.LoggerFactory;
@ -26,10 +28,11 @@ public class CSVUtils {
* Write a time series to a csv file, and returns the file reference.<br>
* Please give the timeSeries already sorted per year
* @param timeSeries
* @param relevantSources
* @param <T>
* @param <T1>
*/
public static <T, T1> File listToCSV(List<TimeSeriesBean<T, T1>> timeSeries){
public static <T, T1> File listToCSV(List<TimeSeriesBean<T, T1>> timeSeries, String[] relevantSources){
if(timeSeries == null || timeSeries.isEmpty()){
logger.warn("The time series provided is null or empty ... " + timeSeries );
@ -44,12 +47,14 @@ public class CSVUtils {
FileOutputStream fo = new FileOutputStream(file);
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fo, "UTF-8"));
Set<String> sources = new HashSet<String>(3);
// discover how the header will look like
boolean isUnitPresent = false;
boolean isValuePresent = false;
boolean isSourcePresent = false;
boolean isAssessmentPresent = false;
for (TimeSeriesBean<T, T1> timeSeriesBean : timeSeries) {
if(timeSeriesBean.isSourcePresent())
isSourcePresent = true;
@ -78,7 +83,7 @@ public class CSVUtils {
headerLine.append(CSV_SEPARATOR);
headerLine.append(TimeSeriesBean.SOURCE_FIELD);
}
if(isAssessmentPresent){
headerLine.append(CSV_SEPARATOR);
headerLine.append(TimeSeriesBean.ASSESSMENT_FIELD);
@ -107,8 +112,10 @@ public class CSVUtils {
if(isSourcePresent){
oneLine.append(CSV_SEPARATOR);
oneLine.append(bean.getSource() != null? bean.getSource() : "");
if(bean.getSource() != null && !bean.getSource().isEmpty())
sources.add(bean.getSource());
}
if(isAssessmentPresent){
oneLine.append(CSV_SEPARATOR);
oneLine.append(bean.getAssessment() != null? bean.getAssessment() : "");
@ -123,7 +130,12 @@ public class CSVUtils {
// file created
logger.debug("CSV file created correctly on this machine!");
for (String source : sources) {
relevantSources[0] += source + "_";
}
relevantSources[0] = relevantSources[0].substring(0, relevantSources[0].length() - 1);
// on exit delete it...
file.deleteOnExit();
return file;
@ -133,6 +145,6 @@ public class CSVUtils {
return null;
}
}
}

View File

@ -52,9 +52,12 @@ public class CommonServiceUtils {
private static final int TIME_SERIES_TAKE_LAST_VALUES = 5;
private static final String REGEX_TAGS = "[^\\s\\w-_.]";
public static final String SYSTEM_TYPE = "system:type";
public static final String SYSTEM_TYPE_STOCK = "GRSF Stock";
public static final String SYSTEM_TYPE_FISHERY = "GRSF Fishery";
// item url property
public static final String ITEM_URL_FIELD = "GRSF Record URL";
private static final String GRSF_ADMIN_ORGANIZATION_NAME = "grsf_admin";
/**
* Retrieve the list of licenses for stocks and fisheries
@ -450,7 +453,7 @@ public class CommonServiceUtils {
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath);
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
customFields.put(CommonServiceUtils.SYSTEM_TYPE, productType.equals(Product_Type.STOCK)? Arrays.asList(SYSTEM_TYPE_STOCK) : Arrays.asList(SYSTEM_TYPE_FISHERY));
}
/**
@ -568,4 +571,33 @@ public class CommonServiceUtils {
}
}).start();
}
/**
* Extend roles to other organization
* @param username
* @param catalogue
* @param organization
* @param admin
*/
public static void extendRoleToOtherOrganizations(String username,
DataCatalogue catalogue, String organization,
RolesCkanGroupOrOrg admin) {
logger.debug("Checking if role extension is needed here");
catalogue.assignRolesOtherOrganization(username, organization, admin);
}
/**
* Evaluate in which organization a record has to be published. The only expcetion is when grsf_admin is involved.
* @param organization
* @param sourceInPath
* @return
*/
public static String evaluateOrganization(String organization, Sources sourceInPath) {
if(sourceInPath.equals(Sources.GRSF) && organization.equals(GRSF_ADMIN_ORGANIZATION_NAME))
return GRSF_ADMIN_ORGANIZATION_NAME;
else
return sourceInPath.getOrigName().toLowerCase();
}
}

View File

@ -493,6 +493,21 @@ public abstract class HelperMethods {
* @throws SQLException
* @throws ClassNotFoundException
*/
public static List<String> getProductsInOrganization(String groupName,
DataCatalogue catalogue) throws ClassNotFoundException, SQLException {
return catalogue.getProductsIdsInGroupOrOrg(groupName, true, 0, Integer.MAX_VALUE);
}
/**
* Retrieve the identifiers of the products in a given organization. It doesn't use CKAN API because they would return at most 1000 ids.
* @param string
* @param catalogue
* @return
* @throws SQLException
* @throws ClassNotFoundException
*/
public static List<String> getProductsInGroup(String groupName,
DataCatalogue catalogue) throws ClassNotFoundException, SQLException {

View File

@ -194,7 +194,8 @@ public class ManageTimeSeriesThread extends Thread{
CkanResourceBase ckanResource = null;
ExternalFile createdFileOnWorkspace = null;
File csvFile = CSVUtils.listToCSV(asList);
String[] relevantSources = new String[1];
File csvFile = CSVUtils.listToCSV(asList, relevantSources);
if(csvFile != null){
for (int i = 0; i < CHANCES; i++) {
@ -207,7 +208,8 @@ public class ManageTimeSeriesThread extends Thread{
if(ckanResource != null){
if(createdFileOnWorkspace == null)
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + (relevantSources[0] != null ? relevantSources[0] + "_" : "")
+ customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
if(createdFileOnWorkspace != null){
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);

View File

@ -233,7 +233,7 @@ public class JTests {
Collections.sort(timeSeries);
File csvFile = CSVUtils.listToCSV(timeSeries);
File csvFile = CSVUtils.listToCSV(timeSeries, null);
// send file
instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null);