diff --git a/pom.xml b/pom.xml
index 669e3af..3374c52 100644
--- a/pom.xml
+++ b/pom.xml
@@ -11,7 +11,7 @@
org.gcube.data-catalogue
grsf-publisher-ws
- 1.3.0-SNAPSHOT
+ 1.4.0-SNAPSHOT
war
grsf-publisher-ws
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java
index 3f9a1a9..a72e90e 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java
@@ -115,7 +115,22 @@ public class FisheryRecord extends Common{
}
public void setFisheryId(String fisheryId) {
- this.fisheryId = fisheryId;
+ String fisheryIdmodified = "";
+ if(fisheryId != null){
+
+ String[] splitCodesValues = fisheryId.split("+");
+ for (int i = 0; i < splitCodesValues.length; i++) {
+ String prefixAndCode = splitCodesValues[i];
+ String prefix = prefixAndCode.split(":")[0];
+ String code = prefixAndCode.split(":")[1];
+ fisheryIdmodified += prefix.toLowerCase() + ":" + code.toUpperCase();
+ if(splitCodesValues.length > 1 && i < (splitCodesValues.length - 1))
+ fisheryIdmodified += "+";
+ }
+ }
+
+ this.fisheryId = fisheryIdmodified;
+
}
public List getFishingArea() {
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java
index 085e5b3..c6df2a2 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java
@@ -180,7 +180,22 @@ public class StockRecord extends Common{
}
public void setStockId(String stockId) {
- this.stockId = stockId;
+
+ String stockIdmodified = "";
+ if(stockId != null){
+
+ String[] splitCodesValues = stockId.split("+");
+ for (int i = 0; i < splitCodesValues.length; i++) {
+ String prefixAndCode = splitCodesValues[i];
+ String prefix = prefixAndCode.split(":")[0];
+ String code = prefixAndCode.split(":")[1];
+ stockIdmodified += prefix.toLowerCase() + ":" + code.toUpperCase();
+ if(splitCodesValues.length > 1 && i < (splitCodesValues.length - 1))
+ stockIdmodified += "+";
+ }
+ }
+
+ this.stockId = stockIdmodified;
}
public List getArea() {
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java
index b068305..0f069ac 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java
@@ -29,7 +29,6 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.utils.Caller;
import org.gcube.common.scope.api.ScopeProvider;
-import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.FisheryRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.DeleteProductBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean;
@@ -40,6 +39,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
+import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
@@ -109,13 +109,16 @@ public class GrsfPublisherFisheryService {
String apiKey = catalogue.getApiKeyFromUsername(username);
String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
+
+ // extend this role to the other organizations in this context
+ CommonServiceUtils.extendRoleToOtherOrganizations(username, catalogue, organization, RolesCkanGroupOrOrg.ADMIN);
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
String authorFullname = HelperMethods.getUserFullname(context, token);
if(authorMail == null || authorFullname == null){
- throw new Exception("Sorry but there was not possible to retrieve your fullname/email!");
+ throw new Exception("Sorry but it was not possible to retrieve your fullname/email!");
}
// The name of the product will be the uuid of the kb. The title will be the fishery's fishery_name. Fishery has also the constraint that
@@ -155,6 +158,8 @@ public class GrsfPublisherFisheryService {
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
+
+ String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking create method..");
@@ -163,7 +168,7 @@ public class GrsfPublisherFisheryService {
apiKey,
futureTitle,
futureName,
- organization,
+ publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer() == null? authorFullname : record.getMaintainer(),
@@ -241,10 +246,10 @@ public class GrsfPublisherFisheryService {
}
// check it is in the right source and it is a fishery
- String grsfTypeValue = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
- String systemTypeValue = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
+ String systemType = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
- if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.FISHERY.getOrigName().equals(grsfTypeValue)){
+ if(fisheryInCkan.getOrganization().getName().equalsIgnoreCase(source) &&
+ CommonServiceUtils.SYSTEM_TYPE_FISHERY.equals(systemType)){
logger.debug("Ok, this is a fishery of the right source, removing it");
boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), apiKey, true);
@@ -300,13 +305,13 @@ public class GrsfPublisherFisheryService {
if(sourceInPath.equals(Sources.GRSF))
datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "fishery", catalogue);
else{
- List fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue);
+ List fullGroupListIds = HelperMethods.getProductsInOrganization(source, catalogue);
String apiKey = catalogue.getApiKeyFromUsername(username);
for (String id : fullGroupListIds) {
CkanDataset dataset = catalogue.getDataset(id, apiKey);
if(dataset != null){
- String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
- if(grsfType.equals(Product_Type.FISHERY.getOrigName()))
+ String type = dataset.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
+ if(CommonServiceUtils.SYSTEM_TYPE_FISHERY.equals(type))
datasetsIds.add(id);
}
}
@@ -476,6 +481,8 @@ public class GrsfPublisherFisheryService {
// replace fields
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
+
+ String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking update method..");
@@ -484,7 +491,7 @@ public class GrsfPublisherFisheryService {
apiKey,
catalogId,
title, name,
- organization,
+ publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer(),
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java
index 0f88d31..5c6db52 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java
@@ -29,7 +29,6 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.utils.Caller;
import org.gcube.common.scope.api.ScopeProvider;
-import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.StockRecord;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.DeleteProductBean;
import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean;
@@ -40,6 +39,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
+import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
@@ -111,6 +111,9 @@ public class GrsfPublisherStockService {
// check it has admin role or throw exception
CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization);
+
+ // extend this role to the other organizations in this context
+ CommonServiceUtils.extendRoleToOtherOrganizations(username, catalogue, organization, RolesCkanGroupOrOrg.ADMIN);
// retrieve the user's email and fullname
String authorMail = HelperMethods.getUserEmail(context, token);
@@ -170,6 +173,8 @@ public class GrsfPublisherStockService {
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
+
+ String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking create method..");
@@ -178,7 +183,7 @@ public class GrsfPublisherStockService {
apiKey,
futureTitle,
futureName,
- organization,
+ publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer() == null? authorFullname : record.getMaintainer(),
@@ -256,10 +261,10 @@ public class GrsfPublisherStockService {
}
// check it is in the right source and it is a stock
- String grsfTypeValue = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
- String systemTypeValue = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
+ String systemType = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
- if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.STOCK.getOrigName().equals(grsfTypeValue)){
+ if(stockInCkan.getOrganization().getName().equalsIgnoreCase(source) &&
+ CommonServiceUtils.SYSTEM_TYPE_STOCK.equals(systemType)){
logger.debug("Ok, this is a stock of the right type, removing it");
boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), apiKey, true);
@@ -314,12 +319,12 @@ public class GrsfPublisherStockService {
if(sourceInPath.equals(Sources.GRSF))
datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "stock", catalogue);
else{
- List fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue);
+ List fullGroupListIds = HelperMethods.getProductsInOrganization(source, catalogue);
for (String id : fullGroupListIds) {
CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username));
if(dataset != null){
- String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY);
- if(grsfType.equals(Product_Type.STOCK.getOrigName()))
+ String type = dataset.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE);
+ if(CommonServiceUtils.SYSTEM_TYPE_STOCK.equals(type))
datasetsIds.add(id);
}
}
@@ -484,6 +489,8 @@ public class GrsfPublisherStockService {
// replace fields
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
+
+ String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath);
logger.info("Invoking update method..");
@@ -492,7 +499,7 @@ public class GrsfPublisherStockService {
apiKey,
catalogId,
title, name,
- organization,
+ publishInOrganization,
authorFullname,
authorMail,
record.getMaintainer(),
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java
index eea3058..4a7e507 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java
@@ -4,7 +4,9 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.TimeSeriesBean;
import org.slf4j.LoggerFactory;
@@ -26,10 +28,11 @@ public class CSVUtils {
* Write a time series to a csv file, and returns the file reference.
* Please give the timeSeries already sorted per year
* @param timeSeries
+ * @param relevantSources
* @param
* @param
*/
- public static File listToCSV(List> timeSeries){
+ public static File listToCSV(List> timeSeries, String[] relevantSources){
if(timeSeries == null || timeSeries.isEmpty()){
logger.warn("The time series provided is null or empty ... " + timeSeries );
@@ -44,12 +47,14 @@ public class CSVUtils {
FileOutputStream fo = new FileOutputStream(file);
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fo, "UTF-8"));
+ Set sources = new HashSet(3);
+
// discover how the header will look like
boolean isUnitPresent = false;
boolean isValuePresent = false;
boolean isSourcePresent = false;
boolean isAssessmentPresent = false;
-
+
for (TimeSeriesBean timeSeriesBean : timeSeries) {
if(timeSeriesBean.isSourcePresent())
isSourcePresent = true;
@@ -78,7 +83,7 @@ public class CSVUtils {
headerLine.append(CSV_SEPARATOR);
headerLine.append(TimeSeriesBean.SOURCE_FIELD);
}
-
+
if(isAssessmentPresent){
headerLine.append(CSV_SEPARATOR);
headerLine.append(TimeSeriesBean.ASSESSMENT_FIELD);
@@ -107,8 +112,10 @@ public class CSVUtils {
if(isSourcePresent){
oneLine.append(CSV_SEPARATOR);
oneLine.append(bean.getSource() != null? bean.getSource() : "");
+ if(bean.getSource() != null && !bean.getSource().isEmpty())
+ sources.add(bean.getSource());
}
-
+
if(isAssessmentPresent){
oneLine.append(CSV_SEPARATOR);
oneLine.append(bean.getAssessment() != null? bean.getAssessment() : "");
@@ -123,7 +130,12 @@ public class CSVUtils {
// file created
logger.debug("CSV file created correctly on this machine!");
-
+
+ for (String source : sources) {
+ relevantSources[0] += source + "_";
+ }
+ relevantSources[0] = relevantSources[0].substring(0, relevantSources[0].length() - 1);
+
// on exit delete it...
file.deleteOnExit();
return file;
@@ -133,6 +145,6 @@ public class CSVUtils {
return null;
}
}
-
+
}
\ No newline at end of file
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java
index 5b67a42..7058e2a 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java
@@ -52,9 +52,12 @@ public class CommonServiceUtils {
private static final int TIME_SERIES_TAKE_LAST_VALUES = 5;
private static final String REGEX_TAGS = "[^\\s\\w-_.]";
public static final String SYSTEM_TYPE = "system:type";
+ public static final String SYSTEM_TYPE_STOCK = "GRSF Stock";
+ public static final String SYSTEM_TYPE_FISHERY = "GRSF Fishery";
// item url property
public static final String ITEM_URL_FIELD = "GRSF Record URL";
+ private static final String GRSF_ADMIN_ORGANIZATION_NAME = "grsf_admin";
/**
* Retrieve the list of licenses for stocks and fisheries
@@ -450,7 +453,7 @@ public class CommonServiceUtils {
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath);
// add the SYSTEM_TYPE
- customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
+ customFields.put(CommonServiceUtils.SYSTEM_TYPE, productType.equals(Product_Type.STOCK)? Arrays.asList(SYSTEM_TYPE_STOCK) : Arrays.asList(SYSTEM_TYPE_FISHERY));
}
/**
@@ -568,4 +571,33 @@ public class CommonServiceUtils {
}
}).start();
}
+
+ /**
+ * Extend roles to other organization
+ * @param username
+ * @param catalogue
+ * @param organization
+ * @param admin
+ */
+ public static void extendRoleToOtherOrganizations(String username,
+ DataCatalogue catalogue, String organization,
+ RolesCkanGroupOrOrg admin) {
+
+ logger.debug("Checking if role extension is needed here");
+ catalogue.assignRolesOtherOrganization(username, organization, admin);
+
+ }
+
+ /**
+ * Evaluate in which organization a record has to be published. The only expcetion is when grsf_admin is involved.
+ * @param organization
+ * @param sourceInPath
+ * @return
+ */
+ public static String evaluateOrganization(String organization, Sources sourceInPath) {
+ if(sourceInPath.equals(Sources.GRSF) && organization.equals(GRSF_ADMIN_ORGANIZATION_NAME))
+ return GRSF_ADMIN_ORGANIZATION_NAME;
+ else
+ return sourceInPath.getOrigName().toLowerCase();
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
index da8d190..7fb3080 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java
@@ -493,6 +493,21 @@ public abstract class HelperMethods {
* @throws SQLException
* @throws ClassNotFoundException
*/
+ public static List getProductsInOrganization(String groupName,
+ DataCatalogue catalogue) throws ClassNotFoundException, SQLException {
+
+ return catalogue.getProductsIdsInGroupOrOrg(groupName, true, 0, Integer.MAX_VALUE);
+
+ }
+
+ /**
+ * Retrieve the identifiers of the products in a given organization. It doesn't use CKAN API because they would return at most 1000 ids.
+ * @param string
+ * @param catalogue
+ * @return
+ * @throws SQLException
+ * @throws ClassNotFoundException
+ */
public static List getProductsInGroup(String groupName,
DataCatalogue catalogue) throws ClassNotFoundException, SQLException {
diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
index 5e30844..9a0ed84 100644
--- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
+++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java
@@ -194,7 +194,8 @@ public class ManageTimeSeriesThread extends Thread{
CkanResourceBase ckanResource = null;
ExternalFile createdFileOnWorkspace = null;
- File csvFile = CSVUtils.listToCSV(asList);
+ String[] relevantSources = new String[1];
+ File csvFile = CSVUtils.listToCSV(asList, relevantSources);
if(csvFile != null){
for (int i = 0; i < CHANCES; i++) {
@@ -207,7 +208,8 @@ public class ManageTimeSeriesThread extends Thread{
if(ckanResource != null){
if(createdFileOnWorkspace == null)
- createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
+ createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + (relevantSources[0] != null ? relevantSources[0] + "_" : "")
+ + customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile);
if(createdFileOnWorkspace != null){
String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true);
diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java
index 7d490a4..0d774a7 100644
--- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java
+++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java
@@ -233,7 +233,7 @@ public class JTests {
Collections.sort(timeSeries);
- File csvFile = CSVUtils.listToCSV(timeSeries);
+ File csvFile = CSVUtils.listToCSV(timeSeries, null);
// send file
instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null);