From 0e63ab07bcaf37e20c90222a370f8da9c8512979 Mon Sep 17 00:00:00 2001 From: Costantino Perciante Date: Tue, 19 Sep 2017 14:42:15 +0000 Subject: [PATCH] semantic identifier is modified by the service, changed system:type to fishery and stocks. Other minor changes in resource names. Version is 1.4.0 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@153263 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 2 +- .../json/input/record/FisheryRecord.java | 17 +++++++++- .../json/input/record/StockRecord.java | 17 +++++++++- .../services/GrsfPublisherFisheryService.java | 27 +++++++++------ .../services/GrsfPublisherStockService.java | 25 +++++++++----- .../grsf_publish_ws/utils/CSVUtils.java | 24 +++++++++---- .../utils/CommonServiceUtils.java | 34 ++++++++++++++++++- .../grsf_publish_ws/utils/HelperMethods.java | 15 ++++++++ .../utils/threads/ManageTimeSeriesThread.java | 6 ++-- .../grsf_publish_ws/JTests.java | 2 +- 10 files changed, 137 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 669e3af..3374c52 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.gcube.data-catalogue grsf-publisher-ws - 1.3.0-SNAPSHOT + 1.4.0-SNAPSHOT war grsf-publisher-ws diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java index 3f9a1a9..a72e90e 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java @@ -115,7 +115,22 @@ public class FisheryRecord extends Common{ } public void setFisheryId(String fisheryId) { - this.fisheryId = fisheryId; + String fisheryIdmodified = ""; + if(fisheryId != null){ + + String[] splitCodesValues = fisheryId.split("+"); + for (int i = 0; i < splitCodesValues.length; i++) { + String prefixAndCode = splitCodesValues[i]; + String prefix = prefixAndCode.split(":")[0]; + String code = prefixAndCode.split(":")[1]; + fisheryIdmodified += prefix.toLowerCase() + ":" + code.toUpperCase(); + if(splitCodesValues.length > 1 && i < (splitCodesValues.length - 1)) + fisheryIdmodified += "+"; + } + } + + this.fisheryId = fisheryIdmodified; + } public List getFishingArea() { diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java index 085e5b3..c6df2a2 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java @@ -180,7 +180,22 @@ public class StockRecord extends Common{ } public void setStockId(String stockId) { - this.stockId = stockId; + + String stockIdmodified = ""; + if(stockId != null){ + + String[] splitCodesValues = stockId.split("+"); + for (int i = 0; i < splitCodesValues.length; i++) { + String prefixAndCode = splitCodesValues[i]; + String prefix = prefixAndCode.split(":")[0]; + String code = prefixAndCode.split(":")[1]; + stockIdmodified += prefix.toLowerCase() + ":" + code.toUpperCase(); + if(splitCodesValues.length > 1 && i < (splitCodesValues.length - 1)) + stockIdmodified += "+"; + } + } + + this.stockId = stockIdmodified; } public List getArea() { diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java index b068305..0f069ac 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java @@ -29,7 +29,6 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.utils.Caller; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.FisheryRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.DeleteProductBean; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean; @@ -40,6 +39,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; +import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.model.CkanDataset; @@ -109,13 +109,16 @@ public class GrsfPublisherFisheryService { String apiKey = catalogue.getApiKeyFromUsername(username); String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin"; CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); + + // extend this role to the other organizations in this context + CommonServiceUtils.extendRoleToOtherOrganizations(username, catalogue, organization, RolesCkanGroupOrOrg.ADMIN); // retrieve the user's email and fullname String authorMail = HelperMethods.getUserEmail(context, token); String authorFullname = HelperMethods.getUserFullname(context, token); if(authorMail == null || authorFullname == null){ - throw new Exception("Sorry but there was not possible to retrieve your fullname/email!"); + throw new Exception("Sorry but it was not possible to retrieve your fullname/email!"); } // The name of the product will be the uuid of the kb. The title will be the fishery's fishery_name. Fishery has also the constraint that @@ -155,6 +158,8 @@ public class GrsfPublisherFisheryService { throw new Exception("Failed to retrieve the namespaces for the key fields!"); customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath); logger.info("Invoking create method.."); @@ -163,7 +168,7 @@ public class GrsfPublisherFisheryService { apiKey, futureTitle, futureName, - organization, + publishInOrganization, authorFullname, authorMail, record.getMaintainer() == null? authorFullname : record.getMaintainer(), @@ -241,10 +246,10 @@ public class GrsfPublisherFisheryService { } // check it is in the right source and it is a fishery - String grsfTypeValue = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); - String systemTypeValue = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); + String systemType = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); - if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.FISHERY.getOrigName().equals(grsfTypeValue)){ + if(fisheryInCkan.getOrganization().getName().equalsIgnoreCase(source) && + CommonServiceUtils.SYSTEM_TYPE_FISHERY.equals(systemType)){ logger.debug("Ok, this is a fishery of the right source, removing it"); boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), apiKey, true); @@ -300,13 +305,13 @@ public class GrsfPublisherFisheryService { if(sourceInPath.equals(Sources.GRSF)) datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "fishery", catalogue); else{ - List fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue); + List fullGroupListIds = HelperMethods.getProductsInOrganization(source, catalogue); String apiKey = catalogue.getApiKeyFromUsername(username); for (String id : fullGroupListIds) { CkanDataset dataset = catalogue.getDataset(id, apiKey); if(dataset != null){ - String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); - if(grsfType.equals(Product_Type.FISHERY.getOrigName())) + String type = dataset.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); + if(CommonServiceUtils.SYSTEM_TYPE_FISHERY.equals(type)) datasetsIds.add(id); } } @@ -476,6 +481,8 @@ public class GrsfPublisherFisheryService { // replace fields customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath); logger.info("Invoking update method.."); @@ -484,7 +491,7 @@ public class GrsfPublisherFisheryService { apiKey, catalogId, title, name, - organization, + publishInOrganization, authorFullname, authorMail, record.getMaintainer(), diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java index 0f88d31..5c6db52 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java @@ -29,7 +29,6 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.utils.Caller; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.StockRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.DeleteProductBean; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean; @@ -40,6 +39,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; +import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.model.CkanDataset; @@ -111,6 +111,9 @@ public class GrsfPublisherStockService { // check it has admin role or throw exception CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); + + // extend this role to the other organizations in this context + CommonServiceUtils.extendRoleToOtherOrganizations(username, catalogue, organization, RolesCkanGroupOrOrg.ADMIN); // retrieve the user's email and fullname String authorMail = HelperMethods.getUserEmail(context, token); @@ -170,6 +173,8 @@ public class GrsfPublisherStockService { throw new Exception("Failed to retrieve the namespaces for the key fields!"); customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath); logger.info("Invoking create method.."); @@ -178,7 +183,7 @@ public class GrsfPublisherStockService { apiKey, futureTitle, futureName, - organization, + publishInOrganization, authorFullname, authorMail, record.getMaintainer() == null? authorFullname : record.getMaintainer(), @@ -256,10 +261,10 @@ public class GrsfPublisherStockService { } // check it is in the right source and it is a stock - String grsfTypeValue = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); - String systemTypeValue = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); + String systemType = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); - if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.STOCK.getOrigName().equals(grsfTypeValue)){ + if(stockInCkan.getOrganization().getName().equalsIgnoreCase(source) && + CommonServiceUtils.SYSTEM_TYPE_STOCK.equals(systemType)){ logger.debug("Ok, this is a stock of the right type, removing it"); boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), apiKey, true); @@ -314,12 +319,12 @@ public class GrsfPublisherStockService { if(sourceInPath.equals(Sources.GRSF)) datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "stock", catalogue); else{ - List fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue); + List fullGroupListIds = HelperMethods.getProductsInOrganization(source, catalogue); for (String id : fullGroupListIds) { CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username)); if(dataset != null){ - String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); - if(grsfType.equals(Product_Type.STOCK.getOrigName())) + String type = dataset.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); + if(CommonServiceUtils.SYSTEM_TYPE_STOCK.equals(type)) datasetsIds.add(id); } } @@ -484,6 +489,8 @@ public class GrsfPublisherStockService { // replace fields customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + String publishInOrganization = CommonServiceUtils.evaluateOrganization(organization, sourceInPath); logger.info("Invoking update method.."); @@ -492,7 +499,7 @@ public class GrsfPublisherStockService { apiKey, catalogId, title, name, - organization, + publishInOrganization, authorFullname, authorMail, record.getMaintainer(), diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java index eea3058..4a7e507 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVUtils.java @@ -4,7 +4,9 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStreamWriter; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.gcube.data_catalogue.grsf_publish_ws.json.input.utils.TimeSeriesBean; import org.slf4j.LoggerFactory; @@ -26,10 +28,11 @@ public class CSVUtils { * Write a time series to a csv file, and returns the file reference.
* Please give the timeSeries already sorted per year * @param timeSeries + * @param relevantSources * @param * @param */ - public static File listToCSV(List> timeSeries){ + public static File listToCSV(List> timeSeries, String[] relevantSources){ if(timeSeries == null || timeSeries.isEmpty()){ logger.warn("The time series provided is null or empty ... " + timeSeries ); @@ -44,12 +47,14 @@ public class CSVUtils { FileOutputStream fo = new FileOutputStream(file); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fo, "UTF-8")); + Set sources = new HashSet(3); + // discover how the header will look like boolean isUnitPresent = false; boolean isValuePresent = false; boolean isSourcePresent = false; boolean isAssessmentPresent = false; - + for (TimeSeriesBean timeSeriesBean : timeSeries) { if(timeSeriesBean.isSourcePresent()) isSourcePresent = true; @@ -78,7 +83,7 @@ public class CSVUtils { headerLine.append(CSV_SEPARATOR); headerLine.append(TimeSeriesBean.SOURCE_FIELD); } - + if(isAssessmentPresent){ headerLine.append(CSV_SEPARATOR); headerLine.append(TimeSeriesBean.ASSESSMENT_FIELD); @@ -107,8 +112,10 @@ public class CSVUtils { if(isSourcePresent){ oneLine.append(CSV_SEPARATOR); oneLine.append(bean.getSource() != null? bean.getSource() : ""); + if(bean.getSource() != null && !bean.getSource().isEmpty()) + sources.add(bean.getSource()); } - + if(isAssessmentPresent){ oneLine.append(CSV_SEPARATOR); oneLine.append(bean.getAssessment() != null? bean.getAssessment() : ""); @@ -123,7 +130,12 @@ public class CSVUtils { // file created logger.debug("CSV file created correctly on this machine!"); - + + for (String source : sources) { + relevantSources[0] += source + "_"; + } + relevantSources[0] = relevantSources[0].substring(0, relevantSources[0].length() - 1); + // on exit delete it... file.deleteOnExit(); return file; @@ -133,6 +145,6 @@ public class CSVUtils { return null; } } - + } \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java index 5b67a42..7058e2a 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java @@ -52,9 +52,12 @@ public class CommonServiceUtils { private static final int TIME_SERIES_TAKE_LAST_VALUES = 5; private static final String REGEX_TAGS = "[^\\s\\w-_.]"; public static final String SYSTEM_TYPE = "system:type"; + public static final String SYSTEM_TYPE_STOCK = "GRSF Stock"; + public static final String SYSTEM_TYPE_FISHERY = "GRSF Fishery"; // item url property public static final String ITEM_URL_FIELD = "GRSF Record URL"; + private static final String GRSF_ADMIN_ORGANIZATION_NAME = "grsf_admin"; /** * Retrieve the list of licenses for stocks and fisheries @@ -450,7 +453,7 @@ public class CommonServiceUtils { CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath); // add the SYSTEM_TYPE - customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); + customFields.put(CommonServiceUtils.SYSTEM_TYPE, productType.equals(Product_Type.STOCK)? Arrays.asList(SYSTEM_TYPE_STOCK) : Arrays.asList(SYSTEM_TYPE_FISHERY)); } /** @@ -568,4 +571,33 @@ public class CommonServiceUtils { } }).start(); } + + /** + * Extend roles to other organization + * @param username + * @param catalogue + * @param organization + * @param admin + */ + public static void extendRoleToOtherOrganizations(String username, + DataCatalogue catalogue, String organization, + RolesCkanGroupOrOrg admin) { + + logger.debug("Checking if role extension is needed here"); + catalogue.assignRolesOtherOrganization(username, organization, admin); + + } + + /** + * Evaluate in which organization a record has to be published. The only expcetion is when grsf_admin is involved. + * @param organization + * @param sourceInPath + * @return + */ + public static String evaluateOrganization(String organization, Sources sourceInPath) { + if(sourceInPath.equals(Sources.GRSF) && organization.equals(GRSF_ADMIN_ORGANIZATION_NAME)) + return GRSF_ADMIN_ORGANIZATION_NAME; + else + return sourceInPath.getOrigName().toLowerCase(); + } } \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java index da8d190..7fb3080 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java @@ -493,6 +493,21 @@ public abstract class HelperMethods { * @throws SQLException * @throws ClassNotFoundException */ + public static List getProductsInOrganization(String groupName, + DataCatalogue catalogue) throws ClassNotFoundException, SQLException { + + return catalogue.getProductsIdsInGroupOrOrg(groupName, true, 0, Integer.MAX_VALUE); + + } + + /** + * Retrieve the identifiers of the products in a given organization. It doesn't use CKAN API because they would return at most 1000 ids. + * @param string + * @param catalogue + * @return + * @throws SQLException + * @throws ClassNotFoundException + */ public static List getProductsInGroup(String groupName, DataCatalogue catalogue) throws ClassNotFoundException, SQLException { diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index 5e30844..9a0ed84 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -194,7 +194,8 @@ public class ManageTimeSeriesThread extends Thread{ CkanResourceBase ckanResource = null; ExternalFile createdFileOnWorkspace = null; - File csvFile = CSVUtils.listToCSV(asList); + String[] relevantSources = new String[1]; + File csvFile = CSVUtils.listToCSV(asList, relevantSources); if(csvFile != null){ for (int i = 0; i < CHANCES; i++) { @@ -207,7 +208,8 @@ public class ManageTimeSeriesThread extends Thread{ if(ckanResource != null){ if(createdFileOnWorkspace == null) - createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); + createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, csvFileName + "_" + (relevantSources[0] != null ? relevantSources[0] + "_" : "") + + customAnnotation.key() + CSVUtils.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); if(createdFileOnWorkspace != null){ String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index 7d490a4..0d774a7 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -233,7 +233,7 @@ public class JTests { Collections.sort(timeSeries); - File csvFile = CSVUtils.listToCSV(timeSeries); + File csvFile = CSVUtils.listToCSV(timeSeries, null); // send file instance.uploadResourceFile(csvFile, datasetName, instance.getApiKeyFromUsername("costantino.perciante"), "random_name.csv", null, null, null);