From 93b9e3a25ee73f803cb5c40eff49c499888c73aa Mon Sep 17 00:00:00 2001 From: Costantino Perciante Date: Wed, 12 Jul 2017 16:05:35 +0000 Subject: [PATCH] added update method. Minor refactor for common code git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@151029 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../grsf_publish_ws/json/input/Base.java | 31 +- .../grsf_publish_ws/json/input/Common.java | 8 +- .../json/input/FisheryRecord.java | 7 +- .../json/output/ResponseBean.java | 3 +- .../json/output/ResponseCreationBean.java | 6 +- .../services/CommonServiceUtils.java | 221 +++++++- .../services/GrsfPublisherFisheryService.java | 507 +++++++++--------- .../services/GrsfPublisherStockService.java | 494 ++++++++--------- .../grsf_publish_ws/utils/CSVHelpers.java | 22 +- .../grsf_publish_ws/utils/HelperMethods.java | 113 ++-- .../threads/AssociationToGroupThread.java | 30 +- .../utils/threads/ManageTimeSeriesThread.java | 11 +- src/main/webapp/WEB-INF/web.xml | 2 +- .../grsf_publish_ws/JTests.java | 2 +- 14 files changed, 819 insertions(+), 638 deletions(-) diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java index d7a975e..d46fc6c 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java @@ -28,6 +28,9 @@ public class Base { private static Logger logger = LoggerFactory.getLogger(Base.class); public static final String UUID_KB_KEY = "UUID Knowledge Base"; + @JsonProperty("catalog_id") //used on patch/update product call + private String catalogId; + @JsonProperty("description") private String description; @@ -68,27 +71,29 @@ public class Base { } /** + * @param id * @param description * @param license * @param author - * @param version * @param authorContact + * @param version * @param maintainer * @param maintainerContact * @param extrasFields * @param extrasResources * @param uuid */ - public Base(String description, String license, String author, - Long version, String authorContact, String maintainer, + public Base(String id, String description, String license, String author, + String authorContact, Long version, String maintainer, String maintainerContact, Map> extrasFields, List extrasResources, String uuid) { super(); + this.catalogId = id; this.description = description; this.license = license; this.author = author; - this.version = version; this.authorContact = authorContact; + this.version = version; this.maintainer = maintainer; this.maintainerContact = maintainerContact; this.extrasFields = extrasFields; @@ -96,6 +101,16 @@ public class Base { this.uuid = uuid; } + + + public String getCatalogId() { + return catalogId; + } + + public void setCatalogId(String catalogId) { + this.catalogId = catalogId; + } + public String getDescription() { return description; } @@ -197,12 +212,12 @@ public class Base { @Override public String toString() { - return "Base [description=" + description + ", license=" + license - + ", author=" + author + ", version=" + version - + ", authorContact=" + authorContact + ", maintainer=" + return "Base [catalogId=" + catalogId + ", description=" + description + ", license=" + + license + ", author=" + author + ", authorContact=" + + authorContact + ", version=" + version + ", maintainer=" + maintainer + ", maintainerContact=" + maintainerContact + ", extrasFields=" + extrasFields + ", extrasResources=" + extrasResources + ", uuid=" + uuid + "]"; } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java index c484c36..49d4302 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java @@ -24,6 +24,7 @@ public class Common extends Base{ public static final String GRSF_TYPE_KEY = "GRSF type"; // stock, fishery public static final String SOURCE_KEY = "Source"; // in case it is a RAM/FIRMS/FishSource record it is not added + // it is added in case of GRSF record @JsonProperty("data_owner") @CustomField(key="Data owner") @@ -50,13 +51,12 @@ public class Common extends Base{ @JsonProperty("traceability_flag") @CustomField(key="Traceability Flag") - @Group(condition="true", groupNameOverValue="traceability-flag") // record is added to group if Traceability Flag is true + @Group(condition="true", groupNameOverValue="traceability-flag") // record is added to group traceability-flag if Traceability Flag is true private Boolean traceabilityFlag; @JsonProperty("status") @CustomField(key="Status") @Group - // @Tag private Status status; @JsonProperty("reporting_year") @@ -70,8 +70,6 @@ public class Common extends Base{ // automatically compiled @JsonProperty("grsf_type") @CustomField(key=GRSF_TYPE_KEY) - //@Tag - //@Group private String grsfType; // automatically compiled @@ -274,4 +272,4 @@ public class Common extends Base{ + ", species=" + species + "]"; } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java index 251726a..7e8d9b2 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java @@ -33,7 +33,6 @@ public class FisheryRecord extends Common{ @JsonProperty("fishing_area") @CustomField(key="Fishing area") - //@Tag private List fishingArea; @JsonProperty("exploited_stocks") @@ -42,7 +41,6 @@ public class FisheryRecord extends Common{ @JsonProperty("management_entity") @CustomField(key="Management entity") - //@Tag private List managementEntity; @JsonProperty("jurisdiction_area") @@ -51,7 +49,6 @@ public class FisheryRecord extends Common{ private List jurisdictionArea; @JsonProperty("production_system_type") - //@Group @Tag @CustomField(key="Production system type") private List productionSystemType; @@ -62,7 +59,6 @@ public class FisheryRecord extends Common{ private List flagState; @JsonProperty("fishing_gear") - //@Tag @CustomField(key="Fishing gear") private List fishingGear; @@ -76,7 +72,6 @@ public class FisheryRecord extends Common{ @JsonProperty("type") @CustomField(key="Type") - //@Tag @Group private Fishery_Type type; @@ -229,4 +224,4 @@ public class FisheryRecord extends Common{ + ", type=" + type + "]"; } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseBean.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseBean.java index 6ecdb63..9d5741c 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseBean.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseBean.java @@ -18,7 +18,6 @@ public class ResponseBean{ * @param success * @param message * @param result - * @param help */ public ResponseBean(boolean success, String message, Object result) { super(); @@ -62,4 +61,4 @@ public class ResponseBean{ return "ResponseBean [success=" + success + ", message=" + message + ", result=" + result + "]"; } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseCreationBean.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseCreationBean.java index 3b1b6d4..553fd61 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseCreationBean.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/output/ResponseCreationBean.java @@ -3,7 +3,7 @@ package org.gcube.data_catalogue.grsf_publish_ws.json.output; import com.fasterxml.jackson.annotation.JsonProperty; /** - * A bean used to reply to a product creation method. + * A bean used to reply to a product creation/patch method. * @author Costantino Perciante at ISTI-CNR */ public class ResponseCreationBean { @@ -30,7 +30,7 @@ public class ResponseCreationBean { * @param productUrl * @param error */ - public ResponseCreationBean(String id, String kbUuid, String productUrl, + public ResponseCreationBean(String id, String kbUuid, String itemUrl, String error) { super(); this.id = id; @@ -77,4 +77,4 @@ public class ResponseCreationBean { + ", itemUrl=" + itemUrl + ", error=" + error + "]"; } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java index 799fa4f..fd580ce 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java @@ -3,11 +3,16 @@ package org.gcube.data_catalogue.grsf_publish_ws.services; import java.beans.PropertyDescriptor; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import javax.servlet.ServletContext; + import org.gcube.common.scope.api.ScopeProvider; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CkanResource; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; @@ -20,23 +25,35 @@ import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource; import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean; +import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean; import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; +import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status; +import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread; +import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.ManageTimeSeriesThread; +import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.WritePostCatalogueManagerThread; +import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; +import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.slf4j.LoggerFactory; +import eu.trentorise.opendata.jackan.model.CkanOrganization; + /** * Services common utils. * @author Costantino Perciante at ISTI-CNR */ +@SuppressWarnings({"rawtypes", "unchecked"}) public class CommonServiceUtils { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonServiceUtils.class); + public static final String DEFAULT_LICENSE = "CC-BY-SA-4.0"; + public static final String GRSF_GROUP_NAME = "grsf-group"; private static final int TIME_SERIES_TAKE_LAST_VALUES = 5; private static final String REGEX_TAGS = "[^\\s\\w-_.]"; public static final String SYSTEM_TYPE = "system:type"; - + // item url property public static final String ITEM_URL_FIELD = "UUID"; @@ -110,16 +127,23 @@ public class CommonServiceUtils { /** * Parse the record to look up tags, groups and resources * @param tags + * @param skipTags * @param groups + * @param skipGroups + * @param resources + * @param skipResources + * @param extras * @param record * @param username - * @param resources + * @param source */ public static void getTagsGroupsResourcesExtrasByRecord( Set tags, boolean skipTags, Set groups, + boolean skipGroups, List resources, + boolean skipResources, Map> extras, Base record, String username, @@ -133,18 +157,23 @@ public class CommonServiceUtils { if(!skipTags) getTagsByField(field, current, record, tags); - getGroupsByField(field, current, record, groups, source); + + if(!skipGroups) + getGroupsByField(field, current, record, groups, source); + getExtrasByField(field, current, record, extras); - getResourcesByField(field, current, record, username, resources); + + if(!skipResources) + getResourcesByField(field, current, record, username, resources); } } while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class - logger.info("Tags are " + tags); - logger.info("Groups are " + groups); - logger.info("Extras are " + extras); - logger.info("Resources without timeseries are " + resources); + logger.debug("Tags are " + tags); + logger.debug("Groups are " + groups); + logger.debug("Extras are " + extras); + logger.debug("Resources without timeseries are " + resources); } /** @@ -211,9 +240,9 @@ public class CommonServiceUtils { // else add all the available elements for (int i = 0; i < asList.size(); i++) { - boolean match = conditionToCheck.equals("") ? true : asList.get(i).toString().trim().matches(conditionToCheck); + boolean match = conditionToCheck.isEmpty() ? true : asList.get(i).toString().trim().matches(conditionToCheck); if(match){ - String groupName = groupNameOverValue.equals("") ? + String groupName = groupNameOverValue.isEmpty() ? HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + asList.get(i).toString().trim()) : source.toString().toLowerCase() + "-" + groupNameOverValue; groups.add(groupName); @@ -224,10 +253,10 @@ public class CommonServiceUtils { }else{ // also convert to the group name that should be on ckan - boolean match = conditionToCheck.equals("") ? true : f.toString().trim().matches(conditionToCheck); + boolean match = conditionToCheck.isEmpty() ? true : f.toString().trim().matches(conditionToCheck); if(match){ - String groupName = groupNameOverValue.equals("") ? + String groupName = groupNameOverValue.isEmpty() ? HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + f.toString().trim()) : source.toString().toLowerCase() + "-" + groupNameOverValue; groups.add(groupName); @@ -236,18 +265,6 @@ public class CommonServiceUtils { } } - // // check if the field is an enumerator, and the enum class is also annotated with @Group - // if(field.getType().isEnum() && field.getType().isAnnotationPresent(Group.class)){ - // - // logger.info("Class " + field.getClass().getSimpleName() + " has annotation @Group"); - // - // // extract the name from the enum class and add it to the groups - // // also convert to the group name that should be on ckan - // String groupName = HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + field.getType().getSimpleName()); - // groups.add(groupName); - // - // } - }catch(Exception e){ logger.error("Failed to read value for field " + field.getName() + " skipping", e); } @@ -347,4 +364,158 @@ public class CommonServiceUtils { } } -} + /** + * Evaluate if the user has the admin role + * Throws exception if he/she doesn't + */ + public static void hasAdminRole(String username, DataCatalogue catalogue, String apiKey, String organization) throws Exception{ + + String role = null; + Iterator> roles = catalogue.getUserRoleByOrganization(username, apiKey).get(organization).entrySet().iterator(); + + while (roles.hasNext()) { + Map.Entry entry = (Map.Entry) roles + .next(); + role = RolesCkanGroupOrOrg.convertToCkanCapacity(entry.getValue()); + } + + logger.info("Role of the user " + username + " is " + role + " in " + organization); + + if(role == null || role.isEmpty() || !role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())) + throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!"); + + } + + /** + * Check this record's name + * @param futureName + * @param catalogue + * @throws Exception on name check + */ + public static void checkName(String futureName, DataCatalogue catalogue) throws Exception { + + if(!HelperMethods.isNameValid(futureName)){ + throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'"); + }else{ + + logger.debug("Checking if such name [" + futureName + "] doesn't exist ..."); + boolean alreadyExists = catalogue.existProductWithNameOrId(futureName); + + if(alreadyExists){ + logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists"); + throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists"); + + } + } + } + + /** + * Validate and check sources + * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + * @param context + * @param contextServlet + * @param sourceInPath + * @param record + * @param resources + * @param groups + * @param customFields + * @param tags + * @param futureTitle + * @param username + * @throws Exception + */ + public static void validateRecordAndMapFields(String context, ServletContext contextServlet, + Sources sourceInPath, Common record, Product_Type productType, Set tags, Map> customFields, Set groups, List resources, String username, String futureTitle) throws Exception { + + // validate the record if it is a GRSF one and set the record type and in manage context + // Status field is needed only in the Manage context for GRSF records + if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){ + if(sourceInPath.equals(Sources.GRSF)){ + //Evaluate the sources + List> recordSources = record.getDatabaseSources(); + String sources = ""; + for (Resource resource : recordSources) { + sources += resource.getName() + ", "; + } + sources = sources.endsWith(", ") ? sources.substring(0, sources.length() -2) : sources; + record.setSourceType(sources); + CommonServiceUtils.validateAggregatedRecord(record); + } + } + + + // set the grsf type: fishery or stock + record.setGrsfType(productType.getOrigName()); + + // evaluate the custom fields/tags, resources and groups + groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + productType.getOrigName().toLowerCase()); //e.g. grsf-fishery + boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records + CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath); + + // manage the refers to + if(sourceInPath.equals(Sources.GRSF)){ + List refersTo = record.getRefersTo(); + if(refersTo == null || refersTo.isEmpty()) + throw new Exception("refers_to is empty"); + for (RefersToBean refersToBean : refersTo) { + resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: " + + refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null)); + } + } + + // add the SYSTEM_TYPE + customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); + } + + /** + * Actions to execute once the dataset has been updated or created. + * @param responseBean + * @param catalogue + * @param namespaces + * @param groups + * @param context + * @param token + * @param futureTitle + * @param authorFullname + * @param contextServlet + * @throws InterruptedException + */ + public static void actionsPostCreateOrUpdate( + String datasetId, String futureName, Common record, String apiKey, String username, String organization, String itemUrl, + ResponseCreationBean responseBean, DataCatalogue catalogue, + Map namespaces, Set groups, String context, + String token, String futureTitle, String authorFullname, ServletContext contextServlet, boolean isUpdated) throws InterruptedException { + + // set info in the response bean + responseBean.setId(datasetId); + responseBean.setItemUrl(itemUrl); + responseBean.setKbUuid(record.getUuid()); + + // manage groups (wait thread to die: ckan doesn't support too much concurrency on same record ...) + if(!groups.isEmpty()){ + logger.info("Launching thread for association to the list of groups " + groups); + AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList(groups), datasetId, organization, username, catalogue, apiKey); + threadGroups.start(); + logger.debug("Waiting association thread to die.."); + threadGroups.join(); + logger.debug("Ok, it died"); + } + + // manage time series as resources + logger.info("Launching thread for time series handling"); + new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start(); + + // write a post if the product has been published in grsf context + if(!isUpdated && context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){ + new WritePostCatalogueManagerThread( + context, + token, + futureTitle, + itemUrl, + false, + new ArrayList(), + authorFullname).start(); + logger.info("Thread to write a post about the new product has been launched"); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java index 4f783d3..311f5df 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java @@ -29,37 +29,27 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.utils.Caller; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Base; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.DeleteProductBean; import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean; import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; -import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread; -import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.ManageTimeSeriesThread; -import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.WritePostCatalogueManagerThread; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; -import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.model.CkanDataset; /** - * Fishery web service methods - * @author Costantino Perciante at ISTI-CNR + * Fishery web service methods. + * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ @Path("{source:firms|FIRMS|ram|RAM|grsf|GRSF|FishSource|fishsource}/fishery/") public class GrsfPublisherFisheryService { - // the default license for these records - private static final String DEFAULT_FISHERY_LICENSE = "CC-BY-SA-4.0"; - // the context @Context ServletContext contextServlet; @@ -77,13 +67,11 @@ public class GrsfPublisherFisheryService { @Path("get-licenses") @Produces(MediaType.APPLICATION_JSON) public Response getLicenses(){ - Status status = Status.OK; Map licenses = CommonServiceUtils.getLicenses(); if(licenses == null) status = Status.INTERNAL_SERVER_ERROR; return Response.status(status).entity(licenses).build(); - } @POST @@ -96,18 +84,16 @@ public class GrsfPublisherFisheryService { @PathParam("source") String source) throws ValidationException{ - // retrieve context and username Caller caller = AuthorizationProvider.instance.get(); String username = caller.getClient().getId(); String context = ScopeProvider.instance.get(); String token = SecurityTokenProvider.instance.get(); - logger.info("Incoming request for creating a fishery record = " + record); - logger.info("Request comes from user " + username + " in context " + context); + logger.info("Incoming request for creating a fishery record = " + record + ".\nRequest comes from user " + username + " in context " + context); ResponseCreationBean responseBean = new ResponseCreationBean(); Status status = Status.INTERNAL_SERVER_ERROR; - String id = ""; + String id = ""; // id of the created record, if everything went ok try{ @@ -116,24 +102,19 @@ public class GrsfPublisherFisheryService { DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ - - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("There was a problem while serving your request. No catalogue instance was found!"); - }else{ String apiKey = catalogue.getApiKeyFromUsername(username); String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin"; - String role = catalogue.getRoleOfUserInOrganization(username, organization, apiKey); + CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); - logger.info("Role of the user " + username + " is " + role); + // retrieve the user's email and fullname + String authorMail = HelperMethods.getUserEmail(context, token); + String authorFullname = HelperMethods.getUserFullname(context, token); - if(role == null || role.isEmpty()) - throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!"); - - if(!role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())){ - status = Status.FORBIDDEN; - throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-admin role!"); + if(authorMail == null || authorFullname == null){ + throw new Exception("Sorry but there was not possible to retrieve your fullname/email!"); } // The name of the product will be the uuid of the kb. The title will be the fishery's fishery_name. Fishery has also the constraint that @@ -141,180 +122,80 @@ public class GrsfPublisherFisheryService { String futureName = record.getUuid(); String futureTitle = record.getFisheryName(); - if(!HelperMethods.isNameValid(futureName)){ + // check name + CommonServiceUtils.checkName(futureName, catalogue); - status = Status.BAD_REQUEST; - throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'"); + Map> customFields = record.getExtrasFields(); + Set tags = new HashSet(); + Set groups = new HashSet(); + List resources = record.getExtrasResources(); + // validate end set sources + CommonServiceUtils.validateRecordAndMapFields(context, contextServlet, sourceInPath, record, Product_Type.FISHERY, tags, customFields, groups, resources, username, futureTitle); + + // check the license id + String license = null; + if(record.getLicense() == null || record.getLicense().isEmpty()) + license = CommonServiceUtils.DEFAULT_LICENSE; + else + if(HelperMethods.existsLicenseId(record.getLicense(), catalogue)) + license = record.getLicense(); + else throw new Exception("Please check the license id!"); + + long version = record.getVersion() == null ? 1 : record.getVersion(); + + // set the visibility of the datatest according the context + boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY)); + + // add the "Product URL" to the record + String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName); + customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl)); + + // convert extras' keys to keys with namespace + Map namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY); + + if(namespaces == null) + throw new Exception("Failed to retrieve the namespaces for the key fields!"); + + customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + logger.info("Invoking create method.."); + + // create the product + id = catalogue.createCKanDatasetMultipleCustomFields( + apiKey, + futureTitle, + futureName, + organization, + authorFullname, + authorMail, + record.getMaintainer() == null? authorFullname : record.getMaintainer(), + record.getMaintainerContact() == null? authorMail : record.getMaintainerContact(), + version, + HelperMethods.removeHTML(record.getDescription()), + license, + new ArrayList(tags), + customFields, + resources, + publicDataset); + + // post actions + if(id != null){ + + logger.info("Created record with identifier " + id); + CommonServiceUtils.actionsPostCreateOrUpdate( + id, futureName, record, apiKey, username, organization, + itemUrl, responseBean, catalogue, namespaces, groups, context, token, futureTitle, authorFullname, + contextServlet, false); + status = Status.CREATED; + }else{ - - logger.debug("Checking if such name [" + futureName + "] doesn't exist yet..."); - boolean alreadyExists = catalogue.existProductWithNameOrId(futureName); - - if(alreadyExists){ - - logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists"); - status = Status.CONFLICT; - throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists"); - - }else{ - - // validate the record if it is a GRSF one and set the record type and in manage context - // Status field is needed only in the Manage context for GRSF records - if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){ - if(sourceInPath.equals(Sources.GRSF)){ - //Evaluate the sources - List> recordSources = record.getDatabaseSources(); - String sources = ""; - for (Resource resource : recordSources) { - sources += resource.getName() + ", "; - } - - sources = sources.endsWith(", ") ? sources.substring(0, sources.length() -2) : sources; - record.setSourceType(sources); - CommonServiceUtils.validateAggregatedRecord(record); - - } - } - - // set the grsf type - record.setGrsfType(Product_Type.FISHERY.getOrigName()); - - // product system type is a list of values for sources records, so remove it (so that no group is generated) - // if(!sourceInPath.equals(Sources.GRSF)) - // record.setProductionSystemType(null); - - // evaluate the custom fields/tags, resources and groups - Map> customFields = record.getExtrasFields(); - Set tags = new HashSet(); - Set groups = new HashSet(); - groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + Product_Type.FISHERY.getOrigName().toLowerCase()); //e.g. grsf-fishery - List resources = record.getExtrasResources(); - boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records - CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, resources, customFields, record, username, sourceInPath); - - // manage the refers to - if(sourceInPath.equals(Sources.GRSF)){ - List refersTo = record.getRefersTo(); - if(refersTo == null || refersTo.isEmpty()) - throw new Exception("refers_to is empty"); - for (RefersToBean refersToBean : refersTo) { - resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: " - + refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null)); - } - } - - // retrieve the user's email and fullname - String authorMail = HelperMethods.getUserEmail(context, token); - String authorFullname = HelperMethods.getUserFullname(context, token); - - if(authorMail == null || authorFullname == null){ - - logger.debug("Author fullname or mail missing, cannot continue"); - status = Status.INTERNAL_SERVER_ERROR; - throw new Exception("Sorry but there was not possible to retrieve your fullname/email!"); - - }else{ - - // check the license id - String license = null; - if(record.getLicense() == null || record.getLicense().isEmpty()) - license = DEFAULT_FISHERY_LICENSE; - else - if(HelperMethods.existsLicenseId(record.getLicense(), catalogue)) - license = record.getLicense(); - else throw new Exception("Please check the license id!"); - - long version = record.getVersion() == null ? 1 : record.getVersion(); - - // set the visibility of the datatest according the context - boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY)); - - // add the SYSTEM_TYPE - customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); - - // convert extras' keys to keys with namespace - Map namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY); - - if(namespaces == null) - throw new Exception("Failed to retrieve the namespaces for the key fields!"); - - customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); - - logger.info("Invoking creation method.."); - - // create the product - id = catalogue.createCKanDatasetMultipleCustomFields( - apiKey, - futureTitle, - futureName, - organization, - authorFullname, - authorMail, - record.getMaintainer(), - record.getMaintainerContact(), - version, - HelperMethods.removeHTML(record.getDescription()), - license, - new ArrayList(tags), - customFields, - resources, - publicDataset); - - if(id != null){ - - logger.info("Item created! Id is " + id); - responseBean.setId(id); - status = Status.CREATED; - String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName); - responseBean.setItemUrl(itemUrl); - responseBean.setKbUuid(record.getUuid()); - - // add the "Product URL" to the field - Map> addField = new HashMap>(); - String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD; - addField.put(modifiedUUIDKey, Arrays.asList(itemUrl)); - catalogue.patchProductCustomFields(id, apiKey, addField); - - if(!groups.isEmpty()){ - logger.info("Launching thread for association to the list of groups " + groups); - AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList(groups), id, organization, username, catalogue); - threadGroups.start(); - logger.info("Waiting association thread to die.."); - threadGroups.join(); - logger.debug("Groups-Thread died"); - } - - // manage time series - logger.info("Launching thread for time series handling"); - new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start(); - - // write a post if the product has been published in grsf context - if(context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){ - // TODO uncomment later - /*new WritePostCatalogueManagerThread( - context, - token, - futureTitle, - itemUrl, - false, - new ArrayList(), - authorFullname).start();*/ - - logger.info("Thread to write a post about the new product has been launched"); - } - - }else{ - - throw new Exception("There was an error during the product generation, sorry"); - - } - } - } + throw new Exception("There was an error during the product generation, sorry"); } } }catch(Exception e){ logger.error("Failed to create fishery record" + e); + status = Status.INTERNAL_SERVER_ERROR; responseBean.setError(e.getMessage()); } @@ -344,61 +225,47 @@ public class GrsfPublisherFisheryService { DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ - - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("There was a problem while serving your request"); - } // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); - logger.info("The request is to delete a fishery object of source " + sourceInPath); + logger.debug("The request is to delete a fishery object of source " + sourceInPath); // retrieve the catalogue instance - CkanDataset fisheryInCkan = catalogue.getDataset(recordToDelete.getId(), catalogue.getApiKeyFromUsername(username)); + String apiKey = catalogue.getApiKeyFromUsername(username); + CkanDataset fisheryInCkan = catalogue.getDataset(recordToDelete.getId(), apiKey); if(fisheryInCkan == null){ - status = Status.NOT_FOUND; throw new Exception("There was a problem while serving your request. This item was not found"); - } // check it is in the right source and it is a fishery String grsfType = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); - String groupToCheck = sourceInPath.equals(Sources.GRSF) ? "grsf-group" : sourceInPath.getOrigName().toLowerCase(); + String groupToCheck = sourceInPath.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : sourceInPath.getOrigName().toLowerCase(); if(catalogue.isDatasetInGroup(groupToCheck, recordToDelete.getId()) && Product_Type.FISHERY.getOrigName().equals(grsfType)){ - logger.warn("Ok, this is a fishery of the right source, removing it"); - boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), catalogue.getApiKeyFromUsername(username), true); + logger.debug("Ok, this is a fishery of the right source, removing it"); + boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), apiKey, true); if(deleted){ - logger.info("Fishery DELETED AND PURGED!"); status = Status.OK; responseBean.setId(fisheryInCkan.getId()); - } else{ - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("Request failed, sorry. Unable to delete/purge the fishery"); - } - }else{ - status = Status.BAD_REQUEST; throw new Exception("The id you are using doesn't belong to a Fishery item having source " + source + "!"); - } }catch(Exception e){ - logger.error("Failed to delete this", e); - status = Status.INTERNAL_SERVER_ERROR; responseBean.setError(e.getMessage()); - } return Response.status(status).entity(responseBean).build(); @@ -410,16 +277,13 @@ public class GrsfPublisherFisheryService { public Response getFisheriesIds( @PathParam("source") String source){ - // retrieve context and username + logger.info("Received call to get fisheries with source " + source); + Caller caller = AuthorizationProvider.instance.get(); String context = ScopeProvider.instance.get(); String username = caller.getClient().getId(); - ResponseBean responseBean = new ResponseBean(); Status status = Status.INTERNAL_SERVER_ERROR; - - logger.info("Received call to get fisheries with source " + source); - List datasetsIds = new ArrayList(); try{ @@ -429,10 +293,7 @@ public class GrsfPublisherFisheryService { DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ - - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("There was a problem while serving your request"); - } // if it is a request for GRSF records, we have Fishery - Stock groups, so it is easy. @@ -440,31 +301,24 @@ public class GrsfPublisherFisheryService { if(sourceInPath.equals(Sources.GRSF)) datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "fishery", catalogue); else{ - List fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue); - + String apiKey = catalogue.getApiKeyFromUsername(username); for (String id : fullGroupListIds) { - - CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username)); + CkanDataset dataset = catalogue.getDataset(id, apiKey); if(dataset != null){ String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); if(grsfType.equals(Product_Type.FISHERY.getOrigName())) datasetsIds.add(id); } - } - } - + status = Status.OK; responseBean.setResult(datasetsIds); responseBean.setSuccess(true); }catch(Exception e){ - logger.error("Failed to fetch this list of ids ", e); - status = Status.INTERNAL_SERVER_ERROR; responseBean.setMessage(e.getMessage()); - } return Response.status(status).entity(responseBean).build(); @@ -476,60 +330,189 @@ public class GrsfPublisherFisheryService { public Response getCatalogueIdAndUrlFromKBID( @QueryParam("name") String name){ - // retrieve context and username String context = ScopeProvider.instance.get(); Caller caller = AuthorizationProvider.instance.get(); String username = caller.getClient().getId(); - ResponseBean responseBean = new ResponseBean(); - Status status = Status.OK; + Status status = Status.INTERNAL_SERVER_ERROR; logger.info("Received call to get the catalogue identifier for the product with name " + name); try{ - DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ throw new Exception("There was a problem while serving your request"); } - CkanDataset dataset = catalogue.getDataset(name, catalogue.getApiKeyFromUsername(username)); if(dataset != null){ - Map result = new HashMap(); result.put("id", dataset.getId()); - - // retrieve the product url - Map customFields = dataset.getExtrasAsHashMap(); - - boolean found = false; - Set KeySet = customFields.keySet(); - for (String key : KeySet) { - if(key.contains(CommonServiceUtils.ITEM_URL_FIELD) && !key.contains(Base.UUID_KB_KEY)){ - result.put("url", customFields.get(key)); - found = true; - break; - } - } - if(!found) - result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId())); - + result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId())); responseBean.setResult(result); responseBean.setSuccess(true); + status = Status.OK; }else{ responseBean.setMessage("Unable to retrieve a catalogue item with name " + name); } - }catch(Exception e){ - logger.error("Failed to retrieve this product", e); - status = Status.INTERNAL_SERVER_ERROR; responseBean.setMessage(e.getMessage()); - } - return Response.status(status).entity(responseBean).build(); } + @POST + @Path("update-product") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response updateFishery( + @NotNull(message="record cannot be null") + @Valid FisheryRecord record, + @PathParam("source") String source) + throws ValidationException{ -} + Caller caller = AuthorizationProvider.instance.get(); + String username = caller.getClient().getId(); + String context = ScopeProvider.instance.get(); + String token = SecurityTokenProvider.instance.get(); + + logger.info("Incoming request for updating a fishery record = " + record + ". Request comes from user " + username + " in context " + context); + + ResponseCreationBean responseBean = new ResponseCreationBean(); + Status status = Status.INTERNAL_SERVER_ERROR; + + // catalog id must be reported + String catalogId = record.getCatalogId(); + + try{ + + if(catalogId == null || catalogId.isEmpty()){ + status = Status.BAD_REQUEST; + throw new Exception("Please specify the 'catalog_id' property"); + } + + DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); + + if(catalogue == null){ + throw new Exception("There was a problem while serving your request. No catalogue instance was found in this context!"); + }else{ + + // get already published record and modify it + CkanDataset recordPublished = catalogue.getDataset(catalogId); + + if(recordPublished == null) + throw new Exception("A record with catalogue id " + catalogId + " does not exist!"); + + // retrieve the user's email and fullname + String authorMail = HelperMethods.getUserEmail(context, token); + String authorFullname = HelperMethods.getUserFullname(context, token); + + if(authorMail == null || authorFullname == null){ + logger.debug("Author fullname or mail missing, cannot continue"); + throw new Exception("Sorry but there was not possible to retrieve your fullname/email!"); + } + + String apiKey = catalogue.getApiKeyFromUsername(username); + String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin"; + + // check he/she has admin role + CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); + + // name, title, product url and are going to remain unchanged (so we keep them from the publisher record); + String name = recordPublished.getName(); + String title = recordPublished.getTitle(); + + // Cast the source to the accepted ones + Sources sourceInPath = Sources.onDeserialize(source); + + // load infos + Map> customFields = record.getExtrasFields(); + Set tags = new HashSet(); + Set groups = new HashSet(); + List resources = record.getExtrasResources(); + + // validate end set sources + CommonServiceUtils.validateRecordAndMapFields( + context, + contextServlet, + sourceInPath, + record, + Product_Type.FISHERY, + tags, + customFields, + groups, + resources, + username, + title); + + // check the license id + String license = null; + if(record.getLicense() == null || record.getLicense().isEmpty()) + license = CommonServiceUtils.DEFAULT_LICENSE; + else + if(HelperMethods.existsLicenseId(record.getLicense(), catalogue)) + license = record.getLicense(); + else throw new Exception("Please check the license id!"); + + long version = record.getVersion() == null ? 1 : record.getVersion(); + + // set the visibility of the datatest according the context + boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY)); + + // add the SYSTEM_TYPE + customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); + + // convert extras' keys to keys with namespace + Map namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY); + + if(namespaces == null) + throw new Exception("Failed to retrieve the namespaces for the key fields!"); + + // retrieve the url + String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD; + String itemUrl = recordPublished.getExtrasAsHashMap().get(modifiedUUIDKey); + customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl)); + + // replace fields + customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + logger.info("Invoking update method.."); + + // update the product + String id = catalogue.updateCKanDataset( + apiKey, + catalogId, + title, name, + organization, + authorFullname, + authorMail, + record.getMaintainer(), + record.getMaintainerContact(), + version, + HelperMethods.removeHTML(record.getDescription()), + license, + new ArrayList(tags), + null, // remove any previous group + customFields, + resources, + publicDataset); + + if(id != null){ + logger.info("Item updated!"); + CommonServiceUtils.actionsPostCreateOrUpdate( + id, name, record, apiKey, username, organization, + itemUrl, responseBean, catalogue, namespaces, groups, context, token, title, authorFullname, + contextServlet, false); + status = Status.OK; + }else{ + status = Status.INTERNAL_SERVER_ERROR; + throw new Exception("There was an error during the item updated, sorry"); + } + } + }catch(Exception e){ + logger.error("Failed to create fishery record" + e); + responseBean.setError(e.getMessage()); + } + return Response.status(status).entity(responseBean).build(); + } +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java index ea64594..209c29d 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java @@ -29,36 +29,28 @@ import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.utils.Caller; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Base; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.DeleteProductBean; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseBean; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean; import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; -import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread; -import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.ManageTimeSeriesThread; -import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.WritePostCatalogueManagerThread; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; -import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.model.CkanDataset; /** - * Stock web service methods - * @author Costantino Perciante at ISTI-CNR + * Stock web service methods. + * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ @Path("{source:firms|FIRMS|ram|RAM|grsf|GRSF|FishSource|fishsource}/stock/") public class GrsfPublisherStockService { - private static final String DEFAULT_STOCK_LICENSE = "CC-BY-SA-4.0"; - // the context @Context ServletContext contextServlet; @@ -93,227 +85,133 @@ public class GrsfPublisherStockService { @NotNull(message="record cannot be null") @Valid StockRecord record, @PathParam("source") String source) throws ValidationException{ - // retrieve context and username Caller caller = AuthorizationProvider.instance.get(); String username = caller.getClient().getId(); String context = ScopeProvider.instance.get(); String token = SecurityTokenProvider.instance.get(); - logger.info("Incoming request for creating a stock record = " + record); - logger.info("Request coming from user " + username + " in context " + context); + logger.info("Incoming request for creating a stock record = " + record + ".\nRequest coming from user " + username + " in context " + context); ResponseCreationBean responseBean = new ResponseCreationBean(); Status status = Status.INTERNAL_SERVER_ERROR; + String id = ""; // id of the created record, if everything went ok try{ // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); - logger.info("The request is to create a stock object of source " + sourceInPath); - DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ - throw new Exception("There was a problem while serving your request"); + throw new Exception("There was a problem while serving your request. No catalogue instance was found!"); }else{ String apiKey = catalogue.getApiKeyFromUsername(username); - String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin";// TODO - String role = catalogue.getRoleOfUserInOrganization(username, organization, apiKey); + String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin"; - logger.info("Role of the user " + username + " is " + role); - - if(role == null || role.isEmpty()) - throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!"); + // check it has admin role or throw exception + CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); - if(!role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())){ - - status = Status.FORBIDDEN; - throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!"); + // retrieve the user's email and fullname + String authorMail = HelperMethods.getUserEmail(context, token); + String authorFullname = HelperMethods.getUserFullname(context, token); + if(authorMail == null || authorFullname == null){ + throw new Exception("Sorry but it was not possible to retrieve your fullname/email!"); } // check the record has a name, at least String futureName = record.getUuid(); String futureTitle = record.getStockName(); - if(!HelperMethods.isNameValid(futureName)){ + // check name and throws exception + CommonServiceUtils.checkName(futureName, catalogue); - status = Status.BAD_REQUEST; - throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'"); + // load other information + Map> customFields = record.getExtrasFields(); + Set tags = new HashSet(); + Set groups = new HashSet(); + List resources = record.getExtrasResources(); - }else{ + // validate end set sources, tags, etc + CommonServiceUtils.validateRecordAndMapFields( + context, + contextServlet, + sourceInPath, + record, + Product_Type.STOCK, + tags, + customFields, + groups, + resources, + username, + futureTitle); - logger.debug("Checking if such 'uuid_knowledge_base' [" + futureName + "] doesn't exist yet..."); - boolean alreadyExist = catalogue.existProductWithNameOrId(futureName); + // check the license id + String license = null; + if(record.getLicense() == null || record.getLicense().isEmpty()) + license = CommonServiceUtils.DEFAULT_LICENSE; + else + if(HelperMethods.existsLicenseId(record.getLicense(), catalogue)) + license = record.getLicense(); + else throw new Exception("Please check the license id!"); - if(alreadyExist){ + // check the version + long version = record.getVersion() == null ? 1 : record.getVersion(); - logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists"); - status = Status.CONFLICT; - throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists"); + // set the visibility of the datatest according the context + boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY)); - }else{ + // add the "Product URL" to the record + String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName); + customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl)); - // validate the record if it is a GRSF one and set the record type and in manage context - // Status field is needed only in the Manage context for GRSF records - if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){ - if(sourceInPath.equals(Sources.GRSF)){ - //Evaluate the sources - List> recordSources = record.getDatabaseSources(); - String sources = ""; - for (Resource resource : recordSources) { - sources += resource.getName() + ", "; - } - sources = sources.endsWith(", ") ? sources.substring(0, sources.length() -2) : sources; - record.setSourceType(sources); - CommonServiceUtils.validateAggregatedRecord(record); - } - } + // convert extras' keys to keys with namespace + Map namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_STOCK); - // set the grsf type - record.setGrsfType(Product_Type.STOCK.getOrigName()); + if(namespaces == null) + throw new Exception("Failed to retrieve the namespaces for the key fields!"); - // evaluate the custom fields/tags, resources and groups - Map> customFields = record.getExtrasFields(); - Set tags = new HashSet(); - Set groups = new HashSet(); - groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + Product_Type.STOCK.getOrigName().toLowerCase()); //e.g. grsf-stock - List resources = record.getExtrasResources(); - boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records - CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, resources, customFields, record, username, sourceInPath); + customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); - // manage the refers to - if(sourceInPath.equals(Sources.GRSF)){ + logger.info("Invoking create method.."); - List refersTo = record.getRefersTo(); - - if(refersTo == null || refersTo.isEmpty()) - throw new Exception("refers_to is empty"); - - for (RefersToBean refersToBean : refersTo) { - resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: " - + refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null)); - } + // create the product + id = catalogue.createCKanDatasetMultipleCustomFields( + apiKey, + futureTitle, + futureName, + organization, + authorFullname, + authorMail, + record.getMaintainer() == null? authorFullname : record.getMaintainer(), + record.getMaintainerContact() == null? authorMail : record.getMaintainerContact(), + version, + HelperMethods.removeHTML(record.getDescription()), + license, + new ArrayList(tags), + customFields, + resources, + publicDataset); - } + if(id != null){ - // retrieve the user's email and fullname - String authorMail = HelperMethods.getUserEmail(context, token); - String authorFullname = HelperMethods.getUserFullname(context, token); + logger.info("Product created! Id is " + id); + CommonServiceUtils.actionsPostCreateOrUpdate( + id, futureName, record, apiKey, username, organization, itemUrl, + responseBean, catalogue, namespaces, groups, context, token, + futureTitle, authorFullname, contextServlet, false); + status = Status.CREATED; - if(authorMail == null || authorFullname == null){ - - logger.debug("Author fullname or mail missing, cannot continue"); - responseBean.setId(null); - status = Status.INTERNAL_SERVER_ERROR; - throw new Exception("Sorry but was not possible to retrieve your fullname/email!"); - - }else{ - - // check the license id - String license = null; - if(record.getLicense() == null || record.getLicense().isEmpty()) - license = DEFAULT_STOCK_LICENSE; - else - if(HelperMethods.existsLicenseId(record.getLicense(), catalogue)) - license = record.getLicense(); - else throw new Exception("Please check the license id!"); - - long version = record.getVersion() == null ? 1 : record.getVersion(); - - // set the visibility of the datatest according the context - boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY)); - - // add the SYSTEM_TYPE - customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); - - logger.info("Invoking creation method.."); - - // convert extras' keys to keys with namespace - Map namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_STOCK); - - if(namespaces == null) - throw new Exception("Failed to retrieve the namespaces for the key fields!"); - - customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); - - // create the product - String id = catalogue.createCKanDatasetMultipleCustomFields( - apiKey, - futureTitle, - futureName, - organization, - authorFullname, - authorMail, - record.getMaintainer(), - record.getMaintainerContact(), - version, - HelperMethods.removeHTML(record.getDescription()), - license, - new ArrayList(tags), - customFields, - resources, - publicDataset); - - if(id != null){ - - logger.info("Product created! Id is " + id); - responseBean.setId(id); - status = Status.CREATED; - String itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName); - responseBean.setItemUrl(itemUrl); - responseBean.setKbUuid(record.getUuid()); - - // add the "Product URL" to the field - Map> addField = new HashMap>(); - String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD; - addField.put(modifiedUUIDKey, Arrays.asList(itemUrl)); - catalogue.patchProductCustomFields(id, apiKey, addField); - - if(!groups.isEmpty()){ - - logger.info("Launching thread for association to the list of groups " + groups); - AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList(groups), id, organization, username, catalogue); - threadGroups.start(); - logger.info("Waiting association thread to die.."); - threadGroups.join(); - logger.debug("Ok, it died"); - - } - - // manage time series - logger.info("Launching thread for time series handling"); - new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start(); - - // write a post if the product has been published in grsf context - if(context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){ - // TODO uncomment later - /*new WritePostCatalogueManagerThread( - context, - token, - futureTitle, - itemUrl, - false, - new ArrayList(), - authorFullname).start(); - logger.info("Thread to write a post about the new product has been launched");*/ - } - - }else - throw new Exception("There was an error during the product generation, sorry"); - } - } - } + }else + throw new Exception("There was an error during the product generation, sorry! Unable to create the dataset"); } - // } }catch(Exception e){ logger.error("Failed to create stock record", e); responseBean.setError(e.getMessage()); } - return Response.status(status).entity(responseBean).build(); - } + } @DELETE @Path("delete-product") @@ -344,11 +242,11 @@ public class GrsfPublisherStockService { // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); - + String apiKey = catalogue.getApiKeyFromUsername(username); logger.info("The request is to delete a stock object of source " + sourceInPath); // retrieve the catalogue instance - CkanDataset stockInCkan = catalogue.getDataset(recordToDelete.getId(), catalogue.getApiKeyFromUsername(username)); + CkanDataset stockInCkan = catalogue.getDataset(recordToDelete.getId(), apiKey); if(stockInCkan == null){ status = Status.NOT_FOUND; @@ -357,39 +255,28 @@ public class GrsfPublisherStockService { // check it is in the right source and it is a fishery String grsfType = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); - String groupToCheck = sourceInPath.equals(Sources.GRSF) ? "grsf-group" : sourceInPath.getOrigName().toLowerCase(); + String groupToCheck = sourceInPath.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : sourceInPath.getOrigName().toLowerCase(); if(catalogue.isDatasetInGroup(groupToCheck, recordToDelete.getId()) && Product_Type.STOCK.getOrigName().equals(grsfType)){ - logger.debug("Ok, this is a stock of the right type, removing it"); - boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), catalogue.getApiKeyFromUsername(username), true); - + boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), apiKey, true); if(deleted){ - logger.info("Stock DELETED AND PURGED!"); status = Status.OK; responseBean.setId(stockInCkan.getId()); - } else{ - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("Request failed, sorry. Unable to delete/purge the stock"); - } - }else{ status = Status.BAD_REQUEST; throw new Exception("The id you are using doesn't belong to a Stock product having source " + source + "!"); } - }catch(Exception e){ - logger.error("Failed to delete this ", e); responseBean.setError(e.getMessage()); - } - return Response.status(status).entity(responseBean).build(); } @@ -403,26 +290,20 @@ public class GrsfPublisherStockService { Caller caller = AuthorizationProvider.instance.get(); String context = ScopeProvider.instance.get(); String username = caller.getClient().getId(); - + Status status = Status.INTERNAL_SERVER_ERROR; ResponseBean responseBean = new ResponseBean(); - Status status = Status.OK; // check it is a stock ... logger.info("Received call to get stocks with source " + source); - List datasetsIds = new ArrayList(); - try{ // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); - DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("There was a problem while serving your request"); - } // if it is a request for GRSF records, we have Fishery - Stock groups, so it is easy. @@ -430,33 +311,23 @@ public class GrsfPublisherStockService { if(sourceInPath.equals(Sources.GRSF)) datasetsIds = HelperMethods.getProductsInGroup(source + "-" + "stock", catalogue); else{ - List fullGroupListIds = HelperMethods.getProductsInGroup(source, catalogue); - for (String id : fullGroupListIds) { - CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username)); if(dataset != null){ String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); if(grsfType.equals(Product_Type.STOCK.getOrigName())) datasetsIds.add(id); } - } - } - responseBean.setResult(datasetsIds); responseBean.setSuccess(true); - + status = Status.OK; }catch(Exception e){ - logger.error("Failed to fetch this list of ids " + source, e); - responseBean.setSuccess(false); responseBean.setMessage(e.getMessage()); - } - return Response.status(status).entity(responseBean).build(); } @@ -470,54 +341,187 @@ public class GrsfPublisherStockService { String context = ScopeProvider.instance.get(); Caller caller = AuthorizationProvider.instance.get(); String username = caller.getClient().getId(); - ResponseBean responseBean = new ResponseBean(); - Status status = Status.OK; - + Status status = Status.INTERNAL_SERVER_ERROR; logger.info("Received call to get the catalogue identifier for the product with name " + name); - try{ - DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ - status = Status.INTERNAL_SERVER_ERROR; throw new Exception("There was a problem while serving your request"); } - CkanDataset dataset = catalogue.getDataset(name, catalogue.getApiKeyFromUsername(username)); if(dataset != null){ - Map result = new HashMap(); result.put("id", dataset.getId()); - - // retrieve the product url - Map customFields = dataset.getExtrasAsHashMap(); - - boolean found = false; - Set KeySet = customFields.keySet(); - for (String key : KeySet) { - if(key.contains(CommonServiceUtils.ITEM_URL_FIELD) && !key.contains(Base.UUID_KB_KEY)){ - result.put("url", customFields.get(key)); - found = true; - break; - } - } - if(!found) - result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId())); - + result.put("url", catalogue.getUnencryptedUrlFromDatasetIdOrName(dataset.getId())); responseBean.setResult(result); responseBean.setSuccess(true); + status = Status.OK; }else{ responseBean.setMessage("Unable to retrieve a catalogue product with name " + name); } - }catch(Exception e){ logger.error("Failed to retrieve this product", e); - responseBean.setSuccess(false); responseBean.setMessage(e.getMessage()); } - return Response.status(status).entity(responseBean).build(); } -} + @POST + @Path("update-product") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response updateFishery( + @NotNull(message="record cannot be null") + @Valid FisheryRecord record, + @PathParam("source") String source) + throws ValidationException{ + + Caller caller = AuthorizationProvider.instance.get(); + String username = caller.getClient().getId(); + String context = ScopeProvider.instance.get(); + String token = SecurityTokenProvider.instance.get(); + + logger.info("Incoming request for updating a stock record = " + record + ". Request comes from user " + username + " in context " + context); + + ResponseCreationBean responseBean = new ResponseCreationBean(); + Status status = Status.INTERNAL_SERVER_ERROR; + + // catalog id must be reported + String catalogId = record.getCatalogId(); + + try{ + + if(catalogId == null || catalogId.isEmpty()){ + status = Status.BAD_REQUEST; + throw new Exception("Please specify the 'catalog_id' property"); + } + + DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); + + if(catalogue == null){ + throw new Exception("There was a problem while serving your request. No catalogue instance was found in this context!"); + }else{ + + // get already published record and modify it + CkanDataset recordPublished = catalogue.getDataset(catalogId); + + if(recordPublished == null) + throw new Exception("A record with catalogue id " + catalogId + " does not exist!"); + + // retrieve the user's email and fullname + String authorMail = HelperMethods.getUserEmail(context, token); + String authorFullname = HelperMethods.getUserFullname(context, token); + + if(authorMail == null || authorFullname == null){ + logger.debug("Author fullname or mail missing, cannot continue"); + throw new Exception("Sorry but there was not possible to retrieve your fullname/email!"); + } + + String apiKey = catalogue.getApiKeyFromUsername(username); + String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin"; + + // check he/she has admin role + CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); + + // name, title, product url and are going to remain unchanged (so we keep them from the publisher record); + String name = recordPublished.getName(); + String title = recordPublished.getTitle(); + + // Cast the source to the accepted ones + Sources sourceInPath = Sources.onDeserialize(source); + + // load infos + Map> customFields = record.getExtrasFields(); + Set tags = new HashSet(); + Set groups = new HashSet(); + List resources = record.getExtrasResources(); + + // validate end set sources + CommonServiceUtils.validateRecordAndMapFields( + context, + contextServlet, + sourceInPath, + record, + Product_Type.STOCK, + tags, + customFields, + groups, + resources, + username, + title); + + // check the license id + String license = null; + if(record.getLicense() == null || record.getLicense().isEmpty()) + license = CommonServiceUtils.DEFAULT_LICENSE; + else + if(HelperMethods.existsLicenseId(record.getLicense(), catalogue)) + license = record.getLicense(); + else throw new Exception("Please check the license id!"); + + long version = record.getVersion() == null ? 1 : record.getVersion(); + + // set the visibility of the datatest according the context + boolean publicDataset = context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY)); + + // add the SYSTEM_TYPE + customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); + + // convert extras' keys to keys with namespace + Map namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY); + + if(namespaces == null) + throw new Exception("Failed to retrieve the namespaces for the key fields!"); + + // retrieve the url + String modifiedUUIDKey = namespaces.containsKey(CommonServiceUtils.ITEM_URL_FIELD) ? namespaces.get(CommonServiceUtils.ITEM_URL_FIELD) : CommonServiceUtils.ITEM_URL_FIELD; + String itemUrl = recordPublished.getExtrasAsHashMap().get(modifiedUUIDKey); + customFields.put(CommonServiceUtils.ITEM_URL_FIELD, Arrays.asList(itemUrl)); + + // replace fields + customFields = HelperMethods.replaceFieldsKey(customFields, namespaces); + + logger.info("Invoking update method.."); + + // update the product + String id = catalogue.updateCKanDataset( + apiKey, + catalogId, + title, name, + organization, + authorFullname, + authorMail, + record.getMaintainer(), + record.getMaintainerContact(), + version, + HelperMethods.removeHTML(record.getDescription()), + license, + new ArrayList(tags), + null, // remove any previous group + customFields, + resources, + publicDataset); + + if(id != null){ + + logger.info("Item updated!"); + CommonServiceUtils.actionsPostCreateOrUpdate( + recordPublished.getId(), name, record, apiKey, username, organization, itemUrl, + responseBean, catalogue, namespaces, groups, context, token, + title, authorFullname, contextServlet, true); + status = Status.OK; + + }else{ + status = Status.INTERNAL_SERVER_ERROR; + throw new Exception("There was an error during the item updated, sorry"); + } + } + }catch(Exception e){ + logger.error("Failed to create fishery record" + e); + responseBean.setError(e.getMessage()); + } + return Response.status(status).entity(responseBean).build(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java index 84b26cc..4c2a6f1 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java @@ -20,6 +20,10 @@ public class CSVHelpers { private static final String CSV_SEPARATOR = ","; private static final String UPLOAD_LOCATION_LOCAL = System.getProperty("java.io.tmpdir"); private static final String GRSF_SUB_PATH = "GRSF_TIME_SERIES"; + public static final String CSV_EXTENSION = ".csv"; + private static final String YEAR_FIELD = "year"; + private static final String VALUE_FIELD = "value"; + private static final String UNIT_FIELD = "unit"; /** * Write a time series to a csv file, and returns the file reference.
@@ -36,7 +40,7 @@ public class CSVHelpers { }else try{ - String fileName = UPLOAD_LOCATION_LOCAL + File.separator + GRSF_SUB_PATH + File.separator + "time_series_" + System.currentTimeMillis() + ".csv"; + String fileName = UPLOAD_LOCATION_LOCAL + File.separator + GRSF_SUB_PATH + File.separator + "time_series_" + System.currentTimeMillis() + CSV_EXTENSION; File file = new File(fileName); file.getParentFile().mkdirs(); file.createNewFile(); @@ -47,16 +51,16 @@ public class CSVHelpers { StringBuffer headerLine = new StringBuffer(); - headerLine.append("year"); + headerLine.append(YEAR_FIELD); headerLine.append(CSV_SEPARATOR); - headerLine.append("value"); + headerLine.append(VALUE_FIELD); // first line is csv header, check the type of object if(timeSeries.get(0).getUnit() != null && !timeSeries.get(0).getUnit().getClass().equals(Void.class)){ isUnitPresent = true; headerLine.append(CSV_SEPARATOR); - headerLine.append("unit"); + headerLine.append(UNIT_FIELD); } @@ -79,13 +83,13 @@ public class CSVHelpers { bw.write(oneLine.toString()); bw.newLine(); - bw.flush(); } - // file created - logger.info("CSV file created correctly on this machine!"); - + bw.flush(); bw.close(); + + // file created + logger.debug("CSV file created correctly on this machine!"); // on exit delete it... file.deleteOnExit(); @@ -96,4 +100,4 @@ public class CSVHelpers { return null; } } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java index 509e8db..5ea8596 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java @@ -30,7 +30,6 @@ import org.gcube.data_catalogue.grsf_publish_ws.utils.cache.CacheImpl; import org.gcube.data_catalogue.grsf_publish_ws.utils.cache.CacheInterface; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory; -import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.Query; import org.gcube.resources.discovery.client.queries.impl.QueryBox; @@ -63,7 +62,7 @@ public abstract class HelperMethods { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HelperMethods.class); private static final String APPLICATION_ID_CATALOGUE_MANAGER = "org.gcube.datacatalogue.ProductCatalogue"; - private static final String NOTIFICATION_MESSAGE = "Dear members,
The product 'PRODUCT_TITLE' has been just published by USER_FULLNAME.
You can find it here: PRODUCT_URL
"; + private static final String NOTIFICATION_MESSAGE = "Dear members,
The item 'PRODUCT_TITLE' has been just published by USER_FULLNAME.
You can find it here: PRODUCT_URL
"; private static final String SOCIAL_SERVICE_APPLICATION_TOKEN = "/2/tokens/generate-application-token/"; private static final String SOCIAL_SERVICE_WRITE_APPLICATION_POST = "/2/posts/write-post-app/"; private static final String MEDIATYPE_JSON = "application/json"; @@ -78,8 +77,10 @@ public abstract class HelperMethods { private static final String PATH_SEPARATOR = "/"; // caches - private static CacheInterface userEmailCache = new CacheImpl(1000 * 60 * 120); - private static CacheInterface userFullnameCache = new CacheImpl(1000 * 60 * 120); + private static CacheInterface userEmailCache = new CacheImpl(1000 * 60 * 60 * 24); + private static CacheInterface userFullnameCache = new CacheImpl(1000 * 60 * 60 * 24); + private static CacheInterface> namespacesCache = new CacheImpl>(1000 * 60 * 60 * 24); + private static CacheInterface catalogueCache = new CacheImpl(1000 * 60 * 60 * 24); /** * Convert a group name to its id on ckan @@ -109,14 +110,18 @@ public abstract class HelperMethods { */ public static DataCatalogue getDataCatalogueRunningInstance(String scope){ - try{ - DataCatalogueImpl instance = DataCatalogueFactory.getFactory().getUtilsPerScope(scope); - return instance; - }catch(Exception e){ - logger.error("Failed to instanciate data catalogue lib", e); + if(catalogueCache.get(scope) != null) + return catalogueCache.get(scope); + else{ + try{ + DataCatalogue instance = DataCatalogueFactory.getFactory().getUtilsPerScope(scope); + catalogueCache.insert(scope, instance); + return instance; + }catch(Exception e){ + logger.error("Failed to instanciate data catalogue lib", e); + return null; + } } - - return null; } /** @@ -496,44 +501,50 @@ public abstract class HelperMethods { public static Map getFieldToFieldNameSpaceMapping(String resourceName){ Map toReturn = new HashMap(); - try { - Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " + - "where $profile/Profile/SecondaryType/string() eq '"+ "ApplicationProfile" + "' and $profile/Profile/Name/string() " + - " eq '" + resourceName + "'" + - "return $profile"); - - DiscoveryClient client = client(); - List appProfile = client.submit(q); - - if (appProfile == null || appProfile.size() == 0) - throw new Exception("Your applicationProfile is not registered in the infrastructure"); - else { - - String elem = appProfile.get(0); - DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement(); - XPathHelper helper = new XPathHelper(node); - - NodeList nodeListKeys = helper.evaluateForNodes("//originalKey"); - NodeList nodeListModifiedKeys = helper.evaluateForNodes("//modifiedKey"); - int sizeKeys = nodeListKeys != null ? nodeListKeys.getLength() : 0; - int sizeKeysModifed = nodeListModifiedKeys != null ? nodeListModifiedKeys.getLength() : 0; - if(sizeKeys != sizeKeysModifed) - throw new Exception("Malformed XML"); - logger.debug("Size is " + sizeKeys); - for (int i = 0; i < sizeKeys; i++) { - toReturn.put(nodeListKeys.item(i).getTextContent(), nodeListModifiedKeys.item(i).getTextContent()); - } - } - } catch (Exception e) { - logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e); - return null; + // check if data are in cache + if(namespacesCache.get(resourceName) != null){ + return namespacesCache.get(resourceName); } + else{ + try { + Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+ "ApplicationProfile" + "' and $profile/Profile/Name/string() " + + " eq '" + resourceName + "'" + + "return $profile"); - logger.debug("Map is " + toReturn); - return toReturn; + DiscoveryClient client = client(); + List appProfile = client.submit(q); + + if (appProfile == null || appProfile.size() == 0) + throw new Exception("Your applicationProfile is not registered in the infrastructure"); + else { + + String elem = appProfile.get(0); + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement(); + XPathHelper helper = new XPathHelper(node); + + NodeList nodeListKeys = helper.evaluateForNodes("//originalKey"); + NodeList nodeListModifiedKeys = helper.evaluateForNodes("//modifiedKey"); + int sizeKeys = nodeListKeys != null ? nodeListKeys.getLength() : 0; + int sizeKeysModifed = nodeListModifiedKeys != null ? nodeListModifiedKeys.getLength() : 0; + if(sizeKeys != sizeKeysModifed) + throw new Exception("Malformed XML"); + logger.debug("Size is " + sizeKeys); + for (int i = 0; i < sizeKeys; i++) { + toReturn.put(nodeListKeys.item(i).getTextContent(), nodeListModifiedKeys.item(i).getTextContent()); + } + } + logger.debug("Map is " + toReturn); + namespacesCache.insert(resourceName, toReturn); + return toReturn; + } catch (Exception e) { + logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e); + return null; + } + } } - + /** * Replace the extras' keys if needed * @param customFields @@ -542,11 +553,11 @@ public abstract class HelperMethods { */ public static Map> replaceFieldsKey(Map> customFields, Map namespaces) { - + Map> toReturn = new HashMap>(); - + Iterator>> iterator = customFields.entrySet().iterator(); - + while (iterator.hasNext()) { Map.Entry> entry = (Map.Entry>) iterator .next(); @@ -555,7 +566,7 @@ public abstract class HelperMethods { else toReturn.put(entry.getKey(), entry.getValue()); } - + return toReturn; } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/AssociationToGroupThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/AssociationToGroupThread.java index 8d44638..553d711 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/AssociationToGroupThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/AssociationToGroupThread.java @@ -27,21 +27,25 @@ public class AssociationToGroupThread extends Thread { private String username; private DataCatalogue catalogue; private String organizationId; + private String apiKey; /** - * @param groupTitle + * + * @param groupsTitles * @param datasetId - * @param username * @param organizationId + * @param username * @param catalogue + * @param apiKey */ public AssociationToGroupThread(List groupsTitles, String datasetId, String organizationId, - String username, DataCatalogue catalogue) { + String username, DataCatalogue catalogue, String apiKey) { this.groupsTitles = groupsTitles; this.datasetId = datasetId; this.username = username; this.catalogue = catalogue; this.organizationId = organizationId; + this.apiKey = apiKey; } @Override @@ -52,16 +56,14 @@ public class AssociationToGroupThread extends Thread { logger.info("Association thread started to put the dataset with id="+ datasetId + " into group with title(s) " + groupsTitles + " for user " + username); // find parents' groups - String userApiKey = catalogue.getApiKeyFromUsername(username); - findHierarchy(groupsTitles, catalogue, userApiKey); + findHierarchy(groupsTitles, catalogue, apiKey); Set uniqueGroups = new HashSet(groupsTitles); - + logger.info("Full set of groups is " + uniqueGroups); - // retrieve the role to be assigned according the one the user has into the organization of the dataset - RolesCkanGroupOrOrg role = RolesCkanGroupOrOrg.valueOf(catalogue.getRoleOfUserInOrganization(username, organizationId, userApiKey).toUpperCase()); + RolesCkanGroupOrOrg role = RolesCkanGroupOrOrg.valueOf(catalogue.getRoleOfUserInOrganization(username, organizationId, apiKey).toUpperCase()); if(!role.equals(RolesCkanGroupOrOrg.ADMIN)) role = RolesCkanGroupOrOrg.MEMBER; // decrease the role to member if it is not an admin @@ -76,7 +78,7 @@ public class AssociationToGroupThread extends Thread { continue; } else{ - boolean putIntoGroup = catalogue.assignDatasetToGroup(groupTitle, datasetId, userApiKey); + boolean putIntoGroup = catalogue.assignDatasetToGroup(groupTitle, datasetId, apiKey); logger.info("Was product put into group " + groupTitle + "? " + putIntoGroup); } } @@ -98,10 +100,10 @@ public class AssociationToGroupThread extends Thread { String apiKey) { ListIterator iterator = groupsTitles.listIterator(); - + while (iterator.hasNext()) { String group = (String) iterator.next(); - + List parents = catalogue.getParentGroups(group, apiKey); if(parents == null || parents.isEmpty()) @@ -110,13 +112,13 @@ public class AssociationToGroupThread extends Thread { for (CkanGroup ckanGroup : parents) { List parentsList = new ArrayList(Arrays.asList(ckanGroup.getName())); findHierarchy(parentsList, catalogue, apiKey); - + for (String parent : parentsList) { iterator.add(parent); } } } - + } -} +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index 97a68af..970f18f 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -41,7 +41,6 @@ import eu.trentorise.opendata.jackan.model.CkanResourceBase; */ public class ManageTimeSeriesThread extends Thread{ - private static final String CSV_FILE_FORMAT = ".csv"; private static final String PATH_SEPARATOR = "/"; // Logger @@ -50,7 +49,7 @@ public class ManageTimeSeriesThread extends Thread{ // try to attach the source at most CANCHES times .. private static final int CANCHES = 3; - private static CacheInterface vreFolderCache = new CacheImpl(1000 * 60 * 120); + private static CacheInterface vreFolderCache = new CacheImpl(1000 * 60 * 60 * 24); private Common record; private String uuidKB; @@ -164,7 +163,7 @@ public class ManageTimeSeriesThread extends Thread{ char firstLetter = uuidKB.charAt(0); // the whole path of the directory is going to be... - String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB) + PATH_SEPARATOR + "csv"; + String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB) + PATH_SEPARATOR + CSVHelpers.CSV_EXTENSION.replace(".", ""); logger.debug("The path under which the time series are going to be saved is " + csvDirectoryForThisProduct); WorkspaceFolder csvFolder = HelperMethods.createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct); @@ -187,7 +186,7 @@ public class ManageTimeSeriesThread extends Thread{ CustomField customAnnotation = field.getAnnotation(CustomField.class); logger.debug("A time series has been just found (from field " + customAnnotation.key() + ")"); - String resourceToAttachOnCkanName = (replaceIllegalChars(productName) + "_" + customAnnotation.key()).replaceAll("\\s", "_").replaceAll("[_]+", "_") + CSV_FILE_FORMAT; + String resourceToAttachOnCkanName = (replaceIllegalChars(productName) + "_" + customAnnotation.key()).replaceAll("\\s", "_").replaceAll("[_]+", "_") + CSVHelpers.CSV_EXTENSION; String resourceToAttachOnCkanDescription = productName + " : " + customAnnotation.key() + " time series"; File csvFile = CSVHelpers.listToCSV(asList); @@ -206,7 +205,7 @@ public class ManageTimeSeriesThread extends Thread{ if(ckanResource != null){ if(createdFileOnWorkspace == null) - createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, uuidKB + CSV_FILE_FORMAT, resourceToAttachOnCkanDescription, csvFile); + createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, uuidKB + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); if(createdFileOnWorkspace != null){ @@ -265,4 +264,4 @@ public class ManageTimeSeriesThread extends Thread{ resourceToAttachName, description); } -} +} \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index a030303..94add68 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -36,7 +36,7 @@ Context of products no longer under manage activities PublicVRE - /gcube/devNext/NextNext + /gcube/devsec/devVRE diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index b45c857..45e0147 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -475,7 +475,7 @@ public class JTests { public void testAssociationThread() throws InterruptedException{ String name = "low-abundance"; DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance("/gcube/devNext/NextNext"); - AssociationToGroupThread threadGroups = new AssociationToGroupThread(Arrays.asList(name), "another-test-test-please-ignore", "grsf", "costantino_perciante", catalogue); + AssociationToGroupThread threadGroups = new AssociationToGroupThread(Arrays.asList(name), "another-test-test-please-ignore", "grsf", "costantino_perciante", catalogue, "apiKey"); threadGroups.start(); threadGroups.join(); logger.info("Thread stopped!");