From 9774bcd2ef8c918cda5d46a3b2675bba926ab78a Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 3 Sep 2018 09:07:06 +0000 Subject: [PATCH] Merged changes made in branch git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@171271 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 4 + pom.xml | 2 +- .../custom_annotations/Group.java | 12 +- .../json/input/record/Common.java | 3 + .../json/input/record/FisheryRecord.java | 6 +- .../json/input/record/StockRecord.java | 11 +- .../utils/CommonServiceUtils.java | 524 ++++++++++-------- 7 files changed, 316 insertions(+), 246 deletions(-) diff --git a/distro/changelog.xml b/distro/changelog.xml index 15b2f90..c90107d 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -9,6 +9,10 @@ Added 'Connected'-'Not Connected' tag to GRSF Records #11766 Added group for SDG flag #11767 Added citation field #11811 + Added sub-groups support for available time series related to GRSF Type "Assessment Unit" #11832 + Added Biomass group #11967 + Changed 'State and trend of Marine Resource' to 'State and Trend' #11968 + Changed 'Scientific advice' to 'Scientific Advice' #11969 diff --git a/pom.xml b/pom.xml index bfc393d..b4d03b0 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.gcube.data-catalogue grsf-publisher-ws - 1.6.0-SNAPSHOT + 1.7.0-SNAPSHOT war grsf-publisher-ws diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/Group.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/Group.java index 305b523..3b9a219 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/Group.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/Group.java @@ -7,12 +7,13 @@ import java.lang.annotation.Target; /** * Group annotation - * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + * @author Costantino Perciante (ISTI - CNR) + * @author Luca Frosini (ISTI - CNR) */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) public @interface Group { - + /** * Define a REGEX condition to be checked before the record * is actually added to the group. @@ -26,4 +27,11 @@ public @interface Group { */ String groupNameOverValue() default ""; + /** + * When the group is created the source is prepended to the group name. + * Set to false to avoid source prepending + * @return + */ + boolean prependSourceToGroupName() default true; + } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Common.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Common.java index 154887c..a694ed9 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Common.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Common.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; * Information that both Stock and Fishery records must contain. * @author Costantino Perciante (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) + * */ public abstract class Common extends Base{ @@ -72,12 +73,14 @@ public abstract class Common extends Base{ @JsonProperty(Constants.CATCHES_JSON_KEY) @CustomField(key=Constants.CATCHES_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.CATCHES_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> catches; @JsonProperty(Constants.LANDINGS_JSON_KEY) @CustomField(key=Constants.LANDINGS_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.LANDINGS_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> landings; diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java index 412447c..e8de0e0 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java @@ -16,10 +16,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** * A fishery record bean. - * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + * @author Costantino Perciante (ISTI - CNR) + * @author Luca Frosini (ISTI - CNR) */ -public class FisheryRecord extends Common{ +public class FisheryRecord extends Common { + @JsonProperty(Constants.FISHERY_NAME_JSON_KEY) @NotNull(message="fishery_name cannot be null") @Size(min=1, message="fishery_name cannot be empty") diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java index d557ea9..16ac9d3 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java @@ -20,7 +20,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** * A stock record bean. - * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + * @author Costantino Perciante (ISTI - CNR) + * @author Luca Frosini (ISTI - CNR) */ public class StockRecord extends Common{ @@ -50,47 +51,55 @@ public class StockRecord extends Common{ @JsonProperty(Constants.FIRMS_ABUNDANCE_LEVEL_JSON_KEY) @CustomField(key=Constants.FIRMS_ABUNDANCE_LEVEL_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.FIRMS_ABUNDANCE_LEVEL_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> abundanceLevelStandard; @JsonProperty(Constants.ABUNDANCE_LEVEL_JSON_KEY) @CustomField(key=Constants.ABUNDANCE_LEVEL_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.ABUNDANCE_LEVEL_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> abundanceLevel; @JsonProperty(Constants.BIOMASS_JSON_KEY) @CustomField(key=Constants.BIOMASS_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.BIOMASS_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> biomass; @JsonProperty(Constants.FISHING_PRESSURE_FIRMS_JSON_KEY) @CustomField(key=Constants.FISHING_PRESSURE_FIRMS_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.FISHING_PRESSURE_FIRMS_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> fishingPressureStandard; @JsonProperty(Constants.FISHING_PRESSURE_JSON_KEY) @CustomField(key=Constants.FISHING_PRESSURE_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.FISHING_PRESSURE_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> fishingPressure; @JsonProperty(Constants.STATE_AND_TREND_MARINE_RESOURCE_JSON_KEY) @CustomField(key=Constants.STATE_AND_TREND_MARINE_RESOURCE_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.STATE_AND_TREND_MARINE_RESOURCE_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> narrativeState; @JsonProperty(Constants.FAO_CATEGORIES_JSON_KEY) @CustomField(key=Constants.FAO_CATEGORIES_CUSTOM_KEY) @TimeSeries + @Group(groupNameOverValue=Constants.FAO_CATEGORIES_CUSTOM_KEY, prependSourceToGroupName=false) @Valid private List> faoState; @JsonProperty(Constants.SCIENTIFIC_ADVICE_JSON_KEY) @CustomField(key=Constants.SCIENTIFIC_ADVICE_CUSTOM_KEY) + @Group(groupNameOverValue=Constants.SCIENTIFIC_ADVICE_CUSTOM_KEY, prependSourceToGroupName=false) private List scientificAdvice; @JsonProperty(Constants.ASSESSOR_JSON_KEY) diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java index 12f55ca..af97d2c 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java @@ -36,6 +36,7 @@ import org.gcube.datacatalogue.common.Constants; import org.gcube.datacatalogue.common.enums.Product_Type; import org.gcube.datacatalogue.common.enums.Sources; import org.gcube.datacatalogue.common.enums.Status; +import org.gcube.datacatalogue.common.enums.Stock_Type; import org.json.simple.JSONObject; import org.slf4j.LoggerFactory; @@ -44,75 +45,77 @@ import eu.trentorise.opendata.jackan.model.CkanLicense; /** * Services common utils. - * @author Costantino Perciante at ISTI-CNR + * @author Costantino Perciante (ISTI - CNR) + * @author Luca Frosini (ISTI - CNR) */ @SuppressWarnings({"rawtypes", "unchecked"}) public class CommonServiceUtils { - + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonServiceUtils.class); private static final int TAG_MAX_SIZE = 100; - private static Map extensionsCheck = new ConcurrentHashMap<>(); - + private static Map extensionsCheck = new ConcurrentHashMap<>(); + /** * Retrieve the list of licenses for stocks and fisheries * @return */ - public static Map getLicenses(DataCatalogue catalogue){ + public static Map getLicenses(DataCatalogue catalogue) { logger.info("Requested licenses..."); - Map toReturn = new HashMap(); + Map toReturn = new HashMap(); List licenses = catalogue.getLicenses(); - - for (CkanLicense ckanLicense : licenses) { + + for(CkanLicense ckanLicense : licenses) { toReturn.put(ckanLicense.getId(), ckanLicense.getTitle()); } return toReturn; } - + /** * Validate an aggregated GRSF record. TODO use @Valid tags * @throws Exception */ public static void validateAggregatedRecord(Common record) throws Exception { - + List refersToList = record.getRefersTo(); String shortTitle = record.getShortName(); Boolean traceabilityFlag = record.isTraceabilityFlag(); Status status = record.getStatus(); - + if(refersToList == null || refersToList.isEmpty()) throw new Exception("refers_to cannot be null/empty"); - + if(traceabilityFlag == null) throw new Exception("traceability_flag cannot be null"); - + if(shortTitle == null || shortTitle.isEmpty()) throw new Exception("short_title cannot be null/empty"); - + if(status == null) throw new Exception("status cannot be null/empty"); - + // check if it is a stock and perform related checks - if(record.getClass().equals(StockRecord.class)){ - + if(record.getClass().equals(StockRecord.class)) { + StockRecord stock = (StockRecord) record; List species = stock.getSpecies(); if(species == null || species.isEmpty()) throw new Exception("species cannot be null/empty in a GRSF record"); } - + // check if it is a stock and perform related checks - if(record.getClass().equals(FisheryRecord.class)){ - + if(record.getClass().equals(FisheryRecord.class)) { + FisheryRecord fishery = (FisheryRecord) record; - + List fishingArea = fishery.getFishingArea(); List jurisdictionArea = fishery.getJurisdictionArea(); - - if((fishingArea == null || fishingArea.isEmpty()) && (jurisdictionArea == null || jurisdictionArea.isEmpty())) + + if((fishingArea == null || fishingArea.isEmpty()) + && (jurisdictionArea == null || jurisdictionArea.isEmpty())) throw new Exception("fishing_area and jurisdiction_area cannot be null/empty at the same time!"); } } - + /** * Parse the record to look up tags, groups and resources * @param tags @@ -126,217 +129,248 @@ public class CommonServiceUtils { * @param username * @param source */ - public static void getTagsGroupsResourcesExtrasByRecord( - Set tags, - boolean skipTags, - Set groups, - boolean skipGroups, - List resources, - boolean skipResources, - Map> extras, - Base record, - String username, - Sources source // it comes from the source type e.g., "grsf-", "ram-" .. - ){ - + public static void getTagsGroupsResourcesExtrasByRecord(Set tags, boolean skipTags, Set groups, + boolean skipGroups, List resources, boolean skipResources, Map> extras, + Base record, String username, Sources source // it comes from the source type e.g., "grsf-", "ram-" .. + ) { + Class current = record.getClass(); - do{ + do { Field[] fields = current.getDeclaredFields(); - for (Field field : fields) { - + for(Field field : fields) { + if(!skipTags) getTagsByField(field, current, record, tags); - + if(!skipGroups) getGroupsByField(field, current, record, groups, source); - + getExtrasByField(field, current, record, extras, source); - + if(!skipResources) getResourcesByField(field, current, record, username, resources); - + } - } - while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class - + } while((current = current.getSuperclass()) != null); // start from the inherited class up to the Object.class + logger.debug("Tags are " + tags); logger.debug("Groups are " + groups); logger.debug("Extras are " + extras); logger.debug("Resources without timeseries are " + resources); } - + /** * Retrieve the list of tags for this object */ - private static void getTagsByField(Field field, Class current, Base record, Set tags){ - if(field.isAnnotationPresent(Tag.class)){ - try{ + private static void getTagsByField(Field field, Class current, Base record, Set tags) { + if(field.isAnnotationPresent(Tag.class)) { + try { Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - if(f instanceof List){ + if(f != null) { + if(f instanceof List) { List asList = ((List) f); - if(!asList.isEmpty()){ - + if(!asList.isEmpty()) { + logger.debug("The object annotated with @Tag is a list. Adding ... "); - + int elementsToConsider = asList.size(); - + // check if it is a time series, in this take the last X elements - if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ - elementsToConsider = Math.min(elementsToConsider, Constants.TIME_SERIES_TAKE_LAST_VALUES); - for (int i = 0; i < elementsToConsider; i++) { - String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS, ""); + if(asList.get(0).getClass().equals(TimeSeriesBean.class)) { + elementsToConsider = Math.min(elementsToConsider, + Constants.TIME_SERIES_TAKE_LAST_VALUES); + for(int i = 0; i < elementsToConsider; i++) { + String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS, + ""); if(finalTag.length() <= TAG_MAX_SIZE) - tags.add(finalTag); + tags.add(finalTag); } - }else{ + } else { // else add all the available elements - for (int i = 0; i < elementsToConsider; i++) { - String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS, ""); + for(int i = 0; i < elementsToConsider; i++) { + String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS, + ""); if(finalTag.length() <= TAG_MAX_SIZE) - tags.add(finalTag); + tags.add(finalTag); } } } - }else{ + } else { logger.debug("The object annotated with @Tag is a simple one. Adding ... "); String finalTag = f.toString().trim().replaceAll(Constants.REGEX_TAGS, ""); logger.debug(finalTag); if(finalTag.length() <= TAG_MAX_SIZE) - tags.add(finalTag); + tags.add(finalTag); } - + } - }catch(Exception e){ + } catch(Exception e) { logger.error("Failed to read value for field " + field.getName() + " skipping", e); } } } - + /** * Retrieve the list of groups' names for this object */ - private static void getGroupsByField(Field field, Class current, Base record, Set groups, Sources source){ - if(field.isAnnotationPresent(Group.class)){ - String conditionToCheck = field.getAnnotation(Group.class).condition(); - String groupNameOverValue = field.getAnnotation(Group.class).groupNameOverValue(); - try{ + private static void getGroupsByField(Field field, Class current, Base record, Set groups, + Sources source) { + if(field.isAnnotationPresent(Group.class)) { + Group group = field.getAnnotation(Group.class); + String conditionToCheck = group.condition(); + String groupNameOverValue = group.groupNameOverValue(); + + + + // See https://support.d4science.org/issues/11832 + boolean assessmentUnit = false; + boolean prependSource = group.prependSourceToGroupName(); + if(record instanceof StockRecord) { + StockRecord stockRecord = (StockRecord) record; + Stock_Type stock_Type = stockRecord.getType(); + if(stock_Type != Stock_Type.Assessment_Unit) { + prependSource = false; + }else { + assessmentUnit = true; + } + } + // end patch for https://support.d4science.org/issues/11832 + + try { Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - if(f instanceof List){ + if(f != null) { + if(f instanceof List) { List asList = ((List) f); - if(!asList.isEmpty()){ - + if(!asList.isEmpty()) { + logger.debug("The object annotated with @Group is a list. Adding ... "); - + // else add all the available elements - for (int i = 0; i < asList.size(); i++) { - boolean match = conditionToCheck.isEmpty() ? true : asList.get(i).toString().trim().matches(conditionToCheck); - if(match){ - String groupName = groupNameOverValue.isEmpty() ? - HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + asList.get(i).toString().trim()) : - source.toString().toLowerCase() + "-" + groupNameOverValue; - groups.add(groupName); + for(int i = 0; i < asList.size(); i++) { + boolean match = conditionToCheck.isEmpty() ? true + : asList.get(i).toString().trim().matches(conditionToCheck); + if(match) { + String groupName = groupNameOverValue.isEmpty() + ? HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + + asList.get(i).toString().trim()) + : source.toString().toLowerCase() + "-" + groupNameOverValue; + if(assessmentUnit && !prependSource) { + groups.add(groupNameOverValue); + }else { + groups.add(groupName); + } } } - + } - }else{ - + + } else { + // also convert to the group name that should be on ckan - boolean match = conditionToCheck.isEmpty() ? true : f.toString().trim().matches(conditionToCheck); - if(match){ - - String groupName = groupNameOverValue.isEmpty() ? - HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + f.toString().trim()) : - source.toString().toLowerCase() + "-" + groupNameOverValue; - groups.add(groupName); - + boolean match = conditionToCheck.isEmpty() ? true + : f.toString().trim().matches(conditionToCheck); + if(match) { + + String groupName = groupNameOverValue.isEmpty() + ? HelperMethods.getGroupNameOnCkan( + source.toString().toLowerCase() + "-" + f.toString().trim()) + : source.toString().toLowerCase() + "-" + groupNameOverValue; + + if(assessmentUnit && !prependSource) { + groups.add(groupNameOverValue); + }else { + groups.add(groupName); + } + + + } } } - - }catch(Exception e){ + + } catch(Exception e) { logger.error("Failed to read value for field " + field.getName() + " skipping", e); } } - + } - + /** * Retrieve the list of extras for this object * @param source */ - private static void getExtrasByField(Field field, Class current, Base record, Map> extras, Sources source){ - if(field.isAnnotationPresent(CustomField.class)){ - try{ + private static void getExtrasByField(Field field, Class current, Base record, Map> extras, + Sources source) { + if(field.isAnnotationPresent(CustomField.class)) { + try { Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); String keyField = field.getAnnotation(CustomField.class).key(); - + // manage no connections nor similar grsf records here for GRSF records only - if(source.equals(Sources.GRSF) && keyField.equals(Constants.SIMILAR_GRSF_RECORDS_CUSTOM_KEY)){ - List asList = (List)f; - if(asList == null || asList.isEmpty()){ + if(source.equals(Sources.GRSF) && keyField.equals(Constants.SIMILAR_GRSF_RECORDS_CUSTOM_KEY)) { + List asList = (List) f; + if(asList == null || asList.isEmpty()) { extras.put(keyField, Arrays.asList(Constants.NO_SIMILAR_GRSF_RECORDS)); return; } - + } - - if(source.equals(Sources.GRSF) && keyField.equals(Constants.CONNECTED_CUSTOM_KEY)){ - List asList = (List)f; - if(asList == null || asList.isEmpty()){ + + if(source.equals(Sources.GRSF) && keyField.equals(Constants.CONNECTED_CUSTOM_KEY)) { + List asList = (List) f; + if(asList == null || asList.isEmpty()) { extras.put(keyField, Arrays.asList(Constants.NO_CONNECTED_RECORDS)); return; } } - - if(f != null){ + + if(f != null) { Set valuesForKey = null; - + // check if the map already contains this key if(extras.containsKey(keyField)) valuesForKey = new HashSet(extras.get(keyField)); else valuesForKey = new HashSet(); - - if(f instanceof List){ - logger.debug("The object " + field.getName() + " is a list and is annotated with @CustomField. Adding ..."); - List asList = (List)f; - if(!asList.isEmpty()){ - + + if(f instanceof List) { + logger.debug("The object " + field.getName() + + " is a list and is annotated with @CustomField. Adding ..."); + List asList = (List) f; + if(!asList.isEmpty()) { + int elementsToConsider = asList.size(); - + // check if it is a time series, in this case take the last X elements - if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ - elementsToConsider = Math.min(elementsToConsider, Constants.TIME_SERIES_TAKE_LAST_VALUES); - for (int i = 0; i < elementsToConsider; i++) { + if(asList.get(0).getClass().equals(TimeSeriesBean.class)) { + elementsToConsider = Math.min(elementsToConsider, + Constants.TIME_SERIES_TAKE_LAST_VALUES); + for(int i = 0; i < elementsToConsider; i++) { // trim and remove html - String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); valuesForKey.add(clean); } - } - else - for (int i = 0; i < elementsToConsider; i++) { - String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + } else + for(int i = 0; i < elementsToConsider; i++) { + String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); valuesForKey.add(clean); } } - - }else{ - String clean = HelperMethods.removeHTML(f.toString().trim()); + + } else { + String clean = HelperMethods.removeHTML(f.toString().trim()); valuesForKey.add(clean); } - + // add to the map extras.put(keyField, new ArrayList(valuesForKey)); } - }catch(Exception e){ + } catch(Exception e) { logger.error("Failed to read value for field " + field.getName() + " skipping", e); } } } - + /** * Retrieve the ResourceBean given the record (extract resources from Database Sources and Source of Information and others) * @param record @@ -345,47 +379,52 @@ public class CommonServiceUtils { * @param resources * @return */ - private static void getResourcesByField(Field field, Class current, Base record, String username, List resources){ - if(field.isAnnotationPresent(CkanResource.class)){ - try{ + private static void getResourcesByField(Field field, Class current, Base record, String username, + List resources) { + if(field.isAnnotationPresent(CkanResource.class)) { + try { Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - - if(f instanceof List){ - - List listOfResources = (List)f; - - for (Resource resource : listOfResources) { - resources.add(new ResourceBean(resource.getUrl(), resource.getName().toString(), resource.getDescription(), null, username, null, null)); + if(f != null) { + + if(f instanceof List) { + + List listOfResources = (List) f; + + for(Resource resource : listOfResources) { + resources.add(new ResourceBean(resource.getUrl(), resource.getName().toString(), + resource.getDescription(), null, username, null, null)); } - - }else{ - - Resource res = (Resource)f; - resources.add(new ResourceBean(res.getUrl(), res.getName().toString(), res.getDescription(), null, username, null, null)); - + + } else { + + Resource res = (Resource) f; + resources.add(new ResourceBean(res.getUrl(), res.getName().toString(), res.getDescription(), + null, username, null, null)); + } } - }catch(Exception e){ + } catch(Exception e) { logger.error("Failed to read value for field " + field.getName() + " skipping", e); } } } - + /** * Evaluate if the user has the admin role * Throws exception if he/she doesn't */ - public static void hasAdminRole(String username, DataCatalogue catalogue, String apiKey, String organization) throws Exception{ - + public static void hasAdminRole(String username, DataCatalogue catalogue, String apiKey, String organization) + throws Exception { + String role = catalogue.getRoleOfUserInOrganization(username, organization, apiKey); logger.info("Role of the user " + username + " is " + role + " in " + organization); - + if(role == null || role.isEmpty() || !role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())) - throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!"); - + throw new Exception( + "You are not authorized to create a product. Please check you have the Catalogue-Administrator role!"); + } - + /** * Check this record's name * @param futureName @@ -393,22 +432,23 @@ public class CommonServiceUtils { * @throws Exception on name check */ public static void checkName(String futureName, DataCatalogue catalogue) throws Exception { - - if(!HelperMethods.isNameValid(futureName)){ - throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'"); - }else{ - + + if(!HelperMethods.isNameValid(futureName)) { + throw new Exception( + "The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'"); + } else { + logger.debug("Checking if such name [" + futureName + "] doesn't exist ..."); boolean alreadyExists = catalogue.existProductWithNameOrId(futureName); - - if(alreadyExists){ + + if(alreadyExists) { logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists"); throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists"); - + } } } - + /** * Validate and check sources * @param apiKey @@ -425,57 +465,62 @@ public class CommonServiceUtils { * @throws Exception */ public static void validateRecordAndMapFields(String apiKey, String context, ServletContext contextServlet, - Sources sourceInPath, Common record, Product_Type productType, Set tags, Map> customFields, - Set groups, List resources, String username, String futureTitle) throws Exception { - + Sources sourceInPath, Common record, Product_Type productType, Set tags, + Map> customFields, Set groups, List resources, String username, + String futureTitle) throws Exception { + // validate the record if it is a GRSF one and set the record type and in manage context // Status field is needed only in the Manage context for GRSF records - if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){ - if(sourceInPath.equals(Sources.GRSF)){ - + if(context.equals((String) contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))) { + if(sourceInPath.equals(Sources.GRSF)) { + List refersTo = record.getRefersTo(); if(refersTo == null || refersTo.isEmpty()) throw new Exception("refers_to is empty for a GRSF record"); - + Set sourcesList = new HashSet(); - + String databaseSource = ""; // we have the id within the catalog of this record. This means that we can retrieve the record and its system:type - for (RefersToBean refersToBean : refersTo) { + for(RefersToBean refersToBean : refersTo) { String sourceOrganization = getRecordOrganization(refersToBean.getId(), apiKey, context); - resources.add(new ResourceBean(refersToBean.getUrl(), sourceOrganization , "", null, username, null, null)); + resources.add(new ResourceBean(refersToBean.getUrl(), sourceOrganization, "", null, username, null, + null)); sourcesList.add(sourceOrganization.toLowerCase()); databaseSource += sourceOrganization + " "; } - + // create the Database Source information customFields.put(Constants.GRSF_DATABASE_SOURCE, Arrays.asList(databaseSource.trim())); - + // append to groups: we need to add this record to the correspondent group of the sources addRecordToGroupSources(groups, new ArrayList(sourcesList), productType, sourceInPath); - + // validate CommonServiceUtils.validateAggregatedRecord(record); } } - + // set the domain record.setDomain(productType.getOrigName()); - + // set system type (it is equal to the GRSF Type for GRSF records, "Legacy" for source records) - record.setSystemType(sourceInPath.equals(Sources.GRSF) ? - productType.equals(Product_Type.FISHERY) ? ((FisheryRecord)record).getType().getOrigName() : ((StockRecord)record).getType().getOrigName() + record.setSystemType( + sourceInPath.equals(Sources.GRSF) + ? productType.equals(Product_Type.FISHERY) ? ((FisheryRecord) record).getType().getOrigName() + : ((StockRecord) record).getType().getOrigName() : Constants.SYSTEM_TYPE_FOR_SOURCES_VALUE); - + logger.debug("Domain is " + productType.getOrigName() + " and system type " + record.getSystemType()); - + // evaluate the custom fields/tags, resources and groups groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + productType.getOrigName().toLowerCase()); //e.g. grsf-fishery boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records - CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath); - + CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, + customFields, record, username, sourceInPath); + } - + /** * Add the record to the group of sources * @param groups @@ -483,18 +528,18 @@ public class CommonServiceUtils { * @param productType * @param sourceInPath */ - private static void addRecordToGroupSources(Set groups, - List sourcesList, Product_Type productType, Sources sourceInPath) { - + private static void addRecordToGroupSources(Set groups, List sourcesList, Product_Type productType, + Sources sourceInPath) { + Collections.sort(sourcesList); // be sure the name are sorted because the groups have been generated this way String groupName = sourceInPath.getOrigName().toLowerCase() + "-" + productType.getOrigName().toLowerCase(); - for (String source : sourcesList) { + for(String source : sourcesList) { groupName += "-" + source; } - + groups.add(groupName); } - + /** * Fetch the system:type property from a record * @param itemIdOrName @@ -502,8 +547,8 @@ public class CommonServiceUtils { * @return null on error * @throws Exception */ - public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{ - + public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception { + DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context); CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey); if(dataset == null) @@ -513,10 +558,10 @@ public class CommonServiceUtils { throw new Exception(Constants.SYSTEM_TYPE_CUSTOM_KEY + " property not set in record " + itemIdOrName); else return systemTypeValue; - + } - - public static String getRecordOrganization(String itemIdOrName, String apiKey, String context) throws Exception{ + + public static String getRecordOrganization(String itemIdOrName, String apiKey, String context) throws Exception { DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context); CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey); if(dataset == null) @@ -524,7 +569,7 @@ public class CommonServiceUtils { else return dataset.getOrganization().getTitle(); } - + /** * Actions to execute once the dataset has been updated or created. * @param responseBean @@ -539,71 +584,69 @@ public class CommonServiceUtils { * @param partialDescription * @throws InterruptedException */ - public static void actionsPostCreateOrUpdate( - final String datasetId, final String futureName, final Common record, final String apiKey, final String username, final String organization, String itemUrl, - ResponseCreationBean responseBean, final DataCatalogue catalogue, - Map namespaces, final Set groups, final String context, - final String token, final String futureTitle, final String authorFullname, final ServletContext contextServlet, final boolean isUpdated, + public static void actionsPostCreateOrUpdate(final String datasetId, final String futureName, final Common record, + final String apiKey, final String username, final String organization, String itemUrl, + ResponseCreationBean responseBean, final DataCatalogue catalogue, Map namespaces, + final Set groups, final String context, final String token, final String futureTitle, + final String authorFullname, final ServletContext contextServlet, final boolean isUpdated, String description) throws InterruptedException { - + // on create, we need to add the item url... the description can be set on create and update instead - if(!isUpdated){ + if(!isUpdated) { itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName); - Map> addField = new HashMap>(); - String modifiedUUIDKey = namespaces.containsKey(Constants.ITEM_URL_FIELD) ? namespaces.get(Constants.ITEM_URL_FIELD) : Constants.ITEM_URL_FIELD; + Map> addField = new HashMap>(); + String modifiedUUIDKey = namespaces.containsKey(Constants.ITEM_URL_FIELD) + ? namespaces.get(Constants.ITEM_URL_FIELD) + : Constants.ITEM_URL_FIELD; addField.put(modifiedUUIDKey, Arrays.asList(itemUrl)); catalogue.patchProductCustomFields(datasetId, apiKey, addField, false); } - + // update description anyway description += "Record URL: " + itemUrl; JSONObject obj = new JSONObject(); obj.put("notes", description); catalogue.patchProductWithJSON(datasetId, obj, apiKey); - + // set info in the response bean responseBean.setId(datasetId); responseBean.setItemUrl(itemUrl); responseBean.setKbUuid(record.getUuid()); - + // it is needed... final String itemUrlForThread = itemUrl; - + new Thread(new Runnable() { - + @Override public void run() { try { // manage groups (wait thread to die: ckan doesn't support too much concurrency on same record ...) - if(!groups.isEmpty()){ + if(!groups.isEmpty()) { logger.info("Launching thread for association to the list of groups " + groups); - AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList(groups), datasetId, organization, username, catalogue, apiKey); + AssociationToGroupThread threadGroups = new AssociationToGroupThread( + new ArrayList(groups), datasetId, organization, username, catalogue, apiKey); threadGroups.start(); threadGroups.join(); } // manage time series as resources logger.info("Launching thread for time series handling"); new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start(); - + // write a post if the product has been published in grsf context - if(!isUpdated && context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){ - new WritePostCatalogueManagerThread( - context, - token, - futureTitle, - itemUrlForThread, - true, - new ArrayList(), - authorFullname).start(); + if(!isUpdated && context + .equals((String) contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))) { + new WritePostCatalogueManagerThread(context, token, futureTitle, itemUrlForThread, true, + new ArrayList(), authorFullname).start(); logger.info("Thread to write a post about the new product has been launched"); } - }catch (InterruptedException e) { + } catch(InterruptedException e) { logger.error("Error", e); } } }).start(); } - + /** * Extend roles to other organization * @param username @@ -611,18 +654,19 @@ public class CommonServiceUtils { * @param organization * @param admin */ - public static void extendRoleToOtherOrganizations(String username, DataCatalogue catalogue, String organization, RolesCkanGroupOrOrg admin) { - + public static void extendRoleToOtherOrganizations(String username, DataCatalogue catalogue, String organization, + RolesCkanGroupOrOrg admin) { + logger.debug("Checking if role extension is needed here"); if(extensionsCheck.containsKey(username) && extensionsCheck.get(username)) return; - else{ + else { catalogue.assignRolesOtherOrganization(username, organization, admin); extensionsCheck.put(username, true); } - + } - + /** * Evaluate in which organization a record has to be published. The only exception is when grsf_admin is involved. * @param organization