diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 6caf699..6a68eaa 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -3,6 +3,9 @@ + + uses + diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java index 08ac149..4ec3c47 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class Base { private static Logger logger = LoggerFactory.getLogger(Base.class); - public static final String UUID_KB_KEY = "UUID Knowledge Base"; + public static final String UUID_KB_KEY = "GRSF UUID"; @JsonProperty("catalog_id") //used on patch/update product call private String catalogId; diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java index 3d5a835..ae637bb 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties(value = {"grsf_type", "source"}) public class Common extends Base{ - public static final String GRSF_TYPE_KEY = "GRSF type"; // stock, fishery + public static final String GRSF_DOMAIN_KEY = "GRSF Domain"; // stock, fishery public static final String SOURCE_KEY = "Source"; // in case it is a RAM/FIRMS/FishSource record it is not added // it is added in case of GRSF record @@ -46,7 +46,7 @@ public class Common extends Base{ private List refersTo; @JsonProperty("short_title") - @CustomField(key="Short Title") + @CustomField(key="Short Name") private String shortTitle; @JsonProperty("traceability_flag") @@ -55,25 +55,13 @@ public class Common extends Base{ private Boolean traceabilityFlag; @JsonProperty("status") - @CustomField(key="Status") + @CustomField(key="Status of the GRSF record") @Group private Status status; - @JsonProperty("reporting_year") - @CustomField(key="Reporting year") - @TimeSeries - @Valid - private List> reportingYear; - - @JsonProperty("reference_year") - @CustomField(key="Reference year") - @TimeSeries - @Valid - private List> referenceYear; - // automatically compiled @JsonProperty("grsf_type") - @CustomField(key=GRSF_TYPE_KEY) + @CustomField(key=GRSF_DOMAIN_KEY) private String grsfType; // automatically compiled @@ -98,7 +86,7 @@ public class Common extends Base{ private List species; @JsonProperty("similar_records") - @CustomField(key="Similar Records") + @CustomField(key="Similar GRSF Records") private List similarRecords; public Common() { @@ -113,8 +101,6 @@ public class Common extends Base{ * @param shortTitle * @param traceabilityFlag * @param status - * @param reportingYear - * @param referenceYear * @param grsfType * @param sourceType * @param catches @@ -126,9 +112,7 @@ public class Common extends Base{ List> databaseSources, List> sourceOfInformation, List refersTo, String shortTitle, - Boolean traceabilityFlag, Status status, - List> reportingYear, - List> referenceYear, String grsfType, + Boolean traceabilityFlag, Status status, String grsfType, String sourceType, List> catches, List> landings, List species, List similarRecords) { @@ -140,8 +124,6 @@ public class Common extends Base{ this.shortTitle = shortTitle; this.traceabilityFlag = traceabilityFlag; this.status = status; - this.reportingYear = reportingYear; - this.referenceYear = referenceYear; this.grsfType = grsfType; this.sourceType = sourceType; this.catches = catches; @@ -177,19 +159,6 @@ public class Common extends Base{ public void setRefersTo(List refersTo) { this.refersTo = refersTo; } - - public List> getReportingYear() { - return reportingYear; - } - - public void setReportingYear(List> reportingYear) { - this.reportingYear = reportingYear; - } - - public void setReferenceYear(List> referenceYear) { - this.referenceYear = referenceYear; - } - public List> getDatabaseSources() { return databaseSources; } @@ -262,10 +231,6 @@ public class Common extends Base{ this.dataOwner = dataOwner; } - public List> getReferenceYear() { - return referenceYear; - } - public List getSimilarRecords() { return similarRecords; } @@ -276,28 +241,14 @@ public class Common extends Base{ @Override public String toString() { - return "Common [" - + (dataOwner != null ? "dataOwner=" + dataOwner + ", " : "") - + (databaseSources != null ? "databaseSources=" - + databaseSources + ", " : "") - + (sourceOfInformation != null ? "sourceOfInformation=" - + sourceOfInformation + ", " : "") - + (refersTo != null ? "refersTo=" + refersTo + ", " : "") - + (shortTitle != null ? "shortTitle=" + shortTitle + ", " : "") - + (traceabilityFlag != null ? "traceabilityFlag=" - + traceabilityFlag + ", " : "") - + (status != null ? "status=" + status + ", " : "") - + (reportingYear != null ? "reportingYear=" + reportingYear - + ", " : "") - + (referenceYear != null ? "referenceYear=" + referenceYear - + ", " : "") - + (grsfType != null ? "grsfType=" + grsfType + ", " : "") - + (sourceType != null ? "sourceType=" + sourceType + ", " : "") - + (catches != null ? "catches=" + catches + ", " : "") - + (landings != null ? "landings=" + landings + ", " : "") - + (species != null ? "species=" + species + ", " : "") - + (similarRecords != null ? "similarRecords=" + similarRecords - : "") + "]"; + return "Common [dataOwner=" + dataOwner + ", databaseSources=" + + databaseSources + ", sourceOfInformation=" + + sourceOfInformation + ", refersTo=" + refersTo + + ", shortTitle=" + shortTitle + ", traceabilityFlag=" + + traceabilityFlag + ", status=" + status + ", grsfType=" + + grsfType + ", sourceType=" + sourceType + ", catches=" + + catches + ", landings=" + landings + ", species=" + species + + ", similarRecords=" + similarRecords + "]"; } } \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java index 7e8d9b2..c2be5fa 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/FisheryRecord.java @@ -24,11 +24,11 @@ public class FisheryRecord extends Common{ @JsonProperty("fishery_name") @NotNull(message="fishery_name cannot be null") @Size(min=1, message="fishery_name cannot be empty") - @CustomField(key="Fishery Name") + @CustomField(key="GRSF Name") private String fisheryName; @JsonProperty("fishery_id") - @CustomField(key="Fishery id") + @CustomField(key="GRSF Semantic identifier") private String fisheryId; @JsonProperty("fishing_area") @@ -36,11 +36,11 @@ public class FisheryRecord extends Common{ private List fishingArea; @JsonProperty("exploited_stocks") - @CustomField(key="Exploited stocks") + @CustomField(key="Resources Exploited") private List exploitingStocks; @JsonProperty("management_entity") - @CustomField(key="Management entity") + @CustomField(key="Management Body/Authority(ies)") private List managementEntity; @JsonProperty("jurisdiction_area") @@ -49,29 +49,21 @@ public class FisheryRecord extends Common{ private List jurisdictionArea; @JsonProperty("production_system_type") + @CustomField(key="Type of Production System") @Tag - @CustomField(key="Production system type") private List productionSystemType; @JsonProperty("flag_state") + @CustomField(key="Flag State") @Tag - @CustomField(key="Flag state") private List flagState; @JsonProperty("fishing_gear") @CustomField(key="Fishing gear") private List fishingGear; - @JsonProperty("environment") - @CustomField(key="Environment") - private String environment; - - @JsonProperty("fishery_uri") - @CustomField(key="Fishery Uri") - private String fisheryUri; - @JsonProperty("type") - @CustomField(key="Type") + @CustomField(key="GRSF Type") @Group private Fishery_Type type; @@ -89,16 +81,13 @@ public class FisheryRecord extends Common{ * @param productionSystemType * @param flagState * @param fishingGear - * @param environment - * @param fisheryUri * @param type */ public FisheryRecord(String fisheryName, String fisheryId, List fishingArea, List exploitingStocks, List managementEntity, List jurisdictionArea, List productionSystemType, - List flagState, List fishingGear, - String environment, String fisheryUri, Fishery_Type type) { + List flagState, List fishingGear, Fishery_Type type) { super(); this.fisheryName = fisheryName; this.fisheryId = fisheryId; @@ -109,19 +98,9 @@ public class FisheryRecord extends Common{ this.productionSystemType = productionSystemType; this.flagState = flagState; this.fishingGear = fishingGear; - this.environment = environment; - this.fisheryUri = fisheryUri; this.type = type; } - public String getFisheryUri() { - return fisheryUri; - } - - public void setFisheryUri(String fisheryUri) { - this.fisheryUri = fisheryUri; - } - public String getFisheryName() { return fisheryName; } @@ -154,14 +133,6 @@ public class FisheryRecord extends Common{ this.jurisdictionArea = jurisdictionArea; } - public String getEnvironment() { - return environment; - } - - public void setEnvironment(String environment) { - this.environment = environment; - } - public Fishery_Type getType() { return type; } @@ -220,7 +191,6 @@ public class FisheryRecord extends Common{ + ", jurisdictionArea=" + jurisdictionArea + ", productionSystemType=" + productionSystemType + ", flagState=" + flagState + ", fishingGear=" + fishingGear - + ", environment=" + environment + ", fisheryUri=" + fisheryUri + ", type=" + type + "]"; } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java index fb0f422..b05ca72 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java @@ -27,36 +27,32 @@ public class StockRecord extends Common{ @JsonProperty("stock_name") @NotNull(message="stock_name cannot be null") @Size(min=2, message="stock_name cannot be empty") - @CustomField(key="Stock Name") + @CustomField(key="GRSF Name") private String stockName; @JsonProperty("stock_id") - @CustomField(key="Stock id") + @CustomField(key="GRSF Semantic identifier") private String stockId; @JsonProperty("assessment_distribution_area") - @CustomField(key="Assessment distribution area") + @CustomField(key="Assessment Area") private List area; @JsonProperty("exploiting_fishery") - @CustomField(key="Exploiting fishery") + @CustomField(key="Exploiting Fishery") private List exploitingFishery; @JsonProperty("management_entity") - @CustomField(key="Management entity") + @CustomField(key="Management Body/Authority(ies)") @Tag private String managementEntity; @JsonProperty("assessment_methods") - @CustomField(key="Assessment methods") + @CustomField(key="Assessment Methods") private List assessmentMethods; - @JsonProperty("state_of_marine_resource") - @CustomField(key="State of marine resources") - private String stateOfMarineResource; - @JsonProperty("standard_abundance_level") - @CustomField(key="Standard Abundance Level") + @CustomField(key="Abundance Level (FIRMS Standard)") @TimeSeries @Valid private List> abundanceLevelStandard; @@ -68,7 +64,7 @@ public class StockRecord extends Common{ private List> abundanceLevel; @JsonProperty("standard_fishing_pressure") - @CustomField(key="Standard Fishing Pressure") + @CustomField(key="Fishing Pressure (FIRMS Standard)") @TimeSeries @Valid private List> fishingPressureStandard; @@ -80,13 +76,13 @@ public class StockRecord extends Common{ private List> fishingPressure; @JsonProperty("narrative_state") - @CustomField(key="Narrative state") + @CustomField(key="State and trend of Marine Resource") @TimeSeries @Valid private List> narrativeState; @JsonProperty("fao_state") - @CustomField(key="Fao State") + @CustomField(key="Fao Categories") @TimeSeries @Valid private List> faoState; @@ -100,17 +96,8 @@ public class StockRecord extends Common{ @Tag private String assessor; - @JsonProperty("stock_uri") - @CustomField(key="Stock Uri") - private String stockUri; - - @JsonProperty("water_area") - @CustomField(key="Water Area") - @Tag - private List waterArea; - @JsonProperty("type") - @CustomField(key="Type") + @CustomField(key="GRSF Type") @Group private Stock_Type type; @@ -125,7 +112,6 @@ public class StockRecord extends Common{ * @param exploitingFishery * @param managementEntity * @param assessmentMethods - * @param stateOfMarineResource * @param abundanceLevelStandard * @param abundanceLevel * @param fishingPressureStandard @@ -134,8 +120,6 @@ public class StockRecord extends Common{ * @param faoState * @param scientificAdvice * @param assessor - * @param stockUri - * @param waterArea * @param type */ public StockRecord( @@ -145,15 +129,13 @@ public class StockRecord extends Common{ List exploitingFishery, String managementEntity, List assessmentMethods, - String stateOfMarineResource, List> abundanceLevelStandard, List> abundanceLevel, List> fishingPressureStandard, List> fishingPressure, List> narrativeState, List> faoState, - List scientificAdvice, String assessor, String stockUri, - List waterArea, Stock_Type type) { + List scientificAdvice, String assessor, Stock_Type type) { super(); this.stockName = stockName; this.stockId = stockId; @@ -161,7 +143,6 @@ public class StockRecord extends Common{ this.exploitingFishery = exploitingFishery; this.managementEntity = managementEntity; this.assessmentMethods = assessmentMethods; - this.stateOfMarineResource = stateOfMarineResource; this.abundanceLevelStandard = abundanceLevelStandard; this.abundanceLevel = abundanceLevel; this.fishingPressureStandard = fishingPressureStandard; @@ -170,8 +151,6 @@ public class StockRecord extends Common{ this.faoState = faoState; this.scientificAdvice = scientificAdvice; this.assessor = assessor; - this.stockUri = stockUri; - this.waterArea = waterArea; this.type = type; } @@ -187,22 +166,6 @@ public class StockRecord extends Common{ this.type = type; } - public String getStockUri() { - return stockUri; - } - - public void setStockUri(String stockUri) { - this.stockUri = stockUri; - } - - public List getWaterArea() { - return waterArea; - } - - public void setWaterArea(List waterArea) { - this.waterArea = waterArea; - } - public String getStockName() { return stockName; } @@ -261,14 +224,6 @@ public class StockRecord extends Common{ this.managementEntity = managementEntity; } - public String getStateOfMarineResource() { - return stateOfMarineResource; - } - - public void setStateOfMarineResource(String stateOfMarineResource) { - this.stateOfMarineResource = stateOfMarineResource; - } - public List> getAbundanceLevelStandard() { return abundanceLevelStandard; } @@ -329,15 +284,13 @@ public class StockRecord extends Common{ + ", area=" + area + ", exploitingFishery=" + exploitingFishery + ", managementEntity=" + managementEntity + ", assessmentMethods=" + assessmentMethods - + ", stateOfMarineResource=" + stateOfMarineResource + ", abundanceLevelStandard=" + abundanceLevelStandard + ", abundanceLevel=" + abundanceLevel + ", fishingPressureStandard=" + fishingPressureStandard + ", fishingPressure=" + fishingPressure + ", narrativeState=" - + narrativeState + ", scientificAdvice=" + scientificAdvice - + ", assessor=" + assessor + ", stockUri=" + stockUri - + ", waterArea=" + waterArea + ", type=" + type + ", faoState=" - + faoState + "]"; + + narrativeState + ", faoState=" + faoState + + ", scientificAdvice=" + scientificAdvice + ", assessor=" + + assessor + ", type=" + type + "]"; } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/TimeSeriesBean.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/TimeSeriesBean.java index 3bd83a2..2fa6926 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/TimeSeriesBean.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/TimeSeriesBean.java @@ -27,7 +27,7 @@ public class TimeSeriesBean implements Comparable>{ @JsonProperty(SOURCE_FIELD) private String source; - + @JsonProperty(ASSESSMENT_FIELD) private String assessment; @@ -90,18 +90,6 @@ public class TimeSeriesBean implements Comparable>{ this.source = source; } - @Override - public String toString() { - - String firstPart = "" + year; - String secondPart = (value != null ? " - " + value : "") - + (unit != null ? " - " + unit : "") - + (source != null ? " (" + source + ")" : "") - + (assessment != null ? " (" + assessment + ")" : ""); - return firstPart + (secondPart != null && !secondPart.isEmpty() ? secondPart : ""); - - } - public String getAssessment() { return assessment; } @@ -114,9 +102,33 @@ public class TimeSeriesBean implements Comparable>{ return unit != null && !unit.getClass().equals(Void.class); } + public boolean isSourcePresent(){ + return source != null && !source.isEmpty(); + } + + public boolean isAssessmentPresent(){ + return assessment != null && !assessment.isEmpty(); + } + + public boolean isValuePresent(){ + return value != null && !value.getClass().equals(Void.class); + } + @Override public int compareTo(TimeSeriesBean o) { return (int) (this.year - o.year); // ascending.. low to highest } + @Override + public String toString() { + + String firstPart = "" + year; + String secondPart = (value != null ? " - " + value : "") + + (unit != null ? " - " + unit : "") + + (source != null ? " (" + source + ")" : "") + + (assessment != null ? " (Ref. year " + assessment + ")" : ""); + return firstPart + (secondPart != null && !secondPart.isEmpty() ? secondPart : ""); + + } + } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java index c3eb303..3e1abe2 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java @@ -46,13 +46,13 @@ public class CommonServiceUtils { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonServiceUtils.class); public static final String DEFAULT_LICENSE = "CC-BY-SA-4.0"; - public static final String GRSF_GROUP_NAME = "grsf-group"; + //public static final String GRSF_GROUP_NAME = "grsf-group"; private static final int TIME_SERIES_TAKE_LAST_VALUES = 5; private static final String REGEX_TAGS = "[^\\s\\w-_.]"; public static final String SYSTEM_TYPE = "system:type"; // item url property - public static final String ITEM_URL_FIELD = "UUID"; + public static final String ITEM_URL_FIELD = "GRSF Record URL"; /** * Retrieve the list of licenses for stocks and fisheries @@ -76,15 +76,11 @@ public class CommonServiceUtils { */ public static void validateAggregatedRecord(Common record) throws Exception { - List> databaseSources = record.getDatabaseSources(); List refersToList = record.getRefersTo(); String shortTitle = record.getShortTitle(); Boolean traceabilityFlag = record.isTraceabilityFlag(); Status status = record.getStatus(); - if(databaseSources == null || databaseSources.isEmpty()) - throw new Exception("database_sources cannot be null/empty"); - if(refersToList == null || refersToList.isEmpty()) throw new Exception("refers_to cannot be null/empty"); @@ -267,7 +263,7 @@ public class CommonServiceUtils { } // add the record among the source group (check for grsf-group) - groups.add(source.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : source.getOrigName().toLowerCase()); + //groups.add(source.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : source.getOrigName().toLowerCase()); } /** @@ -408,6 +404,7 @@ public class CommonServiceUtils { /** * Validate and check sources * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + * @param apiKey * @param context * @param contextServlet * @param sourceInPath @@ -420,8 +417,9 @@ public class CommonServiceUtils { * @param username * @throws Exception */ - public static void validateRecordAndMapFields(String context, ServletContext contextServlet, - Sources sourceInPath, Common record, Product_Type productType, Set tags, Map> customFields, Set groups, List resources, String username, String futureTitle) throws Exception { + public static void validateRecordAndMapFields(String apiKey, String context, ServletContext contextServlet, + Sources sourceInPath, Common record, Product_Type productType, Set tags, Map> customFields, + Set groups, List resources, String username, String futureTitle) throws Exception { // validate the record if it is a GRSF one and set the record type and in manage context // Status field is needed only in the Manage context for GRSF records @@ -452,10 +450,11 @@ public class CommonServiceUtils { if(sourceInPath.equals(Sources.GRSF)){ List refersTo = record.getRefersTo(); if(refersTo == null || refersTo.isEmpty()) - throw new Exception("refers_to is empty"); + throw new Exception("refers_to is empty for a GRSF record"); + + // we have the id within the catalog of this record. This means that we can retrieve the record and its system:type for (RefersToBean refersToBean : refersTo) { - resources.add(new ResourceBean(refersToBean.getUrl(), "Source of item " + futureTitle + " in the catalogue has id: " - + refersToBean.getId(), "Information of a source of the item " + futureTitle, null, username, null, null)); + resources.add(new ResourceBean(refersToBean.getUrl(), getSystemTypeValue(refersToBean.getId(), apiKey, context) , "The source record has id " + refersToBean.getId() + " in the catalog", null, username, null, null)); } } @@ -463,6 +462,24 @@ public class CommonServiceUtils { customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName())); } + /** + * Fetch the system:type property from a record + * @param itemIdOrName + * @param apiKey + * @return null on error + * @throws Exception + */ + public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{ + + DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context); + String systemTypeValue = catalog.getDataset(itemIdOrName, apiKey).getExtrasAsHashMap().get(SYSTEM_TYPE); + if(systemTypeValue == null || systemTypeValue.isEmpty()) + throw new Exception(SYSTEM_TYPE + " property not set in record " + itemIdOrName); + else + return systemTypeValue; + + } + /** * Actions to execute once the dataset has been updated or created. * @param responseBean diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java index 1c2305b..1edc8ca 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java @@ -131,7 +131,7 @@ public class GrsfPublisherFisheryService { List resources = record.getExtrasResources(); // validate end set sources - CommonServiceUtils.validateRecordAndMapFields(context, contextServlet, sourceInPath, record, Product_Type.FISHERY, tags, customFields, groups, resources, username, futureTitle); + CommonServiceUtils.validateRecordAndMapFields(apiKey, context, contextServlet, sourceInPath, record, Product_Type.FISHERY, tags, customFields, groups, resources, username, futureTitle); // check the license id String license = null; @@ -166,14 +166,14 @@ public class GrsfPublisherFisheryService { authorFullname, authorMail, record.getMaintainer() == null? authorFullname : record.getMaintainer(), - record.getMaintainerContact() == null? authorMail : record.getMaintainerContact(), - version, - HelperMethods.removeHTML(record.getDescription()), - license, - new ArrayList(tags), - customFields, - resources, - publicDataset); + record.getMaintainerContact() == null? authorMail : record.getMaintainerContact(), + version, + HelperMethods.removeHTML(record.getDescription()), + license, + new ArrayList(tags), + customFields, + resources, + publicDataset); // post actions if(id != null){ @@ -184,7 +184,7 @@ public class GrsfPublisherFisheryService { null, responseBean, catalogue, namespaces, groups, context, token, futureTitle, authorFullname, contextServlet, false); status = Status.CREATED; - + }else{ throw new Exception("There was an error during the product generation, sorry"); } @@ -220,9 +220,6 @@ public class GrsfPublisherFisheryService { try{ DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); - if(catalogue == null){ - throw new Exception("There was a problem while serving your request"); - } // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); @@ -238,10 +235,10 @@ public class GrsfPublisherFisheryService { } // check it is in the right source and it is a fishery - String grsfType = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); - String groupToCheck = sourceInPath.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : sourceInPath.getOrigName().toLowerCase(); + String grsfTypeValue = fisheryInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); + String systemTypeValue = fisheryInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); - if(catalogue.isDatasetInGroup(groupToCheck, recordToDelete.getId()) && Product_Type.FISHERY.getOrigName().equals(grsfType)){ + if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.FISHERY.getOrigName().equals(grsfTypeValue)){ logger.debug("Ok, this is a fishery of the right source, removing it"); boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), apiKey, true); @@ -302,7 +299,7 @@ public class GrsfPublisherFisheryService { for (String id : fullGroupListIds) { CkanDataset dataset = catalogue.getDataset(id, apiKey); if(dataset != null){ - String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); + String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); if(grsfType.equals(Product_Type.FISHERY.getOrigName())) datasetsIds.add(id); } @@ -392,7 +389,7 @@ public class GrsfPublisherFisheryService { if(catalogue == null){ throw new Exception("There was a problem while serving your request. No catalogue instance was found in this context!"); }else{ - + // get already published record and modify it String apiKey = catalogue.getApiKeyFromUsername(username); CkanDataset recordPublished = catalogue.getDataset(catalogId, apiKey); @@ -409,7 +406,7 @@ public class GrsfPublisherFisheryService { throw new Exception("Sorry but there was not possible to retrieve your fullname/email!"); } - + String organization = HelperMethods.retrieveOrgNameFromScope(context); //"grsf_admin"; // check he/she has admin role @@ -430,6 +427,7 @@ public class GrsfPublisherFisheryService { // validate end set sources CommonServiceUtils.validateRecordAndMapFields( + apiKey, context, contextServlet, sourceInPath, diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java index ca69e6a..e619fa7 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java @@ -134,6 +134,7 @@ public class GrsfPublisherStockService { // validate end set sources, tags, etc CommonServiceUtils.validateRecordAndMapFields( + apiKey, context, contextServlet, sourceInPath, @@ -248,11 +249,12 @@ public class GrsfPublisherStockService { throw new Exception("There was a problem while serving your request. This product was not found"); } - // check it is in the right source and it is a fishery - String grsfType = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); - String groupToCheck = sourceInPath.equals(Sources.GRSF) ? CommonServiceUtils.GRSF_GROUP_NAME : sourceInPath.getOrigName().toLowerCase(); + // check it is in the right source and it is a stock + String grsfTypeValue = stockInCkan.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); + String systemTypeValue = stockInCkan.getExtrasAsHashMap().get(CommonServiceUtils.SYSTEM_TYPE); + + if(systemTypeValue.equalsIgnoreCase(source) && Product_Type.STOCK.getOrigName().equals(grsfTypeValue)){ - if(catalogue.isDatasetInGroup(groupToCheck, recordToDelete.getId()) && Product_Type.STOCK.getOrigName().equals(grsfType)){ logger.debug("Ok, this is a stock of the right type, removing it"); boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), apiKey, true); if(deleted){ @@ -310,7 +312,7 @@ public class GrsfPublisherStockService { for (String id : fullGroupListIds) { CkanDataset dataset = catalogue.getDataset(id, catalogue.getApiKeyFromUsername(username)); if(dataset != null){ - String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_TYPE_KEY); + String grsfType = dataset.getExtrasAsHashMap().get(Common.GRSF_DOMAIN_KEY); if(grsfType.equals(Product_Type.STOCK.getOrigName())) datasetsIds.add(id); } @@ -433,6 +435,7 @@ public class GrsfPublisherStockService { // validate end set sources CommonServiceUtils.validateRecordAndMapFields( + apiKey, context, contextServlet, sourceInPath, diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java index c3b6e01..577dc0c 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CSVHelpers.java @@ -45,10 +45,21 @@ public class CSVHelpers { BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fo, "UTF-8")); // discover how the header will look like - boolean isUnitPresent = timeSeries.get(0).isUnitPresent(); - boolean isValuePresent = true; - boolean isSourcePresent = true; - boolean isAssessmentPresent = true; + boolean isUnitPresent = false; + boolean isValuePresent = false; + boolean isSourcePresent = false; + boolean isAssessmentPresent = false; + + for (TimeSeriesBean timeSeriesBean : timeSeries) { + if(timeSeriesBean.isSourcePresent()) + isSourcePresent = true; + if(timeSeriesBean.isAssessmentPresent()) + isAssessmentPresent = true; + if(timeSeriesBean.isValuePresent()) + isValuePresent = true; + if(timeSeriesBean.isUnitPresent()) + isUnitPresent = true; + } StringBuffer headerLine = new StringBuffer(); headerLine.append(TimeSeriesBean.YEAR_FIELD); @@ -122,5 +133,6 @@ public class CSVHelpers { return null; } } + } \ No newline at end of file diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java index 873efd2..da8d190 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java @@ -108,7 +108,7 @@ public abstract class HelperMethods { * @return * @throws Exception */ - public static DataCatalogue getDataCatalogueRunningInstance(String scope){ + public static DataCatalogue getDataCatalogueRunningInstance(String scope) throws Exception{ if(catalogueCache.get(scope) != null) return catalogueCache.get(scope); @@ -119,7 +119,7 @@ public abstract class HelperMethods { return instance; }catch(Exception e){ logger.error("Failed to instanciate data catalogue lib", e); - return null; + throw new Exception("Failed to retrieve catalogue information"); } } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index 2d81d6a..8cd0ad6 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -160,14 +160,17 @@ public class ManageTimeSeriesThread extends Thread{ String recordTypeFolderName = record.getGrsfType().toLowerCase(); String productName = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockName() : ((FisheryRecord)record).getFisheryName(); + String semanticIdentifier = record.getClass().equals(StockRecord.class) ? ((StockRecord)record).getStockId() : ((FisheryRecord)record).getFisheryId(); char firstLetter = uuidKB.charAt(0); // the whole path of the directory is going to be... - String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB) + PATH_SEPARATOR + CSVHelpers.CSV_EXTENSION.replace(".", ""); + String csvDirectoryForThisProduct = recordTypeFolderName + PATH_SEPARATOR + firstLetter + PATH_SEPARATOR + replaceIllegalChars(uuidKB, "_") + PATH_SEPARATOR + CSVHelpers.CSV_EXTENSION.replace(".", ""); logger.debug("The path under which the time series are going to be saved is " + csvDirectoryForThisProduct); WorkspaceFolder csvFolder = HelperMethods.createOrGetSubFoldersByPath(catalogueFolder, csvDirectoryForThisProduct); - if(csvFolder != null){ + if(csvFolder == null) + logger.error("Failed to create directory where csv files will be deployed in the workspace!!"); + else{ String apiKeyUser = catalogue.getApiKeyFromUsername(username); @@ -179,20 +182,17 @@ public class ManageTimeSeriesThread extends Thread{ try{ Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); if(f != null){ - List asList = (List)f; - if(!asList.isEmpty()){ CustomField customAnnotation = field.getAnnotation(CustomField.class); logger.debug("A time series has been just found (from field " + customAnnotation.key() + ")"); - String resourceToAttachOnCkanName = (replaceIllegalChars(productName) + "_" + customAnnotation.key()).replaceAll("\\s", "_").replaceAll("[_]+", "_") + CSVHelpers.CSV_EXTENSION; - String resourceToAttachOnCkanDescription = productName + " - " + customAnnotation.key() + " time series"; - - File csvFile = CSVHelpers.listToCSV(asList); + String resourceToAttachOnCkanName = customAnnotation.key(); + String resourceToAttachOnCkanDescription = productName + " - " + resourceToAttachOnCkanName + " time series"; CkanResourceBase ckanResource = null; ExternalFile createdFileOnWorkspace = null; + File csvFile = CSVHelpers.listToCSV(asList); if(csvFile != null){ for (int i = 0; i < CHANCES; i++) { @@ -205,7 +205,7 @@ public class ManageTimeSeriesThread extends Thread{ if(ckanResource != null){ if(createdFileOnWorkspace == null) - createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, uuidKB + "_" + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); + createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, replaceIllegalChars(semanticIdentifier, " ") + " " + customAnnotation.key() + CSVHelpers.CSV_EXTENSION, resourceToAttachOnCkanDescription, csvFile); if(createdFileOnWorkspace != null){ String publicUrlToSetOnCkan = createdFileOnWorkspace.getPublicLink(true); @@ -245,8 +245,8 @@ public class ManageTimeSeriesThread extends Thread{ * @param productName * @return */ - private static String replaceIllegalChars(String productName) { - return productName.replaceAll("[/\\[\\],|:*.+]", "_"); + private static String replaceIllegalChars(String value, String replacement) { + return value.replaceAll("[/\\[\\],|:*.+]", replacement); } /** diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index dd7ccc8..7ba4f53 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -412,7 +412,7 @@ public class JTests { } //@Test - public void testHierarchy(){ + public void testHierarchy() throws Exception{ String name = "low-abundance"; DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance("/gcube/devNext/NextNext"); List uniqueGroups = new ArrayList(); @@ -423,7 +423,7 @@ public class JTests { } //@Test - public void testAssociationThread() throws InterruptedException{ + public void testAssociationThread() throws Exception{ String name = "low-abundance"; DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance("/gcube/devNext/NextNext"); AssociationToGroupThread threadGroups = new AssociationToGroupThread(Arrays.asList(name), "another-test-test-please-ignore", "grsf", "costantino_perciante", catalogue, "apiKey");