From be06481c8fd744598f8e68ab3dc75a9bc5d5aedc Mon Sep 17 00:00:00 2001 From: Costantino Perciante Date: Sat, 3 Dec 2016 10:50:38 +0000 Subject: [PATCH] made some optimizations to speed up the publication. Some minor changes git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@135267 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../custom_annotations/TimeSeries.java | 5 +- .../grsf_publish_ws/json/input/Base.java | 8 +- .../grsf_publish_ws/json/input/Common.java | 42 ++- .../json/input/RefersToBean.java | 4 +- .../json/input/StockRecord.java | 1 - .../services/CommonServiceUtils.java | 317 ++++++++++++++++++ .../grsf_publish_ws/services/CommonUtils.java | 306 ----------------- .../services/GrsfPublisherFisheryService.java | 98 +++--- .../services/GrsfPublisherStockService.java | 90 +++-- .../utils/groups/Product_Type.java | 49 +++ .../utils/groups/Record_Type.java | 43 +-- .../grsf_publish_ws/utils/groups/Sources.java | 10 + .../grsf_publish_ws/utils/groups/Status.java | 2 +- .../utils/threads/ManageTimeSeriesThread.java | 10 +- .../grsf_publish_ws/JTests.java | 20 -- 15 files changed, 506 insertions(+), 499 deletions(-) create mode 100644 src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java delete mode 100644 src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonUtils.java create mode 100644 src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Product_Type.java diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/TimeSeries.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/TimeSeries.java index b9f2ade..96ddb79 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/TimeSeries.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/TimeSeries.java @@ -6,7 +6,10 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** - * TimeSeries annotation + * TimeSeries annotation. A list annotated with such field is transformed in a catalogue resources. + * Its elements should look like as (year, value1, value2 ...). + * The list is put in a csv file which is in turn uploaded on a shared folder, to replace the url + * that the catalogue creates for its inner resources. * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ @Retention(RetentionPolicy.RUNTIME) diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java index 0eafa06..159250f 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Base.java @@ -36,14 +36,14 @@ public class Base { // filled automatically by the service @JsonProperty("author") private String author; - - @JsonProperty("version") - private Long version; - + // filled automatically by the service @JsonProperty("author_contact") private String authorContact; + @JsonProperty("version") + private Long version; + @JsonProperty("maintainer") private String maintainer; diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java index c547499..0054dd8 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/Common.java @@ -8,6 +8,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CkanResource; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group; import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag; +import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Record_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status; @@ -21,7 +22,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties(value = {"product_type"}) public class Common extends Base{ - public static final String PRODUCT_TYPE_KEY = "Product type"; + public static final String PRODUCT_TYPE_KEY = "Product type"; // stock, fishery + public static final String RECORD_TYPE_KEY = "Record type"; // aggregated, original @JsonProperty("data_owner") @CustomField(key="Data owner") @@ -41,7 +43,6 @@ public class Common extends Base{ private List> sourceOfInformation; @JsonProperty("refers_to") - @CustomField(key="Refers to") // @NotNull(message="refers_to cannot be null") // @Size(min=1, message="refers_to cannot be empty") @Valid @@ -73,28 +74,23 @@ public class Common extends Base{ @CustomField(key=PRODUCT_TYPE_KEY) @Tag @Group - private String productType; + private String productType; + + // automatically compiled + @JsonProperty("record_type") + @CustomField(key=RECORD_TYPE_KEY) + private Record_Type recordType; public Common() { super(); } - /** - * @param dataOwner - * @param databaseSources - * @param sourceOfInformation - * @param refersTo - * @param shortTitle - * @param traceabilityFlag - * @param status - * @param reportingYear - * @param productType - */ + public Common(String dataOwner, List> databaseSources, List> sourceOfInformation, List refersTo, String shortTitle, Boolean traceabilityFlag, Status status, Long reportingYear, - String productType) { + String productType, Record_Type recordType) { super(); this.dataOwner = dataOwner; this.databaseSources = databaseSources; @@ -105,6 +101,19 @@ public class Common extends Base{ this.status = status; this.reportingYear = reportingYear; this.productType = productType; + this.recordType = recordType; + } + + public Record_Type getRecordType() { + return recordType; + } + + public void setRecordType(Record_Type recordType) { + this.recordType = recordType; + } + + public Boolean getTraceabilityFlag() { + return traceabilityFlag; } public List getRefersTo() { @@ -186,7 +195,8 @@ public class Common extends Base{ + sourceOfInformation + ", refersTo=" + refersTo + ", shortTitle=" + shortTitle + ", traceabilityFlag=" + traceabilityFlag + ", status=" + status + ", reportingYear=" - + reportingYear + ", productType=" + productType + "]"; + + reportingYear + ", productType=" + productType + + ", recordType=" + recordType + "]"; } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/RefersToBean.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/RefersToBean.java index a802864..fcb58e1 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/RefersToBean.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/RefersToBean.java @@ -2,14 +2,16 @@ package org.gcube.data_catalogue.grsf_publish_ws.json.input; import javax.validation.constraints.NotNull; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; /** * RefersToBean are used into aggregated records but not into the sources. - * To check if a product is a source or + * To check if a product is a source or an original record. * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ +@JsonIgnoreProperties(ignoreUnknown = true) public class RefersToBean { @JsonProperty("url") diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java index 122b62d..7ea0e21 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/StockRecord.java @@ -42,7 +42,6 @@ public class StockRecord extends Common{ @JsonProperty("assessment_distribution_area") @CustomField(key="Assessment distribution area") - // @Size(min=1, message="assessment distribution area cannot be empty") private List area; @JsonProperty("exploiting_fishery") diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java new file mode 100644 index 0000000..eacef95 --- /dev/null +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonServiceUtils.java @@ -0,0 +1,317 @@ +package org.gcube.data_catalogue.grsf_publish_ws.services; + +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CkanResource; +import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; +import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group; +import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.Base; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean; +import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; +import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; +import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status; +import org.gcube.datacatalogue.ckanutillibrary.models.ResourceBean; +import org.slf4j.LoggerFactory; + +/** + * Services common utils. + * @author Costantino Perciante at ISTI-CNR + */ +public class CommonServiceUtils { + + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonServiceUtils.class); + private static final int TIME_SERIES_TAKE_LAST_VALUES = 5; + private static final String REGEX_TAGS = "[^\\s\\w-_.]"; + + /** + * Retrieve the list of licenses for stocks and fisheries + * @return + */ + public static Map getLicenses(){ + Map licenses = null; + try{ + logger.info("Requested licenses..."); + licenses = HelperMethods.getLicenses(HelperMethods.getDataCatalogueRunningInstance(ScopeProvider.instance.get())); + }catch(Exception e){ + logger.error("Failed to retrieve the list of licenses"); + return null; + } + return licenses; + } + + /** + * Validate an aggregated GRSF record. TODO use @Valid tags + * @throws Exception + */ + public static void validateAggregatedRecord(Common record) throws Exception { + + List> databaseSources = record.getDatabaseSources(); + List refersToList = record.getRefersTo(); + String shortTitle = record.getShortTitle(); + Boolean traceabilityFlag = record.isTraceabilityFlag(); + Status status = record.getStatus(); + + if(databaseSources == null || databaseSources.isEmpty()) + throw new Exception("database_sources cannot be null/empty"); + + if(refersToList == null || refersToList.isEmpty()) + throw new Exception("refers_to cannot be null/empty"); + + if(traceabilityFlag == null) + throw new Exception("traceability_flag cannot be null"); + + if(shortTitle == null || shortTitle.isEmpty()) + throw new Exception("short_title cannot be null/empty"); + + if(status == null) + throw new Exception("status cannot be null/empty"); + + // check if it is a stock and perfomr related checls + if(record.getClass().equals(StockRecord.class)){ + + StockRecord stock = (StockRecord) record; + List species = stock.getSpecies(); + + if(species == null || species.isEmpty()) + throw new Exception("species cannot be null/empty in a GRSF record"); + } + + + } + + /** + * Parse the record to look up tags, groups and resources + * @param tags + * @param groups + * @param record + * @param username + * @param resources + */ + public static void getTagsGroupsResourcesByRecord( + Set tags, + boolean skipTags, + Set groups, + List resources, + Map> extras, + Base record, + String username, + String groupPrefix // it comes from the source type e.g., "grsf-", "ram-" .. + ){ + + Class current = record.getClass(); + do{ + Field[] fields = current.getDeclaredFields(); + for (Field field : fields) { + + if(!skipTags) + getTagsByField(field, current, record, tags); + getGroupsByField(field, current, record, groups, groupPrefix); + getExtrasByField(field, current, record, extras); + getResourcesByField(field, current, record, username, resources); + + } + } + while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class + + logger.info("Tags are " + tags); + logger.info("Groups is " + groups); + logger.info("Extras is " + extras); + logger.info("Resources " + resources); + } + + /** + * Retrieve the list of tags for this object + */ + private static void getTagsByField(Field field, Class current, Base record, Set tags){ + if(field.isAnnotationPresent(Tag.class)){ + try{ + Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); + if(f != null){ + if(f instanceof List){ + List asList = ((List) f); + if(!asList.isEmpty()){ + + logger.debug("The object annotated with @Tag is a list. Adding ... "); + + int elementsToConsider = asList.size(); + + // check if it is a time series, in this take the last X elements + if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ + elementsToConsider = Math.min(elementsToConsider, TIME_SERIES_TAKE_LAST_VALUES); + + for (int i = (asList.size() - elementsToConsider); i < asList.size(); i++) { + String finalTag = asList.get(i).toString().trim().replaceAll(REGEX_TAGS, ""); + + tags.add(finalTag); + } + }else{ + // else add all the available elements + for (int i = 0; i < elementsToConsider; i++) { + String finalTag = asList.get(i).toString().trim().replaceAll(REGEX_TAGS, ""); + tags.add(finalTag); + } + } + } + }else{ + logger.debug("The object annotated with @Tag is a simple one. Adding ... "); + String finalTag = f.toString().trim().replaceAll(REGEX_TAGS, ""); + logger.debug(finalTag); + tags.add(finalTag); + } + + } + }catch(Exception e){ + logger.error("Failed ot read value for field " + field.getName() + " skipping", e); + } + } + } + + /** + * Retrieve the list of groups' names for this object + */ + private static void getGroupsByField(Field field, Class current, Base record, Set groups, String groupPrefix){ + if(field.isAnnotationPresent(Group.class)){ + try{ + Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(field); + if(f != null){ + if(f instanceof List){ + List asList = ((List) f); + if(!asList.isEmpty()){ + + logger.debug("The object annotated with @Group is a list. Adding ... "); + + // else add all the available elements + for (int i = 0; i < asList.size(); i++) { + String groupName = HelperMethods.getGroupNameOnCkan(groupPrefix + "-" + asList.get(i).toString().trim()); + logger.debug(groupName); + groups.add(groupName); + } + + } + }else{ + + // also convert to the group name that should be on ckan + String groupName = HelperMethods.getGroupNameOnCkan(groupPrefix + "-" + f.toString().trim()); + groups.add(groupName); + } + } + + // check if the field is an enumerator, and the enum class is also annotated with @Group + if(field.getType().isEnum() && field.getType().isAnnotationPresent(Group.class)){ + + logger.info("Class " + field.getClass().getSimpleName() + " has annotation @Group"); + + // extract the name from the enum class and add it to the groups + // also convert to the group name that should be on ckan + String groupName = HelperMethods.getGroupNameOnCkan(groupPrefix + "-" + field.getType().getSimpleName()); + groups.add(groupName); + + } + + }catch(Exception e){ + logger.error("Failed ot read value for field " + field.getName() + " skipping", e); + } + } + } + + /** + * Retrieve the list of extras for this object + */ + private static void getExtrasByField(Field field, Class current, Base record, Map> extras){ + if(field.isAnnotationPresent(CustomField.class)){ + try{ + Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); + String keyField = field.getAnnotation(CustomField.class).key(); + if(f != null){ + List valuesForKey = new ArrayList(); + + // check if the map already contains this key + if(extras.containsKey(keyField)) + valuesForKey = extras.get(keyField); + + if(f instanceof List){ + logger.debug("The object " + field.getName() + " is a list and is annotated with @CustomField. Adding ..."); + List asList = (List)f; + if(!asList.isEmpty()){ + + int elementsToConsider = asList.size(); + + // check if it is a time series, in this case take the last X elements + if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ + elementsToConsider = Math.min(elementsToConsider, TIME_SERIES_TAKE_LAST_VALUES); + + for (int i = (asList.size() - elementsToConsider); i < asList.size(); i++) { + // trim and remove html + String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + valuesForKey.add(clean); + } + + }else{ + for (int i = 0; i < elementsToConsider; i++) { + String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + valuesForKey.add(clean); + } + } + } + }else{ + String clean = HelperMethods.removeHTML(f.toString().trim()); + valuesForKey.add(clean); + } + + // add to the map + extras.put(keyField, valuesForKey); + } + }catch(Exception e){ + logger.error("Failed ot read value for field " + field.getName() + " skipping", e); + } + } + } + + /** + * Retrieve the ResourceBean given the record (extract resources from Database Sources and Source of Information and others) + * @param record + * @param username + * @param tags + * @param groups + * @param resources + * @return + */ + private static void getResourcesByField(Field field, Class current, Base record, String username, List resources){ + if(field.isAnnotationPresent(CkanResource.class)){ + try{ + Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); + if(f != null){ + + if(f instanceof List){ + + List listOfResources = (List)f; + + for (Resource resource : listOfResources) { + resources.add(new ResourceBean(resource.getUrl(), resource.getName().toString(), resource.getDescription(), null, username, null, null)); + } + + }else{ + + Resource res = (Resource)f; + resources.add(new ResourceBean(res.getUrl(), res.getName().toString(), res.getDescription(), null, username, null, null)); + + } + } + }catch(Exception e){ + logger.error("Failed ot read value for field " + field.getName() + " skipping", e); + } + } + } + +} diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonUtils.java deleted file mode 100644 index b8a31ff..0000000 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/CommonUtils.java +++ /dev/null @@ -1,306 +0,0 @@ -package org.gcube.data_catalogue.grsf_publish_ws.services; - -import java.beans.PropertyDescriptor; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CkanResource; -import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField; -import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group; -import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Base; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource; -import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean; -import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; -import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; -import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status; -import org.gcube.datacatalogue.ckanutillibrary.models.ResourceBean; -import org.slf4j.LoggerFactory; - -/** - * Services common utils. - * @author Costantino Perciante at ISTI-CNR - */ -public class CommonUtils { - - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonUtils.class); - private static final int TIME_SERIES_TAKE_LAST_VALUES = 5; - private static final String REGEX_TAGS = "[^\\s\\w-_.]"; - - /** - * Retrieve the list of licenses for stocks and fisheries - * @return - */ - public static Map getLicenses(){ - Map licenses = null; - try{ - logger.info("Requested licenses..."); - licenses = HelperMethods.getLicenses(HelperMethods.getDataCatalogueRunningInstance(ScopeProvider.instance.get())); - }catch(Exception e){ - logger.error("Failed to retrieve the list of licenses"); - return null; - } - return licenses; - } - - /** - * Validate an aggregated GRSF record. TODO use @Valid tags - * @throws Exception - */ - public static void validateAggregatedRecord(Common record) throws Exception { - - List> databaseSources = record.getDatabaseSources(); - List refersToList = record.getRefersTo(); - String shortTitle = record.getShortTitle(); - Boolean traceabilityFlag = record.isTraceabilityFlag(); - Status status = record.getStatus(); - - if(databaseSources == null || databaseSources.isEmpty()) - throw new Exception("database_sources cannot be null/empty"); - - if(refersToList == null || refersToList.isEmpty()) - throw new Exception("refers_to cannot be null/empty"); - - if(traceabilityFlag == null) - throw new Exception("traceability_flag cannot be null"); - - if(shortTitle == null || shortTitle.isEmpty()) - throw new Exception("short_title cannot be null/empty"); - - if(status == null) - throw new Exception("status cannot be null/empty"); - - } - - /** - * Retrieve the list of tags for this object - */ - public static void getTags(Set tags, Base record){ - - Class current = record.getClass(); - do{ - Field[] fields = current.getDeclaredFields(); - for (Field field : fields) { - if(field.isAnnotationPresent(Tag.class)){ - try{ - Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - if(f instanceof List){ - List asList = ((List) f); - if(!asList.isEmpty()){ - - logger.debug("The object annotated with @Tag is a list. Adding ... "); - - int elementsToConsider = asList.size(); - - // check if it is a time series, in this take the last X elements - if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ - elementsToConsider = Math.min(elementsToConsider, TIME_SERIES_TAKE_LAST_VALUES); - - for (int i = (asList.size() - elementsToConsider); i < asList.size(); i++) { - String finalTag = asList.get(i).toString().trim().replaceAll(REGEX_TAGS, ""); - - tags.add(finalTag); - } - }else{ - // else add all the available elements - for (int i = 0; i < elementsToConsider; i++) { - String finalTag = asList.get(i).toString().trim().replaceAll(REGEX_TAGS, ""); - tags.add(finalTag); - } - } - } - }else{ - logger.debug("The object annotated with @Tag is a simple one. Adding ... "); - String finalTag = f.toString().trim().replaceAll(REGEX_TAGS, ""); - logger.debug(finalTag); - tags.add(finalTag); - } - - } - }catch(Exception e){ - logger.error("Failed ot read value for field " + field.getName() + " skipping", e); - } - } - } - } - while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class - - logger.info("Tags are " + tags); - } - - /** - * Retrieve the list of groups' names for this object - */ - public static void getGroups(Set groups, Base record){ - - Class current = record.getClass(); - do{ - Field[] fields = current.getDeclaredFields(); - for (Field field : fields) { - if(field.isAnnotationPresent(Group.class)){ - try{ - Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - - if(f instanceof List){ - List asList = ((List) f); - if(!asList.isEmpty()){ - - logger.debug("The object annotated with @Tag/@Group is a list. Adding ... "); - - // else add all the available elements - for (int i = 0; i < asList.size(); i++) { - String groupName = HelperMethods.getGroupNameOnCkan(asList.get(i).toString().trim()); - logger.debug(groupName); - groups.add(groupName); - } - - } - }else{ - - // also convert to the group name that should be on ckan - String groupName = HelperMethods.getGroupNameOnCkan(f.toString().trim()); - groups.add(groupName); - } - } - - // check if the field is an enumerator, and the enum class is also annotated with @Group - if(field.getType().isEnum() && field.getType().isAnnotationPresent(Group.class)){ - - logger.info("Class " + field.getClass().getSimpleName() + " has annotation @Group"); - - // extract the name from the enum class and add it to the groups - // also convert to the group name that should be on ckan - String groupName = HelperMethods.getGroupNameOnCkan(field.getType().getSimpleName()); - groups.add(groupName); - - } - - }catch(Exception e){ - logger.error("Failed ot read value for field " + field.getName() + " skipping", e); - } - } - } - } - while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class - - logger.info("Groups is " + groups); - } - - /** - * Retrieve the list of extras for this object - */ - public static void getExtras(Map> extras, Base record){ - - Class current = record.getClass(); - do{ - Field[] fields = current.getDeclaredFields(); - for (Field field : fields) { - if(field.isAnnotationPresent(CustomField.class)){ - try{ - Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - String keyField = field.getAnnotation(CustomField.class).key(); - if(f != null){ - List valuesForKey = new ArrayList(); - - // check if the map already contains this key - if(extras.containsKey(keyField)) - valuesForKey = extras.get(keyField); - - if(f instanceof List){ - logger.debug("The object " + field.getName() + " is a list and is annotated with @CustomField. Adding ..."); - List asList = (List)f; - if(!asList.isEmpty()){ - - int elementsToConsider = asList.size(); - - // check if it is a time series, in this case take the last X elements - if(asList.get(0).getClass().equals(TimeSeriesBean.class)){ - elementsToConsider = Math.min(elementsToConsider, TIME_SERIES_TAKE_LAST_VALUES); - - for (int i = (asList.size() - elementsToConsider); i < asList.size(); i++) { - // trim and remove html - String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); - valuesForKey.add(clean); - } - - }else{ - for (int i = 0; i < elementsToConsider; i++) { - String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); - valuesForKey.add(clean); - } - } - } - }else{ - String clean = HelperMethods.removeHTML(f.toString().trim()); - valuesForKey.add(clean); - } - - // add to the map - extras.put(keyField, valuesForKey); - } - }catch(Exception e){ - logger.error("Failed ot read value for field " + field.getName() + " skipping", e); - } - } - } - } - while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class - - logger.info("Extras is " + extras); - } - - /** - * Retrieve the ResourceBean given the record (extract resources from Database Sources and Source of Information and others) - * @param record - * @param username - * @param tags - * @param groups - * @param resources - * @return - */ - public static void getResourcesFromBean(Base record, String username, Set tags, Set groups, List resources){ - - Class current = record.getClass(); - do{ - Field[] fields = current.getDeclaredFields(); - for (Field field : fields) { - if(field.isAnnotationPresent(CkanResource.class)){ - try{ - Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record); - if(f != null){ - - if(f instanceof List){ - - List listOfResources = (List)f; - - for (Resource resource : listOfResources) { - resources.add(new ResourceBean(resource.getUrl(), resource.getName().toString(), resource.getDescription(), null, username, null, null)); - } - - }else{ - - Resource res = (Resource)f; - resources.add(new ResourceBean(res.getUrl(), res.getName().toString(), res.getDescription(), null, username, null, null)); - - } - } - }catch(Exception e){ - logger.error("Failed ot read value for field " + field.getName() + " skipping", e); - } - } - } - } - while((current = current.getSuperclass())!=null); // iterate from the inherited class up to the Object.class - - logger.info("Returning resources " + resources); - } - -} diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java index e5ba991..eb8897b 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java @@ -3,6 +3,7 @@ package org.gcube.data_catalogue.grsf_publish_ws.services; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -30,8 +31,10 @@ import org.gcube.common.scope.api.ScopeProvider; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.DeleteProductBean; import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean; import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; +import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Record_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread; @@ -71,7 +74,7 @@ public class GrsfPublisherFisheryService { public Response getLicenses(){ Status status = Status.OK; - Map licenses = CommonUtils.getLicenses(); + Map licenses = CommonServiceUtils.getLicenses(); if(licenses == null) status = Status.INTERNAL_SERVER_ERROR; return Response.status(status).entity(licenses).build(); @@ -107,18 +110,7 @@ public class GrsfPublisherFisheryService { Sources sourceInPath = Sources.onDeserialize(source); if(sourceInPath == null) - throw new Exception("The specified source in the path is unknown"); - - // // determine the organization in which this product should be put - // String contextInWhichPublish = HelperMethods.getContextFromStatus(record.getStatus(), contextServlet); - // - // if(contextInWhichPublish == null || !contextInWhichPublish.equals(context)){ - // - // // stop, this value must be defined - // status = Status.BAD_REQUEST; - // throw new IllegalArgumentException("Status attribute is not defined or the Token you are using is not correct to perform such request!"); - // - // }else{ + throw new Exception("The specified source in the path is unrecognized. Values accepted are " + Sources.getAsList()); DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ @@ -128,23 +120,13 @@ public class GrsfPublisherFisheryService { }else{ - // // in this case, the status is mandatory - // if(record.getStatus() == null && contextInWhichPublish.equals(HelperMethods.PENDING_CONTEX_KEY)){ - // - // status = Status.BAD_REQUEST; - // throw new Exception("Status information is missing"); - // - // } - // check the user has editor/admin role into the org String organization = HelperMethods.retrieveOrgNameFromScope(context); String role = catalogue.getRoleOfUserInOrganization(username, organization, catalogue.getApiKeyFromUsername(username)); - logger.info("***************************Role of the user " + username + " is " + role); + logger.info("Role of the user " + username + " is " + role); if(!role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString())){ - status = Status.FORBIDDEN; throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-admin role!"); - } // The name of the product will be the uuid of the kb. The title will be the fishery's fishery_name. Fishery has also the constraint that @@ -159,50 +141,47 @@ public class GrsfPublisherFisheryService { throw new Exception("The name requested for the product is not correct! It should contain only alphanumeric characters, and symbols like '.' or '_', '-'"); }else if((fishingArea == null || fishingArea.isEmpty()) && (jurisdictionArea == null || jurisdictionArea.isEmpty()) && sourceInPath.equals(Sources.GRSF)){ - status = Status.BAD_REQUEST; throw new Exception("fishing_area and jurisdiction_area cannot be null/empty at the same time!"); - }else{ logger.debug("Checking if such name [" + futureName + "] doesn't exist yet..."); - boolean alreadyExist = catalogue.existProductWithNameOrId(futureName); + boolean alreadyExists = catalogue.existProductWithNameOrId(futureName); - if(alreadyExist){ + if(alreadyExists){ logger.debug("A product with name " + futureName + " already exists"); status = Status.CONFLICT; throw new Exception("A product with name " + futureName + " already exists"); }else{ - - // validate the record if it is a GRSF one + + // validate the record if it is a GRSF one and set the record type if(sourceInPath.equals(Sources.GRSF)){ - CommonUtils.validateAggregatedRecord(record); - } + record.setRecordType(Record_Type.AGGREGATED); + CommonServiceUtils.validateAggregatedRecord(record); + }else + record.setRecordType(Record_Type.ORIGINAL); // set the type - record.setProductType(Record_Type.FISHERY.getOrigName()); + record.setProductType(Product_Type.FISHERY.getOrigName()); - // evaluate the tags of the product - Set tags = new HashSet(); - CommonUtils.getTags(tags, record); - - // evaluate the groups - Set groups = new HashSet(); - CommonUtils.getGroups(groups, record); - - // add the tag and group for the other sources - if(!sourceInPath.equals(Sources.GRSF)){ - tags.add(sourceInPath.getOrigName()); - groups.add(sourceInPath.getOrigName()); - } - - // evaluate the custom fields + // evaluate the custom fields/tags, resources and groups Map> customFields = record.getExtrasFields(); + Set tags = new HashSet(); + Set groups = new HashSet(); + List resources = record.getExtrasResources(); + boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records + CommonServiceUtils.getTagsGroupsResourcesByRecord(tags, skipTags, groups, resources, customFields, record, username, sourceInPath.getOrigName().toLowerCase()); - // automatically retrieve the other ones - CommonUtils.getExtras(customFields, record); + // manage the refers to + if(sourceInPath.equals(Sources.GRSF)){ + List refersTo = record.getRefersTo(); + for (RefersToBean refersToBean : refersTo) { + resources.add(new ResourceBean(refersToBean.getUrl(), "Source of product " + futureTitle + " in the catalogue has id: " + + refersToBean.getId(), "Information of a source of the product " + futureTitle, null, username, null, null)); + } + } // retrieve the user's email and fullname String authorMail = HelperMethods.getUserEmail(context, token); @@ -216,10 +195,6 @@ public class GrsfPublisherFisheryService { }else{ - // evaluate the resources - List resources = record.getExtrasResources(); - CommonUtils.getResourcesFromBean(record, username, tags, groups, resources); - // check the license id String license = null; if(record.getLicense() == null || record.getLicense().isEmpty()) @@ -232,7 +207,7 @@ public class GrsfPublisherFisheryService { long version = record.getVersion() == null ? 1 : record.getVersion(); logger.info("Invoking creation method.."); - + // create the product id = catalogue.createCKanDatasetMultipleCustomFields( catalogue.getApiKeyFromUsername(username), @@ -295,7 +270,8 @@ public class GrsfPublisherFisheryService { @Produces(MediaType.APPLICATION_JSON) public Response deleteFishery( @NotNull(message="input value is missing") - @Valid DeleteProductBean recordToDelete) throws ValidationException{ + @Valid DeleteProductBean recordToDelete, + @PathParam("source") String source) throws ValidationException{ // retrieve context and username Caller caller = AuthorizationProvider.instance.get(); @@ -317,6 +293,14 @@ public class GrsfPublisherFisheryService { } + // Cast the source to the accepted ones + Sources sourceInPath = Sources.onDeserialize(source); + + if(sourceInPath == null) + throw new Exception("The specified source in the path is unrecogized. Values accepted are [rams, firms, fishsource, grsf]"); + + logger.info("The request is to create a stock object of source " + sourceInPath); + // retrieve the catalogue instance CkanDataset fisheryInCkan = catalogue.getDataset(recordToDelete.getId(), catalogue.getApiKeyFromUsername(username)); @@ -328,7 +312,7 @@ public class GrsfPublisherFisheryService { } // get extras and check there is the product type - if(fisheryInCkan.getExtrasAsHashMap().get(Common.PRODUCT_TYPE_KEY).equals(Record_Type.FISHERY.getOrigName())){ + if(fisheryInCkan.getExtrasAsHashMap().get(Common.PRODUCT_TYPE_KEY).equals(Product_Type.FISHERY.getOrigName())){ logger.warn("Ok, this is a fishery, removing it"); boolean deleted = catalogue.deleteProduct(fisheryInCkan.getId(), catalogue.getApiKeyFromUsername(username), true); if(deleted){ diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java index cfd236b..4dbb010 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java @@ -29,9 +29,11 @@ import org.gcube.common.authorization.library.utils.Caller; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Common; import org.gcube.data_catalogue.grsf_publish_ws.json.input.DeleteProductBean; +import org.gcube.data_catalogue.grsf_publish_ws.json.input.RefersToBean; import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.output.ResponseCreationBean; import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; +import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Product_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Record_Type; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources; import org.gcube.data_catalogue.grsf_publish_ws.utils.threads.AssociationToGroupThread; @@ -71,7 +73,7 @@ public class GrsfPublisherStockService { public Response getLicenses(){ Status status = Status.OK; - Map licenses = CommonUtils.getLicenses(); + Map licenses = CommonServiceUtils.getLicenses(); if(licenses == null) status = Status.INTERNAL_SERVER_ERROR; return Response.status(status).entity(licenses).build(); @@ -83,7 +85,8 @@ public class GrsfPublisherStockService { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public Response publishStock( - @NotNull(message="record cannot be null") @Valid StockRecord record, @PathParam("source") String source) throws ValidationException{ + @NotNull(message="record cannot be null") @Valid StockRecord record, + @PathParam("source") String source) throws ValidationException{ // retrieve context and username Caller caller = AuthorizationProvider.instance.get(); @@ -103,21 +106,10 @@ public class GrsfPublisherStockService { Sources sourceInPath = Sources.onDeserialize(source); if(sourceInPath == null) - throw new Exception("The specified source in the path is unknown"); + throw new Exception("The specified source in the path is unrecogized. Values accepted are [rams, firms, fishsource, grsf]"); logger.info("The request is to create a stock object of source " + sourceInPath); - // // determine the organization in which this product should be put - // String contextInWhichPublish = HelperMethods.getContextFromStatus(record.getStatus(), contextServlet); - // - // if(contextInWhichPublish == null || !contextInWhichPublish.equals(context)){ - // - // // stop, this value must be defined - // status = Status.BAD_REQUEST; - // throw new IllegalArgumentException("Status attribute is not defined or the Token you are using is not correct to perform such request!"); - // - // }else{ - DataCatalogue catalogue = HelperMethods.getDataCatalogueRunningInstance(context); if(catalogue == null){ @@ -126,14 +118,6 @@ public class GrsfPublisherStockService { }else{ - // // in this case, the status is mandatory - // if(record.getStatus() == null && contextInWhichPublish.equals(HelperMethods.PENDING_CONTEX_KEY)){ - // - // status = Status.BAD_REQUEST; - // throw new Exception("Status information is missing"); - // - // } - // check the user has editor/admin role into the org String organization = HelperMethods.retrieveOrgNameFromScope(context); String role = catalogue.getRoleOfUserInOrganization(username, organization, catalogue.getApiKeyFromUsername(username)); @@ -165,35 +149,34 @@ public class GrsfPublisherStockService { throw new Exception("Sorry but a product with such name already exists!"); }else{ - - // validate the record if it is a GRSF one + + // validate the record if it is a GRSF one and set the record type if(sourceInPath.equals(Sources.GRSF)){ - CommonUtils.validateAggregatedRecord(record); - } + record.setRecordType(Record_Type.AGGREGATED); + CommonServiceUtils.validateAggregatedRecord(record); + }else + record.setRecordType(Record_Type.ORIGINAL); // set the type - record.setProductType(Record_Type.STOCK.getOrigName()); + record.setProductType(Product_Type.STOCK.getOrigName()); - // evaluate the tags of the product - Set tags = new HashSet(); - CommonUtils.getTags(tags, record); - - // evaluate the groups - Set groups = new HashSet(); - CommonUtils.getGroups(groups, record); - - // add the tag and group for the other sources - if(!sourceInPath.equals(Sources.GRSF)){ - tags.add(sourceInPath.getOrigName()); - groups.add(sourceInPath.getOrigName()); - } - - // evaluate the custom fields + // evaluate the custom fields/tags, resources and groups Map> customFields = record.getExtrasFields(); + Set tags = new HashSet(); + Set groups = new HashSet(); + List resources = record.getExtrasResources(); + boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records + CommonServiceUtils.getTagsGroupsResourcesByRecord(tags, skipTags, groups, resources, customFields, record, username, sourceInPath.getOrigName().toLowerCase()); - // automatically retrieve the other ones - CommonUtils.getExtras(customFields, record); - + // manage the refers to + if(sourceInPath.equals(Sources.GRSF)){ + List refersTo = record.getRefersTo(); + for (RefersToBean refersToBean : refersTo) { + resources.add(new ResourceBean(refersToBean.getUrl(), "Source of product " + futureTitle + " in the catalogue has id: " + + refersToBean.getId(), "Information of a source of the product " + futureTitle, null, username, null, null)); + } + } + // retrieve the user's email and fullname String authorMail = HelperMethods.getUserEmail(context, token); String authorFullname = HelperMethods.getUserFullname(context, token); @@ -207,10 +190,6 @@ public class GrsfPublisherStockService { }else{ - // evaluate the resources - List resources = record.getExtrasResources(); - CommonUtils.getResourcesFromBean(record, username, tags, groups, resources); - // check the license id String license = null; if(record.getLicense() == null || record.getLicense().isEmpty()) @@ -283,7 +262,8 @@ public class GrsfPublisherStockService { @Produces(MediaType.APPLICATION_JSON) public Response deleteStock( @NotNull(message="missing input value") - @Valid DeleteProductBean recordToDelete) throws ValidationException{ + @Valid DeleteProductBean recordToDelete, + @PathParam("source") String source) throws ValidationException{ // retrieve context and username Caller caller = AuthorizationProvider.instance.get(); @@ -304,6 +284,14 @@ public class GrsfPublisherStockService { throw new Exception("There was a problem while serving your request"); } + + // Cast the source to the accepted ones + Sources sourceInPath = Sources.onDeserialize(source); + + if(sourceInPath == null) + throw new Exception("The specified source in the path is unrecogized. Values accepted are [rams, firms, fishsource, grsf]"); + + logger.info("The request is to create a stock object of source " + sourceInPath); // retrieve the catalogue instance CkanDataset stockInCkan = catalogue.getDataset(recordToDelete.getId(), catalogue.getApiKeyFromUsername(username)); @@ -316,7 +304,7 @@ public class GrsfPublisherStockService { } // get extras and check there is the product type - if(stockInCkan.getExtrasAsHashMap().get(Common.PRODUCT_TYPE_KEY).equals(Record_Type.STOCK.getOrigName())){ + if(stockInCkan.getExtrasAsHashMap().get(Common.PRODUCT_TYPE_KEY).equals(Product_Type.STOCK.getOrigName())){ logger.warn("Ok, this is a stock, removing it"); boolean deleted = catalogue.deleteProduct(stockInCkan.getId(), catalogue.getApiKeyFromUsername(username), true); diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Product_Type.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Product_Type.java new file mode 100644 index 0000000..4851d90 --- /dev/null +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Product_Type.java @@ -0,0 +1,49 @@ +package org.gcube.data_catalogue.grsf_publish_ws.utils.groups; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * The type of product, i.e. Stock or Fishery + * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + */ +public enum Product_Type { + + FISHERY("Fishery"), + STOCK("Stock"); + + private String subGroupNameOrig; + + private Product_Type(String origName) { + this.subGroupNameOrig = origName; + } + + /** + * Return the original name + * @return + */ + public String getOrigName(){ + return subGroupNameOrig; + } + + @JsonValue + public String onSerialize(){ + return subGroupNameOrig.toLowerCase(); + } + + @JsonCreator + public static Product_Type onDeserialize(String recordTypeString) { + if(recordTypeString != null) { + for(Product_Type source : Product_Type.values()) { + if (source.toString().equalsIgnoreCase(recordTypeString.trim())) + return source; + } + } + return null; + } + + @Override + public String toString() { + return getOrigName(); + } +} diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Record_Type.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Record_Type.java index 3083cd4..75312e7 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Record_Type.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Record_Type.java @@ -1,49 +1,22 @@ package org.gcube.data_catalogue.grsf_publish_ws.utils.groups; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; /** - * The type of record, i.e. Stock or Fishery + * The type of record, i.e. Aggregated or Original * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ public enum Record_Type { - - FISHERY("Fishery"), - STOCK("Stock"); - - private String subGroupNameOrig; - private Record_Type(String origName) { - this.subGroupNameOrig = origName; - } + AGGREGATED("Aggregated"), + ORIGINAL("Original"); - /** - * Return the original name - * @return - */ - public String getOrigName(){ - return subGroupNameOrig; - } + private String name; - @JsonValue - public String onSerialize(){ - return subGroupNameOrig.toLowerCase(); - } - - @JsonCreator - public static Record_Type onDeserialize(String recordTypeString) { - if(recordTypeString != null) { - for(Record_Type source : Record_Type.values()) { - if (source.toString().equalsIgnoreCase(recordTypeString.trim())) - return source; - } - } - return null; + private Record_Type(String name) { + this.name = name; } - @Override - public String toString() { - return getOrigName(); + public String getRecordType(){ + return name; } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Sources.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Sources.java index 3de1f9f..9755bbc 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Sources.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Sources.java @@ -1,5 +1,7 @@ package org.gcube.data_catalogue.grsf_publish_ws.utils.groups; +import java.util.Arrays; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; @@ -48,4 +50,12 @@ public enum Sources { public String toString() { return getOrigName(); } + + public static String getAsList(){ + return "[" + Arrays.asList( + FIRMS.toString().toLowerCase(), + RAM.toString().toLowerCase(), + FISHSOURCE.toString().toLowerCase(), + GRSF.toString().toLowerCase()) + "]"; + } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Status.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Status.java index 38a291c..8632fe7 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Status.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/groups/Status.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; /** - * Status Group and sub groups (for both Stock and Fishery) + * Status Group and sub groups (for both Stock and Fishery, only Aggregated records) * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ public enum Status { diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java index f56913f..5b5800a 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/threads/ManageTimeSeriesThread.java @@ -44,6 +44,8 @@ public class ManageTimeSeriesThread extends Thread{ // Logger private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ManageTimeSeriesThread.class); + + // try to attach the source at most CANCHES times .. private static final int CANCHES = 3; private Common record; @@ -128,11 +130,7 @@ public class ManageTimeSeriesThread extends Thread{ throw new IllegalArgumentException("The given record is null!!"); Workspace ws = HomeLibrary.getHomeManagerFactory().getHomeManager().getHome().getWorkspace(); - - // Get a VRE folder by scope WorkspaceSharedFolder vreFolder = ws.getVREFolderByScope(ScopeProvider.instance.get()); - - //Get the VRE Folder catalogue WorkspaceCatalogue catalogueFolder = vreFolder.getVRECatalogue(); logger.debug("Catalogue folder in vre has path " + catalogueFolder.getPath()); @@ -178,7 +176,7 @@ public class ManageTimeSeriesThread extends Thread{ CustomField customAnnotation = field.getAnnotation(CustomField.class); String resourceToAttachName = (productName + "_" + customAnnotation.key()).replaceAll("\\s", "_") + CSV_FILE_FORMAT; - String resourceToAttachDescription = productName + ":" + customAnnotation.key() + " time series"; + String resourceToAttachDescription = productName + " : " + customAnnotation.key() + " time series"; logger.debug("A time series has been just found (from field " + customAnnotation.key() + ")"); File csvFile = CSVHelpers.listToCSV(asList); @@ -194,7 +192,7 @@ public class ManageTimeSeriesThread extends Thread{ ckanResource = uploadFileOnCatalogue(csvFile, packageName, catalogue, username, resourceToAttachName, resourceToAttachDescription, apiKeyUser); //upload this file on the folder of the vre (under .catalogue) and change the url of the resource - if(ckanResource != null){ + else{ ExternalFile createdFileOnWorkspace = HelperMethods.uploadExternalFile(csvFolder, resourceToAttachName, resourceToAttachDescription, csvFile); if(createdFileOnWorkspace != null){ diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index 858f394..332fac3 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -33,7 +33,6 @@ import org.gcube.data_catalogue.grsf_publish_ws.json.input.FisheryRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.Resource; import org.gcube.data_catalogue.grsf_publish_ws.json.input.StockRecord; import org.gcube.data_catalogue.grsf_publish_ws.json.input.TimeSeriesBean; -import org.gcube.data_catalogue.grsf_publish_ws.services.CommonUtils; import org.gcube.data_catalogue.grsf_publish_ws.utils.CSVHelpers; import org.gcube.data_catalogue.grsf_publish_ws.utils.HelperMethods; import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Abundance_Level; @@ -45,7 +44,6 @@ import org.gcube.datacatalogue.ckanutillibrary.DataCatalogue; import org.gcube.datacatalogue.ckanutillibrary.DataCatalogueFactory; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; public class JTests { @@ -220,24 +218,6 @@ public class JTests { } - //@Test - public void testCustomFieldAnnotation() throws JsonProcessingException{ - - StockRecord record = new StockRecord(); - record.setWaterArea(Arrays.asList("aa", "bb", "cc", "dd")); - Map> extras = new HashMap>(); - CommonUtils.getExtras(extras , record); - logger.debug("Extras = " + extras); - - ObjectMapper mapper = new ObjectMapper(); - - //Object to JSON in String - String jsonInString = mapper.writeValueAsString(record); - logger.debug(jsonInString); - - - } - //@Test public void testJSONResource() throws Exception{ DataCatalogueFactory factory = DataCatalogueFactory.getFactory();