From 57c3d65283cc749b481ca22594d163f80b789544 Mon Sep 17 00:00:00 2001 From: Francesco Mangiacrapa Date: Fri, 14 Jun 2019 15:07:35 +0000 Subject: [PATCH] merged with branch version 1.9 at 179217 RN git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@179847 82a268e6-3cf1-43bd-a215-b396298e98cf --- .settings/org.eclipse.wst.common.component | 2 +- distro/changelog.xml | 5 ++- .../json/input/record/Base.java | 2 +- .../json/input/record/FisheryRecord.java | 2 +- .../json/input/record/StockRecord.java | 2 +- .../services/GrsfPublisherFisheryService.java | 9 +++-- .../services/GrsfPublisherStockService.java | 12 ++++--- .../utils/CommonServiceUtils.java | 34 +++++++++++-------- .../grsf_publish_ws/utils/HelperMethods.java | 32 ++++++++++------- .../grsf_publish_ws/JTests.java | 2 +- 10 files changed, 61 insertions(+), 41 deletions(-) diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index d6f218f..7ca4ce4 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -3,7 +3,7 @@ - + uses diff --git a/distro/changelog.xml b/distro/changelog.xml index f459b9f..31829bb 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -2,7 +2,10 @@ - + refers_to can be null while publishing legacy records refs #13347 + Removed the non-ascii clean from extra fields refs #12421 + Properly supporting UTF-8 characters refs #12421 + Title is updated according to Stock/Fishery Name refs #16395 The sources in GRSF VRE are calculated using 'database_sources' field #12861 diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Base.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Base.java index f9ae3a5..6fad4d8 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Base.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Base.java @@ -196,7 +196,7 @@ public abstract class Base { else values = new ArrayList(); - values.add(HelperMethods.removeHTML(value.toString())); + values.add(HelperMethods.removeHTML(value.toString(), false)); extrasFields.put(key, values); } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java index e8de0e0..17369a2 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java @@ -173,7 +173,7 @@ public class FisheryRecord extends Common { @Override public String toString() { - return "FisheryRecord [fisheryName=" + fisheryName + ", fisheryId=" + return super.toString() + " - FisheryRecord [fisheryName=" + fisheryName + ", fisheryId=" + fisheryId + ", fishingArea=" + fishingArea + ", resourcesExploited=" + resourcesExploited + ", jurisdictionArea=" + jurisdictionArea diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java index 16ac9d3..3257939 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java @@ -292,7 +292,7 @@ public class StockRecord extends Common{ @Override public String toString() { - return "StockRecord [stockName=" + stockName + ", stockId=" + stockId + return super.toString() + "- StockRecord [stockName=" + stockName + ", stockId=" + stockId + ", area=" + area + ", exploitingFishery=" + exploitingFishery + ", assessmentMethods=" + assessmentMethods + ", abundanceLevelStandard=" + abundanceLevelStandard diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java index fd80d2d..a46b977 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherFisheryService.java @@ -423,9 +423,12 @@ public class GrsfPublisherFisheryService { // check he/she has admin role CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); - // name, title, product url and are going to remain unchanged (so we keep them from the already published record); + // name, product url and are going to remain unchanged (so we keep them from the already published record); String name = recordPublished.getName(); - String title = recordPublished.getTitle(); + + // The title must change if Fishery Name change + //String title = recordPublished.getTitle(); + String title = record.getFisheryName(); // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); @@ -480,7 +483,7 @@ public class GrsfPublisherFisheryService { // update the product String id = catalogue.updateCKanDataset(apiKey, catalogId, title, name, publishInOrganization, authorFullname, authorMail, record.getMaintainer(), record.getMaintainerContact(), version, - HelperMethods.removeHTML(record.getDescription()), license, new ArrayList(tags), null, // remove any previous group + HelperMethods.removeHTML(record.getDescription(), false), license, new ArrayList(tags), null, // remove any previous group customFields, resources, publicDataset); if(id != null) { diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java index 1825084..897a528 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/services/GrsfPublisherStockService.java @@ -188,7 +188,7 @@ public class GrsfPublisherStockService { publishInOrganization, authorFullname, authorMail, record.getMaintainer() == null ? authorFullname : record.getMaintainer(), record.getMaintainerContact() == null ? authorMail : record.getMaintainerContact(), version, - HelperMethods.removeHTML(record.getDescription()), license, new ArrayList(tags), + HelperMethods.removeHTML(record.getDescription(), false), license, new ArrayList(tags), customFields, resources, publicDataset); if(id != null) { @@ -425,9 +425,13 @@ public class GrsfPublisherStockService { // check he/she has admin role CommonServiceUtils.hasAdminRole(username, catalogue, apiKey, organization); - // name, title, product url and are going to remain unchanged (so we keep them from the publisher record); + // name, product url and are going to remain unchanged (so we keep them from the publisher record); String name = recordPublished.getName(); - String title = recordPublished.getTitle(); + + // The title must change if Stock Name change + //String title = recordPublished.getTitle(); + String title = record.getStockName(); + // Cast the source to the accepted ones Sources sourceInPath = Sources.onDeserialize(source); @@ -482,7 +486,7 @@ public class GrsfPublisherStockService { // update the product String id = catalogue.updateCKanDataset(apiKey, catalogId, title, name, publishInOrganization, authorFullname, authorMail, record.getMaintainer(), record.getMaintainerContact(), version, - HelperMethods.removeHTML(record.getDescription()), license, new ArrayList(tags), null, // remove any previous group + HelperMethods.removeHTML(record.getDescription(), false), license, new ArrayList(tags), null, // remove any previous group customFields, resources, publicDataset); if(id != null) { diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java index 998c413..26e366e 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java @@ -80,21 +80,20 @@ public class CommonServiceUtils { List refersToList = record.getRefersTo(); if(refersToList == null || refersToList.isEmpty()) throw new Exception("refers_to cannot be null/empty"); + + Boolean traceabilityFlag = record.isTraceabilityFlag(); + if(traceabilityFlag == null) + throw new Exception("traceability_flag cannot be null"); + + Status status = record.getStatus(); + if(status == null) + throw new Exception("status cannot be null/empty"); } String shortTitle = record.getShortName(); - Boolean traceabilityFlag = record.isTraceabilityFlag(); - Status status = record.getStatus(); - - if(traceabilityFlag == null) - throw new Exception("traceability_flag cannot be null"); - if(shortTitle == null || shortTitle.isEmpty()) throw new Exception("short_title cannot be null/empty"); - if(status == null) - throw new Exception("status cannot be null/empty"); - // check if it is a stock and perform related checks if(record.getClass().equals(StockRecord.class)) { @@ -349,18 +348,21 @@ public class CommonServiceUtils { Constants.TIME_SERIES_TAKE_LAST_VALUES); for(int i = 0; i < elementsToConsider; i++) { // trim and remove html - String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + // String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + String clean = HelperMethods.removeHTML(asList.get(i).toString().trim(), false); valuesForKey.add(clean); } } else for(int i = 0; i < elementsToConsider; i++) { - String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + // String clean = HelperMethods.removeHTML(asList.get(i).toString().trim()); + String clean = HelperMethods.removeHTML(asList.get(i).toString().trim(), false); valuesForKey.add(clean); } } } else { - String clean = HelperMethods.removeHTML(f.toString().trim()); + // String clean = HelperMethods.removeHTML(f.toString().trim()); + String clean = HelperMethods.removeHTML(f.toString().trim(), false); valuesForKey.add(clean); } @@ -500,9 +502,11 @@ public class CommonServiceUtils { // I'm not in GRSF Admin so the groups must be added using databaseSources List> databaseSources = record.getDatabaseSources(); - for(Resource source : databaseSources) { - Sources sourceName = source.getName(); - sourcesList.add(sourceName.getOrigName().toLowerCase()); + if(databaseSources!=null) { + for(Resource source : databaseSources) { + Sources sourceName = source.getName(); + sourcesList.add(sourceName.getOrigName().toLowerCase()); + } } } diff --git a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java index d15c32d..7c3d9b3 100644 --- a/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java +++ b/src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/HelperMethods.java @@ -9,7 +9,6 @@ import java.io.FileNotFoundException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; -import java.io.UnsupportedEncodingException; import java.sql.SQLException; import java.util.HashMap; import java.util.Iterator; @@ -66,7 +65,7 @@ public abstract class HelperMethods { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HelperMethods.class); private static final String APPLICATION_ID_CATALOGUE_MANAGER = "org.gcube.datacatalogue.ProductCatalogue"; - private static final String NOTIFICATION_MESSAGE = "Dear members,
The record 'PRODUCT_TITLE' has been just published by USER_FULLNAME.
You can find it here: PRODUCT_URL
"; + private static final String NOTIFICATION_MESSAGE = "Dear members,\nThe record 'PRODUCT_TITLE' has been just published by USER_FULLNAME.\nYou can find it here: PRODUCT_URL\n"; private static final String SOCIAL_SERVICE_APPLICATION_TOKEN = "/2/tokens/generate-application-token/"; private static final String SOCIAL_SERVICE_WRITE_APPLICATION_POST = "/2/posts/write-post-app/"; private static final String MEDIATYPE_JSON = "application/json"; @@ -350,26 +349,33 @@ public abstract class HelperMethods { } return result; } - - /** - * Strip out HTML code - * @param html - * @return - * @throws UnsupportedEncodingException - */ - public static String removeHTML(String html) { + + public static String removeHTML(String html, boolean removeAscii) { if(html == null || html.isEmpty()) return html; // remove html and clean String withoutHTML = Jsoup.parse(html).text(); withoutHTML = Jsoup.clean(withoutHTML, Whitelist.basic()); - + if(removeAscii) { // remove non ascii chars ... - withoutHTML = withoutHTML.replaceAll("[^\\p{ASCII}]", " "); + withoutHTML = withoutHTML.replaceAll("[^\\p{ASCII}]", " "); + } return withoutHTML; } - + + /** + * Strip out HTML code + * @param html + * @return + * @throws UnsupportedEncodingException + */ + /* + public static String removeHTML(String html) { + return removeHTML(html, true); + } + */ + /** * Send notification to vre members about the created product by writing a post. * @param productName the title of the product diff --git a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java index 307ba20..00d21ca 100644 --- a/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java +++ b/src/test/java/org/gcube/data_catalogue/grsf_publish_ws/JTests.java @@ -374,7 +374,7 @@ public class JTests { public void removeHTML() throws Exception{ String toTest = "

Based on the results of the MP operation[] asdkljlasdklsa . - * ; for 2015–17 in its 2013 meeting and the outcome of the review of exceptional circumstances in its 2015 meeting, the ESC recommended that there is no need to revise the Extended Commission’s 2013 TAC decision regarding the TAC for 2016–17. The recommended annual TAC for the years 2016-2017 is 14,647.4t.

"; - HelperMethods.removeHTML(toTest); + HelperMethods.removeHTML(toTest, false); }