From 3e0e9252ceb9f69b2833b1a89fce47b3dbb5566b Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Thu, 28 Jul 2022 15:22:14 +0200 Subject: [PATCH] #23715 integrated the `all_fields` parameter --- CHANGELOG.md | 7 +++ pom.xml | 7 +-- .../utillibrary/gcat/GCatCaller.java | 29 +++++----- .../utillibrary/gcat/GCatCallerUtil.java | 34 ++++++----- .../cms/CatalogueContentModeratorSystem.java | 16 +++-- .../server/cms/DataCatalogueCMSImpl.java | 58 ++++++++++++------- src/test/.gitignore | 1 + 7 files changed, 94 insertions(+), 58 deletions(-) create mode 100644 src/test/.gitignore diff --git a/CHANGELOG.md b/CHANGELOG.md index ee6d92f..c857bbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v1.2.0-SNAPSHOT] - 2022-07-28 + +**Enhancements** + +- [#23692] Optimized the listing and the paging of catalogue items +- Moved to maven-portal-bom 3.7.0-SNAPSHOT + ## [v1.1.0] - 2022-05-18 **New Features** diff --git a/pom.xml b/pom.xml index 377b23d..6971770 100644 --- a/pom.xml +++ b/pom.xml @@ -12,8 +12,7 @@ org.gcube.datacatalogue catalogue-util-library - 1.1.0 - + 1.2.0-SNAPSHOT Ckan utility library Utility library to retrieve users information, organizations information and so on from the ckan d4science datacatalogue @@ -42,7 +41,7 @@ org.gcube.distribution maven-portal-bom - 3.6.3 + 3.7.0-SNAPSHOT pom import @@ -101,7 +100,7 @@ org.gcube.data-catalogue gcat-client - [2.0.0-SNAPSHOT, 3.0.0) + [2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT) compile diff --git a/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java b/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java index f2cd002..6179bde 100644 --- a/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java +++ b/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java @@ -34,11 +34,11 @@ public class GCatCaller { private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class); - public final static String MODERATOR_ITEM_STATUS_PARAMETER = Moderated.CM_ITEM_STATUS_QUERY_PARAMETER; - - public final static String MODERATOR_ITEM_ANY_STATUS_VALUE = CMItemStatus.ANY.getValue(); - - public final static String DEFAULT_SORT_VALUE = "name asc"; + public final static String MODERATOR_ITEM_STATUS_PARAMETER = Moderated.CM_ITEM_STATUS_QUERY_PARAMETER; + + public final static String MODERATOR_ITEM_ANY_STATUS_VALUE = CMItemStatus.ANY.getValue(); + + public final static String DEFAULT_SORT_VALUE = "name asc"; /** * Instantiates a new g cat caller. @@ -223,22 +223,24 @@ public class GCatCaller { /** * Gets the list items for CM status. * - * @param status the status - * @param limit the limit - * @param offset the offset - * @param filters the filters + * @param status the status + * @param limit the limit + * @param offset the offset + * @param allFields the all fields. If true returns the all fields of an item + * @param filters the filters * @param sortForField the sort for field * @return the list items for CM status * @throws WebServiceException the web service exception * @throws MalformedURLException the malformed URL exception */ - public String getListItemsForCMStatus(CMItemStatus status, int limit, int offset, Map filters, String sortForField) - throws WebServiceException, MalformedURLException { + public String getListItemsForCMStatus(CMItemStatus status, int limit, int offset, Boolean allFields, + Map filters, String sortForField) throws WebServiceException, MalformedURLException { LOG.trace("getListItemsForCMStatus called"); LOG.info("called getListItemsForCMStatus called with [status: " + status + "], [limit: " + limit + "], [offset: " + offset + "], [filters: " + filters + "]"); - Map queryParams = GCatCallerUtil.genericQueryBuilderFor(status, limit, offset, false, filters, sortForField); + Map queryParams = GCatCallerUtil.genericQueryBuilderFor(status, limit, offset, false, allFields, filters, + sortForField); return getListItemsForQuery(queryParams); } @@ -257,7 +259,8 @@ public class GCatCaller { LOG.trace("countListItemsForCMStatus called"); LOG.info("Calling count list items for [status: " + status + "], [filters: " + filters + "]"); - Map queryParams = GCatCallerUtil.genericQueryBuilderFor(status, null, null, true, filters, null); + Map queryParams = GCatCallerUtil.genericQueryBuilderFor(status, null, null, true, false, + filters, null); String theCount = getListItemsForQuery(queryParams); int count = 0; diff --git a/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCallerUtil.java b/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCallerUtil.java index b7ad6fc..d434df0 100644 --- a/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCallerUtil.java +++ b/src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCallerUtil.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.gcube.gcat.api.GCatConstants; import org.gcube.gcat.api.moderation.CMItemStatus; import org.json.simple.JSONArray; import org.json.simple.parser.JSONParser; @@ -27,17 +28,19 @@ public class GCatCallerUtil { /** * Generic query builder for. * - * @param status the status - * @param limit the limit - * @param offset the offset - * @param count the count - * @param filters the filters + * @param status the status + * @param limit the limit + * @param offset the offset + * @param count the count. If true returns only the count + * @param allFields the all fields. If true returns the all fields of an item + * @param filters the filters + * @param sort the sort * @return the map */ public static Map genericQueryBuilderFor(CMItemStatus status, Integer limit, Integer offset, - Boolean count, Map filters, String sort) { + Boolean count, Boolean allFields, Map filters, String sort) { LOG.info("genericQueryBuilderFor called with [status: " + status + "], [limit: " + limit + "], [offset: " - + offset + "], [count: " + count + "], [filters: " + filters + "]"); + + offset + "], [count: " + count + "], [allFields: " + allFields + "], [filters: " + filters + "]"); Map queryParams = new HashMap(); @@ -62,24 +65,29 @@ public class GCatCallerUtil { if (limit != null) { limit = limit < 0 ? 10 : limit; - queryParams.put("limit", limit + ""); + queryParams.put(GCatConstants.LIMIT_QUERY_PARAMETER, String.valueOf(limit)); } if (offset != null) { offset = offset < 0 ? 0 : offset; - queryParams.put("offset", offset + ""); + queryParams.put(GCatConstants.OFFSET_QUERY_PARAMETER, String.valueOf(offset)); } if (sort != null) { queryParams.put("sort", sort); - }else { - LOG.info("Adding defautl sort: "+GCatCaller.DEFAULT_SORT_VALUE); + } else { + LOG.info("Adding defautl sort: " + GCatCaller.DEFAULT_SORT_VALUE); queryParams.put("sort", GCatCaller.DEFAULT_SORT_VALUE); } - if (count != null) { if (count) { - queryParams.put("count", "true"); + queryParams.put(GCatConstants.COUNT_QUERY_PARAMETER, String.valueOf(count)); + } + } + + if (allFields != null) { + if (allFields) { + queryParams.put(GCatConstants.ALL_FIELDS_QUERY_PARAMETER, String.valueOf(allFields)); } } diff --git a/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/CatalogueContentModeratorSystem.java b/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/CatalogueContentModeratorSystem.java index 8d00881..0af1cd8 100644 --- a/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/CatalogueContentModeratorSystem.java +++ b/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/CatalogueContentModeratorSystem.java @@ -32,18 +32,20 @@ public interface CatalogueContentModeratorSystem { /** * Gets the list items for status. * - * @param theStatus the the status - * @param limit the limit - * @param offset the offset - * @param filters add the input filters to query on CKAN + * @param theStatus the the status + * @param limit the limit + * @param offset the offset + * @param allFields the all fields. If true returns the all fields of an item + * @param filters add the input filters to query on CKAN * @param sortForField the sort for field * @return the list items for status * @throws WebApplicationException the web application exception * @throws MalformedURLException the malformed URL exception * @throws InvalidObjectException the invalid object exception */ - public List getListItemsForStatus(ItemStatus theStatus, int limit, int offset, - Map filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException; + List getListItemsForStatus(ItemStatus theStatus, int limit, int offset, Boolean allFields, + Map filters, String sortForField) + throws WebApplicationException, MalformedURLException, InvalidObjectException; /** * Reject item. @@ -85,4 +87,6 @@ public interface CatalogueContentModeratorSystem { */ void approveItem(String itemName, String moderatorMessage) throws WebApplicationException, MalformedURLException; + + } diff --git a/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/DataCatalogueCMSImpl.java b/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/DataCatalogueCMSImpl.java index c5ba458..10f419c 100644 --- a/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/DataCatalogueCMSImpl.java +++ b/src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/DataCatalogueCMSImpl.java @@ -115,10 +115,11 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem { /** * Gets the list items for status. * - * @param theStatus the the status - * @param limit the limit - * @param offset the offset - * @param filters add the input filters to query on CKAN + * @param theStatus the the status + * @param limit the limit + * @param offset the offset + * @param allFields the all fields. If true returns the all fields of an item + * @param filters add the input filters to query on CKAN * @param sortForField the sort for field * @return the list items for status * @throws WebApplicationException the web application exception @@ -126,30 +127,40 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem { * @throws InvalidObjectException the invalid object exception */ @Override - public List getListItemsForStatus(ItemStatus theStatus, int limit, int offset, - Map filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException { + public List getListItemsForStatus(ItemStatus theStatus, int limit, int offset, Boolean allFields, + Map filters, String sortForField) + throws WebApplicationException, MalformedURLException, InvalidObjectException { LOG.info("called getListItemsForStatus with [status: " + theStatus + "], [limit: " + limit + "], [offset: " - + offset + "], [filters: " + filters + "]"); + + offset + "], [allFields: " + allFields + "], [filters: " + filters + "]"); List listDataset = null; checkNotNull(theStatus); - JSONArray jsonArray = getSourceArrayOfItemsForStatus(theStatus, limit, offset, filters, sortForField); + JSONArray jsonArray = getSourceArrayOfItemsForStatus(theStatus, limit, offset, allFields, filters, sortForField); if (jsonArray != null) { int size = jsonArray.size(); listDataset = new ArrayList(size); LOG.info("reading and converting " + size + " dataset..."); for (int i = 0; i < size; i++) { - String datasetName = null; + String jsonValueDataset = null; try { - datasetName = (String) jsonArray.get(i); - LOG.debug("reading dataset: " + datasetName); - String jsonValueDataset = gCatCaller.getDatasetForName(datasetName); - LOG.trace("the JSON dataset is: " + jsonValueDataset); - CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ); + CkanDataset toCkanDataset = null; + if(!allFields) { + //here the array contains the list of dataset name + String datasetName = (String) jsonArray.get(i); + LOG.debug("going to read from gCat the dataset: " + datasetName); + jsonValueDataset = gCatCaller.getDatasetForName(datasetName); + LOG.trace("the JSON dataset is: " + jsonValueDataset); + }else { + LOG.debug("reading from the array the "+i+"mo json object"); + //here the array contains the list of dataset json fields + jsonValueDataset = (String) jsonArray.get(i); + LOG.trace("the JSON dataset is: " + jsonValueDataset); + } + toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ); LOG.debug("converted as dataset: " + toCkanDataset); listDataset.add(toCkanDataset); } catch (IOException e) { - LOG.warn("Error on reading/converting the dataset name: " + datasetName, e); + LOG.warn("Error on reading/converting the dataset: " + jsonValueDataset, e); } } } @@ -190,10 +201,11 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem { /** * Gets the source array of items for status read by gCatClient. * - * @param theStatus the the status - * @param limit the limit - * @param offset the offset - * @param filters the filters + * @param theStatus the the status + * @param limit the limit + * @param offset the offset + * @param allFields the all fields. If true returns the all fields of an item + * @param filters the filters * @param sortForField the sort for field * @return the source array of items for status * @throws WebApplicationException the web application exception @@ -201,9 +213,10 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem { * @throws InvalidObjectException the invalid object exception */ protected org.json.simple.JSONArray getSourceArrayOfItemsForStatus(ItemStatus theStatus, int limit, int offset, - Map filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException { + Boolean allFields, Map filters, String sortForField) + throws WebApplicationException, MalformedURLException, InvalidObjectException { LOG.info("called getSourceArrayOfItemsForStatus with [status: " + theStatus + "], [limit: " + limit - + "], [offset: " + offset + "], [filters: " + filters + "]"); + + "], [offset: " + offset + "], [allFields: " + allFields + "], [filters: " + filters + "]"); checkNotNull(theStatus); // TODO MUST BE CHANGED FOR THE STATUS org.json.simple.JSONArray jsonArray = null; @@ -216,7 +229,8 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem { */ CMItemStatus cmiStatus = toCMStatus(theStatus); - String datasetNames = gCatCaller.getListItemsForCMStatus(cmiStatus, limit, offset, filters, sortForField); + String datasetNames = gCatCaller.getListItemsForCMStatus(cmiStatus, limit, offset, allFields, filters, + sortForField); if (datasetNames != null) { LOG.debug("for status " + theStatus + " found dataset: " + datasetNames); diff --git a/src/test/.gitignore b/src/test/.gitignore new file mode 100644 index 0000000..cb6eb2c --- /dev/null +++ b/src/test/.gitignore @@ -0,0 +1 @@ +/resources/