gCubeSystem
/
catalogue-util-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

#23715 integrated the `all_fields` parameter

This commit is contained in:
Francesco Mangiacrapa 2022-07-28 15:22:14 +02:00
parent ab358f6bea
commit 3e0e9252ce
7 changed files with 94 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -4,6 +4,13 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [v1.2.0-SNAPSHOT] - 2022-07-28
**Enhancements**
- [#23692] Optimized the listing and the paging of catalogue items
- Moved to maven-portal-bom 3.7.0-SNAPSHOT
## [v1.1.0] - 2022-05-18 ## [v1.1.0] - 2022-05-18
**New Features** **New Features**

7
pom.xml
View File

@ -12,8 +12,7 @@
<groupId>org.gcube.datacatalogue</groupId> <groupId>org.gcube.datacatalogue</groupId>
<artifactId>catalogue-util-library</artifactId> <artifactId>catalogue-util-library</artifactId>
<version>1.1.0</version> <version>1.2.0-SNAPSHOT</version>
<name>Ckan utility library</name> <name>Ckan utility library</name>
<description> <description>
Utility library to retrieve users information, organizations information and so on from the ckan d4science datacatalogue Utility library to retrieve users information, organizations information and so on from the ckan d4science datacatalogue
@ -42,7 +41,7 @@
<dependency> <dependency>
<groupId>org.gcube.distribution</groupId> <groupId>org.gcube.distribution</groupId>
<artifactId>maven-portal-bom</artifactId> <artifactId>maven-portal-bom</artifactId>
<version>3.6.3</version> <version>3.7.0-SNAPSHOT</version>
<type>pom</type> <type>pom</type>
<scope>import</scope> <scope>import</scope>
</dependency> </dependency>
@ -101,7 +100,7 @@
<dependency> <dependency>
<groupId>org.gcube.data-catalogue</groupId> <groupId>org.gcube.data-catalogue</groupId>
<artifactId>gcat-client</artifactId> <artifactId>gcat-client</artifactId>
<version>[2.0.0-SNAPSHOT, 3.0.0)</version> <version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>

29
src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java
View File

@ -34,11 +34,11 @@ public class GCatCaller {
private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class); private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
public final static String MODERATOR_ITEM_STATUS_PARAMETER = Moderated.CM_ITEM_STATUS_QUERY_PARAMETER; public final static String MODERATOR_ITEM_STATUS_PARAMETER = Moderated.CM_ITEM_STATUS_QUERY_PARAMETER;
public final static String MODERATOR_ITEM_ANY_STATUS_VALUE = CMItemStatus.ANY.getValue(); public final static String MODERATOR_ITEM_ANY_STATUS_VALUE = CMItemStatus.ANY.getValue();
public final static String DEFAULT_SORT_VALUE = "name asc"; public final static String DEFAULT_SORT_VALUE = "name asc";
/** /**
* Instantiates a new g cat caller. * Instantiates a new g cat caller.
@ -223,22 +223,24 @@ public class GCatCaller {
/** /**
* Gets the list items for CM status. * Gets the list items for CM status.
* *
* @param status the status * @param status the status
* @param limit the limit * @param limit the limit
* @param offset the offset * @param offset the offset
* @param filters the filters * @param allFields the all fields. If true returns the all fields of an item
* @param filters the filters
* @param sortForField the sort for field * @param sortForField the sort for field
* @return the list items for CM status * @return the list items for CM status
* @throws WebServiceException the web service exception * @throws WebServiceException the web service exception
* @throws MalformedURLException the malformed URL exception * @throws MalformedURLException the malformed URL exception
*/ */
public String getListItemsForCMStatus(CMItemStatus status, int limit, int offset, Map<String, String> filters, String sortForField) public String getListItemsForCMStatus(CMItemStatus status, int limit, int offset, Boolean allFields,
throws WebServiceException, MalformedURLException { Map<String, String> filters, String sortForField) throws WebServiceException, MalformedURLException {
LOG.trace("getListItemsForCMStatus called"); LOG.trace("getListItemsForCMStatus called");
LOG.info("called getListItemsForCMStatus called with [status: " + status + "], [limit: " + limit LOG.info("called getListItemsForCMStatus called with [status: " + status + "], [limit: " + limit
+ "], [offset: " + offset + "], [filters: " + filters + "]"); + "], [offset: " + offset + "], [filters: " + filters + "]");
Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, limit, offset, false, filters, sortForField); Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, limit, offset, false, allFields, filters,
sortForField);
return getListItemsForQuery(queryParams); return getListItemsForQuery(queryParams);
} }
@ -257,7 +259,8 @@ public class GCatCaller {
LOG.trace("countListItemsForCMStatus called"); LOG.trace("countListItemsForCMStatus called");
LOG.info("Calling count list items for [status: " + status + "], [filters: " + filters + "]"); LOG.info("Calling count list items for [status: " + status + "], [filters: " + filters + "]");
Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, null, null, true, filters, null); Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, null, null, true, false,
filters, null);
String theCount = getListItemsForQuery(queryParams); String theCount = getListItemsForQuery(queryParams);
int count = 0; int count = 0;

34
src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCallerUtil.java
View File

@ -6,6 +6,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.gcube.gcat.api.GCatConstants;
import org.gcube.gcat.api.moderation.CMItemStatus; import org.gcube.gcat.api.moderation.CMItemStatus;
import org.json.simple.JSONArray; import org.json.simple.JSONArray;
import org.json.simple.parser.JSONParser; import org.json.simple.parser.JSONParser;
@ -27,17 +28,19 @@ public class GCatCallerUtil {
/** /**
* Generic query builder for. * Generic query builder for.
* *
* @param status the status * @param status the status
* @param limit the limit * @param limit the limit
* @param offset the offset * @param offset the offset
* @param count the count * @param count the count. If true returns only the count
* @param filters the filters * @param allFields the all fields. If true returns the all fields of an item
* @param filters the filters
* @param sort the sort
* @return the map * @return the map
*/ */
public static Map<String, String> genericQueryBuilderFor(CMItemStatus status, Integer limit, Integer offset, public static Map<String, String> genericQueryBuilderFor(CMItemStatus status, Integer limit, Integer offset,
Boolean count, Map<String, String> filters, String sort) { Boolean count, Boolean allFields, Map<String, String> filters, String sort) {
LOG.info("genericQueryBuilderFor called with [status: " + status + "], [limit: " + limit + "], [offset: " LOG.info("genericQueryBuilderFor called with [status: " + status + "], [limit: " + limit + "], [offset: "
+ offset + "], [count: " + count + "], [filters: " + filters + "]"); + offset + "], [count: " + count + "], [allFields: " + allFields + "], [filters: " + filters + "]");
Map<String, String> queryParams = new HashMap<String, String>(); Map<String, String> queryParams = new HashMap<String, String>();
@ -62,24 +65,29 @@ public class GCatCallerUtil {
if (limit != null) { if (limit != null) {
limit = limit < 0 ? 10 : limit; limit = limit < 0 ? 10 : limit;
queryParams.put("limit", limit + ""); queryParams.put(GCatConstants.LIMIT_QUERY_PARAMETER, String.valueOf(limit));
} }
if (offset != null) { if (offset != null) {
offset = offset < 0 ? 0 : offset; offset = offset < 0 ? 0 : offset;
queryParams.put("offset", offset + ""); queryParams.put(GCatConstants.OFFSET_QUERY_PARAMETER, String.valueOf(offset));
} }
if (sort != null) { if (sort != null) {
queryParams.put("sort", sort); queryParams.put("sort", sort);
}else { } else {
LOG.info("Adding defautl sort: "+GCatCaller.DEFAULT_SORT_VALUE); LOG.info("Adding defautl sort: " + GCatCaller.DEFAULT_SORT_VALUE);
queryParams.put("sort", GCatCaller.DEFAULT_SORT_VALUE); queryParams.put("sort", GCatCaller.DEFAULT_SORT_VALUE);
} }
if (count != null) { if (count != null) {
if (count) { if (count) {
queryParams.put("count", "true"); queryParams.put(GCatConstants.COUNT_QUERY_PARAMETER, String.valueOf(count));
}
}
if (allFields != null) {
if (allFields) {
queryParams.put(GCatConstants.ALL_FIELDS_QUERY_PARAMETER, String.valueOf(allFields));
} }
} }

16
src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/CatalogueContentModeratorSystem.java
View File

@ -32,18 +32,20 @@ public interface CatalogueContentModeratorSystem {
/** /**
* Gets the list items for status. * Gets the list items for status.
* *
* @param theStatus the the status * @param theStatus the the status
* @param limit the limit * @param limit the limit
* @param offset the offset * @param offset the offset
* @param filters add the input filters to query on CKAN * @param allFields the all fields. If true returns the all fields of an item
* @param filters add the input filters to query on CKAN
* @param sortForField the sort for field * @param sortForField the sort for field
* @return the list items for status * @return the list items for status
* @throws WebApplicationException the web application exception * @throws WebApplicationException the web application exception
* @throws MalformedURLException the malformed URL exception * @throws MalformedURLException the malformed URL exception
* @throws InvalidObjectException the invalid object exception * @throws InvalidObjectException the invalid object exception
*/ */
public List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset, List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset, Boolean allFields,
Map<String, String> filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException; Map<String, String> filters, String sortForField)
throws WebApplicationException, MalformedURLException, InvalidObjectException;
/** /**
* Reject item. * Reject item.
@ -85,4 +87,6 @@ public interface CatalogueContentModeratorSystem {
*/ */
void approveItem(String itemName, String moderatorMessage) throws WebApplicationException, MalformedURLException; void approveItem(String itemName, String moderatorMessage) throws WebApplicationException, MalformedURLException;
} }

58
src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/DataCatalogueCMSImpl.java
View File

@ -115,10 +115,11 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
/** /**
* Gets the list items for status. * Gets the list items for status.
* *
* @param theStatus the the status * @param theStatus the the status
* @param limit the limit * @param limit the limit
* @param offset the offset * @param offset the offset
* @param filters add the input filters to query on CKAN * @param allFields the all fields. If true returns the all fields of an item
* @param filters add the input filters to query on CKAN
* @param sortForField the sort for field * @param sortForField the sort for field
* @return the list items for status * @return the list items for status
* @throws WebApplicationException the web application exception * @throws WebApplicationException the web application exception
@ -126,30 +127,40 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
* @throws InvalidObjectException the invalid object exception * @throws InvalidObjectException the invalid object exception
*/ */
@Override @Override
public List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset, public List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset, Boolean allFields,
Map<String, String> filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException { Map<String, String> filters, String sortForField)
throws WebApplicationException, MalformedURLException, InvalidObjectException {
LOG.info("called getListItemsForStatus with [status: " + theStatus + "], [limit: " + limit + "], [offset: " LOG.info("called getListItemsForStatus with [status: " + theStatus + "], [limit: " + limit + "], [offset: "
+ offset + "], [filters: " + filters + "]"); + offset + "], [allFields: " + allFields + "], [filters: " + filters + "]");
List<CkanDataset> listDataset = null; List<CkanDataset> listDataset = null;
checkNotNull(theStatus); checkNotNull(theStatus);
JSONArray jsonArray = getSourceArrayOfItemsForStatus(theStatus, limit, offset, filters, sortForField); JSONArray jsonArray = getSourceArrayOfItemsForStatus(theStatus, limit, offset, allFields, filters, sortForField);
if (jsonArray != null) { if (jsonArray != null) {
int size = jsonArray.size(); int size = jsonArray.size();
listDataset = new ArrayList<CkanDataset>(size); listDataset = new ArrayList<CkanDataset>(size);
LOG.info("reading and converting " + size + " dataset..."); LOG.info("reading and converting " + size + " dataset...");
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
String datasetName = null; String jsonValueDataset = null;
try { try {
datasetName = (String) jsonArray.get(i); CkanDataset toCkanDataset = null;
LOG.debug("reading dataset: " + datasetName); if(!allFields) {
String jsonValueDataset = gCatCaller.getDatasetForName(datasetName); //here the array contains the list of dataset name
LOG.trace("the JSON dataset is: " + jsonValueDataset); String datasetName = (String) jsonArray.get(i);
CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ); LOG.debug("going to read from gCat the dataset: " + datasetName);
jsonValueDataset = gCatCaller.getDatasetForName(datasetName);
LOG.trace("the JSON dataset is: " + jsonValueDataset);
}else {
LOG.debug("reading from the array the "+i+"mo json object");
//here the array contains the list of dataset json fields
jsonValueDataset = (String) jsonArray.get(i);
LOG.trace("the JSON dataset is: " + jsonValueDataset);
}
toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ);
LOG.debug("converted as dataset: " + toCkanDataset); LOG.debug("converted as dataset: " + toCkanDataset);
listDataset.add(toCkanDataset); listDataset.add(toCkanDataset);
} catch (IOException e) { } catch (IOException e) {
LOG.warn("Error on reading/converting the dataset name: " + datasetName, e); LOG.warn("Error on reading/converting the dataset: " + jsonValueDataset, e);
} }
} }
} }
@ -190,10 +201,11 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
/** /**
* Gets the source array of items for status read by gCatClient. * Gets the source array of items for status read by gCatClient.
* *
* @param theStatus the the status * @param theStatus the the status
* @param limit the limit * @param limit the limit
* @param offset the offset * @param offset the offset
* @param filters the filters * @param allFields the all fields. If true returns the all fields of an item
* @param filters the filters
* @param sortForField the sort for field * @param sortForField the sort for field
* @return the source array of items for status * @return the source array of items for status
* @throws WebApplicationException the web application exception * @throws WebApplicationException the web application exception
@ -201,9 +213,10 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
* @throws InvalidObjectException the invalid object exception * @throws InvalidObjectException the invalid object exception
*/ */
protected org.json.simple.JSONArray getSourceArrayOfItemsForStatus(ItemStatus theStatus, int limit, int offset, protected org.json.simple.JSONArray getSourceArrayOfItemsForStatus(ItemStatus theStatus, int limit, int offset,
Map<String, String> filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException { Boolean allFields, Map<String, String> filters, String sortForField)
throws WebApplicationException, MalformedURLException, InvalidObjectException {
LOG.info("called getSourceArrayOfItemsForStatus with [status: " + theStatus + "], [limit: " + limit LOG.info("called getSourceArrayOfItemsForStatus with [status: " + theStatus + "], [limit: " + limit
+ "], [offset: " + offset + "], [filters: " + filters + "]"); + "], [offset: " + offset + "], [allFields: " + allFields + "], [filters: " + filters + "]");
checkNotNull(theStatus); checkNotNull(theStatus);
// TODO MUST BE CHANGED FOR THE STATUS // TODO MUST BE CHANGED FOR THE STATUS
org.json.simple.JSONArray jsonArray = null; org.json.simple.JSONArray jsonArray = null;
@ -216,7 +229,8 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
*/ */
CMItemStatus cmiStatus = toCMStatus(theStatus); CMItemStatus cmiStatus = toCMStatus(theStatus);
String datasetNames = gCatCaller.getListItemsForCMStatus(cmiStatus, limit, offset, filters, sortForField); String datasetNames = gCatCaller.getListItemsForCMStatus(cmiStatus, limit, offset, allFields, filters,
sortForField);
if (datasetNames != null) { if (datasetNames != null) {
LOG.debug("for status " + theStatus + " found dataset: " + datasetNames); LOG.debug("for status " + theStatus + " found dataset: " + datasetNames);

1
src/test/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/resources/