gCubeSystem
/
catalogue-util-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

#23715 integrated the `all_fields` parameter

This commit is contained in:
Francesco Mangiacrapa 2022-07-28 15:22:14 +02:00
parent ab358f6bea
commit 3e0e9252ce
7 changed files with 94 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -4,6 +4,13 @@
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [v1.2.0-SNAPSHOT] - 2022-07-28
**Enhancements**
- [#23692] Optimized the listing and the paging of catalogue items
- Moved to maven-portal-bom 3.7.0-SNAPSHOT
## [v1.1.0] - 2022-05-18
**New Features**

7
pom.xml
View File

@ -12,8 +12,7 @@
<groupId>org.gcube.datacatalogue</groupId>
<artifactId>catalogue-util-library</artifactId>
<version>1.1.0</version>
<version>1.2.0-SNAPSHOT</version>
<name>Ckan utility library</name>
<description>
Utility library to retrieve users information, organizations information and so on from the ckan d4science datacatalogue
@ -42,7 +41,7 @@
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-portal-bom</artifactId>
<version>3.6.3</version>
<version>3.7.0-SNAPSHOT</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@ -101,7 +100,7 @@
<dependency>
<groupId>org.gcube.data-catalogue</groupId>
<artifactId>gcat-client</artifactId>
<version>[2.0.0-SNAPSHOT, 3.0.0)</version>
<version>[2.0.0-SNAPSHOT, 3.0.0-SNAPSHOT)</version>
<scope>compile</scope>
</dependency>

29
src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java
View File

@ -34,11 +34,11 @@ public class GCatCaller {
private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
public final static String MODERATOR_ITEM_STATUS_PARAMETER = Moderated.CM_ITEM_STATUS_QUERY_PARAMETER;
public final static String MODERATOR_ITEM_ANY_STATUS_VALUE = CMItemStatus.ANY.getValue();
public final static String DEFAULT_SORT_VALUE = "name asc";
public final static String MODERATOR_ITEM_STATUS_PARAMETER = Moderated.CM_ITEM_STATUS_QUERY_PARAMETER;
public final static String MODERATOR_ITEM_ANY_STATUS_VALUE = CMItemStatus.ANY.getValue();
public final static String DEFAULT_SORT_VALUE = "name asc";
/**
* Instantiates a new g cat caller.
@ -223,22 +223,24 @@ public class GCatCaller {
/**
* Gets the list items for CM status.
*
* @param status the status
* @param limit the limit
* @param offset the offset
* @param filters the filters
* @param status the status
* @param limit the limit
* @param offset the offset
* @param allFields the all fields. If true returns the all fields of an item
* @param filters the filters
* @param sortForField the sort for field
* @return the list items for CM status
* @throws WebServiceException the web service exception
* @throws MalformedURLException the malformed URL exception
*/
public String getListItemsForCMStatus(CMItemStatus status, int limit, int offset, Map<String, String> filters, String sortForField)
throws WebServiceException, MalformedURLException {
public String getListItemsForCMStatus(CMItemStatus status, int limit, int offset, Boolean allFields,
Map<String, String> filters, String sortForField) throws WebServiceException, MalformedURLException {
LOG.trace("getListItemsForCMStatus called");
LOG.info("called getListItemsForCMStatus called with [status: " + status + "], [limit: " + limit
+ "], [offset: " + offset + "], [filters: " + filters + "]");
Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, limit, offset, false, filters, sortForField);
Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, limit, offset, false, allFields, filters,
sortForField);
return getListItemsForQuery(queryParams);
}
@ -257,7 +259,8 @@ public class GCatCaller {
LOG.trace("countListItemsForCMStatus called");
LOG.info("Calling count list items for [status: " + status + "], [filters: " + filters + "]");
Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, null, null, true, filters, null);
Map<String, String> queryParams = GCatCallerUtil.genericQueryBuilderFor(status, null, null, true, false,
filters, null);
String theCount = getListItemsForQuery(queryParams);
int count = 0;

34
src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCallerUtil.java
View File

@ -6,6 +6,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.gcube.gcat.api.GCatConstants;
import org.gcube.gcat.api.moderation.CMItemStatus;
import org.json.simple.JSONArray;
import org.json.simple.parser.JSONParser;
@ -27,17 +28,19 @@ public class GCatCallerUtil {
/**
* Generic query builder for.
*
* @param status the status
* @param limit the limit
* @param offset the offset
* @param count the count
* @param filters the filters
* @param status the status
* @param limit the limit
* @param offset the offset
* @param count the count. If true returns only the count
* @param allFields the all fields. If true returns the all fields of an item
* @param filters the filters
* @param sort the sort
* @return the map
*/
public static Map<String, String> genericQueryBuilderFor(CMItemStatus status, Integer limit, Integer offset,
Boolean count, Map<String, String> filters, String sort) {
Boolean count, Boolean allFields, Map<String, String> filters, String sort) {
LOG.info("genericQueryBuilderFor called with [status: " + status + "], [limit: " + limit + "], [offset: "
+ offset + "], [count: " + count + "], [filters: " + filters + "]");
+ offset + "], [count: " + count + "], [allFields: " + allFields + "], [filters: " + filters + "]");
Map<String, String> queryParams = new HashMap<String, String>();
@ -62,24 +65,29 @@ public class GCatCallerUtil {
if (limit != null) {
limit = limit < 0 ? 10 : limit;
queryParams.put("limit", limit + "");
queryParams.put(GCatConstants.LIMIT_QUERY_PARAMETER, String.valueOf(limit));
}
if (offset != null) {
offset = offset < 0 ? 0 : offset;
queryParams.put("offset", offset + "");
queryParams.put(GCatConstants.OFFSET_QUERY_PARAMETER, String.valueOf(offset));
}
if (sort != null) {
queryParams.put("sort", sort);
}else {
LOG.info("Adding defautl sort: "+GCatCaller.DEFAULT_SORT_VALUE);
} else {
LOG.info("Adding defautl sort: " + GCatCaller.DEFAULT_SORT_VALUE);
queryParams.put("sort", GCatCaller.DEFAULT_SORT_VALUE);
}
if (count != null) {
if (count) {
queryParams.put("count", "true");
queryParams.put(GCatConstants.COUNT_QUERY_PARAMETER, String.valueOf(count));
}
}
if (allFields != null) {
if (allFields) {
queryParams.put(GCatConstants.ALL_FIELDS_QUERY_PARAMETER, String.valueOf(allFields));
}
}

16
src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/CatalogueContentModeratorSystem.java
View File

@ -32,18 +32,20 @@ public interface CatalogueContentModeratorSystem {
/**
* Gets the list items for status.
*
* @param theStatus the the status
* @param limit the limit
* @param offset the offset
* @param filters add the input filters to query on CKAN
* @param theStatus the the status
* @param limit the limit
* @param offset the offset
* @param allFields the all fields. If true returns the all fields of an item
* @param filters add the input filters to query on CKAN
* @param sortForField the sort for field
* @return the list items for status
* @throws WebApplicationException the web application exception
* @throws MalformedURLException the malformed URL exception
* @throws InvalidObjectException the invalid object exception
*/
public List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset,
Map<String, String> filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException;
List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset, Boolean allFields,
Map<String, String> filters, String sortForField)
throws WebApplicationException, MalformedURLException, InvalidObjectException;
/**
* Reject item.
@ -85,4 +87,6 @@ public interface CatalogueContentModeratorSystem {
*/
void approveItem(String itemName, String moderatorMessage) throws WebApplicationException, MalformedURLException;
}

58
src/main/java/org/gcube/datacatalogue/utillibrary/server/cms/DataCatalogueCMSImpl.java
View File

@ -115,10 +115,11 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
/**
* Gets the list items for status.
*
* @param theStatus the the status
* @param limit the limit
* @param offset the offset
* @param filters add the input filters to query on CKAN
* @param theStatus the the status
* @param limit the limit
* @param offset the offset
* @param allFields the all fields. If true returns the all fields of an item
* @param filters add the input filters to query on CKAN
* @param sortForField the sort for field
* @return the list items for status
* @throws WebApplicationException the web application exception
@ -126,30 +127,40 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
* @throws InvalidObjectException the invalid object exception
*/
@Override
public List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset,
Map<String, String> filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException {
public List<CkanDataset> getListItemsForStatus(ItemStatus theStatus, int limit, int offset, Boolean allFields,
Map<String, String> filters, String sortForField)
throws WebApplicationException, MalformedURLException, InvalidObjectException {
LOG.info("called getListItemsForStatus with [status: " + theStatus + "], [limit: " + limit + "], [offset: "
+ offset + "], [filters: " + filters + "]");
+ offset + "], [allFields: " + allFields + "], [filters: " + filters + "]");
List<CkanDataset> listDataset = null;
checkNotNull(theStatus);
JSONArray jsonArray = getSourceArrayOfItemsForStatus(theStatus, limit, offset, filters, sortForField);
JSONArray jsonArray = getSourceArrayOfItemsForStatus(theStatus, limit, offset, allFields, filters, sortForField);
if (jsonArray != null) {
int size = jsonArray.size();
listDataset = new ArrayList<CkanDataset>(size);
LOG.info("reading and converting " + size + " dataset...");
for (int i = 0; i < size; i++) {
String datasetName = null;
String jsonValueDataset = null;
try {
datasetName = (String) jsonArray.get(i);
LOG.debug("reading dataset: " + datasetName);
String jsonValueDataset = gCatCaller.getDatasetForName(datasetName);
LOG.trace("the JSON dataset is: " + jsonValueDataset);
CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ);
CkanDataset toCkanDataset = null;
if(!allFields) {
//here the array contains the list of dataset name
String datasetName = (String) jsonArray.get(i);
LOG.debug("going to read from gCat the dataset: " + datasetName);
jsonValueDataset = gCatCaller.getDatasetForName(datasetName);
LOG.trace("the JSON dataset is: " + jsonValueDataset);
}else {
LOG.debug("reading from the array the "+i+"mo json object");
//here the array contains the list of dataset json fields
jsonValueDataset = (String) jsonArray.get(i);
LOG.trace("the JSON dataset is: " + jsonValueDataset);
}
toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ);
LOG.debug("converted as dataset: " + toCkanDataset);
listDataset.add(toCkanDataset);
} catch (IOException e) {
LOG.warn("Error on reading/converting the dataset name: " + datasetName, e);
LOG.warn("Error on reading/converting the dataset: " + jsonValueDataset, e);
}
}
}
@ -190,10 +201,11 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
/**
* Gets the source array of items for status read by gCatClient.
*
* @param theStatus the the status
* @param limit the limit
* @param offset the offset
* @param filters the filters
* @param theStatus the the status
* @param limit the limit
* @param offset the offset
* @param allFields the all fields. If true returns the all fields of an item
* @param filters the filters
* @param sortForField the sort for field
* @return the source array of items for status
* @throws WebApplicationException the web application exception
@ -201,9 +213,10 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
* @throws InvalidObjectException the invalid object exception
*/
protected org.json.simple.JSONArray getSourceArrayOfItemsForStatus(ItemStatus theStatus, int limit, int offset,
Map<String, String> filters, String sortForField) throws WebApplicationException, MalformedURLException, InvalidObjectException {
Boolean allFields, Map<String, String> filters, String sortForField)
throws WebApplicationException, MalformedURLException, InvalidObjectException {
LOG.info("called getSourceArrayOfItemsForStatus with [status: " + theStatus + "], [limit: " + limit
+ "], [offset: " + offset + "], [filters: " + filters + "]");
+ "], [offset: " + offset + "], [allFields: " + allFields + "], [filters: " + filters + "]");
checkNotNull(theStatus);
// TODO MUST BE CHANGED FOR THE STATUS
org.json.simple.JSONArray jsonArray = null;
@ -216,7 +229,8 @@ public class DataCatalogueCMSImpl implements CatalogueContentModeratorSystem {
*/
CMItemStatus cmiStatus = toCMStatus(theStatus);
String datasetNames = gCatCaller.getListItemsForCMStatus(cmiStatus, limit, offset, filters, sortForField);
String datasetNames = gCatCaller.getListItemsForCMStatus(cmiStatus, limit, offset, allFields, filters,
sortForField);
if (datasetNames != null) {
LOG.debug("for status " + theStatus + " found dataset: " + datasetNames);

1
src/test/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/resources/