gCubeSystem
/
catalogue-util-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

#23108 added method `performQuery`

This commit is contained in:
Francesco Mangiacrapa 2022-04-04 17:27:52 +02:00
parent 4cb97f41fa
commit c9f64ab384
8 changed files with 207 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.classpath
View File

@ -13,11 +13,6 @@
</attributes>
</classpathentry>
<classpathentry kind="src" path="src/main/resources"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
@ -26,9 +21,14 @@
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="owner.project.facets" value="java"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>

2
.settings/org.eclipse.jdt.core.prefs
View File

@ -10,6 +10,6 @@ org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8

2
.settings/org.eclipse.wst.common.project.facet.core.xml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<faceted-project>
<installed facet="java" version="1.8"/>
<installed facet="jst.utility" version="1.0"/>
<installed facet="java" version="1.8"/>
</faceted-project>

1
CHANGELOG.md
View File

@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
* [#21643] Integrated with the content-moderator-system facilities
* [#22838] Integrated the collection provided by gCat to read the service configurations
* [#23108] The Moderation facility accessible to Catalogue Editor/Admin in read only mode
## [v1.0.3] - 2022-01-21

2
pom.xml
View File

@ -31,6 +31,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<gwtVersion>2.7.0</gwtVersion>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<apache.http.version>4.4.1</apache.http.version>
<gcube.jackson.version>2.8.11</gcube.jackson.version>
</properties>

8
src/main/java/org/gcube/datacatalogue/utillibrary/gcat/GCatCaller.java
View File

@ -35,7 +35,9 @@ public class GCatCaller {
private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
public final static String MODERATOR_FIELD_NAME = Moderated.SYSTEM_CM_ITEM_STATUS.replaceAll(":", "");
public final static String MODERATOR_ITEM_STATUS_FIELD_NAME = Moderated.SYSTEM_CM_ITEM_STATUS.replaceAll(":", "");
public static final String MODERATOR_SYSTEM_CM_PREFIX = Moderated.SYSTEM_CM_PREFIX.replaceAll(":", "");
/**
* Instantiates a new g cat caller.
@ -235,7 +237,7 @@ public class GCatCaller {
Map<String, String> queryParams = new HashMap<String, String>();
// queryParams.put(Moderated.SYSTEM_CM_ITEM_STATUS, status.getValue());
String modStatusField = GCatCaller.MODERATOR_FIELD_NAME;
String modStatusField = GCatCaller.MODERATOR_ITEM_STATUS_FIELD_NAME;
queryParams.put("q", "extras_" + modStatusField + ":" + status.getValue());
if (limit < 0)
@ -268,7 +270,7 @@ public class GCatCaller {
Map<String, String> queryParams = new HashMap<String, String>();
// queryParams.put(Moderated.SYSTEM_CM_ITEM_STATUS, status.getValue());
String modStatusField = GCatCaller.MODERATOR_FIELD_NAME;
String modStatusField = GCatCaller.MODERATOR_ITEM_STATUS_FIELD_NAME;
queryParams.put("q", "extras_" + modStatusField + ":" + status.getValue());
queryParams.put("count", "true");
String theCount = getListItemsForQuery(queryParams);

20
src/main/java/org/gcube/datacatalogue/utillibrary/server/DataCatalogueImpl.java
View File

@ -3,6 +3,7 @@ package org.gcube.datacatalogue.utillibrary.server;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import java.net.MalformedURLException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
@ -16,6 +17,8 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.xml.ws.WebServiceException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
@ -612,6 +615,8 @@ public class DataCatalogueImpl implements DataCatalogue {
* Checks if is notification to users enabled.
*
* @return true, if is notification to users enabled
*
* YOU NEED TO USE gCat
*/
/*
* (non-Javadoc)
@ -620,6 +625,7 @@ public class DataCatalogueImpl implements DataCatalogue {
* isNotificationToUsersEnabled()
*/
@Override
@Deprecated
public boolean isNotificationToUsersEnabled() {
return ALERT_USERS_ON_POST_CREATION;
}
@ -1058,6 +1064,20 @@ public class DataCatalogueImpl implements DataCatalogue {
return null;
}
/**
* Perform query.
*
* @param queryParameters the query parameters
* @return the list of items
* @throws WebServiceException the web service exception
* @throws MalformedURLException the malformed URL exception
*/
public String performQuery(Map<String, String> queryParameters) throws WebServiceException, MalformedURLException {
checkNotNull(queryParameters);
String results = gCatCaller.getListItemsForQuery(queryParameters);
return results;
}
/**
* *****************************************************************************
*

322
src/test/java/org/gcube/datacatalogue/utillibrary/test/TestDataCatalogueLib.java
View File

@ -1,6 +1,5 @@
package org.gcube.datacatalogue.utillibrary.test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@ -12,6 +11,7 @@ import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datacatalogue.utillibrary.ckan.MarshUnmarshCkanObject;
import org.gcube.datacatalogue.utillibrary.ckan.MarshUnmarshCkanObject.METHOD;
import org.gcube.datacatalogue.utillibrary.gcat.GCatCaller;
import org.gcube.datacatalogue.utillibrary.server.ApplicationProfileScopePerUrlReader;
import org.gcube.datacatalogue.utillibrary.server.DataCatalogueFactory;
import org.gcube.datacatalogue.utillibrary.server.DataCatalogueImpl;
@ -21,16 +21,12 @@ import org.gcube.datacatalogue.utillibrary.shared.jackan.model.CkanGroup;
import org.gcube.datacatalogue.utillibrary.shared.jackan.model.CkanOrganization;
import org.gcube.datacatalogue.utillibrary.shared.jackan.model.CkanUser;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.LoggerFactory;
/**
* The Class TestDataCatalogueLib.
*
* @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy)
* Jun 1, 2020
* @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy) Jun 1, 2020
*/
public class TestDataCatalogueLib {
@ -38,11 +34,13 @@ public class TestDataCatalogueLib {
private DataCatalogueFactory factory;
private String scope = "/gcube/devsec/devVRE";
//private String scope = "/gcube";
//private String scope = "/pred4s/preprod/preVRE";
//private String scope = "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti";
//private String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab";
//private String testUser = "costantino_perciante";
// private String scope = "/gcube";
// private String scope = "/pred4s/preprod/preVRE";
// private String scope =
// "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti";
// private String scope =
// "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab";
// private String testUser = "costantino_perciante";
private String testUser = "francesco.mangiacrapa";
private String authorizationToken = "";
String subjectId = "aa_father4";
@ -55,23 +53,23 @@ public class TestDataCatalogueLib {
* @throws Exception the exception
*/
//@Before
public void before() throws Exception{
public void before() throws Exception {
factory = DataCatalogueFactory.getFactory();
}
/**
* Factory test.
*
* @throws Exception the exception
*/
//@Test
public void factoryTest() throws Exception{
// @Test
public void factoryTest() throws Exception {
DataCatalogueFactory factory = DataCatalogueFactory.getFactory();
while(true){
while (true) {
factory.getUtilsPerScope(scope);
Thread.sleep(60* 1000 * 3);
Thread.sleep(60 * 1000 * 3);
factory.getUtilsPerScope(scope);
break;
}
@ -82,14 +80,14 @@ public class TestDataCatalogueLib {
}
}
/**
* Gets the scope per url.
*
* @return the scope per url
*/
//@Test
public void getScopePerUrl(){
// @Test
public void getScopePerUrl() {
ScopeProvider.instance.set(scope);
String url = "https://dev2.d4science.org/group/devvre/ckan";
@ -101,16 +99,15 @@ public class TestDataCatalogueLib {
String scopeToUse2 = ApplicationProfileScopePerUrlReader.getScopePerUrl(url2);
LOG.debug("Retrieved scope is " + scopeToUse2);
}
/**
* Gets the user role by group.
*
* @return the user role by group
* @throws Exception the exception
*/
//@Test
public void getUserRoleByGroup() throws Exception{
// @Test
public void getUserRoleByGroup() throws Exception {
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String username = testUser;
long init = System.currentTimeMillis();
@ -118,30 +115,29 @@ public class TestDataCatalogueLib {
long end = System.currentTimeMillis();
LOG.debug("Time taken " + (end - init));
}
/**
* Gets the user role by group.
*
* @return the user role by group
* @throws Exception the exception
*/
//@Test
public void getDataset() throws Exception{
// @Test
public void getDataset() throws Exception {
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String username = testUser;
CkanDataset dataset = instance.getDataset("sarda-sarda", username);
LOG.debug("Got dataset: " +dataset.getName() + ", with id: "+dataset.getId());
LOG.debug("Got dataset: " + dataset.getName() + ", with id: " + dataset.getId());
}
/**
* Gets the user role by organization.
*
* @return the user role by organization
* @throws Exception the exception
*/
//@Test
public void getUserRoleByOrganization() throws Exception{
// @Test
public void getUserRoleByOrganization() throws Exception {
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String username = testUser;
long init = System.currentTimeMillis();
@ -149,169 +145,155 @@ public class TestDataCatalogueLib {
long end = System.currentTimeMillis();
LOG.debug("Time taken " + (end - init));
}
//@Test
public void getRoleOfUserInOrganization() throws Exception{
// @Test
public void getRoleOfUserInOrganization() throws Exception {
String orgName = "devvre";
DataCatalogueImpl utils = factory.getUtilsPerScope(scope);
CkanOrganization org = utils.getOrganizationByIdOrName(orgName);
String role = utils.getRoleOfUserInOrganization(testUser, org.getName());
LOG.debug("The user "+testUser+" in the org "+org.getName() + " has the role "+role);
LOG.debug("The user " + testUser + " in the org " + org.getName() + " has the role " + role);
}
//@Test
public void getOrganizationForName() throws Exception{
// @Test
public void getOrganizationForName() throws Exception {
String orgName = "devvre";
DataCatalogueImpl utils = factory.getUtilsPerScope(scope);
CkanOrganization org = utils.getOrganizationByIdOrName(orgName);
LOG.debug("The "+CkanOrganization.class.getSimpleName()+" is: "+org.getName());
LOG.debug("LandingPages of "+CkanOrganization.class.getSimpleName()+" for name " + utils.getLandingPages());
if(org.getUsers()!=null) {
LOG.debug("The " + CkanOrganization.class.getSimpleName() + " is: " + org.getName());
LOG.debug("LandingPages of " + CkanOrganization.class.getSimpleName() + " for name " + utils.getLandingPages());
if (org.getUsers() != null) {
for (CkanUser user : org.getUsers()) {
LOG.debug("User: "+user.getName());
LOG.debug("User: " + user.getName());
}
}
}
//@Test
public void getOrganizationsNamesByUser() throws Exception{
// @Test
public void getOrganizationsNamesByUser() throws Exception {
DataCatalogueImpl utils = factory.getUtilsPerScope(scope);
List<CkanOrganization> listOrgs = utils.getOrganizationsByUser(testUser);
LOG.debug("User :"+testUser+" found in the Organization/s:");
LOG.debug("User :" + testUser + " found in the Organization/s:");
for (CkanOrganization ckanOrganization : listOrgs) {
LOG.debug("Org: "+ckanOrganization.getName());
LOG.debug("Org: " + ckanOrganization.getName());
}
}
//@Test
public void getGroupsNamesByUser() throws Exception{
DataCatalogueImpl utils = factory.getUtilsPerScope(scope);
List<CkanGroup> listGroups = utils.getGroupsByUser(testUser);
LOG.debug("User :"+testUser+" found in the Group/s:");
for (CkanGroup ckanGroup : listGroups) {
LOG.debug("Group: "+ckanGroup.getName());
}
}
//@Test
public void getLandingPages() throws Exception{
// @Test
public void getGroupsNamesByUser() throws Exception {
DataCatalogueImpl utils = factory.getUtilsPerScope(scope);
List<CkanGroup> listGroups = utils.getGroupsByUser(testUser);
LOG.debug("User :" + testUser + " found in the Group/s:");
for (CkanGroup ckanGroup : listGroups) {
LOG.debug("Group: " + ckanGroup.getName());
}
}
// @Test
public void getLandingPages() throws Exception {
DataCatalogueImpl utils = factory.getUtilsPerScope(scope);
LOG.debug("Landing pages " + utils.getLandingPages());
}
//@Test
public void countDatasetForOganization() throws Exception{
DataCatalogueImpl utils = new DataCatalogueImpl(scope,true);
// @Test
public void countDatasetForOganization() throws Exception {
DataCatalogueImpl utils = new DataCatalogueImpl(scope, true);
List<String> listOrgIds = utils.getOrganizationsIds();
System.out.println("'ORGANIZATION NAME': 'NUMBER OF PACKAGE'");
for (String orgID : listOrgIds) {
CkanOrganization org = utils.getOrganizationByIdOrName(orgID);
int packageCount = org.getPackageCount();
//logger.info("ORG: "+org.getName() + " has "+packageCount+ " package/s");
System.out.println(org.getName()+": "+packageCount);
// logger.info("ORG: "+org.getName() + " has "+packageCount+ " package/s");
System.out.println(org.getName() + ": " + packageCount);
}
}
//@Test
public void createDataset() throws Exception{
// @Test
public void createDataset() throws Exception {
try {
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(authorizationToken);
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String licenseId = instance.getLicenses().get(0).getId();
Map<String, List<String>> customFieldsMultiple = new HashMap<String, List<String>>();
for (int i = 0; i < 10; i++) {
List<String> values = new ArrayList<String>();
for (int j = 0; j < new Random().nextInt(3); j++) {
values.add("value-random-"+ new Random().nextInt(10));
values.add("value-random-" + new Random().nextInt(10));
}
customFieldsMultiple.put("key-random-"+new Random().nextInt(10), values);
customFieldsMultiple.put("key-random-" + new Random().nextInt(10), values);
}
customFieldsMultiple.put("empty-key", Arrays.asList(""));
customFieldsMultiple.put("system:type", Arrays.asList("EmptyProfile"));
boolean setSearchable = true;
boolean setPublic = true;
List<ResourceBean> resources = null;
String orgName = scope.split("/")[3].toLowerCase();
//System.out.println("Org name by VRE: "+orgName);
LOG.debug("Org name by VRE: "+orgName);
// System.out.println("Org name by VRE: "+orgName);
LOG.debug("Org name by VRE: " + orgName);
int random = new Random().nextInt();
String datasetTitle = "a dataset created by catalogue-util-library "+random;
String datasetTitle = "a dataset created by catalogue-util-library " + random;
String datasetName = datasetTitle.replace(" ", "");
String createdDataset = instance.createCkanDatasetMultipleCustomFields(
testUser,
"a dataset created by catalogue-util-library "+random,
datasetName,
orgName,
testUser,
null,
null,
null,
1,
"the description",
licenseId,
Arrays.asList("tag 1","tag 2","tag 3"),
customFieldsMultiple,
resources,
setPublic,
setSearchable,
true);
LOG.info("Created the dataset with id: " +createdDataset);
}catch (Exception e) {
String createdDataset = instance.createCkanDatasetMultipleCustomFields(testUser,
"a dataset created by catalogue-util-library " + random, datasetName, orgName, testUser, null, null,
null, 1, "the description", licenseId, Arrays.asList("tag 1", "tag 2", "tag 3"),
customFieldsMultiple, resources, setPublic, setSearchable, true);
LOG.info("Created the dataset with id: " + createdDataset);
} catch (Exception e) {
e.printStackTrace();
}
}
//@Test
public void createGroup() throws Exception{
// @Test
public void createGroup() throws Exception {
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(authorizationToken);
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
int random = new Random().nextInt();
String groupTitle = "a grop created by catalogue-util-library "+random;
String groupTitle = "a grop created by catalogue-util-library " + random;
String groupName = groupTitle.replace(" ", "");
CkanGroup ckanGroup = instance.createGroup(groupName, groupTitle, "description");
LOG.info("Created the group: "+ckanGroup);
LOG.info("Created the group: " + ckanGroup);
}
//@Test
// @Test
public void addResource() throws Exception {
ScopeProvider.instance.set(scope);
@ -323,11 +305,11 @@ public class TestDataCatalogueLib {
for (int i = 1; i < 3; i++) {
ResourceBean resourceBean = new ResourceBean("https://data-dev.d4science.net/ie8Y", "resouce " + i,
"description " + i, null, testUser, dataset.getId(), null);
instance.addResourceToDataset(resourceBean,testUser,dataset.getOrganization().getName());
instance.addResourceToDataset(resourceBean, testUser, dataset.getOrganization().getName());
}
}
//@Test
// @Test
public void deleteResource() throws Exception {
ScopeProvider.instance.set(scope);
@ -337,43 +319,81 @@ public class TestDataCatalogueLib {
String resourceId = "ce7295cf-47db-4faf-901f-4fec2d3fae7a";
boolean deleted = instance.deleteResourceFromDataset(resourceId, testUser);
LOG.info("Deleted the resource? "+deleted);
LOG.info("Deleted the resource? " + deleted);
}
//@Test
public void patchFieldsForDataset() throws Exception{
// @Test
public void patchFieldsForDataset() throws Exception {
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(authorizationToken);
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
Map<String, String> customFieldsToChange = new HashMap<String, String>();
customFieldsToChange.put("key-random-0", "patched");
instance.patchFieldsForDataset(productName, customFieldsToChange);
}
//@Test
public void getUrlProduct() throws Exception{
// @Test
public void getUrlProduct() throws Exception {
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(authorizationToken);
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String datasetName = productName;
String url = instance.getUnencryptedUrlFromDatasetIdOrName(datasetName);
LOG.debug("url is " + url);
}
//@Test
public void marshallingDataset() throws Exception{
// @Test
public void marshallingDataset() throws Exception {
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String username = testUser;
CkanDataset dataset = instance.getDataset("sarda-sarda", username);
LOG.debug("Got dataset: " +dataset.getName() + ", with id: "+dataset.getId() +" is private: "+dataset.isPriv());
String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset,METHOD.TO_CREATE);
LOG.debug("Got dataset: " + dataset.getName() + ", with id: " + dataset.getId() + " is private: "
+ dataset.isPriv());
String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset, METHOD.TO_CREATE);
System.out.println(jsonValueDataset);
dataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset, METHOD.TO_READ);
System.out.println(dataset);
}
//@Test
public void performQuery() throws Exception {
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
List<String> emails2 = new ArrayList<String>(2);
//emails2.add("luca.frosini@isti.cnr.it");
//emails2.add("francesco.mangiacrapa@cnr.it");
List<String> emails = new ArrayList<String>(2);
StringBuilder queryMails = new StringBuilder();
queryMails.append("(");
for (String email : emails2) {
emails.add(email);
queryMails.append(" '" + email + "'");
}
queryMails.append(")");
// queryParams.put(Moderated.SYSTEM_CM_ITEM_STATUS, status.getValue());
String system_cm_prefix = GCatCaller.MODERATOR_ITEM_STATUS_FIELD_NAME;
StringBuilder queryValuesBuilder = new StringBuilder();
queryValuesBuilder.append("author_email:" + queryMails.toString());
queryValuesBuilder.append(" AND ");
queryValuesBuilder.append("extras_" + system_cm_prefix + ":*");
Map<String, String> query = new HashMap<String, String>(1);
query.put("q", queryValuesBuilder.toString());
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(authorizationToken);
String listItems = instance.performQuery(query);
System.out.println("item returned: " + listItems);
}
}