Added empty thrash API

This commit is contained in:
Luca Frosini 2021-12-20 13:46:50 +01:00
parent a8732e2045
commit 8c3b4b5495
8 changed files with 595 additions and 24 deletions

View File

@ -8,8 +8,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- Users are created/referenced in the form <Surname Name> and not vice-versa [#21479]
- Added support for moderation [#21342]
- Added support for IAM authz [#21628]
- Added items bulk delete/purge [#21685]
- Using UriResolverManager to get item URL in place of direct HTTP call [22549]
- Added items bulk delete/purge [#22299]
- Using UriResolverManager to get item URL in place of direct HTTP call [#22549]
- Added empty trash API [#13322]
## [v2.0.0] [r5.2.0] - 2021-05-04

View File

@ -112,6 +112,13 @@
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.19</version>
</dependency>
<!-- ehCAChe -->
<dependency>

View File

@ -151,7 +151,7 @@ public class CKANPackage extends CKAN implements Moderated {
/*
* Return the CKAN organization name using the current context name
*/
protected static String getOrganizationName(ScopeBean scopeBean) {
public static String getOrganizationName(ScopeBean scopeBean) {
String contextName = scopeBean.name();
return contextName.toLowerCase().replace(" ", "_");
}
@ -386,6 +386,12 @@ public class CKANPackage extends CKAN implements Moderated {
}
protected void reuseInstance() {
this.name = null;
this.result = null;
this.itemID = null;
}
/**
* @param purge indicate if the item
* @return the name list of deleted items
@ -418,7 +424,7 @@ public class CKANPackage extends CKAN implements Moderated {
int alreadyTriedAndNotDeletedAgain = 0;
for(JsonNode node : results) {
try {
this.name = null;
this.reuseInstance();
this.result = node;
this.name = node.get(NAME_KEY).asText();
this.itemID = node.get(ID_KEY).asText();
@ -949,27 +955,15 @@ public class CKANPackage extends CKAN implements Moderated {
@Override
public void purge() {
try {
setApiKey(CKANUtility.getSysAdminAPI());
readItem();
checkModerationDelete();
if(ckanUser.getRole()!=Role.ADMIN && !isItemCreator()) {
throw new ForbiddenException("Only " + Role.ADMIN.getPortalRole() + "s and item creator are entitled to purge an item");
}
checkModerationDelete();
setApiKey(CKANUtility.getSysAdminAPI());
readItem();
// try {
// readItem();
// } catch(WebApplicationException e) {
// // If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged
// // with SysAdmin right.
// Status status = Status.fromStatusCode(e.getResponse().getStatusInfo().getStatusCode());
// if(status != Status.NOT_FOUND) {
// throw e;
// }
//
// }
if(result.has(RESOURCES_KEY)) {
itemID = result.get(ID_KEY).asText();
ArrayNode arrayNode = (ArrayNode) result.get(RESOURCES_KEY);
@ -1109,6 +1103,10 @@ public class CKANPackage extends CKAN implements Moderated {
}
parameters.put(INCLUDE_PRIVATE_KEY, String.valueOf(true));
}else{
if(ckanUser.getRole()==Role.ADMIN) {
parameters.put(INCLUDE_PRIVATE_KEY, String.valueOf(true));
}
}
return parameters;
@ -1469,5 +1467,7 @@ public class CKANPackage extends CKAN implements Moderated {
throw new InternalServerErrorException(e);
}
}
}

View File

@ -0,0 +1,318 @@
package org.gcube.gcat.persistence.ckan;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.WebApplicationException;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.encryption.encrypter.StringEncrypter;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.gcat.api.roles.Role;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.postgresql.core.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CKANPackageTrash {
protected static final Logger logger = LoggerFactory.getLogger(CKANPackageTrash.class);
// CKAN Instance info
private final static String SERVICE_ENDPOINT_CATEGORY= "Database";
private final static String SERVICE_ENDPOINT_NAME = "CKanDatabase";
// property to retrieve the master service endpoint into the /root scope
private final static String IS_MASTER_ROOT_KEY_PROPERTY = "IS_ROOT_MASTER"; // true, false.. missing means false as well
private static final String GROUP_TABLE_KEY = "group";
private static final String GROUP_ID_KEY = "id";
private static final String GROUP_NAME_KEY = "name";
private static final String PACKAGE_TABLE_KEY = "package";
private static final String PACKAGE_NAME_KEY = "name";
private static final String PACKAGE_TYPE_KEY = "type";
private static final String PACKAGE_TYPE_VALUE = "dataset";
private static final String PACKAGE_STATE_KEY = "state";
private static final String PACKAGE_STATE_VALUE = "deleted";
private static final String PACKAGE_OWNER_ORG_KEY = "owner_org";
protected ObjectMapper mapper;
protected final CKANUser ckanUser;
protected final CKANInstance ckanInstance;
protected final Set<String> supportedOrganizations;
protected boolean ownOnly;
private String url;
private String username;
private String password;
public CKANPackageTrash() {
mapper = new ObjectMapper();
ckanUser = CKANUserCache.getCurrrentCKANUser();
ckanInstance = CKANInstance.getInstance();
supportedOrganizations = ckanInstance.getSupportedOrganizations();
for(String supportedOrganization : supportedOrganizations) {
ckanUser.addUserToOrganization(supportedOrganization);
}
getConfigurationFromIS();
ownOnly = true;
}
public void setOwnOnly(boolean ownOnly) {
this.ownOnly = ownOnly;
}
/**
* Retrieve endpoints information from IS for DataCatalogue URL
* @return list of endpoints for ckan data catalogue
* @throws Exception
*/
protected List<ServiceEndpoint> getServiceEndpoints() {
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'");
query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> serviceEndpoints = client.submit(query);
if(serviceEndpoints.size() == 0) {
logger.error("There is no {} having category {} and name {} in this context.",
ServiceEndpoint.class.getSimpleName(), SERVICE_ENDPOINT_CATEGORY, SERVICE_ENDPOINT_NAME);
throw new InternalServerErrorException("No CKAN configuration on IS");
}
return serviceEndpoints;
}
protected void getConfigurationFromIS() {
try {
List<ServiceEndpoint> serviceEndpoints = getServiceEndpoints();
ServiceEndpoint serviceEndpoint = null;
if(serviceEndpoints.size() > 1) {
logger.info("Too many {} having category {} and name {} in this context. Looking for the one that has the property {}",
ServiceEndpoint.class.getSimpleName(), SERVICE_ENDPOINT_CATEGORY,
SERVICE_ENDPOINT_NAME);
for(ServiceEndpoint se : serviceEndpoints) {
Iterator<AccessPoint> accessPointIterator = se.profile().accessPoints().iterator();
while(accessPointIterator.hasNext()) {
ServiceEndpoint.AccessPoint accessPoint = accessPointIterator.next();
// get the is master property
Property entry = accessPoint.propertyMap().get(IS_MASTER_ROOT_KEY_PROPERTY);
String isMaster = entry != null ? entry.value() : null;
if(isMaster == null || !isMaster.equals("true")) {
continue;
}
// set this variable
serviceEndpoint = se;
break;
}
}
// if none of them was master, throw an exception
if(serviceEndpoint == null) {
throw new InternalServerErrorException(
"Too many CKAN configuration on IS and no one with MASTER property");
}
} else {
serviceEndpoint = serviceEndpoints.get(0);
}
Iterator<AccessPoint> accessPointIterator = serviceEndpoint.profile().accessPoints().iterator();
while(accessPointIterator.hasNext()) {
AccessPoint accessPoint = accessPointIterator.next();
// add this host
String host = accessPoint.address();
String db = accessPoint.name();
url = String.format("jdbc:postgresql://%s/%s", host, db);
username = accessPoint.username();
password = StringEncrypter.getEncrypter().decrypt(accessPoint.password());
}
} catch(WebApplicationException e) {
throw e;
} catch(Exception e) {
throw new InternalServerErrorException("Error while getting configuration on IS", e);
}
}
protected Connection getConnection() throws Exception {
Class.forName("org.postgresql.Driver");
Connection connection = DriverManager.getConnection(url, username, password);
logger.trace("Database {} opened successfully", url);
connection.setAutoCommit(false);
return connection;
}
protected String getQuotedString(String string) throws SQLException {
StringBuilder builder = new StringBuilder();
builder.append("'");
Utils.escapeLiteral(builder, string, false);
builder.append("'");
return builder.toString();
}
protected ArrayNode getItems() throws WebApplicationException {
Connection connection = null;
try {
StringBuffer stringBufferOrg = new StringBuffer();
stringBufferOrg.append("SELECT ");
stringBufferOrg.append(GROUP_ID_KEY);
stringBufferOrg.append(" FROM \"");
stringBufferOrg.append(GROUP_TABLE_KEY);
stringBufferOrg.append("\" WHERE ");
stringBufferOrg.append(GROUP_NAME_KEY);
stringBufferOrg.append(" IN ");
stringBufferOrg.append("(");
boolean first = true;
for(String organizationName : supportedOrganizations) {
if(first) {
first = false;
}else {
stringBufferOrg.append(",");
}
stringBufferOrg.append(getQuotedString(organizationName));
}
stringBufferOrg.append(")");
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("SELECT ");
stringBuffer.append(PACKAGE_NAME_KEY);
stringBuffer.append(" FROM ");
stringBuffer.append(PACKAGE_TABLE_KEY);
stringBuffer.append(" WHERE ");
stringBuffer.append(PACKAGE_TYPE_KEY);
stringBuffer.append("=");
stringBuffer.append(getQuotedString(PACKAGE_TYPE_VALUE));
stringBuffer.append(" AND ");
stringBuffer.append(PACKAGE_STATE_KEY);
stringBuffer.append("=");
stringBuffer.append(getQuotedString(PACKAGE_STATE_VALUE));
if(ownOnly || ckanUser.getRole()!=Role.ADMIN) {
// add only own items
stringBuffer.append(" AND ");
stringBuffer.append(CKANPackage.AUTHOR_EMAIL_KEY);
stringBuffer.append("=");
stringBuffer.append(getQuotedString(ckanUser.getEMail()));
}
stringBuffer.append(" AND ");
stringBuffer.append(PACKAGE_OWNER_ORG_KEY);
stringBuffer.append(" IN (");
stringBuffer.append(stringBufferOrg);
stringBuffer.append(")");
ArrayNode items = mapper.createArrayNode();
connection = getConnection();
Statement statement = connection.createStatement();
String sql = stringBuffer.toString();
logger.trace("Going to request the following query: {}", sql);
ResultSet resultSet = statement.executeQuery(sql);
while (resultSet.next()) {
String id = resultSet.getString(PACKAGE_NAME_KEY);
items.add(id);
}
return items;
} catch (WebApplicationException e) {
throw e;
} catch (Exception e) {
throw new WebApplicationException(e);
}finally {
if(connection!=null) {
try {
connection.close();
} catch (SQLException e) {
}
}
}
}
public String list() throws WebApplicationException {
try {
return mapper.writeValueAsString(getItems());
} catch (WebApplicationException e) {
throw e;
} catch (Exception e) {
throw new WebApplicationException(e);
}
}
public ObjectNode removeAll() throws WebApplicationException {
ObjectNode objectNode = mapper.createObjectNode();
ArrayNode deleted = mapper.createArrayNode();
ArrayNode notDeleted = mapper.createArrayNode();
ArrayNode itemNames = getItems();
CKANPackage ckanPackage = new CKANPackage();
for(int i=0; i<itemNames.size(); i++) {
String name = itemNames.get(i).asText();
try {
ckanPackage.reuseInstance();
ckanPackage.setName(name);
ckanPackage.purge();
deleted.add(name);
}catch (Exception e) {
notDeleted.add(name);
}
try {
Thread.sleep(TimeUnit.MILLISECONDS.toMillis(300));
} catch (InterruptedException e) {
}
}
objectNode.set("deleted", deleted);
objectNode.set("failed", notDeleted);
return objectNode;
}
public String empty() throws WebApplicationException {
try {
return mapper.writeValueAsString(removeAll());
} catch (WebApplicationException e) {
throw e;
} catch (Exception e) {
throw new WebApplicationException(e);
}
}
}

View File

@ -0,0 +1,44 @@
package org.gcube.gcat.rest;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.gcube.gcat.ResourceInitializer;
import org.gcube.gcat.api.GCatConstants;
import org.gcube.gcat.persistence.ckan.CKANPackageTrash;
/**
* @author Luca Frosini (ISTI - CNR)
*/
@Path("trash")
public class Trash extends BaseREST {
@GET
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String list() {
CKANPackageTrash ckanPackageTrash = new CKANPackageTrash();
return ckanPackageTrash.list();
}
@DELETE
public Response delete(@QueryParam(GCatConstants.OWN_ONLY_QUERY_PARAMETER) @DefaultValue("true") Boolean own_only) {
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
CKANPackageTrash ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.empty();
}
});
thread.start();
return Response.status(Status.ACCEPTED).build();
}
}

View File

@ -51,7 +51,7 @@ public class CKANPackageTest extends ContextTest {
private static final String URL_KEY = "url";
private static final String PRIVATE_KEY = "private";
private static final String ITEM_NAME_VALUE = "restful_transaction_model";
public static final String ITEM_NAME_VALUE = "restful_transaction_model";
private static final String LICENSE_VALUE = "CC-BY-SA-4.0";
private static final String EXTRAS_TYPE_VALUE_VALUE = "EmptyProfile";
@ -530,9 +530,13 @@ public class CKANPackageTest extends ContextTest {
@Test
//(expected = NotFoundException.class)
public void delete() throws Exception {
delete(true);
}
public void delete(boolean purge) throws Exception {
CKANPackage ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
ckanPackage.delete(true);
ckanPackage.delete(purge);
}
@Test
@ -650,10 +654,20 @@ public class CKANPackageTest extends ContextTest {
public void listTest() {
CKANPackage ckanPackage = new CKANPackage();
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false");
mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "False");
UriInfo uriInfo = getUriInfo(mvm);
ckanPackage.setUriInfo(uriInfo);
String res = ckanPackage.list(10, 0);
logger.debug("{}", res);
}
@Test
public void teet() {
Boolean b = Boolean.parseBoolean("false");
logger.debug("{}", b);
b = Boolean.parseBoolean("False");
logger.debug("{}", b);
}
}

View File

@ -0,0 +1,178 @@
package org.gcube.gcat.persistence.ckan;
import javax.ws.rs.NotFoundException;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.gcat.ContextTest;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI-CNR)
*/
public class CKanPackageTrashTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(CKanPackageTrashTest.class);
protected boolean find(ArrayNode ids, String name) {
boolean found = false;
for(int i=0; i<ids.size(); i++) {
if(name.compareTo(ids.get(i).asText())==0) {
found = true;
break;
}
}
return found;
}
@Test
public void testListAndEmptyTrash() throws Exception {
// Cleanign the env
ContextTest.setContextByName(VRE);
CKANPackageTest ckanPackageTest = new CKANPackageTest();
ckanPackageTest.delete(true);
ckanPackageTest = new CKANPackageTest();
ckanPackageTest.create();
ckanPackageTest.delete(false);
CKANPackageTrash ckanPackageTrash = new CKANPackageTrash();
ArrayNode ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
boolean found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(found);
ckanPackageTest.delete(true);
ids = ckanPackageTrash.getItems();
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
logger.debug("{}", ids);
}
public static final String NON_CATALOGUE_ADMIN_USER = "lucio.lelii";
@Test
public void testListAndEmptyTrashFromAdmin() throws Exception {
// Cleanign the env
ContextTest.setContextByName(VRE);
CKANPackageTest ckanPackageTest = new CKANPackageTest();
try {
ckanPackageTest.delete(true);
}catch (NotFoundException e) {
// It is expected. the env was clean
}
ContextTest.setContextByName(NON_CATALOGUE_ADMIN_USER+"_"+VRE);
ckanPackageTest = new CKANPackageTest();
ckanPackageTest.create();
ckanPackageTest.delete(false);
// I'm admin
ContextTest.setContextByName(VRE);
CKANPackageTrash ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(false);
ArrayNode ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
boolean found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(found);
ckanPackageTrash.setOwnOnly(true);
ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
ContextTest.setContextByName(NON_CATALOGUE_ADMIN_USER+"_"+VRE);
ckanPackageTest.delete(true);
// I'm admin
ContextTest.setContextByName(VRE);
ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(false);
ids = ckanPackageTrash.getItems();
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
logger.debug("{}", ids);
}
@Test
public void testListAndEmptyTrashFromNonAdmin() throws Exception {
// Cleaning the env
ContextTest.setContextByName(VRE);
CKANPackageTest ckanPackageTest = new CKANPackageTest();
try {
ckanPackageTest.delete(true);
}catch (NotFoundException e) {
// It is expected. the env was clean
}
ContextTest.setContextByName(VRE);
ckanPackageTest = new CKANPackageTest();
ckanPackageTest.create();
ckanPackageTest.delete(false);
// He is not admin
ContextTest.setContextByName(NON_CATALOGUE_ADMIN_USER+"_"+VRE);
CKANPackageTrash ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(false);
ArrayNode ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
boolean found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
ckanPackageTrash.setOwnOnly(true);
ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
ContextTest.setContextByName(VRE);
ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(false);
ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(found);
ckanPackageTrash.setOwnOnly(true);
ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(found);
ckanPackageTest.delete(true);
ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(false);
ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
ckanPackageTrash.setOwnOnly(true);
ids = ckanPackageTrash.getItems();
logger.debug("{}", ids);
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
ContextTest.setContextByName(NON_CATALOGUE_ADMIN_USER+"_"+VRE);
ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(false);
ids = ckanPackageTrash.getItems();
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
logger.debug("{}", ids);
ckanPackageTrash = new CKANPackageTrash();
ckanPackageTrash.setOwnOnly(true);
ids = ckanPackageTrash.getItems();
found = find(ids, CKANPackageTest.ITEM_NAME_VALUE);
Assert.assertTrue(!found);
logger.debug("{}", ids);
}
}

View File

@ -1,6 +1,7 @@
package org.gcube.gcat.utils;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.encryption.encrypter.StringEncrypter;
import org.gcube.gcat.ContextTest;
import org.junit.Test;
import org.slf4j.Logger;
@ -16,4 +17,12 @@ public class ConstantsTest extends ContextTest {
Constants.getCatalogueSecurityToken());
}
@Test
public void decrypt() throws Exception {
StringEncrypter stringEncrypter = StringEncrypter.getEncrypter();
String decrypted = stringEncrypter.decrypt("w0KVc+78b2yUQsndDh/cXyyRquuwyILTygmoF0Y5Dls=");
logger.debug("{}", decrypted);
}
}