diff --git a/CHANGELOG.md b/CHANGELOG.md index e0b2517..c86fe76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Users are created/referenced in the form and not vice-versa [#21479] - Added support for moderation [#21342] - Added support for IAM authz [#21628] -- Added items bulk delete/purge [#21685] -- Using UriResolverManager to get item URL in place of direct HTTP call [22549] +- Added items bulk delete/purge [#22299] +- Using UriResolverManager to get item URL in place of direct HTTP call [#22549] +- Added empty trash API [#13322] ## [v2.0.0] [r5.2.0] - 2021-05-04 diff --git a/pom.xml b/pom.xml index 4b1af99..0359d56 100644 --- a/pom.xml +++ b/pom.xml @@ -112,6 +112,13 @@ org.gcube.core common-smartgears + + + org.postgresql + postgresql + 42.2.19 + + diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java index 816eeb6..266a985 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java @@ -151,7 +151,7 @@ public class CKANPackage extends CKAN implements Moderated { /* * Return the CKAN organization name using the current context name */ - protected static String getOrganizationName(ScopeBean scopeBean) { + public static String getOrganizationName(ScopeBean scopeBean) { String contextName = scopeBean.name(); return contextName.toLowerCase().replace(" ", "_"); } @@ -386,6 +386,12 @@ public class CKANPackage extends CKAN implements Moderated { } + protected void reuseInstance() { + this.name = null; + this.result = null; + this.itemID = null; + } + /** * @param purge indicate if the item * @return the name list of deleted items @@ -418,7 +424,7 @@ public class CKANPackage extends CKAN implements Moderated { int alreadyTriedAndNotDeletedAgain = 0; for(JsonNode node : results) { try { - this.name = null; + this.reuseInstance(); this.result = node; this.name = node.get(NAME_KEY).asText(); this.itemID = node.get(ID_KEY).asText(); @@ -949,27 +955,15 @@ public class CKANPackage extends CKAN implements Moderated { @Override public void purge() { try { + setApiKey(CKANUtility.getSysAdminAPI()); + readItem(); + + checkModerationDelete(); + if(ckanUser.getRole()!=Role.ADMIN && !isItemCreator()) { throw new ForbiddenException("Only " + Role.ADMIN.getPortalRole() + "s and item creator are entitled to purge an item"); } - checkModerationDelete(); - - setApiKey(CKANUtility.getSysAdminAPI()); - readItem(); - -// try { -// readItem(); -// } catch(WebApplicationException e) { -// // If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged -// // with SysAdmin right. -// Status status = Status.fromStatusCode(e.getResponse().getStatusInfo().getStatusCode()); -// if(status != Status.NOT_FOUND) { -// throw e; -// } -// -// } - if(result.has(RESOURCES_KEY)) { itemID = result.get(ID_KEY).asText(); ArrayNode arrayNode = (ArrayNode) result.get(RESOURCES_KEY); @@ -1109,6 +1103,10 @@ public class CKANPackage extends CKAN implements Moderated { } parameters.put(INCLUDE_PRIVATE_KEY, String.valueOf(true)); + }else{ + if(ckanUser.getRole()==Role.ADMIN) { + parameters.put(INCLUDE_PRIVATE_KEY, String.valueOf(true)); + } } return parameters; @@ -1469,5 +1467,7 @@ public class CKANPackage extends CKAN implements Moderated { throw new InternalServerErrorException(e); } } + + } diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackageTrash.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackageTrash.java new file mode 100644 index 0000000..244f7ca --- /dev/null +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackageTrash.java @@ -0,0 +1,318 @@ +package org.gcube.gcat.persistence.ckan; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.ws.rs.InternalServerErrorException; +import javax.ws.rs.WebApplicationException; + +import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; +import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; +import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; +import org.gcube.common.encryption.encrypter.StringEncrypter; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.Property; +import org.gcube.gcat.api.roles.Role; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.postgresql.core.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CKANPackageTrash { + + protected static final Logger logger = LoggerFactory.getLogger(CKANPackageTrash.class); + + // CKAN Instance info + private final static String SERVICE_ENDPOINT_CATEGORY= "Database"; + private final static String SERVICE_ENDPOINT_NAME = "CKanDatabase"; + + // property to retrieve the master service endpoint into the /root scope + private final static String IS_MASTER_ROOT_KEY_PROPERTY = "IS_ROOT_MASTER"; // true, false.. missing means false as well + + + private static final String GROUP_TABLE_KEY = "group"; + private static final String GROUP_ID_KEY = "id"; + private static final String GROUP_NAME_KEY = "name"; + + private static final String PACKAGE_TABLE_KEY = "package"; + + private static final String PACKAGE_NAME_KEY = "name"; + + private static final String PACKAGE_TYPE_KEY = "type"; + private static final String PACKAGE_TYPE_VALUE = "dataset"; + + private static final String PACKAGE_STATE_KEY = "state"; + private static final String PACKAGE_STATE_VALUE = "deleted"; + + private static final String PACKAGE_OWNER_ORG_KEY = "owner_org"; + + protected ObjectMapper mapper; + + protected final CKANUser ckanUser; + protected final CKANInstance ckanInstance; + protected final Set supportedOrganizations; + + protected boolean ownOnly; + + private String url; + private String username; + private String password; + + public CKANPackageTrash() { + mapper = new ObjectMapper(); + + ckanUser = CKANUserCache.getCurrrentCKANUser(); + ckanInstance = CKANInstance.getInstance(); + + supportedOrganizations = ckanInstance.getSupportedOrganizations(); + for(String supportedOrganization : supportedOrganizations) { + ckanUser.addUserToOrganization(supportedOrganization); + } + + getConfigurationFromIS(); + + ownOnly = true; + } + + public void setOwnOnly(boolean ownOnly) { + this.ownOnly = ownOnly; + } + + /** + * Retrieve endpoints information from IS for DataCatalogue URL + * @return list of endpoints for ckan data catalogue + * @throws Exception + */ + protected List getServiceEndpoints() { + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'"); + query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List serviceEndpoints = client.submit(query); + if(serviceEndpoints.size() == 0) { + logger.error("There is no {} having category {} and name {} in this context.", + ServiceEndpoint.class.getSimpleName(), SERVICE_ENDPOINT_CATEGORY, SERVICE_ENDPOINT_NAME); + throw new InternalServerErrorException("No CKAN configuration on IS"); + } + return serviceEndpoints; + } + + protected void getConfigurationFromIS() { + try { + List serviceEndpoints = getServiceEndpoints(); + ServiceEndpoint serviceEndpoint = null; + + if(serviceEndpoints.size() > 1) { + logger.info("Too many {} having category {} and name {} in this context. Looking for the one that has the property {}", + ServiceEndpoint.class.getSimpleName(), SERVICE_ENDPOINT_CATEGORY, + SERVICE_ENDPOINT_NAME); + + for(ServiceEndpoint se : serviceEndpoints) { + Iterator accessPointIterator = se.profile().accessPoints().iterator(); + while(accessPointIterator.hasNext()) { + ServiceEndpoint.AccessPoint accessPoint = accessPointIterator.next(); + + // get the is master property + Property entry = accessPoint.propertyMap().get(IS_MASTER_ROOT_KEY_PROPERTY); + String isMaster = entry != null ? entry.value() : null; + + if(isMaster == null || !isMaster.equals("true")) { + continue; + } + + // set this variable + serviceEndpoint = se; + break; + } + } + + // if none of them was master, throw an exception + if(serviceEndpoint == null) { + throw new InternalServerErrorException( + "Too many CKAN configuration on IS and no one with MASTER property"); + } + + } else { + serviceEndpoint = serviceEndpoints.get(0); + } + + Iterator accessPointIterator = serviceEndpoint.profile().accessPoints().iterator(); + while(accessPointIterator.hasNext()) { + AccessPoint accessPoint = accessPointIterator.next(); + + // add this host + String host = accessPoint.address(); + String db = accessPoint.name(); + url = String.format("jdbc:postgresql://%s/%s", host, db); + username = accessPoint.username(); + password = StringEncrypter.getEncrypter().decrypt(accessPoint.password()); + } + + } catch(WebApplicationException e) { + throw e; + } catch(Exception e) { + throw new InternalServerErrorException("Error while getting configuration on IS", e); + } + + } + + protected Connection getConnection() throws Exception { + Class.forName("org.postgresql.Driver"); + Connection connection = DriverManager.getConnection(url, username, password); + logger.trace("Database {} opened successfully", url); + connection.setAutoCommit(false); + return connection; + } + + protected String getQuotedString(String string) throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("'"); + Utils.escapeLiteral(builder, string, false); + builder.append("'"); + return builder.toString(); + } + + protected ArrayNode getItems() throws WebApplicationException { + Connection connection = null; + try { + StringBuffer stringBufferOrg = new StringBuffer(); + stringBufferOrg.append("SELECT "); + stringBufferOrg.append(GROUP_ID_KEY); + stringBufferOrg.append(" FROM \""); + stringBufferOrg.append(GROUP_TABLE_KEY); + stringBufferOrg.append("\" WHERE "); + stringBufferOrg.append(GROUP_NAME_KEY); + stringBufferOrg.append(" IN "); + stringBufferOrg.append("("); + boolean first = true; + for(String organizationName : supportedOrganizations) { + if(first) { + first = false; + }else { + stringBufferOrg.append(","); + } + stringBufferOrg.append(getQuotedString(organizationName)); + } + stringBufferOrg.append(")"); + + + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("SELECT "); + stringBuffer.append(PACKAGE_NAME_KEY); + stringBuffer.append(" FROM "); + stringBuffer.append(PACKAGE_TABLE_KEY); + stringBuffer.append(" WHERE "); + stringBuffer.append(PACKAGE_TYPE_KEY); + stringBuffer.append("="); + stringBuffer.append(getQuotedString(PACKAGE_TYPE_VALUE)); + stringBuffer.append(" AND "); + stringBuffer.append(PACKAGE_STATE_KEY); + stringBuffer.append("="); + stringBuffer.append(getQuotedString(PACKAGE_STATE_VALUE)); + + if(ownOnly || ckanUser.getRole()!=Role.ADMIN) { + // add only own items + stringBuffer.append(" AND "); + stringBuffer.append(CKANPackage.AUTHOR_EMAIL_KEY); + stringBuffer.append("="); + stringBuffer.append(getQuotedString(ckanUser.getEMail())); + } + + stringBuffer.append(" AND "); + stringBuffer.append(PACKAGE_OWNER_ORG_KEY); + stringBuffer.append(" IN ("); + stringBuffer.append(stringBufferOrg); + stringBuffer.append(")"); + + ArrayNode items = mapper.createArrayNode(); + connection = getConnection(); + + Statement statement = connection.createStatement(); + + String sql = stringBuffer.toString(); + logger.trace("Going to request the following query: {}", sql); + ResultSet resultSet = statement.executeQuery(sql); + + while (resultSet.next()) { + String id = resultSet.getString(PACKAGE_NAME_KEY); + items.add(id); + } + + return items; + } catch (WebApplicationException e) { + throw e; + } catch (Exception e) { + throw new WebApplicationException(e); + }finally { + if(connection!=null) { + try { + connection.close(); + } catch (SQLException e) { + + } + } + } + } + + public String list() throws WebApplicationException { + try { + return mapper.writeValueAsString(getItems()); + } catch (WebApplicationException e) { + throw e; + } catch (Exception e) { + throw new WebApplicationException(e); + } + } + + public ObjectNode removeAll() throws WebApplicationException { + ObjectNode objectNode = mapper.createObjectNode(); + ArrayNode deleted = mapper.createArrayNode(); + ArrayNode notDeleted = mapper.createArrayNode(); + + ArrayNode itemNames = getItems(); + CKANPackage ckanPackage = new CKANPackage(); + for(int i=0; i mvm = new MultivaluedHashMap(); - mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false"); + mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "False"); UriInfo uriInfo = getUriInfo(mvm); ckanPackage.setUriInfo(uriInfo); String res = ckanPackage.list(10, 0); logger.debug("{}", res); } + + @Test + public void teet() { + Boolean b = Boolean.parseBoolean("false"); + logger.debug("{}", b); + b = Boolean.parseBoolean("False"); + logger.debug("{}", b); + } + + } diff --git a/src/test/java/org/gcube/gcat/persistence/ckan/CKanPackageTrashTest.java b/src/test/java/org/gcube/gcat/persistence/ckan/CKanPackageTrashTest.java new file mode 100644 index 0000000..a2a6600 --- /dev/null +++ b/src/test/java/org/gcube/gcat/persistence/ckan/CKanPackageTrashTest.java @@ -0,0 +1,178 @@ +package org.gcube.gcat.persistence.ckan; + +import javax.ws.rs.NotFoundException; + +import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; +import org.gcube.gcat.ContextTest; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI-CNR) + */ +public class CKanPackageTrashTest extends ContextTest { + + private static Logger logger = LoggerFactory.getLogger(CKanPackageTrashTest.class); + + protected boolean find(ArrayNode ids, String name) { + boolean found = false; + for(int i=0; i