Implemented and tested bulk delete

This commit is contained in:
Luca Frosini 2021-12-03 17:41:38 +01:00
parent abc354ee1b
commit 58674cec7a
5 changed files with 170 additions and 182 deletions

View File

@ -7,6 +7,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -16,7 +17,6 @@ import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.NotAllowedException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response.Status;
import org.apache.http.MethodNotSupportedException;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
@ -73,7 +73,7 @@ public class CKANPackage extends CKAN implements Moderated {
protected static final String ORGANIZATION_FILTER_TEMPLATE = GCatConstants.ORGANIZATION_PARAMETER + ":%s";
protected static final String ORGANIZATION_REGEX = GCatConstants.ORGANIZATION_PARAMETER + ":[a-zA-Z0-9_\"]*";
protected static final String ORGANIZATION_REGEX = GCatConstants.ORGANIZATION_PARAMETER + ":[a-zA-Z0-9_\\-\"]*";
protected static final Pattern ORGANIZATION_REGEX_PATTERN;
@ -331,7 +331,30 @@ public class CKANPackage extends CKAN implements Moderated {
}
}
protected Map<String,String> getListingParameters(int limit, int offset) {
protected Map<String,String> addFieldsFilters(Map<String,String> parameters, String... requiredFields){
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("[");
stringBuffer.append("'");
stringBuffer.append(ID_KEY);
stringBuffer.append("'");
stringBuffer.append(",");
stringBuffer.append("'");
stringBuffer.append(NAME_KEY);
stringBuffer.append("'");
for(String requiredField : requiredFields) {
if(requiredField!=null && requiredField.compareTo("")!=0) {
stringBuffer.append(",");
stringBuffer.append("'");
stringBuffer.append(requiredField);
stringBuffer.append("'");
}
}
stringBuffer.append("]");
parameters.put("fl", stringBuffer.toString());
return parameters;
}
protected Map<String,String> getListingParameters(int limit, int offset, String... requiredFields) {
Map<String,String> parameters = new HashMap<>();
if(limit <= 0) {
// According to CKAN documentation
@ -349,6 +372,8 @@ public class CKANPackage extends CKAN implements Moderated {
MultivaluedMap<String,String> queryParameters = uriInfo.getQueryParameters();
parameters = checkListParameters(queryParameters, parameters);
parameters = addFieldsFilters(parameters, requiredFields);
parameters = addModerationStatusFilter(parameters);
return parameters;
@ -370,19 +395,21 @@ public class CKANPackage extends CKAN implements Moderated {
queryParameters.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, Boolean.TRUE.toString());
}
Map<String,String> parameters = getListingParameters(0,0);
int limit = 25;
int offset = 0;
Map<String,String> parameters = getListingParameters(limit,offset, "resources");
ObjectNode objectNode = mapper.createObjectNode();
ArrayNode deleted = mapper.createArrayNode();
objectNode.set("deleted", deleted);
ArrayNode notDeleted = mapper.createArrayNode();
objectNode.set("failed", notDeleted);
sendGetRequest(LIST, parameters);
ArrayNode results = (ArrayNode) result.get(RESULTS_KEY);
Set<String> notDeletedSet = new HashSet<>();
while(results.size()>0) {
int alreadyTriedAndNotDeletedAgain = 0;
for(JsonNode node : results) {
try {
this.name = null;
@ -391,10 +418,23 @@ public class CKANPackage extends CKAN implements Moderated {
this.itemID = node.get(ID_KEY).asText();
delete(purge);
deleted.add(name);
if(notDeletedSet.contains(name)) {
notDeletedSet.remove(name);
}
try {
Thread.sleep(TimeUnit.MILLISECONDS.toMillis(300));
} catch (InterruptedException e) {
}
} catch(Exception e) {
try {
if(name!=null) {
notDeleted.add(name);
if(notDeletedSet.contains(name)) {
alreadyTriedAndNotDeletedAgain++;
}else {
notDeleted.add(name);
notDeletedSet.add(name);
}
logger.error("Error while trying to delete item with name {}", name);
}else {
logger.error("Unable to get the name of {}.",mapper.writeValueAsString(node));
@ -405,10 +445,35 @@ public class CKANPackage extends CKAN implements Moderated {
}
}
try {
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
} catch (InterruptedException e) {
}
if(purge) {
setApiKey(CKANUtility.getApiKey());
}
if(limit==alreadyTriedAndNotDeletedAgain) {
offset++;
parameters = getListingParameters(limit,offset, "resources");
}
sendGetRequest(LIST, parameters);
results = (ArrayNode) result.get(RESULTS_KEY);
}
if(notDeleted.size()!=notDeletedSet.size()) {
notDeleted = mapper.createArrayNode();
for(String name : notDeletedSet) {
notDeleted.add(name);
}
}
objectNode.set("deleted", deleted);
objectNode.set("failed", notDeleted);
return getAsString(objectNode);
}
@ -873,31 +938,34 @@ public class CKANPackage extends CKAN implements Moderated {
@Override
public void purge() {
try {
delete();
} catch(WebApplicationException e) {
// If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged
// with SysAdmin right.
Status status = Status.fromStatusCode(e.getResponse().getStatusInfo().getStatusCode());
if(status != Status.NOT_FOUND) {
throw e;
}
}
try {
setApiKey(CKANUtility.getSysAdminAPI());
readItem();
if(ckanUser.getRole()!=Role.ADMIN && !isItemCreator()) {
throw new ForbiddenException("Only " + Role.ADMIN.getPortalRole() + "s and item creator are entitled to purge an the item");
throw new ForbiddenException("Only " + Role.ADMIN.getPortalRole() + "s and item creator are entitled to purge an item");
}
checkModerationDelete();
setApiKey(CKANUtility.getSysAdminAPI());
readItem();
// try {
// readItem();
// } catch(WebApplicationException e) {
// // If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged
// // with SysAdmin right.
// Status status = Status.fromStatusCode(e.getResponse().getStatusInfo().getStatusCode());
// if(status != Status.NOT_FOUND) {
// throw e;
// }
//
// }
if(result.has(RESOURCES_KEY)) {
itemID = result.get(ID_KEY).asText();
ArrayNode arrayNode = (ArrayNode) result.get(RESOURCES_KEY);
for(JsonNode jsonNode : arrayNode) {
CKANResource ckanResource = new CKANResource(itemID);
ckanResource.setPreviousRepresentation(jsonNode);
ckanResource.deleteFile(); // Only delete file is required because the item has been deleted
ckanResource.deleteFile(); // Only delete file is required because the item will be purged at the end
}
}
super.purge();

View File

@ -1,7 +1,5 @@
package org.gcube.gcat.rest;
import java.util.Map;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
@ -22,6 +20,8 @@ import org.gcube.gcat.annotation.PATCH;
import org.gcube.gcat.annotation.PURGE;
import org.gcube.gcat.api.GCatConstants;
import org.gcube.gcat.persistence.ckan.CKANPackage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
@ -29,6 +29,8 @@ import org.gcube.gcat.persistence.ckan.CKANPackage;
@Path(Item.ITEMS)
public class Item extends REST<CKANPackage> implements org.gcube.gcat.api.interfaces.Item<Response,Response> {
private final Logger logger = LoggerFactory.getLogger(Item.class);
public static final String ITEM_ID_PARAMETER = "ITEM_ID";
protected String moderationMessage;
@ -37,53 +39,20 @@ public class Item extends REST<CKANPackage> implements org.gcube.gcat.api.interf
super(ITEMS, ITEM_ID_PARAMETER, CKANPackage.class);
}
/*
* Not used as REST method, implemented to respect {@link org.gcube.gcat.api.interfaces.Item} interface
*/
@Override
public int count() throws WebServiceException {
CKANPackage ckan = getInstance();
return ckan.count();
}
/*
* Not used as REST method, implemented to respect {@link org.gcube.gcat.api.interfaces.Item} interface
*/
@Override
public String list(Map<String,String> parameters) throws WebServiceException {
CKANPackage ckan = getInstance();
return ckan.list(parameters);
}
@GET
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String list(@QueryParam(GCatConstants.LIMIT_PARAMETER) @DefaultValue("10") int limit,
@QueryParam(GCatConstants.OFFSET_PARAMETER) @DefaultValue("0") int offset,
@QueryParam(GCatConstants.COUNT_PARAMETER) @DefaultValue("false") Boolean countOnly) {
if(countOnly) {
int count = count();
CKANPackage ckan = getInstance();
int count = ckan.count();
return createCountJson(count);
}else {
return list(limit, offset);
}
}
@DELETE
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String deleteAll(@QueryParam(GCatConstants.PURGE_QUERY_PARAMETER) @DefaultValue("false") Boolean purge) {
CKANPackage ckan = getInstance();
return ckan.deleteAll(purge);
}
@PURGE
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String purgeAll() {
CKANPackage ckan = getInstance();
return ckan.deleteAll(true);
}
/*
* Not used as REST method, implemented to respect {@link org.gcube.gcat.api.interfaces.Item} interface
*/
@ -147,6 +116,33 @@ public class Item extends REST<CKANPackage> implements org.gcube.gcat.api.interf
return delete(name, new Boolean(purge));
}
protected void deleteAll(boolean purge) {
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
CKANPackage ckan = getInstance();
String ret = ckan.deleteAll(purge);
logger.info("Result of delete all is {}", ret);
}
});
thread.start();
}
@DELETE
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@Override
public Response bulkDelete(@QueryParam(GCatConstants.PURGE_QUERY_PARAMETER) @DefaultValue("false") boolean purge) {
deleteAll(purge);
return Response.status(Status.ACCEPTED).build();
}
@PURGE
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@Override
public Response bulkPurge() {
return bulkDelete(true);
}
@POST
@Path("/{" + ITEM_ID_PARAMETER + "}")
@Consumes(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@ -162,5 +158,5 @@ public class Item extends REST<CKANPackage> implements org.gcube.gcat.api.interf
}
return responseBuilder.build();
}
}

View File

@ -3,6 +3,7 @@ package org.gcube.gcat.utils;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.WebApplicationException;
@ -53,6 +54,7 @@ public class HTTPCall {
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setInstanceFollowRedirects(false);
connection.setRequestProperty(USER_AGENT_KEY, USER_AGENT_NAME);
connection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(1));
// connection.setRequestMethod(HEAD.class.getSimpleName());
int responseCode = connection.getResponseCode();

View File

@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*
*/
public class ContextTest {
@ -26,11 +25,9 @@ public class ContextTest {
protected static Properties properties;
protected static final String PROPERTIES_FILENAME = "token.properties";
public static final String PARENT_DEFAULT_TEST_SCOPE;
public static final String DEFAULT_TEST_SCOPE;
public static final String ALTERNATIVE_TEST_SCOPE;
public static final String DEV_VRE;
public static final String ROOT;
public static final String VO;
public static final String VRE;
static {
properties = new Properties();
@ -46,11 +43,10 @@ public class ContextTest {
//DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE";
// DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE";
PARENT_DEFAULT_TEST_SCOPE = "/gcube";
DEFAULT_TEST_SCOPE = PARENT_DEFAULT_TEST_SCOPE + "/devsec";
ALTERNATIVE_TEST_SCOPE = DEFAULT_TEST_SCOPE + "/devVRE";
ROOT = "/gcube";
VO = ROOT + "/devsec";
VRE = VO + "/devVRE";
DEV_VRE = ALTERNATIVE_TEST_SCOPE;
}
public static void set(Secret secret) throws Exception {
@ -80,7 +76,7 @@ public class ContextTest {
@BeforeClass
public static void beforeClass() throws Exception {
setContextByName(DEFAULT_TEST_SCOPE);
setContextByName(VRE);
}
@AfterClass

View File

@ -66,105 +66,10 @@ public class CKANPackageTest extends ContextTest {
ContextTest.setContextByName("/gcube/devsec/devVRE");
CKANPackage ckanPackage = new CKANPackage();
UriInfo uriInfo = new UriInfo() {
@Override
public URI resolve(URI uri) {
return null;
}
@Override
public URI relativize(URI uri) {
return null;
}
@Override
public UriBuilder getRequestUriBuilder() {
return null;
}
@Override
public URI getRequestUri() {
return null;
}
@Override
public MultivaluedMap<String, String> getQueryParameters(boolean decode) {
return null;
}
@Override
public MultivaluedMap<String, String> getQueryParameters() {
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
mvm.add(Moderated.CM_ITEM_STATUS_QUERY_PARAMETER, CMItemStatus.PENDING.getValue());
return mvm;
}
@Override
public List<PathSegment> getPathSegments(boolean decode) {
return null;
}
@Override
public List<PathSegment> getPathSegments() {
return null;
}
@Override
public MultivaluedMap<String, String> getPathParameters(boolean decode) {
return null;
}
@Override
public MultivaluedMap<String, String> getPathParameters() {
return null;
}
@Override
public String getPath(boolean decode) {
return null;
}
@Override
public String getPath() {
return null;
}
@Override
public List<String> getMatchedURIs(boolean decode) {
return null;
}
@Override
public List<String> getMatchedURIs() {
return null;
}
@Override
public List<Object> getMatchedResources() {
return null;
}
@Override
public UriBuilder getBaseUriBuilder() {
return null;
}
@Override
public URI getBaseUri() {
return null;
}
@Override
public UriBuilder getAbsolutePathBuilder() {
return null;
}
@Override
public URI getAbsolutePath() {
return null;
}
};
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
mvm.add(Moderated.CM_ITEM_STATUS_QUERY_PARAMETER, CMItemStatus.PENDING.getValue());
UriInfo uriInfo = getUriInfo(mvm);
ckanPackage.setUriInfo(uriInfo);
ObjectMapper mapper = new ObjectMapper();
@ -174,7 +79,7 @@ public class CKANPackageTest extends ContextTest {
logger.debug("List:\n{}", mapper.writeValueAsString(gotList));
}
protected UriInfo getListingUriInfo(final MultivaluedMap<String, String> queryParameters) {
protected UriInfo getUriInfo(MultivaluedMap<String, String> queryParameters) {
UriInfo uriInfo = new UriInfo() {
@Override
@ -326,7 +231,7 @@ public class CKANPackageTest extends ContextTest {
Map<String,String> parameters = null;
try {
ckanPackage.setUriInfo(getListingUriInfo(queryParameters));
ckanPackage.setUriInfo(getUriInfo(queryParameters));
parameters = ckanPackage.getListingParameters(10, 0);
}catch (ForbiddenException e) {
if(includeFakeOrganization) {
@ -656,14 +561,13 @@ public class CKANPackageTest extends ContextTest {
@Test
public void generateLinoTokenModeration() throws Exception {
UserInfo userInfo = new UserInfo("leonardo.candela", new ArrayList<>());
String userToken = authorizationService().generateUserToken(userInfo, ALTERNATIVE_TEST_SCOPE);
String userToken = authorizationService().generateUserToken(userInfo, VRE);
logger.debug(userToken);
}
@Test
public void testModeration() throws Exception {
ContextTest.setContextByName(DEV_VRE);
CKANPackage ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
try {
@ -677,13 +581,13 @@ public class CKANPackageTest extends ContextTest {
createPackage(mapper);
ContextTest.setContextByName("leonardo.candela_"+DEV_VRE);
ContextTest.setContextByName("leonardo.candela_"+VRE);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
ckanPackage.message("Please add the notes.");
ContextTest.setContextByName(DEV_VRE);
ContextTest.setContextByName(VRE);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
String readItem = ckanPackage.read();
@ -694,13 +598,13 @@ public class CKANPackageTest extends ContextTest {
ckanPackage.message("I hope now it can be approved.");
ContextTest.setContextByName("leonardo.candela_"+DEV_VRE);
ContextTest.setContextByName("leonardo.candela_"+VRE);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
ckanPackage.reject("You must specify the institution.");
ContextTest.setContextByName(DEV_VRE);
ContextTest.setContextByName(VRE);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
readItem = ckanPackage.read();
@ -710,16 +614,38 @@ public class CKANPackageTest extends ContextTest {
ckanPackage.update(mapper.writeValueAsString(readItemObjectNode));
ContextTest.setContextByName("pasquale.pagano_"+DEV_VRE);
ContextTest.setContextByName("pasquale.pagano_"+VRE);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
ckanPackage.approve("It seems fine now");
ContextTest.setContextByName(DEV_VRE);
ContextTest.setContextByName(VRE);
ckanPackage = new CKANPackage();
ckanPackage.setName(ITEM_NAME_VALUE);
ckanPackage.purge();
}
@Test
public void deleteAllTest() {
CKANPackage ckanPackage = new CKANPackage();
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
mvm.add(GCatConstants.Q_KEY, "organization:eosc-pillar-f2ds");
mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false");
UriInfo uriInfo = getUriInfo(mvm);
ckanPackage.setUriInfo(uriInfo);
String res = ckanPackage.deleteAll(true);
logger.debug("{}", res);
}
@Test
public void listTest() {
CKANPackage ckanPackage = new CKANPackage();
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false");
UriInfo uriInfo = getUriInfo(mvm);
ckanPackage.setUriInfo(uriInfo);
String res = ckanPackage.list(10, 0);
logger.debug("{}", res);
}
}