diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java index 7c88209..4a325fe 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java @@ -7,6 +7,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -16,7 +17,6 @@ import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.NotAllowedException; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response.Status; import org.apache.http.MethodNotSupportedException; import org.gcube.com.fasterxml.jackson.databind.JsonNode; @@ -73,7 +73,7 @@ public class CKANPackage extends CKAN implements Moderated { protected static final String ORGANIZATION_FILTER_TEMPLATE = GCatConstants.ORGANIZATION_PARAMETER + ":%s"; - protected static final String ORGANIZATION_REGEX = GCatConstants.ORGANIZATION_PARAMETER + ":[a-zA-Z0-9_\"]*"; + protected static final String ORGANIZATION_REGEX = GCatConstants.ORGANIZATION_PARAMETER + ":[a-zA-Z0-9_\\-\"]*"; protected static final Pattern ORGANIZATION_REGEX_PATTERN; @@ -331,7 +331,30 @@ public class CKANPackage extends CKAN implements Moderated { } } - protected Map getListingParameters(int limit, int offset) { + protected Map addFieldsFilters(Map parameters, String... requiredFields){ + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("["); + stringBuffer.append("'"); + stringBuffer.append(ID_KEY); + stringBuffer.append("'"); + stringBuffer.append(","); + stringBuffer.append("'"); + stringBuffer.append(NAME_KEY); + stringBuffer.append("'"); + for(String requiredField : requiredFields) { + if(requiredField!=null && requiredField.compareTo("")!=0) { + stringBuffer.append(","); + stringBuffer.append("'"); + stringBuffer.append(requiredField); + stringBuffer.append("'"); + } + } + stringBuffer.append("]"); + parameters.put("fl", stringBuffer.toString()); + return parameters; + } + + protected Map getListingParameters(int limit, int offset, String... requiredFields) { Map parameters = new HashMap<>(); if(limit <= 0) { // According to CKAN documentation @@ -349,6 +372,8 @@ public class CKANPackage extends CKAN implements Moderated { MultivaluedMap queryParameters = uriInfo.getQueryParameters(); parameters = checkListParameters(queryParameters, parameters); + parameters = addFieldsFilters(parameters, requiredFields); + parameters = addModerationStatusFilter(parameters); return parameters; @@ -370,19 +395,21 @@ public class CKANPackage extends CKAN implements Moderated { queryParameters.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, Boolean.TRUE.toString()); } - Map parameters = getListingParameters(0,0); + int limit = 25; + int offset = 0; + Map parameters = getListingParameters(limit,offset, "resources"); ObjectNode objectNode = mapper.createObjectNode(); ArrayNode deleted = mapper.createArrayNode(); - objectNode.set("deleted", deleted); - ArrayNode notDeleted = mapper.createArrayNode(); - objectNode.set("failed", notDeleted); sendGetRequest(LIST, parameters); ArrayNode results = (ArrayNode) result.get(RESULTS_KEY); + Set notDeletedSet = new HashSet<>(); + while(results.size()>0) { + int alreadyTriedAndNotDeletedAgain = 0; for(JsonNode node : results) { try { this.name = null; @@ -391,10 +418,23 @@ public class CKANPackage extends CKAN implements Moderated { this.itemID = node.get(ID_KEY).asText(); delete(purge); deleted.add(name); + if(notDeletedSet.contains(name)) { + notDeletedSet.remove(name); + } + try { + Thread.sleep(TimeUnit.MILLISECONDS.toMillis(300)); + } catch (InterruptedException e) { + + } } catch(Exception e) { try { if(name!=null) { - notDeleted.add(name); + if(notDeletedSet.contains(name)) { + alreadyTriedAndNotDeletedAgain++; + }else { + notDeleted.add(name); + notDeletedSet.add(name); + } logger.error("Error while trying to delete item with name {}", name); }else { logger.error("Unable to get the name of {}.",mapper.writeValueAsString(node)); @@ -405,10 +445,35 @@ public class CKANPackage extends CKAN implements Moderated { } } + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException e) { + + } + if(purge) { + setApiKey(CKANUtility.getApiKey()); + } + + if(limit==alreadyTriedAndNotDeletedAgain) { + offset++; + parameters = getListingParameters(limit,offset, "resources"); + } + sendGetRequest(LIST, parameters); results = (ArrayNode) result.get(RESULTS_KEY); + } + if(notDeleted.size()!=notDeletedSet.size()) { + notDeleted = mapper.createArrayNode(); + for(String name : notDeletedSet) { + notDeleted.add(name); + } + } + + objectNode.set("deleted", deleted); + objectNode.set("failed", notDeleted); + return getAsString(objectNode); } @@ -873,31 +938,34 @@ public class CKANPackage extends CKAN implements Moderated { @Override public void purge() { try { - delete(); - } catch(WebApplicationException e) { - // If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged - // with SysAdmin right. - Status status = Status.fromStatusCode(e.getResponse().getStatusInfo().getStatusCode()); - if(status != Status.NOT_FOUND) { - throw e; - } - } - - try { - setApiKey(CKANUtility.getSysAdminAPI()); - readItem(); - if(ckanUser.getRole()!=Role.ADMIN && !isItemCreator()) { - throw new ForbiddenException("Only " + Role.ADMIN.getPortalRole() + "s and item creator are entitled to purge an the item"); + throw new ForbiddenException("Only " + Role.ADMIN.getPortalRole() + "s and item creator are entitled to purge an item"); } + checkModerationDelete(); + + setApiKey(CKANUtility.getSysAdminAPI()); + readItem(); + +// try { +// readItem(); +// } catch(WebApplicationException e) { +// // If the item was deleted but not purged we obtain Not Found. This is accepted. The item has to be purged +// // with SysAdmin right. +// Status status = Status.fromStatusCode(e.getResponse().getStatusInfo().getStatusCode()); +// if(status != Status.NOT_FOUND) { +// throw e; +// } +// +// } + if(result.has(RESOURCES_KEY)) { itemID = result.get(ID_KEY).asText(); ArrayNode arrayNode = (ArrayNode) result.get(RESOURCES_KEY); for(JsonNode jsonNode : arrayNode) { CKANResource ckanResource = new CKANResource(itemID); ckanResource.setPreviousRepresentation(jsonNode); - ckanResource.deleteFile(); // Only delete file is required because the item has been deleted + ckanResource.deleteFile(); // Only delete file is required because the item will be purged at the end } } super.purge(); diff --git a/src/main/java/org/gcube/gcat/rest/Item.java b/src/main/java/org/gcube/gcat/rest/Item.java index bdcc0b7..1f52fa6 100644 --- a/src/main/java/org/gcube/gcat/rest/Item.java +++ b/src/main/java/org/gcube/gcat/rest/Item.java @@ -1,7 +1,5 @@ package org.gcube.gcat.rest; -import java.util.Map; - import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -22,6 +20,8 @@ import org.gcube.gcat.annotation.PATCH; import org.gcube.gcat.annotation.PURGE; import org.gcube.gcat.api.GCatConstants; import org.gcube.gcat.persistence.ckan.CKANPackage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Luca Frosini (ISTI - CNR) @@ -29,6 +29,8 @@ import org.gcube.gcat.persistence.ckan.CKANPackage; @Path(Item.ITEMS) public class Item extends REST implements org.gcube.gcat.api.interfaces.Item { + private final Logger logger = LoggerFactory.getLogger(Item.class); + public static final String ITEM_ID_PARAMETER = "ITEM_ID"; protected String moderationMessage; @@ -37,53 +39,20 @@ public class Item extends REST implements org.gcube.gcat.api.interf super(ITEMS, ITEM_ID_PARAMETER, CKANPackage.class); } - /* - * Not used as REST method, implemented to respect {@link org.gcube.gcat.api.interfaces.Item} interface - */ - @Override - public int count() throws WebServiceException { - CKANPackage ckan = getInstance(); - return ckan.count(); - } - - /* - * Not used as REST method, implemented to respect {@link org.gcube.gcat.api.interfaces.Item} interface - */ - @Override - public String list(Map parameters) throws WebServiceException { - CKANPackage ckan = getInstance(); - return ckan.list(parameters); - } - @GET @Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8) public String list(@QueryParam(GCatConstants.LIMIT_PARAMETER) @DefaultValue("10") int limit, @QueryParam(GCatConstants.OFFSET_PARAMETER) @DefaultValue("0") int offset, @QueryParam(GCatConstants.COUNT_PARAMETER) @DefaultValue("false") Boolean countOnly) { if(countOnly) { - int count = count(); + CKANPackage ckan = getInstance(); + int count = ckan.count(); return createCountJson(count); }else { return list(limit, offset); } } - - @DELETE - @Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8) - public String deleteAll(@QueryParam(GCatConstants.PURGE_QUERY_PARAMETER) @DefaultValue("false") Boolean purge) { - CKANPackage ckan = getInstance(); - return ckan.deleteAll(purge); - } - - @PURGE - @Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8) - public String purgeAll() { - CKANPackage ckan = getInstance(); - return ckan.deleteAll(true); - - } - /* * Not used as REST method, implemented to respect {@link org.gcube.gcat.api.interfaces.Item} interface */ @@ -147,6 +116,33 @@ public class Item extends REST implements org.gcube.gcat.api.interf return delete(name, new Boolean(purge)); } + protected void deleteAll(boolean purge) { + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + CKANPackage ckan = getInstance(); + String ret = ckan.deleteAll(purge); + logger.info("Result of delete all is {}", ret); + } + }); + thread.start(); + } + + @DELETE + @Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8) + @Override + public Response bulkDelete(@QueryParam(GCatConstants.PURGE_QUERY_PARAMETER) @DefaultValue("false") boolean purge) { + deleteAll(purge); + return Response.status(Status.ACCEPTED).build(); + } + + @PURGE + @Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8) + @Override + public Response bulkPurge() { + return bulkDelete(true); + } + @POST @Path("/{" + ITEM_ID_PARAMETER + "}") @Consumes(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8) @@ -162,5 +158,5 @@ public class Item extends REST implements org.gcube.gcat.api.interf } return responseBuilder.build(); } - + } diff --git a/src/main/java/org/gcube/gcat/utils/HTTPCall.java b/src/main/java/org/gcube/gcat/utils/HTTPCall.java index dd9a06c..9be434c 100644 --- a/src/main/java/org/gcube/gcat/utils/HTTPCall.java +++ b/src/main/java/org/gcube/gcat/utils/HTTPCall.java @@ -3,6 +3,7 @@ package org.gcube.gcat.utils; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.util.concurrent.TimeUnit; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.WebApplicationException; @@ -53,6 +54,7 @@ public class HTTPCall { HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setInstanceFollowRedirects(false); connection.setRequestProperty(USER_AGENT_KEY, USER_AGENT_NAME); + connection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(1)); // connection.setRequestMethod(HEAD.class.getSimpleName()); int responseCode = connection.getResponseCode(); diff --git a/src/test/java/org/gcube/gcat/ContextTest.java b/src/test/java/org/gcube/gcat/ContextTest.java index d68683a..12d35f7 100644 --- a/src/test/java/org/gcube/gcat/ContextTest.java +++ b/src/test/java/org/gcube/gcat/ContextTest.java @@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory; /** * @author Luca Frosini (ISTI - CNR) - * */ public class ContextTest { @@ -26,11 +25,9 @@ public class ContextTest { protected static Properties properties; protected static final String PROPERTIES_FILENAME = "token.properties"; - public static final String PARENT_DEFAULT_TEST_SCOPE; - public static final String DEFAULT_TEST_SCOPE; - public static final String ALTERNATIVE_TEST_SCOPE; - - public static final String DEV_VRE; + public static final String ROOT; + public static final String VO; + public static final String VRE; static { properties = new Properties(); @@ -46,11 +43,10 @@ public class ContextTest { //DEFAULT_TEST_SCOPE_NAME = "/pred4s/preprod/preVRE"; // DEFAULT_TEST_SCOPE_NAME = "/gcube/devsec/devVRE"; - PARENT_DEFAULT_TEST_SCOPE = "/gcube"; - DEFAULT_TEST_SCOPE = PARENT_DEFAULT_TEST_SCOPE + "/devsec"; - ALTERNATIVE_TEST_SCOPE = DEFAULT_TEST_SCOPE + "/devVRE"; + ROOT = "/gcube"; + VO = ROOT + "/devsec"; + VRE = VO + "/devVRE"; - DEV_VRE = ALTERNATIVE_TEST_SCOPE; } public static void set(Secret secret) throws Exception { @@ -80,7 +76,7 @@ public class ContextTest { @BeforeClass public static void beforeClass() throws Exception { - setContextByName(DEFAULT_TEST_SCOPE); + setContextByName(VRE); } @AfterClass diff --git a/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java b/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java index 591a9d8..3cb8c44 100644 --- a/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java +++ b/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java @@ -66,105 +66,10 @@ public class CKANPackageTest extends ContextTest { ContextTest.setContextByName("/gcube/devsec/devVRE"); CKANPackage ckanPackage = new CKANPackage(); - UriInfo uriInfo = new UriInfo() { - - @Override - public URI resolve(URI uri) { - return null; - } - - @Override - public URI relativize(URI uri) { - return null; - } - - @Override - public UriBuilder getRequestUriBuilder() { - return null; - } - - @Override - public URI getRequestUri() { - return null; - } - - @Override - public MultivaluedMap getQueryParameters(boolean decode) { - return null; - } - - @Override - public MultivaluedMap getQueryParameters() { - MultivaluedMap mvm = new MultivaluedHashMap(); - mvm.add(Moderated.CM_ITEM_STATUS_QUERY_PARAMETER, CMItemStatus.PENDING.getValue()); - return mvm; - } - - @Override - public List getPathSegments(boolean decode) { - return null; - } - - @Override - public List getPathSegments() { - return null; - } - - @Override - public MultivaluedMap getPathParameters(boolean decode) { - return null; - } - - @Override - public MultivaluedMap getPathParameters() { - return null; - } - - @Override - public String getPath(boolean decode) { - return null; - } - - @Override - public String getPath() { - return null; - } - - @Override - public List getMatchedURIs(boolean decode) { - return null; - } - - @Override - public List getMatchedURIs() { - return null; - } - - @Override - public List getMatchedResources() { - return null; - } - - @Override - public UriBuilder getBaseUriBuilder() { - return null; - } - - @Override - public URI getBaseUri() { - return null; - } - - @Override - public UriBuilder getAbsolutePathBuilder() { - return null; - } - - @Override - public URI getAbsolutePath() { - return null; - } - }; + MultivaluedMap mvm = new MultivaluedHashMap(); + mvm.add(Moderated.CM_ITEM_STATUS_QUERY_PARAMETER, CMItemStatus.PENDING.getValue()); + + UriInfo uriInfo = getUriInfo(mvm); ckanPackage.setUriInfo(uriInfo); ObjectMapper mapper = new ObjectMapper(); @@ -174,7 +79,7 @@ public class CKANPackageTest extends ContextTest { logger.debug("List:\n{}", mapper.writeValueAsString(gotList)); } - protected UriInfo getListingUriInfo(final MultivaluedMap queryParameters) { + protected UriInfo getUriInfo(MultivaluedMap queryParameters) { UriInfo uriInfo = new UriInfo() { @Override @@ -326,7 +231,7 @@ public class CKANPackageTest extends ContextTest { Map parameters = null; try { - ckanPackage.setUriInfo(getListingUriInfo(queryParameters)); + ckanPackage.setUriInfo(getUriInfo(queryParameters)); parameters = ckanPackage.getListingParameters(10, 0); }catch (ForbiddenException e) { if(includeFakeOrganization) { @@ -656,14 +561,13 @@ public class CKANPackageTest extends ContextTest { @Test public void generateLinoTokenModeration() throws Exception { UserInfo userInfo = new UserInfo("leonardo.candela", new ArrayList<>()); - String userToken = authorizationService().generateUserToken(userInfo, ALTERNATIVE_TEST_SCOPE); + String userToken = authorizationService().generateUserToken(userInfo, VRE); logger.debug(userToken); } @Test public void testModeration() throws Exception { - ContextTest.setContextByName(DEV_VRE); CKANPackage ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); try { @@ -677,13 +581,13 @@ public class CKANPackageTest extends ContextTest { createPackage(mapper); - ContextTest.setContextByName("leonardo.candela_"+DEV_VRE); + ContextTest.setContextByName("leonardo.candela_"+VRE); ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); ckanPackage.message("Please add the notes."); - ContextTest.setContextByName(DEV_VRE); + ContextTest.setContextByName(VRE); ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); String readItem = ckanPackage.read(); @@ -694,13 +598,13 @@ public class CKANPackageTest extends ContextTest { ckanPackage.message("I hope now it can be approved."); - ContextTest.setContextByName("leonardo.candela_"+DEV_VRE); + ContextTest.setContextByName("leonardo.candela_"+VRE); ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); ckanPackage.reject("You must specify the institution."); - ContextTest.setContextByName(DEV_VRE); + ContextTest.setContextByName(VRE); ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); readItem = ckanPackage.read(); @@ -710,16 +614,38 @@ public class CKANPackageTest extends ContextTest { ckanPackage.update(mapper.writeValueAsString(readItemObjectNode)); - ContextTest.setContextByName("pasquale.pagano_"+DEV_VRE); + ContextTest.setContextByName("pasquale.pagano_"+VRE); ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); ckanPackage.approve("It seems fine now"); - ContextTest.setContextByName(DEV_VRE); + ContextTest.setContextByName(VRE); ckanPackage = new CKANPackage(); ckanPackage.setName(ITEM_NAME_VALUE); ckanPackage.purge(); } + @Test + public void deleteAllTest() { + CKANPackage ckanPackage = new CKANPackage(); + MultivaluedMap mvm = new MultivaluedHashMap(); + mvm.add(GCatConstants.Q_KEY, "organization:eosc-pillar-f2ds"); + mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false"); + UriInfo uriInfo = getUriInfo(mvm); + ckanPackage.setUriInfo(uriInfo); + String res = ckanPackage.deleteAll(true); + logger.debug("{}", res); + } + + @Test + public void listTest() { + CKANPackage ckanPackage = new CKANPackage(); + MultivaluedMap mvm = new MultivaluedHashMap(); + mvm.add(GCatConstants.OWN_ONLY_QUERY_PARAMETER, "false"); + UriInfo uriInfo = getUriInfo(mvm); + ckanPackage.setUriInfo(uriInfo); + String res = ckanPackage.list(10, 0); + logger.debug("{}", res); + } }