From e3e8ffd39336e6322f9ac97b9d7629ad23a04f97 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Thu, 12 May 2022 16:25:58 +0200 Subject: [PATCH] Changed GRSF tests --- CHANGELOG.md | 5 ++ pom.xml | 2 +- .../org/gcube/gcat/persistence/ckan/CKAN.java | 2 +- .../gcat/persistence/ckan/CKANPackage.java | 23 ++++- .../gcat/persistence/ckan/CKANResource.java | 2 +- .../gcube/gcat/persistence/ckan/CKANUser.java | 2 +- .../persistence/ckan/CKANPackageTest.java | 2 +- .../{ckan => grsf}/GRSFUtilities.java | 86 +++++++++++++------ src/test/resources/logback-test.xml | 3 +- 9 files changed, 91 insertions(+), 36 deletions(-) rename src/test/java/org/gcube/gcat/persistence/{ckan => grsf}/GRSFUtilities.java (75%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24583d0..e9967e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for gCube Catalogue (gCat) Service +## [v2.3.0-SNAPSHOT] + +- Added moderation link in moderation message [#23142] +- Switched moderation messages to notification [#23317] + ## [v2.2.0] - Switched gcat credentials to new IAM authz [#21628][#22727] diff --git a/pom.xml b/pom.xml index 78ccd48..769aba2 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.gcube.data-catalogue gcat war - 2.2.0 + 2.3.0-SNAPSHOT gCube Catalogue (gCat) Service This service allows any client to publish on the gCube Catalogue. diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java index 8190dc8..493b2af 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java @@ -360,7 +360,7 @@ public abstract class CKAN { } } - protected void purge() { + public void purge() { sendPostRequest(PURGE, createJsonNodeWithNameAsID()); } diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java index 82d6e3d..56fe2d9 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java @@ -504,6 +504,23 @@ public class CKANPackage extends CKAN implements Moderated { return getAsString(arrayNode); } + public String listNoCheck(int limit, int offset) { + Map parameters = new HashMap<>(); + if(limit <= 0) { + // According to CKAN documentation + // the number of matching rows to return. There is a hard limit of 1000 datasets per query. + // see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search + limit = 1000; + } + parameters.put(ROWS_KEY, String.valueOf(limit)); + + if(offset < 0) { + offset = 0; + } + parameters.put(START_KEY, String.valueOf(offset)); + return list(parameters); + } + @Override public String list(int limit, int offset) { Map parameters = getListingParameters(limit, offset); @@ -983,9 +1000,9 @@ public class CKANPackage extends CKAN implements Moderated { /** * Used for bulk purging. Internal use only */ - protected void purgeNoCheckNoDeleteFiles() { -// setApiKey(CKANUtility.getSysAdminAPI()); -// super.purge(); + public void purgeNoCheckNoDeleteFiles() { + setApiKey(CKANUtility.getSysAdminAPI()); + super.purge(); } /* diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java index 64903db..d5743cb 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANResource.java @@ -471,7 +471,7 @@ public class CKANResource extends CKAN { } @Override - protected void purge() { + public void purge() { String[] moreAllowed = new String[] {HEAD.class.getSimpleName(), GET.class.getSimpleName(), PUT.class.getSimpleName(), DELETE.class.getSimpleName()}; throw new NotAllowedException(OPTIONS.class.getSimpleName(), moreAllowed); diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java index d0379a4..b8c444f 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANUser.java @@ -215,7 +215,7 @@ public class CKANUser extends CKAN { return ret; } - protected void addUserToOrganization(String organizationName, String ckanUsername, String role) { + public void addUserToOrganization(String organizationName, String ckanUsername, String role) { logger.trace("Going to add user {} to organization {} with role {}", ckanUsername, organizationName, role); CKANOrganization ckanOrganization = new CKANOrganization(); ckanOrganization.setApiKey(CKANUtility.getSysAdminAPI()); diff --git a/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java b/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java index de150ff..6998e90 100644 --- a/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java +++ b/src/test/java/org/gcube/gcat/persistence/ckan/CKANPackageTest.java @@ -75,7 +75,7 @@ public class CKANPackageTest extends ContextTest { logger.debug("List:\n{}", mapper.writeValueAsString(gotList)); } - protected UriInfo getUriInfo(MultivaluedMap queryParameters) { + public static UriInfo getUriInfo(MultivaluedMap queryParameters) { UriInfo uriInfo = new UriInfo() { @Override diff --git a/src/test/java/org/gcube/gcat/persistence/ckan/GRSFUtilities.java b/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java similarity index 75% rename from src/test/java/org/gcube/gcat/persistence/ckan/GRSFUtilities.java rename to src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java index acf5440..d0ec8b1 100644 --- a/src/test/java/org/gcube/gcat/persistence/ckan/GRSFUtilities.java +++ b/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java @@ -1,4 +1,4 @@ -package org.gcube.gcat.persistence.ckan; +package org.gcube.gcat.persistence.grsf; import java.io.IOException; import java.util.HashMap; @@ -7,14 +7,26 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore; +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.UriInfo; + import org.gcube.com.fasterxml.jackson.core.JsonProcessingException; import org.gcube.com.fasterxml.jackson.databind.JsonNode; import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.utils.manager.SecretManagerProvider; import org.gcube.gcat.ContextTest; +import org.gcube.gcat.persistence.ckan.CKANGroup; +import org.gcube.gcat.persistence.ckan.CKANGroupTest; +import org.gcube.gcat.persistence.ckan.CKANOrganization; +import org.gcube.gcat.persistence.ckan.CKANOrganizationTest; +import org.gcube.gcat.persistence.ckan.CKANPackage; +import org.gcube.gcat.persistence.ckan.CKANPackageTest; +import org.gcube.gcat.persistence.ckan.CKANUser; +import org.gcube.gcat.persistence.ckan.CKANUtility; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -25,7 +37,7 @@ import org.slf4j.LoggerFactory; */ public class GRSFUtilities extends ContextTest { - private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class); + private static Logger logger = LoggerFactory.getLogger(GRSFUtilities.class); public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF"; public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin"; @@ -137,6 +149,8 @@ public class GRSFUtilities extends ContextTest { // @Test public void createNewGRSFGroups() throws ObjectNotFound, Exception { +// ContextTest.setContext(""); + logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); String sysAdminAPI = CKANUtility.getSysAdminAPI(); String[] groupNames = new String[] { "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", @@ -157,28 +171,35 @@ public class GRSFUtilities extends ContextTest { node.put("name", getGroupId(name)); String json = objectMapper.writeValueAsString(node); logger.info(json); - // ckanGroupToCreate.create(json); + ckanGroupToCreate.create(json); } - } // @Test public void deleteNewGRSFGroups() throws ObjectNotFound, Exception { + ContextTest.setContext(""); + logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); + logger.debug(SecretManagerProvider.instance.get().getContext()); + String sysAdminAPI = CKANUtility.getSysAdminAPI(); - String[] groupNames = new String[] { - "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", - "Catch", "Landing", - "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", - "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", - "FAO Stock Status Category", "Scientific Advice", - "GRSF SDG Flag", "GRSF Traceability Flag" - }; +// String[] groupNames = new String[] { +// "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", +// "Catch", "Landing", +// "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", +// "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", +// "FAO Stock Status Category", "Scientific Advice", +// "GRSF SDG Flag", "GRSF Traceability Flag" +// }; +// + List groupNames = CKANGroupTest.listGroup(); + for(String name : groupNames) { CKANGroup ckanGroupToCreate = new CKANGroup(); ckanGroupToCreate.setApiKey(sysAdminAPI); - ckanGroupToCreate.setName(getGroupId(name)); +// ckanGroupToCreate.setName(getGroupId(name)); + ckanGroupToCreate.setName(name); try { - ckanGroupToCreate.purge(); +// ckanGroupToCreate.purge(); }catch (Exception e) { } @@ -186,28 +207,34 @@ public class GRSFUtilities extends ContextTest { } - // @Test + @Test public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception { - String username = "grsf_publisher"; - // username = "luca_frosini"; - // username = "francesco_mangiacrapa"; + ContextTest.setContext(""); + logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); + logger.debug(SecretManagerProvider.instance.get().getContext()); + + String[] usernames = new String[] { "grsf_publisher", "luca_frosini", "francesco_mangiacrapa"}; + String sysAdminAPI = CKANUtility.getSysAdminAPI(); List groupNames = CKANGroupTest.listGroup(); for(String groupName : groupNames) { CKANUser ckanUser = new CKANUser(); ckanUser.setApiKey(sysAdminAPI); - ckanUser.setName(username); - ckanUser.addToGroup(groupName); + for(String username : usernames) { + ckanUser.setName(username); + ckanUser.addToGroup(groupName); + } } List orgs = CKANOrganizationTest.listOrg(); for(String org : orgs) { CKANUser ckanUser = new CKANUser(); ckanUser.setApiKey(sysAdminAPI); - ckanUser.addUserToOrganization(org, username, "admin"); + for(String username : usernames) { + ckanUser.addUserToOrganization(org, username, "admin"); + } } - } // @Test @@ -239,10 +266,13 @@ public class GRSFUtilities extends ContextTest { private void purgeGRSFRecords(int limit, int offset) throws Exception { logger.debug("Going to purge {} records, starting from {}", limit, limit*offset); CKANPackage ckanPackage = new CKANPackage(); + MultivaluedMap mvm = new MultivaluedHashMap(); + UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm); + ckanPackage.setUriInfo(uriInfo); ObjectMapper mapper = new ObjectMapper(); boolean go = true; while(go) { - String ret = ckanPackage.list(limit, offset); + String ret = ckanPackage.listNoCheck(limit, offset); JsonNode gotList = mapper.readTree(ret); Assert.assertTrue(gotList instanceof ArrayNode); ArrayNode itemsArray = (ArrayNode) gotList; @@ -252,14 +282,14 @@ public class GRSFUtilities extends ContextTest { logger.debug("Going to purge record with name {}", name); ckanPackage.setName(name); try { - ckanPackage.purgeNoCheckNoDeleteFiles(); + // ckanPackage.purgeNoCheckNoDeleteFiles(); if(go) { break; } }catch (Exception e) { // TODO: handle exception } - // Thread.sleep(500); + Thread.sleep(500); } }else { go=false; @@ -267,9 +297,11 @@ public class GRSFUtilities extends ContextTest { } } - @JsonIgnore // @Test public void purgeAllGRSFRecords() throws Exception { + ContextTest.setContext(""); + logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); + logger.debug(SecretManagerProvider.instance.get().getContext()); purgeGRSFRecords(100, 0); } diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index 4da43d9..2e2f160 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -10,7 +10,8 @@ - + +