From 0cf6e58dd47c6d04a52ba9bf606b415cdec8fb7c Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Fri, 14 Jul 2023 10:59:33 +0200 Subject: [PATCH] Removed GRSF utilities moved in the dedicated component --- .../gcat/persistence/grsf/GRSFUtilities.java | 424 ------------------ 1 file changed, 424 deletions(-) delete mode 100644 src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java diff --git a/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java b/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java deleted file mode 100644 index aa605f9..0000000 --- a/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java +++ /dev/null @@ -1,424 +0,0 @@ -package org.gcube.gcat.persistence.grsf; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; - -import javax.ws.rs.core.MultivaluedHashMap; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.UriInfo; - -import org.gcube.com.fasterxml.jackson.core.JsonProcessingException; -import org.gcube.com.fasterxml.jackson.databind.JsonNode; -import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; -import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; -import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; -import org.gcube.common.authorization.client.exceptions.ObjectNotFound; -import org.gcube.common.authorization.utils.manager.SecretManagerProvider; -import org.gcube.common.scope.impl.ScopeBean; -import org.gcube.gcat.ContextTest; -import org.gcube.gcat.api.GCatConstants; -import org.gcube.gcat.persistence.ckan.CKANGroup; -import org.gcube.gcat.persistence.ckan.CKANGroupTest; -import org.gcube.gcat.persistence.ckan.CKANOrganization; -import org.gcube.gcat.persistence.ckan.CKANOrganizationTest; -import org.gcube.gcat.persistence.ckan.CKANPackage; -import org.gcube.gcat.persistence.ckan.CKANPackageTest; -import org.gcube.gcat.persistence.ckan.CKANUser; -import org.gcube.gcat.persistence.ckan.CKANUtility; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author Luca Frosini (ISTI - CNR) - */ -public class GRSFUtilities extends ContextTest { - - private static Logger logger = LoggerFactory.getLogger(GRSFUtilities.class); - - public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF"; - public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin"; - public static final String PRODUCTION_GRSF_PRE = "/d4science.research-infrastructures.eu/FARM/GRSF_Pre"; - - public static final String PREPROD_GRSF_PRE = "/pred4s/preprod/GRSF_Pre"; - - @Before - public void before() throws Exception { -// ContextTest.setContextByName(PREPROD_GRSF_PRE); -// ContextTest.setContextByName(PRODUCTION_GRSF); -// ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN); -// ContextTest.setContextByName(PRODUCTION_GRSF_PRE); -// logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); -// logger.debug(SecretManagerProvider.instance.get().getContext()); - } - - private void create(Set createdGroup, Map groups, String name) throws JsonProcessingException, IOException { - if(createdGroup.contains(name)) { - return; - } - String sysAdminAPI = ""; // CKANUtility.getSysAdminAPI(); - CKANGroup ckanGroupToCreate = new CKANGroup(); - ckanGroupToCreate.setApiKey(sysAdminAPI); - ckanGroupToCreate.setName(name); - // ckanGroupToCreate.purge(); - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(groups.get(name)); - ArrayNode array = (ArrayNode) jsonNode.get("groups"); - for(JsonNode node : array) { - String parentName = node.get("name").asText(); - if(!createdGroup.contains(parentName)) { - create(createdGroup, groups, parentName); - } - } - ckanGroupToCreate.create(groups.get(name)); - createdGroup.add(name); - } - - // @Test - public void createGRSFGroups() throws ObjectNotFound, Exception { - String key = CKANUtility.getSysAdminAPI(); - List groupNames = CKANGroupTest.listGroup(); - Map groups = new HashMap<>(); - for(String name : groupNames) { - CKANGroup ckanGroup = new CKANGroup(); - ckanGroup.setApiKey(key); - ckanGroup.setName(name); - String read = ckanGroup.read(); - groups.put(name, read); - } - - Set createdGroup = new HashSet<>(); - - for(String name : groupNames) { - create(createdGroup, groups, name); - } - } - - public static String getGroupNameOnCkan(String origName){ - if(origName == null) { - throw new IllegalArgumentException("origName cannot be null"); - } - String modified = origName.replaceAll("\\(", ""); - modified = modified.replaceAll("\\)", ""); - modified = modified.trim().toLowerCase().replaceAll("[^A-Za-z0-9-]", "-"); - if(modified.startsWith("-")) { - modified = modified.substring(1); - } - if(modified.endsWith("-")) { - modified = modified.substring(0, modified.length() -1); - } - return modified; - } - - public static final String GROUP_SUFFIX = "-group"; - - public static String getGroupId(String name) { - StringBuffer stringBuffer = new StringBuffer(); - stringBuffer.append(getGroupNameOnCkan(name)); - if(!name.endsWith(GROUP_SUFFIX)) { - stringBuffer.append(GROUP_SUFFIX); - } - return stringBuffer.toString(); - } - - @Test - public void testGroupName() { - String[] groupNames = new String[] { - "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", - "Catch", "Landing", "Assessment Method", - "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", - "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", - "FAO Stock Status Category", "Scientific Advice", - "GRSF SDG Flag", "GRSF Traceability Flag" - }; - for(String name : groupNames) { - String ckanGroupName = getGroupId(name); - String groupName = CKANGroup.fromGroupTitleToName(name) + GROUP_SUFFIX; - logger.info("{} -> {}", name, ckanGroupName, groupName); - Assert.assertTrue(ckanGroupName.compareTo(groupName)==0); - } - - - } - - // @Test - public void createNewGRSFGroups() throws ObjectNotFound, Exception { - String sysAdminAPI = CKANUtility.getSysAdminAPI(); - String[] groupNames = new String[] { - "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", - "Catch", "Landing", "Assessment Method", - "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", - "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", - "FAO Stock Status Category", "Scientific Advice", - "GRSF SDG Flag", "GRSF Traceability Flag" - }; - groupNames = new String[] {"Assessment Method"}; - ObjectMapper objectMapper = new ObjectMapper(); - for(String name : groupNames) { - CKANGroup ckanGroupToCreate = new CKANGroup(); - ckanGroupToCreate.setApiKey(sysAdminAPI); - ckanGroupToCreate.setName(name); - ObjectNode node = objectMapper.createObjectNode(); - node.put("display_name", name); - node.put("title", name); - node.put("name", getGroupId(name)); - String json = objectMapper.writeValueAsString(node); - logger.info(json); - ckanGroupToCreate.create(json); - } - } - - // @Test - public void deleteNewGRSFGroups() throws ObjectNotFound, Exception { - String sysAdminAPI = CKANUtility.getSysAdminAPI(); -// String[] groupNames = new String[] { -// "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", -// "Catch", "Landing", -// "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", -// "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", -// "FAO Stock Status Category", "Scientific Advice", -// "GRSF SDG Flag", "GRSF Traceability Flag" -// }; -// - List groupNames = CKANGroupTest.listGroup(); - logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); - logger.debug(SecretManagerProvider.instance.get().getContext()); - - for(String name : groupNames) { - CKANGroup ckanGroupToCreate = new CKANGroup(); - ckanGroupToCreate.setApiKey(sysAdminAPI); -// ckanGroupToCreate.setName(getGroupId(name)); - ckanGroupToCreate.setName(name); - try { -// ckanGroupToCreate.purge(); - }catch (Exception e) { - - } - } - } - - - // @Test - public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception { - String[] usernames = new String[] { "grsf_publisher", "luca_frosini", "francesco_mangiacrapa"}; - - String sysAdminAPI = CKANUtility.getSysAdminAPI(); - List groupNames = CKANGroupTest.listGroup(); - for(String groupName : groupNames) { - CKANUser ckanUser = new CKANUser(); - ckanUser.setApiKey(sysAdminAPI); - for(String username : usernames) { - ckanUser.setName(username); - ckanUser.addToGroup(groupName); - } - } - - List orgs = CKANOrganizationTest.listOrg(); - for(String org : orgs) { - CKANUser ckanUser = new CKANUser(); - ckanUser.setApiKey(sysAdminAPI); - for(String username : usernames) { - ckanUser.addUserToOrganization(org, username, "admin"); - } - } - } - - @Test - public void manageOrganizations() throws JsonProcessingException { - String context = SecretManagerProvider.instance.get().getContext(); - ScopeBean scopeBean = new ScopeBean(context); - - String sysAdminAPI = CKANUtility.getSysAdminAPI(); - Map organizations = new HashMap<>(); - organizations.put(scopeBean.name(), scopeBean.name()); - organizations.put("FIRMS", "FIRMS"); - organizations.put("FishSource", "FishSource"); - organizations.put("RAM", "RAM"); - organizations.put("SDG", "FAO SDG 14.4.1 Questionnaire"); - ObjectMapper objectMapper = new ObjectMapper(); - for(String org : organizations.keySet()) { - CKANOrganization ckanOrganization = new CKANOrganization(); - ckanOrganization.setApiKey(sysAdminAPI); - ckanOrganization.setName(org.toLowerCase()); - ObjectNode node = objectMapper.createObjectNode(); - String fancyName = organizations.get(org); - node.put("display_name", fancyName); - node.put("title", fancyName); - node.put("name", org.toLowerCase()); - String json = objectMapper.writeValueAsString(node); - try { - ckanOrganization.create(json); - // ckanOrganization.purge(); - }catch (Exception e) { - - } - } - - } - - protected void printLine(File file, String line) throws Exception { - try (FileWriter fw = new FileWriter(file, true); - BufferedWriter bw = new BufferedWriter(fw); - PrintWriter out = new PrintWriter(bw)) { - out.println(line); - out.flush(); - } catch (IOException e) { - throw e; - } - } - - - public Map getListParameter(int limit, int offset, String type, String org) { - Map parameters = new HashMap<>(); - if(limit <= 0) { - // According to CKAN documentation - // the number of matching rows to return. There is a hard limit of 1000 datasets per query. - // see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search - limit = 1000; - } - parameters.put(CKANPackage.ROWS_KEY, String.valueOf(limit)); - - if(offset < 0) { - offset = 0; - } - parameters.put(CKANPackage.START_KEY, String.valueOf(offset)); - - String q = null; - - if(type!=null) { - // This filter by type - StringWriter qStringWriter = new StringWriter(); - qStringWriter.append("extras_systemtype:"); - qStringWriter.append(type); - q = qStringWriter.toString(); - - } - - if(org!=null) { - StringWriter qStringWriter = new StringWriter(); - qStringWriter.append("organization:"); - qStringWriter.append(org); - if(q!=null) { - q = q + " AND " + qStringWriter.toString(); - } - } - - if(q!=null) { - q = q + " AND (StatusoftheRecord:Approved OR StatusoftheRecord:Archived)"; - // q = q + " AND (groups:stock-group)"; - } - - - if(q!=null) { - parameters.put(GCatConstants.Q_KEY, q); - } - - - return parameters; - } - - // @Test - public void list() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - int limit = 100; - // String[] contexts = new String[] {PRODUCTION_GRSF, PRODUCTION_GRSF_ADMIN}; - String[] contexts = new String[] {PRODUCTION_GRSF}; - for(String context : contexts) { - ContextTest.setContextByName(context); - logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); - logger.debug(SecretManagerProvider.instance.get().getContext()); - CKANPackage ckanPackage = new CKANPackage(); - SortedSet sortedSet = new TreeSet<>(); - String[] types = new String[] {"Marine Resource", "Assessment Unit"}; - for(String type : types) { - int offset = 0; - boolean go = true; - ScopeBean scopeBean = new ScopeBean(context); - while(go) { - Map parameters = getListParameter(limit, offset==0 ? 0 : (offset*limit), type, scopeBean.name().toLowerCase()); - String ret = ckanPackage.list(parameters); - JsonNode gotList = mapper.readTree(ret); - Assert.assertTrue(gotList instanceof ArrayNode); - ArrayNode itemsArray = (ArrayNode) gotList; - if(itemsArray.size()>0) { - for(JsonNode jsonNode : itemsArray) { - String name = jsonNode.asText(); - sortedSet.add(name); - } - offset++; - }else { - go=false; - } - } - File file = new File(scopeBean.name() + "-" + type.replace(" ", "_")+".txt"); - if(file.exists()) { - file.delete(); - } - for(String name : sortedSet) { - printLine(file, name); - } - } - } - } - - private void purgeGRSFRecords(int limit, int offset) throws Exception { - logger.debug("Going to purge {} records, starting from {}", limit, limit*offset); - CKANPackage ckanPackage = new CKANPackage(); - MultivaluedMap mvm = new MultivaluedHashMap(); - UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm); - ckanPackage.setUriInfo(uriInfo); - ObjectMapper mapper = new ObjectMapper(); - boolean go = true; - while(go) { - Map parameters = getListParameter(limit, offset, null, null); - String ret = ckanPackage.list(parameters); - JsonNode gotList = mapper.readTree(ret); - Assert.assertTrue(gotList instanceof ArrayNode); - ArrayNode itemsArray = (ArrayNode) gotList; - if(itemsArray.size()>0) { - for(JsonNode jsonNode : itemsArray) { - String name = jsonNode.asText(); - logger.debug("Going to purge record with name {}", name); - ckanPackage.setName(name); - try { - ckanPackage.purgeNoCheckNoDeleteFiles(); - }catch (Exception e) { - logger.error("Unable to purge record with name {}", name, e); - } - } - Thread.sleep(500); - }else { - go=false; - } - } - } - - // @Test - public void purgeAllGRSFRecords() throws Exception { - purgeGRSFRecords(100, 0); - } - - @Test - public void purgeSpecificGRSFRecord() throws Exception { - CKANPackage ckanPackage = new CKANPackage(); - MultivaluedMap mvm = new MultivaluedHashMap(); - UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm); - ckanPackage.setUriInfo(uriInfo); - ckanPackage.setName("70ae6895-7d3d-4f4a-86f9-bcb17d41bff6"); - ckanPackage.purgeNoCheckNoDeleteFiles(); - } - -}