package org.gcube.gcat.persistence.ckan; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore; import org.gcube.com.fasterxml.jackson.core.JsonProcessingException; import org.gcube.com.fasterxml.jackson.databind.JsonNode; import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.gcat.ContextTest; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Luca Frosini (ISTI - CNR) */ public class GRSFUtilities extends ContextTest { private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class); public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF"; public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin"; public static final String PRODUCTION_GRSF_PRE = "/d4science.research-infrastructures.eu/FARM/GRSF_Pre"; public static final String PREPROD_GRSF_PRE = "/pred4s/preprod/GRSF_Pre"; public GRSFUtilities() throws ObjectNotFound, Exception { ContextTest.setContextByName(PREPROD_GRSF_PRE); } @Test public void test() throws ObjectNotFound, Exception { ContextTest.setContextByName(PREPROD_GRSF_PRE); // // ContextTest.setContextByName(PRODUCTION_GRSF); // ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN); // ContextTest.setContextByName(PRODUCTION_GRSF_PRE); } private void create(Set createdGroup, Map groups, String name) throws JsonProcessingException, IOException { if(createdGroup.contains(name)) { return; } String sysAdminAPI = ""; // CKANUtility.getSysAdminAPI(); CKANGroup ckanGroupToCreate = new CKANGroup(); ckanGroupToCreate.setApiKey(sysAdminAPI); ckanGroupToCreate.setName(name); // ckanGroupToCreate.purge(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode = objectMapper.readTree(groups.get(name)); ArrayNode array = (ArrayNode) jsonNode.get("groups"); for(JsonNode node : array) { String parentName = node.get("name").asText(); if(!createdGroup.contains(parentName)) { create(createdGroup, groups, parentName); } } ckanGroupToCreate.create(groups.get(name)); createdGroup.add(name); } // @Test public void createGRSFGroups() throws ObjectNotFound, Exception { String key = CKANUtility.getSysAdminAPI(); List groupNames = CKANGroupTest.listGroup(); Map groups = new HashMap<>(); for(String name : groupNames) { CKANGroup ckanGroup = new CKANGroup(); ckanGroup.setApiKey(key); ckanGroup.setName(name); String read = ckanGroup.read(); groups.put(name, read); } Set createdGroup = new HashSet<>(); for(String name : groupNames) { create(createdGroup, groups, name); } } public static String getGroupNameOnCkan(String origName){ if(origName == null) { throw new IllegalArgumentException("origName cannot be null"); } String modified = origName.replaceAll("\\(", ""); modified = modified.replaceAll("\\)", ""); modified = modified.trim().toLowerCase().replaceAll("[^A-Za-z0-9-]", "-"); if(modified.startsWith("-")) { modified = modified.substring(1); } if(modified.endsWith("-")) { modified = modified.substring(0, modified.length() -1); } return modified; } public static final String GROUP_SUFFIX = "-group"; public static String getGroupId(String name) { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append(getGroupNameOnCkan(name)); if(!name.endsWith(GROUP_SUFFIX)) { stringBuffer.append(GROUP_SUFFIX); } return stringBuffer.toString(); } @Test public void testGroupName() { String[] groupNames = new String[] { "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", "Catch", "Landing", "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", "FAO Stock Status Category", "Scientific Advice", "GRSF SDG Flag", "GRSF Traceability Flag" }; for(String name : groupNames) { String ckanGroupName = getGroupId(name); String groupName = CKANGroup.fromGroupTitleToName(name) + GROUP_SUFFIX; logger.info("{} -> {}", name, ckanGroupName, groupName); Assert.assertTrue(ckanGroupName.compareTo(groupName)==0); } } // @Test public void createNewGRSFGroups() throws ObjectNotFound, Exception { String sysAdminAPI = CKANUtility.getSysAdminAPI(); String[] groupNames = new String[] { "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", "Catch", "Landing", "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", "FAO Stock Status Category", "Scientific Advice", "GRSF SDG Flag", "GRSF Traceability Flag" }; ObjectMapper objectMapper = new ObjectMapper(); for(String name : groupNames) { CKANGroup ckanGroupToCreate = new CKANGroup(); ckanGroupToCreate.setApiKey(sysAdminAPI); ckanGroupToCreate.setName(name); ObjectNode node = objectMapper.createObjectNode(); node.put("display_name", name); node.put("title", name); node.put("name", getGroupId(name)); String json = objectMapper.writeValueAsString(node); logger.info(json); // ckanGroupToCreate.create(json); } } // @Test public void deleteNewGRSFGroups() throws ObjectNotFound, Exception { String sysAdminAPI = CKANUtility.getSysAdminAPI(); String[] groupNames = new String[] { "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM", "Catch", "Landing", "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass", "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend", "FAO Stock Status Category", "Scientific Advice", "GRSF SDG Flag", "GRSF Traceability Flag" }; for(String name : groupNames) { CKANGroup ckanGroupToCreate = new CKANGroup(); ckanGroupToCreate.setApiKey(sysAdminAPI); ckanGroupToCreate.setName(getGroupId(name)); try { ckanGroupToCreate.purge(); }catch (Exception e) { } } } // @Test public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception { String username = "grsf_publisher"; // username = "luca_frosini"; // username = "francesco_mangiacrapa"; String sysAdminAPI = CKANUtility.getSysAdminAPI(); List groupNames = CKANGroupTest.listGroup(); for(String groupName : groupNames) { CKANUser ckanUser = new CKANUser(); ckanUser.setApiKey(sysAdminAPI); ckanUser.setName(username); ckanUser.addToGroup(groupName); } List orgs = CKANOrganizationTest.listOrg(); for(String org : orgs) { CKANUser ckanUser = new CKANUser(); ckanUser.setApiKey(sysAdminAPI); ckanUser.addUserToOrganization(org, username, "admin"); } } // @Test public void manageOrganizations() throws JsonProcessingException { String sysAdminAPI = CKANUtility.getSysAdminAPI(); String[] organizations = new String[] { "GRSF_Pre", "FIRMS", "FishSource", "RAM" }; ObjectMapper objectMapper = new ObjectMapper(); for(String org : organizations) { CKANOrganization ckanOrganization = new CKANOrganization(); ckanOrganization.setApiKey(sysAdminAPI); ckanOrganization.setName(org.toLowerCase()); ObjectNode node = objectMapper.createObjectNode(); node.put("display_name", org); node.put("title", org); node.put("name", org.toLowerCase()); String json = objectMapper.writeValueAsString(node); try { // ckanOrganization.create(json); // ckanOrganization.purge(); }catch (Exception e) { } } } private void purgeGRSFRecords(int limit, int offset) throws Exception { logger.debug("Going to purge {} records, starting from {}", limit, limit*offset); CKANPackage ckanPackage = new CKANPackage(); ObjectMapper mapper = new ObjectMapper(); boolean go = true; while(go) { String ret = ckanPackage.list(limit, offset); JsonNode gotList = mapper.readTree(ret); Assert.assertTrue(gotList instanceof ArrayNode); ArrayNode itemsArray = (ArrayNode) gotList; if(itemsArray.size()>0) { for(JsonNode jsonNode : itemsArray) { String name = jsonNode.asText(); logger.debug("Going to purge record with name {}", name); ckanPackage.setName(name); try { ckanPackage.purgeNoCheckNoDeleteFiles(); if(go) { break; } }catch (Exception e) { // TODO: handle exception } // Thread.sleep(500); } }else { go=false; } } } @JsonIgnore // @Test public void purgeAllGRSFRecords() throws Exception { purgeGRSFRecords(100, 0); } }