diff --git a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java index bffb802..b221991 100644 --- a/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java +++ b/src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java @@ -67,9 +67,9 @@ public class CKANPackage extends CKAN implements Moderated { // limit in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search - protected static final String ROWS_KEY = "rows"; + public static final String ROWS_KEY = "rows"; // offset in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search - protected static final String START_KEY = "start"; + public static final String START_KEY = "start"; protected static final String ORGANIZATION_FILTER_TEMPLATE = GCatConstants.ORGANIZATION_PARAMETER + ":%s"; @@ -504,30 +504,6 @@ public class CKANPackage extends CKAN implements Moderated { return getAsString(arrayNode); } - public String listNoCheck(int limit, int offset) { - Map parameters = new HashMap<>(); - if(limit <= 0) { - // According to CKAN documentation - // the number of matching rows to return. There is a hard limit of 1000 datasets per query. - // see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search - limit = 1000; - } - parameters.put(ROWS_KEY, String.valueOf(limit)); - - if(offset < 0) { - offset = 0; - } - parameters.put(START_KEY, String.valueOf(offset)); - - // This filter by type -// StringWriter qStringWriter = new StringWriter(); -// qStringWriter.append("extras_systemtype:"); -// qStringWriter.append(""); -// parameters.put(GCatConstants.Q_KEY, qStringWriter.toString()); - - return list(parameters); - } - @Override public String list(int limit, int offset) { Map parameters = getListingParameters(limit, offset); diff --git a/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java b/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java index efc325b..4ffb42b 100644 --- a/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java +++ b/src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java @@ -1,11 +1,18 @@ package org.gcube.gcat.persistence.grsf; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; @@ -18,7 +25,9 @@ import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.utils.manager.SecretManagerProvider; +import org.gcube.common.scope.impl.ScopeBean; import org.gcube.gcat.ContextTest; +import org.gcube.gcat.api.GCatConstants; import org.gcube.gcat.persistence.ckan.CKANGroup; import org.gcube.gcat.persistence.ckan.CKANGroupTest; import org.gcube.gcat.persistence.ckan.CKANOrganization; @@ -49,13 +58,11 @@ public class GRSFUtilities extends ContextTest { @Before public void before() throws Exception { // ContextTest.setContextByName(PREPROD_GRSF_PRE); - - ContextTest.setContextByName(PRODUCTION_GRSF); +// ContextTest.setContextByName(PRODUCTION_GRSF); // ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN); // ContextTest.setContextByName(PRODUCTION_GRSF_PRE); - - logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); - logger.debug(SecretManagerProvider.instance.get().getContext()); +// logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); +// logger.debug(SecretManagerProvider.instance.get().getContext()); } private void create(Set createdGroup, Map groups, String name) throws JsonProcessingException, IOException { @@ -254,6 +261,111 @@ public class GRSFUtilities extends ContextTest { } + protected void printLine(File file, String line) throws Exception { + try (FileWriter fw = new FileWriter(file, true); + BufferedWriter bw = new BufferedWriter(fw); + PrintWriter out = new PrintWriter(bw)) { + out.println(line); + out.flush(); + } catch (IOException e) { + throw e; + } + } + + + public Map getListParameter(int limit, int offset, String type, String org) { + Map parameters = new HashMap<>(); + if(limit <= 0) { + // According to CKAN documentation + // the number of matching rows to return. There is a hard limit of 1000 datasets per query. + // see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search + limit = 1000; + } + parameters.put(CKANPackage.ROWS_KEY, String.valueOf(limit)); + + if(offset < 0) { + offset = 0; + } + parameters.put(CKANPackage.START_KEY, String.valueOf(offset)); + + String q = null; + + if(type!=null) { + // This filter by type + StringWriter qStringWriter = new StringWriter(); + qStringWriter.append("extras_systemtype:"); + qStringWriter.append(type); + q = qStringWriter.toString(); + + } + + if(org!=null) { + StringWriter qStringWriter = new StringWriter(); + qStringWriter.append("organization:"); + qStringWriter.append(org); + if(q!=null) { + q = q + " AND " + qStringWriter.toString(); + } + } + + if(q!=null) { + q = q + " AND (StatusoftheRecord:Approved OR StatusoftheRecord:Archived)"; + // q = q + " AND (groups:stock-group)"; + } + + + if(q!=null) { + parameters.put(GCatConstants.Q_KEY, q); + } + + + return parameters; + } + + // @Test + public void list() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + int limit = 100; + // String[] contexts = new String[] {PRODUCTION_GRSF, PRODUCTION_GRSF_ADMIN}; + String[] contexts = new String[] {PRODUCTION_GRSF}; + for(String context : contexts) { + ContextTest.setContextByName(context); + logger.debug(SecretManagerProvider.instance.get().getUser().getUsername()); + logger.debug(SecretManagerProvider.instance.get().getContext()); + CKANPackage ckanPackage = new CKANPackage(); + SortedSet sortedSet = new TreeSet<>(); + String[] types = new String[] {"Marine Resource", "Assessment Unit"}; + for(String type : types) { + int offset = 0; + boolean go = true; + ScopeBean scopeBean = new ScopeBean(context); + while(go) { + Map parameters = getListParameter(limit, offset==0 ? 0 : (offset*limit), type, scopeBean.name().toLowerCase()); + String ret = ckanPackage.list(parameters); + JsonNode gotList = mapper.readTree(ret); + Assert.assertTrue(gotList instanceof ArrayNode); + ArrayNode itemsArray = (ArrayNode) gotList; + if(itemsArray.size()>0) { + for(JsonNode jsonNode : itemsArray) { + String name = jsonNode.asText(); + sortedSet.add(name); + } + offset++; + }else { + go=false; + } + } + File file = new File(scopeBean.name() + "-" + type.replace(" ", "_")+".txt"); + if(file.exists()) { + file.delete(); + } + for(String name : sortedSet) { + printLine(file, name); + } + } + } + } + private void purgeGRSFRecords(int limit, int offset) throws Exception { logger.debug("Going to purge {} records, starting from {}", limit, limit*offset); CKANPackage ckanPackage = new CKANPackage(); @@ -263,7 +375,8 @@ public class GRSFUtilities extends ContextTest { ObjectMapper mapper = new ObjectMapper(); boolean go = true; while(go) { - String ret = ckanPackage.listNoCheck(limit, offset); + Map parameters = getListParameter(limit, offset, null, null); + String ret = ckanPackage.list(parameters); JsonNode gotList = mapper.readTree(ret); Assert.assertTrue(gotList instanceof ArrayNode); ArrayNode itemsArray = (ArrayNode) gotList; @@ -285,7 +398,7 @@ public class GRSFUtilities extends ContextTest { } } - @Test + // @Test public void purgeAllGRSFRecords() throws Exception { purgeGRSFRecords(100, 0); }