Merge branch 'feature/23142' of gitea@code-repo.d4science.org:gCubeSystem/gcat.git into feature/23142

src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java

@@ -67,9 +67,9 @@ public class CKANPackage extends CKAN implements Moderated {
 	
 	
 	// limit in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
-	protected static final String ROWS_KEY = "rows";
+	public static final String ROWS_KEY = "rows";
 	// offset in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
-	protected static final String START_KEY = "start";
+	public static final String START_KEY = "start";
 	
 	protected static final String ORGANIZATION_FILTER_TEMPLATE = GCatConstants.ORGANIZATION_PARAMETER + ":%s";
 	
@@ -504,30 +504,6 @@ public class CKANPackage extends CKAN implements Moderated {
 		return getAsString(arrayNode);
 	}
 	
-	public String listNoCheck(int limit, int offset) {
-		Map<String,String> parameters = new HashMap<>();
-		if(limit <= 0) {
-			// According to CKAN documentation
-			// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
-			// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
-			limit = 1000;
-		}
-		parameters.put(ROWS_KEY, String.valueOf(limit));
-		
-		if(offset < 0) {
-			offset = 0;
-		}
-		parameters.put(START_KEY, String.valueOf(offset));
-		
-		// This filter by type
-//		StringWriter qStringWriter = new StringWriter();
-//		qStringWriter.append("extras_systemtype:");
-//		qStringWriter.append("");
-//		parameters.put(GCatConstants.Q_KEY,	qStringWriter.toString());
-		
-		return list(parameters);
-	}
-	
 	@Override
 	public String list(int limit, int offset) {
 		Map<String,String> parameters = getListingParameters(limit, offset);

src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java

@@ -1,11 +1,18 @@
 package org.gcube.gcat.persistence.grsf;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 
 import javax.ws.rs.core.MultivaluedHashMap;
 import javax.ws.rs.core.MultivaluedMap;
@@ -18,7 +25,9 @@ import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
 import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
 import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
 import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
+import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.gcat.ContextTest;
+import org.gcube.gcat.api.GCatConstants;
 import org.gcube.gcat.persistence.ckan.CKANGroup;
 import org.gcube.gcat.persistence.ckan.CKANGroupTest;
 import org.gcube.gcat.persistence.ckan.CKANOrganization;
@@ -49,13 +58,11 @@ public class GRSFUtilities extends ContextTest {
 	@Before
 	public void before() throws Exception {
 //		ContextTest.setContextByName(PREPROD_GRSF_PRE);
-		
+//		ContextTest.setContextByName(PRODUCTION_GRSF);
-		ContextTest.setContextByName(PRODUCTION_GRSF);
 //		ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN);
 //		ContextTest.setContextByName(PRODUCTION_GRSF_PRE);
-
+//		logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
-		logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
+//		logger.debug(SecretManagerProvider.instance.get().getContext());
-		logger.debug(SecretManagerProvider.instance.get().getContext());
 	}
 	
 	private void create(Set<String> createdGroup, Map<String, String> groups, String name) throws JsonProcessingException, IOException {
@@ -254,6 +261,111 @@ public class GRSFUtilities extends ContextTest {
 		
 	}
 	
+	protected void printLine(File file, String line) throws Exception {
+		try (FileWriter fw = new FileWriter(file, true);
+				BufferedWriter bw = new BufferedWriter(fw);
+				PrintWriter out = new PrintWriter(bw)) {
+			out.println(line);
+			out.flush();
+		} catch (IOException e) {
+			throw e;
+		}
+	}
+	
+	
+	public Map<String,String> getListParameter(int limit, int offset, String type, String org) {
+		Map<String,String> parameters = new HashMap<>();
+		if(limit <= 0) {
+			// According to CKAN documentation
+			// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
+			// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
+			limit = 1000;
+		}
+		parameters.put(CKANPackage.ROWS_KEY, String.valueOf(limit));
+		
+		if(offset < 0) {
+			offset = 0;
+		}
+		parameters.put(CKANPackage.START_KEY, String.valueOf(offset));
+		
+		String q = null;
+		
+		if(type!=null) {
+			// This filter by type
+			StringWriter qStringWriter = new StringWriter();
+			qStringWriter.append("extras_systemtype:");
+			qStringWriter.append(type);
+			q = qStringWriter.toString();
+			
+		}
+		
+		if(org!=null) {
+			StringWriter qStringWriter = new StringWriter();
+			qStringWriter.append("organization:");
+			qStringWriter.append(org);
+			if(q!=null) {
+				q = q + " AND " + qStringWriter.toString();
+			}
+		}
+		
+		if(q!=null) {
+			q = q + " AND (StatusoftheRecord:Approved OR StatusoftheRecord:Archived)";
+			// q = q + " AND (groups:stock-group)";
+		}
+		
+		
+		if(q!=null) {
+			parameters.put(GCatConstants.Q_KEY,	q);
+		}
+		
+		
+		return parameters;
+	}
+	
+	// @Test
+	public void list()  throws Exception {
+		ObjectMapper mapper = new ObjectMapper();
+		int limit = 100;
+		// String[] contexts = new String[] {PRODUCTION_GRSF, PRODUCTION_GRSF_ADMIN};
+		String[] contexts = new String[] {PRODUCTION_GRSF};
+		for(String context : contexts) {
+			ContextTest.setContextByName(context);
+			logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
+			logger.debug(SecretManagerProvider.instance.get().getContext());
+			CKANPackage ckanPackage = new CKANPackage();
+			SortedSet<String> sortedSet = new TreeSet<>();
+			String[] types = new String[] {"Marine Resource", "Assessment Unit"};
+			for(String type : types) {
+				int offset = 0;
+				boolean go = true;
+				ScopeBean scopeBean = new ScopeBean(context);
+				while(go) {
+					Map<String,String> parameters = getListParameter(limit, offset==0 ? 0 : (offset*limit), type, scopeBean.name().toLowerCase());
+					String ret = ckanPackage.list(parameters);
+					JsonNode gotList = mapper.readTree(ret);
+					Assert.assertTrue(gotList instanceof ArrayNode);
+					ArrayNode itemsArray = (ArrayNode) gotList;
+					if(itemsArray.size()>0) {
+						for(JsonNode jsonNode : itemsArray) {
+							String name = jsonNode.asText();
+							sortedSet.add(name);
+						}
+						offset++;
+					}else {
+						go=false;
+					}
+				}
+				File file = new File(scopeBean.name() + "-" + type.replace(" ", "_")+".txt");
+				if(file.exists()) {
+					file.delete();
+				}
+				for(String name : sortedSet) {
+					printLine(file, name);
+				}
+			}
+		}
+	}
+	
 	private void purgeGRSFRecords(int limit, int offset) throws Exception {
 		logger.debug("Going to purge {} records, starting from {}", limit, limit*offset);
 		CKANPackage ckanPackage = new CKANPackage();
@@ -263,7 +375,8 @@ public class GRSFUtilities extends ContextTest {
 		ObjectMapper mapper = new ObjectMapper();
 		boolean go = true;
 		while(go) {
-			String ret = ckanPackage.listNoCheck(limit, offset);
+			Map<String,String> parameters = getListParameter(limit, offset, null, null);
+			String ret = ckanPackage.list(parameters);
 			JsonNode gotList = mapper.readTree(ret);
 			Assert.assertTrue(gotList instanceof ArrayNode);
 			ArrayNode itemsArray = (ArrayNode) gotList;
@@ -285,7 +398,7 @@ public class GRSFUtilities extends ContextTest {
 		}
 	}
 	
-	@Test
+	// @Test
 	public void purgeAllGRSFRecords() throws Exception {
 		purgeGRSFRecords(100, 0);
 	}