gCubeSystem
/
gcat
16
0
Fork 0
Code Issues Pull Requests Releases 13 Wiki Activity

Moved specific code for GRSF to the dedicated test class

This commit is contained in:
Luca Frosini 2022-05-24 17:35:48 +02:00
parent 08312a1a19
commit 3ec771527c
2 changed files with 122 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java
View File

@ -67,9 +67,9 @@ public class CKANPackage extends CKAN implements Moderated {
// limit in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
protected static final String ROWS_KEY = "rows";
public static final String ROWS_KEY = "rows";
// offset in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
protected static final String START_KEY = "start";
public static final String START_KEY = "start";
protected static final String ORGANIZATION_FILTER_TEMPLATE = GCatConstants.ORGANIZATION_PARAMETER + ":%s";
@ -504,30 +504,6 @@ public class CKANPackage extends CKAN implements Moderated {
return getAsString(arrayNode);
}
public String listNoCheck(int limit, int offset) {
Map<String,String> parameters = new HashMap<>();
if(limit <= 0) {
// According to CKAN documentation
// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
limit = 1000;
}
parameters.put(ROWS_KEY, String.valueOf(limit));
if(offset < 0) {
offset = 0;
}
parameters.put(START_KEY, String.valueOf(offset));
// This filter by type
// StringWriter qStringWriter = new StringWriter();
// qStringWriter.append("extras_systemtype:");
// qStringWriter.append("");
// parameters.put(GCatConstants.Q_KEY, qStringWriter.toString());
return list(parameters);
}
@Override
public String list(int limit, int offset) {
Map<String,String> parameters = getListingParameters(limit, offset);

127
src/test/java/org/gcube/gcat/persistence/grsf/GRSFUtilities.java
View File

@ -1,11 +1,18 @@
package org.gcube.gcat.persistence.grsf;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
@ -18,7 +25,9 @@ import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.gcat.ContextTest;
import org.gcube.gcat.api.GCatConstants;
import org.gcube.gcat.persistence.ckan.CKANGroup;
import org.gcube.gcat.persistence.ckan.CKANGroupTest;
import org.gcube.gcat.persistence.ckan.CKANOrganization;
@ -49,13 +58,11 @@ public class GRSFUtilities extends ContextTest {
@Before
public void before() throws Exception {
// ContextTest.setContextByName(PREPROD_GRSF_PRE);
ContextTest.setContextByName(PRODUCTION_GRSF);
// ContextTest.setContextByName(PRODUCTION_GRSF);
// ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN);
// ContextTest.setContextByName(PRODUCTION_GRSF_PRE);
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
logger.debug(SecretManagerProvider.instance.get().getContext());
// logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
// logger.debug(SecretManagerProvider.instance.get().getContext());
}
private void create(Set<String> createdGroup, Map<String, String> groups, String name) throws JsonProcessingException, IOException {
@ -254,6 +261,111 @@ public class GRSFUtilities extends ContextTest {
}
protected void printLine(File file, String line) throws Exception {
try (FileWriter fw = new FileWriter(file, true);
BufferedWriter bw = new BufferedWriter(fw);
PrintWriter out = new PrintWriter(bw)) {
out.println(line);
out.flush();
} catch (IOException e) {
throw e;
}
}
public Map<String,String> getListParameter(int limit, int offset, String type, String org) {
Map<String,String> parameters = new HashMap<>();
if(limit <= 0) {
// According to CKAN documentation
// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
limit = 1000;
}
parameters.put(CKANPackage.ROWS_KEY, String.valueOf(limit));
if(offset < 0) {
offset = 0;
}
parameters.put(CKANPackage.START_KEY, String.valueOf(offset));
String q = null;
if(type!=null) {
// This filter by type
StringWriter qStringWriter = new StringWriter();
qStringWriter.append("extras_systemtype:");
qStringWriter.append(type);
q = qStringWriter.toString();
}
if(org!=null) {
StringWriter qStringWriter = new StringWriter();
qStringWriter.append("organization:");
qStringWriter.append(org);
if(q!=null) {
q = q + " AND " + qStringWriter.toString();
}
}
if(q!=null) {
q = q + " AND (StatusoftheRecord:Approved OR StatusoftheRecord:Archived)";
// q = q + " AND (groups:stock-group)";
}
if(q!=null) {
parameters.put(GCatConstants.Q_KEY, q);
}
return parameters;
}
// @Test
public void list() throws Exception {
ObjectMapper mapper = new ObjectMapper();
int limit = 100;
// String[] contexts = new String[] {PRODUCTION_GRSF, PRODUCTION_GRSF_ADMIN};
String[] contexts = new String[] {PRODUCTION_GRSF};
for(String context : contexts) {
ContextTest.setContextByName(context);
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
logger.debug(SecretManagerProvider.instance.get().getContext());
CKANPackage ckanPackage = new CKANPackage();
SortedSet<String> sortedSet = new TreeSet<>();
String[] types = new String[] {"Marine Resource", "Assessment Unit"};
for(String type : types) {
int offset = 0;
boolean go = true;
ScopeBean scopeBean = new ScopeBean(context);
while(go) {
Map<String,String> parameters = getListParameter(limit, offset==0 ? 0 : (offset*limit), type, scopeBean.name().toLowerCase());
String ret = ckanPackage.list(parameters);
JsonNode gotList = mapper.readTree(ret);
Assert.assertTrue(gotList instanceof ArrayNode);
ArrayNode itemsArray = (ArrayNode) gotList;
if(itemsArray.size()>0) {
for(JsonNode jsonNode : itemsArray) {
String name = jsonNode.asText();
sortedSet.add(name);
}
offset++;
}else {
go=false;
}
}
File file = new File(scopeBean.name() + "-" + type.replace(" ", "_")+".txt");
if(file.exists()) {
file.delete();
}
for(String name : sortedSet) {
printLine(file, name);
}
}
}
}
private void purgeGRSFRecords(int limit, int offset) throws Exception {
logger.debug("Going to purge {} records, starting from {}", limit, limit*offset);
CKANPackage ckanPackage = new CKANPackage();
@ -263,7 +375,8 @@ public class GRSFUtilities extends ContextTest {
ObjectMapper mapper = new ObjectMapper();
boolean go = true;
while(go) {
String ret = ckanPackage.listNoCheck(limit, offset);
Map<String,String> parameters = getListParameter(limit, offset, null, null);
String ret = ckanPackage.list(parameters);
JsonNode gotList = mapper.readTree(ret);
Assert.assertTrue(gotList instanceof ArrayNode);
ArrayNode itemsArray = (ArrayNode) gotList;
@ -285,7 +398,7 @@ public class GRSFUtilities extends ContextTest {
}
}
@Test
// @Test
public void purgeAllGRSFRecords() throws Exception {
purgeGRSFRecords(100, 0);
}