Removed GRSF utilities moved in the dedicated component
This commit is contained in:
luca.frosini
parent
1913af6663
commit
0cf6e58dd4
|
|
@ -1,424 +0,0 @@
|
||||||
package org.gcube.gcat.persistence.grsf;
|
|
||||||
|
|
||||||
import java.io.BufferedWriter;
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileWriter;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.io.StringWriter;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.SortedSet;
|
|
||||||
import java.util.TreeSet;
|
|
||||||
|
|
||||||
import javax.ws.rs.core.MultivaluedHashMap;
|
|
||||||
import javax.ws.rs.core.MultivaluedMap;
|
|
||||||
import javax.ws.rs.core.UriInfo;
|
|
||||||
|
|
||||||
import org.gcube.com.fasterxml.jackson.core.JsonProcessingException;
|
|
||||||
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
|
|
||||||
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
|
|
||||||
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
|
|
||||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
|
||||||
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
|
||||||
import org.gcube.common.scope.impl.ScopeBean;
|
|
||||||
import org.gcube.gcat.ContextTest;
|
|
||||||
import org.gcube.gcat.api.GCatConstants;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANGroup;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANGroupTest;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANOrganization;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANOrganizationTest;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANPackage;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANPackageTest;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANUser;
|
|
||||||
import org.gcube.gcat.persistence.ckan.CKANUtility;
|
|
||||||
import org.junit.Assert;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
|
||||||
*/
|
|
||||||
public class GRSFUtilities extends ContextTest {
|
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(GRSFUtilities.class);
|
|
||||||
|
|
||||||
public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF";
|
|
||||||
public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin";
|
|
||||||
public static final String PRODUCTION_GRSF_PRE = "/d4science.research-infrastructures.eu/FARM/GRSF_Pre";
|
|
||||||
|
|
||||||
public static final String PREPROD_GRSF_PRE = "/pred4s/preprod/GRSF_Pre";
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void before() throws Exception {
|
|
||||||
// ContextTest.setContextByName(PREPROD_GRSF_PRE);
|
|
||||||
// ContextTest.setContextByName(PRODUCTION_GRSF);
|
|
||||||
// ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN);
|
|
||||||
// ContextTest.setContextByName(PRODUCTION_GRSF_PRE);
|
|
||||||
// logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
|
||||||
// logger.debug(SecretManagerProvider.instance.get().getContext());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void create(Set<String> createdGroup, Map<String, String> groups, String name) throws JsonProcessingException, IOException {
|
|
||||||
if(createdGroup.contains(name)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
String sysAdminAPI = ""; // CKANUtility.getSysAdminAPI();
|
|
||||||
CKANGroup ckanGroupToCreate = new CKANGroup();
|
|
||||||
ckanGroupToCreate.setApiKey(sysAdminAPI);
|
|
||||||
ckanGroupToCreate.setName(name);
|
|
||||||
// ckanGroupToCreate.purge();
|
|
||||||
ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
JsonNode jsonNode = objectMapper.readTree(groups.get(name));
|
|
||||||
ArrayNode array = (ArrayNode) jsonNode.get("groups");
|
|
||||||
for(JsonNode node : array) {
|
|
||||||
String parentName = node.get("name").asText();
|
|
||||||
if(!createdGroup.contains(parentName)) {
|
|
||||||
create(createdGroup, groups, parentName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ckanGroupToCreate.create(groups.get(name));
|
|
||||||
createdGroup.add(name);
|
|
||||||
}
|
|
||||||
|
|
||||||
// @Test
|
|
||||||
public void createGRSFGroups() throws ObjectNotFound, Exception {
|
|
||||||
String key = CKANUtility.getSysAdminAPI();
|
|
||||||
List<String> groupNames = CKANGroupTest.listGroup();
|
|
||||||
Map<String, String> groups = new HashMap<>();
|
|
||||||
for(String name : groupNames) {
|
|
||||||
CKANGroup ckanGroup = new CKANGroup();
|
|
||||||
ckanGroup.setApiKey(key);
|
|
||||||
ckanGroup.setName(name);
|
|
||||||
String read = ckanGroup.read();
|
|
||||||
groups.put(name, read);
|
|
||||||
}
|
|
||||||
|
|
||||||
Set<String> createdGroup = new HashSet<>();
|
|
||||||
|
|
||||||
for(String name : groupNames) {
|
|
||||||
create(createdGroup, groups, name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getGroupNameOnCkan(String origName){
|
|
||||||
if(origName == null) {
|
|
||||||
throw new IllegalArgumentException("origName cannot be null");
|
|
||||||
}
|
|
||||||
String modified = origName.replaceAll("\\(", "");
|
|
||||||
modified = modified.replaceAll("\\)", "");
|
|
||||||
modified = modified.trim().toLowerCase().replaceAll("[^A-Za-z0-9-]", "-");
|
|
||||||
if(modified.startsWith("-")) {
|
|
||||||
modified = modified.substring(1);
|
|
||||||
}
|
|
||||||
if(modified.endsWith("-")) {
|
|
||||||
modified = modified.substring(0, modified.length() -1);
|
|
||||||
}
|
|
||||||
return modified;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static final String GROUP_SUFFIX = "-group";
|
|
||||||
|
|
||||||
public static String getGroupId(String name) {
|
|
||||||
StringBuffer stringBuffer = new StringBuffer();
|
|
||||||
stringBuffer.append(getGroupNameOnCkan(name));
|
|
||||||
if(!name.endsWith(GROUP_SUFFIX)) {
|
|
||||||
stringBuffer.append(GROUP_SUFFIX);
|
|
||||||
}
|
|
||||||
return stringBuffer.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGroupName() {
|
|
||||||
String[] groupNames = new String[] {
|
|
||||||
"GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
|
||||||
"Catch", "Landing", "Assessment Method",
|
|
||||||
"Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
|
|
||||||
"Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
|
|
||||||
"FAO Stock Status Category", "Scientific Advice",
|
|
||||||
"GRSF SDG Flag", "GRSF Traceability Flag"
|
|
||||||
};
|
|
||||||
for(String name : groupNames) {
|
|
||||||
String ckanGroupName = getGroupId(name);
|
|
||||||
String groupName = CKANGroup.fromGroupTitleToName(name) + GROUP_SUFFIX;
|
|
||||||
logger.info("{} -> {}", name, ckanGroupName, groupName);
|
|
||||||
Assert.assertTrue(ckanGroupName.compareTo(groupName)==0);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// @Test
|
|
||||||
public void createNewGRSFGroups() throws ObjectNotFound, Exception {
|
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
|
||||||
String[] groupNames = new String[] {
|
|
||||||
"GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
|
||||||
"Catch", "Landing", "Assessment Method",
|
|
||||||
"Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
|
|
||||||
"Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
|
|
||||||
"FAO Stock Status Category", "Scientific Advice",
|
|
||||||
"GRSF SDG Flag", "GRSF Traceability Flag"
|
|
||||||
};
|
|
||||||
groupNames = new String[] {"Assessment Method"};
|
|
||||||
ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
for(String name : groupNames) {
|
|
||||||
CKANGroup ckanGroupToCreate = new CKANGroup();
|
|
||||||
ckanGroupToCreate.setApiKey(sysAdminAPI);
|
|
||||||
ckanGroupToCreate.setName(name);
|
|
||||||
ObjectNode node = objectMapper.createObjectNode();
|
|
||||||
node.put("display_name", name);
|
|
||||||
node.put("title", name);
|
|
||||||
node.put("name", getGroupId(name));
|
|
||||||
String json = objectMapper.writeValueAsString(node);
|
|
||||||
logger.info(json);
|
|
||||||
ckanGroupToCreate.create(json);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// @Test
|
|
||||||
public void deleteNewGRSFGroups() throws ObjectNotFound, Exception {
|
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
|
||||||
// String[] groupNames = new String[] {
|
|
||||||
// "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
|
||||||
// "Catch", "Landing",
|
|
||||||
// "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
|
|
||||||
// "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
|
|
||||||
// "FAO Stock Status Category", "Scientific Advice",
|
|
||||||
// "GRSF SDG Flag", "GRSF Traceability Flag"
|
|
||||||
// };
|
|
||||||
//
|
|
||||||
List<String> groupNames = CKANGroupTest.listGroup();
|
|
||||||
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
|
||||||
logger.debug(SecretManagerProvider.instance.get().getContext());
|
|
||||||
|
|
||||||
for(String name : groupNames) {
|
|
||||||
CKANGroup ckanGroupToCreate = new CKANGroup();
|
|
||||||
ckanGroupToCreate.setApiKey(sysAdminAPI);
|
|
||||||
// ckanGroupToCreate.setName(getGroupId(name));
|
|
||||||
ckanGroupToCreate.setName(name);
|
|
||||||
try {
|
|
||||||
// ckanGroupToCreate.purge();
|
|
||||||
}catch (Exception e) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// @Test
|
|
||||||
public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception {
|
|
||||||
String[] usernames = new String[] { "grsf_publisher", "luca_frosini", "francesco_mangiacrapa"};
|
|
||||||
|
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
|
||||||
List<String> groupNames = CKANGroupTest.listGroup();
|
|
||||||
for(String groupName : groupNames) {
|
|
||||||
CKANUser ckanUser = new CKANUser();
|
|
||||||
ckanUser.setApiKey(sysAdminAPI);
|
|
||||||
for(String username : usernames) {
|
|
||||||
ckanUser.setName(username);
|
|
||||||
ckanUser.addToGroup(groupName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
List<String> orgs = CKANOrganizationTest.listOrg();
|
|
||||||
for(String org : orgs) {
|
|
||||||
CKANUser ckanUser = new CKANUser();
|
|
||||||
ckanUser.setApiKey(sysAdminAPI);
|
|
||||||
for(String username : usernames) {
|
|
||||||
ckanUser.addUserToOrganization(org, username, "admin");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void manageOrganizations() throws JsonProcessingException {
|
|
||||||
String context = SecretManagerProvider.instance.get().getContext();
|
|
||||||
ScopeBean scopeBean = new ScopeBean(context);
|
|
||||||
|
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
|
||||||
Map<String, String> organizations = new HashMap<>();
|
|
||||||
organizations.put(scopeBean.name(), scopeBean.name());
|
|
||||||
organizations.put("FIRMS", "FIRMS");
|
|
||||||
organizations.put("FishSource", "FishSource");
|
|
||||||
organizations.put("RAM", "RAM");
|
|
||||||
organizations.put("SDG", "FAO SDG 14.4.1 Questionnaire");
|
|
||||||
ObjectMapper objectMapper = new ObjectMapper();
|
|
||||||
for(String org : organizations.keySet()) {
|
|
||||||
CKANOrganization ckanOrganization = new CKANOrganization();
|
|
||||||
ckanOrganization.setApiKey(sysAdminAPI);
|
|
||||||
ckanOrganization.setName(org.toLowerCase());
|
|
||||||
ObjectNode node = objectMapper.createObjectNode();
|
|
||||||
String fancyName = organizations.get(org);
|
|
||||||
node.put("display_name", fancyName);
|
|
||||||
node.put("title", fancyName);
|
|
||||||
node.put("name", org.toLowerCase());
|
|
||||||
String json = objectMapper.writeValueAsString(node);
|
|
||||||
try {
|
|
||||||
ckanOrganization.create(json);
|
|
||||||
// ckanOrganization.purge();
|
|
||||||
}catch (Exception e) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void printLine(File file, String line) throws Exception {
|
|
||||||
try (FileWriter fw = new FileWriter(file, true);
|
|
||||||
BufferedWriter bw = new BufferedWriter(fw);
|
|
||||||
PrintWriter out = new PrintWriter(bw)) {
|
|
||||||
out.println(line);
|
|
||||||
out.flush();
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public Map<String,String> getListParameter(int limit, int offset, String type, String org) {
|
|
||||||
Map<String,String> parameters = new HashMap<>();
|
|
||||||
if(limit <= 0) {
|
|
||||||
// According to CKAN documentation
|
|
||||||
// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
|
|
||||||
// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
|
|
||||||
limit = 1000;
|
|
||||||
}
|
|
||||||
parameters.put(CKANPackage.ROWS_KEY, String.valueOf(limit));
|
|
||||||
|
|
||||||
if(offset < 0) {
|
|
||||||
offset = 0;
|
|
||||||
}
|
|
||||||
parameters.put(CKANPackage.START_KEY, String.valueOf(offset));
|
|
||||||
|
|
||||||
String q = null;
|
|
||||||
|
|
||||||
if(type!=null) {
|
|
||||||
// This filter by type
|
|
||||||
StringWriter qStringWriter = new StringWriter();
|
|
||||||
qStringWriter.append("extras_systemtype:");
|
|
||||||
qStringWriter.append(type);
|
|
||||||
q = qStringWriter.toString();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if(org!=null) {
|
|
||||||
StringWriter qStringWriter = new StringWriter();
|
|
||||||
qStringWriter.append("organization:");
|
|
||||||
qStringWriter.append(org);
|
|
||||||
if(q!=null) {
|
|
||||||
q = q + " AND " + qStringWriter.toString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if(q!=null) {
|
|
||||||
q = q + " AND (StatusoftheRecord:Approved OR StatusoftheRecord:Archived)";
|
|
||||||
// q = q + " AND (groups:stock-group)";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if(q!=null) {
|
|
||||||
parameters.put(GCatConstants.Q_KEY, q);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
return parameters;
|
|
||||||
}
|
|
||||||
|
|
||||||
// @Test
|
|
||||||
public void list() throws Exception {
|
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
|
||||||
int limit = 100;
|
|
||||||
// String[] contexts = new String[] {PRODUCTION_GRSF, PRODUCTION_GRSF_ADMIN};
|
|
||||||
String[] contexts = new String[] {PRODUCTION_GRSF};
|
|
||||||
for(String context : contexts) {
|
|
||||||
ContextTest.setContextByName(context);
|
|
||||||
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
|
||||||
logger.debug(SecretManagerProvider.instance.get().getContext());
|
|
||||||
CKANPackage ckanPackage = new CKANPackage();
|
|
||||||
SortedSet<String> sortedSet = new TreeSet<>();
|
|
||||||
String[] types = new String[] {"Marine Resource", "Assessment Unit"};
|
|
||||||
for(String type : types) {
|
|
||||||
int offset = 0;
|
|
||||||
boolean go = true;
|
|
||||||
ScopeBean scopeBean = new ScopeBean(context);
|
|
||||||
while(go) {
|
|
||||||
Map<String,String> parameters = getListParameter(limit, offset==0 ? 0 : (offset*limit), type, scopeBean.name().toLowerCase());
|
|
||||||
String ret = ckanPackage.list(parameters);
|
|
||||||
JsonNode gotList = mapper.readTree(ret);
|
|
||||||
Assert.assertTrue(gotList instanceof ArrayNode);
|
|
||||||
ArrayNode itemsArray = (ArrayNode) gotList;
|
|
||||||
if(itemsArray.size()>0) {
|
|
||||||
for(JsonNode jsonNode : itemsArray) {
|
|
||||||
String name = jsonNode.asText();
|
|
||||||
sortedSet.add(name);
|
|
||||||
}
|
|
||||||
offset++;
|
|
||||||
}else {
|
|
||||||
go=false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File file = new File(scopeBean.name() + "-" + type.replace(" ", "_")+".txt");
|
|
||||||
if(file.exists()) {
|
|
||||||
file.delete();
|
|
||||||
}
|
|
||||||
for(String name : sortedSet) {
|
|
||||||
printLine(file, name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void purgeGRSFRecords(int limit, int offset) throws Exception {
|
|
||||||
logger.debug("Going to purge {} records, starting from {}", limit, limit*offset);
|
|
||||||
CKANPackage ckanPackage = new CKANPackage();
|
|
||||||
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
|
|
||||||
UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm);
|
|
||||||
ckanPackage.setUriInfo(uriInfo);
|
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
|
||||||
boolean go = true;
|
|
||||||
while(go) {
|
|
||||||
Map<String,String> parameters = getListParameter(limit, offset, null, null);
|
|
||||||
String ret = ckanPackage.list(parameters);
|
|
||||||
JsonNode gotList = mapper.readTree(ret);
|
|
||||||
Assert.assertTrue(gotList instanceof ArrayNode);
|
|
||||||
ArrayNode itemsArray = (ArrayNode) gotList;
|
|
||||||
if(itemsArray.size()>0) {
|
|
||||||
for(JsonNode jsonNode : itemsArray) {
|
|
||||||
String name = jsonNode.asText();
|
|
||||||
logger.debug("Going to purge record with name {}", name);
|
|
||||||
ckanPackage.setName(name);
|
|
||||||
try {
|
|
||||||
ckanPackage.purgeNoCheckNoDeleteFiles();
|
|
||||||
}catch (Exception e) {
|
|
||||||
logger.error("Unable to purge record with name {}", name, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Thread.sleep(500);
|
|
||||||
}else {
|
|
||||||
go=false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// @Test
|
|
||||||
public void purgeAllGRSFRecords() throws Exception {
|
|
||||||
purgeGRSFRecords(100, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void purgeSpecificGRSFRecord() throws Exception {
|
|
||||||
CKANPackage ckanPackage = new CKANPackage();
|
|
||||||
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
|
|
||||||
UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm);
|
|
||||||
ckanPackage.setUriInfo(uriInfo);
|
|
||||||
ckanPackage.setName("70ae6895-7d3d-4f4a-86f9-bcb17d41bff6");
|
|
||||||
ckanPackage.purgeNoCheckNoDeleteFiles();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue