Changed GRSF tests
This commit is contained in:
parent
d5ef34dd49
commit
e3e8ffd393
|
@ -2,6 +2,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for gCube Catalogue (gCat) Service
|
# Changelog for gCube Catalogue (gCat) Service
|
||||||
|
|
||||||
|
## [v2.3.0-SNAPSHOT]
|
||||||
|
|
||||||
|
- Added moderation link in moderation message [#23142]
|
||||||
|
- Switched moderation messages to notification [#23317]
|
||||||
|
|
||||||
## [v2.2.0]
|
## [v2.2.0]
|
||||||
|
|
||||||
- Switched gcat credentials to new IAM authz [#21628][#22727]
|
- Switched gcat credentials to new IAM authz [#21628][#22727]
|
||||||
|
|
2
pom.xml
2
pom.xml
|
@ -12,7 +12,7 @@
|
||||||
<groupId>org.gcube.data-catalogue</groupId>
|
<groupId>org.gcube.data-catalogue</groupId>
|
||||||
<artifactId>gcat</artifactId>
|
<artifactId>gcat</artifactId>
|
||||||
<packaging>war</packaging>
|
<packaging>war</packaging>
|
||||||
<version>2.2.0</version>
|
<version>2.3.0-SNAPSHOT</version>
|
||||||
<name>gCube Catalogue (gCat) Service</name>
|
<name>gCube Catalogue (gCat) Service</name>
|
||||||
<description>
|
<description>
|
||||||
This service allows any client to publish on the gCube Catalogue.
|
This service allows any client to publish on the gCube Catalogue.
|
||||||
|
|
|
@ -360,7 +360,7 @@ public abstract class CKAN {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void purge() {
|
public void purge() {
|
||||||
sendPostRequest(PURGE, createJsonNodeWithNameAsID());
|
sendPostRequest(PURGE, createJsonNodeWithNameAsID());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -504,6 +504,23 @@ public class CKANPackage extends CKAN implements Moderated {
|
||||||
return getAsString(arrayNode);
|
return getAsString(arrayNode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String listNoCheck(int limit, int offset) {
|
||||||
|
Map<String,String> parameters = new HashMap<>();
|
||||||
|
if(limit <= 0) {
|
||||||
|
// According to CKAN documentation
|
||||||
|
// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
|
||||||
|
// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
|
||||||
|
limit = 1000;
|
||||||
|
}
|
||||||
|
parameters.put(ROWS_KEY, String.valueOf(limit));
|
||||||
|
|
||||||
|
if(offset < 0) {
|
||||||
|
offset = 0;
|
||||||
|
}
|
||||||
|
parameters.put(START_KEY, String.valueOf(offset));
|
||||||
|
return list(parameters);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String list(int limit, int offset) {
|
public String list(int limit, int offset) {
|
||||||
Map<String,String> parameters = getListingParameters(limit, offset);
|
Map<String,String> parameters = getListingParameters(limit, offset);
|
||||||
|
@ -983,9 +1000,9 @@ public class CKANPackage extends CKAN implements Moderated {
|
||||||
/**
|
/**
|
||||||
* Used for bulk purging. Internal use only
|
* Used for bulk purging. Internal use only
|
||||||
*/
|
*/
|
||||||
protected void purgeNoCheckNoDeleteFiles() {
|
public void purgeNoCheckNoDeleteFiles() {
|
||||||
// setApiKey(CKANUtility.getSysAdminAPI());
|
setApiKey(CKANUtility.getSysAdminAPI());
|
||||||
// super.purge();
|
super.purge();
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -471,7 +471,7 @@ public class CKANResource extends CKAN {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void purge() {
|
public void purge() {
|
||||||
String[] moreAllowed = new String[] {HEAD.class.getSimpleName(), GET.class.getSimpleName(),
|
String[] moreAllowed = new String[] {HEAD.class.getSimpleName(), GET.class.getSimpleName(),
|
||||||
PUT.class.getSimpleName(), DELETE.class.getSimpleName()};
|
PUT.class.getSimpleName(), DELETE.class.getSimpleName()};
|
||||||
throw new NotAllowedException(OPTIONS.class.getSimpleName(), moreAllowed);
|
throw new NotAllowedException(OPTIONS.class.getSimpleName(), moreAllowed);
|
||||||
|
|
|
@ -215,7 +215,7 @@ public class CKANUser extends CKAN {
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addUserToOrganization(String organizationName, String ckanUsername, String role) {
|
public void addUserToOrganization(String organizationName, String ckanUsername, String role) {
|
||||||
logger.trace("Going to add user {} to organization {} with role {}", ckanUsername, organizationName, role);
|
logger.trace("Going to add user {} to organization {} with role {}", ckanUsername, organizationName, role);
|
||||||
CKANOrganization ckanOrganization = new CKANOrganization();
|
CKANOrganization ckanOrganization = new CKANOrganization();
|
||||||
ckanOrganization.setApiKey(CKANUtility.getSysAdminAPI());
|
ckanOrganization.setApiKey(CKANUtility.getSysAdminAPI());
|
||||||
|
|
|
@ -75,7 +75,7 @@ public class CKANPackageTest extends ContextTest {
|
||||||
logger.debug("List:\n{}", mapper.writeValueAsString(gotList));
|
logger.debug("List:\n{}", mapper.writeValueAsString(gotList));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected UriInfo getUriInfo(MultivaluedMap<String, String> queryParameters) {
|
public static UriInfo getUriInfo(MultivaluedMap<String, String> queryParameters) {
|
||||||
UriInfo uriInfo = new UriInfo() {
|
UriInfo uriInfo = new UriInfo() {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package org.gcube.gcat.persistence.ckan;
|
package org.gcube.gcat.persistence.grsf;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
@ -7,14 +7,26 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore;
|
import javax.ws.rs.core.MultivaluedHashMap;
|
||||||
|
import javax.ws.rs.core.MultivaluedMap;
|
||||||
|
import javax.ws.rs.core.UriInfo;
|
||||||
|
|
||||||
import org.gcube.com.fasterxml.jackson.core.JsonProcessingException;
|
import org.gcube.com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
|
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
|
||||||
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
|
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
|
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
|
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||||
|
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
||||||
import org.gcube.gcat.ContextTest;
|
import org.gcube.gcat.ContextTest;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANGroup;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANGroupTest;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANOrganization;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANOrganizationTest;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANPackage;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANPackageTest;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANUser;
|
||||||
|
import org.gcube.gcat.persistence.ckan.CKANUtility;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
@ -25,7 +37,7 @@ import org.slf4j.LoggerFactory;
|
||||||
*/
|
*/
|
||||||
public class GRSFUtilities extends ContextTest {
|
public class GRSFUtilities extends ContextTest {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class);
|
private static Logger logger = LoggerFactory.getLogger(GRSFUtilities.class);
|
||||||
|
|
||||||
public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF";
|
public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF";
|
||||||
public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin";
|
public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin";
|
||||||
|
@ -137,6 +149,8 @@ public class GRSFUtilities extends ContextTest {
|
||||||
|
|
||||||
// @Test
|
// @Test
|
||||||
public void createNewGRSFGroups() throws ObjectNotFound, Exception {
|
public void createNewGRSFGroups() throws ObjectNotFound, Exception {
|
||||||
|
// ContextTest.setContext("");
|
||||||
|
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
||||||
String[] groupNames = new String[] {
|
String[] groupNames = new String[] {
|
||||||
"GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
"GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
||||||
|
@ -157,28 +171,35 @@ public class GRSFUtilities extends ContextTest {
|
||||||
node.put("name", getGroupId(name));
|
node.put("name", getGroupId(name));
|
||||||
String json = objectMapper.writeValueAsString(node);
|
String json = objectMapper.writeValueAsString(node);
|
||||||
logger.info(json);
|
logger.info(json);
|
||||||
// ckanGroupToCreate.create(json);
|
ckanGroupToCreate.create(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// @Test
|
// @Test
|
||||||
public void deleteNewGRSFGroups() throws ObjectNotFound, Exception {
|
public void deleteNewGRSFGroups() throws ObjectNotFound, Exception {
|
||||||
|
ContextTest.setContext("");
|
||||||
|
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
||||||
|
logger.debug(SecretManagerProvider.instance.get().getContext());
|
||||||
|
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
||||||
String[] groupNames = new String[] {
|
// String[] groupNames = new String[] {
|
||||||
"GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
// "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
|
||||||
"Catch", "Landing",
|
// "Catch", "Landing",
|
||||||
"Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
|
// "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
|
||||||
"Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
|
// "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
|
||||||
"FAO Stock Status Category", "Scientific Advice",
|
// "FAO Stock Status Category", "Scientific Advice",
|
||||||
"GRSF SDG Flag", "GRSF Traceability Flag"
|
// "GRSF SDG Flag", "GRSF Traceability Flag"
|
||||||
};
|
// };
|
||||||
|
//
|
||||||
|
List<String> groupNames = CKANGroupTest.listGroup();
|
||||||
|
|
||||||
for(String name : groupNames) {
|
for(String name : groupNames) {
|
||||||
CKANGroup ckanGroupToCreate = new CKANGroup();
|
CKANGroup ckanGroupToCreate = new CKANGroup();
|
||||||
ckanGroupToCreate.setApiKey(sysAdminAPI);
|
ckanGroupToCreate.setApiKey(sysAdminAPI);
|
||||||
ckanGroupToCreate.setName(getGroupId(name));
|
// ckanGroupToCreate.setName(getGroupId(name));
|
||||||
|
ckanGroupToCreate.setName(name);
|
||||||
try {
|
try {
|
||||||
ckanGroupToCreate.purge();
|
// ckanGroupToCreate.purge();
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -186,28 +207,34 @@ public class GRSFUtilities extends ContextTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// @Test
|
@Test
|
||||||
public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception {
|
public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception {
|
||||||
String username = "grsf_publisher";
|
ContextTest.setContext("");
|
||||||
// username = "luca_frosini";
|
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
||||||
// username = "francesco_mangiacrapa";
|
logger.debug(SecretManagerProvider.instance.get().getContext());
|
||||||
|
|
||||||
|
String[] usernames = new String[] { "grsf_publisher", "luca_frosini", "francesco_mangiacrapa"};
|
||||||
|
|
||||||
|
|
||||||
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
String sysAdminAPI = CKANUtility.getSysAdminAPI();
|
||||||
List<String> groupNames = CKANGroupTest.listGroup();
|
List<String> groupNames = CKANGroupTest.listGroup();
|
||||||
for(String groupName : groupNames) {
|
for(String groupName : groupNames) {
|
||||||
CKANUser ckanUser = new CKANUser();
|
CKANUser ckanUser = new CKANUser();
|
||||||
ckanUser.setApiKey(sysAdminAPI);
|
ckanUser.setApiKey(sysAdminAPI);
|
||||||
ckanUser.setName(username);
|
for(String username : usernames) {
|
||||||
ckanUser.addToGroup(groupName);
|
ckanUser.setName(username);
|
||||||
|
ckanUser.addToGroup(groupName);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
List<String> orgs = CKANOrganizationTest.listOrg();
|
List<String> orgs = CKANOrganizationTest.listOrg();
|
||||||
for(String org : orgs) {
|
for(String org : orgs) {
|
||||||
CKANUser ckanUser = new CKANUser();
|
CKANUser ckanUser = new CKANUser();
|
||||||
ckanUser.setApiKey(sysAdminAPI);
|
ckanUser.setApiKey(sysAdminAPI);
|
||||||
ckanUser.addUserToOrganization(org, username, "admin");
|
for(String username : usernames) {
|
||||||
|
ckanUser.addUserToOrganization(org, username, "admin");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// @Test
|
// @Test
|
||||||
|
@ -239,10 +266,13 @@ public class GRSFUtilities extends ContextTest {
|
||||||
private void purgeGRSFRecords(int limit, int offset) throws Exception {
|
private void purgeGRSFRecords(int limit, int offset) throws Exception {
|
||||||
logger.debug("Going to purge {} records, starting from {}", limit, limit*offset);
|
logger.debug("Going to purge {} records, starting from {}", limit, limit*offset);
|
||||||
CKANPackage ckanPackage = new CKANPackage();
|
CKANPackage ckanPackage = new CKANPackage();
|
||||||
|
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
|
||||||
|
UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm);
|
||||||
|
ckanPackage.setUriInfo(uriInfo);
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
boolean go = true;
|
boolean go = true;
|
||||||
while(go) {
|
while(go) {
|
||||||
String ret = ckanPackage.list(limit, offset);
|
String ret = ckanPackage.listNoCheck(limit, offset);
|
||||||
JsonNode gotList = mapper.readTree(ret);
|
JsonNode gotList = mapper.readTree(ret);
|
||||||
Assert.assertTrue(gotList instanceof ArrayNode);
|
Assert.assertTrue(gotList instanceof ArrayNode);
|
||||||
ArrayNode itemsArray = (ArrayNode) gotList;
|
ArrayNode itemsArray = (ArrayNode) gotList;
|
||||||
|
@ -252,14 +282,14 @@ public class GRSFUtilities extends ContextTest {
|
||||||
logger.debug("Going to purge record with name {}", name);
|
logger.debug("Going to purge record with name {}", name);
|
||||||
ckanPackage.setName(name);
|
ckanPackage.setName(name);
|
||||||
try {
|
try {
|
||||||
ckanPackage.purgeNoCheckNoDeleteFiles();
|
// ckanPackage.purgeNoCheckNoDeleteFiles();
|
||||||
if(go) {
|
if(go) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
// TODO: handle exception
|
// TODO: handle exception
|
||||||
}
|
}
|
||||||
// Thread.sleep(500);
|
Thread.sleep(500);
|
||||||
}
|
}
|
||||||
}else {
|
}else {
|
||||||
go=false;
|
go=false;
|
||||||
|
@ -267,9 +297,11 @@ public class GRSFUtilities extends ContextTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonIgnore
|
|
||||||
// @Test
|
// @Test
|
||||||
public void purgeAllGRSFRecords() throws Exception {
|
public void purgeAllGRSFRecords() throws Exception {
|
||||||
|
ContextTest.setContext("");
|
||||||
|
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
|
||||||
|
logger.debug(SecretManagerProvider.instance.get().getContext());
|
||||||
purgeGRSFRecords(100, 0);
|
purgeGRSFRecords(100, 0);
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,8 @@
|
||||||
|
|
||||||
|
|
||||||
<logger name="org.gcube" level="ERROR" />
|
<logger name="org.gcube" level="ERROR" />
|
||||||
<logger name="org.gcube.gcat" level="TRACE" />
|
<logger name="org.gcube.gcat" level="ERROR" />
|
||||||
|
<logger name="org.gcube.gcat.persistence.grsf" level="TRACE" />
|
||||||
|
|
||||||
<root level="WARN">
|
<root level="WARN">
|
||||||
<appender-ref ref="STDOUT" />
|
<appender-ref ref="STDOUT" />
|
||||||
|
|
Loading…
Reference in New Issue