Merge remote-tracking branch 'origin/feature/23142'

Conflicts:
	src/main/java/org/gcube/gcat/social/SocialMessage.java
This commit is contained in:
Luca Frosini 2022-05-27 17:21:59 +02:00
commit 8ce89f4140
16 changed files with 683 additions and 73 deletions

View File

@ -2,6 +2,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for gCube Catalogue (gCat) Service
## [v2.3.0-SNAPSHOT]
- Added moderation link in moderation message [#23142]
- Switched moderation messages to notification [#23317]
## [v2.2.0]
- Switched gcat credentials to new IAM authz [#21628][#22727]

18
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.data-catalogue</groupId>
<artifactId>gcat</artifactId>
<packaging>war</packaging>
<version>2.2.0</version>
<version>2.3.0-SNAPSHOT</version>
<name>gCube Catalogue (gCat) Service</name>
<description>
This service allows any client to publish on the gCube Catalogue.
@ -64,11 +64,11 @@
<artifactId>gcat-api</artifactId>
<version>[2.0.0,3.0.0-SNAPSHOT)</version>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.gcube.common</groupId> -->
<!-- <artifactId>authorization-control-library</artifactId> -->
<!-- <version>[1.0.0,2.0.0-SNAPSHOT)</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.gcube.social-networking</groupId>
<artifactId>social-service-client</artifactId>
<version>1.1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>authorization-utils</artifactId>
@ -227,12 +227,6 @@
<version>${enunciate.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.geotoolkit</groupId>
<artifactId>geotk-xml-base</artifactId>
<version>3.20-geoapi-3.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>

View File

@ -1,9 +1,18 @@
package org.gcube.gcat.moderation.thread;
import java.util.HashMap;
import java.util.Map;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.gcat.api.moderation.CMItemStatus;
import org.gcube.gcat.moderation.thread.social.SocialMessageModerationThread;
import org.gcube.gcat.moderation.thread.social.notifications.SocialNotificationModerationThread;
import org.gcube.gcat.persistence.ckan.CKANUser;
import org.gcube.portlets.user.uriresolvermanager.UriResolverManager;
import org.gcube.portlets.user.uriresolvermanager.resolvers.query.CatalogueResolverQueryString.MODERATION_OP;
import org.gcube.portlets.user.uriresolvermanager.resolvers.query.CatalogueResolverQueryStringBuilder;
/**
* @author Luca Frosini (ISTI - CNR)
@ -14,6 +23,8 @@ public abstract class ModerationThread {
protected String itemName;
protected String itemTitle;
protected String itemURL;
protected String moderationURL;
protected boolean create;
protected CMItemStatus cmItemStatus;
@ -24,14 +35,15 @@ public abstract class ModerationThread {
public static ModerationThread getDefaultInstance() {
// return new FakeModerationThread();
return new SocialMessageModerationThread();
// return new SocialMessageModerationThread();
return new SocialNotificationModerationThread();
}
public ModerationThread() {
this.objectMapper = new ObjectMapper();
this.itemAuthor = false;
this.create = false;
cmItemStatus = CMItemStatus.PENDING;
this.cmItemStatus = CMItemStatus.PENDING;
}
public void setItemCoordinates(String itemID, String itemName, String itemTitle, String itemURL) {
@ -49,6 +61,32 @@ public abstract class ModerationThread {
this.ckanUser = ckanUser;
}
public String getModerationURL() {
if(moderationURL==null) {
try {
SecretManager secretManager = SecretManagerProvider.instance.get();
String context = secretManager.getContext();
UriResolverManager resolver = new UriResolverManager("CTLG");
Map<String, String> params = new HashMap<String, String>();
params.put("gcube_scope", context); //e.g. /gcube/devsec/devVRE
params.put("entity_context", "organization");
params.put("entity_name", CatalogueConfiguration.getOrganizationName(context)); //e.g. devvre
CatalogueResolverQueryStringBuilder builder = new CatalogueResolverQueryStringBuilder(itemName); //item name under moderation
builder.itemStatus(cmItemStatus.name()). //e.g. pending, approved, rejected
moderation(MODERATION_OP.show);
String queryString = builder.buildQueryParametersToQueryString();
params.put(CatalogueResolverQueryStringBuilder.QUERY_STRING_PARAMETER, queryString);
moderationURL = resolver.getLink(params, true);
}catch (Exception e) {
return itemURL;
}
}
return moderationURL;
}
/**
* The message is sent as gCat
* @param message

View File

@ -1,4 +1,4 @@
package org.gcube.gcat.social;
package org.gcube.gcat.moderation.thread.social.messages;
import java.util.ArrayList;
import java.util.Collection;

View File

@ -0,0 +1,98 @@
package org.gcube.gcat.moderation.thread.social.messages;
import java.net.HttpURLConnection;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.socialservice.SocialService;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.gcat.utils.Constants;
import org.gcube.gcat.utils.HTTPUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class SocialMessage extends Thread {
private static final Logger logger = LoggerFactory.getLogger(SocialMessage.class);
public static final String ITEM_URL = "Item URL";
// https://wiki.gcube-system.org/gcube/Social_Networking_Service#Send_a_message
protected static final String SOCIAL_SERVICE_SEND_MESSAGE_PATH = "/2/messages/write-message";
protected static final String RESPONSE_SUCCESS_KEY = "success";
protected static final String RESPONSE_MESSAGE_KEY = "message";
protected final ObjectMapper objectMapper;
protected Message message;
public SocialMessage() throws Exception {
super();
this.objectMapper = new ObjectMapper();
}
public Message getMessage() {
return message;
}
public void setMessage(Message message) {
this.message = message;
}
@Override
public void run() {
try {
logger.info("Going to send Message {}", message);
// write message
sendSocialMessage();
} catch(Exception e) {
logger.error("Error while executing post creation actions", e);
}
}
public void sendSocialMessage() {
try {
String basePath = SocialService.getSocialService().getServiceBasePath();
if(basePath == null) {
logger.info("Unable to send a message because there is no social networking service available");
return;
}
basePath = basePath.endsWith("/") ? basePath : basePath + "/";
String messageString = objectMapper.writeValueAsString(message);
logger.debug("The message that is going to be send is\n{}", messageString);
GXHTTPStringRequest gxhttpStringRequest = GXHTTPStringRequest.newRequest(basePath);
gxhttpStringRequest.from(Constants.CATALOGUE_NAME);
gxhttpStringRequest.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON);
SecretManager secretManager = SecretManagerProvider.instance.get();
gxhttpStringRequest.setSecurityToken(secretManager.getCurrentSecretHolder().getSecrets().first().getToken());
gxhttpStringRequest.path(SOCIAL_SERVICE_SEND_MESSAGE_PATH);
HttpURLConnection httpURLConnection = gxhttpStringRequest.post(messageString);
String ret = HTTPUtility.getResultAsString(httpURLConnection);
JsonNode jsonNode = objectMapper.readTree(ret);
if(jsonNode.get(RESPONSE_SUCCESS_KEY).asBoolean()) {
logger.info("Message sent : {}", messageString);
} else {
logger.info("Failed to write the message {}. Reason {}", messageString,
jsonNode.get(RESPONSE_MESSAGE_KEY).asText());
}
} catch(Exception e) {
logger.error("Unable to send the message : " + message.toString(), e);
}
}
}

View File

@ -1,4 +1,4 @@
package org.gcube.gcat.moderation.thread.social;
package org.gcube.gcat.moderation.thread.social.messages;
import java.io.StringWriter;
import java.util.Set;
@ -9,8 +9,6 @@ import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.gcat.api.moderation.CMItemStatus;
import org.gcube.gcat.api.moderation.Moderated;
import org.gcube.gcat.moderation.thread.ModerationThread;
import org.gcube.gcat.social.Message;
import org.gcube.gcat.social.SocialMessage;
import org.gcube.gcat.social.SocialUsers;
import org.gcube.gcat.utils.Constants;
import org.slf4j.Logger;
@ -37,7 +35,11 @@ public class SocialMessageModerationThread extends ModerationThread {
stringBuffer.append(itemID);
stringBuffer.append("\n");
stringBuffer.append("URL: ");
stringBuffer.append(itemURL);
if(cmItemStatus == CMItemStatus.APPROVED) {
stringBuffer.append(itemURL);
}else {
stringBuffer.append(getModerationURL());
}
stringBuffer.append("\n\n");
return stringBuffer;
}
@ -197,3 +199,4 @@ public class SocialMessageModerationThread extends ModerationThread {
}
}

View File

@ -0,0 +1,297 @@
package org.gcube.gcat.moderation.thread.social.notifications;
import java.net.URL;
import java.util.Set;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.gcat.api.moderation.CMItemStatus;
import org.gcube.gcat.api.moderation.Moderated;
import org.gcube.gcat.moderation.thread.ModerationThread;
import org.gcube.gcat.persistence.ckan.CKANUser;
import org.gcube.gcat.social.SocialUsers;
import org.gcube.gcat.utils.Constants;
import org.gcube.social_networking.social_networking_client_library.NotificationClient;
import org.gcube.social_networking.socialnetworking.model.beans.catalogue.CatalogueEvent;
import org.gcube.social_networking.socialnetworking.model.beans.catalogue.CatalogueEventType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class SocialNotificationModerationThread extends ModerationThread {
private static final Logger logger = LoggerFactory.getLogger(SocialNotificationModerationThread.class);
public static final String AUTHOR = "Author";
protected CatalogueEventType catalogueEventType;
protected boolean comment;
protected static final boolean notificationSentByGCat;
static {
notificationSentByGCat = false;
}
public SocialNotificationModerationThread() {
super();
this.comment = false;
}
/**
* Create the message for an item that is created/updated
*/
protected void notifyItemToBeManaged() throws Exception {
/*
* An example of created message is:
*
* [mister x] created/updated the item "[TITLE]". You are kindly requested to review it and decide either to APPROVE or REJECT it. [Go to catalogue]
*
*/
String fullName = ckanUser.getNameSurname();
StringBuffer stringBuffer = new StringBuffer();
if(notificationSentByGCat) {
stringBuffer.append(fullName);
}
stringBuffer.append(create ? " created " : " updated ");
stringBuffer.append("the item ");
stringBuffer = addQuotedTitle(stringBuffer);
stringBuffer.append(". You are kindly requested to review it and decide either to APPROVE or REJECT it. ");
postMessage(stringBuffer.toString());
}
public void postItemCreated() throws Exception {
create = true;
cmItemStatus = CMItemStatus.PENDING;
catalogueEventType = CatalogueEventType.ITEM_SUBMITTED;
notifyItemToBeManaged();
}
public void postItemUpdated() throws Exception {
create = false;
cmItemStatus = CMItemStatus.PENDING;
catalogueEventType = CatalogueEventType.ITEM_UPDATED;
notifyItemToBeManaged();
}
protected StringBuffer addUserWithRole(String fullName, String role, StringBuffer stringBuffer, boolean addUserFullName) {
if(addUserFullName) {
stringBuffer.append(fullName);
}
if(role!=null) {
stringBuffer.append(" [");
stringBuffer.append(role);
stringBuffer.append("] ");
}
return stringBuffer;
}
protected StringBuffer addUserWithRole(String fullName, String role, StringBuffer stringBuffer) {
return addUserWithRole(fullName, role, stringBuffer, notificationSentByGCat);
}
public void postItemManaged(String userMessage) throws Exception {
/*
* [mister x] rejected the item "[TITLE]" with this accompanying message "[MESSAGE]". To resubmit it [Go to catalogue]
*
* [mister x] approved the item "[TITLE]" with this accompanying message "[MESSAGE]". [Go to catalogue]
*/
create = false;
String fullName = ckanUser.getNameSurname();
StringBuffer stringBuffer = new StringBuffer();
stringBuffer = addUserWithRole(fullName, Moderated.CATALOGUE_MODERATOR, stringBuffer);
stringBuffer.append(cmItemStatus.getValue());
stringBuffer.append(" the item ");
stringBuffer = addQuotedTitle(stringBuffer);
if(userMessage!=null && userMessage.length()>0) {
stringBuffer.append(" with this accompanying message \"");
stringBuffer.append(userMessage);
stringBuffer.append("\"");
}
stringBuffer.append(".");
if(cmItemStatus == CMItemStatus.REJECTED) {
stringBuffer.append(" To resubmit it ");
}
postMessage(stringBuffer.toString());
}
@Override
public void postItemRejected(String userMessage) throws Exception {
create = false;
cmItemStatus = CMItemStatus.REJECTED;
catalogueEventType = CatalogueEventType.ITEM_REJECTED;
postItemManaged(userMessage);
}
@Override
public void postItemApproved(String userMessage) throws Exception {
create = false;
cmItemStatus = CMItemStatus.APPROVED;
catalogueEventType = CatalogueEventType.ITEM_PUBLISHED;
postItemManaged(userMessage);
}
protected StringBuffer addQuotedTitle(StringBuffer stringBuffer, String quotingCharacter) {
stringBuffer.append(quotingCharacter);
stringBuffer.append(itemTitle);
stringBuffer.append(quotingCharacter);
return stringBuffer;
}
protected StringBuffer addQuotedTitle(StringBuffer stringBuffer) {
return addQuotedTitle(stringBuffer, "\"");
}
protected String getSubject() {
StringBuffer stringBuffer = new StringBuffer();
String fullName = ckanUser.getNameSurname();
if(!comment) {
switch (catalogueEventType) {
case ITEM_SUBMITTED:
stringBuffer.append(fullName);
stringBuffer.append(" created the item ");
break;
case ITEM_UPDATED:
stringBuffer.append(fullName);
stringBuffer.append(" updated the item ");
break;
case ITEM_REJECTED:
case ITEM_PUBLISHED:
addUserWithRole(fullName, Moderated.CATALOGUE_MODERATOR, stringBuffer, true);
stringBuffer.append(cmItemStatus.getValue());
stringBuffer.append(" the item ");
break;
default:
break;
}
}else {
addUserWithRole(fullName, itemAuthor ? SocialNotificationModerationThread.AUTHOR : Moderated.CATALOGUE_MODERATOR, stringBuffer, true);
stringBuffer.append("commented on the item ");
}
stringBuffer = addQuotedTitle(stringBuffer);
return stringBuffer.toString();
}
protected CatalogueEvent getCatalogueEvent(String messageString) throws Exception {
CatalogueEvent catalogueEvent = new CatalogueEvent();
catalogueEvent.setType(catalogueEventType);
catalogueEvent.setNotifyText(messageString);
catalogueEvent.setItemId(getSubject());
if(cmItemStatus == CMItemStatus.APPROVED) {
catalogueEvent.setItemURL(new URL(itemURL));
}else {
catalogueEvent.setItemURL(new URL(getModerationURL()));
}
Set<String> users = SocialUsers.getUsernamesByRole(Moderated.CATALOGUE_MODERATOR);
users.add("luca.frosini");
SecretManager secretManager = SecretManagerProvider.instance.get();
String username = secretManager.getUser().getUsername();
// Adding the user is generating the event
users.add(username);
// Adding item creator
users.add(CKANUser.getUsernameFromCKANUsername(ckanUser.getName()));
catalogueEvent.setIdsToNotify(users.toArray(new String[users.size()]));
catalogueEvent.setIdsAsGroup(false);
return catalogueEvent;
}
@Override
protected void postMessage(String messageString) throws Exception {
CatalogueEvent catalogueEvent = getCatalogueEvent(messageString);
SecretManager secretManager = SecretManagerProvider.instance.get();
Secret secret = Constants.getCatalogueSecret();
if(notificationSentByGCat) {
secretManager.startSession(secret);
}
try {
sendNotification(catalogueEvent);
}finally {
if(notificationSentByGCat) {
secretManager.endSession();
}
}
}
@Override
public void postUserMessage(CMItemStatus cmItemStatus, String userMessage) throws Exception {
/*
* [mister x] ([Role]) commented on the item "[TITLE]" as follows "[MESSAGE]". [Go to catalogue]
*/
this.create = false;
this.cmItemStatus = cmItemStatus;
this.comment = true;
switch (cmItemStatus) {
case PENDING:
catalogueEventType = CatalogueEventType.ITEM_UPDATED;
break;
case APPROVED:
catalogueEventType = CatalogueEventType.ITEM_PUBLISHED;
break;
case REJECTED:
catalogueEventType = CatalogueEventType.ITEM_REJECTED;
break;
default:
break;
}
String fullName = ckanUser.getNameSurname();
StringBuffer stringBuffer = new StringBuffer();
stringBuffer = addUserWithRole(fullName, itemAuthor ? SocialNotificationModerationThread.AUTHOR : Moderated.CATALOGUE_MODERATOR, stringBuffer);
stringBuffer.append("commented on the item ");
stringBuffer = addQuotedTitle(stringBuffer);
stringBuffer.append(" as follows \"");
stringBuffer.append(userMessage);
stringBuffer.append("\".");
CatalogueEvent catalogueEvent = getCatalogueEvent(stringBuffer.toString());
SecretManager secretManager = SecretManagerProvider.instance.get();
Secret secret = Constants.getCatalogueSecret();
if(notificationSentByGCat) {
secretManager.startSession(secret);
}
try {
sendNotification(catalogueEvent);
}finally {
if(notificationSentByGCat) {
secretManager.endSession();
}
}
}
protected void sendNotification(CatalogueEvent catalogueEvent) throws Exception {
Thread thread = new Thread() {
public void run() {
try {
NotificationClient nc = new NotificationClient();
nc.sendCatalogueEvent(catalogueEvent);
} catch(Exception e) {
logger.error("Error while sending notification.", e);
}
}
};
thread.run();
// thread.start();
}
@Override
protected void createModerationThread() throws Exception {
create = true;
cmItemStatus = CMItemStatus.PENDING;
}
}

View File

@ -360,7 +360,7 @@ public abstract class CKAN {
}
}
protected void purge() {
public void purge() {
sendPostRequest(PURGE, createJsonNodeWithNameAsID());
}

View File

@ -67,9 +67,9 @@ public class CKANPackage extends CKAN implements Moderated {
// limit in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
protected static final String ROWS_KEY = "rows";
public static final String ROWS_KEY = "rows";
// offset in https://docs.ckan.org/en/latest/api/index.html#ckan.logic.action.get.package_search
protected static final String START_KEY = "start";
public static final String START_KEY = "start";
protected static final String ORGANIZATION_FILTER_TEMPLATE = GCatConstants.ORGANIZATION_PARAMETER + ":%s";
@ -983,9 +983,9 @@ public class CKANPackage extends CKAN implements Moderated {
/**
* Used for bulk purging. Internal use only
*/
protected void purgeNoCheckNoDeleteFiles() {
// setApiKey(CKANUtility.getSysAdminAPI());
// super.purge();
public void purgeNoCheckNoDeleteFiles() {
setApiKey(CKANUtility.getSysAdminAPI());
super.purge();
}
/*

View File

@ -471,7 +471,7 @@ public class CKANResource extends CKAN {
}
@Override
protected void purge() {
public void purge() {
String[] moreAllowed = new String[] {HEAD.class.getSimpleName(), GET.class.getSimpleName(),
PUT.class.getSimpleName(), DELETE.class.getSimpleName()};
throw new NotAllowedException(OPTIONS.class.getSimpleName(), moreAllowed);

View File

@ -199,11 +199,20 @@ public class CKANUser extends CKAN {
}
protected static String getCKANUsername(String username) {
if(username == null)
if(username == null) {
return null;
}
return username.trim().replaceAll("\\.", "_");
}
public static String getUsernameFromCKANUsername(String ckanUsername) {
if(ckanUsername == null) {
return null;
}
return ckanUsername.trim().replaceAll("_", ".");
}
public static String getCKANUsername() {
String username = SecretManagerProvider.instance.get().getUser().getUsername();
return getCKANUsername(username);
@ -215,7 +224,7 @@ public class CKANUser extends CKAN {
return ret;
}
protected void addUserToOrganization(String organizationName, String ckanUsername, String role) {
public void addUserToOrganization(String organizationName, String ckanUsername, String role) {
logger.trace("Going to add user {} to organization {} with role {}", ckanUsername, organizationName, role);
CKANOrganization ckanOrganization = new CKANOrganization();
ckanOrganization.setApiKey(CKANUtility.getSysAdminAPI());

View File

@ -4,7 +4,6 @@ import org.gcube.gcat.ContextTest;
import org.gcube.gcat.api.moderation.CMItemStatus;
import org.gcube.gcat.moderation.thread.ModerationThread;
import org.gcube.gcat.persistence.ckan.CKANUser;
import org.junit.Ignore;
import org.junit.Test;
/**
@ -12,23 +11,47 @@ import org.junit.Test;
*/
public class ModerationThreadTest extends ContextTest {
@Test
@Ignore
public void test() throws Exception {
protected ModerationThread getModerationThread(CKANUser ckanUser) {
ModerationThread moderationThread = ModerationThread.getDefaultInstance();
moderationThread.setItemCoordinates("e31a6ba8-66ef-47b8-b61f-99a1366b4a69", "my_first_restful_transaction_model", "RESTful Transaction Model", "https://data.dev.d4science.org/ctlg/devVRE/my_first_restful_transaction_model");
moderationThread.setItemCoordinates("b1040e70-774f-47b6-95e9-f24efca50caf", "my_first_restful_transaction_model", "RESTful Transaction Model", "https://data.dev.d4science.org/ctlg/devVRE/my_first_restful_transaction_model");
moderationThread.setCKANUser(ckanUser);
return moderationThread;
}
@Test
// @Ignore
public void testModerationThread() throws Exception {
ContextTest.setContextByName("pasquale.pagano_/gcube/devsec/devVRE");
CKANUser ckanUser = new CKANUser();
ckanUser.setName(CKANUser.getCKANUsername());
ckanUser.read();
moderationThread.setCKANUser(ckanUser);
ModerationThread moderationThread = getModerationThread(ckanUser);
moderationThread.postItemCreated();
moderationThread = getModerationThread(ckanUser);
moderationThread.postItemUpdated();
moderationThread = getModerationThread(ckanUser);
moderationThread.postUserMessage(CMItemStatus.PENDING, "Pensaci Bene");
moderationThread = getModerationThread(ckanUser);
moderationThread.postItemRejected(null);
moderationThread = getModerationThread(ckanUser);
moderationThread.postItemRejected("reject con messaggio: Non mi garba");
moderationThread = getModerationThread(ckanUser);
moderationThread.postItemApproved(null);
moderationThread = getModerationThread(ckanUser);
moderationThread.postItemApproved("approve con messaggio: Ora mi garba");
moderationThread = getModerationThread(ckanUser);
moderationThread.setItemAuthor(true);
moderationThread.postUserMessage(CMItemStatus.APPROVED, "Grazie");
Thread.sleep(1000);
}

View File

@ -75,7 +75,7 @@ public class CKANPackageTest extends ContextTest {
logger.debug("List:\n{}", mapper.writeValueAsString(gotList));
}
protected UriInfo getUriInfo(MultivaluedMap<String, String> queryParameters) {
public static UriInfo getUriInfo(MultivaluedMap<String, String> queryParameters) {
UriInfo uriInfo = new UriInfo() {
@Override

View File

@ -1,6 +1,7 @@
package org.gcube.gcat.persistence.ckan;
import org.gcube.gcat.ContextTest;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -22,6 +23,16 @@ public class CKANUserTest extends ContextTest {
return user;
}
@Test
public void getUsernameFromCkanUsername() {
String username = "luca.frosini";
String ckanUsername = CKANUser.getCKANUsername(username);
Assert.assertTrue(ckanUsername.compareTo("luca_frosini")==0);
String gotUsername = CKANUser.getUsernameFromCKANUsername(ckanUsername);
Assert.assertTrue(gotUsername.compareTo(username)==0);
}
@Test
public void list() throws Exception {
CKANUser ckanUser = getCKANUser();

View File

@ -1,21 +1,43 @@
package org.gcube.gcat.persistence.ckan;
package org.gcube.gcat.persistence.grsf;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.UriInfo;
import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore;
import org.gcube.com.fasterxml.jackson.core.JsonProcessingException;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.gcat.ContextTest;
import org.gcube.gcat.api.GCatConstants;
import org.gcube.gcat.persistence.ckan.CKANGroup;
import org.gcube.gcat.persistence.ckan.CKANGroupTest;
import org.gcube.gcat.persistence.ckan.CKANOrganization;
import org.gcube.gcat.persistence.ckan.CKANOrganizationTest;
import org.gcube.gcat.persistence.ckan.CKANPackage;
import org.gcube.gcat.persistence.ckan.CKANPackageTest;
import org.gcube.gcat.persistence.ckan.CKANUser;
import org.gcube.gcat.persistence.ckan.CKANUtility;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -25,7 +47,7 @@ import org.slf4j.LoggerFactory;
*/
public class GRSFUtilities extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(CKANGroupTest.class);
private static Logger logger = LoggerFactory.getLogger(GRSFUtilities.class);
public static final String PRODUCTION_GRSF = "/d4science.research-infrastructures.eu/FARM/GRSF";
public static final String PRODUCTION_GRSF_ADMIN = "/d4science.research-infrastructures.eu/FARM/GRSF_Admin";
@ -33,17 +55,14 @@ public class GRSFUtilities extends ContextTest {
public static final String PREPROD_GRSF_PRE = "/pred4s/preprod/GRSF_Pre";
public GRSFUtilities() throws ObjectNotFound, Exception {
ContextTest.setContextByName(PREPROD_GRSF_PRE);
}
@Test
public void test() throws ObjectNotFound, Exception {
ContextTest.setContextByName(PREPROD_GRSF_PRE);
//
@Before
public void before() throws Exception {
// ContextTest.setContextByName(PREPROD_GRSF_PRE);
// ContextTest.setContextByName(PRODUCTION_GRSF);
// ContextTest.setContextByName(PRODUCTION_GRSF_ADMIN);
// ContextTest.setContextByName(PRODUCTION_GRSF_PRE);
// logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
// logger.debug(SecretManagerProvider.instance.get().getContext());
}
private void create(Set<String> createdGroup, Map<String, String> groups, String name) throws JsonProcessingException, IOException {
@ -157,28 +176,33 @@ public class GRSFUtilities extends ContextTest {
node.put("name", getGroupId(name));
String json = objectMapper.writeValueAsString(node);
logger.info(json);
// ckanGroupToCreate.create(json);
ckanGroupToCreate.create(json);
}
}
// @Test
public void deleteNewGRSFGroups() throws ObjectNotFound, Exception {
String sysAdminAPI = CKANUtility.getSysAdminAPI();
String[] groupNames = new String[] {
"GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
"Catch", "Landing",
"Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
"Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
"FAO Stock Status Category", "Scientific Advice",
"GRSF SDG Flag", "GRSF Traceability Flag"
};
// String[] groupNames = new String[] {
// "GRSF", "Legacy", "Fishery", "Stock", "FIRMS", "FishSource", "RAM",
// "Catch", "Landing",
// "Abundance Level", "Abundance Level (FIRMS Standard)", "Biomass",
// "Fishing Pressure", "Fishing Pressure (FIRMS Standard)", "State and Trend",
// "FAO Stock Status Category", "Scientific Advice",
// "GRSF SDG Flag", "GRSF Traceability Flag"
// };
//
List<String> groupNames = CKANGroupTest.listGroup();
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
logger.debug(SecretManagerProvider.instance.get().getContext());
for(String name : groupNames) {
CKANGroup ckanGroupToCreate = new CKANGroup();
ckanGroupToCreate.setApiKey(sysAdminAPI);
ckanGroupToCreate.setName(getGroupId(name));
// ckanGroupToCreate.setName(getGroupId(name));
ckanGroupToCreate.setName(name);
try {
ckanGroupToCreate.purge();
// ckanGroupToCreate.purge();
}catch (Exception e) {
}
@ -188,26 +212,27 @@ public class GRSFUtilities extends ContextTest {
// @Test
public void associateUserToAllCKANGroupsAndOrganization() throws ObjectNotFound, Exception {
String username = "grsf_publisher";
// username = "luca_frosini";
// username = "francesco_mangiacrapa";
String[] usernames = new String[] { "grsf_publisher", "luca_frosini", "francesco_mangiacrapa"};
String sysAdminAPI = CKANUtility.getSysAdminAPI();
List<String> groupNames = CKANGroupTest.listGroup();
for(String groupName : groupNames) {
CKANUser ckanUser = new CKANUser();
ckanUser.setApiKey(sysAdminAPI);
ckanUser.setName(username);
ckanUser.addToGroup(groupName);
for(String username : usernames) {
ckanUser.setName(username);
ckanUser.addToGroup(groupName);
}
}
List<String> orgs = CKANOrganizationTest.listOrg();
for(String org : orgs) {
CKANUser ckanUser = new CKANUser();
ckanUser.setApiKey(sysAdminAPI);
ckanUser.addUserToOrganization(org, username, "admin");
for(String username : usernames) {
ckanUser.addUserToOrganization(org, username, "admin");
}
}
}
// @Test
@ -236,13 +261,122 @@ public class GRSFUtilities extends ContextTest {
}
protected void printLine(File file, String line) throws Exception {
try (FileWriter fw = new FileWriter(file, true);
BufferedWriter bw = new BufferedWriter(fw);
PrintWriter out = new PrintWriter(bw)) {
out.println(line);
out.flush();
} catch (IOException e) {
throw e;
}
}
public Map<String,String> getListParameter(int limit, int offset, String type, String org) {
Map<String,String> parameters = new HashMap<>();
if(limit <= 0) {
// According to CKAN documentation
// the number of matching rows to return. There is a hard limit of 1000 datasets per query.
// see https://docs.ckan.org/en/2.6/api/index.html#ckan.logic.action.get.package_search
limit = 1000;
}
parameters.put(CKANPackage.ROWS_KEY, String.valueOf(limit));
if(offset < 0) {
offset = 0;
}
parameters.put(CKANPackage.START_KEY, String.valueOf(offset));
String q = null;
if(type!=null) {
// This filter by type
StringWriter qStringWriter = new StringWriter();
qStringWriter.append("extras_systemtype:");
qStringWriter.append(type);
q = qStringWriter.toString();
}
if(org!=null) {
StringWriter qStringWriter = new StringWriter();
qStringWriter.append("organization:");
qStringWriter.append(org);
if(q!=null) {
q = q + " AND " + qStringWriter.toString();
}
}
if(q!=null) {
q = q + " AND (StatusoftheRecord:Approved OR StatusoftheRecord:Archived)";
// q = q + " AND (groups:stock-group)";
}
if(q!=null) {
parameters.put(GCatConstants.Q_KEY, q);
}
return parameters;
}
// @Test
public void list() throws Exception {
ObjectMapper mapper = new ObjectMapper();
int limit = 100;
// String[] contexts = new String[] {PRODUCTION_GRSF, PRODUCTION_GRSF_ADMIN};
String[] contexts = new String[] {PRODUCTION_GRSF};
for(String context : contexts) {
ContextTest.setContextByName(context);
logger.debug(SecretManagerProvider.instance.get().getUser().getUsername());
logger.debug(SecretManagerProvider.instance.get().getContext());
CKANPackage ckanPackage = new CKANPackage();
SortedSet<String> sortedSet = new TreeSet<>();
String[] types = new String[] {"Marine Resource", "Assessment Unit"};
for(String type : types) {
int offset = 0;
boolean go = true;
ScopeBean scopeBean = new ScopeBean(context);
while(go) {
Map<String,String> parameters = getListParameter(limit, offset==0 ? 0 : (offset*limit), type, scopeBean.name().toLowerCase());
String ret = ckanPackage.list(parameters);
JsonNode gotList = mapper.readTree(ret);
Assert.assertTrue(gotList instanceof ArrayNode);
ArrayNode itemsArray = (ArrayNode) gotList;
if(itemsArray.size()>0) {
for(JsonNode jsonNode : itemsArray) {
String name = jsonNode.asText();
sortedSet.add(name);
}
offset++;
}else {
go=false;
}
}
File file = new File(scopeBean.name() + "-" + type.replace(" ", "_")+".txt");
if(file.exists()) {
file.delete();
}
for(String name : sortedSet) {
printLine(file, name);
}
}
}
}
private void purgeGRSFRecords(int limit, int offset) throws Exception {
logger.debug("Going to purge {} records, starting from {}", limit, limit*offset);
CKANPackage ckanPackage = new CKANPackage();
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
UriInfo uriInfo = CKANPackageTest.getUriInfo(mvm);
ckanPackage.setUriInfo(uriInfo);
ObjectMapper mapper = new ObjectMapper();
boolean go = true;
while(go) {
String ret = ckanPackage.list(limit, offset);
Map<String,String> parameters = getListParameter(limit, offset, null, null);
String ret = ckanPackage.list(parameters);
JsonNode gotList = mapper.readTree(ret);
Assert.assertTrue(gotList instanceof ArrayNode);
ArrayNode itemsArray = (ArrayNode) gotList;
@ -253,21 +387,17 @@ public class GRSFUtilities extends ContextTest {
ckanPackage.setName(name);
try {
ckanPackage.purgeNoCheckNoDeleteFiles();
if(go) {
break;
}
}catch (Exception e) {
// TODO: handle exception
logger.error("Unable to purge record with name {}", name, e);
}
// Thread.sleep(500);
}
Thread.sleep(500);
}else {
go=false;
}
}
}
@JsonIgnore
// @Test
public void purgeAllGRSFRecords() throws Exception {
purgeGRSFRecords(100, 0);

View File

@ -11,6 +11,8 @@
<logger name="org.gcube" level="ERROR" />
<logger name="org.gcube.gcat" level="TRACE" />
<logger name="org.gcube.gcat.persistence.grsf" level="TRACE" />
<logger name="org.gcube.social_networking.social_networking_client_library.utils" level="TRACE" />
<root level="WARN">
<appender-ref ref="STDOUT" />