gCubeSystem
/
gcat
16
0
Fork 0
Code Issues Pull Requests Releases 13 Wiki Activity

Adding CKAN DB in configuration

This commit is contained in:
Luca Frosini 2022-02-16 22:34:30 +01:00
parent bd22fb48c3
commit b66b1fcd56
11 changed files with 161 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pom.xml
View File

@ -47,6 +47,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-encryption</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope</artifactId>

3
src/main/java/org/gcube/gcat/persistence/ckan/CatalogueConfigurationFactory.java → src/main/java/org/gcube/gcat/configuration/CatalogueConfigurationFactory.java
View File

@ -1,11 +1,10 @@
package org.gcube.gcat.persistence.ckan;
package org.gcube.gcat.configuration;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.gcat.configuration.GCoreISConfigurationProxy;
/**
* @author Luca Frosini (ISTI - CNR)

139
src/main/java/org/gcube/gcat/configuration/GCoreISConfigurationProxy.java
View File

@ -26,6 +26,7 @@ import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.resources.gcore.ServiceEndpoint.Runtime;
import org.gcube.common.resources.gcore.common.Platform;
import org.gcube.common.resources.gcore.utils.Group;
import org.gcube.gcat.api.configuration.CKANDB;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.informationsystem.publisher.RegistryPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
@ -40,20 +41,27 @@ import org.gcube.smartgears.context.container.ContainerContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/
public class GCoreISConfigurationProxy {
private static final Logger logger = LoggerFactory.getLogger(GCoreISConfigurationProxy.class);
// property to retrieve the master service endpoint into the /root scope
private final static String IS_ROOT_MASTER_PROPERTY_KEY = "IS_ROOT_MASTER"; // true, false.. missing means false as
public final static String IS_ROOT_MASTER_PROPERTY_KEY = "IS_ROOT_MASTER"; // true, false.. missing means false as
private final static String DEFAULT_ORGANIZATION_PROPERTY_KEY = "DEFAULT_ORGANIZATION";
private final static String SUPPORTED_ORGANIZATION_PROPERTY_KEY = "SUPPORTED_ORGANIZATION";
private final static String API_KEY_PROPERTY_KEY = "API_KEY";
private final static String SOLR_INDEX_ADDRESS_PROPERTY_KEY = "SOLR_INDEX_ADDRESS";
private final static String SOCIAL_POST_PROPERTY_KEY = "SOCIAL_POST";
private final static String ALERT_USERS_ON_POST_CREATION_PROPERTY_KEY = "ALERT_USERS_ON_POST_CREATION";
private final static String MODERATION_ENABLED_KEY_PROPERTY_KEY = "MODERATION_ENABLED";
public final static String DEFAULT_ORGANIZATION_PROPERTY_KEY = "DEFAULT_ORGANIZATION";
public final static String SUPPORTED_ORGANIZATION_PROPERTY_KEY = "SUPPORTED_ORGANIZATION";
public final static String API_KEY_PROPERTY_KEY = "API_KEY";
public final static String SOLR_INDEX_ADDRESS_PROPERTY_KEY = "SOLR_INDEX_ADDRESS";
public final static String SOCIAL_POST_PROPERTY_KEY = "SOCIAL_POST";
public final static String ALERT_USERS_ON_POST_CREATION_PROPERTY_KEY = "ALERT_USERS_ON_POST_CREATION";
public final static String MODERATION_ENABLED_KEY_PROPERTY_KEY = "MODERATION_ENABLED";
public final static String CKAN_DB_URL_PROPERTY_KEY = "CKAN_DB_URL";
public final static String CKAN_DB_USERNAME_PROPERTY_KEY = "CKAN_DB_USERNAME";
public final static String CKAN_DB_PASSWORD_PROPERTY_KEY = "CKAN_DB_PASSWORD";
// CKAN Instance info
private final static String CATEGORY = "Application";
@ -117,9 +125,8 @@ public class GCoreISConfigurationProxy {
Map<String, Property> propertyMap = accessPoint.propertyMap();
// retrieve sys admin token
String sysAdminToken = propertyMap.get(API_KEY_PROPERTY_KEY).value();
sysAdminToken = StringEncrypter.getEncrypter().decrypt(sysAdminToken);
catalogueConfiguration.setSysAdminToken(sysAdminToken);
String encryptedSysAdminToken = propertyMap.get(API_KEY_PROPERTY_KEY).value();
catalogueConfiguration.setSysAdminToken(StringEncrypter.getEncrypter().decrypt(encryptedSysAdminToken));
String defaultOrganization = null;
@ -194,6 +201,24 @@ public class GCoreISConfigurationProxy {
catalogueConfiguration.setDefaultOrganization(defaultOrganization);
}
}
CKANDB ckanDB = null;
if (propertyMap.containsKey(CKAN_DB_URL_PROPERTY_KEY)) {
String ckanDBURL = propertyMap.get(CKAN_DB_URL_PROPERTY_KEY).value().trim();
ckanDB = new CKANDB();
ckanDB.setUrl(ckanDBURL);
String ckanDBUsername = propertyMap.get(CKAN_DB_USERNAME_PROPERTY_KEY).value().trim();
ckanDB.setUsername(ckanDBUsername);
// Password is encrypted
String ckanDBPassword = propertyMap.get(CKAN_DB_PASSWORD_PROPERTY_KEY).value().trim();
ckanDB.setPassword(StringEncrypter.getEncrypter().decrypt(ckanDBPassword));
}else {
mustBeUpdated = true;
ckanDB = getCKANDBFromIS();
}
catalogueConfiguration.setCkanDB(ckanDB);
if(mustBeUpdated) {
logger.warn("The ServiceEndpoint with ID {} in context {} should be updated", serviceEndpoint.id(), context);
@ -270,6 +295,93 @@ public class GCoreISConfigurationProxy {
return serviceEndpoint;
}
// CKAN Instance info
private final static String CKAN_DB_SERVICE_ENDPOINT_CATEGORY= "Database";
private final static String CKAN_DB_SERVICE_ENDPOINT_NAME = "CKanDatabase";
/**
* Retrieve endpoints information from IS for DataCatalogue URL
* @return list of endpoints for ckan data catalogue
* @throws Exception
*/
protected List<ServiceEndpoint> getCKANDBServiceEndpoints() {
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + CKAN_DB_SERVICE_ENDPOINT_CATEGORY + "'");
query.addCondition("$resource/Profile/Name/text() eq '" + CKAN_DB_SERVICE_ENDPOINT_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> serviceEndpoints = client.submit(query);
if(serviceEndpoints.size() == 0) {
logger.error("There is no {} having category {} and name {} in this context.",
ServiceEndpoint.class.getSimpleName(), CKAN_DB_SERVICE_ENDPOINT_CATEGORY, CKAN_DB_SERVICE_ENDPOINT_NAME);
throw new InternalServerErrorException("No CKAN configuration on IS");
}
return serviceEndpoints;
}
protected CKANDB getCKANDBFromIS() {
try {
List<ServiceEndpoint> serviceEndpoints = getCKANDBServiceEndpoints();
ServiceEndpoint serviceEndpoint = null;
if(serviceEndpoints.size() > 1) {
logger.info("Too many {} having category {} and name {} in this context. Looking for the one that has the property {}",
ServiceEndpoint.class.getSimpleName(), CKAN_DB_SERVICE_ENDPOINT_CATEGORY,
CKAN_DB_SERVICE_ENDPOINT_NAME);
for(ServiceEndpoint se : serviceEndpoints) {
Iterator<AccessPoint> accessPointIterator = se.profile().accessPoints().iterator();
while(accessPointIterator.hasNext()) {
ServiceEndpoint.AccessPoint accessPoint = accessPointIterator.next();
// get the is master property
Property entry = accessPoint.propertyMap().get(IS_ROOT_MASTER_PROPERTY_KEY);
String isMaster = entry != null ? entry.value() : null;
if(isMaster == null || !isMaster.equals("true")) {
continue;
}
// set this variable
serviceEndpoint = se;
break;
}
}
// if none of them was master, throw an exception
if(serviceEndpoint == null) {
throw new InternalServerErrorException(
"Too many CKAN configuration on IS and no one with MASTER property");
}
} else {
serviceEndpoint = serviceEndpoints.get(0);
}
Iterator<AccessPoint> accessPointIterator = serviceEndpoint.profile().accessPoints().iterator();
while(accessPointIterator.hasNext()) {
AccessPoint accessPoint = accessPointIterator.next();
String host = accessPoint.address();
String db = accessPoint.name();
CKANDB ckanDB = new CKANDB();
String url = String.format("jdbc:postgresql://%s/%s", host, db);
ckanDB.setUrl(url);
ckanDB.setUsername(accessPoint.username());
ckanDB.setPassword(StringEncrypter.getEncrypter().decrypt(accessPoint.password()));
return ckanDB;
}
return null;
} catch(WebApplicationException e) {
throw e;
} catch(Exception e) {
throw new InternalServerErrorException("Error while getting configuration on IS", e);
}
}
public static final String GENERIC_RESOURCE_SECONDARY_TYPE_FOR_ORGANIZATIONS = "ApplicationProfile";
public static final String GENERIC_RESOURCE_NAME_FOR_ORGANIZATIONS = "Supported CKAN Organizations";
@ -503,9 +615,8 @@ public class GCoreISConfigurationProxy {
catalogueConfiguration.setCkanURL(ckanURL);
// retrieve sys admin token
String sysAdminToken = propertyMap.get(API_KEY_PROPERTY_KEY).value();
sysAdminToken = StringEncrypter.getEncrypter().decrypt(sysAdminToken);
catalogueConfiguration.setSysAdminToken(sysAdminToken);
String encryptedSysAdminToken = propertyMap.get(API_KEY_PROPERTY_KEY).value();
catalogueConfiguration.setSysAdminToken(StringEncrypter.getEncrypter().decrypt(encryptedSysAdminToken));
}
rootMaster = isRootMaster(serviceEndpoint);

1
src/main/java/org/gcube/gcat/persistence/ckan/CKAN.java
View File

@ -22,6 +22,7 @@ import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.NullNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
import org.gcube.gcat.utils.HTTPUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

1
src/main/java/org/gcube/gcat/persistence/ckan/CKANPackage.java
View File

@ -30,6 +30,7 @@ import org.gcube.gcat.api.moderation.CMItemVisibility;
import org.gcube.gcat.api.moderation.Moderated;
import org.gcube.gcat.api.moderation.ModerationContent;
import org.gcube.gcat.api.roles.Role;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
import org.gcube.gcat.moderation.thread.ModerationThread;
import org.gcube.gcat.oldutils.Validator;
import org.gcube.gcat.profile.MetadataUtility;

110
src/main/java/org/gcube/gcat/persistence/ckan/CKANPackageTrash.java
View File

@ -1,32 +1,22 @@
package org.gcube.gcat.persistence.ckan;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.WebApplicationException;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.encryption.encrypter.StringEncrypter;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.gcat.api.configuration.CKANDB;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.gcat.api.roles.Role;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
import org.postgresql.core.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -35,14 +25,6 @@ public class CKANPackageTrash {
protected static final Logger logger = LoggerFactory.getLogger(CKANPackageTrash.class);
// CKAN Instance info
private final static String SERVICE_ENDPOINT_CATEGORY= "Database";
private final static String SERVICE_ENDPOINT_NAME = "CKanDatabase";
// property to retrieve the master service endpoint into the /root scope
private final static String IS_MASTER_ROOT_KEY_PROPERTY = "IS_ROOT_MASTER"; // true, false.. missing means false as well
private static final String GROUP_TABLE_KEY = "group";
private static final String GROUP_ID_KEY = "id";
private static final String GROUP_NAME_KEY = "name";
@ -67,10 +49,6 @@ public class CKANPackageTrash {
protected boolean ownOnly;
private String url;
private String username;
private String password;
public CKANPackageTrash() {
mapper = new ObjectMapper();
@ -82,8 +60,6 @@ public class CKANPackageTrash {
ckanUser.addUserToOrganization(supportedOrganization);
}
getConfigurationFromIS();
ownOnly = true;
}
@ -91,86 +67,12 @@ public class CKANPackageTrash {
this.ownOnly = ownOnly;
}
/**
* Retrieve endpoints information from IS for DataCatalogue URL
* @return list of endpoints for ckan data catalogue
* @throws Exception
*/
protected List<ServiceEndpoint> getServiceEndpoints() {
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'");
query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> serviceEndpoints = client.submit(query);
if(serviceEndpoints.size() == 0) {
logger.error("There is no {} having category {} and name {} in this context.",
ServiceEndpoint.class.getSimpleName(), SERVICE_ENDPOINT_CATEGORY, SERVICE_ENDPOINT_NAME);
throw new InternalServerErrorException("No CKAN configuration on IS");
}
return serviceEndpoints;
}
protected void getConfigurationFromIS() {
try {
List<ServiceEndpoint> serviceEndpoints = getServiceEndpoints();
ServiceEndpoint serviceEndpoint = null;
if(serviceEndpoints.size() > 1) {
logger.info("Too many {} having category {} and name {} in this context. Looking for the one that has the property {}",
ServiceEndpoint.class.getSimpleName(), SERVICE_ENDPOINT_CATEGORY,
SERVICE_ENDPOINT_NAME);
for(ServiceEndpoint se : serviceEndpoints) {
Iterator<AccessPoint> accessPointIterator = se.profile().accessPoints().iterator();
while(accessPointIterator.hasNext()) {
ServiceEndpoint.AccessPoint accessPoint = accessPointIterator.next();
// get the is master property
Property entry = accessPoint.propertyMap().get(IS_MASTER_ROOT_KEY_PROPERTY);
String isMaster = entry != null ? entry.value() : null;
if(isMaster == null || !isMaster.equals("true")) {
continue;
}
// set this variable
serviceEndpoint = se;
break;
}
}
// if none of them was master, throw an exception
if(serviceEndpoint == null) {
throw new InternalServerErrorException(
"Too many CKAN configuration on IS and no one with MASTER property");
}
} else {
serviceEndpoint = serviceEndpoints.get(0);
}
Iterator<AccessPoint> accessPointIterator = serviceEndpoint.profile().accessPoints().iterator();
while(accessPointIterator.hasNext()) {
AccessPoint accessPoint = accessPointIterator.next();
// add this host
String host = accessPoint.address();
String db = accessPoint.name();
url = String.format("jdbc:postgresql://%s/%s", host, db);
username = accessPoint.username();
password = StringEncrypter.getEncrypter().decrypt(accessPoint.password());
}
} catch(WebApplicationException e) {
throw e;
} catch(Exception e) {
throw new InternalServerErrorException("Error while getting configuration on IS", e);
}
}
protected Connection getConnection() throws Exception {
Class.forName("org.postgresql.Driver");
CKANDB ckanDB = configuration.getCkanDB();
String url = ckanDB.getUrl();
String username = ckanDB.getUsername();
String password = ckanDB.getPassword();
Connection connection = DriverManager.getConnection(url, username, password);
logger.trace("Database {} opened successfully", url);
connection.setAutoCommit(false);

2
src/main/java/org/gcube/gcat/persistence/ckan/CKANUtility.java
View File

@ -2,6 +2,8 @@ package org.gcube.gcat.persistence.ckan;
import javax.ws.rs.InternalServerErrorException;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
/**
* @author Luca Frosini (ISTI - CNR)
*/

38
src/main/java/org/gcube/gcat/rest/Configuration.java
View File

@ -23,7 +23,7 @@ import org.gcube.gcat.ResourceInitializer;
import org.gcube.gcat.annotation.PATCH;
import org.gcube.gcat.annotation.PURGE;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.gcat.persistence.ckan.CatalogueConfigurationFactory;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -73,24 +73,6 @@ public class Configuration extends BaseREST implements org.gcube.gcat.api.interf
}
}
@PUT
@Consumes(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String createOrUpdate(String context, String json) throws WebServiceException {
try {
ObjectMapper mapper = new ObjectMapper();
CatalogueConfiguration catalogueConfiguration = mapper.readValue(json, CatalogueConfiguration.class);
checkContext(context, catalogueConfiguration);
return createOrUpdate(catalogueConfiguration);
}catch (WebServiceException e) {
throw e;
}catch (Exception e) {
throw new InternalServerErrorException(e);
}
}
@POST
@Consumes(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@ -112,7 +94,7 @@ public class Configuration extends BaseREST implements org.gcube.gcat.api.interf
throw new InternalServerErrorException(e);
}
}
@GET
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String read(String context) throws WebServiceException {
@ -141,6 +123,22 @@ public class Configuration extends BaseREST implements org.gcube.gcat.api.interf
}
}
@PUT
@Consumes(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
@Produces(ResourceInitializer.APPLICATION_JSON_CHARSET_UTF_8)
public String createOrUpdate(String context, String json) throws WebServiceException {
try {
ObjectMapper mapper = new ObjectMapper();
CatalogueConfiguration catalogueConfiguration = mapper.readValue(json, CatalogueConfiguration.class);
checkContext(context, catalogueConfiguration);
return createOrUpdate(catalogueConfiguration);
}catch (WebServiceException e) {
throw e;
}catch (Exception e) {
throw new InternalServerErrorException(e);
}
}
@Override
public String update(String json) throws WebServiceException {
try {

2
src/main/java/org/gcube/gcat/social/SocialPost.java
View File

@ -15,8 +15,8 @@ import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.socialservice.SocialService;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.gcat.persistence.ckan.CatalogueConfigurationFactory;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
import org.gcube.gcat.persistence.ckan.CKANUserCache;
import org.gcube.gcat.utils.Constants;
import org.gcube.gcat.utils.HTTPUtility;

2
src/test/java/org/gcube/gcat/configuration/GCoreISConfigurationProxyTest.java
View File

@ -25,7 +25,7 @@ public class GCoreISConfigurationProxyTest extends ContextTest {
@Test
public void testCatalogueConfiguration() throws Exception {
ContextTest.setContextByName("/gcube/devsec");
ContextTest.setContextByName("/gcube/devNext");
String context = SecretManager.instance.get().getContext();
GCoreISConfigurationProxy gCoreISConfigurationProxy = new GCoreISConfigurationProxy(context);
CatalogueConfiguration catalogueConfiguration = gCoreISConfigurationProxy.getCatalogueConfigurationFromIS();

1
src/test/java/org/gcube/gcat/persistence/ckan/CKANInstanceTest.java
View File

@ -3,6 +3,7 @@ package org.gcube.gcat.persistence.ckan;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.gcat.ContextTest;
import org.gcube.gcat.api.configuration.CatalogueConfiguration;
import org.gcube.gcat.configuration.CatalogueConfigurationFactory;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;