ckan-util-library/src/main/java/org/gcube/datacatalogue/ckanutillibrary/CKanUtilsImpl.java

759 lines
24 KiB
Java
Raw Normal View History

package org.gcube.datacatalogue.ckanutillibrary;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import net.htmlparser.jericho.Renderer;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;
import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datacatalogue.ckanutillibrary.models.CKanUserWrapper;
import org.gcube.datacatalogue.ckanutillibrary.models.ResourceBean;
import org.gcube.datacatalogue.ckanutillibrary.models.RolesIntoOrganization;
import org.gcube.datacatalogue.ckanutillibrary.models.State;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.CheckedCkanClient;
import eu.trentorise.opendata.jackan.CkanClient;
import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpResponse;
import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpStatus;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.methods.HttpPost;
import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.StringEntity;
import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.CloseableHttpClient;
import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.HttpClientBuilder;
import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanLicense;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanPair;
import eu.trentorise.opendata.jackan.model.CkanResource;
import eu.trentorise.opendata.jackan.model.CkanTag;
import eu.trentorise.opendata.jackan.model.CkanUser;
/**
* This is the Ckan Utils implementation class.
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
*/
public class CKanUtilsImpl implements CKanUtilsInterface{
private static final Logger logger = LoggerFactory.getLogger(CKanUtilsImpl.class);
private String CKAN_CATALOGUE_URL;
private String CKAN_DB_NAME;
private String CKAN_DB_USER;
private String CKAN_DB_PASSWORD;
private String CKAN_DB_URL;
private Integer CKAN_DB_PORT;
// Connection to the db
private Connection connection;
public CKanUtilsImpl(String scope) throws Exception{
String currentScope = ScopeProvider.instance.get();
try{
ScopeProvider.instance.set(scope);
CKanRunningCluster runningInstance = new CKanRunningCluster(scope);
CKAN_DB_URL = runningInstance.getDatabaseHosts().get(0);
CKAN_DB_NAME = runningInstance.getDataBaseName();
CKAN_DB_USER = runningInstance.getDataBaseUser();
CKAN_DB_PASSWORD = StringEncrypter.getEncrypter().decrypt(runningInstance.getDataBasePassword());
logger.debug("Plain password first 3 chars are " + CKAN_DB_PASSWORD.substring(0, 3));
CKAN_DB_PORT = runningInstance.getDatabasePorts().get(0);
CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0);
// create db connection
Class.forName("org.postgresql.Driver");
connection = DriverManager.getConnection(
"jdbc:postgresql://" + CKAN_DB_URL + ":" + CKAN_DB_PORT + "/" + CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD);
}catch(Exception e){
logger.error("Error while trying to connect to ckan database/catalogue ", e);
}finally{
// set the scope back
ScopeProvider.instance.set(currentScope);
}
}
@Override
public String getApiKeyFromUser(String username) {
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
logger.debug("Request api key for user = " + username);
String apiToReturn = null;
try{
String query = "SELECT \"apikey\" FROM \"user\" WHERE \"name\"=? and \"state\"=?;";
PreparedStatement preparedStatement = connection.prepareStatement(query);
preparedStatement.setString(1, ckanUsername);
preparedStatement.setString(2, State.ACTIVE.toString());
ResultSet rs = preparedStatement.executeQuery();
while (rs.next()) {
apiToReturn = rs.getString("apikey");
break;
}
}catch(Exception e){
logger.error("Unable to retrieve key for user " + ckanUsername, e);
}
logger.debug("Api key retrieved for user " + ckanUsername);
return apiToReturn;
}
@Override
public CKanUserWrapper getUserFromApiKey(String apiKey) {
logger.debug("Request user whose api key is = " + apiKey);
CKanUserWrapper user = new CKanUserWrapper();
try{
String query = "SELECT * FROM \"user\" WHERE \"apikey\"=? and \"state\"=?;";
PreparedStatement preparedStatement = connection.prepareStatement(query);
preparedStatement.setString(1, apiKey);
preparedStatement.setString(2, State.ACTIVE.toString());
ResultSet rs = preparedStatement.executeQuery();
while (rs.next()) {
// check if it is active
if(State.DELETED.equals(rs.getString("state")))
break;
user.setId(rs.getString("id"));
user.setName(rs.getString("name"));
user.setApiKey(apiKey);
user.setCreationTimestamp(rs.getTimestamp("created").getTime());
user.setAbout(rs.getString("about"));
user.setOpenId(rs.getString("openid"));
user.setFullName(rs.getString("fullname"));
user.setEmail(rs.getString("email"));
user.setAdmin(rs.getBoolean("sysadmin"));
break;
}
}catch(Exception e){
logger.error("Unable to retrieve user with api key " + apiKey, e);
}
logger.debug("User retrieved");
return user;
}
@Override
public List<CkanOrganization> getOrganizationsByUser(String username) {
logger.debug("Requested organizations for user " + username);
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
List<String> organizationIds = getOrganizationsIds();
String userId = getUserIdByUsername(ckanUsername);
// list to return
List<CkanOrganization> toReturn = new ArrayList<CkanOrganization>();
// get the CkanClient to retrieve the organization from the id
CkanClient client = new CkanClient(CKAN_CATALOGUE_URL);
try{
// for each org id, check if the user is included
for (String orgId : organizationIds) {
String query = "SELECT * FROM \"member\" WHERE \"table_id\"=? and \"group_id\"=? and \"table_name\"=? and \"state\"=?;";
PreparedStatement preparedStatement = connection.prepareStatement(query);
preparedStatement.setString(1, userId);
preparedStatement.setString(2, orgId);
preparedStatement.setString(3, "user");
preparedStatement.setString(4, State.ACTIVE.toString());
ResultSet rs = preparedStatement.executeQuery();
while (rs.next()) {
// the role within the organization doesn't matter
logger.debug("User " + ckanUsername + " belongs to organization with id " + orgId);
toReturn.add(client.getOrganization(orgId));
}
}
}catch(Exception e){
logger.error("Unable to get user's organizations", e);
}
return toReturn;
}
@Override
public Map<String, List<RolesIntoOrganization>> getGroupsAndRolesByUser(
String username, List<RolesIntoOrganization> rolesToMatch) {
logger.debug("Requested roles the user " + username + " has into its organizations");
logger.debug("Roles to check are " + rolesToMatch);
Map<String, List<RolesIntoOrganization>> toReturn = new HashMap<String, List<RolesIntoOrganization>>();
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
// retrieve the user and if it is a sys_admin, for every organizations will be created in the map, add also
// the sys_admin role
boolean isSysAdmin = false;
if(rolesToMatch.contains(RolesIntoOrganization.SYSADMIN)){
isSysAdmin = isSysAdmin(ckanUsername);
}
try{
// get id from the user
String userId = getUserIdByUsername(ckanUsername);
// use the above method to require the list of user's organizations
List<CkanOrganization> usersOrganizations = getOrganizationsByUser(ckanUsername);
for (CkanOrganization ckanOrganization : usersOrganizations) {
// get the org id
String orgId = ckanOrganization.getId();
// go to the member table, that says which role has this user into the org
String query = "SELECT * FROM \"member\" WHERE \"table_id\"=? and \"group_id\"=? and \"table_name\"=? and \"state\"=?;";
PreparedStatement preparedStatement = connection.prepareStatement(query);
preparedStatement.setString(1, userId);
preparedStatement.setString(2, orgId);
preparedStatement.setString(3, "user");
preparedStatement.setString(4, "active");
ResultSet rs = preparedStatement.executeQuery();
// prepare the data to put into the hashmap
List<RolesIntoOrganization> rolesIntoOrg = new ArrayList<RolesIntoOrganization>();
if(isSysAdmin)
rolesIntoOrg.add(RolesIntoOrganization.SYSADMIN);
while(rs.next()){
// check
String role = rs.getString("capacity");
if(rolesToMatch.contains(RolesIntoOrganization.valueOf(role))){
rolesIntoOrg.add(RolesIntoOrganization.valueOf(role));
logger.debug("User " + ckanUsername + " has role " + role + " into organization " + ckanOrganization.getName());
}
}
if(!rolesIntoOrg.isEmpty())
toReturn.put(orgId, rolesIntoOrg);
}
}catch(Exception e){
logger.error("Unable to analyze user's roles", e);
}
return toReturn;
}
/**
* Returns the user id given his username
* @param username
* @return the id on success, null otherwise
*/
private String getUserIdByUsername(String username) {
logger.debug("Request user id whose username is = " + username);
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
String userId = null;
try{
String query = "SELECT \"id\" FROM \"user\" WHERE \"name\"=? and \"state\"=?;";
PreparedStatement preparedStatement = connection.prepareStatement(query);
preparedStatement.setString(1, ckanUsername);
preparedStatement.setString(2, State.ACTIVE.toString());
ResultSet rs = preparedStatement.executeQuery();
while (rs.next()) {
userId = rs.getString("id");
break;
}
}catch(Exception e){
logger.error("Unable to retrieve user with name " + ckanUsername, e);
}
logger.debug("User id retrieved");
return userId;
}
/**
* Retrieve the list of organizations ids
* @return
*/
private List<String> getOrganizationsIds(){
logger.debug("Request organization ids");
List<String> toReturn = new ArrayList<String>();
try{
String query = "SELECT \"id\" FROM \"group\" WHERE \"is_organization\"=? and \"state\"=?;";
PreparedStatement preparedStatement = connection.prepareStatement(query);
preparedStatement.setBoolean(1, true);
preparedStatement.setString(2, State.ACTIVE.toString());
ResultSet rs = preparedStatement.executeQuery();
while (rs.next()) {
toReturn.add(rs.getString("id"));
}
logger.debug("Organizations' ids retrieved");
}catch(Exception e){
logger.error("Unable to retrieve list of organization ids", e);
}
return toReturn;
}
@Override
public String getCKANDBUrl() {
return CKAN_DB_URL;
}
@Override
public String getCatalogueUrl() {
return CKAN_CATALOGUE_URL;
}
@Override
public List<String> getOrganizationsNamesByUser(String username) {
logger.debug("Requested organizations for user " + username);
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
List<CkanOrganization> orgs = getOrganizationsByUser(ckanUsername);
List<String> orgsName = new ArrayList<String>();
for (CkanOrganization ckanOrganization : orgs) {
orgsName.add(ckanOrganization.getName());
logger.debug("Organization name is " + ckanOrganization.getName());
}
return orgsName;
}
@Override
public String findLicenseIdByLicense(String chosenLicense) {
logger.debug("Requested license id");
String ckanPortalUrl = getCatalogueUrl();
CkanClient client = new CkanClient(ckanPortalUrl);
//retrieve the list of available licenses
List<CkanLicense> licenses = client.getLicenseList();
for (CkanLicense ckanLicense : licenses) {
if(ckanLicense.getTitle().equals(chosenLicense))
return ckanLicense.getId();
}
return null;
}
@Override
public List<String> getLicenseTitles() {
logger.debug("Request for CKAN licenses");
// get the url and the api key of the user
String ckanPortalUrl = getCatalogueUrl();
List<String> result = new ArrayList<String>();
CkanClient client = new CkanClient(ckanPortalUrl);
//retrieve the list of available licenses
List<CkanLicense> licenses = client.getLicenseList();
for (CkanLicense ckanLicense : licenses) {
result.add(ckanLicense.getTitle());
logger.debug("License is " + ckanLicense.getTitle() + " and id " + ckanLicense.getId());
}
return result;
}
@Override
public boolean setDatasetPrivate(boolean priv, String organizationId,
String datasetId, String username) {
String pathSetPrivate = "/api/3/action/bulk_update_private";
String pathSetPublic = "/api/3/action/bulk_update_public";
String token = null;
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
if(ckanUsername == null || ckanUsername.isEmpty()){
logger.error("The owner parameter is mandatory");
return false;
}else{
token = getApiKeyFromUser(ckanUsername);
if(token == null){
logger.error("Unable to retrieve user's token");
return false;
}
}
// Request parameters to be replaced
String parameter = "{"
+ "\"org_id\":\"ORGANIZATION_ID\","
+ "\"datasets\":[\"DATASET_ID\"]"
+ "}";
if(organizationId != null && !organizationId.isEmpty() && datasetId != null && !datasetId.isEmpty()){
// replace with right data
parameter = parameter.replace("ORGANIZATION_ID", organizationId);
parameter = parameter.replace("DATASET_ID", datasetId);
CloseableHttpClient httpClient = HttpClientBuilder.create().build();
if(priv){
try {
HttpPost request = new HttpPost(CKAN_CATALOGUE_URL + pathSetPrivate);
request.addHeader("Authorization", token);
StringEntity params = new StringEntity(parameter);
request.setEntity(params);
HttpResponse response = httpClient.execute(request);
logger.debug("[PRIVATE]Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
return true;
}catch (Exception ex) {
logger.error("Error while trying to set private the dataset ", ex);
}
}else
{
try {
HttpPost request = new HttpPost(CKAN_CATALOGUE_URL + pathSetPublic);
StringEntity params =new StringEntity(parameter);
request.addHeader("Authorization", token);
request.setEntity(params);
HttpResponse response = httpClient.execute(request);
logger.debug("[PUBLIC]Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
return true;
}catch (Exception ex) {
logger.error("Error while trying to set public the dataset ", ex);
}
}
}
return false;
}
/**
* Utility method to check if a something at this url actually exists
* @param URLName
* @return
*/
private static boolean resourceExists(String URLName){
try {
HttpURLConnection.setFollowRedirects(true);
HttpURLConnection con = (HttpURLConnection) new URL(URLName).openConnection();
con.setRequestMethod("HEAD");
logger.debug("Return code is " + con.getResponseCode());
return (con.getResponseCode() == HttpURLConnection.HTTP_OK);
}
catch (Exception e) {
logger.error("Exception while checking url", e);
return false;
}
}
@Override
public String addResourceToDataset(ResourceBean resourceBean) {
logger.debug("Request to add a resource described by this bean " + resourceBean);
try{
if(resourceExists(resourceBean.getUrl())){
// retrieve ckan's catalog url
String ckanPortalUrl = getCatalogueUrl();
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(resourceBean.getOwner());
// retrieve the api key for this user
String apiKey = getApiKeyFromUser(ckanUsername);
CkanResource resource = new CkanResource(ckanPortalUrl, resourceBean.getDatasetId());
resource.setName(resourceBean.getName());
// escape description
Source description = new Source(resourceBean.getDescription());
Segment htmlSeg = new Segment(description, 0, description.length());
Renderer htmlRend = new Renderer(htmlSeg);
resource.setDescription(htmlRend.toString());
resource.setUrl(resourceBean.getUrl());
resource.setOwner(ckanUsername);
// Checked client
CheckedCkanClient client = new CheckedCkanClient(ckanPortalUrl, apiKey);
CkanResource createdRes = client.createResource(resource);
if(createdRes != null){
logger.debug("Resource " + createdRes.getName() + " is now available");
return createdRes.getId();
}
}else
logger.error("There is no resource at this url " + resourceBean.getUrl());
}catch(Exception e){
logger.error("Unable to create the resource described by the bean " + resourceBean, e);
}
return null;
}
@Override
public boolean deleteResourceFromDataset(String username, String resourceId) {
logger.error("Request to delete a resource with id " + resourceId + " coming by user " + username);
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
try{
CheckedCkanClient client = new CheckedCkanClient(getCatalogueUrl(), getApiKeyFromUser(ckanUsername));
client.deleteResource(resourceId);
return true;
}catch(Exception e){
logger.error("Unable to delete resource whose id is " + resourceId, e);
}
return false;
}
/**
* Generate the catalogue's dataset name from its title
* @param title
* @return
*/
private String nameFromTitle(String title) {
String convertedName = title.replaceAll(" ", "_");
convertedName = convertedName.replaceAll("\\.", "_");
convertedName = convertedName.toLowerCase();
if(convertedName.endsWith("_"))
convertedName = convertedName.substring(0, convertedName.length() - 2);
return convertedName;
}
@Override
public String createCKanDataset(String username, String withId,
String title, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail,
long version, String description, String licenseId,
List<String> tags, Map<String, String> customFields,
List<ResourceBean> resources, boolean setPublic) {
logger.debug("Request for dataset creation");
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
CheckedCkanClient client = new CheckedCkanClient(getCatalogueUrl(), getApiKeyFromUser(ckanUsername));
// create the base dataset and fill it
CkanDataset dataset = new CkanDataset();
// set values
dataset.setId(withId);
// get the name from the title
dataset.setName(nameFromTitle(title));
dataset.setTitle(title);
CkanOrganization orgOwner = client.getOrganization(organizationNameOrId);
dataset.setOwnerOrg(orgOwner.getId());
dataset.setAuthor(author);
dataset.setAuthorEmail(authorMail);
dataset.setMaintainer(maintainer);
dataset.setMaintainerEmail(maintainerMail);
dataset.setVersion(String.valueOf(version));
// description must be escaped
Source descriptionEscaped = new Source(description);
Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
Renderer htmlRend = new Renderer(htmlSeg);
dataset.setNotes(htmlRend.toString());
logger.debug("Description (escaped is ) " + htmlRend.toString());
dataset.setLicenseId(licenseId);
// set the tags, if any
if(tags != null && !tags.isEmpty()){
// convert to ckan tags
List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
for (String stringTag : tags) {
ckanTags.add(new CkanTag(stringTag));
}
dataset.setTags(ckanTags);
}
// set the custom fields, if any
if(customFields != null && !customFields.isEmpty()){
// iterate and create
Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
List<CkanPair> extras = new ArrayList<CkanPair>(customFields.entrySet().size());
while (iterator.hasNext()) {
Map.Entry<String, String> entry = (Map.Entry<String, String>) iterator.next();
extras.add(new CkanPair(entry.getKey(), entry.getValue()));
}
dataset.setExtras(extras);
}
// check if we need to add the resources
if(resources != null && !resources.isEmpty()){
logger.debug("We need to add resources to the dataset");
try{
List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
for(ResourceBean resource: resources){
CkanResource newResource = new CkanResource();
newResource.setDescription(resource.getDescription());
newResource.setId(resource.getId());
newResource.setUrl(resource.getUrl());
newResource.setName(resource.getName());
newResource.setMimetype(resource.getMimeType());
newResource.setOwner(ckanUsername);
resourcesCkan.add(newResource);
}
// add to the dataset
dataset.setResources(resourcesCkan);
}catch(Exception e){
logger.error("Unable to add those resources to the dataset", e);
}
}
// try to create
CkanDataset res = null;
try{
res = client.createDataset(dataset);
if(res != null){
logger.debug("Dataset with name " + res.getName() + " has been created. Setting visibility");
// set visibility
setDatasetPrivate(
!setPublic, // swap to private
res.getOrganization().getId(),
res.getId(),
ckanUsername);
return res.getId();
}
}catch(Exception e){
// try to update
logger.error("Error while creating the dataset, probably it already exists.", e);
}
return null;
}
@Override
public String getUrlFromDatasetIdOrName(String username, String datasetIdOrName) {
logger.debug("Request coming for dataset url of dataset with name/id " + datasetIdOrName);
String ckanUsername = fromOwnerToCKanOwner(username);
// the url of the dataset looks like "getCatalogueUrl() + /dataset/ + dataset name"
try{
CheckedCkanClient client = new CheckedCkanClient(getCatalogueUrl(), getApiKeyFromUser(ckanUsername));
CkanDataset dataset = client.getDataset(datasetIdOrName);
if(dataset != null){
return getCatalogueUrl() + "/dataset/" + dataset.getName();
}
}catch(Exception e){
logger.error("Error while retrieving dataset with id/name=" + datasetIdOrName, e);
}
return null;
}
/**
* Ckan username has _ instead of . (that is, costantino.perciante -> costantino_perciante)
* @param owner
* @return
*/
private static String fromOwnerToCKanOwner(String owner){
return owner.replaceAll("\\.", "_");
}
@Override
public boolean checkRole(String username, String organizationName,
RolesIntoOrganization correspondentRoleToCheck) {
// TODO it must be defined
return true;
}
@Override
public boolean isSysAdmin(String username) {
// in order to avoid errors, the username is always converted
String ckanUsername = fromOwnerToCKanOwner(username);
CheckedCkanClient checkedClient = new CheckedCkanClient(getCatalogueUrl(), getApiKeyFromUser(ckanUsername));
CkanUser user = checkedClient.getUser(getUserIdByUsername(ckanUsername));
return user.isSysadmin();
}
@Override
protected void finalize() throws Throwable {
super.finalize();
logger.debug("Closing connection on finalize()");
connection.close();
}
}