Merge pull request #40 from conwetlab/develop

Merge develop for release 0.3
This commit is contained in:
Francisco de la Vega 2018-01-04 13:18:02 +01:00 committed by GitHub
commit 1f9d13f3e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 360 additions and 271 deletions

23
.travis.yml Normal file
View File

@ -0,0 +1,23 @@
sudo: required
language: python
python:
- "2.7"
env:
- CKANVERSION=2.7.2 POSTGISVERSION=2
services:
- redis-server
- postgresql
addons:
firefox: "46.0"
install:
- bash bin/travis-build.bash
before_script:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- sleep 3 # give xvfb some time to start
script:
- sh bin/travis-run.sh
after_success: coveralls
branches:
only:
- develop

View File

@ -1,4 +1,4 @@
CKAN Private Datasets [![Build Status](https://build.conwet.fi.upm.es/jenkins/buildStatus/icon?job=ckan_privatedatasets-develop)](https://build.conwet.fi.upm.es/jenkins/job/ckan_privatedatasets-develop/)
CKAN Private Datasets [![Build Status](https://travis-ci.org/conwetlab/ckanext-privatedatasets.svg?branch=develop)](https://travis-ci.org/conwetlab/ckanext-privatedatasets) [![Coverage Status](https://coveralls.io/repos/github/conwetlab/ckanext-privatedatasets/badge.svg?branch=develop)](https://coveralls.io/github/conwetlab/ckanext-privatedatasets?branch=develop)
=====================
This CKAN extension allows a user to create private datasets that only certain users will be able to see. When a dataset is being created, it's possible to specify the list of users that can see this dataset. In addition, the extension provides an HTTP API that allows to add users programmatically.

View File

@ -1,107 +0,0 @@
#!/bin/bash
set -xe
trap 'jobs -p | xargs --no-run-if-empty kill' INT TERM EXIT
export PATH=$PATH:/usr/local/bin
export PIP_DOWNLOAD_CACHE=~/.pip_cache
WD=`pwd`
DB_HOST_IP=${DB_HOST_IP:=127.0.0.1}
POSTGRES_PORT=${POSTGRES_PORT:=5432}
echo "Downloading CKAN..."
git clone https://github.com/ckan/ckan
cd ckan
git checkout release-v2.5.3
cd $WD
echo "Checking Solr..."
SOLR_ACTIVE=`nc -z localhost 8983; echo $?`
if [ $SOLR_ACTIVE -ne 0 ]
then
echo "Downloading Solr..."
CACHE_DIR=~/.cache
FILE=solr-4.10.4.tgz
SOLAR_UNZIP_FOLDER=solr-4.10.4
# If the solar folder does not exist, we have to build it
if [ ! -d "$CACHE_DIR/$SOLAR_UNZIP_FOLDER" ]
then
# Download the solar installation file if it does not exist
wget --no-verbose --timestamping --directory-prefix=$CACHE_DIR http://archive.apache.org/dist/lucene/solr/4.10.4/$FILE
# Unzip the folder
tar -xf "$CACHE_DIR/$FILE" --directory "$CACHE_DIR"
# Delete the downloaded tar.gz
rm "$CACHE_DIR/$FILE"
fi
echo "Configuring and starting Solr..."
ln -s "$CACHE_DIR/$SOLAR_UNZIP_FOLDER" .
mv "$SOLAR_UNZIP_FOLDER/example/solr/collection1/conf/schema.xml" "$SOLAR_UNZIP_FOLDER/example/solr/collection1/conf/schema.xml.bak"
ln -s $WD/ckan/ckan/config/solr/schema.xml "$SOLAR_UNZIP_FOLDER/example/solr/collection1/conf/schema.xml"
cd $SOLAR_UNZIP_FOLDER/example
java -jar start.jar 2>&1 > /dev/null &
cd $WD
else
echo "Solar is already installed..."
fi
echo "Setting up virtualenv..."
virtualenv --no-site-packages virtualenv
source virtualenv/bin/activate
pip install --upgrade pip
# Force html5lib version to be used
pip install html5lib==0.9999999
echo "Installing CKAN dependencies..."
cd ckan
python setup.py develop
pip install -r requirements.txt
pip install -r dev-requirements.txt
cd ..
echo "Removing databases from old executions..."
sudo -u postgres psql -c "DROP DATABASE IF EXISTS datastore_test;"
sudo -u postgres psql -c "DROP DATABASE IF EXISTS ckan_test;"
sudo -u postgres psql -c "DROP USER IF EXISTS ckan_default;"
echo "Creating the PostgreSQL user and database..."
sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;'
echo "Modifying the configuration to setup properly the Postgres port..."
mkdir -p data/storage
echo "
sqlalchemy.url = postgresql://ckan_default:pass@$DB_HOST_IP:$POSTGRES_PORT/ckan_test
ckan.datastore.write_url = postgresql://ckan_default:pass@$DB_HOST_IP:$POSTGRES_PORT/datastore_test
ckan.datastore.read_url = postgresql://datastore_default:pass@$DB_HOST_IP:$POSTGRES_PORT/datastore_test
ckan.storage_path=data/storage" >> test.ini
echo "Initializing the database..."
sed -i "s/\(postgresql:\/\/.\+\)@localhost\(:[0-9]\+\)\?/\1@$DB_HOST_IP:$POSTGRES_PORT/g" ckan/test-core.ini
cd ckan
paster db init -c test-core.ini
cd ..
echo "Installing ckanext-privatedatasets and its requirements..."
python setup.py develop
echo "Running tests..."
python setup.py nosetests

47
bin/travis-build.bash Executable file
View File

@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -e
echo "This is travis-build.bash..."
echo "Installing the packages that CKAN requires..."
sudo apt-get update -qq
sudo apt-get install solr-jetty
echo "Installing CKAN and its Python dependencies..."
git clone https://github.com/ckan/ckan
cd ckan
git checkout ckan-$CKANVERSION
python setup.py develop
sed -i "s|psycopg2==2.4.5|psycopg2==2.7.1|g" requirements.txt
pip install -r requirements.txt --allow-all-external
pip install -r dev-requirements.txt --allow-all-external
cd -
echo "Setting up Solr..."
# solr is multicore for tests on ckan master now, but it's easier to run tests
# on Travis single-core still.
# see https://github.com/ckan/ckan/issues/2972
sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini
printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
sudo service jetty restart
echo "Creating the PostgreSQL user and database..."
sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
sudo -u postgres psql -c "CREATE DATABASE ckan_test WITH OWNER ckan_default;"
sudo -u postgres psql -c "CREATE DATABASE datastore_test WITH OWNER ckan_default;"
echo "Initialising the database..."
cd ckan
paster db init -c test-core.ini
cd -
echo "Installing ckanext-privatedatasets and its requirements..."
python setup.py develop
echo "travis-build.bash is done."

3
bin/travis-run.sh Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
python setup.py nosetests

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2014 CoNWeT Lab., Universidad Politécnica de Madrid
# Copyright (c) 2014 - 2017 CoNWeT Lab., Universidad Politécnica de Madrid
# This file is part of CKAN Private Dataset Extension.
@ -52,92 +52,8 @@ def package_acquired(context, request_data):
:rtype: dict
'''
log.info('Notification received: %s' % request_data)
# Check access
plugins.toolkit.check_access(constants.PACKAGE_ACQUIRED, context, request_data)
# Get the parser from the configuration
class_path = config.get(PARSER_CONFIG_PROP, '')
if class_path != '':
try:
cls = class_path.split(':')
class_package = cls[0]
class_name = cls[1]
parser_cls = getattr(importlib.import_module(class_package), class_name)
parser = parser_cls()
except Exception as e:
raise plugins.toolkit.ValidationError({'message': '%s: %s' % (type(e).__name__, str(e))})
else:
raise plugins.toolkit.ValidationError({'message': '%s not configured' % PARSER_CONFIG_PROP})
# Parse the result using the parser set in the configuration
# Expected result: {'errors': ["...", "...", ...]
# 'users_datasets': [{'user': 'user_name', 'datasets': ['ds1', 'ds2', ...]}, ...]}
result = parser.parse_notification(request_data)
warns = []
# Introduce the users into the datasets
for user_info in result['users_datasets']:
for dataset_id in user_info['datasets']:
try:
context_pkg_show = context.copy()
context_pkg_show['ignore_auth'] = True
context_pkg_show[constants.CONTEXT_CALLBACK] = True
dataset = plugins.toolkit.get_action('package_show')(context_pkg_show, {'id': dataset_id})
# This operation can only be performed with private datasets
# This check is redundant since the package_update function will throw an exception
# if a list of allowed users is included in a public dataset. However, this check
# should be performed in order to avoid strange future exceptions
if dataset.get('private', None) is True:
# Create the array if it does not exist
if constants.ALLOWED_USERS not in dataset or dataset[constants.ALLOWED_USERS] is None:
dataset[constants.ALLOWED_USERS] = []
# Add the user only if it is not in the list
if user_info['user'] not in dataset[constants.ALLOWED_USERS]:
dataset[constants.ALLOWED_USERS].append(user_info['user'])
context_pkg_update = context.copy()
context_pkg_update['ignore_auth'] = True
# Set creator as the user who is performing the changes
user_show = plugins.toolkit.get_action('user_show')
creator_user_id = dataset.get('creator_user_id', '')
user_show_context = {'ignore_auth': True}
user = user_show(user_show_context, {'id': creator_user_id})
context_pkg_update['user'] = user.get('name', '')
plugins.toolkit.get_action('package_update')(context_pkg_update, dataset)
log.info('Allowed Users added correctly')
else:
log.warn('The user %s is already allowed to access the %s dataset' % (user_info['user'], dataset_id))
else:
log.warn('Dataset %s is public. Allowed Users cannot be added')
warns.append('Unable to upload the dataset %s: It\'s a public dataset' % dataset_id)
except plugins.toolkit.ObjectNotFound:
# If a dataset does not exist in the instance, an error message will be returned to the user.
# However the process won't stop and the process will continue with the remaining datasets.
log.warn('Dataset %s was not found in this instance' % dataset_id)
warns.append('Dataset %s was not found in this instance' % dataset_id)
except plugins.toolkit.ValidationError as e:
# Some datasets does not allow to introduce the list of allowed users since this property is
# only valid for private datasets outside an organization. In this case, a wanr will return
# but the process will continue
# WARN: This exception should not be risen anymore since public datasets are not updated.
message = '%s(%s): %s' % (dataset_id, constants.ALLOWED_USERS, e.error_dict[constants.ALLOWED_USERS][0])
log.warn(message)
warns.append(message)
# Return warnings that inform about non-existing datasets
if len(warns) > 0:
return {'warns': warns}
context['method'] = 'grant'
return _process_package(context, request_data)
def acquisitions_list(context, data_dict):
'''
@ -194,3 +110,123 @@ def acquisitions_list(context, data_dict):
pass
return result
def revoke_access(context, request_data):
'''
API action to be called in order to revoke access grants of an user.
This API should be called to delete the user from the list of allowed users.
Since each service can provide a different way of pushing the data, the received
data will be forwarded to the parser set in the preferences. This parser should
return a dict similar to the following one:
{'errors': ["...", "...", ...]
'users_datasets': [{'user': 'user_name', 'datasets': ['ds1', 'ds2', ...]}, ...]}
1) 'errors' contains the list of errors. It should be empty if no errors arised
while the notification is parsed
2) 'users_datasets' is the lists of datasets available for each user (each element
of this list is a dictionary with two fields: user and datasets).
:parameter request_data: Depends on the parser
:type request_data: dict
:return: A list of warnings or None if the list of warnings is empty
:rtype: dict
'''
context['method'] = 'revoke'
return _process_package(context, request_data)
def _process_package(context, request_data):
log.info('Notification received: %s' % request_data)
# Check access
method = constants.PACKAGE_ACQUIRED if context.get('method') == 'grant' else constants.PACKAGE_DELETED
plugins.toolkit.check_access(method, context, request_data)
# Get the parser from the configuration
class_path = config.get(PARSER_CONFIG_PROP, '')
if class_path != '':
try:
cls = class_path.split(':')
class_package = cls[0]
class_name = cls[1]
parser_cls = getattr(importlib.import_module(class_package), class_name)
parser = parser_cls()
except Exception as e:
raise plugins.toolkit.ValidationError({'message': '%s: %s' % (type(e).__name__, str(e))})
else:
raise plugins.toolkit.ValidationError({'message': '%s not configured' % PARSER_CONFIG_PROP})
# Parse the result using the parser set in the configuration
# Expected result: {'errors': ["...", "...", ...]
# 'users_datasets': [{'user': 'user_name', 'datasets': ['ds1', 'ds2', ...]}, ...]}
result = parser.parse_notification(request_data)
warns = []
for user_info in result['users_datasets']:
for dataset_id in user_info['datasets']:
try:
context_pkg_show = context.copy()
context_pkg_show['ignore_auth'] = True
context_pkg_show[constants.CONTEXT_CALLBACK] = True
dataset = plugins.toolkit.get_action('package_show')(context_pkg_show, {'id': dataset_id})
# This operation can only be performed with private datasets
# This check is redundant since the package_update function will throw an exception
# if a list of allowed users is included in a public dataset. However, this check
# should be performed in order to avoid strange future exceptions
if dataset.get('private', None) is True:
# Create the array if it does not exist
if constants.ALLOWED_USERS not in dataset or dataset[constants.ALLOWED_USERS] is None:
dataset[constants.ALLOWED_USERS] = []
method = context['method'] == 'grant'
present = user_info['user'] in dataset[constants.ALLOWED_USERS]
# Deletes the user only if it is in the list
if (not method and present) or (method and not present):
if method:
dataset[constants.ALLOWED_USERS].append(user_info['user'])
else:
dataset[constants.ALLOWED_USERS].remove(user_info['user'])
context_pkg_update = context.copy()
context_pkg_update['ignore_auth'] = True
# Set creator as the user who is performing the changes
user_show = plugins.toolkit.get_action('user_show')
creator_user_id = dataset.get('creator_user_id', '')
user_show_context = {'ignore_auth': True}
user = user_show(user_show_context, {'id': creator_user_id})
context_pkg_update['user'] = user.get('name', '')
plugins.toolkit.get_action('package_update')(context_pkg_update, dataset)
log.info('Action %s access to dataset ended successfully' % context['method'])
else:
log.warn('Action %s access to dataset not completed. The dataset %s already %s access to the user %s' % (context['method'], dataset_id, context['method'], user_info['user']))
else:
log.warn('Dataset %s is public. Cannot %s access to users' % (dataset_id, context['method']))
warns.append('Unable to upload the dataset %s: It\'s a public dataset' % dataset_id)
except plugins.toolkit.ObjectNotFound:
# If a dataset does not exist in the instance, an error message will be returned to the user.
# However the process won't stop and the process will continue with the remaining datasets.
log.warn('Dataset %s was not found in this instance' % dataset_id)
warns.append('Dataset %s was not found in this instance' % dataset_id)
except plugins.toolkit.ValidationError as e:
# Some datasets does not allow to introduce the list of allowed users since this property is
# only valid for private datasets outside an organization. In this case, a wanr will return
# but the process will continue
# WARN: This exception should not be risen anymore since public datasets are not updated.
message = '%s(%s): %s' % (dataset_id, constants.ALLOWED_USERS, e.error_dict[constants.ALLOWED_USERS][0])
log.warn(message)
warns.append(message)
# Return warnings that inform about non-existing datasets
if len(warns) > 0:
return {'warns': warns}

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2014 CoNWeT Lab., Universidad Politécnica de Madrid
# Copyright (c) 2014 - 2017 CoNWeT Lab., Universidad Politécnica de Madrid
# This file is part of CKAN Private Dataset Extension.
@ -140,3 +140,8 @@ def package_acquired(context, data_dict):
def acquisitions_list(context, data_dict):
# Users can get only their acquisitions list
return {'success': context['user'] == data_dict['user']}
@tk.auth_allow_anonymous_access
def revoke_access(context, data_dict):
# TODO: Check functionality and improve security(if needed)
return {'success': True}

View File

@ -24,3 +24,4 @@ SEARCHABLE = 'searchable'
ACQUIRE_URL = 'acquire_url'
CONTEXT_CALLBACK = 'updating_via_cb'
PACKAGE_ACQUIRED = 'package_acquired'
PACKAGE_DELETED = 'revoke_access'

View File

@ -27,7 +27,6 @@ from ckan.common import request
class FiWareNotificationParser(object):
def parse_notification(self, request_data):
my_host = request.host
fields = ['customer_name', 'resources']
@ -62,3 +61,4 @@ class FiWareNotificationParser(object):
raise tk.ValidationError({'message': 'Invalid resource format'})
return {'users_datasets': [{'user': user_name, 'datasets': datasets}]}

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2014 CoNWeT Lab., Universidad Politécnica de Madrid
# Copyright (c) 2014 - 2017 CoNWeT Lab., Universidad Politécnica de Madrid
# This file is part of CKAN Private Dataset Extension.
@ -21,6 +21,9 @@ import ckan.lib.search as search
import ckan.model as model
import ckan.plugins as p
import ckan.plugins.toolkit as tk
from ckan.lib.plugins import DefaultPermissionLabels
import auth
import actions
import constants
@ -32,7 +35,7 @@ import helpers as helpers
HIDDEN_FIELDS = [constants.ALLOWED_USERS, constants.SEARCHABLE]
class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm, DefaultPermissionLabels):
p.implements(p.IDatasetForm)
p.implements(p.IAuthFunctions)
@ -41,6 +44,7 @@ class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
p.implements(p.IActions)
p.implements(p.IPackageController, inherit=True)
p.implements(p.ITemplateHelpers)
p.implements(p.IPermissionLabels)
######################################################################
############################ DATASET FORM ############################
@ -112,7 +116,8 @@ class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
'package_update': auth.package_update,
# 'resource_show': auth.resource_show,
constants.PACKAGE_ACQUIRED: auth.package_acquired,
constants.ACQUISITIONS_LIST: auth.acquisitions_list}
constants.ACQUISITIONS_LIST: auth.acquisitions_list,
constants.PACKAGE_DELETED: auth.revoke_access}
# resource_show is not required in CKAN 2.3 because it delegates to
# package_show
@ -145,6 +150,8 @@ class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
return m
######################################################################
############################## IACTIONS ##############################
######################################################################
@ -152,7 +159,8 @@ class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
def get_actions(self):
return {
constants.PACKAGE_ACQUIRED: actions.package_acquired,
constants.ACQUISITIONS_LIST: actions.acquisitions_list
constants.ACQUISITIONS_LIST: actions.acquisitions_list,
constants.PACKAGE_DELETED: actions.revoke_access
}
######################################################################
@ -287,6 +295,24 @@ class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):
return search_results
####
def get_dataset_labels(self, dataset_obj):
labels = super(PrivateDatasets, self).get_dataset_labels(
dataset_obj)
if getattr(dataset_obj, 'searchable', False):
labels.append('searchable')
return labels
def get_user_dataset_labels(self, user_obj):
labels = super(PrivateDatasets, self).get_user_dataset_labels(
user_obj)
labels.append('searchable')
return labels
######################################################################
######################### ITEMPLATESHELPER ###########################
######################################################################

View File

@ -49,7 +49,7 @@
{% endfor %}
</select>
<span class="info-block info-inline">
<i class="icon-info-sign"></i>
<i class="icon-info-sign fa fa-info-circle"></i>
{% trans %}
Private datasets can only be accessed by certain users, while public datasets can be accessed by anyone.
{% endtrans %}
@ -68,7 +68,7 @@
{% endfor %}
</select>
<span class="info-block info-inline">
<i class="icon-info-sign"></i>
<i class="icon-info-sign fa fa-info-circle"></i>
{% trans %}
Searchable datasets can be searched by anyone, while not-searchable datasets can only be accessed by entering directly its URL.
{% endtrans %}

View File

@ -10,6 +10,6 @@ Example:
#}
<a href={{ url_dest }} class="btn btn-mini" target="_blank">
<i class="icon-shopping-cart"></i>
<i class="icon-shopping-cart fa fa-shopping-cart"></i>
{{ _('Acquire') }}
</a>

View File

@ -64,7 +64,6 @@ class ActionsTest(unittest.TestCase):
])
def test_class_cannot_be_loaded(self, class_path, class_name, path_exist, class_exist, expected_error):
class_package = class_path
class_package += ':' + class_name if class_name else ''
actions.config = {PARSER_CONFIG_PROP: class_package}
@ -79,8 +78,7 @@ class ActionsTest(unittest.TestCase):
actions.importlib.import_module = MagicMock(side_effect=ImportError(IMPORT_ERROR_MSG) if not path_exist else None,
return_value=package if path_exist else None)
# Call the function
if expected_error:
with self.assertRaises(actions.plugins.toolkit.ValidationError) as cm:
actions.package_acquired({}, {})
@ -163,18 +161,14 @@ class ActionsTest(unittest.TestCase):
])
def test_add_users(self, users_info, datasets_not_found, not_updatable_datasets, allowed_users=[]):
parse_result = {'users_datasets': []}
parse_result = {'users_datasets': [{'user': user, 'datasets': users_info[user]} for user in users_info]}
creator_user = {'name': 'ckan', 'id': '1234'}
# Transform user_info
for user in users_info:
parse_result['users_datasets'].append({'user': user, 'datasets': users_info[user]})
parse_notification, package_show, package_update, user_show = self.configure_mocks(parse_result,
datasets_not_found, not_updatable_datasets, allowed_users, creator_user)
# Call the function
context = {'user': 'user1', 'model': 'model', 'auth_obj': {'id': 1}}
context = {'user': 'user1', 'model': 'model', 'auth_obj': {'id': 1}, 'method': 'grant'}
result = actions.package_acquired(context, users_info)
# Calculate the list of warns
@ -295,3 +289,80 @@ class ActionsTest(unittest.TestCase):
expected_acquired_datasets.append(pkg)
self.assertEquals(expected_acquired_datasets, result)
@parameterized.expand([
# Simple Test: one user and one dataset
({'user1': ['ds1']}, [], [], None), #Test with and non-existing list of allowed users
({'user2': ['ds1']}, [], [], []), #Test remove a non-existing user
({'user3': ['ds1']}, [], [], ['user3']), #Test remove an existing user
({'user4': ['ds1']}, [], [], ['another_user']), #Test remove non-existing from an already populated list
({'user5': ['ds1']}, [], [], ['another_user', 'user5']),
({'user6': ['ds1']}, ['ds1'], [], None), #Test remove from an unknown place
({'user61': ['ds1']}, ['ds1'], [], []),
({'user62': ['ds1']}, ['ds1'], [], ['user6']),
({'user7': ['ds1']}, [], ['ds1'], []), #Tests deleting from a public dataset
({'user8': ['ds1']}, [], ['ds1'], ['another_user']),
({'user9': ['ds1']}, [], ['ds1'], ['another_user', 'another_one']),
({'user91': ['ds1']}, ['ds1'], ['ds1'], ['another_user', 'another_one']), # Checking the behaviour when the unknown dataset is public
({'user92': ['ds1']}, ['ds1'], ['ds1'], ['another_user', 'another_one','user92']),
# Complex test: some users and some datasets
({'user1': ['ds1', 'ds2', 'ds3', 'ds4'], 'user2': ['ds5', 'ds6', 'ds7']}, ['ds3', 'ds6'], ['ds4', 'ds7'], []),
({'user3': ['ds1', 'ds2', 'ds3', 'ds4'], 'user4': ['ds5', 'ds6', 'ds7']}, ['ds3', 'ds6'], ['ds4', 'ds7'], ['another_user']),
({'user5': ['ds1', 'ds2', 'ds3', 'ds4'], 'user6': ['ds5', 'ds6', 'ds7']}, ['ds3', 'ds6'], ['ds4', 'ds7'], ['another_user', 'another_one']),
({'user7': ['ds1', 'ds2', 'ds3', 'ds4'], 'user8': ['ds5', 'ds6', 'ds7']}, ['ds3', 'ds6'], ['ds4', 'ds7'], ['another_user', 'another_one', 'user7'])
])
def test_delete_users(self, users_info, datasets_not_found, not_updatable_datasets, allowed_users=[]):
parse_result = {'users_datasets': []}
creator_user = {'name': 'ckan', 'id': '1234'}
# Transform user_info
for user in users_info:
parse_result['users_datasets'].append({'user': user, 'datasets': users_info[user]})
parse_delete, package_show, package_update, user_show = self.configure_mocks(parse_result,
datasets_not_found, not_updatable_datasets, allowed_users, creator_user)
# Call the function
context = {'user': 'user1', 'model': 'model', 'auth_obj': {'id': 1}, 'method': 'revoke'}
result = actions.revoke_access(context, users_info)
# Calculate the list of warns
warns = []
for user_datasets in parse_result['users_datasets']:
for dataset_id in user_datasets['datasets']:
if dataset_id in datasets_not_found:
warns.append('Dataset %s was not found in this instance' % dataset_id)
elif dataset_id in not_updatable_datasets:
# warns.append('%s(%s): %s' % (dataset_id, 'allowed_users', ADD_USERS_ERROR))
warns.append('Unable to upload the dataset %s: It\'s a public dataset' % dataset_id)
expected_result = {'warns': warns} if len(warns) > 0 else None
# Check that the returned result is as expected
self.assertEquals(expected_result, result)
# Check that the initial functions (check_access and parse_notification) has been called properly
parse_delete.assert_called_once_with(users_info)
actions.plugins.toolkit.check_access.assert_called_once_with('revoke_access', context, users_info)
for user_datasets in parse_result['users_datasets']:
for dataset_id in user_datasets['datasets']:
# The show function is always called
context_show = context.copy()
context_show['ignore_auth'] = True
context_show['updating_via_cb'] = True
package_show.assert_any_call(context_show, {'id': dataset_id})
# The update function is called only when the show function does not throw an exception and
# when the user is not in the list of allowed users.
if dataset_id not in datasets_not_found and allowed_users is not None and user_datasets['user'] in allowed_users and dataset_id not in not_updatable_datasets:
# Calculate the list of allowed_users
expected_allowed_users = list(allowed_users)
expected_allowed_users.remove(user_datasets['user'])
context_update = context.copy()
context_update['ignore_auth'] = True
context_update['user'] = creator_user['name']
package_update.assert_any_call(context_update, {'id': dataset_id, 'allowed_users': expected_allowed_users, 'private': True, 'creator_user_id': creator_user['id']})

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2014 CoNWeT Lab., Universidad Politécnica de Madrid
# Copyright (c) 2014 - 2017 CoNWeT Lab., Universidad Politécnica de Madrid
# This file is part of CKAN Private Dataset Extension.
@ -251,6 +251,9 @@ class AuthTest(unittest.TestCase):
def test_package_acquired(self):
self.assertTrue(auth.package_acquired({}, {})['success'])
def test_package_deleted(self):
self.assertTrue(auth.revoke_access({},{})['success'])
@parameterized.expand([
({'user': 'user_1'}, {'user': 'user_1'}, True),
({'user': 'user_2'}, {'user': 'user_1'}, False),

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2014 CoNWeT Lab., Universidad Politécnica de Madrid
# Copyright (c) 2014 - 2017 CoNWeT Lab., Universidad Politécnica de Madrid
# This file is part of CKAN Private Dataset Extension.
@ -65,7 +65,8 @@ class PluginTest(unittest.TestCase):
('resource_show', plugin.auth.resource_show),
('resource_show', plugin.auth.resource_show, True, False),
('package_acquired', plugin.auth.package_acquired),
('acquisitions_list', plugin.auth.acquisitions_list)
('acquisitions_list', plugin.auth.acquisitions_list),
('revoke_access', plugin.auth.revoke_access)
])
def test_auth_function(self, function_name, expected_function, is_ckan_23=False, expected=True):
plugin.tk.check_ckan_version = MagicMock(return_value=is_ckan_23)
@ -97,7 +98,8 @@ class PluginTest(unittest.TestCase):
@parameterized.expand([
('package_acquired', plugin.actions.package_acquired),
('acquisitions_list', plugin.actions.acquisitions_list)
('acquisitions_list', plugin.actions.acquisitions_list),
('revoke_access', plugin.actions.revoke_access)
])
def test_actions_function(self, function_name, expected_function):
actions = self.privateDatasets.get_actions()

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2014 CoNWeT Lab., Universidad Politécnica de Madrid
# Copyright (c) 2014 - 2017 CoNWeT Lab., Universidad Politécnica de Madrid
# This file is part of CKAN Private Dataset Extension.
@ -17,9 +17,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with CKAN Private Dataset Extension. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from nose_parameterized import parameterized
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select, WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from subprocess import Popen
import ckan.lib.search.index as search_index
@ -70,6 +74,7 @@ class TestSelenium(unittest.TestCase):
self.driver = webdriver.Remote(os.environ['WEB_DRIVER_URL'], webdriver.DesiredCapabilities.FIREFOX.copy())
self.base_url = os.environ['CKAN_SERVER_URL']
else:
self.driver = webdriver.Firefox()
self.base_url = 'http://127.0.0.1:5000/'
@ -108,7 +113,11 @@ class TestSelenium(unittest.TestCase):
def login(self, username, password):
driver = self.driver
driver.get(self.base_url)
driver.find_element_by_link_text('Log in').click()
login_btn = WebDriverWait(driver, 15).until(
EC.element_to_be_clickable((By.LINK_TEXT, 'Log in'))
)
login_btn.click()
driver.find_element_by_id('field-login').clear()
driver.find_element_by_id('field-login').send_keys(username)
driver.find_element_by_id('field-password').clear()
@ -217,38 +226,31 @@ class TestSelenium(unittest.TestCase):
driver = self.driver
driver.find_element_by_link_text('Datasets').click()
if searchable:
if searchable or owner or in_org:
xpath = '//div[@id=\'content\']/div[3]/div/section/div/ul/li/div/h3/span'
# Check the label
if owner:
self.assertEqual('OWNER', driver.find_element_by_xpath(xpath).text)
if not acquired and private and not in_org:
self.assertEqual('PRIVATE', driver.find_element_by_xpath(xpath).text)
elif acquired and not owner and private:
self.assertEqual('ACQUIRED', driver.find_element_by_xpath(xpath).text)
elif owner:
self.assertEqual('OWNER', driver.find_element_by_xpath(xpath).text)
# When a user cannot access a dataset, the link is no longer provided
else:
# If the dataset is not searchable, a link to it could not be found in the dataset search page
# If the dataset is not searchable and the user is not the owner, a link to it could not be found in the dataset search page
self.assertEquals(None, re.search(dataset_url, driver.page_source))
# Access the dataset
driver.get(self.base_url + 'dataset/' + dataset_url)
if not acquired and private and not in_org:
xpath = '//div[@id=\'content\']/div/div'
buy_msg = 'This private dataset can be acquired. To do so, please click here'
if acquire_url is not None:
self.assertTrue(driver.find_element_by_xpath(xpath).text.startswith(buy_msg))
self.assertEquals(acquire_url, driver.find_element_by_link_text('here').get_attribute('href'))
xpath += '[2]' # The unauthorized message is in a different Path
else:
src = driver.page_source
self.assertEquals(None, re.search(buy_msg, src))
# If the user has not access to the dataset the 404 error page is displayed
xpath = '//*[@id="content"]/div[2]/article/div/h1'
msg = '404 Not Found'
self.assertTrue('/user/login' in driver.current_url)
self.assertTrue(driver.find_element_by_xpath(xpath).text.startswith('Unauthorized to read package %s' % dataset_url))
self.assertEquals(driver.find_element_by_xpath(xpath).text, msg)
else:
self.assertEquals(self.base_url + 'dataset/%s' % dataset_url, driver.current_url)
@ -274,10 +276,6 @@ class TestSelenium(unittest.TestCase):
self.register(user, user, '%s@conwet.com' % user, user)
@parameterized.expand([
# (['user1', 'user2', 'user3'], True, True, ['user2'], 'http://store.conwet.com/'),
# (['user1', 'user2', 'user3'], True, True, ['user3']),
# (['user1', 'user2', 'user3'], False, True, ['user3']),
# (['user1', 'user2', 'user3'], True, False, ['user2']),
(['user1', 'user2', 'user3'], True, True, [], 'http://store.conwet.com/'),
(['user1', 'user2', 'user3'], True, True, []),
(['user1', 'user2', 'user3'], False, True, []),
@ -298,7 +296,9 @@ class TestSelenium(unittest.TestCase):
url = get_dataset_url(pkg_name)
self.create_ds(pkg_name, 'Example description', ['tag1', 'tag2', 'tag3'], private, searchable,
allowed_users, acquire_url, 'http://upm.es', 'UPM Main', 'Example Description', 'CSV')
self.check_ds_values(url, private, searchable, allowed_users, acquire_url)
self.check_user_access(pkg_name, url, True, True, False, private, searchable, acquire_url)
self.check_acquired(pkg_name, url, False, private)
@ -310,28 +310,17 @@ class TestSelenium(unittest.TestCase):
acquired = user in allowed_users
self.check_user_access(pkg_name, url, False, acquired, False, private, searchable, acquire_url)
# The user is logged out when they try to access a private dataset and they are not included
# in the list of allowed users.
if not acquired and private:
self.login(user, user)
self.check_acquired(pkg_name, url, acquired, private)
@parameterized.expand([
# (['a'] , 'http://upm.es', 'Allowed users: Name must be at least 2 characters long'),
# (['a a'], 'http://upm.es', 'Allowed users: Url must be purely lowercase alphanumeric (ascii) characters and these symbols: -_'),
(['upm', 'a'], 'http://upm.es', 'Allowed users: Must be at least 2 characters long'),
(['upm', 'a a a'], 'http://upm.es', 'Allowed users: Must be purely lowercase alphanumeric (ascii) characters and these symbols: -_'),
(['upm', 'a?-vz'], 'http://upm.es', 'Allowed users: Must be purely lowercase alphanumeric (ascii) characters and these symbols: -_'),
(['thisisaveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylongname'],
'http://upm.es', 'Allowed users: Name must be a maximum of 100 characters long'),
(['conwet'], 'ftp://google.es', 'Acquire URL: The URL "ftp://google.es" is not valid.'),
# (['conwet'], 'http://google*.com', 'Acquire URL: The URL "http://google*.com" is not valid.'),
# (['conwet'], 'http://google+.com', 'Acquire URL: The URL "http://google+.com" is not valid.'),
# (['conwet'], 'http://google/.com', 'Acquire URL: The URL "http://google/.com" is not valid.'),
(['conwet'], 'google', 'Acquire URL: The URL "google" is not valid.'),
(['conwet'], 'http://google', 'Acquire URL: The URL "http://google" is not valid.'),
# (['conwet'], 'http://google:es', 'Acquire URL: The URL "http://google:es" is not valid.'),
(['conwet'], 'www.google.es', 'Acquire URL: The URL "www.google.es" is not valid.')
])
@ -449,21 +438,6 @@ class TestSelenium(unittest.TestCase):
self.check_ds_values(url_path, dataset['private'], dataset['searchable'], final_users, acquire_url)
@parameterized.expand([
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], True, True, [], 'http://store.conwet.com/'),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], True, True, []),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], False, True, []),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], True, False, []),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], True, True, ['user3'], 'http://store.conwet.com/'),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], True, True, ['user3']),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], False, True, ['user3']),
# (['user1', 'user2', 'user3'], [{'name': 'CoNWeT', 'users': ['user2']}], True, False, ['user3']),
# More complex
# (['user1', 'user2', 'user3', 'user4', 'user5', 'user6'], [{'name': 'CoNWeT', 'users': ['user2', 'user3']}], True, True, ['user4', 'user5'], 'http://store.conwet.com/'),
# (['user1', 'user2', 'user3', 'user4', 'user5', 'user6'], [{'name': 'CoNWeT', 'users': ['user2', 'user3']}], True, True, ['user4', 'user5']),
# (['user1', 'user2', 'user3', 'user4', 'user5', 'user6'], [{'name': 'CoNWeT', 'users': ['user2', 'user3']}], False, True, ['user4', 'user5']),
# (['user1', 'user2', 'user3', 'user4', 'user5', 'user6'], [{'name': 'CoNWeT', 'users': ['user2', 'user3']}], True, False, ['user4', 'user5']),
# Even if user6 is in another organization, he/she won't be able to access the dataset
(['user1', 'user2', 'user3', 'user4', 'user5', 'user6'], [{'name': 'CoNWeT', 'users': ['user2', 'user3']},
{'name': 'UPM', 'users': ['user6']}], True, True, ['user4', 'user5'], 'http://store.conwet.com/'),
@ -503,11 +477,6 @@ class TestSelenium(unittest.TestCase):
in_org = user in orgs[0]['users']
self.check_user_access(pkg_name, url, False, acquired, in_org, private, searchable, acquire_url)
# The user is logged out when they try to access a private dataset and they are not included
# in the list of allowed users.
if not acquired and private and not in_org:
self.login(user, user)
self.check_acquired(pkg_name, url, acquired, private)
def test_bug_16(self):

View File

@ -6,8 +6,18 @@ port = 5000
[app:main]
use = config:./ckan/test-core.ini
ckan.site_url = http://127.0.0.1:5000/
ckan.legacy_templates = no
ckan.plugins = privatedatasets
ckan.privatedatasets.parser = ckanext.privatedatasets.parsers.fiware:FiWareNotificationParser
ckan.privatedatasets.show_acquire_url_on_create = True
ckan.privatedatasets.show_acquire_url_on_edit = True
ckan.privatedatasets.show_acquire_url_on_edit = True
sqlalchemy.url = postgresql://ckan_default:pass@127.0.0.1:5432/ckan_test
ckan.datastore.write_url = postgresql://ckan_default:pass@127.0.0.1:5432/datastore_test
ckan.datastore.read_url = postgresql://datastore_default:pass@127.0.0.1:5432/datastore_test
ckan.storage_path=data/storage