[#13] Remove or deprecate unused code
Mostly in controllers, dictization and plugin, either related to the old templates pre-dataset type or old authorization.
This commit is contained in:
amercader
parent
6c02c87f8d
commit
2ee27164c3
28
README.rst
28
README.rst
|
|
@ -94,32 +94,10 @@ The commands should be run with the pyenv activated and refer to your sites conf
|
|||
|
||||
paster --plugin=ckanext-harvest harvester sources --config=mysite.ini
|
||||
|
||||
Authorization Profiles
|
||||
======================
|
||||
Authorization
|
||||
=============
|
||||
|
||||
Starting from CKAN 1.6.1, the harvester extension offers the ability to use
|
||||
different authorization profiles. These can be defined in your ini file as::
|
||||
|
||||
ckan.harvest.auth.profile = <profile_name>
|
||||
|
||||
The two available profiles right now are:
|
||||
|
||||
* `default`: This is the default profile, the same one that this extension has
|
||||
used historically. Basically, only sysadmins can manage anything related to
|
||||
harvesting, including creating and editing harvest sources or running harvest
|
||||
jobs.
|
||||
|
||||
* `publisher`: When using this profile, sysadmins can still perform any
|
||||
harvesting related action, but in addition, users belonging to a publisher
|
||||
(with role `admin`) can manage and run their own harvest sources and jobs.
|
||||
Note that this requires CKAN core to also use the `publisher` authorization
|
||||
profile, i.e you will also need to add::
|
||||
|
||||
ckan.auth.profile = publisher
|
||||
|
||||
To know more about the CKAN publisher auth profile, visit:
|
||||
|
||||
http://oldwiki.ckan.org/Working_with_the_publisher_auth_profile
|
||||
TODO
|
||||
|
||||
|
||||
The CKAN harvester
|
||||
|
|
|
|||
|
|
@ -4,19 +4,13 @@ from lxml.etree import XMLSyntaxError
|
|||
from pylons.i18n import _
|
||||
|
||||
from ckan import model
|
||||
from ckan.model.group import Group
|
||||
|
||||
import ckan.lib.helpers as h, json
|
||||
from ckan.lib.base import BaseController, c, g, request, \
|
||||
response, session, render, config, abort, redirect
|
||||
|
||||
from ckan.lib.navl.dictization_functions import DataError
|
||||
from ckan.logic import NotFound, ValidationError, get_action, NotAuthorized
|
||||
from ckanext.harvest.plugin import DATASET_TYPE_NAME
|
||||
from ckanext.harvest.logic.schema import harvest_source_form_to_db_schema
|
||||
|
||||
from ckan.lib.helpers import Page,pager_url
|
||||
import ckan.plugins as p
|
||||
import ckan.lib.helpers as h, json
|
||||
from ckan.lib.base import BaseController, c, \
|
||||
response, render, abort, redirect
|
||||
|
||||
from ckanext.harvest.plugin import DATASET_TYPE_NAME
|
||||
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -29,211 +23,29 @@ class ViewController(BaseController):
|
|||
|
||||
super(ViewController,self).__before__(action, **params)
|
||||
|
||||
#TODO: remove
|
||||
c.publisher_auth = (config.get('ckan.harvest.auth.profile',None) == 'publisher')
|
||||
|
||||
c.dataset_type = DATASET_TYPE_NAME
|
||||
|
||||
def _get_publishers(self):
|
||||
groups = None
|
||||
user = model.User.get(c.user)
|
||||
if c.publisher_auth:
|
||||
if user.sysadmin:
|
||||
groups = Group.all(group_type='publisher')
|
||||
elif c.userobj:
|
||||
groups = c.userobj.get_groups('publisher')
|
||||
else: # anonymous user shouldn't have access to this page anyway.
|
||||
groups = []
|
||||
|
||||
# Be explicit about which fields we make available in the template
|
||||
groups = [ {
|
||||
'name': g.name,
|
||||
'id': g.id,
|
||||
'title': g.title,
|
||||
} for g in groups ]
|
||||
|
||||
return groups
|
||||
|
||||
|
||||
def index(self):
|
||||
context = {'model':model, 'user':c.user,'session':model.Session}
|
||||
try:
|
||||
# Request all harvest sources
|
||||
c.sources = get_action('harvest_source_list')(context,{})
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
if c.publisher_auth:
|
||||
c.sources = sorted(c.sources,key=lambda source : source['publisher_title'])
|
||||
|
||||
c.status = config.get('ckan.harvest.status')
|
||||
|
||||
return render('index.html')
|
||||
|
||||
def new(self,data = None,errors = None, error_summary = None):
|
||||
|
||||
if ('save' in request.params) and not data:
|
||||
return self._save_new()
|
||||
|
||||
data = data or {}
|
||||
errors = errors or {}
|
||||
error_summary = error_summary or {}
|
||||
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
harvesters_info = get_action('harvesters_info_show')(context,{})
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'harvesters': harvesters_info}
|
||||
|
||||
c.groups = self._get_publishers()
|
||||
|
||||
vars['form_items'] = self._make_autoform_items(harvesters_info)
|
||||
|
||||
c.form = render('source/old_new_source_form.html', extra_vars=vars)
|
||||
return render('source/new.html')
|
||||
|
||||
|
||||
|
||||
def _save_new(self):
|
||||
try:
|
||||
data_dict = dict(request.params)
|
||||
self._check_data_dict(data_dict)
|
||||
context = {'model':model, 'user':c.user, 'session':model.Session,
|
||||
'schema':harvest_source_form_schema()}
|
||||
|
||||
source = get_action('harvest_source_create')(context,data_dict)
|
||||
|
||||
# Create a harvest job for the new source
|
||||
get_action('harvest_job_create')(context,{'source_id':source['id']})
|
||||
|
||||
h.flash_success(_('New harvest source added successfully.'
|
||||
'A new harvest job for the source has also been created.'))
|
||||
redirect('/harvest/%s' % source['id'])
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
except DataError,e:
|
||||
abort(400, 'Integrity Error')
|
||||
except ValidationError,e:
|
||||
errors = e.error_dict
|
||||
error_summary = e.error_summary if hasattr(e,'error_summary') else None
|
||||
return self.new(data_dict, errors, error_summary)
|
||||
|
||||
def edit(self, id, data = None,errors = None, error_summary = None):
|
||||
|
||||
if ('save' in request.params) and not data:
|
||||
return self._save_edit(id)
|
||||
|
||||
|
||||
if not data:
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
|
||||
old_data = get_action('harvest_source_show')(context, {'id':id})
|
||||
except NotFound:
|
||||
abort(404, _('Harvest Source not found'))
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
data = data or old_data
|
||||
errors = errors or {}
|
||||
error_summary = error_summary or {}
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
harvesters_info = get_action('harvesters_info_show')(context,{})
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'harvesters': harvesters_info}
|
||||
|
||||
c.groups = self._get_publishers()
|
||||
|
||||
vars['form_items'] = self._make_autoform_items(harvesters_info)
|
||||
|
||||
c.form = render('source/old_new_source_form.html', extra_vars=vars)
|
||||
|
||||
return render('source/edit.html')
|
||||
|
||||
def _save_edit(self,id):
|
||||
try:
|
||||
data_dict = dict(request.params)
|
||||
data_dict['id'] = id
|
||||
self._check_data_dict(data_dict)
|
||||
context = {'model':model, 'user':c.user, 'session':model.Session,
|
||||
'schema':harvest_source_form_schema()}
|
||||
|
||||
source = get_action('harvest_source_update')(context,data_dict)
|
||||
|
||||
h.flash_success(_('Harvest source edited successfully.'))
|
||||
redirect('/harvest/%s' %id)
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
except DataError,e:
|
||||
abort(400, _('Integrity Error'))
|
||||
except NotFound, e:
|
||||
abort(404, _('Harvest Source not found'))
|
||||
except ValidationError,e:
|
||||
errors = e.error_dict
|
||||
error_summary = e.error_summary if hasattr(e,'error_summary') else None
|
||||
return self.edit(id,data_dict, errors, error_summary)
|
||||
|
||||
def _check_data_dict(self, data_dict):
|
||||
'''Check if the return data is correct'''
|
||||
|
||||
# TODO: remove frequency once it is added to the frontend!
|
||||
surplus_keys_schema = ['id','publisher_id','user_id','config','save','frequency']
|
||||
schema_keys = harvest_source_form_to_db_schema().keys()
|
||||
keys_in_schema = set(schema_keys) - set(surplus_keys_schema)
|
||||
|
||||
# user_id is not yet used, we'll set the logged user one for the time being
|
||||
if not data_dict.get('user_id',None):
|
||||
if c.userobj:
|
||||
data_dict['user_id'] = c.userobj.id
|
||||
if keys_in_schema - set(data_dict.keys()):
|
||||
log.info(_('Incorrect form fields posted'))
|
||||
raise DataError(data_dict)
|
||||
|
||||
def read(self,id):
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
c.source = get_action('harvest_source_show')(context, {'id':id})
|
||||
c.page = Page(
|
||||
collection=c.source['status']['packages'],
|
||||
page=request.params.get('page', 1),
|
||||
items_per_page=20,
|
||||
url=pager_url
|
||||
)
|
||||
|
||||
return render('source/read.html')
|
||||
except NotFound:
|
||||
abort(404,_('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
|
||||
|
||||
def delete(self,id):
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
get_action('harvest_source_delete')(context, {'id':id})
|
||||
p.toolkit.get_action('harvest_source_delete')(context, {'id':id})
|
||||
|
||||
h.flash_success(_('Harvesting source successfully inactivated'))
|
||||
redirect(h.url_for('harvest'))
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
|
||||
def refresh(self, id):
|
||||
try:
|
||||
context = {'model':model, 'user':c.user, 'session':model.Session}
|
||||
get_action('harvest_job_create')(context,{'source_id':id})
|
||||
p.toolkit.get_action('harvest_job_create')(context,{'source_id':id})
|
||||
h.flash_success(_('Refresh requested, harvesting will take place within 15 minutes.'))
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
except Exception, e:
|
||||
if 'Can not create jobs on inactive sources' in str(e):
|
||||
|
|
@ -251,7 +63,7 @@ class ViewController(BaseController):
|
|||
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
obj = get_action('harvest_object_show')(context, {'id':id})
|
||||
obj = p.toolkit.get_action('harvest_object_show')(context, {'id':id})
|
||||
|
||||
# Check content type. It will probably be either XML or JSON
|
||||
try:
|
||||
|
|
@ -278,9 +90,9 @@ class ViewController(BaseController):
|
|||
|
||||
response.headers['Content-Length'] = len(content)
|
||||
return content.encode('utf-8')
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest object not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
except Exception, e:
|
||||
msg = 'An error occurred: [%s]' % str(e)
|
||||
|
|
@ -293,10 +105,9 @@ class ViewController(BaseController):
|
|||
context = {'model': model, 'user': c.user}
|
||||
source_dict = p.toolkit.get_action('harvest_source_show')(context,
|
||||
{'id': source_id})
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404, p.toolkit._('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
except Exception, e:
|
||||
msg = 'An error occurred: [%s]' % str(e)
|
||||
|
|
@ -308,20 +119,20 @@ class ViewController(BaseController):
|
|||
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
c.job = get_action('harvest_job_show')(context, {'id': id})
|
||||
c.job_report = get_action('harvest_job_report')(context, {'id': id})
|
||||
c.job = p.toolkit.get_action('harvest_job_show')(context, {'id': id})
|
||||
c.job_report = p.toolkit.get_action('harvest_job_report')(context, {'id': id})
|
||||
|
||||
if not source_dict:
|
||||
source_dict = get_action('harvest_source_show')(context, {'id': c.job['source_id']})
|
||||
source_dict = p.toolkit.get_action('harvest_source_show')(context, {'id': c.job['source_id']})
|
||||
|
||||
c.harvest_source = source_dict
|
||||
c.is_last_job = is_last
|
||||
|
||||
return render('source/job/read.html')
|
||||
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest job not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
except Exception, e:
|
||||
msg = 'An error occurred: [%s]' % str(e)
|
||||
|
|
@ -330,22 +141,22 @@ class ViewController(BaseController):
|
|||
def about(self, id):
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
c.harvest_source = get_action('harvest_source_show')(context, {'id':id})
|
||||
c.harvest_source = p.toolkit.get_action('harvest_source_show')(context, {'id':id})
|
||||
return render('source/about.html')
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
def admin(self, id):
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
p.toolkit.check_access('harvest_source_update', context, {'id': id})
|
||||
c.harvest_source = get_action('harvest_source_show')(context, {'id':id})
|
||||
c.harvest_source = p.toolkit.get_action('harvest_source_show')(context, {'id':id})
|
||||
return render('source/admin.html')
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized:
|
||||
abort(401,self.not_auth_message)
|
||||
|
||||
def show_last_job(self, source):
|
||||
|
|
@ -364,51 +175,15 @@ class ViewController(BaseController):
|
|||
|
||||
try:
|
||||
context = {'model':model, 'user':c.user}
|
||||
c.harvest_source = get_action('harvest_source_show')(context, {'id': source})
|
||||
c.jobs = get_action('harvest_job_list')(context, {'source_id': c.harvest_source['id']})
|
||||
c.harvest_source = p.toolkit.get_action('harvest_source_show')(context, {'id': source})
|
||||
c.jobs = p.toolkit.get_action('harvest_job_list')(context, {'source_id': c.harvest_source['id']})
|
||||
|
||||
return render('source/job/list.html')
|
||||
|
||||
except NotFound:
|
||||
except p.toolkit.ObjectNotFound:
|
||||
abort(404,_('Harvest source not found'))
|
||||
except NotAuthorized,e:
|
||||
except p.toolkit.NotAuthorized, e:
|
||||
abort(401,self.not_auth_message)
|
||||
except Exception, e:
|
||||
msg = 'An error occurred: [%s]' % str(e)
|
||||
abort(500,msg)
|
||||
|
||||
|
||||
def _make_autoform_items(self, harvesters_info):
|
||||
states = [{'text': 'active', 'value': 'True'},
|
||||
{'text': 'withdrawn', 'value': 'False'},]
|
||||
|
||||
harvest_list = []
|
||||
harvest_descriptions = p.toolkit.literal('<ul>')
|
||||
for harvester in harvesters_info:
|
||||
harvest_list.append({'text':harvester['title'], 'value': harvester['name']})
|
||||
harvest_descriptions += p.toolkit.literal('<li><span class="harvester-title">')
|
||||
harvest_descriptions += harvester['title']
|
||||
harvest_descriptions += p.toolkit.literal('</span>: ')
|
||||
harvest_descriptions += harvester['description']
|
||||
harvest_descriptions += p.toolkit.literal('</li>')
|
||||
harvest_descriptions += p.toolkit.literal('</ul>')
|
||||
|
||||
items = [
|
||||
{'name': 'url', 'control': 'input', 'label': _('URL'), 'placeholder': _(''), 'extra_info': 'This should include the http:// part of the URL'},
|
||||
{'name': 'type', 'control': 'select', 'options': harvest_list, 'label': _('Source type'), 'placeholder': _(''), 'extra_info': 'Which type of source does the URL above represent? '},
|
||||
{'control': 'html', 'html': harvest_descriptions},
|
||||
{'name': 'title', 'control': 'input', 'label': _('Title'), 'placeholder': _(''), 'extra_info': 'This will be shown as the datasets source.'},
|
||||
{'name': 'description', 'control': 'textarea', 'label': _('Description'), 'placeholder': _(''), 'extra_info':'You can add your own notes here about what the URL above represents to remind you later.'},]
|
||||
|
||||
if c.groups:
|
||||
pubs = []
|
||||
for group in c.groups:
|
||||
pubs.append({'text':group['title'], 'value': group['id']})
|
||||
items.append({'name': 'publisher_id', 'control': 'select', 'options': pubs, 'label': _('Publisher'), 'placeholder': _('')})
|
||||
|
||||
items += [
|
||||
{'name': 'config', 'control': 'textarea', 'label': _('Configuration'), 'placeholder': _(''), 'extra_info': ''},
|
||||
{'name': 'active', 'control': 'select', 'options': states, 'label': _('State'), 'placeholder': _(''), 'extra_text': ''},
|
||||
]
|
||||
|
||||
return items
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import re
|
||||
import logging
|
||||
|
||||
from ckan import logic
|
||||
|
|
@ -137,15 +136,3 @@ def _check_for_existing_jobs(context, source_id):
|
|||
exist = len(exist_new + exist_running) > 0
|
||||
|
||||
return exist
|
||||
|
||||
|
||||
def _error_summary(error_dict):
|
||||
error_summary = {}
|
||||
for key, error in error_dict.iteritems():
|
||||
error_summary[_prettify(key)] = error[0]
|
||||
return error_summary
|
||||
|
||||
def _prettify(field_name):
|
||||
field_name = re.sub('(?<!\w)[Uu]rl(?!\w)', 'URL', field_name.replace('_', ' ').capitalize())
|
||||
return field_name.replace('_', ' ')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
from sqlalchemy import or_, func
|
||||
from sqlalchemy import or_
|
||||
from ckan.model import User
|
||||
import datetime
|
||||
|
||||
|
|
@ -16,7 +16,7 @@ from ckanext.harvest.model import (HarvestSource, HarvestJob, HarvestObject)
|
|||
from ckanext.harvest.logic.dictization import (harvest_source_dictize,
|
||||
harvest_job_dictize,
|
||||
harvest_object_dictize)
|
||||
from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@side_effect_free
|
||||
|
|
@ -99,6 +99,9 @@ def harvest_source_show_status(context, data_dict):
|
|||
|
||||
@side_effect_free
|
||||
def harvest_source_list(context, data_dict):
|
||||
'''
|
||||
TODO: Use package search
|
||||
'''
|
||||
|
||||
check_access('harvest_source_list',context,data_dict)
|
||||
|
||||
|
|
@ -113,6 +116,10 @@ def harvest_source_list(context, data_dict):
|
|||
|
||||
@side_effect_free
|
||||
def harvest_source_for_a_dataset(context, data_dict):
|
||||
'''
|
||||
TODO: Deprecated, harvest source id is added as an extra to each dataset
|
||||
automatically
|
||||
'''
|
||||
'''For a given dataset, return the harvest source that
|
||||
created or last updated it, otherwise NotFound.'''
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,10 @@ from ckanext.harvest.model import HarvestSource, HarvestJob, HarvestObject, \
|
|||
|
||||
|
||||
def harvest_source_dictize(source, context):
|
||||
'''
|
||||
TODO: Deprecated
|
||||
'''
|
||||
|
||||
out = source.as_dict()
|
||||
|
||||
out['publisher_title'] = u''
|
||||
|
|
@ -90,6 +94,9 @@ def harvest_object_dictize(obj, context):
|
|||
return out
|
||||
|
||||
def _get_source_status(source, context):
|
||||
'''
|
||||
TODO: Deprecated, use harvest_source_show_status instead
|
||||
'''
|
||||
|
||||
model = context.get('model')
|
||||
detailed = context.get('detailed',True)
|
||||
|
|
|
|||
|
|
@ -173,25 +173,6 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
|
|||
|
||||
self.startup = True
|
||||
|
||||
auth_profile = config.get('ckan.harvest.auth.profile',None)
|
||||
|
||||
if auth_profile:
|
||||
# Check if auth profile exists
|
||||
module_root = 'ckanext.harvest.logic.auth'
|
||||
module_path = '%s.%s' % (module_root, auth_profile)
|
||||
try:
|
||||
module = __import__(module_path)
|
||||
except ImportError,e:
|
||||
raise ImportError('Unknown auth profile: %s' % auth_profile)
|
||||
|
||||
# If we are using the publisher auth profile, make sure CKAN core
|
||||
# also uses it.
|
||||
if auth_profile == 'publisher' and \
|
||||
not config.get('ckan.auth.profile','') == 'publisher':
|
||||
raise Exception('You must enable the "publisher" auth profile'
|
||||
+' in CKAN in order to use it on the harvest extension'
|
||||
+' (adding "ckan.auth.profile=publisher" to your ini file)')
|
||||
|
||||
# Setup harvest model
|
||||
model_setup()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue