source extras field type
This commit is contained in:
kindly
parent
ebe246fe99
commit
5b50126670
|
|
@ -7,7 +7,7 @@ import ckan.plugins as p
|
|||
|
||||
from ckanext.harvest.model import UPDATE_FREQUENCIES
|
||||
from ckanext.harvest.plugin import DATASET_TYPE_NAME
|
||||
|
||||
from ckanext.harvest.interfaces import IHarvester
|
||||
|
||||
def package_list_for_source(source_id):
|
||||
'''
|
||||
|
|
@ -82,3 +82,12 @@ def link_for_harvest_object(id=None, guid=None, text=None):
|
|||
link = '<a href="{url}">{text}</a>'.format(url=url, text=text)
|
||||
|
||||
return p.toolkit.literal(link)
|
||||
|
||||
def harvest_source_extra_fields():
|
||||
fields = {}
|
||||
for harvester in p.PluginImplementations(IHarvester):
|
||||
if not hasattr(harvester, 'extra_schema'):
|
||||
continue
|
||||
fields[harvester.info()['name']] = harvester.extra_schema().keys()
|
||||
return fields
|
||||
|
||||
|
|
|
|||
|
|
@ -16,8 +16,10 @@ from ckan.lib.navl.validators import (ignore_missing,
|
|||
from ckanext.harvest.logic.validators import (harvest_source_url_validator,
|
||||
harvest_source_type_exists,
|
||||
harvest_source_config_validator,
|
||||
harvest_source_extra_validator,
|
||||
harvest_source_frequency_exists,
|
||||
dataset_type_exists,
|
||||
harvest_source_convert_from_config,
|
||||
)
|
||||
|
||||
def harvest_source_schema():
|
||||
|
|
@ -35,7 +37,6 @@ def harvest_source_schema():
|
|||
'state': [ignore_missing],
|
||||
'config': [ignore_missing, harvest_source_config_validator, convert_to_extras],
|
||||
'extras': default_extras_schema(),
|
||||
'__extras': [ignore],
|
||||
}
|
||||
|
||||
extras_schema = default_extras_schema()
|
||||
|
|
@ -48,7 +49,7 @@ def harvest_source_schema():
|
|||
def harvest_source_form_to_db_schema():
|
||||
|
||||
schema = harvest_source_schema()
|
||||
|
||||
schema['__extras'] = [harvest_source_extra_validator]
|
||||
schema['save'] = [ignore]
|
||||
schema.pop("id")
|
||||
|
||||
|
|
@ -60,7 +61,7 @@ def harvest_source_db_to_form_schema():
|
|||
schema.update({
|
||||
'source_type': [convert_from_extras, ignore_missing],
|
||||
'frequency': [convert_from_extras, ignore_missing],
|
||||
'config': [convert_from_extras, ignore_missing],
|
||||
'config': [convert_from_extras, harvest_source_convert_from_config, ignore_missing],
|
||||
'owner_org': [ignore_missing]
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import urlparse
|
||||
import json
|
||||
|
||||
from ckan.lib.navl.dictization_functions import Invalid
|
||||
from ckan.lib.navl.dictization_functions import Invalid, validate
|
||||
from ckan import model
|
||||
from ckan.plugins import PluginImplementations
|
||||
|
||||
|
|
@ -8,7 +9,7 @@ from ckanext.harvest.plugin import DATASET_TYPE_NAME
|
|||
from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES
|
||||
from ckanext.harvest.interfaces import IHarvester
|
||||
|
||||
|
||||
from ckan.lib.navl.validators import keep_extras
|
||||
|
||||
def harvest_source_id_exists(value, context):
|
||||
|
||||
|
|
@ -101,6 +102,71 @@ def harvest_source_config_validator(key,data,errors,context):
|
|||
else:
|
||||
return data[key]
|
||||
|
||||
def keep_not_empty_extras(key, data, errors, context):
|
||||
extras = data.pop(key, {})
|
||||
for extras_key, value in extras.iteritems():
|
||||
if value:
|
||||
data[key[:-1] + (extras_key,)] = value
|
||||
|
||||
def harvest_source_extra_validator(key,data,errors,context):
|
||||
harvester_type = data.get(('source_type',),'')
|
||||
|
||||
#gather all extra fields to use as whitelist of what
|
||||
#can be added to top level data_dict
|
||||
all_extra_fields = set()
|
||||
for harvester in PluginImplementations(IHarvester):
|
||||
if not hasattr(harvester, 'extra_schema'):
|
||||
continue
|
||||
all_extra_fields.update(harvester.extra_schema().keys())
|
||||
|
||||
extra_schema = {'__extras': [keep_not_empty_extras]}
|
||||
for harvester in PluginImplementations(IHarvester):
|
||||
if not hasattr(harvester, 'extra_schema'):
|
||||
continue
|
||||
info = harvester.info()
|
||||
if not info['name'] == harvester_type:
|
||||
continue
|
||||
extra_schema.update(harvester.extra_schema())
|
||||
break
|
||||
|
||||
extra_data, extra_errors = validate(data.get(key, {}), extra_schema)
|
||||
for key in extra_data.keys():
|
||||
#only allow keys that appear in at least one harvester
|
||||
if key not in all_extra_fields:
|
||||
extra_data.pop(key)
|
||||
|
||||
for key, value in extra_data.iteritems():
|
||||
data[(key,)] = value
|
||||
|
||||
for key, value in extra_errors.iteritems():
|
||||
errors[(key,)] = value
|
||||
|
||||
## need to get config out of extras as __extra runs
|
||||
## after rest of validation
|
||||
package_extras = data.get(('extras',), [])
|
||||
|
||||
for num, extra in enumerate(list(package_extras)):
|
||||
if extra['key'] == 'config':
|
||||
# remove config extra so we can add back cleanly later
|
||||
package_extras.pop(num)
|
||||
config_dict = json.loads(extra.get('value') or '{}')
|
||||
break
|
||||
else:
|
||||
config_dict = {}
|
||||
config_dict.update(extra_data)
|
||||
if config_dict:
|
||||
package_extras.append(dict(key='config',
|
||||
value=json.dumps(config_dict)))
|
||||
if package_extras:
|
||||
data[('extras',)] = package_extras
|
||||
|
||||
def harvest_source_convert_from_config(key,data,errors,context):
|
||||
config = data[key]
|
||||
if config:
|
||||
config_dict = json.loads(config)
|
||||
for key, value in config_dict.iteritems():
|
||||
data[(key,)] = value
|
||||
|
||||
def harvest_source_active_validator(value,context):
|
||||
if isinstance(value,basestring):
|
||||
if value.lower() == 'true':
|
||||
|
|
|
|||
|
|
@ -231,6 +231,7 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
|
|||
p.toolkit.add_template_directory(config, templates)
|
||||
p.toolkit.add_public_directory(config, 'public')
|
||||
p.toolkit.add_resource('fanstatic_library', 'ckanext-harvest')
|
||||
p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field')
|
||||
|
||||
## IActions
|
||||
|
||||
|
|
@ -260,6 +261,7 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
|
|||
'harvester_types': harvest_helpers.harvester_types,
|
||||
'harvest_frequencies': harvest_helpers.harvest_frequencies,
|
||||
'link_for_harvest_object': harvest_helpers.link_for_harvest_object,
|
||||
'harvest_source_extra_fields': harvest_helpers.harvest_source_extra_fields,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
|
||||
[depends]
|
||||
|
||||
main = base/main
|
||||
|
||||
[groups]
|
||||
|
||||
main =
|
||||
extra_fields.js
|
||||
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
{% import 'macros/form.html' as form %}
|
||||
{% resource 'harvest-extra-field/main' %}
|
||||
|
||||
<form id="source-new" class="form-horizontal" method="post" >
|
||||
|
||||
|
|
@ -25,9 +26,13 @@
|
|||
<label class="control-label">Source type</label>
|
||||
<div class="controls">
|
||||
{% for harvester in h.harvesters_info() %}
|
||||
{% set checked = harvester['name'] == ( data.source_type or 'ckan' ) %}
|
||||
{% set checked = False %}
|
||||
{# select first option if nothing in data #}
|
||||
{% if data.source_type == harvester['name'] or (not data.source_type and loop.first) %}
|
||||
{% set checked = True %}
|
||||
{% endif %}
|
||||
<label class="radio">
|
||||
<input type="radio" name="source_type" value="{{ harvester['name'] }}" {{ "checked " if checked }}>
|
||||
<input type="radio" name="source_type" value="{{ harvester['name'] }}" {{ "checked " if checked }} data-module="harvest-type-change">
|
||||
{{ harvester['title'] }}
|
||||
<i class="icon-question-sign" title="{{ harvester['description'] }}" data-toggle="tooltip"></i>
|
||||
</label>
|
||||
|
|
@ -37,7 +42,9 @@
|
|||
|
||||
{{ form.select('frequency', id='field-frequency', label=_('Frequency of update'), options=h.harvest_frequencies(), selected=data.frequency, error=errors.frequency) }}
|
||||
|
||||
{% block extra_config %}
|
||||
{{ form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) }}
|
||||
{% endblock extra_config %}
|
||||
|
||||
{# if we have a default group then this wants remembering #}
|
||||
{% if data.group_id %}
|
||||
|
|
|
|||
Loading…
Reference in New Issue