diff --git a/ckanext/harvest/helpers.py b/ckanext/harvest/helpers.py index d92bdb6..6181a77 100644 --- a/ckanext/harvest/helpers.py +++ b/ckanext/harvest/helpers.py @@ -7,7 +7,7 @@ import ckan.plugins as p from ckanext.harvest.model import UPDATE_FREQUENCIES from ckanext.harvest.plugin import DATASET_TYPE_NAME - +from ckanext.harvest.interfaces import IHarvester def package_list_for_source(source_id): ''' @@ -82,3 +82,12 @@ def link_for_harvest_object(id=None, guid=None, text=None): link = '{text}'.format(url=url, text=text) return p.toolkit.literal(link) + +def harvest_source_extra_fields(): + fields = {} + for harvester in p.PluginImplementations(IHarvester): + if not hasattr(harvester, 'extra_schema'): + continue + fields[harvester.info()['name']] = harvester.extra_schema().keys() + return fields + diff --git a/ckanext/harvest/logic/schema.py b/ckanext/harvest/logic/schema.py index d1eb875..e2724a5 100644 --- a/ckanext/harvest/logic/schema.py +++ b/ckanext/harvest/logic/schema.py @@ -16,8 +16,10 @@ from ckan.lib.navl.validators import (ignore_missing, from ckanext.harvest.logic.validators import (harvest_source_url_validator, harvest_source_type_exists, harvest_source_config_validator, + harvest_source_extra_validator, harvest_source_frequency_exists, dataset_type_exists, + harvest_source_convert_from_config, ) def harvest_source_schema(): @@ -35,7 +37,6 @@ def harvest_source_schema(): 'state': [ignore_missing], 'config': [ignore_missing, harvest_source_config_validator, convert_to_extras], 'extras': default_extras_schema(), - '__extras': [ignore], } extras_schema = default_extras_schema() @@ -48,7 +49,7 @@ def harvest_source_schema(): def harvest_source_form_to_db_schema(): schema = harvest_source_schema() - + schema['__extras'] = [harvest_source_extra_validator] schema['save'] = [ignore] schema.pop("id") @@ -60,7 +61,7 @@ def harvest_source_db_to_form_schema(): schema.update({ 'source_type': [convert_from_extras, ignore_missing], 'frequency': [convert_from_extras, ignore_missing], - 'config': [convert_from_extras, ignore_missing], + 'config': [convert_from_extras, harvest_source_convert_from_config, ignore_missing], 'owner_org': [ignore_missing] }) diff --git a/ckanext/harvest/logic/validators.py b/ckanext/harvest/logic/validators.py index 97efea8..3e7f810 100644 --- a/ckanext/harvest/logic/validators.py +++ b/ckanext/harvest/logic/validators.py @@ -1,6 +1,7 @@ import urlparse +import json -from ckan.lib.navl.dictization_functions import Invalid +from ckan.lib.navl.dictization_functions import Invalid, validate from ckan import model from ckan.plugins import PluginImplementations @@ -8,7 +9,7 @@ from ckanext.harvest.plugin import DATASET_TYPE_NAME from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES from ckanext.harvest.interfaces import IHarvester - +from ckan.lib.navl.validators import keep_extras def harvest_source_id_exists(value, context): @@ -101,6 +102,71 @@ def harvest_source_config_validator(key,data,errors,context): else: return data[key] +def keep_not_empty_extras(key, data, errors, context): + extras = data.pop(key, {}) + for extras_key, value in extras.iteritems(): + if value: + data[key[:-1] + (extras_key,)] = value + +def harvest_source_extra_validator(key,data,errors,context): + harvester_type = data.get(('source_type',),'') + + #gather all extra fields to use as whitelist of what + #can be added to top level data_dict + all_extra_fields = set() + for harvester in PluginImplementations(IHarvester): + if not hasattr(harvester, 'extra_schema'): + continue + all_extra_fields.update(harvester.extra_schema().keys()) + + extra_schema = {'__extras': [keep_not_empty_extras]} + for harvester in PluginImplementations(IHarvester): + if not hasattr(harvester, 'extra_schema'): + continue + info = harvester.info() + if not info['name'] == harvester_type: + continue + extra_schema.update(harvester.extra_schema()) + break + + extra_data, extra_errors = validate(data.get(key, {}), extra_schema) + for key in extra_data.keys(): + #only allow keys that appear in at least one harvester + if key not in all_extra_fields: + extra_data.pop(key) + + for key, value in extra_data.iteritems(): + data[(key,)] = value + + for key, value in extra_errors.iteritems(): + errors[(key,)] = value + + ## need to get config out of extras as __extra runs + ## after rest of validation + package_extras = data.get(('extras',), []) + + for num, extra in enumerate(list(package_extras)): + if extra['key'] == 'config': + # remove config extra so we can add back cleanly later + package_extras.pop(num) + config_dict = json.loads(extra.get('value') or '{}') + break + else: + config_dict = {} + config_dict.update(extra_data) + if config_dict: + package_extras.append(dict(key='config', + value=json.dumps(config_dict))) + if package_extras: + data[('extras',)] = package_extras + +def harvest_source_convert_from_config(key,data,errors,context): + config = data[key] + if config: + config_dict = json.loads(config) + for key, value in config_dict.iteritems(): + data[(key,)] = value + def harvest_source_active_validator(value,context): if isinstance(value,basestring): if value.lower() == 'true': diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index a668d81..e781a9d 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -231,6 +231,7 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm): p.toolkit.add_template_directory(config, templates) p.toolkit.add_public_directory(config, 'public') p.toolkit.add_resource('fanstatic_library', 'ckanext-harvest') + p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field') ## IActions @@ -260,6 +261,7 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm): 'harvester_types': harvest_helpers.harvester_types, 'harvest_frequencies': harvest_helpers.harvest_frequencies, 'link_for_harvest_object': harvest_helpers.link_for_harvest_object, + 'harvest_source_extra_fields': harvest_helpers.harvest_source_extra_fields, } diff --git a/ckanext/harvest/public/ckanext/harvest/javascript/resource.config b/ckanext/harvest/public/ckanext/harvest/javascript/resource.config new file mode 100644 index 0000000..7a1cb79 --- /dev/null +++ b/ckanext/harvest/public/ckanext/harvest/javascript/resource.config @@ -0,0 +1,10 @@ + +[depends] + +main = base/main + +[groups] + +main = + extra_fields.js + diff --git a/ckanext/harvest/templates_new/source/new_source_form.html b/ckanext/harvest/templates_new/source/new_source_form.html index a4c9641..e39455b 100644 --- a/ckanext/harvest/templates_new/source/new_source_form.html +++ b/ckanext/harvest/templates_new/source/new_source_form.html @@ -1,4 +1,5 @@ {% import 'macros/form.html' as form %} +{% resource 'harvest-extra-field/main' %}
@@ -25,9 +26,13 @@
{% for harvester in h.harvesters_info() %} - {% set checked = harvester['name'] == ( data.source_type or 'ckan' ) %} + {% set checked = False %} + {# select first option if nothing in data #} + {% if data.source_type == harvester['name'] or (not data.source_type and loop.first) %} + {% set checked = True %} + {% endif %} @@ -37,7 +42,9 @@ {{ form.select('frequency', id='field-frequency', label=_('Frequency of update'), options=h.harvest_frequencies(), selected=data.frequency, error=errors.frequency) }} + {% block extra_config %} {{ form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) }} + {% endblock extra_config %} {# if we have a default group then this wants remembering #} {% if data.group_id %}