diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index 0960a38..9a6a9df 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -10,7 +10,7 @@ from ckan import model from ckan.model import Session, Package from ckan.logic import ValidationError, NotFound, get_action -from ckan.logic.schema import default_package_schema +from ckan.logic.schema import default_create_package_schema from ckan.lib.navl.validators import ignore_missing,ignore from ckan.lib.munge import munge_title_to_name,substitute_ascii_equivalents @@ -124,7 +124,7 @@ class HarvesterBase(SingletonPlugin): ''' try: # Change default schema - schema = default_package_schema() + schema = default_create_package_schema() schema['id'] = [ignore_missing, unicode] schema['__junk'] = [ignore] diff --git a/ckanext/harvest/logic/action/create.py b/ckanext/harvest/logic/action/create.py index aceff8e..0f2933a 100644 --- a/ckanext/harvest/logic/action/create.py +++ b/ckanext/harvest/logic/action/create.py @@ -8,7 +8,7 @@ from ckanext.harvest.logic import HarvestJobExists from ckanext.harvest.plugin import DATASET_TYPE_NAME from ckanext.harvest.model import (HarvestSource, HarvestJob) from ckanext.harvest.logic.dictization import harvest_job_dictize -from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema +from ckanext.harvest.logic.schema import harvest_source_show_package_schema from ckanext.harvest.logic.action.get import harvest_source_list,harvest_job_list log = logging.getLogger(__name__) @@ -58,7 +58,7 @@ def harvest_source_create(context,data_dict): context['extras_as_string'] = True package_dict = logic.get_action('package_create')(context, data_dict) - context['schema'] = harvest_source_db_to_form_schema() + context['schema'] = harvest_source_show_package_schema() source = logic.get_action('package_show')(context, package_dict) return source diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index ddd0b05..59550ff 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -24,8 +24,7 @@ from ckanext.harvest.queue import get_gather_publisher from ckanext.harvest.model import HarvestSource, HarvestJob, HarvestObject from ckanext.harvest.logic import HarvestJobExists -from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema - +from ckanext.harvest.logic.schema import harvest_source_show_package_schema from ckanext.harvest.logic.action.get import harvest_source_show, harvest_job_list, _get_sources_for_user @@ -79,7 +78,7 @@ def harvest_source_update(context,data_dict): context['extras_as_string'] = True package_dict = logic.get_action('package_update')(context, data_dict) - context['schema'] = harvest_source_db_to_form_schema() + context['schema'] = harvest_source_show_package_schema() source = logic.get_action('package_show')(context, package_dict) return source diff --git a/ckanext/harvest/logic/schema.py b/ckanext/harvest/logic/schema.py index 56ecd3e..7a0c9e2 100644 --- a/ckanext/harvest/logic/schema.py +++ b/ckanext/harvest/logic/schema.py @@ -47,7 +47,7 @@ def harvest_source_schema(): return schema -def harvest_source_form_to_db_schema(): +def harvest_source_create_package_schema(): schema = harvest_source_schema() schema['__extras'] = [harvest_source_extra_validator] @@ -56,7 +56,13 @@ def harvest_source_form_to_db_schema(): return schema -def harvest_source_db_to_form_schema(): +def harvest_source_update_package_schema(): + + schema = harvest_source_create_package_schema() + + return schema + +def harvest_source_show_package_schema(): schema = harvest_source_schema() schema.update({ diff --git a/ckanext/harvest/logic/validators.py b/ckanext/harvest/logic/validators.py index 907cd1c..5a6111d 100644 --- a/ckanext/harvest/logic/validators.py +++ b/ckanext/harvest/logic/validators.py @@ -1,3 +1,4 @@ +import logging import urlparse import json @@ -11,6 +12,8 @@ from ckanext.harvest.interfaces import IHarvester from ckan.lib.navl.validators import keep_extras +log = logging.getLogger(__name__) + def harvest_source_id_exists(value, context): result = HarvestSource.get(value,None) @@ -149,7 +152,11 @@ def harvest_source_extra_validator(key,data,errors,context): if extra['key'] == 'config': # remove config extra so we can add back cleanly later package_extras.pop(num) - config_dict = json.loads(extra.get('value') or '{}') + try: + config_dict = json.loads(extra.get('value') or '{}') + except ValueError: + log.error('Wrong JSON provided in config, skipping') + config_dict = {} break else: config_dict = {} diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index 868a9a0..46b7b20 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -135,63 +135,37 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm): p.toolkit.c.dataset_type = DATASET_TYPE_NAME - def form_to_db_schema_options(self, options): - ''' - Similar to form_to_db_schema but with further options to allow - slightly different schemas, eg for creation or deletion on the API. - ''' - schema = self.form_to_db_schema() - # Tweak the default schema to allow using the same id as the harvest source - # if creating datasets for the harvest sources - if self.startup: - schema['id'] = [unicode] - return schema - - def form_to_db_schema(self): + def create_package_schema(self): ''' Returns the schema for mapping package data from a form to a format suitable for the database. ''' - from ckanext.harvest.logic.schema import harvest_source_form_to_db_schema + from ckanext.harvest.logic.schema import harvest_source_create_package_schema + schema = harvest_source_create_package_schema() + if self.startup: + schema['id'] = [unicode] - return harvest_source_form_to_db_schema() + return schema - def db_to_form_schema_options(self, options): + def update_package_schema(self): ''' - Similar to db_to_form_schema but with further options to allow - slightly different schemas, eg for creation or deletion on the API. + Returns the schema for mapping package data from a form to a format + suitable for the database. ''' - return self.db_to_form_schema() + from ckanext.harvest.logic.schema import harvest_source_update_package_schema + schema = harvest_source_update_package_schema() - def db_to_form_schema(self): + return schema + + def show_package_schema(self): ''' Returns the schema for mapping package data from the database into a format suitable for the form ''' - from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema + from ckanext.harvest.logic.schema import harvest_source_show_package_schema - return harvest_source_db_to_form_schema() - - def check_data_dict(self, data_dict, schema=None): - '''Check if the return data is correct, mostly for checking out - if spammers are submitting only part of the form''' - - surplus_keys_schema = ['__extras', '__junk', 'extras', 'notes', - 'extras_validation', 'save', 'return_to', 'type', - 'state', 'owner_org', 'frequency', 'config', - 'organization'] - - if not schema: - schema = self.form_to_db_schema() - schema_keys = schema.keys() - keys_in_schema = set(schema_keys) - set(surplus_keys_schema) - - missing_keys = keys_in_schema - set(data_dict.keys()) - if missing_keys: - msg = 'Incorrect form fields posted, missing %s' % missing_keys - log.info(msg) - raise dictization_functions.DataError(msg) + return harvest_source_show_package_schema() def configure(self, config):