Merge branch '29-new-idataset-form' into release-v2.0

ckanext/harvest/harvesters/base.py

@@ -10,7 +10,7 @@ from ckan import model
 from ckan.model import Session, Package
 from ckan.logic import ValidationError, NotFound, get_action
 
-from ckan.logic.schema import default_package_schema
+from ckan.logic.schema import default_create_package_schema
 from ckan.lib.navl.validators import ignore_missing,ignore
 from ckan.lib.munge import munge_title_to_name,substitute_ascii_equivalents
 
@@ -124,7 +124,7 @@ class HarvesterBase(SingletonPlugin):
         '''
         try:
             # Change default schema
-            schema = default_package_schema()
+            schema = default_create_package_schema()
             schema['id'] = [ignore_missing, unicode]
             schema['__junk'] = [ignore]
 

ckanext/harvest/logic/action/create.py

@@ -8,7 +8,7 @@ from ckanext.harvest.logic import HarvestJobExists
 from ckanext.harvest.plugin import DATASET_TYPE_NAME
 from ckanext.harvest.model import (HarvestSource, HarvestJob)
 from ckanext.harvest.logic.dictization import harvest_job_dictize
-from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema
+from ckanext.harvest.logic.schema import harvest_source_show_package_schema
 from ckanext.harvest.logic.action.get import harvest_source_list,harvest_job_list
 
 log = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ def harvest_source_create(context,data_dict):
     context['extras_as_string'] = True
     package_dict = logic.get_action('package_create')(context, data_dict)
 
-    context['schema'] = harvest_source_db_to_form_schema()
+    context['schema'] = harvest_source_show_package_schema()
     source = logic.get_action('package_show')(context, package_dict)
 
     return source

ckanext/harvest/logic/action/update.py

@@ -24,8 +24,7 @@ from ckanext.harvest.queue import get_gather_publisher
 
 from ckanext.harvest.model import HarvestSource, HarvestJob, HarvestObject
 from ckanext.harvest.logic import HarvestJobExists
-from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema
+from ckanext.harvest.logic.schema import harvest_source_show_package_schema
-
 
 from ckanext.harvest.logic.action.get import harvest_source_show, harvest_job_list, _get_sources_for_user
 
@@ -79,7 +78,7 @@ def harvest_source_update(context,data_dict):
     context['extras_as_string'] = True
     package_dict = logic.get_action('package_update')(context, data_dict)
 
-    context['schema'] = harvest_source_db_to_form_schema()
+    context['schema'] = harvest_source_show_package_schema()
     source = logic.get_action('package_show')(context, package_dict)
 
     return source

ckanext/harvest/logic/schema.py

@@ -47,7 +47,7 @@ def harvest_source_schema():
 
     return schema
 
-def harvest_source_form_to_db_schema():
+def harvest_source_create_package_schema():
 
     schema = harvest_source_schema()
     schema['__extras'] = [harvest_source_extra_validator]
@@ -56,7 +56,13 @@ def harvest_source_form_to_db_schema():
 
     return schema
 
-def harvest_source_db_to_form_schema():
+def harvest_source_update_package_schema():
+
+    schema = harvest_source_create_package_schema()
+
+    return schema
+
+def harvest_source_show_package_schema():
 
     schema = harvest_source_schema()
     schema.update({

ckanext/harvest/logic/validators.py

@@ -1,3 +1,4 @@
+import logging
 import urlparse
 import json
 
@@ -11,6 +12,8 @@ from ckanext.harvest.interfaces import IHarvester
 
 from ckan.lib.navl.validators import keep_extras
 
+log = logging.getLogger(__name__)
+
 def harvest_source_id_exists(value, context):
 
     result = HarvestSource.get(value,None)
@@ -149,7 +152,11 @@ def harvest_source_extra_validator(key,data,errors,context):
         if extra['key'] == 'config':
             # remove config extra so we can add back cleanly later
             package_extras.pop(num)
+            try:
                 config_dict = json.loads(extra.get('value') or '{}')
+            except ValueError:
+                log.error('Wrong JSON provided in config, skipping')
+                config_dict = {}
             break
     else:
         config_dict = {}

ckanext/harvest/plugin.py

@@ -135,63 +135,37 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
 
         p.toolkit.c.dataset_type = DATASET_TYPE_NAME
 
-    def form_to_db_schema_options(self, options):
-        '''
-            Similar to form_to_db_schema but with further options to allow
-            slightly different schemas, eg for creation or deletion on the API.
-        '''
-        schema = self.form_to_db_schema()
 
-        # Tweak the default schema to allow using the same id as the harvest source
+    def create_package_schema(self):
-        # if creating datasets for the harvest sources
-        if self.startup:
-            schema['id'] = [unicode]
-        return schema
-
-    def form_to_db_schema(self):
         '''
         Returns the schema for mapping package data from a form to a format
         suitable for the database.
         '''
-        from ckanext.harvest.logic.schema import harvest_source_form_to_db_schema
+        from ckanext.harvest.logic.schema import harvest_source_create_package_schema
+        schema = harvest_source_create_package_schema()
+        if self.startup:
+            schema['id'] = [unicode]
 
-        return harvest_source_form_to_db_schema()
+        return schema
 
-    def db_to_form_schema_options(self, options):
+    def update_package_schema(self):
         '''
-            Similar to db_to_form_schema but with further options to allow
+        Returns the schema for mapping package data from a form to a format
-            slightly different schemas, eg for creation or deletion on the API.
+        suitable for the database.
         '''
-        return self.db_to_form_schema()
+        from ckanext.harvest.logic.schema import harvest_source_update_package_schema
+        schema = harvest_source_update_package_schema()
 
-    def db_to_form_schema(self):
+        return schema
+
+    def show_package_schema(self):
         '''
         Returns the schema for mapping package data from the database into a
         format suitable for the form
         '''
-        from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema
+        from ckanext.harvest.logic.schema import harvest_source_show_package_schema
 
-        return harvest_source_db_to_form_schema()
+        return harvest_source_show_package_schema()
-
-    def check_data_dict(self, data_dict, schema=None):
-        '''Check if the return data is correct, mostly for checking out
-        if spammers are submitting only part of the form'''
-
-        surplus_keys_schema = ['__extras', '__junk', 'extras', 'notes',
-                               'extras_validation', 'save', 'return_to', 'type',
-                               'state', 'owner_org', 'frequency', 'config',
-                               'organization']
-
-        if not schema:
-            schema = self.form_to_db_schema()
-        schema_keys = schema.keys()
-        keys_in_schema = set(schema_keys) - set(surplus_keys_schema)
-
-        missing_keys = keys_in_schema - set(data_dict.keys())
-        if missing_keys:
-            msg = 'Incorrect form fields posted, missing %s' % missing_keys
-            log.info(msg)
-            raise dictization_functions.DataError(msg)
 
     def configure(self, config):