Various fixes for the harvest source dataset type forms

Add a db to form schema to show the fields stored in extras. Validate
the source url on the Package object.

ckanext/harvest/logic/schema.py

@@ -1,7 +1,9 @@
 from ckan.lib.base import config
+from ckan.logic.schema import default_extras_schema
 from ckan.logic.validators import (package_id_exists,
                                    name_validator,
                                    package_name_validator,
+                                   ignore_not_package_admin,
                                    )
 from ckan.logic.converters import convert_to_extras
 
@@ -59,8 +61,9 @@ def harvest_source_schema():
         'title': [if_empty_same_as("name"), unicode],
         'notes': [ignore_missing, unicode],
         'frequency': [ignore_missing, unicode, harvest_source_frequency_exists, convert_to_extras],
-        'state': [ignore_missing, harvest_source_active_validator],
+        'state': [ignore_not_package_admin, ignore_missing],
-        'config': [ignore_missing, harvest_source_config_validator, convert_to_extras]
+        'config': [ignore_missing, harvest_source_config_validator, convert_to_extras],
+        'extras': default_extras_schema(),
     }
 
     return schema

ckanext/harvest/logic/validators.py

@@ -1,7 +1,7 @@
 import urlparse
 
 from ckan.lib.navl.dictization_functions import Invalid
-from ckan.model import Session
+from ckan import model
 from ckan.plugins import PluginImplementations
 
 from ckanext.harvest.plugin import DATASET_TYPE_NAME
@@ -44,16 +44,26 @@ def _normalize_url(url):
     return check_url
 
 def harvest_source_url_validator(key,data,errors,context):
-    new_url = _normalize_url(data[key])
+    package = context.get("package")
-    source_id = data.get(('id',),'')
-    if source_id:
-        # When editing a source we need to avoid its own URL
-        existing_sources = Session.query(HarvestSource.url,HarvestSource.active) \
-                       .filter(HarvestSource.id!=source_id).all()
-    else:
-        existing_sources = Session.query(HarvestSource.url,HarvestSource.active).all()
 
-    for url,active in existing_sources:
+    if package:
+        package_id = package.id
+    else:
+        package_id = data.get(key[:-1] + ("id",))
+
+    new_url = _normalize_url(data[key])
+    #pkg_id = data.get(('id',),'')
+
+    q = model.Session.query(model.Package.url, model.Package.state) \
+               .filter(model.Package.type==DATASET_TYPE_NAME)
+
+    if package_id:
+        # When editing a source we need to avoid its own URL
+        q = q.filter(model.Package.id!=package_id)
+
+    existing_sources = q.all()
+
+    for url, state in existing_sources:
         url = _normalize_url(url)
         if url == new_url:
             raise Invalid('There already is a Harvest Source for this URL: %s' % data[key])

ckanext/harvest/plugin.py

@@ -3,9 +3,11 @@ from logging import getLogger
 from pylons import config
 
 from ckan import logic
+from ckan.logic.converters import convert_from_extras
 import ckan.plugins as p
 from ckan.lib.plugins import DefaultDatasetForm
 from ckan.lib.navl import dictization_functions
+from ckan.lib.navl.validators import ignore_missing
 
 from ckanext.harvest.model import setup as model_setup
 from ckanext.harvest.model import HarvestSource, HarvestJob
@@ -72,12 +74,29 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
 
         return harvest_source_form_schema()
 
+    def db_to_form_schema(self):
+        '''
+        Returns the schema for mapping package data from the database into a
+        format suitable for the form
+        '''
+        from ckanext.harvest.logic.schema import harvest_source_schema
+
+        schema = harvest_source_schema()
+        schema.update({
+            'source_type': [convert_from_extras, ignore_missing],
+            'frequency': [convert_from_extras, ignore_missing],
+            'config': [convert_from_extras, ignore_missing],
+        })
+
+        return schema
+
     def check_data_dict(self, data_dict, schema=None):
         '''Check if the return data is correct, mostly for checking out
         if spammers are submitting only part of the form'''
 
-        surplus_keys_schema = ['__extras', '__junk', 'extras_validation', 'save',
+        surplus_keys_schema = ['__extras', '__junk', 'extras',
-                               'return_to', 'type', 'state']
+                               'extras_validation', 'save', 'return_to', 'type',
+                               'state']
 
         #TODO: state and delete
 

ckanext/harvest/templates_new/source/new_source_form.html

@@ -19,7 +19,7 @@
 
   {{ form.markdown('notes', id='field-notes', label=_('Description'), value=data.notes, error=errors.notes) }}
 
-  {{ form.select('source_type', id='field-source_type', label=_('Source type'), options=c.harvester_types, error=errors.source_type) }}
+  {{ form.select('source_type', id='field-source_type', label=_('Source type'), options=c.harvester_types, selected=data.source_type, error=errors.source_type) }}
     <div class="controls">
       <ul>
       {% for harvester in c.harvesters_info %}
@@ -28,7 +28,7 @@
       </ul>
     </div>
 
-  {{ form.select('frequency', id='field-frequency', label=_('Frequency of update'), options=c.frequencies, error=errors.frequency) }}
+  {{ form.select('frequency', id='field-frequency', label=_('Frequency of update'), options=c.frequencies, selected=data.frequency, error=errors.frequency) }}
 
   {{ form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) }}