Merge branch 'error_creation_moved_to_model'

ckanext/harvest/harvesters/base.py

@@ -2,8 +2,7 @@ import logging
 import re
 import uuid
 
-from sqlalchemy.sql import update,and_, bindparam
+from sqlalchemy.sql import update, bindparam
-from sqlalchemy.exc import InvalidRequestError
 from pylons import config
 
 from ckan import plugins as p
@@ -12,11 +11,11 @@ from ckan.model import Session, Package, PACKAGE_NAME_MAX_LENGTH
 from ckan.logic import ValidationError, NotFound, get_action
 
 from ckan.logic.schema import default_create_package_schema
-from ckan.lib.navl.validators import ignore_missing,ignore
+from ckan.lib.navl.validators import ignore_missing, ignore
-from ckan.lib.munge import munge_title_to_name,substitute_ascii_equivalents
+from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents
 
-from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \
+from ckanext.harvest.model import (HarvestObject, HarvestGatherError,
-                                    HarvestObjectError
+                                   HarvestObjectError)
 
 from ckan.plugins.core import SingletonPlugin, implements
 from ckanext.harvest.interfaces import IHarvester
@@ -46,7 +45,11 @@ log = logging.getLogger(__name__)
 
 class HarvesterBase(SingletonPlugin):
     '''
-    Generic class for  harvesters with helper functions
+    Generic base class for harvesters, providing a number of useful functions.
+
+    A harvester doesn't have to derive from this - it could just have:
+
+        implements(IHarvester)
     '''
     implements(IHarvester)
 
@@ -149,31 +152,8 @@ class HarvesterBase(SingletonPlugin):
             return ideal_name[:PACKAGE_NAME_MAX_LENGTH-APPEND_MAX_CHARS] + \
                 str(uuid.uuid4())[:APPEND_MAX_CHARS]
 
-
+    _save_gather_error = HarvestGatherError.create
-    def _save_gather_error(self, message, job):
+    _save_object_error = HarvestObjectError.create
-        err = HarvestGatherError(message=message, job=job)
-        try:
-            err.save()
-        except InvalidRequestError:
-            Session.rollback()
-            err.save()
-        finally:
-            log.error(message)
-
-
-    def _save_object_error(self, message, obj, stage=u'Fetch', line=None):
-        err = HarvestObjectError(message=message,
-                                 object=obj,
-                                 stage=stage,
-                                 line=line)
-        try:
-            err.save()
-        except InvalidRequestError, e:
-            Session.rollback()
-            err.save()
-        finally:
-            log_message = '{0}, line {1}'.format(message,line) if line else message
-            log.debug(log_message)
 
     def _get_user_name(self):
         '''

ckanext/harvest/model/__init__.py

@@ -10,6 +10,7 @@ from sqlalchemy import ForeignKey
 from sqlalchemy import types
 from sqlalchemy.engine.reflection import Inspector
 from sqlalchemy.orm import backref, relation
+from sqlalchemy.exc import InvalidRequestError
 
 from ckan import model
 from ckan import logic
@@ -153,13 +154,43 @@ class HarvestGatherError(HarvestDomainObject):
     '''Gather errors are raised during the **gather** stage of a harvesting
        job.
     '''
-    pass
+    @classmethod
+    def create(cls, message, job):
+        '''
+        Helper function to create an error object and save it.
+        '''
+        err = cls(message=message, job=job)
+        try:
+            err.save()
+        except InvalidRequestError:
+            Session.rollback()
+            err.save()
+        finally:
+            # No need to alert administrator so don't log as an error
+            log.info(message)
+
 
 class HarvestObjectError(HarvestDomainObject):
     '''Object errors are raised during the **fetch** or **import** stage of a
        harvesting job, and are referenced to a specific harvest object.
     '''
-    pass
+    @classmethod
+    def create(cls, message, object, stage=u'Fetch', line=None):
+        '''
+        Helper function to create an error object and save it.
+        '''
+        err = cls(message=message, object=object,
+                  stage=stage, line=line)
+        try:
+            err.save()
+        except InvalidRequestError:
+            Session.rollback()
+            err.save()
+        finally:
+            log_message = '{0}, line {1}'.format(message, line) \
+                          if line else message
+            log.debug(log_message)
+
 
 def harvest_object_before_insert_listener(mapper,connection,target):
     '''