diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index 01a7943..90a1b14 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -2,8 +2,7 @@ import logging import re import uuid -from sqlalchemy.sql import update,and_, bindparam -from sqlalchemy.exc import InvalidRequestError +from sqlalchemy.sql import update, bindparam from pylons import config from ckan import plugins as p @@ -12,11 +11,11 @@ from ckan.model import Session, Package, PACKAGE_NAME_MAX_LENGTH from ckan.logic import ValidationError, NotFound, get_action from ckan.logic.schema import default_create_package_schema -from ckan.lib.navl.validators import ignore_missing,ignore -from ckan.lib.munge import munge_title_to_name,substitute_ascii_equivalents +from ckan.lib.navl.validators import ignore_missing, ignore +from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents -from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \ - HarvestObjectError +from ckanext.harvest.model import (HarvestObject, HarvestGatherError, + HarvestObjectError) from ckan.plugins.core import SingletonPlugin, implements from ckanext.harvest.interfaces import IHarvester @@ -46,7 +45,11 @@ log = logging.getLogger(__name__) class HarvesterBase(SingletonPlugin): ''' - Generic class for harvesters with helper functions + Generic base class for harvesters, providing a number of useful functions. + + A harvester doesn't have to derive from this - it could just have: + + implements(IHarvester) ''' implements(IHarvester) @@ -149,31 +152,8 @@ class HarvesterBase(SingletonPlugin): return ideal_name[:PACKAGE_NAME_MAX_LENGTH-APPEND_MAX_CHARS] + \ str(uuid.uuid4())[:APPEND_MAX_CHARS] - - def _save_gather_error(self, message, job): - err = HarvestGatherError(message=message, job=job) - try: - err.save() - except InvalidRequestError: - Session.rollback() - err.save() - finally: - log.error(message) - - - def _save_object_error(self, message, obj, stage=u'Fetch', line=None): - err = HarvestObjectError(message=message, - object=obj, - stage=stage, - line=line) - try: - err.save() - except InvalidRequestError, e: - Session.rollback() - err.save() - finally: - log_message = '{0}, line {1}'.format(message,line) if line else message - log.debug(log_message) + _save_gather_error = HarvestGatherError.create + _save_object_error = HarvestObjectError.create def _get_user_name(self): ''' diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index 5a813a5..a87eb58 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -10,6 +10,7 @@ from sqlalchemy import ForeignKey from sqlalchemy import types from sqlalchemy.engine.reflection import Inspector from sqlalchemy.orm import backref, relation +from sqlalchemy.exc import InvalidRequestError from ckan import model from ckan import logic @@ -153,13 +154,43 @@ class HarvestGatherError(HarvestDomainObject): '''Gather errors are raised during the **gather** stage of a harvesting job. ''' - pass + @classmethod + def create(cls, message, job): + ''' + Helper function to create an error object and save it. + ''' + err = cls(message=message, job=job) + try: + err.save() + except InvalidRequestError: + Session.rollback() + err.save() + finally: + # No need to alert administrator so don't log as an error + log.info(message) + class HarvestObjectError(HarvestDomainObject): '''Object errors are raised during the **fetch** or **import** stage of a harvesting job, and are referenced to a specific harvest object. ''' - pass + @classmethod + def create(cls, message, object, stage=u'Fetch', line=None): + ''' + Helper function to create an error object and save it. + ''' + err = cls(message=message, object=object, + stage=stage, line=line) + try: + err.save() + except InvalidRequestError: + Session.rollback() + err.save() + finally: + log_message = '{0}, line {1}'.format(message, line) \ + if line else message + log.debug(log_message) + def harvest_object_before_insert_listener(mapper,connection,target): '''