CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'db-error'

This commit is contained in:
amercader 2015-11-20 12:29:37 +00:00
parent ede50aa3fb 59be6e2c71
commit 920df684ae
1 changed files with 26 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
ckanext/harvest/queue.py
View File

@ -3,12 +3,13 @@ import datetime
import json import json
import pika import pika
import sqlalchemy
from ckan.lib.base import config from ckan.lib.base import config
from ckan.plugins import PluginImplementations from ckan.plugins import PluginImplementations
from ckan import model from ckan import model
from ckanext.harvest.model import HarvestJob, HarvestObject,HarvestGatherError from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError
from ckanext.harvest.interfaces import IHarvester from ckanext.harvest.interfaces import IHarvester
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -221,8 +222,18 @@ def gather_callback(channel, method, header, body):
# Get a publisher for the fetch queue # Get a publisher for the fetch queue
publisher = get_fetch_publisher() publisher = get_fetch_publisher()
try:
job = HarvestJob.get(id) job = HarvestJob.get(id)
except sqlalchemy.exc.OperationalError, e:
# Occasionally we see: sqlalchemy.exc.OperationalError
# "SSL connection has been closed unexpectedly"
log.exception(e)
log.error('Connection Error during gather of job %s: %r %r',
id, e, e.args)
# By not sending the ack, it will be retried later.
# Try to clear the issue with a remove.
model.Session.remove()
return
if not job: if not job:
log.error('Harvest job does not exist: %s' % id) log.error('Harvest job does not exist: %s' % id)
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)
@ -313,7 +324,18 @@ def fetch_callback(channel, method, header, body):
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)
return False return False
try:
obj = HarvestObject.get(id) obj = HarvestObject.get(id)
except sqlalchemy.exc.OperationalError, e:
# Occasionally we see: sqlalchemy.exc.OperationalError
# "SSL connection has been closed unexpectedly"
log.exception(e)
log.error('Connection Error during gather of harvest object %s: %r %r',
id, e, e.args)
# By not sending the ack, it will be retried later.
# Try to clear the issue with a remove.
model.Session.remove()
return
if not obj: if not obj:
log.error('Harvest object does not exist: %s' % id) log.error('Harvest object does not exist: %s' % id)
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)