CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Catch, record and recover from temporary db problems.

This commit is contained in:
David Read 2015-07-22 10:25:11 +01:00
parent 83cca925c9
commit de17e0ae8c
1 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
ckanext/harvest/queue.py
View File

@ -209,8 +209,17 @@ def gather_callback(channel, method, header, body):
# Get a publisher for the fetch queue # Get a publisher for the fetch queue
publisher = get_fetch_publisher() publisher = get_fetch_publisher()
job = HarvestJob.get(id) try:
job = HarvestJob.get(id)
except Exception, e:
# Occasionally we see:
# sqlalchemy.exc.OperationalError "SSL connection has been closed unexpectedly"
log.exception(e)
log.error('Connection Error during gather of %s: %r %r' % (id, e, e.args))
# By not sending the ack, it will be retried later.
# Try to clear the issue with a remove.
model.Session.remove()
return
if not job: if not job:
log.error('Harvest job does not exist: %s' % id) log.error('Harvest job does not exist: %s' % id)
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)
@ -280,8 +289,17 @@ def fetch_callback(channel, method, header, body):
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)
return False return False
try:
obj = HarvestObject.get(id) obj = HarvestObject.get(id)
except Exception, e:
# Occasionally we see: sqlalchemy.exc.OperationalError
# "SSL connection has been closed unexpectedly"
log.exception(e)
log.error('Connection Error during gather of %s: %r %r' % (id, e, e.args))
# By not sending the ack, it will be retried later.
# Try to clear the issue with a remove.
model.Session.remove()
return
if not obj: if not obj:
log.error('Harvest object does not exist: %s' % id) log.error('Harvest object does not exist: %s' % id)
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)