CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement retry times for harvest objects

This commit is contained in:
amercader 2012-11-15 18:11:35 +00:00
parent 202c9d9fcc
commit 03fd1884f4
2 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ckanext/harvest/model/__init__.py
View File

@ -200,7 +200,7 @@ def define_harvester_tables():
Column('import_finished', types.DateTime), Column('import_finished', types.DateTime),
Column('state', types.UnicodeText, default=u'WAITING'), Column('state', types.UnicodeText, default=u'WAITING'),
Column('metadata_modified_date', types.DateTime), Column('metadata_modified_date', types.DateTime),
Column('retry_times',types.Integer), Column('retry_times',types.Integer, default=0),
Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')), Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')),
Column('harvest_source_id', types.UnicodeText, ForeignKey('harvest_source.id')), Column('harvest_source_id', types.UnicodeText, ForeignKey('harvest_source.id')),
Column('package_id', types.UnicodeText, ForeignKey('package.id'), nullable=True), Column('package_id', types.UnicodeText, ForeignKey('package.id'), nullable=True),
@ -375,6 +375,7 @@ ALTER TABLE harvest_object_extra
ADD CONSTRAINT harvest_object_extra_harvest_object_id_fkey FOREIGN KEY (harvest_object_id) REFERENCES harvest_object(id); ADD CONSTRAINT harvest_object_extra_harvest_object_id_fkey FOREIGN KEY (harvest_object_id) REFERENCES harvest_object(id);
UPDATE harvest_object set state = 'COMPLETE'; UPDATE harvest_object set state = 'COMPLETE';
UPDATE harvest_object set retry_times = 0;
UPDATE harvest_source set frequency = 'MANUAL'; UPDATE harvest_source set frequency = 'MANUAL';
""" """

9
ckanext/harvest/queue.py
View File

@ -158,6 +158,14 @@ def fetch_callback(channel, method, header, body):
channel.basic_ack(method.delivery_tag) channel.basic_ack(method.delivery_tag)
return False return False
obj.retry_times += 1
obj.save()
if obj.retry_times >= 5:
log.error('Too many consecutive retries for object {0}'.format(obj.id))
channel.basic_ack(method.delivery_tag)
return False
# Send the harvest object to the plugins that implement # Send the harvest object to the plugins that implement
# the Harvester interface, only if the source type # the Harvester interface, only if the source type
# matches # matches
@ -171,7 +179,6 @@ def fetch_callback(channel, method, header, body):
success = harvester.fetch_stage(obj) success = harvester.fetch_stage(obj)
obj.fetch_finished = datetime.datetime.now() obj.fetch_finished = datetime.datetime.now()
obj.save() obj.save()
#TODO: retry times?
if success: if success:
# If no errors where found, call the import method # If no errors where found, call the import method
obj.import_started = datetime.datetime.now() obj.import_started = datetime.datetime.now()