CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

purge_queues command now has warning about impact of Redis flushall, plus add some (log) output when you run a purge.

This commit is contained in:
David Read 2015-10-21 16:12:40 +00:00
parent 6360681a8f
commit d1f84295f8
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ckanext/harvest/commands/harvester.py
View File

@ -45,6 +45,8 @@ class Harvester(CkanCommand):
harvester purge_queues harvester purge_queues
- removes all jobs from fetch and gather queue - removes all jobs from fetch and gather queue
WARNING: if using Redis, this command purges any other data you have
in Redis too!
harvester [-j] [-o] [--segments={segments}] import [{source-id}] harvester [-j] [-o] [--segments={segments}] import [{source-id}]
- perform the import stage with the last fetched objects, for a certain - perform the import stage with the last fetched objects, for a certain

5
ckanext/harvest/queue.py
View File

@ -82,10 +82,13 @@ def purge_queues():
if backend in ('amqp', 'ampq'): if backend in ('amqp', 'ampq'):
channel = connection.channel() channel = connection.channel()
channel.queue_purge(queue=get_gather_queue_name()) channel.queue_purge(queue=get_gather_queue_name())
log.info('AMQP queue purged: %s', get_gather_queue_name())
channel.queue_purge(queue=get_fetch_queue_name()) channel.queue_purge(queue=get_fetch_queue_name())
log.info('AMQP queue purged: %s', get_fetch_queue_name())
return return
if backend == 'redis': if backend == 'redis':
connection.flushall() connection.flushall()
log.info('Redis flushed')
def resubmit_jobs(): def resubmit_jobs():
if config.get('ckan.harvest.mq.type') != 'redis': if config.get('ckan.harvest.mq.type') != 'redis':
@ -95,7 +98,7 @@ def resubmit_jobs():
for key in harvest_object_pending: for key in harvest_object_pending:
date_of_key = datetime.datetime.strptime(redis.get(key), date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f") "%Y-%m-%d %H:%M:%S.%f")
if (datetime.datetime.now() - date_of_key).seconds > 180: # 3 minuites for fetch and import max if (datetime.datetime.now() - date_of_key).seconds > 180: # 3 minutes for fetch and import max
redis.rpush('harvest_object_id', redis.rpush('harvest_object_id',
json.dumps({'harvest_object_id': key.split(':')[-1]}) json.dumps({'harvest_object_id': key.split(':')[-1]})
) )