Merge branch 'master' of github.com:ckan/ckanext-harvest

CHANGELOG.rst

@@ -12,8 +12,6 @@ Unreleased_
 ***********
 
 
-
-
 ***********
 1.3.2_ - 2020-10-08
 ***********
@@ -23,6 +21,11 @@ Changed
 
 - Calculate timeouts based on last finished object instead of job creation time #418
 
+Fixed
+-----
+
+- Fix resubmitting harvest objects to Redis fetch queue #421
+
 
 ***********
 1.3.1_ - 2020-09-01

ckanext/harvest/queue.py

@@ -167,8 +167,14 @@ def resubmit_objects():
         .filter_by(state='WAITING') \
         .all()
 
+    objects_in_queue = []
+    fetch_routing_key = get_fetch_routing_key()
+
+    objects_in_queue = [json.loads(o)['harvest_object_id']
+                        for o in redis.lrange(fetch_routing_key, 0, -1)]
+
     for object_id, in waiting_objects:
-        if not redis.get(object_id):
+        if object_id not in objects_in_queue:
             log.debug('Re-sent object {} to the fetch queue'.format(object_id))
             publisher.send({'harvest_object_id': object_id})