CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Only send harvesting jobs from active sources to queue

This commit is contained in:
Adrià Mercader 2011-04-15 15:50:32 +01:00
parent f492ed9c7a
commit d92b1df8c6
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ckanext/harvest/lib/__init__.py
View File

@ -210,6 +210,10 @@ def create_harvest_job(source_id):
except: except:
raise Exception('Source %s does not exist' % source_id) raise Exception('Source %s does not exist' % source_id)
# Check if the source is active
if not source.active:
raise Exception('Can not create jobs on inactive sources')
# Check if there already is an unrun job for this source # Check if there already is an unrun job for this source
exists = get_harvest_jobs(source=source,status=u'New') exists = get_harvest_jobs(source=source,status=u'New')
if len(exists): if len(exists):
@ -230,12 +234,15 @@ def run_harvest_jobs():
# Send each job to the gather queue # Send each job to the gather queue
publisher = get_gather_publisher() publisher = get_gather_publisher()
sent_jobs = []
for job in jobs: for job in jobs:
if job['source']['active']:
publisher.send({'harvest_job_id': job['id']}) publisher.send({'harvest_job_id': job['id']})
log.info('Sent job %s to the gather queue' % job['id']) log.info('Sent job %s to the gather queue' % job['id'])
sent_jobs.append(job)
publisher.close() publisher.close()
return jobs return sent_jobs
def get_harvest_object(id,default=Exception,attr=None): def get_harvest_object(id,default=Exception,attr=None):
obj = HarvestObject.get(id,default=default,attr=attr) obj = HarvestObject.get(id,default=default,attr=attr)