CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'abort-by-job'

This commit is contained in:
amercader 2015-12-10 13:22:40 +00:00
parent 82fe2d0e53 b53682f267
commit b7a0343763
2 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ckanext/harvest/commands/harvester.py
View File

@ -38,7 +38,7 @@ class Harvester(CkanCommand):
harvester jobs harvester jobs
- lists harvest jobs - lists harvest jobs
harvester job_abort {source-id/name} harvester job_abort {source-id/source-name/obj-id}
- marks a job as "Aborted" so that the source can be restarted afresh. - marks a job as "Aborted" so that the source can be restarted afresh.
It ensures that the job's harvest objects status are also marked It ensures that the job's harvest objects status are also marked
finished. You should ensure that neither the job nor its objects are finished. You should ensure that neither the job nor its objects are
@ -358,19 +358,15 @@ class Harvester(CkanCommand):
def job_abort(self): def job_abort(self):
if len(self.args) >= 2: if len(self.args) >= 2:
source_id_or_name = unicode(self.args[1]) job_or_source_id_or_name = unicode(self.args[1])
else: else:
print 'Please provide a source id' print 'Please provide a job id or source name/id'
sys.exit(1) sys.exit(1)
context = {'model': model, 'session': model.Session,
'user': self.admin_user['name']}
source = get_action('harvest_source_show')(
context, {'id': source_id_or_name})
context = {'model': model, 'user': self.admin_user['name'], context = {'model': model, 'user': self.admin_user['name'],
'session': model.Session} 'session': model.Session}
job = get_action('harvest_job_abort')(context, job = get_action('harvest_job_abort')(
{'source_id': source['id']}) context, {'id': job_or_source_id_or_name})
print 'Job status: {0}'.format(job['status']) print 'Job status: {0}'.format(job['status'])
def run_harvester(self): def run_harvester(self):

30
ckanext/harvest/logic/action/update.py
View File

@ -495,6 +495,11 @@ def harvest_job_abort(context, data_dict):
marks them "ERROR", so any left in limbo are cleaned up. Does not actually marks them "ERROR", so any left in limbo are cleaned up. Does not actually
stop running any queued harvest fetchs/objects. stop running any queued harvest fetchs/objects.
Specify either id or source_id.
:param id: the job id to abort, or the id or name of the harvest source
with a job to abort
:type id: string
:param source_id: the name or id of the harvest source with a job to abort :param source_id: the name or id of the harvest source with a job to abort
:type source_id: string :type source_id: string
''' '''
@ -503,18 +508,25 @@ def harvest_job_abort(context, data_dict):
model = context['model'] model = context['model']
source_id = data_dict.get('source_id') source_or_job_id = data_dict.get('source_id') or data_dict.get('id')
source = harvest_source_show(context, {'id': source_id}) if source_or_job_id:
try:
# HarvestJob set status to 'Finished' source = harvest_source_show(context, {'id': source_or_job_id})
# Don not use harvest_job_list since it can use a lot of memory except NotFound:
last_job = model.Session.query(HarvestJob) \ job = get_action('harvest_job_show')(
context, {'id': source_or_job_id})
else:
# HarvestJob set status to 'Aborted'
# Do not use harvest_job_list since it can use a lot of memory
# Get the most recent job for the source
job = model.Session.query(HarvestJob) \
.filter_by(source_id=source['id']) \ .filter_by(source_id=source['id']) \
.order_by(HarvestJob.created.desc()).first() .order_by(HarvestJob.created.desc()).first()
if not last_job: if not job:
raise NotFound('Error: source has no jobs') raise NotFound('Error: source has no jobs')
job = get_action('harvest_job_show')(context, job_id = job.id
{'id': last_job.id}) job = get_action('harvest_job_show')(
context, {'id': job_id})
if job['status'] != 'Finished': if job['status'] != 'Finished':
# i.e. New or Running # i.e. New or Running