CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #192 from metaodi/consistent-api-response 

Make sure all status keys are always returned
This commit is contained in:
David Read 2015-11-17 12:34:27 +00:00
parent 5f96346487 8e02aedc65
commit cc79eeb942
2 changed files with 24 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ckanext/harvest/logic/dictization.py
View File

@ -37,7 +37,7 @@ def harvest_job_dictize(job, context):
func.count(HarvestObject.id).label('total_objects'))\
.filter_by(harvest_job_id=job.id)\
.group_by(HarvestObject.report_status).all()
out['stats'] = {}
out['stats'] = {'added': 0, 'updated': 0, 'errors': 0, 'deleted': 0}
for status, count in stats:
out['stats'][status] = count

45
ckanext/harvest/tests/test_queue.py
View File

@ -10,6 +10,7 @@ from ckan.plugins.core import SingletonPlugin, implements
import json
import ckan.logic as logic
from ckan import model
from nose.tools import assert_equal
class TestHarvester(SingletonPlugin):
@ -32,14 +33,14 @@ class TestHarvester(SingletonPlugin):
return []
def fetch_stage(self, harvest_object):
assert harvest_object.state == "FETCH"
assert_equal(harvest_object.state, "FETCH")
assert harvest_object.fetch_started != None
harvest_object.content = json.dumps({'name': harvest_object.guid})
harvest_object.save()
return True
def import_stage(self, harvest_object):
assert harvest_object.state == "IMPORT"
assert_equal(harvest_object.state, "IMPORT")
assert harvest_object.fetch_finished != None
assert harvest_object.import_started != None
@ -172,13 +173,13 @@ class TestHarvestQueue(object):
assert count == 3
all_objects = model.Session.query(HarvestObject).filter_by(current=True).all()
assert len(all_objects) == 3
assert all_objects[0].state == 'COMPLETE'
assert all_objects[0].report_status == 'added'
assert all_objects[1].state == 'COMPLETE'
assert all_objects[1].report_status == 'added'
assert all_objects[2].state == 'COMPLETE'
assert all_objects[2].report_status == 'added'
assert_equal(len(all_objects), 3)
assert_equal(all_objects[0].state, 'COMPLETE')
assert_equal(all_objects[0].report_status, 'added')
assert_equal(all_objects[1].state, 'COMPLETE')
assert_equal(all_objects[1].report_status, 'added')
assert_equal(all_objects[2].state, 'COMPLETE')
assert_equal(all_objects[2].report_status, 'added')
## fire run again to check if job is set to Finished
try:
@ -194,17 +195,17 @@ class TestHarvestQueue(object):
{'id': job_id}
)
assert harvest_job['status'] == u'Finished'
assert harvest_job['stats'] == {'added': 3}
assert_equal(harvest_job['status'], u'Finished')
assert_equal(harvest_job['stats'], {'added': 3, 'updated': 0, 'errors': 0, 'deleted': 0})
harvest_source_dict = logic.get_action('harvest_source_show')(
context,
{'id': harvest_source['id']}
)
assert harvest_source_dict['status']['last_job']['stats'] == {'added': 3}
assert harvest_source_dict['status']['total_datasets'] == 3
assert harvest_source_dict['status']['job_count'] == 1
assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 3, 'updated': 0, 'errors': 0, 'deleted': 0})
assert_equal(harvest_source_dict['status']['total_datasets'], 3)
assert_equal(harvest_source_dict['status']['job_count'], 1)
########### Second run ########################
@ -242,16 +243,16 @@ class TestHarvestQueue(object):
count = model.Session.query(model.Package) \
.filter(model.Package.type=='dataset') \
.count()
assert count == 3
assert_equal(count, 3)
all_objects = model.Session.query(HarvestObject).filter_by(report_status='added').all()
assert len(all_objects) == 3, len(all_objects)
assert_equal(len(all_objects), 3)
all_objects = model.Session.query(HarvestObject).filter_by(report_status='updated').all()
assert len(all_objects) == 2, len(all_objects)
assert_equal(len(all_objects), 2)
all_objects = model.Session.query(HarvestObject).filter_by(report_status='deleted').all()
assert len(all_objects) == 1, len(all_objects)
assert_equal(len(all_objects), 1)
# run to make sure job is marked as finshed
try:
@ -266,7 +267,7 @@ class TestHarvestQueue(object):
context,
{'id': job_id}
)
assert harvest_job['stats'] == {'updated': 2, 'deleted': 1}
assert_equal(harvest_job['stats'], {'added': 0, 'updated': 2, 'errors': 0, 'deleted': 1})
context['detailed'] = True
harvest_source_dict = logic.get_action('harvest_source_show')(
@ -274,6 +275,6 @@ class TestHarvestQueue(object):
{'id': harvest_source['id']}
)
assert harvest_source_dict['status']['last_job']['stats'] == {'updated': 2, 'deleted': 1}
assert harvest_source_dict['status']['total_datasets'] == 2
assert harvest_source_dict['status']['job_count'] == 2
assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 0, 'updated': 2, 'errors': 0, 'deleted': 1})
assert_equal(harvest_source_dict['status']['total_datasets'], 2)
assert_equal(harvest_source_dict['status']['job_count'], 2)