CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #192 from metaodi/consistent-api-response 

Make sure all status keys are always returned
This commit is contained in:
David Read 2015-11-17 12:34:27 +00:00
parent 5f96346487 8e02aedc65
commit cc79eeb942
2 changed files with 24 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ckanext/harvest/logic/dictization.py
View File

@ -37,7 +37,7 @@ def harvest_job_dictize(job, context):
func.count(HarvestObject.id).label('total_objects'))\ func.count(HarvestObject.id).label('total_objects'))\
.filter_by(harvest_job_id=job.id)\ .filter_by(harvest_job_id=job.id)\
.group_by(HarvestObject.report_status).all() .group_by(HarvestObject.report_status).all()
out['stats'] = {} out['stats'] = {'added': 0, 'updated': 0, 'errors': 0, 'deleted': 0}
for status, count in stats: for status, count in stats:
out['stats'][status] = count out['stats'][status] = count

45
ckanext/harvest/tests/test_queue.py
View File

@ -10,6 +10,7 @@ from ckan.plugins.core import SingletonPlugin, implements
import json import json
import ckan.logic as logic import ckan.logic as logic
from ckan import model from ckan import model
from nose.tools import assert_equal
class TestHarvester(SingletonPlugin): class TestHarvester(SingletonPlugin):
@ -32,14 +33,14 @@ class TestHarvester(SingletonPlugin):
return [] return []
def fetch_stage(self, harvest_object): def fetch_stage(self, harvest_object):
assert harvest_object.state == "FETCH" assert_equal(harvest_object.state, "FETCH")
assert harvest_object.fetch_started != None assert harvest_object.fetch_started != None
harvest_object.content = json.dumps({'name': harvest_object.guid}) harvest_object.content = json.dumps({'name': harvest_object.guid})
harvest_object.save() harvest_object.save()
return True return True
def import_stage(self, harvest_object): def import_stage(self, harvest_object):
assert harvest_object.state == "IMPORT" assert_equal(harvest_object.state, "IMPORT")
assert harvest_object.fetch_finished != None assert harvest_object.fetch_finished != None
assert harvest_object.import_started != None assert harvest_object.import_started != None
@ -172,13 +173,13 @@ class TestHarvestQueue(object):
assert count == 3 assert count == 3
all_objects = model.Session.query(HarvestObject).filter_by(current=True).all() all_objects = model.Session.query(HarvestObject).filter_by(current=True).all()
assert len(all_objects) == 3 assert_equal(len(all_objects), 3)
assert all_objects[0].state == 'COMPLETE' assert_equal(all_objects[0].state, 'COMPLETE')
assert all_objects[0].report_status == 'added' assert_equal(all_objects[0].report_status, 'added')
assert all_objects[1].state == 'COMPLETE' assert_equal(all_objects[1].state, 'COMPLETE')
assert all_objects[1].report_status == 'added' assert_equal(all_objects[1].report_status, 'added')
assert all_objects[2].state == 'COMPLETE' assert_equal(all_objects[2].state, 'COMPLETE')
assert all_objects[2].report_status == 'added' assert_equal(all_objects[2].report_status, 'added')
## fire run again to check if job is set to Finished ## fire run again to check if job is set to Finished
try: try:
@ -194,17 +195,17 @@ class TestHarvestQueue(object):
{'id': job_id} {'id': job_id}
) )
assert harvest_job['status'] == u'Finished' assert_equal(harvest_job['status'], u'Finished')
assert harvest_job['stats'] == {'added': 3} assert_equal(harvest_job['stats'], {'added': 3, 'updated': 0, 'errors': 0, 'deleted': 0})
harvest_source_dict = logic.get_action('harvest_source_show')( harvest_source_dict = logic.get_action('harvest_source_show')(
context, context,
{'id': harvest_source['id']} {'id': harvest_source['id']}
) )
assert harvest_source_dict['status']['last_job']['stats'] == {'added': 3} assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 3, 'updated': 0, 'errors': 0, 'deleted': 0})
assert harvest_source_dict['status']['total_datasets'] == 3 assert_equal(harvest_source_dict['status']['total_datasets'], 3)
assert harvest_source_dict['status']['job_count'] == 1 assert_equal(harvest_source_dict['status']['job_count'], 1)
########### Second run ######################## ########### Second run ########################
@ -242,16 +243,16 @@ class TestHarvestQueue(object):
count = model.Session.query(model.Package) \ count = model.Session.query(model.Package) \
.filter(model.Package.type=='dataset') \ .filter(model.Package.type=='dataset') \
.count() .count()
assert count == 3 assert_equal(count, 3)
all_objects = model.Session.query(HarvestObject).filter_by(report_status='added').all() all_objects = model.Session.query(HarvestObject).filter_by(report_status='added').all()
assert len(all_objects) == 3, len(all_objects) assert_equal(len(all_objects), 3)
all_objects = model.Session.query(HarvestObject).filter_by(report_status='updated').all() all_objects = model.Session.query(HarvestObject).filter_by(report_status='updated').all()
assert len(all_objects) == 2, len(all_objects) assert_equal(len(all_objects), 2)
all_objects = model.Session.query(HarvestObject).filter_by(report_status='deleted').all() all_objects = model.Session.query(HarvestObject).filter_by(report_status='deleted').all()
assert len(all_objects) == 1, len(all_objects) assert_equal(len(all_objects), 1)
# run to make sure job is marked as finshed # run to make sure job is marked as finshed
try: try:
@ -266,7 +267,7 @@ class TestHarvestQueue(object):
context, context,
{'id': job_id} {'id': job_id}
) )
assert harvest_job['stats'] == {'updated': 2, 'deleted': 1} assert_equal(harvest_job['stats'], {'added': 0, 'updated': 2, 'errors': 0, 'deleted': 1})
context['detailed'] = True context['detailed'] = True
harvest_source_dict = logic.get_action('harvest_source_show')( harvest_source_dict = logic.get_action('harvest_source_show')(
@ -274,6 +275,6 @@ class TestHarvestQueue(object):
{'id': harvest_source['id']} {'id': harvest_source['id']}
) )
assert harvest_source_dict['status']['last_job']['stats'] == {'updated': 2, 'deleted': 1} assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 0, 'updated': 2, 'errors': 0, 'deleted': 1})
assert harvest_source_dict['status']['total_datasets'] == 2 assert_equal(harvest_source_dict['status']['total_datasets'], 2)
assert harvest_source_dict['status']['job_count'] == 2 assert_equal(harvest_source_dict['status']['job_count'], 2)