From f9b87fff0c950fb50b7fee2acc9349e6e5dbbd84 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Tue, 17 Nov 2015 11:43:11 +0100 Subject: [PATCH 1/2] Make sure all possible status are always returned It makes it harded to parse the API response if you always have to check if 'deleted' is set and if what value it has. I think to simply return 0 for all status value is good practice. --- ckanext/harvest/logic/dictization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/logic/dictization.py b/ckanext/harvest/logic/dictization.py index 23024d7..ce88ee5 100644 --- a/ckanext/harvest/logic/dictization.py +++ b/ckanext/harvest/logic/dictization.py @@ -37,7 +37,7 @@ def harvest_job_dictize(job, context): func.count(HarvestObject.id).label('total_objects'))\ .filter_by(harvest_job_id=job.id)\ .group_by(HarvestObject.report_status).all() - out['stats'] = {} + out['stats'] = {'added': 0, 'updated': 0, 'errors': 0, 'deleted': 0} for status, count in stats: out['stats'][status] = count From 8e02aedc659fba2465f65c98cbde5000bc16ddbd Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Tue, 17 Nov 2015 12:09:24 +0100 Subject: [PATCH 2/2] Fix tests --- ckanext/harvest/tests/test_queue.py | 45 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/ckanext/harvest/tests/test_queue.py b/ckanext/harvest/tests/test_queue.py index d5b3888..bdd0954 100644 --- a/ckanext/harvest/tests/test_queue.py +++ b/ckanext/harvest/tests/test_queue.py @@ -10,6 +10,7 @@ from ckan.plugins.core import SingletonPlugin, implements import json import ckan.logic as logic from ckan import model +from nose.tools import assert_equal class TestHarvester(SingletonPlugin): @@ -32,14 +33,14 @@ class TestHarvester(SingletonPlugin): return [] def fetch_stage(self, harvest_object): - assert harvest_object.state == "FETCH" + assert_equal(harvest_object.state, "FETCH") assert harvest_object.fetch_started != None harvest_object.content = json.dumps({'name': harvest_object.guid}) harvest_object.save() return True def import_stage(self, harvest_object): - assert harvest_object.state == "IMPORT" + assert_equal(harvest_object.state, "IMPORT") assert harvest_object.fetch_finished != None assert harvest_object.import_started != None @@ -172,13 +173,13 @@ class TestHarvestQueue(object): assert count == 3 all_objects = model.Session.query(HarvestObject).filter_by(current=True).all() - assert len(all_objects) == 3 - assert all_objects[0].state == 'COMPLETE' - assert all_objects[0].report_status == 'added' - assert all_objects[1].state == 'COMPLETE' - assert all_objects[1].report_status == 'added' - assert all_objects[2].state == 'COMPLETE' - assert all_objects[2].report_status == 'added' + assert_equal(len(all_objects), 3) + assert_equal(all_objects[0].state, 'COMPLETE') + assert_equal(all_objects[0].report_status, 'added') + assert_equal(all_objects[1].state, 'COMPLETE') + assert_equal(all_objects[1].report_status, 'added') + assert_equal(all_objects[2].state, 'COMPLETE') + assert_equal(all_objects[2].report_status, 'added') ## fire run again to check if job is set to Finished try: @@ -194,17 +195,17 @@ class TestHarvestQueue(object): {'id': job_id} ) - assert harvest_job['status'] == u'Finished' - assert harvest_job['stats'] == {'added': 3} + assert_equal(harvest_job['status'], u'Finished') + assert_equal(harvest_job['stats'], {'added': 3, 'updated': 0, 'errors': 0, 'deleted': 0}) harvest_source_dict = logic.get_action('harvest_source_show')( context, {'id': harvest_source['id']} ) - assert harvest_source_dict['status']['last_job']['stats'] == {'added': 3} - assert harvest_source_dict['status']['total_datasets'] == 3 - assert harvest_source_dict['status']['job_count'] == 1 + assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 3, 'updated': 0, 'errors': 0, 'deleted': 0}) + assert_equal(harvest_source_dict['status']['total_datasets'], 3) + assert_equal(harvest_source_dict['status']['job_count'], 1) ########### Second run ######################## @@ -242,16 +243,16 @@ class TestHarvestQueue(object): count = model.Session.query(model.Package) \ .filter(model.Package.type=='dataset') \ .count() - assert count == 3 + assert_equal(count, 3) all_objects = model.Session.query(HarvestObject).filter_by(report_status='added').all() - assert len(all_objects) == 3, len(all_objects) + assert_equal(len(all_objects), 3) all_objects = model.Session.query(HarvestObject).filter_by(report_status='updated').all() - assert len(all_objects) == 2, len(all_objects) + assert_equal(len(all_objects), 2) all_objects = model.Session.query(HarvestObject).filter_by(report_status='deleted').all() - assert len(all_objects) == 1, len(all_objects) + assert_equal(len(all_objects), 1) # run to make sure job is marked as finshed try: @@ -266,7 +267,7 @@ class TestHarvestQueue(object): context, {'id': job_id} ) - assert harvest_job['stats'] == {'updated': 2, 'deleted': 1} + assert_equal(harvest_job['stats'], {'added': 0, 'updated': 2, 'errors': 0, 'deleted': 1}) context['detailed'] = True harvest_source_dict = logic.get_action('harvest_source_show')( @@ -274,6 +275,6 @@ class TestHarvestQueue(object): {'id': harvest_source['id']} ) - assert harvest_source_dict['status']['last_job']['stats'] == {'updated': 2, 'deleted': 1} - assert harvest_source_dict['status']['total_datasets'] == 2 - assert harvest_source_dict['status']['job_count'] == 2 + assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 0, 'updated': 2, 'errors': 0, 'deleted': 1}) + assert_equal(harvest_source_dict['status']['total_datasets'], 2) + assert_equal(harvest_source_dict['status']['job_count'], 2)