CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #200 from ckan/fix-error-report-status 

Fix job stats key
This commit is contained in:
David Read 2015-11-30 17:49:14 +00:00
parent 154a3f42cd 6c3efe53df
commit 673a64a820
2 changed files with 74 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
ckanext/harvest/logic/dictization.py
View File

@ -1,9 +1,8 @@
from sqlalchemy import distinct, func from sqlalchemy import distinct, func
import ckan.logic as logic
from ckan.model import Package, Group from ckan.model import Package, Group
from ckanext.harvest.model import HarvestSource, HarvestJob, HarvestObject, \ from ckanext.harvest.model import (HarvestSource, HarvestJob, HarvestObject,
HarvestGatherError, HarvestObjectError HarvestGatherError, HarvestObjectError)
def harvest_source_dictize(source, context): def harvest_source_dictize(source, context):
@ -23,9 +22,9 @@ def harvest_source_dictize(source, context):
out['status'] = _get_source_status(source, context) out['status'] = _get_source_status(source, context)
return out return out
def harvest_job_dictize(job, context): def harvest_job_dictize(job, context):
out = job.as_dict() out = job.as_dict()
@ -38,13 +37,14 @@ def harvest_job_dictize(job, context):
.filter_by(harvest_job_id=job.id)\ .filter_by(harvest_job_id=job.id)\
.group_by(HarvestObject.report_status).all() .group_by(HarvestObject.report_status).all()
out['stats'] = {'added': 0, 'updated': 0, 'not modified': 0, out['stats'] = {'added': 0, 'updated': 0, 'not modified': 0,
'errors': 0, 'deleted': 0} 'errored': 0, 'deleted': 0}
for status, count in stats: for status, count in stats:
out['stats'][status] = count out['stats'][status] = count
# We actually want to check which objects had errors, because they # We actually want to check which objects had errors, because they
# could have been added/updated anyway (eg bbox errors) # could have been added/updated anyway (eg bbox errors)
count = model.Session.query(func.distinct(HarvestObjectError.harvest_object_id)) \ count = model.Session.query(
func.distinct(HarvestObjectError.harvest_object_id)) \
.join(HarvestObject) \ .join(HarvestObject) \
.filter(HarvestObject.harvest_job_id == job.id) \ .filter(HarvestObject.harvest_job_id == job.id) \
.count() .count()
@ -59,7 +59,8 @@ def harvest_job_dictize(job, context):
out['stats']['errored'] = out['stats'].get('errored', 0) + count out['stats']['errored'] = out['stats'].get('errored', 0) + count
if context.get('return_error_summary', True): if context.get('return_error_summary', True):
q = model.Session.query(HarvestObjectError.message, \ q = model.Session.query(
HarvestObjectError.message,
func.count(HarvestObjectError.message).label('error_count')) \ func.count(HarvestObjectError.message).label('error_count')) \
.join(HarvestObject) \ .join(HarvestObject) \
.filter(HarvestObject.harvest_job_id == job.id) \ .filter(HarvestObject.harvest_job_id == job.id) \
@ -67,7 +68,8 @@ def harvest_job_dictize(job, context):
.order_by('error_count desc') \ .order_by('error_count desc') \
.limit(context.get('error_summmary_limit', 20)) .limit(context.get('error_summmary_limit', 20))
out['object_error_summary'] = q.all() out['object_error_summary'] = q.all()
q = model.Session.query(HarvestGatherError.message, \ q = model.Session.query(
HarvestGatherError.message,
func.count(HarvestGatherError.message).label('error_count')) \ func.count(HarvestGatherError.message).label('error_count')) \
.filter(HarvestGatherError.harvest_job_id == job.id) \ .filter(HarvestGatherError.harvest_job_id == job.id) \
.group_by(HarvestGatherError.message) \ .group_by(HarvestGatherError.message) \
@ -76,6 +78,7 @@ def harvest_job_dictize(job, context):
out['gather_error_summary'] = q.all() out['gather_error_summary'] = q.all()
return out return out
def harvest_object_dictize(obj, context): def harvest_object_dictize(obj, context):
out = obj.as_dict() out = obj.as_dict()
out['source'] = obj.harvest_source_id out['source'] = obj.harvest_source_id
@ -94,6 +97,7 @@ def harvest_object_dictize(obj, context):
return out return out
def _get_source_status(source, context): def _get_source_status(source, context):
''' '''
TODO: Deprecated, use harvest_source_show_status instead TODO: Deprecated, use harvest_source_show_status instead
@ -110,7 +114,8 @@ def _get_source_status(source, context):
'job_count': 0, 'job_count': 0,
'next_harvest': '', 'next_harvest': '',
'last_harvest_request': '', 'last_harvest_request': '',
'last_harvest_statistics':{'added':0,'updated':0,'errors':0,'deleted':0}, 'last_harvest_statistics':
{'added': 0, 'updated': 0, 'errors': 0, 'deleted': 0},
'last_harvest_errors': {'gather': [], 'object': []}, 'last_harvest_errors': {'gather': [], 'object': []},
'overall_statistics': {'added': 0, 'errors': 0}, 'overall_statistics': {'added': 0, 'errors': 0},
'packages': []} 'packages': []}
@ -143,23 +148,28 @@ def _get_source_status(source, context):
statistics['added'] = harvest_job_dict['stats'].get('new', 0) statistics['added'] = harvest_job_dict['stats'].get('new', 0)
statistics['updated'] = harvest_job_dict['stats'].get('updated', 0) statistics['updated'] = harvest_job_dict['stats'].get('updated', 0)
statistics['deleted'] = harvest_job_dict['stats'].get('deleted', 0) statistics['deleted'] = harvest_job_dict['stats'].get('deleted', 0)
statistics['errors'] = (harvest_job_dict['stats'].get('errored',0) + statistics['errors'] = (
harvest_job_dict['stats'].get('errored', 0) +
len(last_job.gather_errors)) len(last_job.gather_errors))
if detailed: if detailed:
# We have the gathering errors in last_job.gather_errors, so let's also # We have the gathering errors in last_job.gather_errors, so let's
# get also the object errors. # also get also the object errors.
object_errors = model.Session.query(HarvestObjectError).join(HarvestObject) \ object_errors = model.Session.query(HarvestObjectError)\
.join(HarvestObject) \
.filter(HarvestObject.job == last_job) .filter(HarvestObject.job == last_job)
for gather_error in last_job.gather_errors: for gather_error in last_job.gather_errors:
out['last_harvest_errors']['gather'].append(gather_error.message) out['last_harvest_errors']['gather'].append(gather_error.message)
for object_error in object_errors: for object_error in object_errors:
err = {'object_id':object_error.object.id,'object_guid':object_error.object.guid,'message': object_error.message} err = {'object_id': object_error.object.id,
'object_guid': object_error.object.guid,
'message': object_error.message}
out['last_harvest_errors']['object'].append(err) out['last_harvest_errors']['object'].append(err)
# Overall statistics # Overall statistics
packages = model.Session.query(distinct(HarvestObject.package_id),Package.name) \ packages = model.Session.query(distinct(HarvestObject.package_id),
Package.name) \
.join(Package).join(HarvestSource) \ .join(Package).join(HarvestSource) \
.filter(HarvestObject.source == source) \ .filter(HarvestObject.source == source) \
.filter(HarvestObject.current == True) \ .filter(HarvestObject.current == True) \
@ -182,4 +192,3 @@ def _get_source_status(source, context):
out['last_harvest_request'] = 'Not yet harvested' out['last_harvest_request'] = 'Not yet harvested'
return out return out

8
ckanext/harvest/tests/test_queue.py
View File

@ -189,14 +189,14 @@ class TestHarvestQueue(object):
) )
assert_equal(harvest_job['status'], u'Finished') assert_equal(harvest_job['status'], u'Finished')
assert_equal(harvest_job['stats'], {'added': 3, 'updated': 0, 'not modified': 0, 'errors': 0, 'deleted': 0}) assert_equal(harvest_job['stats'], {'added': 3, 'updated': 0, 'not modified': 0, 'errored': 0, 'deleted': 0})
harvest_source_dict = logic.get_action('harvest_source_show')( harvest_source_dict = logic.get_action('harvest_source_show')(
context, context,
{'id': harvest_source['id']} {'id': harvest_source['id']}
) )
assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 3, 'updated': 0, 'not modified': 0, 'errors': 0, 'deleted': 0}) assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 3, 'updated': 0, 'not modified': 0, 'errored': 0, 'deleted': 0})
assert_equal(harvest_source_dict['status']['total_datasets'], 3) assert_equal(harvest_source_dict['status']['total_datasets'], 3)
assert_equal(harvest_source_dict['status']['job_count'], 1) assert_equal(harvest_source_dict['status']['job_count'], 1)
@ -252,7 +252,7 @@ class TestHarvestQueue(object):
context, context,
{'id': job_id} {'id': job_id}
) )
assert_equal(harvest_job['stats'], {'added': 0, 'updated': 2, 'not modified': 0, 'errors': 0, 'deleted': 1}) assert_equal(harvest_job['stats'], {'added': 0, 'updated': 2, 'not modified': 0, 'errored': 0, 'deleted': 1})
context['detailed'] = True context['detailed'] = True
harvest_source_dict = logic.get_action('harvest_source_show')( harvest_source_dict = logic.get_action('harvest_source_show')(
@ -260,6 +260,6 @@ class TestHarvestQueue(object):
{'id': harvest_source['id']} {'id': harvest_source['id']}
) )
assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 0, 'updated': 2, 'not modified': 0, 'errors': 0, 'deleted': 1}) assert_equal(harvest_source_dict['status']['last_job']['stats'], {'added': 0, 'updated': 2, 'not modified': 0, 'errored': 0, 'deleted': 1})
assert_equal(harvest_source_dict['status']['total_datasets'], 2) assert_equal(harvest_source_dict['status']['total_datasets'], 2)
assert_equal(harvest_source_dict['status']['job_count'], 2) assert_equal(harvest_source_dict['status']['job_count'], 2)