From 3d32a188026fc0d824e168678278c785d98a8a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Mercader?= Date: Tue, 5 Apr 2011 12:55:58 +0100 Subject: [PATCH] [refactoring] Return dictionaries from the common functions, and use them in the CLI" --- ckanext/harvest/commands/harvester.py | 47 ++++++++++++------------ ckanext/harvest/lib/__init__.py | 51 +++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index 202ae74..43bda6a 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -121,7 +121,7 @@ class Harvester(CkanCommand): self.print_there_are('harvest source', sources) # Create a Harvest Job for the new Source - create_harvest_job(source.id) + create_harvest_job(source['id']) print 'A new Harvest Job for this source has also been created' def remove_harvest_source(self): @@ -233,13 +233,14 @@ class Harvester(CkanCommand): self.print_harvest_source(source) def print_harvest_source(self, source): - print 'Source id: %s' % source.id - print ' url: %s' % source.url - print ' type: %s' % source.type - print ' active: %s' % source.active - print ' user: %s' % source.user_id - print 'publisher: %s' % source.publisher_id - print ' objects: %s' % len(source.objects) + print 'Source id: %s' % source['id'] + print ' url: %s' % source['url'] + print ' type: %s' % source['type'] + print ' active: %s' % source['active'] + print ' user: %s' % source['user_id'] + print 'publisher: %s' % source['publisher_id'] + print ' jobs: %s' % len(source['jobs']) + print ' objects: %s' % len(source['objects']) print '' def print_harvest_jobs(self, jobs): @@ -249,20 +250,22 @@ class Harvester(CkanCommand): self.print_harvest_job(job) def print_harvest_job(self, job): - print 'Job id: %s' % job.id - print 'status: %s' % job.status - print 'source: %s' % job.source.id - print ' url: %s' % job.source.url - #print "report: %s" % job.report - #TODO: print errors - ''' - if job.report and job.report['added']: - for package_id in job.report['added']: - print " doc: %s" % package_id - if job.report and job.report['errors']: - for msg in job.report['errors']: - print " error: %s" % msg - ''' + print ' Job id: %s' % job['id'] + print ' status: %s' % job['status'] + print ' source: %s' % job['source']['id'] + print ' url: %s' % job['source']['url'] + print ' objects: %s' % len(job['objects']) + + print 'gather_errors: %s' % len(job['gather_errors']) + if (len(job['gather_errors']) > 0): + for error in job['gather_errors']: + print ' %s' % error['message'] + + print 'object_errors: %s' % len(job['object_errors']) + if (len(job['object_errors']) > 0): + for error in job['object_errors']: + print ' %s' % error['message'] + print '' def print_there_are(self, what, sequence, condition=''): diff --git a/ckanext/harvest/lib/__init__.py b/ckanext/harvest/lib/__init__.py index daab33b..aa4d53e 100644 --- a/ckanext/harvest/lib/__init__.py +++ b/ckanext/harvest/lib/__init__.py @@ -7,11 +7,45 @@ from ckanext.harvest.model import HarvestSource, HarvestJob log = __import__("logging").getLogger(__name__) +def _source_as_dict(source): + out = source.as_dict() + out['jobs'] = [] + out['objects'] = [] + + for job in source.jobs: + out['jobs'].append(job.as_dict()) + + for obj in source.objects: + out['objects'].append(obj.as_dict()) + + return out + +def _job_as_dict(job): + out = job.as_dict() + out['source'] = job.source.as_dict() + out['objects'] = [] + out['gather_errors'] = [] + out['object_errors'] = [] + + for obj in job.objects: + out['objects'].append(obj.as_dict()) + + for error in job.gather_errors: + out['gather_errors'].append(error.as_dict()) + + for error in job.gather_errors: + out['object_errors'].append(error.as_dict()) + + return out + + def get_harvest_source(id,default=Exception,attr=None): - return HarvestSource.get(id,default=default,attr=attr) + source = HarvestSource.get(id,default=default,attr=attr) + return _source_as_dict(source) def get_harvest_sources(**kwds): - return HarvestSource.filter(**kwds).all() + sources = HarvestSource.filter(**kwds).all() + return [_source_as_dict(source) for source in sources] def create_harvest_source(source_dict): if not 'url' in source_dict or not source_dict['url'] or \ @@ -26,15 +60,16 @@ def create_harvest_source(source_dict): source = HarvestSource() source.url = source_dict['url'] source.type = source_dict['type'] - print str(source_dict['active']) opt = ['active','description','user_id','publisher_id'] for o in opt: if o in source_dict and source_dict[o] is not None: source.__setattr__(o,source_dict[o]) source.save() + - return source + return _source_as_dict(source) + def delete_harvest_source(source_id): try: @@ -50,10 +85,12 @@ def delete_harvest_source(source_id): return True def get_harvest_job(id,attr=None): - return HarvestJob.get(id,attr) + job = HarvestJob.get(id,attr) + return _job_as_dict(job) def get_harvest_jobs(**kwds): - return HarvestJob.filter(**kwds).all() + jobs = HarvestJob.filter(**kwds).all() + return [_job_as_dict(job) for job in jobs] def create_harvest_job(source_id): # Check if source exists @@ -72,7 +109,7 @@ def create_harvest_job(source_id): job.save() - return job + return _job_as_dict(job) def delete_harvest_job(job_id): try: