[refactoring] Return dictionaries from the common functions, and use them in the CLI"
This commit is contained in:
Adrià Mercader
parent
dca4ff328f
commit
3d32a18802
|
|
@ -121,7 +121,7 @@ class Harvester(CkanCommand):
|
|||
self.print_there_are('harvest source', sources)
|
||||
|
||||
# Create a Harvest Job for the new Source
|
||||
create_harvest_job(source.id)
|
||||
create_harvest_job(source['id'])
|
||||
print 'A new Harvest Job for this source has also been created'
|
||||
|
||||
def remove_harvest_source(self):
|
||||
|
|
@ -233,13 +233,14 @@ class Harvester(CkanCommand):
|
|||
self.print_harvest_source(source)
|
||||
|
||||
def print_harvest_source(self, source):
|
||||
print 'Source id: %s' % source.id
|
||||
print ' url: %s' % source.url
|
||||
print ' type: %s' % source.type
|
||||
print ' active: %s' % source.active
|
||||
print ' user: %s' % source.user_id
|
||||
print 'publisher: %s' % source.publisher_id
|
||||
print ' objects: %s' % len(source.objects)
|
||||
print 'Source id: %s' % source['id']
|
||||
print ' url: %s' % source['url']
|
||||
print ' type: %s' % source['type']
|
||||
print ' active: %s' % source['active']
|
||||
print ' user: %s' % source['user_id']
|
||||
print 'publisher: %s' % source['publisher_id']
|
||||
print ' jobs: %s' % len(source['jobs'])
|
||||
print ' objects: %s' % len(source['objects'])
|
||||
print ''
|
||||
|
||||
def print_harvest_jobs(self, jobs):
|
||||
|
|
@ -249,20 +250,22 @@ class Harvester(CkanCommand):
|
|||
self.print_harvest_job(job)
|
||||
|
||||
def print_harvest_job(self, job):
|
||||
print 'Job id: %s' % job.id
|
||||
print 'status: %s' % job.status
|
||||
print 'source: %s' % job.source.id
|
||||
print ' url: %s' % job.source.url
|
||||
#print "report: %s" % job.report
|
||||
#TODO: print errors
|
||||
'''
|
||||
if job.report and job.report['added']:
|
||||
for package_id in job.report['added']:
|
||||
print " doc: %s" % package_id
|
||||
if job.report and job.report['errors']:
|
||||
for msg in job.report['errors']:
|
||||
print " error: %s" % msg
|
||||
'''
|
||||
print ' Job id: %s' % job['id']
|
||||
print ' status: %s' % job['status']
|
||||
print ' source: %s' % job['source']['id']
|
||||
print ' url: %s' % job['source']['url']
|
||||
print ' objects: %s' % len(job['objects'])
|
||||
|
||||
print 'gather_errors: %s' % len(job['gather_errors'])
|
||||
if (len(job['gather_errors']) > 0):
|
||||
for error in job['gather_errors']:
|
||||
print ' %s' % error['message']
|
||||
|
||||
print 'object_errors: %s' % len(job['object_errors'])
|
||||
if (len(job['object_errors']) > 0):
|
||||
for error in job['object_errors']:
|
||||
print ' %s' % error['message']
|
||||
|
||||
print ''
|
||||
|
||||
def print_there_are(self, what, sequence, condition=''):
|
||||
|
|
|
|||
|
|
@ -7,11 +7,45 @@ from ckanext.harvest.model import HarvestSource, HarvestJob
|
|||
|
||||
log = __import__("logging").getLogger(__name__)
|
||||
|
||||
def _source_as_dict(source):
|
||||
out = source.as_dict()
|
||||
out['jobs'] = []
|
||||
out['objects'] = []
|
||||
|
||||
for job in source.jobs:
|
||||
out['jobs'].append(job.as_dict())
|
||||
|
||||
for obj in source.objects:
|
||||
out['objects'].append(obj.as_dict())
|
||||
|
||||
return out
|
||||
|
||||
def _job_as_dict(job):
|
||||
out = job.as_dict()
|
||||
out['source'] = job.source.as_dict()
|
||||
out['objects'] = []
|
||||
out['gather_errors'] = []
|
||||
out['object_errors'] = []
|
||||
|
||||
for obj in job.objects:
|
||||
out['objects'].append(obj.as_dict())
|
||||
|
||||
for error in job.gather_errors:
|
||||
out['gather_errors'].append(error.as_dict())
|
||||
|
||||
for error in job.gather_errors:
|
||||
out['object_errors'].append(error.as_dict())
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def get_harvest_source(id,default=Exception,attr=None):
|
||||
return HarvestSource.get(id,default=default,attr=attr)
|
||||
source = HarvestSource.get(id,default=default,attr=attr)
|
||||
return _source_as_dict(source)
|
||||
|
||||
def get_harvest_sources(**kwds):
|
||||
return HarvestSource.filter(**kwds).all()
|
||||
sources = HarvestSource.filter(**kwds).all()
|
||||
return [_source_as_dict(source) for source in sources]
|
||||
|
||||
def create_harvest_source(source_dict):
|
||||
if not 'url' in source_dict or not source_dict['url'] or \
|
||||
|
|
@ -26,15 +60,16 @@ def create_harvest_source(source_dict):
|
|||
source = HarvestSource()
|
||||
source.url = source_dict['url']
|
||||
source.type = source_dict['type']
|
||||
print str(source_dict['active'])
|
||||
opt = ['active','description','user_id','publisher_id']
|
||||
for o in opt:
|
||||
if o in source_dict and source_dict[o] is not None:
|
||||
source.__setattr__(o,source_dict[o])
|
||||
|
||||
source.save()
|
||||
|
||||
|
||||
return source
|
||||
return _source_as_dict(source)
|
||||
|
||||
|
||||
def delete_harvest_source(source_id):
|
||||
try:
|
||||
|
|
@ -50,10 +85,12 @@ def delete_harvest_source(source_id):
|
|||
return True
|
||||
|
||||
def get_harvest_job(id,attr=None):
|
||||
return HarvestJob.get(id,attr)
|
||||
job = HarvestJob.get(id,attr)
|
||||
return _job_as_dict(job)
|
||||
|
||||
def get_harvest_jobs(**kwds):
|
||||
return HarvestJob.filter(**kwds).all()
|
||||
jobs = HarvestJob.filter(**kwds).all()
|
||||
return [_job_as_dict(job) for job in jobs]
|
||||
|
||||
def create_harvest_job(source_id):
|
||||
# Check if source exists
|
||||
|
|
@ -72,7 +109,7 @@ def create_harvest_job(source_id):
|
|||
|
||||
job.save()
|
||||
|
||||
return job
|
||||
return _job_as_dict(job)
|
||||
|
||||
def delete_harvest_job(job_id):
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in New Issue