diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index c4aed46..1f06b05 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -3,8 +3,8 @@ import re from pprint import pprint from ckan.lib.cli import CkanCommand -#from ckanext.harvest.lib import * -#from ckanext.harvest.queue import get_gather_consumer, get_fetch_consumer +from ckanext.harvest.lib import * +from ckanext.harvest.queue import get_gather_consumer, get_fetch_consumer class Harvester(CkanCommand): '''Harvests remotely mastered metadata @@ -223,7 +223,6 @@ class Harvester(CkanCommand): print ' user: %s' % source['user_id'] print 'publisher: %s' % source['publisher_id'] print ' jobs: %s' % len(source['jobs']) - print ' objects: %s' % len(source['objects']) print '' def print_harvest_jobs(self, jobs): @@ -243,11 +242,6 @@ class Harvester(CkanCommand): if (len(job['gather_errors']) > 0): for error in job['gather_errors']: print ' %s' % error['message'] - - print 'object_errors: %s' % len(job['object_errors']) - if (len(job['object_errors']) > 0): - for error in job['object_errors']: - print ' %s' % error['message'] print '' diff --git a/ckanext/harvest/lib/__init__.py b/ckanext/harvest/lib/__init__.py index 97defb6..bc0561f 100644 --- a/ckanext/harvest/lib/__init__.py +++ b/ckanext/harvest/lib/__init__.py @@ -10,13 +10,10 @@ log = __import__("logging").getLogger(__name__) def _source_as_dict(source): out = source.as_dict() out['jobs'] = [] - out['objects'] = [] for job in source.jobs: out['jobs'].append(job.as_dict()) - for obj in source.objects: - out['objects'].append(obj.as_dict()) #TODO: Get some report data @@ -27,7 +24,6 @@ def _job_as_dict(job): out['source'] = job.source.as_dict() out['objects'] = [] out['gather_errors'] = [] - out['object_errors'] = [] for obj in job.objects: out['objects'].append(obj.as_dict()) @@ -35,9 +31,6 @@ def _job_as_dict(job): for error in job.gather_errors: out['gather_errors'].append(error.as_dict()) - for error in job.gather_errors: - out['object_errors'].append(error.as_dict()) - return out def _object_as_dict(obj): diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index de0c962..c14fa87 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -82,7 +82,10 @@ class HarvestObject(HarvestDomainObject): packages, RDF graphs, etc. ''' - pass + + @property + def source(self): + return self.job.source class HarvestGatherError(HarvestDomainObject): '''Gather errors are raised during the **gather** stage of a harvesting @@ -121,7 +124,6 @@ harvest_object_table = Table('harvest_object', metadata, Column('guid', types.UnicodeText, default=''), Column('created', DateTime, default=datetime.datetime.utcnow), Column('content', types.UnicodeText, nullable=True), - Column('source_id', types.UnicodeText, ForeignKey('harvest_source.id')), Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')), Column('fetch_started', DateTime), Column('fetch_finished', DateTime), @@ -138,7 +140,6 @@ harvest_gather_error_table = Table('harvest_gather_error',metadata, # New table harvest_object_error_table = Table('harvest_object_error',metadata, Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')), Column('harvest_object_id', types.UnicodeText, ForeignKey('harvest_object.id')), Column('message',types.UnicodeText), Column('stage', types.UnicodeText), @@ -149,10 +150,6 @@ mapper( HarvestSource, harvest_source_table, properties={ - 'objects': relation( - HarvestObject, - backref=u'source', - ), 'jobs': relation( HarvestJob, backref=u'source', @@ -196,10 +193,6 @@ mapper( HarvestObjectError, harvest_object_error_table, properties={ - 'job':relation( - HarvestJob, - backref='object_errors' - ), 'object':relation( HarvestObject, backref='errors'