CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

take tables out of global scope at import time

This commit is contained in:
David Raznick 2011-05-31 18:02:07 +01:00
parent 2b98080266
commit 264b606c48
1 changed files with 113 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
ckanext/harvest/model/__init__.py
View File

@ -19,8 +19,17 @@ __all__ = [
] ]
harvest_source_table = None
harvest_job_table = None
harvest_object_table = None
harvest_gather_error_table = None
harvest_object_error_table = None
def setup(): def setup():
if harvest_source_table is None:
create_harvester_tables()
metadata.create_all() metadata.create_all()
class HarvestError(Exception): class HarvestError(Exception):
pass pass
@ -92,106 +101,113 @@ class HarvestObjectError(HarvestDomainObject):
''' '''
pass pass
def create_harvester_tables():
harvest_source_table = Table('harvest_source', metadata, global harvest_source_table
Column('id', types.UnicodeText, primary_key=True, default=make_uuid), global harvest_job_table
Column('url', types.UnicodeText, nullable=False), global harvest_object_table
Column('description', types.UnicodeText, default=u''), global harvest_gather_error_table
Column('config', types.UnicodeText, default=u''), global harvest_object_error_table
Column('created', DateTime, default=datetime.datetime.utcnow),
Column('type',types.UnicodeText,nullable=False),
Column('active',types.Boolean,default=True),
Column('user_id', types.UnicodeText, default=u''),
Column('publisher_id', types.UnicodeText, default=u''),
)
# Was harvesting_job
harvest_job_table = Table('harvest_job', metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('created', DateTime, default=datetime.datetime.utcnow),
Column('gather_started', DateTime),
Column('gather_finished', DateTime),
Column('source_id', types.UnicodeText, ForeignKey('harvest_source.id')),
Column('status', types.UnicodeText, default=u'New', nullable=False),
)
# Was harvested_document
harvest_object_table = Table('harvest_object', metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('guid', types.UnicodeText, default=''),
Column('gathered', DateTime, default=datetime.datetime.utcnow),
Column('fetch_started', DateTime),
Column('content', types.UnicodeText, nullable=True),
Column('fetch_finished', DateTime),
Column('metadata_modified_date', DateTime),
Column('retry_times',types.Integer),
Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')),
Column('package_id', types.UnicodeText, ForeignKey('package.id'), nullable=True),
)
# New table
harvest_gather_error_table = Table('harvest_gather_error',metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')),
Column('message', types.UnicodeText),
Column('created', DateTime, default=datetime.datetime.utcnow),
)
# New table
harvest_object_error_table = Table('harvest_object_error',metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('harvest_object_id', types.UnicodeText, ForeignKey('harvest_object.id')),
Column('message',types.UnicodeText),
Column('stage', types.UnicodeText),
Column('created', DateTime, default=datetime.datetime.utcnow),
)
mapper( harvest_source_table = Table('harvest_source', metadata,
HarvestSource, Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
harvest_source_table, Column('url', types.UnicodeText, nullable=False),
properties={ Column('description', types.UnicodeText, default=u''),
'jobs': relation( Column('config', types.UnicodeText, default=u''),
HarvestJob, Column('created', DateTime, default=datetime.datetime.utcnow),
backref=u'source', Column('type',types.UnicodeText,nullable=False),
order_by=harvest_job_table.c.created, Column('active',types.Boolean,default=True),
), Column('user_id', types.UnicodeText, default=u''),
}, Column('publisher_id', types.UnicodeText, default=u''),
) )
# Was harvesting_job
harvest_job_table = Table('harvest_job', metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('created', DateTime, default=datetime.datetime.utcnow),
Column('gather_started', DateTime),
Column('gather_finished', DateTime),
Column('source_id', types.UnicodeText, ForeignKey('harvest_source.id')),
Column('status', types.UnicodeText, default=u'New', nullable=False),
)
# Was harvested_document
harvest_object_table = Table('harvest_object', metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('guid', types.UnicodeText, default=''),
Column('gathered', DateTime, default=datetime.datetime.utcnow),
Column('fetch_started', DateTime),
Column('content', types.UnicodeText, nullable=True),
Column('fetch_finished', DateTime),
Column('metadata_modified_date', DateTime),
Column('retry_times',types.Integer),
Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')),
Column('package_id', types.UnicodeText, ForeignKey('package.id'), nullable=True),
)
# New table
harvest_gather_error_table = Table('harvest_gather_error',metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')),
Column('message', types.UnicodeText),
Column('created', DateTime, default=datetime.datetime.utcnow),
)
# New table
harvest_object_error_table = Table('harvest_object_error',metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
Column('harvest_object_id', types.UnicodeText, ForeignKey('harvest_object.id')),
Column('message',types.UnicodeText),
Column('stage', types.UnicodeText),
Column('created', DateTime, default=datetime.datetime.utcnow),
)
mapper( mapper(
HarvestJob, HarvestSource,
harvest_job_table, harvest_source_table,
) properties={
'jobs': relation(
HarvestJob,
backref=u'source',
order_by=harvest_job_table.c.created,
),
},
)
mapper( mapper(
HarvestObject, HarvestJob,
harvest_object_table, harvest_job_table,
properties={ )
'package':relation(
Package,
backref='harvest_objects',
),
'job': relation(
HarvestJob,
backref=u'objects',
),
},
)
mapper( mapper(
HarvestGatherError, HarvestObject,
harvest_gather_error_table, harvest_object_table,
properties={ properties={
'job':relation( 'package':relation(
HarvestJob, Package,
backref='gather_errors' backref='harvest_objects',
), ),
}, 'job': relation(
) HarvestJob,
backref=u'objects',
),
},
)
mapper( mapper(
HarvestObjectError, HarvestGatherError,
harvest_object_error_table, harvest_gather_error_table,
properties={ properties={
'object':relation( 'job':relation(
HarvestObject, HarvestJob,
backref='errors' backref='gather_errors'
), ),
}, },
) )
mapper(
HarvestObjectError,
harvest_object_error_table,
properties={
'object':relation(
HarvestObject,
backref='errors'
),
},
)