CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better harvest source dataset migration 

Current implementation only checked for the first source to exist and
didn't allow to rerun the migration for other sources if there was an
error. With the new one, all non existing sources are migrated each
time.
This commit is contained in:
amercader 2013-05-24 14:49:55 +01:00
parent 1d54edfdaa
commit 1792180e4f
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
ckanext/harvest/model/__init__.py
View File

@ -72,12 +72,13 @@ def setup():
migrate_v3() migrate_v3()
# Check if this instance has harvest source datasets # Check if this instance has harvest source datasets
source_id = Session.query(HarvestSource.id).first() source_ids = Session.query(HarvestSource.id).all()
if source_id: source_package_ids = Session.query(model.Package.id).filter(model.Package.type==u'harvest').all()
pkg = Session.query(model.Package).filter(model.Package.id==source_id[0]).first() sources_to_migrate = set(source_ids) - set(source_package_ids)
if not pkg: if sources_to_migrate:
log.debug('Creating harvest source datasets from existing sources') log.debug('Creating harvest source datasets for %i existing sources', len(sources_to_migrate))
migrate_v3_create_datasets() sources_to_migrate = [s[0] for s in sources_to_migrate]
migrate_v3_create_datasets(sources_to_migrate)
else: else:
log.debug('Harvest table creation deferred') log.debug('Harvest table creation deferred')
@ -414,7 +415,7 @@ class PackageIdHarvestSourceIdMismatch(Exception):
""" """
pass pass
def migrate_v3_create_datasets(): def migrate_v3_create_datasets(source_ids=None):
import pylons import pylons
from paste.registry import Registry from paste.registry import Registry
@ -423,12 +424,20 @@ def migrate_v3_create_datasets():
registry.prepare() registry.prepare()
registry.register(pylons.translator, MockTranslator()) registry.register(pylons.translator, MockTranslator())
sources = model.Session.query(HarvestSource).all() sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource) \
.filter(HarvestSource.id.in_(source_ids)) \
.all()
if not sources: if not sources:
log.debug('No harvest sources to migrate') log.debug('No harvest sources to migrate')
return return
site_user_name = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True},{})['name'] site_user_name = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True},{})['name']
context = {'model': model, context = {'model': model,