CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[#253] Fix default_groups by saving the dicts to the config object, since saving it to the harvester object doesnt work in the real world. This is a lot more efficient than doing group_show for every dataset imported.

This commit is contained in:
David Read 2016-06-27 12:01:35 +01:00
parent b737a419dc
commit 78933fb775
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -128,20 +128,24 @@ class CKANHarvester(HarvesterBase):
raise ValueError('default_groups must be a *list* of group'
' names/ids')
if config_obj['default_groups'] and \
not isinstance(config_obj['default_groups'][0], str):
not isinstance(config_obj['default_groups'][0],
basestring):
raise ValueError('default_groups must be a list of group '
'names/ids (i.e. strings)')
# Check if default groups exist
context = {'model': model, 'user': toolkit.c.user}
self.default_group_dicts = []
config_obj['default_group_dicts'] = []
for group_name_or_id in config_obj['default_groups']:
try:
group = get_action('group_show')(
context, {'id': group_name_or_id})
self.default_group_dicts.append(group)
# save the dict to the config object, as we'll need it
# in the import_stage of every dataset
config_obj['default_group_dicts'].append(group)
except NotFound, e:
raise ValueError('Default group not found')
config = json.dumps(config_obj)
if 'default_extras' in config_obj:
if not isinstance(config_obj['default_extras'], dict):
@ -496,7 +500,7 @@ class CKANHarvester(HarvesterBase):
package_dict['groups'] = []
existing_group_ids = [g['id'] for g in package_dict['groups']]
package_dict['groups'].extend(
[g for g in self.default_group_dicts
[g for g in self.config['default_group_dicts']
if g['id'] not in existing_group_ids])
# Set default extras if needed

8
ckanext/harvest/logic/validators.py
View File

@ -134,12 +134,14 @@ def harvest_source_config_validator(key, data, errors, context):
if info['name'] == harvester_type:
if hasattr(harvester, 'validate_config'):
try:
return harvester.validate_config(data[key])
config = harvester.validate_config(data[key])
except Exception, e:
raise Invalid('Error parsing the configuration options: %s'
% e)
else:
return data[key]
if config is not None:
# save an edited config, for use during the harvest
data[key] = config
# no value is returned for this sort of validator/converter
def keep_not_empty_extras(key, data, errors, context):