CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed creation/import of groups and organizations when harvesting from remote ckan instance

This commit is contained in:
Petar Efnushev 2016-05-20 16:38:48 +02:00
parent 28c5c68c36
commit c154365371
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -60,12 +60,17 @@ class CKANHarvester(HarvesterBase):
raise ContentFetchError('HTTP general exception: %s' % e) raise ContentFetchError('HTTP general exception: %s' % e)
return http_response.read() return http_response.read()
def _get_group(self, base_url, group_name): def _get_group(self, base_url, group):
url = base_url + self._get_action_api_offset() + '/group_show?id=' + \ url = base_url + self._get_action_api_offset() + '/group_show?id=' + \
munge_name(group_name) group['id']
try: try:
content = self._get_content(url) content = self._get_content(url)
return json.loads(content) data = json.loads(content)
if self.action_api_version == 3:
return data.pop('result')
return data
except (ContentFetchError, ValueError): except (ContentFetchError, ValueError):
log.debug('Could not fetch/decode remote group') log.debug('Could not fetch/decode remote group')
raise RemoteResourceError('Could not fetch/decode remote group') raise RemoteResourceError('Could not fetch/decode remote group')
@ -123,10 +128,10 @@ class CKANHarvester(HarvesterBase):
# Check if default groups exist # Check if default groups exist
context = {'model': model, 'user': c.user} context = {'model': model, 'user': c.user}
for group_name in config_obj['default_groups']: for group_ in config_obj['default_groups']:
try: try:
group = get_action('group_show')( group = get_action('group_show')(
context, {'id': group_name}) context, {'id': group_['id']})
except NotFound, e: except NotFound, e:
raise ValueError('Default group not found') raise ValueError('Default group not found')
@ -404,36 +409,30 @@ class CKANHarvester(HarvesterBase):
# check if remote groups exist locally, otherwise remove # check if remote groups exist locally, otherwise remove
validated_groups = [] validated_groups = []
for group_name in package_dict['groups']: for group_ in package_dict['groups']:
try: try:
data_dict = {'id': group_name} data_dict = {'id': group_['id']}
group = get_action('group_show')(context, data_dict) group = get_action('group_show')(context, data_dict)
if self.api_version == 1: validated_groups.append({'id': group['id'], 'name': group['name']})
validated_groups.append(group['name'])
else:
validated_groups.append(group['id'])
except NotFound, e: except NotFound, e:
log.info('Group %s is not available', group_name) log.info('Group %s is not available', group_)
if remote_groups == 'create': if remote_groups == 'create':
try: try:
group = self._get_group(harvest_object.source.url, group_name) group = self._get_group(harvest_object.source.url, group_)
except RemoteResourceError: except RemoteResourceError:
log.error('Could not get remote group %s', group_name) log.error('Could not get remote group %s', group_)
continue continue
for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name']: for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name']:
group.pop(key, None) group.pop(key, None)
get_action('group_create')(context, group) get_action('group_create')(context, group)
log.info('Group %s has been newly created', group_name) log.info('Group %s has been newly created', group_)
if self.api_version == 1: validated_groups.append({'id': group['id'], 'name': group['name']})
validated_groups.append(group['name'])
else:
validated_groups.append(group['id'])
package_dict['groups'] = validated_groups package_dict['groups'] = validated_groups
# Local harvest source organization # Local harvest source organization
source_dataset = get_action('package_show')(context, {'id': harvest_object.source.id}) source_dataset = get_action('package_show')(context, {'id': harvest_object.source.id})
local_org = source_dataset.get('owner_org') local_org = source_dataset.get('owner_org')