From d50eb6fca8eb0b0aa4844614a96090644783e30d Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Fri, 4 Oct 2013 15:34:22 +0200 Subject: [PATCH 1/2] Harvesting of remote organisations similar to remote groups --- README.rst | 9 +++++- ckanext/harvest/harvesters/ckanharvester.py | 33 +++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 555d865..773d074 100644 --- a/README.rst +++ b/README.rst @@ -200,6 +200,12 @@ field. The currently supported configuration options are: present in the local CKAN. Setting it to 'create' will make an attempt to create the groups by copying the details from the remote CKAN. +* remote_orgs: By default, remote organizations are ignored. Setting this property + enables the harvester to import remote organizations. There are two alternatives. + Setting it to 'only_local' will just import organizations which id is already + present in the local CKAN. Setting it to 'create' will make an attempt to + create the organizations by copying the details from the remote CKAN. + Here is an example of a configuration object (the one that must be entered in the configuration field):: @@ -212,7 +218,8 @@ the configuration field):: "user":"harverster-user", "api_key":"", "read_only": true, - "remote_groups": "only_local" + "remote_groups": "only_local", + "remote_orgs": "create" } diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index cd582e3..353c61c 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -296,8 +296,37 @@ class CKANHarvester(HarvesterBase): package_dict['groups'] = validated_groups - # Ignore remote orgs for the time being - package_dict.pop('owner_org', None) + remote_orgs = self.config.get('remote_orgs', None) + if not remote_groups in ('only_local', 'create'): + # Ignore remote groups + package_dict.pop('owner_org', None) + else: + if not 'owner_org' in package_dict: + package_dict['owner_org'] = None + + # check if remote org exist locally, otherwise remove + validated_org = None + remote_org = package_dict['owner_org'] + context = {'model': model, 'session': Session, 'user': 'harvest'} + + try: + data_dict = {'id': remote_org} + org = get_action('organization_show')(context, data_dict) + validated_org = org['id'] + except NotFound, e: + log.info('Organization %s is not available' % remote_org) + if remote_groups == 'create': + try: + org = self._get_group(harvest_object.source.url, remote_org) + for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name', 'type']: + org.pop(key, None) + get_action('organization_create')(context, org) + log.info('Organization %s has been newly created' % remote_org) + validated_org = org['id'] + except: + log.error('Could not get remote org %s' % remote_org) + + package_dict['owner_org'] = validated_org # Set default groups if needed default_groups = self.config.get('default_groups', []) From dd1acd0c6ba1d2d16c2fd2c963a625b209a1bc1c Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Mon, 7 Oct 2013 11:22:19 +0200 Subject: [PATCH 2/2] Use remote_orgs for organizations --- ckanext/harvest/harvesters/ckanharvester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index 353c61c..b08fe1d 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -297,7 +297,7 @@ class CKANHarvester(HarvesterBase): package_dict['groups'] = validated_groups remote_orgs = self.config.get('remote_orgs', None) - if not remote_groups in ('only_local', 'create'): + if not remote_orgs in ('only_local', 'create'): # Ignore remote groups package_dict.pop('owner_org', None) else: @@ -315,7 +315,7 @@ class CKANHarvester(HarvesterBase): validated_org = org['id'] except NotFound, e: log.info('Organization %s is not available' % remote_org) - if remote_groups == 'create': + if remote_orgs == 'create': try: org = self._get_group(harvest_object.source.url, remote_org) for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name', 'type']: