Merge branch 'metaodi-add-harvesting-of-organizations'

README.rst

@@ -200,6 +200,12 @@ field. The currently supported configuration options are:
     present in the local CKAN. Setting it to 'create' will make an attempt to
     create the groups by copying the details from the remote CKAN.
 
+*   remote_orgs: By default, remote organizations are ignored. Setting this property
+    enables the harvester to import remote organizations. There are two alternatives.
+    Setting it to 'only_local' will just import organizations which id is already
+    present in the local CKAN. Setting it to 'create' will make an attempt to
+    create the organizations by copying the details from the remote CKAN.
+
 Here is an example of a configuration object (the one that must be entered in
 the configuration field)::
 
@@ -212,7 +218,8 @@ the configuration field)::
      "user":"harverster-user",
      "api_key":"<REMOTE_API_KEY>",
      "read_only": true,
-     "remote_groups": "only_local"
+     "remote_groups": "only_local",
+     "remote_orgs": "create"
     }
 
 

ckanext/harvest/harvesters/ckanharvester.py

@@ -296,8 +296,38 @@ class CKANHarvester(HarvesterBase):
 
                 package_dict['groups'] = validated_groups
 
-            # Ignore remote orgs for the time being
+            remote_orgs = self.config.get('remote_orgs', None)
+            if not remote_orgs in ('only_local', 'create'):
+                # Ignore remote groups
                 package_dict.pop('owner_org', None)
+            else:
+                if not 'owner_org' in package_dict:
+                    package_dict['owner_org'] = None
+
+                # check if remote org exist locally, otherwise remove
+                validated_org = None
+                remote_org = package_dict['owner_org']
+                context = {'model': model, 'session': Session, 'user': 'harvest'}
+                
+                if remote_org:
+                    try:
+                        data_dict = {'id': remote_org}
+                        org = get_action('organization_show')(context, data_dict)
+                        validated_org = org['id']
+                    except NotFound, e:
+                        log.info('Organization %s is not available' % remote_org)
+                        if remote_orgs == 'create':
+                            try:
+                                org = self._get_group(harvest_object.source.url, remote_org)
+                                for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name', 'type']:
+                                    org.pop(key, None)
+                                get_action('organization_create')(context, org)
+                                log.info('Organization %s has been newly created' % remote_org)
+                                validated_org = org['id']
+                            except:
+                                log.error('Could not get remote org %s' % remote_org)
+
+                package_dict['owner_org'] = validated_org
 
             # Set default groups if needed
             default_groups = self.config.get('default_groups', [])