CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve exception handling with custom exception 

1. Try whenever possible to catch specific exceptions
2. Raise custom exception where appropriate
3. Fix the exception handling in _get_group and _get_organization
This commit is contained in:
Stefan Oderbolz 2015-01-15 00:07:26 +01:00
parent 0fd38e0e54
commit ef35c21e2a
1 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -40,8 +40,14 @@ class CKANHarvester(HarvesterBase):
api_key = self.config.get('api_key',None) api_key = self.config.get('api_key',None)
if api_key: if api_key:
http_request.add_header('Authorization',api_key) http_request.add_header('Authorization',api_key)
http_response = urllib2.urlopen(http_request)
try:
http_response = urllib2.urlopen(http_request)
except urllib2.HTTPError, e:
raise ContentFetchError(
'Could not fetch url: %s, HTTP error code: %s' %
(url, e.code)
)
return http_response.read() return http_response.read()
def _get_group(self, base_url, group_name): def _get_group(self, base_url, group_name):
@ -49,8 +55,9 @@ class CKANHarvester(HarvesterBase):
try: try:
content = self._get_content(url) content = self._get_content(url)
return json.loads(content) return json.loads(content)
except Exception, e: except (ContentFetchError, ValueError):
raise e log.debug('Could not fetch/decode remote group');
raise RemoteResourceError('Could not fetch/decode remote group')
def _get_organization(self, base_url, org_name): def _get_organization(self, base_url, org_name):
url = base_url + self._get_action_api_offset() + '/organization_show?id=' + org_name url = base_url + self._get_action_api_offset() + '/organization_show?id=' + org_name
@ -58,8 +65,9 @@ class CKANHarvester(HarvesterBase):
content = self._get_content(url) content = self._get_content(url)
content_dict = json.loads(content) content_dict = json.loads(content)
return content_dict['result'] return content_dict['result']
except Exception, e: except (ContentFetchError, ValueError, KeyError):
raise e log.debug('Could not fetch/decode remote group');
raise RemoteResourceError('Could not fetch/decode remote organization')
def _set_config(self,config_str): def _set_config(self,config_str):
if config_str: if config_str:
@ -294,7 +302,7 @@ class CKANHarvester(HarvesterBase):
if remote_groups == 'create': if remote_groups == 'create':
try: try:
group = self._get_group(harvest_object.source.url, group_name) group = self._get_group(harvest_object.source.url, group_name)
except: except RemoteResourceError:
log.error('Could not get remote group %s' % group_name) log.error('Could not get remote group %s' % group_name)
continue continue
@ -339,10 +347,9 @@ class CKANHarvester(HarvesterBase):
try: try:
try: try:
org = self._get_organization(harvest_object.source.url, remote_org) org = self._get_organization(harvest_object.source.url, remote_org)
except: except RemoteResourceError:
# fallback if remote CKAN exposes organizations as groups # fallback if remote CKAN exposes organizations as groups
# this especially targets older versions of CKAN # this especially targets older versions of CKAN
log.debug('_get_organization() failed, now trying _get_group()')
org = self._get_group(harvest_object.source.url, remote_org) org = self._get_group(harvest_object.source.url, remote_org)
for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name', 'type']: for key in ['packages', 'created', 'users', 'groups', 'tags', 'extras', 'display_name', 'type']:
@ -350,7 +357,7 @@ class CKANHarvester(HarvesterBase):
get_action('organization_create')(context, org) get_action('organization_create')(context, org)
log.info('Organization %s has been newly created' % remote_org) log.info('Organization %s has been newly created' % remote_org)
validated_org = org['id'] validated_org = org['id']
except: except (RemoteResourceError, ValidationError):
log.error('Could not get remote org %s' % remote_org) log.error('Could not get remote org %s' % remote_org)
package_dict['owner_org'] = validated_org or local_org package_dict['owner_org'] = validated_org or local_org
@ -425,3 +432,8 @@ class CKANHarvester(HarvesterBase):
except Exception, e: except Exception, e:
self._save_object_error('%r'%e,harvest_object,'Import') self._save_object_error('%r'%e,harvest_object,'Import')
class ContentFetchError(Exception):
pass
class RemoteResourceError(Exception):
pass