If not revisions then we get a 404, so deal with that better.

This commit is contained in:
David Read 2015-11-23 21:36:45 +00:00
parent 4405066fab
commit 392c13d828
1 changed files with 13 additions and 3 deletions

View File

@ -45,6 +45,9 @@ class CKANHarvester(HarvesterBase):
try:
http_response = urllib2.urlopen(http_request)
except urllib2.HTTPError, e:
if e.getcode() == 404:
raise ContentNotFoundError('HTTP error: %s' % e.code)
else:
raise ContentFetchError('HTTP error: %s' % e.code)
except urllib2.URLError, e:
raise ContentFetchError('URL error: %s' % e.reason)
@ -205,11 +208,15 @@ class CKANHarvester(HarvesterBase):
revision = json.loads(content)
package_ids = revision['packages']
else:
log.info('No packages have been updated on the remote CKAN instance since the last harvest job')
log.info('No revisions since last harvest %s',
last_time)
return []
except ContentNotFoundError, e:
log.info('No revisions since last harvest %s', last_time)
return []
except ContentFetchError, e:
# Any error at all indicates that revision filtering is not
# Any other error indicates that revision filtering is not
# working for whatever reason, so fallback to just getting
# all the packages, which is expensive but reliable.
log.info('CKAN instance %s does not suport revision '
@ -468,5 +475,8 @@ class CKANHarvester(HarvesterBase):
class ContentFetchError(Exception):
pass
class ContentNotFoundError(ContentFetchError):
pass
class RemoteResourceError(Exception):
pass