CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

If not revisions then we get a 404, so deal with that better.

This commit is contained in:
David Read 2015-11-23 21:36:45 +00:00
parent 4405066fab
commit 392c13d828
1 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -45,6 +45,9 @@ class CKANHarvester(HarvesterBase):
try: try:
http_response = urllib2.urlopen(http_request) http_response = urllib2.urlopen(http_request)
except urllib2.HTTPError, e: except urllib2.HTTPError, e:
if e.getcode() == 404:
raise ContentNotFoundError('HTTP error: %s' % e.code)
else:
raise ContentFetchError('HTTP error: %s' % e.code) raise ContentFetchError('HTTP error: %s' % e.code)
except urllib2.URLError, e: except urllib2.URLError, e:
raise ContentFetchError('URL error: %s' % e.reason) raise ContentFetchError('URL error: %s' % e.reason)
@ -205,11 +208,15 @@ class CKANHarvester(HarvesterBase):
revision = json.loads(content) revision = json.loads(content)
package_ids = revision['packages'] package_ids = revision['packages']
else: else:
log.info('No packages have been updated on the remote CKAN instance since the last harvest job') log.info('No revisions since last harvest %s',
last_time)
return [] return []
except ContentNotFoundError, e:
log.info('No revisions since last harvest %s', last_time)
return []
except ContentFetchError, e: except ContentFetchError, e:
# Any error at all indicates that revision filtering is not # Any other error indicates that revision filtering is not
# working for whatever reason, so fallback to just getting # working for whatever reason, so fallback to just getting
# all the packages, which is expensive but reliable. # all the packages, which is expensive but reliable.
log.info('CKAN instance %s does not suport revision ' log.info('CKAN instance %s does not suport revision '
@ -468,5 +475,8 @@ class CKANHarvester(HarvesterBase):
class ContentFetchError(Exception): class ContentFetchError(Exception):
pass pass
class ContentNotFoundError(ContentFetchError):
pass
class RemoteResourceError(Exception): class RemoteResourceError(Exception):
pass pass