CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #191 from raphaelstolt/master 

Catch JSONDecodeError when no JSON content
This commit is contained in:
David Read 2015-11-16 11:42:32 +00:00
parent 6415cdd740 084723abb7
commit 5f96346487
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -6,6 +6,7 @@ from ckan.model import Session, Package
from ckan.logic import ValidationError, NotFound, get_action from ckan.logic import ValidationError, NotFound, get_action
from ckan.lib.helpers import json from ckan.lib.helpers import json
from ckan.lib.munge import munge_name from ckan.lib.munge import munge_name
from simplejson.scanner import JSONDecodeError
from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \ from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \
HarvestObjectError HarvestObjectError
@ -221,10 +222,13 @@ class CKANHarvester(HarvesterBase):
try: try:
content = self._get_content(url) content = self._get_content(url)
package_ids = json.loads(content)
except ContentFetchError,e: except ContentFetchError,e:
self._save_gather_error('Unable to get content for URL: %s: %s' % (url, str(e)),harvest_job) self._save_gather_error('Unable to get content for URL: %s: %s' % (url, str(e)),harvest_job)
return None return None
package_ids = json.loads(content) except JSONDecodeError,e:
self._save_gather_error('Unable to decode content for URL: %s: %s' % (url, str(e)),harvest_job)
return None
if org_filter_include: if org_filter_include:
package_ids = set(package_ids) & include_pkg_ids package_ids = set(package_ids) & include_pkg_ids