From 98bfd50f4774991d35d9edf1bc372eb07de5cc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Mercader?= Date: Tue, 7 Jun 2011 13:35:11 +0100 Subject: [PATCH] Load config in the CKAN harvester --- ckanext/harvest/harvesters/ckanharvester.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index 23ab27f..835d1d9 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -18,6 +18,7 @@ class CKANHarvester(HarvesterBase): ''' A Harvester for CKAN instances ''' + config = None #TODO: check different API versions api_version = '2' @@ -40,6 +41,13 @@ class CKANHarvester(HarvesterBase): except Exception, e: raise e + def _set_config(self,config_str): + if config_str: + self.config = json.loads(config_str) + log.debug('Using config: %r', self.config) + else: + self.config = {} + def info(self): return { 'name': 'ckan', @@ -62,6 +70,9 @@ class CKANHarvester(HarvesterBase): get_all_packages = True package_ids = [] + if not self.config: + self._set_config(harvest_job.source.config) + # Check if this source has been harvested before previous_job = Session.query(HarvestJob) \ .filter(HarvestJob.source==harvest_job.source) \ @@ -145,6 +156,10 @@ class CKANHarvester(HarvesterBase): def fetch_stage(self,harvest_object): log.debug('In CKANHarvester fetch_stage') + + if not self.config: + self._set_config(harvest_object.job.source.config) + # Get source URL url = harvest_object.source.url.rstrip('/') url = url + self._get_rest_api_offset() + '/package/' + harvest_object.guid @@ -173,6 +188,9 @@ class CKANHarvester(HarvesterBase): harvest_object, 'Import') return False + if not self.config: + self._set_config(harvest_object.job.source.config) + try: package_dict = json.loads(harvest_object.content) return self._create_or_update_package(package_dict,harvest_object)