CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'api-version-fix' of git://github.com/fraunhoferfokus/ckanext-harvest into fraunhoferfokus-api-version-fix

This commit is contained in:
amercader 2013-05-31 17:15:43 +01:00
parent 361abcfc07 87cae31c75
commit a6a0196a4e
3 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.rst
View File

@ -151,8 +151,8 @@ The CKAN harvesters support a number of configuration options to control their
behaviour. Those need to be defined as a JSON object in the configuration form behaviour. Those need to be defined as a JSON object in the configuration form
field. The currently supported configuration options are: field. The currently supported configuration options are:
* api_version: You can force the harvester to use either version '1' or '2' of * api_version: You can force the harvester to use either version 1 or 2 of
the CKAN API. Default is '2'. the CKAN API. Default is 2.
* default_tags: A list of tags that will be added to all harvested datasets. * default_tags: A list of tags that will be added to all harvested datasets.
Tags don't need to previously exist. Tags don't need to previously exist.
@ -160,7 +160,7 @@ field. The currently supported configuration options are:
* default_groups: A list of groups to which the harvested datasets will be * default_groups: A list of groups to which the harvested datasets will be
added to. The groups must exist. Note that you must use ids or names to added to. The groups must exist. Note that you must use ids or names to
define the groups according to the API version you defined (names for version define the groups according to the API version you defined (names for version
'1', ids for version '2'). 1, ids for version 2).
* default_extras: A dictionary of key value pairs that will be added to extras * default_extras: A dictionary of key value pairs that will be added to extras
of the harvested datasets. You can use the following replacement strings, of the harvested datasets. You can use the following replacement strings,
@ -204,7 +204,7 @@ Here is an example of a configuration object (the one that must be entered in
the configuration field):: the configuration field)::
{ {
"api_version":"1", "api_version": 1,
"default_tags":["new-tag-1","new-tag-2"], "default_tags":["new-tag-1","new-tag-2"],
"default_groups":["my-own-group"], "default_groups":["my-own-group"],
"default_extras":{"new_extra":"Test","harvest_url":"{harvest_source_url}/dataset/{dataset_id}"}, "default_extras":{"new_extra":"Test","harvest_url":"{harvest_source_url}/dataset/{dataset_id}"},

10
ckanext/harvest/harvesters/base.py
View File

@ -130,11 +130,15 @@ class HarvesterBase(SingletonPlugin):
# Check API version # Check API version
if self.config: if self.config:
api_version = self.config.get('api_version','2') try:
api_version = int(self.config.get('api_version', 2))
except ValueError:
raise ValueError('api_version must be an integer')
#TODO: use site user when available #TODO: use site user when available
user_name = self.config.get('user',u'harvest') user_name = self.config.get('user', u'harvest')
else: else:
api_version = '2' api_version = 2
user_name = u'harvest' user_name = u'harvest'
context = { context = {

15
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -20,13 +20,13 @@ class CKANHarvester(HarvesterBase):
''' '''
config = None config = None
api_version = '2' api_version = 2
def _get_rest_api_offset(self): def _get_rest_api_offset(self):
return '/api/%s/rest' % self.api_version return '/api/%d/rest' % self.api_version
def _get_search_api_offset(self): def _get_search_api_offset(self):
return '/api/%s/search' % self.api_version return '/api/%d/search' % self.api_version
def _get_content(self, url): def _get_content(self, url):
http_request = urllib2.Request( http_request = urllib2.Request(
@ -53,7 +53,10 @@ class CKANHarvester(HarvesterBase):
self.config = json.loads(config_str) self.config = json.loads(config_str)
if 'api_version' in self.config: if 'api_version' in self.config:
self.api_version = self.config['api_version'] try:
self.api_version = int(self.config['api_version'])
except ValueError:
raise ValueError('api_version must be an integer')
log.debug('Using config: %r', self.config) log.debug('Using config: %r', self.config)
else: else:
@ -263,7 +266,7 @@ class CKANHarvester(HarvesterBase):
try: try:
data_dict = {'id': group_name} data_dict = {'id': group_name}
group = get_action('group_show')(context, data_dict) group = get_action('group_show')(context, data_dict)
if self.api_version == '1': if self.api_version == 1:
validated_groups.append(group['name']) validated_groups.append(group['name'])
else: else:
validated_groups.append(group['id']) validated_groups.append(group['id'])
@ -280,7 +283,7 @@ class CKANHarvester(HarvesterBase):
group.pop(key, None) group.pop(key, None)
get_action('group_create')(context, group) get_action('group_create')(context, group)
log.info('Group %s has been newly created' % group_name) log.info('Group %s has been newly created' % group_name)
if self.api_version == '1': if self.api_version == 1:
validated_groups.append(group['name']) validated_groups.append(group['name'])
else: else:
validated_groups.append(group['id']) validated_groups.append(group['id'])