CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/etri-odp/ckanext-harvest into etri-odp-master

This commit is contained in:
amercader 2018-10-25 12:47:26 +02:00
parent 3e5f98e708 0f8d448864
commit 753d29d5ff
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ckanext/harvest/harvesters/ckanharvester.py
View File

@ -154,6 +154,11 @@ class CKANHarvester(HarvesterBase):
raise ValueError('Harvest configuration cannot contain both ' raise ValueError('Harvest configuration cannot contain both '
'organizations_filter_include and organizations_filter_exclude') 'organizations_filter_include and organizations_filter_exclude')
if 'groups_filter_include' in config_obj \
and 'groups_filter_exclude' in config_obj:
raise ValueError('Harvest configuration cannot contain both '
'groups_filter_include and groups_filter_exclude')
if 'user' in config_obj: if 'user' in config_obj:
# Check if user exists # Check if user exists
context = {'model': model, 'user': toolkit.c.user} context = {'model': model, 'user': toolkit.c.user}
@ -195,6 +200,15 @@ class CKANHarvester(HarvesterBase):
fq_terms.extend( fq_terms.extend(
'-organization:%s' % org_name for org_name in org_filter_exclude) '-organization:%s' % org_name for org_name in org_filter_exclude)
groups_filter_include = self.config.get('groups_filter_include', [])
groups_filter_exclude = self.config.get('groups_filter_exclude', [])
if groups_filter_include:
fq_terms.append(' OR '.join(
'groups:%s' % group_name for group_name in groups_filter_include))
elif groups_filter_exclude:
fq_terms.extend(
'-groups:%s' % group_name for group_name in groups_filter_exclude)
# Ideally we can request from the remote CKAN only those datasets # Ideally we can request from the remote CKAN only those datasets
# modified since the last completely successful harvest. # modified since the last completely successful harvest.
last_error_free_job = self.last_error_free_job(harvest_job) last_error_free_job = self.last_error_free_job(harvest_job)