From ade5f83e3838f3bd0c3a8868749bbbcae4fa1923 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Mon, 5 Aug 2013 23:07:25 +0200 Subject: [PATCH 1/7] Change key of data_dict from 'type' to 'source_type' --- ckanext/harvest/commands/harvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index df35965..20f47ab 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -191,7 +191,7 @@ class Harvester(CkanCommand): try: data_dict = { 'url':url, - 'type':type, + 'source_type':type, 'config':config, 'frequency':frequency, 'active':active, From 1249564be53b8a6d2987aefd5ad1dc9607e19225 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Mon, 5 Aug 2013 23:46:21 +0200 Subject: [PATCH 2/7] Add additional name argument when creating new harvest source --- ckanext/harvest/commands/harvester.py | 37 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index 20f47ab..a6a1bbb 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -15,7 +15,7 @@ class Harvester(CkanCommand): harvester initdb - Creates the necessary tables in the database - harvester source {url} {type} [{config}] [{active}] [{user-id}] [{publisher-id}] [{frequency}] + harvester source {name} {url} {type} [{config}] [{active}] [{user-id}] [{publisher-id}] [{frequency}] - create new harvest source harvester rmsource {id} @@ -156,34 +156,40 @@ class Harvester(CkanCommand): def create_harvest_source(self): if len(self.args) >= 2: - url = unicode(self.args[1]) + name = unicode(self.args[1] + else: + print 'Please provide a source name' + sys.exit(1) + if len(self.args) >= 3: + url = unicode(self.args[2]) else: print 'Please provide a source URL' sys.exit(1) - if len(self.args) >= 3: - type = unicode(self.args[2]) + if len(self.args) >= 4: + type = unicode(self.args[3]) else: print 'Please provide a source type' sys.exit(1) - if len(self.args) >= 4: - config = unicode(self.args[3]) + + if len(self.args) >= 5: + config = unicode(self.args[4]) else: config = None - if len(self.args) >= 5: - active = not(self.args[4].lower() == 'false' or \ - self.args[4] == '0') + if len(self.args) >= 6: + active = not(self.args[5].lower() == 'false' or \ + self.args[5] == '0') else: active = True - if len(self.args) >= 6: - user_id = unicode(self.args[5]) + if len(self.args) >= 7: + user_id = unicode(self.args[6]) else: user_id = u'' - if len(self.args) >= 7: - publisher_id = unicode(self.args[6]) + if len(self.args) >= 8: + publisher_id = unicode(self.args[7]) else: publisher_id = u'' - if len(self.args) >= 8: - frequency = unicode(self.args[7]) + if len(self.args) >= 9: + frequency = unicode(self.args[8]) if not frequency: frequency = 'MANUAL' else: @@ -192,6 +198,7 @@ class Harvester(CkanCommand): data_dict = { 'url':url, 'source_type':type, + 'name':name, 'config':config, 'frequency':frequency, 'active':active, From 7ae9d6e208e851b15484ec6cbed41092131acbf3 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Mon, 5 Aug 2013 23:50:30 +0200 Subject: [PATCH 3/7] Made print method more robust against KeyErrors This is especially needed if you create a new harvest source which does not have all the optional arguments. Before this lead to a KeyError after the creation of the source. Now this simply output 'None'. --- ckanext/harvest/commands/harvester.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index a6a1bbb..8d71724 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -306,14 +306,14 @@ class Harvester(CkanCommand): self.print_harvest_source(source) def print_harvest_source(self, source): - print 'Source id: %s' % source['id'] - print ' url: %s' % source['url'] - print ' type: %s' % source['type'] - print ' active: %s' % source['active'] - print ' user: %s' % source['user_id'] - print 'publisher: %s' % source['publisher_id'] - print 'frequency: %s' % source['frequency'] - print ' jobs: %s' % source['status']['job_count'] + print 'Source id: %s' % source.get('id') + print ' url: %s' % source.get('url') + print ' type: %s' % source.get('type') + print ' active: %s' % source.get('active') + print ' user: %s' % source.get('user_id') + print 'publisher: %s' % source.get('publisher_id') + print 'frequency: %s' % source.get('frequency') + print ' jobs: %s' % source.get('status').get('job_count') print '' def print_harvest_jobs(self, jobs): @@ -323,9 +323,9 @@ class Harvester(CkanCommand): self.print_harvest_job(job) def print_harvest_job(self, job): - print ' Job id: %s' % job['id'] - print ' status: %s' % job['status'] - print ' source: %s' % job['source_id'] + print ' Job id: %s' % job.get('id') + print ' status: %s' % job.get('status') + print ' source: %s' % job.get('source_id') print ' objects: %s' % len(job.get('objects', [])) print 'gather_errors: %s' % len(job.get('gather_errors', [])) From ad0bbea3a0006fba6989170ff59ac07626cab1b4 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Tue, 6 Aug 2013 00:03:36 +0200 Subject: [PATCH 4/7] Added new 'name' parameter to README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1d4a855..0264d99 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,7 @@ The following operations can be run from the command line using the harvester initdb - Creates the necessary tables in the database - harvester source {url} {type} [{config}] [{active}] [{user-id}] [{publisher-id}] [{frequency}] + harvester source {name} {url} {type} [{config}] [{active}] [{user-id}] [{publisher-id}] [{frequency}] - create new harvest source harvester rmsource {id} From 8e33262026dd7bc5bfe7b8057e327d8f1caf45fd Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 14 Aug 2013 11:31:23 +0100 Subject: [PATCH 5/7] [#56] Fix syntax error and wrong type --- ckanext/harvest/commands/harvester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index 8d71724..6707854 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -156,7 +156,7 @@ class Harvester(CkanCommand): def create_harvest_source(self): if len(self.args) >= 2: - name = unicode(self.args[1] + name = unicode(self.args[1]) else: print 'Please provide a source name' sys.exit(1) @@ -308,7 +308,7 @@ class Harvester(CkanCommand): def print_harvest_source(self, source): print 'Source id: %s' % source.get('id') print ' url: %s' % source.get('url') - print ' type: %s' % source.get('type') + print ' type: %s' % source.get('source_type') print ' active: %s' % source.get('active') print ' user: %s' % source.get('user_id') print 'publisher: %s' % source.get('publisher_id') From 3494727d3f2912bbd857149486eabff0a07aab8e Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 14 Aug 2013 11:43:32 +0100 Subject: [PATCH 6/7] [#56] Increase max params number --- ckanext/harvest/commands/harvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index 6707854..2acf88d 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -70,7 +70,7 @@ class Harvester(CkanCommand): summary = __doc__.split('\n')[0] usage = __doc__ - max_args = 8 + max_args = 9 min_args = 0 def __init__(self,name): From ffea49ca62ce79f5b0d89804cf6314dfb0f3929e Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 14 Aug 2013 11:54:51 +0100 Subject: [PATCH 7/7] [#56] Update parameters on source create command Add missing title and owner_org fields, remove deprecated user_id and publisher_id --- README.rst | 2 +- ckanext/harvest/commands/harvester.py | 42 +++++++++++++-------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 0264d99..555d865 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,7 @@ The following operations can be run from the command line using the harvester initdb - Creates the necessary tables in the database - harvester source {name} {url} {type} [{config}] [{active}] [{user-id}] [{publisher-id}] [{frequency}] + harvester source {name} {url} {type} [{title}] [{active}] [{owner_org}] [{frequency}] [{config}] - create new harvest source harvester rmsource {id} diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py index 2acf88d..ce6473a 100644 --- a/ckanext/harvest/commands/harvester.py +++ b/ckanext/harvest/commands/harvester.py @@ -1,5 +1,4 @@ import sys -import re from pprint import pprint from ckan import model @@ -15,7 +14,7 @@ class Harvester(CkanCommand): harvester initdb - Creates the necessary tables in the database - harvester source {name} {url} {type} [{config}] [{active}] [{user-id}] [{publisher-id}] [{frequency}] + harvester source {name} {url} {type} [{title}] [{active}] [{owner_org}] [{frequency}] [{config}] - create new harvest source harvester rmsource {id} @@ -172,38 +171,40 @@ class Harvester(CkanCommand): sys.exit(1) if len(self.args) >= 5: - config = unicode(self.args[4]) + title = unicode(self.args[4]) else: - config = None + title = None if len(self.args) >= 6: active = not(self.args[5].lower() == 'false' or \ self.args[5] == '0') else: active = True if len(self.args) >= 7: - user_id = unicode(self.args[6]) + owner_org = unicode(self.args[6]) else: - user_id = u'' + owner_org = None if len(self.args) >= 8: - publisher_id = unicode(self.args[7]) - else: - publisher_id = u'' - if len(self.args) >= 9: - frequency = unicode(self.args[8]) + frequency = unicode(self.args[7]) if not frequency: frequency = 'MANUAL' else: frequency = 'MANUAL' + if len(self.args) >= 9: + config = unicode(self.args[8]) + else: + config = None + try: data_dict = { - 'url':url, - 'source_type':type, - 'name':name, - 'config':config, - 'frequency':frequency, + 'name': name, + 'url': url, + 'source_type': type, + 'title': title, 'active':active, - 'user_id':user_id, - 'publisher_id':publisher_id} + 'owner_org': owner_org, + 'frequency': frequency, + 'config': config, + } context = {'model':model, 'session':model.Session, 'user': self.admin_user['name']} source = get_action('harvest_source_create')(context,data_dict) @@ -309,9 +310,8 @@ class Harvester(CkanCommand): print 'Source id: %s' % source.get('id') print ' url: %s' % source.get('url') print ' type: %s' % source.get('source_type') - print ' active: %s' % source.get('active') - print ' user: %s' % source.get('user_id') - print 'publisher: %s' % source.get('publisher_id') + print ' active: %s' % (source.get('active') if 'active' in source else source.get('state') == 'active') + print 'owner org: %s' % source.get('owner_org') print 'frequency: %s' % source.get('frequency') print ' jobs: %s' % source.get('status').get('job_count') print ''