From 4dfd091aecc27932c9c9fdb683ecba7401b11a60 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Wed, 14 Aug 2013 11:52:11 +0200 Subject: [PATCH 1/5] Make the /harvest page more robust if source.status is not set This prevents exceptions from appearing in the log from Jinja: [error] [client 1.2.3.4] Error - : 'dict object' has no attribute 'status' --- .../harvest/templates_new/snippets/source_item.html | 4 +++- ckanext/harvest/templates_new/source/admin.html | 2 +- ckanext/harvest/templates_new/source/admin_base.html | 2 +- ckanext/harvest/templates_new/source/read_base.html | 11 +++++++++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ckanext/harvest/templates_new/snippets/source_item.html b/ckanext/harvest/templates_new/snippets/source_item.html index 200d90f..18f7b6e 100644 --- a/ckanext/harvest/templates_new/snippets/source_item.html +++ b/ckanext/harvest/templates_new/snippets/source_item.html @@ -38,7 +38,9 @@ Example: {% endif %}

- {{ _('Datasets') }}: {{ source.status.total_datasets }} + {% if source.status %} + {{ _('Datasets') }}: {{ source.status.total_datasets }} + {% endif %} {% if not within_organization and source.organization %} — {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization_read', id=source.organization.name)) }} {% endif %} diff --git a/ckanext/harvest/templates_new/source/admin.html b/ckanext/harvest/templates_new/source/admin.html index 95a6cbc..a2ee908 100644 --- a/ckanext/harvest/templates_new/source/admin.html +++ b/ckanext/harvest/templates_new/source/admin.html @@ -3,7 +3,7 @@ {% block primary_content_inner %}

Last Harvest Job

- {% if source.status.last_job %} + {% if source.status and source.status.last_job %} {% snippet "snippets/job_details.html", job=source.status.last_job %}
diff --git a/ckanext/harvest/templates_new/source/admin_base.html b/ckanext/harvest/templates_new/source/admin_base.html index 94fdf69..6878700 100644 --- a/ckanext/harvest/templates_new/source/admin_base.html +++ b/ckanext/harvest/templates_new/source/admin_base.html @@ -8,7 +8,7 @@ {% endblock %} {% block actions_content %} - {% if source.status.last_job and (source.status.last_job.status == 'New' or source.status.last_job.status == 'Running') %} + {% if source.status and source.status.last_job and (source.status.last_job.status == 'New' or source.status.last_job.status == 'Running') %}
  • Reharvest
    • {% else %} {% set locale = h.dump_json({'content': _('This will re-run the harvesting for this source. Any updates at the source will overwrite the local datasets. Sources with a large number of datasets may take a significant amount of time to finish harvesting. Please confirm you would like us to start reharvesting.')}) %} diff --git a/ckanext/harvest/templates_new/source/read_base.html b/ckanext/harvest/templates_new/source/read_base.html index 9e48e1d..1abcc19 100644 --- a/ckanext/harvest/templates_new/source/read_base.html +++ b/ckanext/harvest/templates_new/source/read_base.html @@ -21,8 +21,15 @@ {% endif %}
    -
    {{ _('Datasets') }}
    -
    {{ c.harvest_source.status.total_datasets }}
    +
    + {{ _('Datasets') }} +
    + +
    + {% if c.harvest_source.status %} + {{ c.harvest_source.status.total_datasets }} + {% endif %} +
    From f26baf6c09522832f4c7497f07b153c15de2a0ac Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Thu, 15 Aug 2013 13:24:04 +0200 Subject: [PATCH 2/5] Hide both the label and the number of datasets when 'status' is not available --- ckanext/harvest/templates_new/source/read_base.html | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ckanext/harvest/templates_new/source/read_base.html b/ckanext/harvest/templates_new/source/read_base.html index 1abcc19..9965afb 100644 --- a/ckanext/harvest/templates_new/source/read_base.html +++ b/ckanext/harvest/templates_new/source/read_base.html @@ -21,15 +21,10 @@ {% endif %}
    -
    - {{ _('Datasets') }} -
    - -
    - {% if c.harvest_source.status %} - {{ c.harvest_source.status.total_datasets }} - {% endif %} -
    + {% if c.harvest_source.status %} +
    {{ _('Datasets') }}
    +
    {{ c.harvest_source.status.total_datasets }}
    + {% endif %}
    From 39ad78d90aaed57badd35b2386c93966c62079da Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 15 Aug 2013 14:37:12 +0100 Subject: [PATCH 3/5] [#59] Ignore auth in the CKAN harvester --- ckanext/harvest/harvesters/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index 06801c6..c143179 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -147,6 +147,7 @@ class HarvesterBase(SingletonPlugin): 'user': user_name, 'api_version': api_version, 'schema': schema, + 'ignore_auth': True, } tags = package_dict.get('tags', []) From b25fffda932e74e479e8df0eccbda17ac03dc350 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 15 Aug 2013 14:37:55 +0100 Subject: [PATCH 4/5] [#36] Fix bug on API version checking --- ckanext/harvest/harvesters/ckanharvester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index 2548ab6..1b82897 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -51,7 +51,8 @@ class CKANHarvester(HarvesterBase): def _set_config(self,config_str): if config_str: self.config = json.loads(config_str) - self.api_version = int(self.config['api_version']) + if 'api_version' in self.config: + self.api_version = int(self.config['api_version']) log.debug('Using config: %r', self.config) else: From 01ca5c0dfd22e897d4f83289572d4fc3fd9a16bd Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 15 Aug 2013 14:38:33 +0100 Subject: [PATCH 5/5] [#61] Ignore harvest sources on the CKAN harvester --- ckanext/harvest/harvesters/ckanharvester.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index 1b82897..cd582e3 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -245,6 +245,10 @@ class CKANHarvester(HarvesterBase): try: package_dict = json.loads(harvest_object.content) + if package_dict.get('type') == 'harvest': + log.warn('Remote dataset is a harvest source, ignoring...') + return False + # Set default tags if needed default_tags = self.config.get('default_tags',[]) if default_tags: