Merge branch '7-harvest-source-templates' into source_extra_config_validation

ckanext/harvest/fanstatic_library/styles/harvest.css

@@ -5,85 +5,19 @@ header.with-filter {
 header.with-filter h1 {
   margin-top: 0;
 }
-.harvest-jobs {
+[data-diff] {
-  border: 1px solid #dddddd;
-  -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
-}
-.harvest-jobs a {
-  display: block;
-  padding: 8px 15px;
-  color: #666;
-  border-bottom: 1px dotted #dddddd;
-}
-.harvest-jobs a h4 {
-  font-size: 16px;
-  line-height: 1.3;
   color: #000;
-}
+  background-color: #DDD;
-.harvest-jobs a p {
+  text-shadow: none;
-  clear: both;
-  overflow: hidden;
-}
-.harvest-jobs a:hover {
-  background-color: #f6f6f6;
-  text-decoration: none;
-}
-.harvest-jobs a:hover h4 {
-  text-decoration: underline;
-}
-.harvest-jobs a:hover .harvest-diff .diff-added {
-  background-color: #79db69;
-}
-.harvest-jobs a:hover .harvest-diff .diff-updated {
-  background-color: #a277ff;
-}
-.harvest-jobs a:hover .harvest-diff .diff-deleted {
-  background-color: #dc7c7f;
-}
-.harvest-jobs a:last-child {
-  border-bottom-width: 0;
-}
-.harvest-errors {
-  float: right;
-}
-.harvest-errors .badge {
-  float: left;
-  font-size: 16px;
-  line-height: 1.3;
-  margin-top: 5px;
-}
-.harvest-errors .badge:first-child {
-  padding-right: 8px;
-  -webkit-border-radius: 100px 0 0 100px;
-  -moz-border-radius: 100px 0 0 100px;
-  border-radius: 100px 0 0 100px;
-}
-.harvest-errors .badge:last-child {
   font-weight: normal;
-  padding-left: 8px;
-  -webkit-border-radius: 0 100px 100px 0;
-  -moz-border-radius: 0 100px 100px 0;
-  border-radius: 0 100px 100px 0;
 }
-.harvest-diff span {
+[data-diff="added"] {
-  display: inline-block;
-  color: #000;
-  padding: 2px 10px;
-  margin-right: 5px;
-  font-size: 12px;
-  -webkit-border-radius: 5px;
-  -moz-border-radius: 5px;
-  border-radius: 5px;
-}
-.harvest-diff .diff-added {
   background-color: #9ee592;
 }
-.harvest-diff .diff-updated {
+[data-diff="updated"] {
   background-color: #c5aaff;
 }
-.harvest-diff .diff-deleted {
+[data-diff="deleted"] {
   background-color: #e7a4a6;
 }
 .harvest-error-summary .count {

ckanext/harvest/fanstatic_library/styles/harvest.less

@@ -9,86 +9,21 @@ header.with-filter {
   }
 }
 
-.harvest-jobs {
+[data-diff] {
-  border: 1px solid @borderColor;
-  .border-radius(3px);
-  a {
-    display: block;
-    padding: 8px 15px;
-    color: #666;
-    border-bottom: 1px dotted @borderColor;
-    h4 {
-      font-size: 16px;
-      line-height: 1.3;
   color: #000;
-    }
+  background-color: #DDD;
-    p {
+  text-shadow: none;
-      clear: both;
-      overflow: hidden;
-    }
-    &:hover {
-      background-color: @hoverColor;
-      text-decoration: none;
-      h4 {
-        text-decoration: underline;
-      }
-      .harvest-diff {
-        .diff-added {
-          background-color: darken(@diffAdded, 10%);
-        }
-        .diff-updated {
-          background-color: darken(@diffUpdated, 10%);
-        }
-        .diff-deleted {
-          background-color: darken(@diffDeleted, 10%);
-        }
-      }
-    }
-    &:last-child {
-      border-bottom-width: 0;
-    }
-  }
-}
-
-.harvest-errors {
-  .clearfix();
-  float: right;
-  .badge {
-    float: left;
-    font-size: 16px;
-    line-height: 1.3;
-    margin-top: 5px;
-    &:first-child {
-      padding-right: 8px;
-      .border-radius(100px 0 0 100px);
-    }
-    &:last-child {
   font-weight: normal;
-      padding-left: 8px;
-      .border-radius(0 100px 100px 0);
 }
-  }
+[data-diff="added"] {
-}
-
-.harvest-diff {
-  span {
-    display: inline-block;
-    color: #000;
-    padding: 2px 10px;
-    margin-right: 5px;
-    font-size: 12px;
-    .border-radius(5px);
-  }
-  .diff-added {
   background-color: @diffAdded;
 }
-  .diff-updated {
+[data-diff="updated"] {
   background-color: @diffUpdated;
 }
-  .diff-deleted {
+[data-diff="deleted"] {
   background-color: @diffDeleted;
 }
-}
 
 .harvest-error-summary {
   .count {

ckanext/harvest/logic/action/get.py

@@ -8,7 +8,7 @@ from ckan.plugins import PluginImplementations
 from ckanext.harvest.interfaces import IHarvester
 
 import ckan.plugins as p
-from ckan.logic import NotFound, check_access
+from ckan.logic import NotFound, check_access, side_effect_free
 
 from ckanext.harvest import model as harvest_model
 
@@ -19,7 +19,7 @@ from ckanext.harvest.logic.dictization import (harvest_source_dictize,
 from ckanext.harvest.logic.schema import harvest_source_db_to_form_schema
 log = logging.getLogger(__name__)
 
-
+@side_effect_free
 def harvest_source_show(context,data_dict):
     '''
     Returns the metadata of a harvest source
@@ -42,6 +42,7 @@ def harvest_source_show(context,data_dict):
 
     return source_dict
 
+@side_effect_free
 def harvest_source_show_status(context, data_dict):
     '''
     Returns a status report for a harvest source
@@ -60,7 +61,7 @@ def harvest_source_show_status(context, data_dict):
 
     source = harvest_model.HarvestSource.get(data_dict['id'])
     if not source:
-        raise p.toolkit.NotFound('Harvest source {0} does not exist'.format(data_dict['id']))
+        raise p.toolkit.ObjectNotFound('Harvest source {0} does not exist'.format(data_dict['id']))
 
     out = {
            'job_count': 0,
@@ -96,7 +97,7 @@ def harvest_source_show_status(context, data_dict):
 
     return out
 
-
+@side_effect_free
 def harvest_source_list(context, data_dict):
 
     check_access('harvest_source_list',context,data_dict)
@@ -110,6 +111,7 @@ def harvest_source_list(context, data_dict):
     context.update({'detailed':False})
     return [harvest_source_dictize(source, context) for source in sources]
 
+@side_effect_free
 def harvest_source_for_a_dataset(context, data_dict):
     '''For a given dataset, return the harvest source that
     created or last updated it, otherwise NotFound.'''
@@ -130,6 +132,7 @@ def harvest_source_for_a_dataset(context, data_dict):
 
     return harvest_source_dictize(source,context)
 
+@side_effect_free
 def harvest_job_show(context,data_dict):
 
     check_access('harvest_job_show',context,data_dict)
@@ -143,6 +146,7 @@ def harvest_job_show(context,data_dict):
 
     return harvest_job_dictize(job,context)
 
+@side_effect_free
 def harvest_job_report(context, data_dict):
 
     check_access('harvest_job_show', context, data_dict)
@@ -154,6 +158,24 @@ def harvest_job_report(context, data_dict):
     if not job:
         raise NotFound
 
+    report = {
+        'gather_errors': [],
+        'object_errors': []
+    }
+
+    # Gather errors
+    q = model.Session.query(harvest_model.HarvestGatherError) \
+                      .join(harvest_model.HarvestJob) \
+                      .filter(harvest_model.HarvestGatherError.harvest_job_id==job.id) \
+                      .order_by(harvest_model.HarvestGatherError.created.desc())
+
+    for error in q.all():
+        report['gather_errors'].append({
+            'message': error.message
+        })
+
+    # Object errors
+
     # Check if the harvester for this job's source has a method for returning
     # the URL to the original document
     original_url_builder = None
@@ -167,19 +189,18 @@ def harvest_job_report(context, data_dict):
                       .filter(harvest_model.HarvestObject.harvest_job_id==job.id) \
                       .order_by(harvest_model.HarvestObjectError.harvest_object_id)
 
-    report = {}
     for error, guid in q.all():
-        if not error.harvest_object_id in report:
+        if not error.harvest_object_id in report['object_errors']:
-            report[error.harvest_object_id] = {
+            report['object_errors'][error.harvest_object_id] = {
                 'guid': guid,
                 'errors': []
             }
             if original_url_builder:
                 url = original_url_builder(error.harvest_object_id)
                 if url:
-                    report[error.harvest_object_id]['original_url'] = url
+                    report['object_errors'][error.harvest_object_id]['original_url'] = url
 
-        report[error.harvest_object_id]['errors'].append({
+        report['object_errors'][error.harvest_object_id]['errors'].append({
             'message': error.message,
             'line': error.line,
             'type': error.stage
@@ -187,6 +208,7 @@ def harvest_job_report(context, data_dict):
 
     return report
 
+@side_effect_free
 def harvest_job_list(context,data_dict):
 
     check_access('harvest_job_list',context,data_dict)
@@ -212,6 +234,7 @@ def harvest_job_list(context,data_dict):
     context['return_error_summary'] = False
     return [harvest_job_dictize(job, context) for job in jobs]
 
+@side_effect_free
 def harvest_object_show(context,data_dict):
 
     check_access('harvest_object_show',context,data_dict)
@@ -224,6 +247,7 @@ def harvest_object_show(context,data_dict):
 
     return harvest_object_dictize(obj,context)
 
+@side_effect_free
 def harvest_object_list(context,data_dict):
 
     check_access('harvest_object_list',context,data_dict)
@@ -246,6 +270,7 @@ def harvest_object_list(context,data_dict):
 
     return [getattr(obj,'id') for obj in objects]
 
+@side_effect_free
 def harvesters_info_show(context,data_dict):
 
     check_access('harvesters_info_show',context,data_dict)

ckanext/harvest/logic/dictization.py

@@ -37,10 +37,6 @@ def harvest_job_dictize(job, context):
         for status, count in stats:
             out['stats'][status] = count
 
-    out['gather_errors'] = []
-    for error in job.gather_errors:
-        out['gather_errors'].append(error.as_dict())
-
     if context.get('return_error_summary', True):
         q = model.Session.query(HarvestObjectError.message, \
                                 func.count(HarvestObjectError.message).label('error_count')) \
@@ -49,9 +45,14 @@ def harvest_job_dictize(job, context):
                           .group_by(HarvestObjectError.message) \
                           .order_by('error_count desc') \
                           .limit(context.get('error_summmary_limit', 20))
-
+        out['object_error_summary'] = q.all()
-        out['error_summary'] = q.all()
+        q = model.Session.query(HarvestGatherError.message, \
-
+                                func.count(HarvestGatherError.message).label('error_count')) \
+                          .filter(HarvestGatherError.harvest_job_id==job.id) \
+                          .group_by(HarvestGatherError.message) \
+                          .order_by('error_count desc') \
+                          .limit(context.get('error_summmary_limit', 20))
+        out['gather_error_summary'] = q.all()
     return out
 
 def harvest_object_dictize(obj, context):

ckanext/harvest/templates_new/snippets/job_details.html

@@ -15,18 +15,18 @@ Example:
 
 {% set stats = job.stats %}
 
-<p class="harvest-errors">
+{% if job.status == 'Finished' %}
+  <p>
+    <span class="label label-important">
       {% if 'errored' in stats and stats['errored'] > 0 %}
-    <span class="badge badge-important">{{ stats['errored'] }}</span>
+        {{ stats['errored'] }}
       {% else %}
-    <span class="badge">0</span>
+        0
       {% endif %}
-  <span class="badge">{{ _('errors') }}</span>
+      {{ _('errors') }}
-</p>
+    </span>
-
-<p class="harvest-diff">
     {% for action in ['added', 'updated', 'deleted'] %}
-    <span class="diff-{{ action }}">
+      <span class="label" data-diff="{{ action }}">
         {% if action in stats and stats[action] > 0 %}
           {{ stats[action] }}
         {% else %}
@@ -36,8 +36,9 @@ Example:
       </span>
     {% endfor %}
   </p>
+{% endif %}
 
-<h3>{{ _('Details') }}</h3>
+<h3 class="hide-heading">{{ _('Details') }}</h3>
 <table class="table table-striped table-bordered table-condensed">
   <colgroup>
     <col width="15">

ckanext/harvest/templates_new/snippets/source_item.html

@@ -29,13 +29,18 @@ Example:
       {% endif %}
     </h3>
 
-    <!-- TODO: nicer -->
+    {% if source.notes %}
-    <div>{{ source.url }}</div>
+      <p>{{ source.notes }}</p>
+    {% else %}
+      <p class="empty">{{ _('There is no description for this harvest source') }}</p>
+    {% endif %}
 
-    <div><span>Type: {{ source_type }}</span><span>Total datasets: {{ source.status.total_datasets }}</span></div>
+    <p class="muted">
-    <div style="color:#EB7C91"><span>Last harvest: {{ source.status.last_job.gather_finished }}</span></div>
+      {{ _('Datasets') }}: {{ source.status.total_datasets }}
+      {% if source.organization %}
+      &mdash; {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization_read', id=source.organization.name)) }}</a>
+      {% endif %}
+    </p>
 
-    <a style="color:#EB7C91" href="{{ h.url_for('harvesting_job_create', id=source.id) }}">Refresh</a>
-    <a style="color:#EB7C91" href="{{ h.url_for('{0}_edit'.format(source.type), id=source.name) }}">Edit</a>
   </div>
 </li>

ckanext/harvest/templates_new/source/job/list.html

@@ -10,9 +10,9 @@
   <header class="with-filter">
     <ul class="nav nav-pills pull-right">
       <li class="disabled"><a>{{ _('Show me:') }}</a></li>
-      <li{% if c.filter_nav == 'Finished' %} class="active"{% endif %}>{{ h.nav_named_link(_('Finished jobs'), 'harvest_job_list', source=source.name) }}</li>
+      <li{% if c.filter_nav == 'Finished' %} class="active"{% endif %}>{{ h.nav_named_link(_('Finished'), 'harvest_job_list', source=source.name) }}</li>
-      <li{% if c.filter_nav == 'New' %} class="active"{% endif %}>{{ h.nav_named_link(_('New jobs'), 'harvest_job_list', source=source.name, status='New') }}</li>
+      <li{% if c.filter_nav == 'New' %} class="active"{% endif %}>{{ h.nav_named_link(_('New'), 'harvest_job_list', source=source.name, status='New') }}</li>
-      <li{% if c.filter_nav == 'Running' %} class="active"{% endif %}>{{ h.nav_named_link(_('Running jobs'), 'harvest_job_list', source=source.name, status='Running') }}</li>
+      <li{% if c.filter_nav == 'Running' %} class="active"{% endif %}>{{ h.nav_named_link(_('Running'), 'harvest_job_list', source=source.name, status='Running') }}</li>
     </ul>
     <h1>{{ _('Harvest Jobs') }}</h1>
   </header>
@@ -20,25 +20,33 @@
   {% if c.jobs|length == 0 %}
     <p class="empty">{{ _('No jobs yet for this source') }}</p>
   {% else %}
-    <ul class="harvest-jobs unstyled">
+    <ul class="dataset-list unstyled">
       {% for job in c.jobs %}
-        <li>
+        <li class="dataset-item">
+          <div class="dataset-content">
+            <h3 class="dataset-heading">
               <a href="{{ h.url_for(controller=controller, action='show_job', source=source.name, id=job.id) }}">
-            {% if 'errored' in job.stats and job.stats['errored'] > 0 %}
+                {{ _('Job: ') }} {{ job.id }}
-              <span class="harvest-errors">
+              </a>
-                <span class="badge badge-important">{{ job.stats['errored'] }}</span>
+            </h3>
-                <span class="badge">{{ _('errors') }}</span>
-              </span>
-            {% endif %}
-            <h4>{{ _('Job: ') }} {{ job.id }}</h4>
               <p>
               {{ _('Started:') }} {{ h.render_datetime(job.gather_started, with_hours=True) or _('Not yet') }}
               &mdash;
               {{ _('Finished:') }} {{ h.render_datetime(job.gather_finished, with_hours=True) or _('Not yet') }}
             </p>
-            <p class="harvest-diff">
+          </div>
+          {% if job.status == 'Finished' %}
+            <ul class="dataset-resources unstyled">
+              {% if 'errored' in job.stats and job.stats['errored'] > 0 %}
+                <li>
+                  <span class="label label-important">
+                    {{ job.stats['errored'] }} {{ _('errors') }}
+                  </span>
+                </li>
+              {% endif %}
               {% for action in ['added', 'updated', 'deleted'] %}
-                <span class="diff-{{ action }}" title="{{ _(action) }}">
+                <li>
+                  <span class="label" data-diff="{{ action }}" title="{{ _(action) }}">
                     {% if action in job.stats and job.stats[action] > 0 %}
                       {{ job.stats[action] }}
                     {% else %}
@@ -46,9 +54,10 @@
                     {% endif %}
                     {{ _(action) }}
                   </span>
+                </li>
               {% endfor %}
-            </p>
+            </ul>
-          </a>
+          {% endif %}
         </li>
        {% endfor %}
     </ul>

ckanext/harvest/templates_new/source/job/read.html

@@ -12,11 +12,16 @@
   <h1>{{ _('Job Report') }}</h1>
   {% snippet 'snippets/job_details.html', job=c.job %}
 
-  <h2>{{ _('Error Summary') }}</h2>
+  {% if c.job.status == 'Finished' %}
+
     {% if c.job.error_summary|length == 0 %}
+      <h2>{{ _('Error Summary') }}</h2>
       <p class="empty">{{ _('No errors for this job') }}</p>
     {% else %}
-    <p class="empty">{{ _('Only the 20 most frequent errors are shown') }}</p>
+      <h2>
+        {{ _('Error Summary') }}
+        <small>{{ _('Only the 20 most frequent errors are shown') }}</small>
+      </h2>
       <table class="table table-striped table-bordered table-condensed harvest-error-summary">
         <colgroup>
           <col width="8">
@@ -39,8 +44,11 @@
       </table>
     {% endif %}
 
-  <h2>{{ _('Error Report') }}</h2>
+    {% if c.job.error_summary|length %}
-  <p class="lead">{{ c.job_report.keys()|length}} documents with errors</p>
+      <h2>
+        {{ _('Error Report') }}
+        <small>{{ c.job_report.keys()|length}} {{ _('documents with errors') }}</small>
+      </h2>
 
       <table class="table table-bordered table-hover harvest-error-list">
         <tbody>
@@ -72,4 +80,8 @@
           {% endfor %}
         </tbody>
       </table>
+    {% endif %}
+
+  {% endif %}
+
 {% endblock %}