CatalogueRevamping
/
harvester-d4science
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch '7-harvest-source-templates' into 2.0-dataset-sources

This commit is contained in:
amercader 2013-03-06 15:21:46 +00:00
parent 182fbf054a 74633d0803
commit f15f458906
37 changed files with 772 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -10,3 +10,4 @@ dist
development.ini
*.swp
*~
node_modules

27
ckanext/harvest/controllers/view.py
View File

@ -317,7 +317,7 @@ class ViewController(BaseController):
c.harvest_source = source_dict
c.is_last_job = is_last
return render('job/read.html')
return render('source/job/read.html')
except NotFound:
abort(404,_('Harvest job not found'))
@ -327,11 +327,34 @@ class ViewController(BaseController):
msg = 'An error occurred: [%s]' % str(e)
abort(500,msg)
def about(self, id):
try:
context = {'model':model, 'user':c.user}
c.harvest_source = get_action('harvest_source_show')(context, {'id':id})
return render('source/about.html')
except NotFound:
abort(404,_('Harvest source not found'))
except NotAuthorized,e:
abort(401,self.not_auth_message)
def admin(self, id):
try:
context = {'model':model, 'user':c.user}
p.toolkit.check_access('harvest_source_update', context, {'id': id})
c.harvest_source = get_action('harvest_source_show')(context, {'id':id})
return render('source/admin.html')
except NotFound:
abort(404,_('Harvest source not found'))
except NotAuthorized,e:
abort(401,self.not_auth_message)
def show_last_job(self, source):
source_dict = self._get_source_for_job(source)
if not source_dict['status']['last_job']:
abort(404, _('No jobs yet for this source'))
return self.show_job(source_dict['status']['last_job']['id'],
source_dict=source_dict,
is_last=True)
@ -344,7 +367,7 @@ class ViewController(BaseController):
c.harvest_source = get_action('harvest_source_show')(context, {'id': source})
c.jobs = get_action('harvest_job_list')(context, {'source_id': c.harvest_source['id']})
return render('job/list.html')
return render('source/job/list.html')
except NotFound:
abort(404,_('Harvest source not found'))

38
ckanext/harvest/fanstatic_library/styles/harvest.css Normal file
View File

@ -0,0 +1,38 @@
header.with-filter {
clear: both;
overflow: hidden;
}
header.with-filter h1 {
margin-top: 0;
}
[data-diff] {
color: #000;
background-color: #DDD;
text-shadow: none;
font-weight: normal;
}
[data-diff="added"] {
background-color: #9ee592;
}
[data-diff="updated"] {
background-color: #c5aaff;
}
[data-diff="deleted"] {
background-color: #e7a4a6;
}
.harvest-error-summary .count {
text-align: right;
}
.harvest-error-list h5 {
margin-top: 0;
}
.harvest-error-list .error {
padding-left: 20px;
}
.harvest-types label.radio {
font-weight: normal;
margin-bottom: 0;
}
.harvest-types label.radio i {
color: #999;
}

49
ckanext/harvest/fanstatic_library/styles/harvest.less Normal file
View File

@ -0,0 +1,49 @@
@import 'mixins.less';
@import 'variables.less';
header.with-filter {
clear: both;
overflow: hidden;
h1 {
margin-top: 0;
}
}
[data-diff] {
color: #000;
background-color: #DDD;
text-shadow: none;
font-weight: normal;
}
[data-diff="added"] {
background-color: @diffAdded;
}
[data-diff="updated"] {
background-color: @diffUpdated;
}
[data-diff="deleted"] {
background-color: @diffDeleted;
}
.harvest-error-summary {
.count {
text-align: right;
}
}
.harvest-error-list {
h5 {
margin-top: 0;
}
.error {
padding-left: 20px;
}
}
.harvest-types label.radio {
font-weight: normal;
margin-bottom: 0;
i {
color: #999;
}
}

34
ckanext/harvest/fanstatic_library/styles/less Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/env node
// This file is only used to generate the harvest.css
var path = require('path'),
nodeWatch = require('nodewatch'),
exec = require('child_process').exec,
watch = path.join(__dirname),
lastArg = process.argv.slice().pop();
function now() {
return new Date().toISOString().replace('T', ' ').substr(0, 19);
}
function compile(event, filename) {
var start = Date.now();
exec('`npm bin`/lessc ' + __dirname + '/harvest.less > ' + __dirname + '/harvest.css', function (err, stdout, stderr) {
var duration = Date.now() - start;
if (err) {
console.log('An error occurred running the less command:');
console.log(err.message);
}
else if (stderr || stdout) {
console.log(stdout, stderr);
} else {
console.log('[%s] recompiled in %sms', now(), duration);
}
});
}
nodeWatch.add(watch).onChange(compile);
compile();

16
ckanext/harvest/fanstatic_library/styles/mixins.less Normal file
View File

@ -0,0 +1,16 @@
.clearfix() {
}
.border-radius(@radius) {
-webkit-border-radius: @radius;
-moz-border-radius: @radius;
border-radius: @radius;
}
.box-shadow(@shadowA, @shadowB:X, ...){
@props: ~`"@{arguments}".replace(/[\[\]]|\,\sX/g, '')`;
-webkit-box-shadow: @props;
-moz-box-shadow: @props;
box-shadow: @props;
}

6
ckanext/harvest/fanstatic_library/styles/variables.less Normal file
View File

@ -0,0 +1,6 @@
@borderColor: #DDD;
@hoverColor: #F6F6F6;
@diffAdded: #9EE592;
@diffUpdated: #C5AAFF;
@diffDeleted: #E7A4A6;

18
ckanext/harvest/helpers.py
View File

@ -7,7 +7,7 @@ import ckan.plugins as p
from ckanext.harvest.model import UPDATE_FREQUENCIES
from ckanext.harvest.plugin import DATASET_TYPE_NAME
from ckanext.harvest.interfaces import IHarvester
def package_list_for_source(source_id):
'''
@ -45,8 +45,11 @@ def package_list_for_source(source_id):
)
pager.items = query['results']
out = h.snippet('snippets/package_list.html', packages=query['results'])
out += pager.pager()
if query['results']:
out = h.snippet('snippets/package_list.html', packages=query['results'])
out += pager.pager()
else:
out = h.snippet('snippets/package_list_empty.html')
return out
@ -79,3 +82,12 @@ def link_for_harvest_object(id=None, guid=None, text=None):
link = '<a href="{url}">{text}</a>'.format(url=url, text=text)
return p.toolkit.literal(link)
def harvest_source_extra_fields():
fields = {}
for harvester in p.PluginImplementations(IHarvester):
if not hasattr(harvester, 'extra_schema'):
continue
fields[harvester.info()['name']] = harvester.extra_schema().keys()
return fields

12
ckanext/harvest/logic/action/get.py
View File

@ -65,7 +65,6 @@ def harvest_source_show_status(context, data_dict):
out = {
'job_count': 0,
'next_job': p.toolkit._('Not yet scheduled'),
'last_job': None,
'total_datasets': 0,
}
@ -78,13 +77,8 @@ def harvest_source_show_status(context, data_dict):
out['job_count'] = job_count
# Get next scheduled job
next_job = harvest_model.HarvestJob.filter(source=source,status=u'New').first()
if next_job:
out['next_job'] = p.toolkit._('Scheduled')
# Get the last finished job
last_job = harvest_model.HarvestJob.filter(source=source,status=u'Finished') \
# Get the most recent job
last_job = harvest_model.HarvestJob.filter(source=source) \
.order_by(harvest_model.HarvestJob.created.desc()).first()
if not last_job:
@ -223,7 +217,7 @@ def harvest_job_list(context,data_dict):
session = context['session']
source_id = data_dict.get('source_id',False)
status = data_dict.get('status',False)
status = data_dict.get('status', False)
query = session.query(HarvestJob)

1
ckanext/harvest/logic/action/update.py
View File

@ -217,6 +217,7 @@ def harvest_jobs_run(context,data_dict):
last_object = session.query(HarvestObject) \
.filter(HarvestObject.harvest_job_id==job['id']) \
.filter(HarvestObject.import_finished!=None) \
.order_by(HarvestObject.import_finished.desc()) \
.first()
if last_object:

16
ckanext/harvest/logic/dictization.py
View File

@ -37,6 +37,22 @@ def harvest_job_dictize(job, context):
for status, count in stats:
out['stats'][status] = count
# We actually want to check which objects had errors, because they
# could have been added/updated anyway (eg bbox errors)
count = model.Session.query(func.distinct(HarvestObjectError.harvest_object_id)) \
.join(HarvestObject) \
.filter(HarvestObject.harvest_job_id==job.id) \
.count()
if count > 0:
out['stats']['errored'] = count
# Add gather errors to the error count
count = model.Session.query(HarvestGatherError) \
.filter(HarvestGatherError.harvest_job_id==job.id) \
.count()
if count > 0:
out['stats']['errored'] = out['stats'].get('errored', 0) + count
if context.get('return_error_summary', True):
q = model.Session.query(HarvestObjectError.message, \
func.count(HarvestObjectError.message).label('error_count')) \

7
ckanext/harvest/logic/schema.py
View File

@ -16,8 +16,10 @@ from ckan.lib.navl.validators import (ignore_missing,
from ckanext.harvest.logic.validators import (harvest_source_url_validator,
harvest_source_type_exists,
harvest_source_config_validator,
harvest_source_extra_validator,
harvest_source_frequency_exists,
dataset_type_exists,
harvest_source_convert_from_config,
)
def harvest_source_schema():
@ -35,7 +37,6 @@ def harvest_source_schema():
'state': [ignore_missing],
'config': [ignore_missing, harvest_source_config_validator, convert_to_extras],
'extras': default_extras_schema(),
'__extras': [ignore],
}
extras_schema = default_extras_schema()
@ -48,7 +49,7 @@ def harvest_source_schema():
def harvest_source_form_to_db_schema():
schema = harvest_source_schema()
schema['__extras'] = [harvest_source_extra_validator]
schema['save'] = [ignore]
schema.pop("id")
@ -60,7 +61,7 @@ def harvest_source_db_to_form_schema():
schema.update({
'source_type': [convert_from_extras, ignore_missing],
'frequency': [convert_from_extras, ignore_missing],
'config': [convert_from_extras, ignore_missing],
'config': [convert_from_extras, harvest_source_convert_from_config, ignore_missing],
'owner_org': [ignore_missing]
})

72
ckanext/harvest/logic/validators.py
View File

@ -1,6 +1,7 @@
import urlparse
import json
from ckan.lib.navl.dictization_functions import Invalid
from ckan.lib.navl.dictization_functions import Invalid, validate
from ckan import model
from ckan.plugins import PluginImplementations
@ -8,7 +9,7 @@ from ckanext.harvest.plugin import DATASET_TYPE_NAME
from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES
from ckanext.harvest.interfaces import IHarvester
from ckan.lib.navl.validators import keep_extras
def harvest_source_id_exists(value, context):
@ -101,6 +102,73 @@ def harvest_source_config_validator(key,data,errors,context):
else:
return data[key]
def keep_not_empty_extras(key, data, errors, context):
extras = data.pop(key, {})
for extras_key, value in extras.iteritems():
if value:
data[key[:-1] + (extras_key,)] = value
def harvest_source_extra_validator(key,data,errors,context):
harvester_type = data.get(('source_type',),'')
#gather all extra fields to use as whitelist of what
#can be added to top level data_dict
all_extra_fields = set()
for harvester in PluginImplementations(IHarvester):
if not hasattr(harvester, 'extra_schema'):
continue
all_extra_fields.update(harvester.extra_schema().keys())
extra_schema = {'__extras': [keep_not_empty_extras]}
for harvester in PluginImplementations(IHarvester):
if not hasattr(harvester, 'extra_schema'):
continue
info = harvester.info()
if not info['name'] == harvester_type:
continue
extra_schema.update(harvester.extra_schema())
break
extra_data, extra_errors = validate(data.get(key, {}), extra_schema)
for key in extra_data.keys():
#only allow keys that appear in at least one harvester
if key not in all_extra_fields:
extra_data.pop(key)
for key, value in extra_data.iteritems():
data[(key,)] = value
for key, value in extra_errors.iteritems():
errors[(key,)] = value
## need to get config out of extras as __extra runs
## after rest of validation
package_extras = data.get(('extras',), [])
for num, extra in enumerate(list(package_extras)):
if extra['key'] == 'config':
# remove config extra so we can add back cleanly later
package_extras.pop(num)
config_dict = json.loads(extra.get('value') or '{}')
break
else:
config_dict = {}
config_dict.update(extra_data)
if config_dict:
config = json.dumps(config_dict)
package_extras.append(dict(key='config',
value=config))
data[('config',)] = config
if package_extras:
data[('extras',)] = package_extras
def harvest_source_convert_from_config(key,data,errors,context):
config = data[key]
if config:
config_dict = json.loads(config)
for key, value in config_dict.iteritems():
data[(key,)] = value
def harvest_source_active_validator(value,context):
if isinstance(value,basestring):
if value.lower() == 'true':

8
ckanext/harvest/plugin.py
View File

@ -107,7 +107,7 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
def setup_template_variables(self, context, data_dict):
p.toolkit.c.harvest_source = p.toolkit.c.pkg
p.toolkit.c.harvest_source = p.toolkit.c.pkg_dict
p.toolkit.c.dataset_type = DATASET_TYPE_NAME
@ -212,6 +212,9 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
map.connect('harvest_object_show', '/harvest/object/:id', controller=controller, action='show_object')
map.connect('{0}_admin'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/admin/:id', controller=controller, action='admin')
map.connect('{0}_about'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/about/:id', controller=controller, action='about')
map.connect('harvest_job_list', '/' + DATASET_TYPE_NAME + '/{source}/job', controller=controller, action='list_jobs')
map.connect('harvest_job_show_last', '/' + DATASET_TYPE_NAME + '/{source}/job/last', controller=controller, action='show_last_job')
map.connect('harvest_job_show', '/' + DATASET_TYPE_NAME + '/{source}/job/{id}', controller=controller, action='show_job')
@ -227,6 +230,8 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
templates = 'templates_new'
p.toolkit.add_template_directory(config, templates)
p.toolkit.add_public_directory(config, 'public')
p.toolkit.add_resource('fanstatic_library', 'ckanext-harvest')
p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field')
## IActions
@ -256,6 +261,7 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
'harvester_types': harvest_helpers.harvester_types,
'harvest_frequencies': harvest_helpers.harvest_frequencies,
'link_for_harvest_object': harvest_helpers.link_for_harvest_object,
'harvest_source_extra_fields': harvest_helpers.harvest_source_extra_fields,
}

39
ckanext/harvest/public/ckanext/harvest/javascript/extra_fields.js Normal file
View File

@ -0,0 +1,39 @@
ckan.module('harvest-type-change', function (jQuery, _) {
return {
initialize: function () {
var self, harvest_source_type;
self = this;
harvest_source_type = this.el.attr('value');
this.el.change(function(){
self.sandbox.publish('harvest-source-type-select', harvest_source_type);
})
if (this.el.attr("checked") === "checked"){
self.sandbox.publish('harvest-source-type-select', harvest_source_type);
}
},
}
})
ckan.module('harvest-extra-form-change', function (jQuery, _) {
return {
initialize: function () {
var self, item, i, control_groups, control_group, item_name;
self = this;
self.sandbox.subscribe('harvest-source-type-select', function(source_type) {
form_items = self.options.formItems;
items = form_items[source_type] || [];
control_groups = self.el.find('.control-group');
for (i=0;i<control_groups.length;i++){
control_group = $(control_groups[i])
item_name = control_group.find('input').attr('name');
if ($.inArray(item_name, items) === -1){
control_group.hide();
} else{
control_group.show();
}
}
})
},
}
})

10
ckanext/harvest/public/ckanext/harvest/javascript/resource.config Normal file
View File

@ -0,0 +1,10 @@
[depends]
main = base/main
[groups]
main =
extra_fields.js

2
ckanext/harvest/queue.py
View File

@ -213,7 +213,7 @@ def fetch_callback(channel, method, header, body):
.all()) == 2:
obj.report_status = 'updated'
else:
obj.report_status = 'new'
obj.report_status = 'added'
obj.save()
model.Session.remove()
channel.basic_ack(method.delivery_tag)

6
ckanext/harvest/templates_new/base.html Normal file
View File

@ -0,0 +1,6 @@
{% ckan_extends %}
{% block styles %}
{{ super() }}
{% resource 'ckanext-harvest/styles/harvest.css' %}
{% endblock %}

56
ckanext/harvest/templates_new/job/list.html
View File

@ -1,56 +0,0 @@
{% extends "page.html" %}
{% block subtitle %}{{ _('Harvest Jobs')}}{% endblock %}
{% block breadcrumb_content %}
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li>{{ h.nav_named_link(c.harvest_source.title|truncate(30), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
<li class="active">{{ h.nav_link(_('Jobs'), controller='ckanext.harvest.controllers.view:ViewController', action='list_jobs', source=c.harvest_source.name)}}</li>
{% endblock %}
{% block primary %}
<article class="module">
<div class="module-content">
<h1 class="page-heading">{{ _('Harvest Jobs') }}</h1>
{% if c.jobs|length == 0 %}
<p>{{ _('No jobs yet for this source') }}</p>
{% else %}
<table class="table table-striped table-bordered table-condensed error-summary">
<thead>
<tr>
<th>{{ _('Created') }}</th>
<th>{{ _('Finished') }}</th>
<th>{{ _('Status') }}</th>
<th>{{ _('Added') }}</th>
<th>{{ _('Updated') }}</th>
<th>{{ _('Deleted') }}</th>
<th>{{ _('Errors') }}</th>
<th>{{ _('Full report') }}</th>
</tr>
</thead>
<tbody>
{% for job in c.jobs %}
<tr>
<td>{{ h.render_datetime(job.gather_started, with_hours=True) }}</td>
<td>{{ h.render_datetime(job.finished, with_hours=True) }}</td>
<td>{{ _(job.status) }}</td>
{% for action in ['added', 'updated', 'deleted', 'errored'] %}
{% if action in job.stats and job.stats[action] > 0 %}
<td>{{ job.stats[action] }}</td>
{% else %}
<td>0</td>
{% endif %}
{% endfor %}
<td><a href="{{ h.url_for(controller='ckanext.harvest.controllers.view:ViewController', action='show_job', source=c.harvest_source.name, id=job.id) }}">{{ _('view') }}</a></td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
</div>
</article>
{% endblock %}

62
ckanext/harvest/templates_new/job/read.html
View File

@ -1,62 +0,0 @@
{% extends "page.html" %}
{% block subtitle %}{{ _('Harvest Job Report')}}{% endblock %}
{% block breadcrumb_content %}
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li>{{ h.nav_named_link(c.harvest_source.title|truncate(30), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
<li>{{ h.nav_link(_('Jobs'), controller='ckanext.harvest.controllers.view:ViewController', action='list_jobs', source=c.harvest_source.name)}}</li>
{% if c.is_last_job %}
<li class="active">{{ h.nav_link(_('Last'), controller='ckanext.harvest.controllers.view:ViewController', action='show_last_job', source=c.harvest_source.name)}}</li>
{% else %}
<li class="active">{{ h.nav_link(c.job.id|truncate(30), controller='ckanext.harvest.controllers.view:ViewController', action='show_job', id=c.job.id, source=c.harvest_source.name)}}</li>
{% endif %}
{% endblock %}
{% block primary %}
<article class="module">
<div class="module-content">
<h1 class="page-heading">{{ _('Harvest Job Report') }}</h1>
{% snippet 'snippets/job_details.html', job=c.job %}
<h3>{{ _('Error Report') }}</h3>
<div style='font-size: 1.5em; margin: 1em 0;'>
{{ c.job_report.keys()|length}} documents with errors
</div>
{% for harvest_object_id in c.job_report.keys() %}
<div>
<div>
{{ c.job_report[harvest_object_id].guid }}
{% if 'original_url' in c.job_report[harvest_object_id] %}
(<a href="{{ c.job_report[harvest_object_id].original_url }}">{{ _('Remote content') }}</a>)
{% endif %}
({{ h.link_for_harvest_object(harvest_object_id,text=_('Local content')) }})
</div>
{% for error in c.job_report[harvest_object_id].errors %}
<div style="margin-left: 2em">{{ error.message }}
{% if error.line %}
<span>(line {{error.line}})</span>
{% endif %}
</div>
{% endfor %}
</div>
{% endfor %}
</div>
</article>
{% endblock %}
{% block styles %}
{{ super() }}
<!-- TODO: cleanup and resource -->
<link type="text/css" rel="stylesheet" media="all" href="/ckanext/harvest/style.css" />
{% endblock %}

71
ckanext/harvest/templates_new/snippets/job_details.html
View File

@ -13,19 +13,45 @@ Example:
#}
<div style='margin: 1em 0;'>
{% set stats = job.stats %}
{% for action in ['added', 'updated', 'deleted', 'errored'] %}
{% if action in stats and stats[action] > 0 %}
<span>{{ stats[action] }} {{ _(action) }}</span>
{% else %}
<span>0 {{ _(action) }}</span>
{% endif %}
{% endfor %}
</div>
{% set stats = job.stats %}
<h3>{{ _('Details') }}</h3>
{% if job.status == 'Finished' %}
<p>
<span class="label label-important">
{% if 'errored' in stats and stats['errored'] > 0 %}
{{ stats['errored'] }}
{% else %}
0
{% endif %}
{{ _('errors') }}
</span>
{% for action in ['added', 'updated', 'deleted'] %}
<span class="label" data-diff="{{ action }}">
{% if action in stats and stats[action] > 0 %}
{{ stats[action] }}
{% else %}
0
{% endif %}
{{ _(action) }}
</span>
{% endfor %}
</p>
{% endif %}
<h3 class="hide-heading">{{ _('Details') }}</h3>
<table class="table table-striped table-bordered table-condensed">
<colgroup>
<col width="15">
<col width="85">
</colgroup>
<tr>
<th>{{ _('Id') }}</th>
<td>{{ job.id }}</td>
</tr>
<tr>
<th>{{ _('Created') }}</th>
<td>{{ h.render_datetime(job.created, with_hours=True) }}</td>
</tr>
<tr>
<th>{{ _('Started') }}</th>
<td>{{ h.render_datetime(job.gather_started, with_hours=True) }}</td>
@ -39,26 +65,3 @@ Example:
<td>{{ _(job.status) }}</td>
</tr>
</table>
<h3>{{ _('Error Summary') }}</h3>
{% if job.error_summary|length == 0 %}
<p>{{ _('No errors for this job') }}</p>
{% else %}
<p>{{ _('Only the 20 most frequent errors are shown') }}</p>
<table class="table table-striped table-bordered table-condensed error-summary">
<thead>
<tr>
<th>{{ _('Message') }}</th>
<th>{{ _('Count') }}</th>
</tr>
</thead>
<tbody>
{% for error in job.error_summary %}
<tr>
<td>{{ error[0] }}</td>
<td>{{ error[1] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}

32
ckanext/harvest/templates_new/snippets/job_error_summary.html Normal file
View File

@ -0,0 +1,32 @@
{#
Displays a table with a summary of the most common errors for a job
error_summary - List of tuples with (message, count)
Example:
{% snippet 'snippets/job_error_summary.html', summary=c.job.object_error_summary %}
#}
<table class="table table-striped table-bordered table-condensed harvest-error-summary">
<colgroup>
<col width="8">
<col width="92">
</colgroup>
<thead>
<tr>
<th class="count">{{ _('Count') }}</th>
<th>{{ _('Message') }}</th>
</tr>
</thead>
<tbody>
{% for error in summary %}
<tr>
<td class="count">{{ error[1] }}</td>
<td>{{ error[0] }}</td>
</tr>
{% endfor %}
</tbody>
</table>

1
ckanext/harvest/templates_new/snippets/package_list_empty.html Normal file
View File

@ -0,0 +1 @@
<p class="empty">There are no datasets associated to this harvest source.</p>

17
ckanext/harvest/templates_new/snippets/source_item.html
View File

@ -29,13 +29,18 @@ Example:
{% endif %}
</h3>
<!-- TODO: nicer -->
<div>{{ source.url }}</div>
{% if source.notes %}
<p>{{ source.notes }}</p>
{% else %}
<p class="empty">{{ _('There is no description for this harvest source') }}</p>
{% endif %}
<div><span>Type: {{ source_type }}</span><span>Total datasets: {{ source.status.total_datasets }}</span></div>
<div style="color:#EB7C91"><span>Last harvest: {{ source.status.last_job.gather_finished }}</span> <span>Next harvest: {{ source.status.next_job }}</span></div>
<p class="muted">
{{ _('Datasets') }}: {{ source.status.total_datasets }}
{% if source.organization %}
&mdash; {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization_read', id=source.organization.name)) }}</a>
{% endif %}
</p>
<a style="color:#EB7C91" href="{{ h.url_for('harvesting_job_create', id=source.id) }}">Refresh</a>
<a style="color:#EB7C91" href="{{ h.url_for('{0}_edit'.format(source.type), id=source.name) }}">Edit</a>
</div>
</li>

13
ckanext/harvest/templates_new/source/about.html Normal file
View File

@ -0,0 +1,13 @@
{% extends "source/read_base.html" %}
{% block primary_content_inner %}
<section class="module-content">
<h1>{{ source.title or source.name }}</h1>
{% if source.notes %}
<p>{{ h.markdown_extract(source.notes)|urlize }}</p>
{% else %}
<p class="empty">{{ _('There is no description for this harvest source') }}</p>
{% endif %}
</section>
{% snippet "package/snippets/additional_info.html", pkg_dict=source %}
{% endblock %}

18
ckanext/harvest/templates_new/source/admin.html Normal file
View File

@ -0,0 +1,18 @@
{% extends "source/admin_base.html" %}
{% block primary_content_inner %}
<section class="module-content">
<h1>Last Harvest Job</h1>
{% if source.status.last_job %}
{% snippet "snippets/job_details.html", job=source.status.last_job %}
<div class="form-actions">
<a href="{{ h.url_for(controller='ckanext.harvest.controllers.view:ViewController', action='show_last_job', source=source.name) }}" class="btn pull-right">
<i class="icon-briefcase"></i>
{{ _('View full job report') }}
</a>
</div>
{% else %}
<p class="empty">{{ _('No jobs yet for this source') }}</p>
{% endif %}
</section>
{% endblock %}

25
ckanext/harvest/templates_new/source/admin_base.html Normal file
View File

@ -0,0 +1,25 @@
{% extends "source/read_base.html" %}
{% block breadcrumb_content_root_selected %}{% endblock %}
{% block breadcrumb_content %}
{{ super() }}
<li class="active"><a href="">{{ _('Admin') }}</a></li>
{% endblock %}
{% block actions_content %}
{% if source.status.last_job and (source.status.last_job.status == 'New' or source.status.last_job.status == 'Running') %}
<li><a class="btn disabled" rel="tooltip" title="There already is an unrun job for this source"><i class="icon-refresh icon-large"></i> Refresh</a></li>
{% else %}
<li>{{ h.nav_named_link(_('Refresh'), 'harvesting_job_create', id=source.id, class_='btn', icon='refresh')}}</li>
{% endif %}
<li>{{ h.nav_named_link(_('View harvest source'), '{0}_read'.format(c.dataset_type), id=source.name, class_='btn', icon='eye-open')}}</li>
{% endblock %}
{% block page_header %}
{% snippet 'snippets/page_header.html', items=[
h.build_nav_icon('{0}_admin'.format(c.dataset_type), _('Dashboard'), id=source.name, icon='dashboard'),
h.build_nav_icon('harvest_job_list'.format(c.dataset_type), _('Jobs'), source=source.name, icon='reorder'),
h.build_nav_icon('{0}_edit'.format(c.dataset_type), _('Edit'), id=source.name, icon='edit'),
] %}
{% endblock %}

28
ckanext/harvest/templates_new/source/base.html Normal file
View File

@ -0,0 +1,28 @@
{% extends "page.html" %}
{% set source = c.pkg_dict or c.harvest_source %}
{% if source %}
{% set authorized_user = h.check_access('harvest_source_update', {'id':source.id }) %}
{% else %}
{% set authorized_user = h.check_access('harvest_source_create') %}
{% endif %}
{% block subtitle %}{{ source.title or source.name }}{% endblock %}
{% block breadcrumb_content_root_selected %} class="active"{% endblock %}
{#
TODO: once #354 is merged in CKAN core we can re-adjust the truncation
lengths here (as the breadcrumbs are a little different in #354)
#}
{% block breadcrumb_content %}
{% if source.organization %}
<li>{{ h.nav_named_link(_('Organizations'), 'organizations_index') }}</li>
<li>{{ h.nav_named_link(source.organization.title or source.organization.name|truncate(10), 'organization_read', id=source.organization.name) }}</li>
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li{{ self.breadcrumb_content_root_selected() }}>{{ h.nav_named_link(c.harvest_source.title|truncate(10), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
{% else %}
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li{{ self.breadcrumb_content_root_selected() }}>{{ h.nav_named_link(c.harvest_source.title|truncate(30), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
{% endif %}
{% endblock %}

27
ckanext/harvest/templates_new/source/base_form_page.html
View File

@ -1,27 +0,0 @@
{% extends "page.html" %}
{% block primary_content %}
<section class="module">
<div class="module-content">
{% block form %}{{ c.form | safe }}{% endblock %}
</div>
</section>
{% endblock %}
{% block secondary_content %}
{% block info_module %}
<section class="module module-narrow">
<h2 class="module-heading"><i class="icon-large icon-info-sign"></i> {{ _('Harvest sources') }}</h2>
<div class="module-content">
<p>
{% trans %}
Harvest sources allow importing metadata from other catalogues
as CKAN datasets. These can be other CKAN instances or other
protocols and formats. (TODO: Review)
{% endtrans %}
</p>
</div>
</section>
{% endblock %}
{% endblock %}

20
ckanext/harvest/templates_new/source/edit.html
View File

@ -1,15 +1,9 @@
{% extends "source/base_form_page.html" %}
{% block breadcrumb_content %}
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li>{{ h.nav_named_link(c.harvest_source.title|truncate(30), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
<li class="active">{{ h.nav_named_link(_('Edit Harvest Source'), '{0}_edit'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
{% endblock %}
{% block actions_content %}
<li>{{ h.nav_named_link(_('View harvest source'), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name, class_='btn', icon='eye-open')}}</li>
{% endblock %}
<!-- TODO back to source -->
{% extends "source/admin_base.html" %}
{% block subtitle %}{{ _('Edit harvest source') }}{% endblock %}
{% block primary_content_inner %}
<div class="module-content">
{% block form %}{{ c.form | safe }}{% endblock %}
</div>
{% endblock %}

63
ckanext/harvest/templates_new/source/job/list.html Normal file
View File

@ -0,0 +1,63 @@
{% extends "source/admin_base.html" %}
{% set controller = 'ckanext.harvest.controllers.view:ViewController' %}
{% block subtitle %}{{ _('Harvest Jobs')}} - {{ super() }}{% endblock %}
{% block primary_content_inner %}
<div class="module-content">
<h1 class="results">{{ _('Harvest Jobs') }}</h1>
{% if c.jobs|length == 0 %}
<p class="empty">{{ _('No jobs yet for this source') }}</p>
{% else %}
<ul class="dataset-list unstyled">
{% for job in c.jobs %}
<li class="dataset-item">
<div class="dataset-content">
<h3 class="dataset-heading">
<a href="{{ h.url_for(controller=controller, action='show_job', source=source.name, id=job.id) }}">
{{ _('Job: ') }} {{ job.id }}
</a>
{% if job.status != 'Finished' %}
<span class="label">{{ job.status }}</span>
{% endif %}
</h3>
<p>
{{ _('Started:') }} {{ h.render_datetime(job.gather_started, with_hours=True) or _('Not yet') }}
&mdash;
{{ _('Finished:') }} {{ h.render_datetime(job.finished, with_hours=True) or _('Not yet') }}
</p>
</div>
{% if job.status == 'Finished' %}
<ul class="dataset-resources unstyled">
{% if 'errored' in job.stats and job.stats['errored'] > 0 %}
<li>
<span class="label label-important">
{{ job.stats['errored'] }} {{ _('errors') }}
</span>
</li>
{% endif %}
{% for action in ['added', 'updated', 'deleted'] %}
<li>
<span class="label" data-diff="{{ action }}" title="{{ _(action) }}">
{% if action in job.stats and job.stats[action] > 0 %}
{{ job.stats[action] }}
{% else %}
0
{% endif %}
{{ _(action) }}
</span>
</li>
{% endfor %}
</ul>
{% endif %}
</li>
{% endfor %}
</ul>
{% endif %}
</div>
{% endblock %}

95
ckanext/harvest/templates_new/source/job/read.html Normal file
View File

@ -0,0 +1,95 @@
{% extends "source/admin_base.html" %}
{% block subtitle %}{{ _('Job Report') }} - {{ super() }}{% endblock %}
{% block primary_content_inner %}
<div class="module-content">
<p class="pull-right">
{{ h.nav_named_link(_('Back to job list'), 'harvest_job_list', source=source.name, class_='btn', icon='arrow-left')}}
</p>
<h1>{{ _('Job Report') }}</h1>
{% snippet 'snippets/job_details.html', job=c.job %}
{% if c.job.status == 'Finished' %}
{% if c.job.object_error_summary|length == 0 and c.job.gather_error_summary|length == 0 %}
<h2>{{ _('Error Summary') }}</h2>
<p class="empty">{{ _('No errors for this job') }}</p>
{% else %}
<h2>
{{ _('Error Summary') }}
<small>{{ _('Only the 20 most frequent errors are shown') }}</small>
</h2>
{% if c.job.gather_error_summary|length > 0 %}
<h3>{{ _('Job Errors') }}</h3>
{% snippet 'snippets/job_error_summary.html', summary=c.job.gather_error_summary %}
{% endif %}
{% if c.job.object_error_summary|length > 0 %}
<h3>{{ _('Document Errors') }}</h3>
{% snippet 'snippets/job_error_summary.html', summary=c.job.object_error_summary %}
{% endif %}
{% endif %}
{% if c.job_report.gather_errors|length > 0 or c.job_report.object_errors.keys()|length > 0 %}
<h2>
{{ _('Error Report') }}
</h2>
{% if c.job_report.gather_errors|length > 0 %}
<h3>{{ _('Job Errors') }}</h3>
<table class="table table-bordered table-hover harvest-error-list">
<tbody>
{% for error in c.job_report.gather_errors %}
<tr>
<td>
<div class="error">
{{ error.message }}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if c.job_report.object_errors.keys()|length > 0 %}
<h3>{{ _('Document Errors') }}
<small>{{ c.job_report.object_errors.keys()|length}} {{ _('documents with errors') }}</small>
</h3>
<table class="table table-bordered table-hover harvest-error-list">
<tbody>
{% for harvest_object_id in c.job_report.object_errors.keys() %}
{% set object = c.job_report.object_errors[harvest_object_id] %}
<tr>
<td>
<span class="btn-group pull-right">
{% if 'original_url' in object%}
<a href="{{ object.original_url }}" class="btn btn-small">
{{ _('Remote content') }}
</a>
{% endif %}
<a href="{{ h.url_for('harvest_object_show', id=harvest_object_id) }}" class="btn btn-small">
{{ _('Local content') }}
</a>
</span>
<h5>{{ object.guid }}</h5>
{% for error in object.errors %}
<div class="error">
{{ error.message }}
{% if error.line %}
<span class="line">(line {{ error.line }})</span>
{% endif %}
</div>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% endif %}
{% endif %}
{% endblock %}

27
ckanext/harvest/templates_new/source/new.html
View File

@ -1,9 +1,34 @@
{% extends "source/base_form_page.html" %}
{% extends "source/admin_base.html" %}
{% block breadcrumb_content %}
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li class="active">{{ h.nav_named_link(_('Create Harvest Source'), '{0}_new'.format(c.dataset_type)) }}</li>
{% endblock %}
{% block actions_content %}
{% endblock %}
{% block subtitle %}{{ _('Create harvest source') }}{% endblock %}
{% block primary_content %}
<section class="module">
<div class="module-content">
{% block form %}{{ c.form | safe }}{% endblock %}
</div>
</section>
{% endblock %}
{% block secondary_content %}
<section class="module module-narrow">
<h2 class="module-heading"><i class="icon-large icon-info-sign"></i> {{ _('Harvest sources') }}</h2>
<div class="module-content">
<p>
{% trans %}
Harvest sources allow importing metadata from other catalogues
as CKAN datasets. These can be other CKAN instances or other
protocols and formats. (TODO: Review)
{% endtrans %}
</p>
</div>
</section>
{% endblock %}

26
ckanext/harvest/templates_new/source/new_source_form.html
View File

@ -1,11 +1,12 @@
{% import 'macros/form.html' as form %}
{% resource 'harvest-extra-field/main' %}
<form id="source-new" class="form-horizontal" method="post" >
{% block errors %}{{ form.errors(error_summary) }}{% endblock %}
{% call form.input('url', id='field-url', label=_('URL'), value=data.url, error=errors.url, classes=['control-full', 'control-large']) %}
<span class="info-block icon-large icon-info-sign">
<span class="info-block">
{{ _('This should include the http:// part of the URL') }}
</span>
{% endcall %}
@ -21,18 +22,29 @@
{{ form.markdown('notes', id='field-notes', label=_('Description'), value=data.notes, error=errors.notes) }}
{{ form.select('source_type', id='field-source_type', label=_('Source type'), options=h.harvester_types(), selected=data.source_type, error=errors.source_type) }}
<div class="harvest-types control-group">
<label class="control-label">Source type</label>
<div class="controls">
<ul>
{% for harvester in h.harvesters_info() %}
<li><b>{{ harvester['title']}}</b>: {{harvester['description'] }}</li>
{% set checked = False %}
{# select first option if nothing in data #}
{% if data.source_type == harvester['name'] or (not data.source_type and loop.first) %}
{% set checked = True %}
{% endif %}
<label class="radio">
<input type="radio" name="source_type" value="{{ harvester['name'] }}" {{ "checked " if checked }} data-module="harvest-type-change">
{{ harvester['title'] }}
<i class="icon-question-sign" title="{{ harvester['description'] }}" data-toggle="tooltip"></i>
</label>
{% endfor %}
</ul>
</div>
</div>
{{ form.select('frequency', id='field-frequency', label=_('Frequency of update'), options=h.harvest_frequencies(), selected=data.frequency, error=errors.frequency) }}
{% block extra_config %}
{{ form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) }}
{% endblock extra_config %}
{# if we have a default group then this wants remembering #}
{% if data.group_id %}
@ -58,6 +70,8 @@
{% endif %}
{% endif %}
<input id="save" name="save" value="Save" type="submit" class="btn"/>
<p class="form-actions">
<input id="save" name="save" value="Save" type="submit" class="btn btn-primary">
</p>
</form>

106
ckanext/harvest/templates_new/source/read.html
View File

@ -1,102 +1,8 @@
{% extends "page.html" %}
{% set pkg = c.pkg_dict %}
{% set authorized_user = h.check_access('harvest_source_update', {'id':pkg.id }) %}
{% block subtitle %}{{ pkg.title or pkg.name }}{% endblock %}
{% block breadcrumb_content %}
<li>{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
<li class="active">{{ h.nav_named_link(c.harvest_source.title|truncate(30), '{0}_read'.format(c.dataset_type), id=c.harvest_source.name) }}</li>
{% endblock %}
{% block actions_content %}
{% if authorized_user %}
<li>{{ h.nav_named_link(_('Edit'), '{0}_edit'.format(c.dataset_type), id=c.harvest_source.name, class_='btn btn-primary', icon='wrench')}}</li>
{% endif %}
<li>{{ h.follow_button('dataset', pkg.id) }}</li>
{% endblock %}
{% block primary_content %}
{% block package_revision_info %}
{% if c.pkg_revision_id %}
<div class="module info">
<p class="module-content">
{% set timestamp = h.render_datetime(c.pkg_revision_timestamp, with_hours=True) %}
{% set url = h.url(controller='package', action='read', id=pkg.name) %}
{% if c.pkg_revision_not_latest %}
{% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the <a href="{{ url }}">current revision</a>.{% endtrans %}
{% else %}
{% trans timestamp=timestamp %}This is the current revision of this dataset, as edited at {{ timestamp }}.{% endtrans %}
{% endif %}
</div>
</div>
{% endif %}
{% endblock %}
<article class="module prose">
{% block package_content %}
{% block package_description %}
<section class="module-content">
<h1>{{ pkg.title or pkg.name }}
{% if pkg.state.startswith('draft') %}
[{{ _('Draft') }}]
{% endif %}
</h1>
{% if c.pkg_notes_formatted %}
<div class="notes embedded-content">
{{ c.pkg_notes_formatted }}
</div>
{% endif %}
<span class="insert-comment-thread"></span>
</section>
{% endblock %}
{% block package_additional_info %}
{% snippet "package/snippets/additional_info.html", pkg_dict=pkg %}
{% endblock %}
{% if authorized_user %}
<section class="module-content additional-info">
<h2>Last Harvest Job</h2>
<a href="{{ h.url_for(controller='ckanext.harvest.controllers.view:ViewController', action='show_last_job', source=pkg.name) }}">{{ _('View full job report') }}</a>
{% snippet "snippets/job_details.html", job=pkg.status.last_job %}
</section>
{% endif %}
<section class="module-content">
<h3>Datasets</h3>
{{ h.package_list_for_source(c.pkg_dict['id']) }}
</section>
{% endblock %}
</article>
{% endblock %}
{% block secondary_content %}
{% block secondary_help_content %}{% endblock %}
{% block package_groups %}
{% for group in pkg.groups %}
{% snippet "snippets/group.html", group=group, truncate=70 %}
{% endfor %}
{% endblock %}
{% block package_social %}
{% snippet "snippets/social.html" %}
{% endblock %}
{% block package_license %}
{% snippet "snippets/license.html", pkg_dict=pkg %}
{% endblock %}
{% extends "source/read_base.html" %}
{% block primary_content_inner %}
<section class="module-content">
<h1 class="hide-heading">Datasets</h1>
{{ h.package_list_for_source(source.id) }}
</section>
{% endblock %}

42
ckanext/harvest/templates_new/source/read_base.html Normal file
View File

@ -0,0 +1,42 @@
{% extends "source/base.html" %}
{% block actions_content %}
{% if authorized_user %}
<li>{{ h.nav_named_link(_('Admin'), '{0}_admin'.format(c.dataset_type), id=c.harvest_source.name, class_='btn btn-primary', icon='wrench')}}</li>
{% endif %}
{% endblock %}
{# TODO: once #354 is merged in CKAN core .profile-info doesn't exist #}
{% block secondary_content %}
<div class="module context-info profile-info">
<section class="module-content">
<h1 class="heading">{{ c.harvest_source.title }}</h1>
{% if c.harvest_source.notes %}
<p>
{{ h.markdown_extract(c.harvest_source.notes, 180) }}
{{ h.nav_named_link(_('read more'), '{0}_about'.format(c.dataset_type), id=c.harvest_source.name) }}
</p>
{% else %}
<p class="empty">{{ _('There is no description for this harvest source') }}</p>
{% endif %}
<div class="nums">
<dl>
<dt>{{ _('Datasets') }}</dt>
<dd>{{ c.harvest_source.status.total_datasets }}</dd>
</dl>
</div>
</section>
</div>
{% endblock %}
{% block primary_content %}
<article class="module prose">
{% block page_header %}
{% snippet 'snippets/page_header.html', items=[
h.build_nav_icon('{0}_read'.format(c.dataset_type), _('Datasets'), id=source.name, icon='sitemap'),
h.build_nav_icon('{0}_about'.format(c.dataset_type), _('About'), id=source.name, icon='info-sign'),
] %}
{% endblock %}
{% block primary_content_inner %}{% endblock %}
</article>
{% endblock %}

7
ckanext/harvest/templates_new/source/search.html
View File

@ -2,6 +2,13 @@
{% block subtitle %}{{ _("Harvest sources") }}{% endblock %}
{% block add_action_content %}
<a href="{{ h.url_for('{0}_new'.format(c.dataset_type)) }}" class="btn btn-primary">
<i class="icon-plus-sign-alt"></i>
{{ _('Add Harvest Source') }}
</a>
{% endblock %}
{% block breadcrumb_content %}
<li class="active">{{ h.nav_named_link(_('Harvest Sources'), '{0}_search'.format(c.dataset_type)) }}</li>
{% endblock %}