Merge branch '12-org-source-listing' into 2.0-dataset-sources

ckanext/harvest/commands/harvester.py

@@ -322,9 +322,8 @@ class Harvester(CkanCommand):
         print '      objects: %s' % len(job.get('objects', []))
 
         print 'gather_errors: %s' % len(job.get('gather_errors', []))
-        if (len(job['gather_errors']) > 0):
+        for error in job.get('gather_errors', []):
-            for error in job.get('gather_errors', []):
+            print '               %s' % error['message']
-                print '               %s' % error['message']
 
         print ''
 

ckanext/harvest/controllers/organization.py

@@ -0,0 +1,208 @@
+import logging
+import genshi
+from urllib import urlencode
+
+from ckan import plugins as p
+from ckan.lib.base import c, model, request, render, h, g
+from ckan.lib.base import abort
+import ckan.lib.maintain as maintain
+import ckan.lib.search as search
+import ckan.new_authz
+
+from ckan.controllers.group import GroupController
+
+try:
+    from collections import OrderedDict # 2.7
+except ImportError:
+    from sqlalchemy.util import OrderedDict
+
+log = logging.getLogger(__name__)
+
+class OrganizationController(GroupController):
+
+    def source_list(self, id, limit=20):
+        group_type = self._get_group_type(id.split('@')[0])
+        context = {'model': model, 'session': model.Session,
+                   'user': c.user or c.author,
+                   'schema': self._db_to_form_schema(group_type=group_type),
+                   'for_view': True}
+        data_dict = {'id': id}
+
+        # unicode format (decoded from utf8)
+        q = c.q = request.params.get('q', '')
+
+        try:
+            c.group_dict = self._action('group_show')(context, data_dict)
+            c.group = context['group']
+        except p.toolkit.NotFound:
+            abort(404, p.toolkit._('Group not found'))
+        except p.toolkit.NotAuthorized:
+            abort(401, p.toolkit._('Unauthorized to read group %s') % id)
+
+        self._read(id, limit, dataset_type='harvest_source')
+        return render('source/org_source_list.html')
+
+    def _read(self, id, limit, dataset_type=None):
+        ''' This is common code used by both read and bulk_process'''
+        group_type = self._get_group_type(id.split('@')[0])
+        context = {'model': model, 'session': model.Session,
+                   'user': c.user or c.author,
+                   'schema': self._db_to_form_schema(group_type=group_type),
+                   'for_view': True, 'extras_as_string': True}
+
+        q = c.q = request.params.get('q', '')
+        # Search within group
+        q += ' groups: "%s"' % c.group_dict.get('name')
+
+        try:
+            description_formatted = ckan.misc.MarkdownFormat().to_html(
+            c.group_dict.get('description', ''))
+            c.description_formatted = genshi.HTML(description_formatted)
+        except Exception, e:
+            error_msg = "<span class='inline-warning'>%s</span>" %\
+                        p.toolkit._("Cannot render description")
+            c.description_formatted = genshi.HTML(error_msg)
+
+        context['return_query'] = True
+
+        # c.group_admins is used by CKAN's legacy (Genshi) templates only,
+        # if we drop support for those then we can delete this line.
+        c.group_admins = ckan.new_authz.get_group_or_org_admin_ids(c.group.id)
+
+        try:
+            page = int(request.params.get('page', 1))
+        except ValueError, e:
+            abort(400, ('"page" parameter must be an integer'))
+
+        # most search operations should reset the page counter:
+        params_nopage = [(k, v) for k, v in request.params.items()
+                         if k != 'page']
+        #sort_by = request.params.get('sort', 'name asc')
+        sort_by = request.params.get('sort', None)
+
+        def search_url(params):
+            if group_type == 'organization':
+                if c.action == 'bulk_process':
+                    url = self._url_for(controller='organization',
+                                        action='bulk_process',
+                                        id=id)
+                else:
+                    url = self._url_for(controller='organization',
+                                        action='read',
+                                        id=id)
+            else:
+                url = self._url_for(controller='group', action='read',
+                                id=id)
+            params = [(k, v.encode('utf-8') if isinstance(v, basestring)
+                       else str(v)) for k, v in params]
+            return url + u'?' + urlencode(params)
+
+        def drill_down_url(**by):
+            return h.add_url_param(alternative_url=None,
+                                   controller='group', action='read',
+                                   extras=dict(id=c.group_dict.get('name')),
+                                   new_params=by)
+
+        c.drill_down_url = drill_down_url
+
+        def remove_field(key, value=None, replace=None):
+            return h.remove_url_param(key, value=value, replace=replace,
+                                  controller='group', action='read',
+                                  extras=dict(id=c.group_dict.get('name')))
+
+        c.remove_field = remove_field
+
+        def pager_url(q=None, page=None):
+            params = list(params_nopage)
+            params.append(('page', page))
+            return search_url(params)
+
+        try:
+            c.fields = []
+            search_extras = {}
+            for (param, value) in request.params.items():
+                if not param in ['q', 'page', 'sort'] \
+                        and len(value) and not param.startswith('_'):
+                    if not param.startswith('ext_'):
+                        c.fields.append((param, value))
+                        q += ' %s: "%s"' % (param, value)
+                    else:
+                        search_extras[param] = value
+
+            fq = 'capacity:"public"'
+            user_member_of_orgs = [org['id'] for org
+                                   in h.organizations_available('read')]
+
+            if (c.group and c.group.id in user_member_of_orgs):
+                fq = ''
+                context['ignore_capacity_check'] = True
+
+            facets = OrderedDict()
+
+            default_facet_titles = {'groups': p.toolkit._('Groups'),
+                              'tags': p.toolkit._('Tags'),
+                              'res_format': p.toolkit._('Formats'),
+                              'license': p.toolkit._('Licence'), }
+
+            for facet in g.facets:
+                if facet in default_facet_titles:
+                    facets[facet] = default_facet_titles[facet]
+                else:
+                    facets[facet] = facet
+
+            if dataset_type:
+                fq = fq + 'dataset_type:"{dataset_type}"'.format(dataset_type=dataset_type)
+
+            # Facet titles
+            for plugin in p.PluginImplementations(p.IFacets):
+                if self.group_type == 'organization':
+                    facets = plugin.organization_facets(
+                        facets, self.group_type, dataset_type)
+                else:
+                    facets = plugin.group_facets(
+                        facets, self.group_type, dataset_type)
+
+            if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs):
+                del facets['capacity']
+
+            c.facet_titles = facets
+
+            data_dict = {
+                'q': q,
+                'fq': fq,
+                'facet.field': facets.keys(),
+                'rows': limit,
+                'sort': sort_by,
+                'start': (page - 1) * limit,
+                'extras': search_extras
+            }
+
+            query = p.toolkit.get_action('package_search')(context, data_dict)
+
+            c.page = h.Page(
+                collection=query['results'],
+                page=page,
+                url=pager_url,
+                item_count=query['count'],
+                items_per_page=limit
+            )
+
+            c.facets = query['facets']
+            maintain.deprecate_context_item(
+              'facets',
+              'Use `c.search_facets` instead.')
+
+            c.search_facets = query['search_facets']
+            c.search_facets_limits = {}
+            for facet in c.facets.keys():
+                limit = int(request.params.get('_%s_limit' % facet, 10))
+                c.search_facets_limits[facet] = limit
+            c.page.items = query['results']
+
+            c.sort_by_selected = sort_by
+
+        except search.SearchError, se:
+            log.error('Group search error: %r', se.args)
+            c.query_error = True
+            c.facets = {}
+            c.page = h.Page(collection=[])

ckanext/harvest/plugin.py

@@ -219,6 +219,8 @@ class Harvest(p.SingletonPlugin, DefaultDatasetForm):
         map.connect('harvest_job_show_last', '/' + DATASET_TYPE_NAME + '/{source}/job/last', controller=controller, action='show_last_job')
         map.connect('harvest_job_show', '/' + DATASET_TYPE_NAME + '/{source}/job/{id}', controller=controller, action='show_job')
 
+        org_controller = 'ckanext.harvest.controllers.organization:OrganizationController'
+        map.connect('harvest_source_org_list', '/organization/' + DATASET_TYPE_NAME + '/' + '{id}', controller=org_controller, action='source_list')
 
         return map
 

ckanext/harvest/templates_new/snippets/source_item.html

@@ -7,6 +7,7 @@ hide_resources - If true hides the resources (default: false).
 banner         - If true displays a popular banner (default: false).
 truncate       - The length to trucate the description to (default: 180)
 truncate_title - The length to truncate the title to (default: 80).
+show_organization - Boolean on whether to show the related organization
 
 Example:
 
@@ -17,11 +18,12 @@ Example:
 {% set truncate_title = truncate_title or 80 %}
 {% set title = source.title or source.name %}
 {% set source_type = h.get_pkg_dict_extra(source, 'source_type') %}
+{% set url = h.url_for('harvest_source_edit', id=source.name) if within_organization else h.url_for('harvest_source_read', id=source.name) %}
 
 <li class="{{ item_class or "dataset-item" }}">
   <div class="dataset-content">
     <h3 class="dataset-heading">
-      {{ h.link_to(h.truncate(title, truncate_title), h.url_for('harvest_source_read', id=source.name)) }}
+      {{ h.link_to(h.truncate(title, truncate_title), url) }}
       {% if source.get(state, '').startswith('draft') %}
         <span class="label label-info">{{ _('Draft') }}</span>
       {% elif source.get(state, '').startswith('deleted') %}
@@ -37,7 +39,7 @@ Example:
 
     <p class="muted">
       {{ _('Datasets') }}: {{ source.status.total_datasets }}
-      {% if source.organization %}
+      {% if not within_organization and source.organization %}
       &mdash; {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization_read', id=source.organization.name)) }}</a>
       {% endif %}
     </p>

ckanext/harvest/templates_new/snippets/source_list.html

@@ -17,7 +17,7 @@ Example:
 {% if sources %}
   <ul class="{{ list_class or 'dataset-list unstyled' }}">
     {% for source in sources %}
-      {% snippet 'snippets/source_item.html', source=source, item_class=item_class, hide_resources=hide_resources, banner=banner, truncate=truncate, truncate_title=truncate_title %}
+      {% snippet 'snippets/source_item.html', source=source, item_class=item_class, hide_resources=hide_resources, banner=banner, truncate=truncate, truncate_title=truncate_title, within_organization=within_organization %}
     {% endfor %}
   </ul>
 {% endif %}

ckanext/harvest/templates_new/source/org_source_list.html

@@ -0,0 +1,33 @@
+{% extends 'organization/edit_base.html' %}
+
+{% block subtitle %}{{ _('Harvest Sources') }} - {{ super() }}{% endblock %}
+
+{% block primary_content_inner %}
+  <div class="clearfix">
+    <h1 class="hide-heading">{{ _('Harvest Sources') }}</h1>
+    <div class="primary">
+      <h3 class="results">
+        {%- if c.page.item_count -%}
+          {{ c.page.item_count }} harvest sources{{ _(" found for \"{query}\"").format(query=c.q) if c.q }}
+        {%- elif request.params -%}
+          {{ _('Sorry no harvest sources found for "{query}"').format(query=c.q) }}
+        {%- else -%}
+          {{ _('Harvest Sources') }}
+        {%- endif -%}
+      </h3>
+      {% snippet 'snippets/source_list.html', sources=c.page.items, within_organization=true %}
+    </div>
+    <aside class="tertiary">
+      <form method="GET">
+        <span class="control-group search-normal">
+          <input type="text" class="search" name="q" value="{{ c.q }}" autocomplete="off" placeholder="{{ _('Search sources...') }}" />
+          <button type="submit"><i class="icon-search"></i> <span>{{ _('Search') }}</span></button>
+        </span>
+      </form>
+      {{ h.snippet('snippets/facet_list.html', title=_('Frequency'), name='frequency', within_tertiary=true, extras={'id':c.group_dict.name}) }}
+      {{ h.snippet('snippets/facet_list.html', title=_('Type'), name='type', within_tertiary=true, extras={'id':c.group_dict.name}) }}
+    </aside>
+  </div>
+  {{ c.page.pager() }}
+{% endblock %}
+

ckanext/harvest/templates_new/source/search.html

@@ -71,7 +71,7 @@
         {% endtrans %}
       {% endif %}
 
-      {{ h.snippet('snippets/source_list.html', sources=c.page.items) }}
+      {{ h.snippet('snippets/source_list.html', sources=c.page.items, show_organization=true) }}
 
     </div>