-
- {#Added by Francesco Mangiacrapa; see: 8964 #}
-
-
{% endblock %}
-
-
+
{#
The pre_primary block can be used to add content to before the
rendering of the main content columns of the page.
@@ -87,8 +53,26 @@
{% block pre_primary %}
{% endblock %}
+ {% block secondary %}
+
+ {% endblock %}
+
{% block primary %}
-
+
{#
The primary_content block can be used to add content to the page.
This is the main block that is likely to be used within a template.
@@ -127,24 +111,6 @@
{% endblock %}
{% endblock %}
-
- {% block secondary %}
-
- {% endblock %}
{% endblock %}
@@ -152,9 +118,9 @@
{% endblock -%}
{#
- Override the footer on a page by page basis by extending this block. If
- making sitewide header changes it is preferable to override the footer.html-u
- file.
+ Override the footer on a page by page basis by extending this block. If
+ making sitewide header changes it is preferable to override the footer.html-u
+ file.
#}
{%- block footer %}
{% include "footer.html" %}
@@ -165,7 +131,7 @@
{% asset 'base/main' %}
{% asset 'base/ckan' %}
{% if g.tracking_enabled %}
- {% asset 'base/tracking.js' %}
+ {% asset 'base/tracking' %}
{% endif %}
{{ super() }}
{% endblock -%}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/revision/__init__.py b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/revision/__init__.py
new file mode 100644
index 0000000..b9445d5
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/revision/__init__.py
@@ -0,0 +1,3 @@
+# encoding: utf-8
+
+# empty file needed for pylons to find templates in this directory
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/add_dataset.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/add_dataset.html
new file mode 100644
index 0000000..bfe63ae
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/add_dataset.html
@@ -0,0 +1,11 @@
+{# Adds 'Add Dataset' button #}
+
+{% set dataset_type = dataset_type if dataset_type else h.default_package_type() %}
+{% set link_label = h.humanize_entity_type('package', dataset_type, 'add link') or _("Add Dataset") %}
+{% set route_name = dataset_type ~ '.new' %}
+
+{% if group %}
+ {% link_for link_label, named_route=route_name, group=group, class_='btn btn-primary', icon='plus-square' %}
+{% else %}
+ {% link_for link_label, named_route=route_name, class_='btn btn-primary', icon='plus-square' %}
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/additional_info.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/additional_info.html
new file mode 100644
index 0000000..fb21cdf
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/additional_info.html
@@ -0,0 +1,25 @@
+{# This snippet creates an Additional Info Table
+
+extras is a list of tuples of the form (key, value)
+
+#}
+{% if extras %}
+
{{ _('Additional Info') }}
+
+
+
+ {{ _('Field') }}
+ {{ _('Value') }}
+
+
+
+ {% for extra in extras %}
+ {% set key, value = extra %}
+
+ {{ _(key|e) }}
+ {{ value }}
+
+ {% endfor %}
+
+
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context.html
new file mode 100644
index 0000000..edca7e1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context.html
@@ -0,0 +1,10 @@
+{%if type == 'user' %}
+
{{ dict.name }}
+ {% snippet 'snippets/context/user.html', id=dict.id, name=dict.name, about=dict.about, is_me='false', num_followers=dict.num_followers, number_created_packages=dict.number_created_packages %}
+{%elif type == 'dataset' %}
+
{{ dict.title }}
+ {% snippet 'snippets/context/dataset.html', id=dict.id, name=dict.name, notes=dict.notes, num_resources=dict.num_resources, num_tags=dict.num_tags %}
+{%elif type == 'group' %}
+
{{ dict.title }}
+ {% snippet 'snippets/context/group.html', id=dict.id, name=dict.name, description=dict.description, num_followers=dict.num_followers, package_count=dict.package_count %}
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/dataset.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/dataset.html
new file mode 100644
index 0000000..ea0a262
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/dataset.html
@@ -0,0 +1,21 @@
+
+ {% if notes != 'null' %}
+ {% set notes = notes|striptags %}
+
+ {{ notes|truncate(160) }}
+
+ {% endif %}
+
+ {{ h.follow_button('dataset', id) }}
+
+
+
+ {{ _('Resources') }}
+ {{ h.SI_number_span(num_resources) }}
+
+
+ {{ _('Tags') }}
+ {{ h.SI_number_span(num_tags) }}
+
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/group.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/group.html
new file mode 100644
index 0000000..95a7d7b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/group.html
@@ -0,0 +1,23 @@
+{% set dataset_type = h.default_package_type() %}
+
+
+ {% if description != 'null' %}
+ {% set description = description|striptags %}
+
+ {{ description|truncate(160) }}
+
+ {% endif %}
+
+ {{ h.follow_button('group', id) }}
+
+
+
+ {{ _('Followers') }}
+ {{ h.SI_number_span(num_followers) }}
+
+
+ {{ h.humanize_entity_type('package', dataset_type, 'facet label') or _('Datasets') }}
+ {{ h.SI_number_span(package_count) }}
+
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/user.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/user.html
new file mode 100644
index 0000000..ab8589f
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/context/user.html
@@ -0,0 +1,25 @@
+{% set dataset_type = h.default_package_type() %}
+
+
+ {% if about != 'null' %}
+ {% set about = about|striptags %}
+
+ {{ about|truncate(160) }}
+
+ {% endif %}
+
+ {% if is_me == 'false' %}
+ {{ h.follow_button('user', id) }}
+ {% endif %}
+
+
+
+ {{ _('Followers') }}
+ {{ h.SI_number_span(num_followers) }}
+
+
+ {{ h.humanize_entity_type('package', dataset_type, 'facet label') or _('Datasets') }}
+ {{ h.SI_number_span(number_created_packages) }}
+
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/csrf_input.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/csrf_input.html
new file mode 100644
index 0000000..b495045
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/csrf_input.html
@@ -0,0 +1,2 @@
+{# Adds a csrf_token to the forms as a hidden input. #}
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/custom_form_fields.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/custom_form_fields.html
index f0af62b..9aa2d9f 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/custom_form_fields.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/custom_form_fields.html
@@ -10,24 +10,15 @@ Example:
#}
{% import "macros/form.html" as form %}
-
-{% set d4science_cms_obj_placeholders = h.d4science_get_content_moderator_system_placeholder() %}
-{% set d4science_system_type_obj = h.d4science_theme_get_systemtype_field_dict_from_session() %}
+
-
{% for extra in extras %}
- {% set is_system_field = extra.key.startswith(d4science_cms_obj_placeholders.prefix) or extra.key.startswith(d4science_system_type_obj.id) %}
- {% set apply_classes = [] %}
- {% if is_system_field %}
- {% set apply_classes = ['disabled-div'] %}
- {% endif %}
{% set prefix = 'extras__%d__' % loop.index0 %}
{{ form.custom(
names=(prefix ~ 'key', prefix ~ 'value', prefix ~ 'deleted'),
id='field-extras-%d' % loop.index,
label=_('Custom Field'),
values=(extra.key, extra.value, extra.deleted),
- classes=apply_classes,
error=errors[prefix ~ 'key'] or errors[prefix ~ 'value']
) }}
{% endfor %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/datapusher_status.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/datapusher_status.html
new file mode 100644
index 0000000..8cd6a39
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/datapusher_status.html
@@ -0,0 +1,14 @@
+{# Datapusher status indicator
+
+resource: the resource
+
+#}
+{% if resource.datastore_active %}
+ {% set job = h.datapusher_status(resource.id) %}
+ {% set title = _('Datapusher status: {status}.').format(status=job.status) %}
+ {% if job.status == 'unknown' %}
+
+ {% else %}
+
+ {% endif %}
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/disqus_trackback.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/disqus_trackback.html
new file mode 100644
index 0000000..64a5cb7
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/disqus_trackback.html
@@ -0,0 +1,4 @@
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/facet_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/facet_list.html
index e985503..127fe99 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/facet_list.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/facet_list.html
@@ -2,98 +2,90 @@
Construct a facet module populated with links to filtered results.
name
- The field name identifying the facet field, eg. "tags"
+The field name identifying the facet field, eg. "tags"
title
- The title of the facet, eg. "Tags", or "Tag Cloud"
+The title of the facet, eg. "Tags", or "Tag Cloud"
label_function
- Renders the human-readable label for each facet value.
- If defined, this should be a callable that accepts a `facet_item`.
- eg. lambda facet_item: facet_item.display_name.upper()
- By default it displays the facet item's display name, which should
- usually be good enough
+Renders the human-readable label for each facet value.
+If defined, this should be a callable that accepts a `facet_item`.
+eg. lambda facet_item: facet_item.display_name.upper()
+By default it displays the facet item's display name, which should
+usually be good enough
if_empty
- A string, which if defined, and the list of possible facet items is empty,
- is displayed in lieu of an empty list.
+A string, which if defined, and the list of possible facet items is empty,
+is displayed in lieu of an empty list.
count_label
- A callable which accepts an integer, and returns a string. This controls
- how a facet-item's count is displayed.
+A callable which accepts an integer, and returns a string. This controls
+how a facet-item's count is displayed.
extras
- Extra info passed into the add/remove params to make the url
+Extra info passed into the add/remove params to make the url
alternative_url
- URL to use when building the necessary URLs, instead of the default
- ones returned by url_for. Useful eg for dataset types.
+URL to use when building the necessary URLs, instead of the default
+ones returned by url_for. Useful eg for dataset types.
hide_empty
- Do not show facet if there are none, Default: false.
+Do not show facet if there are none, Default: false.
-within_tertiary
- Boolean for when a facet list should appear in the the right column of the
- page and not the left column.
+search_facets
+Dictionary with search facets
#}
{% block facet_list %}
- {% set hide_empty = hide_empty or false %}
- {% with items = items or h.get_facet_items_dict(name) %}
- {% if items or not hide_empty %}
- {% if within_tertiary %}
- {% set nav_class = 'nav nav-pills nav-stacked' %}
- {% set nav_item_class = ' ' %}
- {% set wrapper_class = 'nav-facet nav-facet-tertiary' %}
- {% endif %}
- {% block facet_list_item %}
-
- {% block facet_list_heading %}
-
-
- {% endblock %}
- {% block facet_list_items %}
- {% with items = items or h.get_facet_items_dict(name) %}
- {% if items %}
-
-
- {% for item in items %}
-
- {% set href = h.remove_url_param(name, item.name, extras=extras, alternative_url=alternative_url) if item.active else h.add_url_param(new_params={name: item.name}, extras=extras, alternative_url=alternative_url) %}
- {% set label = label_function(item) if label_function else item.display_name %}
- {# Updated by Francesco Mangiacrapa, see #11747 #}
- {#
- {% set label_truncated = h.truncate(label, 22) if not label_function else label %}
- #}
- {% set count = count_label(item['count']) if count_label else ('(%d)' % item['count']) %}
-
-
- {{ label }} {{ count }}
-
-
- {% endfor %}
-
-
+ {% set hide_empty = hide_empty or false %}
+ {% with items = items or h.get_facet_items_dict(name, search_facets) %}
+ {% if items or not hide_empty %}
+ {% block facet_list_item %}
+
+ {% block facet_list_heading %}
+
+
+ {% endblock %}
+ {% block facet_list_items %}
+ {% with items = items or h.get_facet_items_dict(name, search_facets) %}
+ {% if items %}
+
+
+ {% for item in items %}
+ {% set href = h.remove_url_param(name, item.name, extras=extras, alternative_url=alternative_url) if item.active else h.add_url_param(new_params={name: item.name}, extras=extras, alternative_url=alternative_url) %}
+ {% set label = label_function(item) if label_function else item.display_name %}
+ {% set label_truncated = label|truncate(22) if not label_function else label %}
+ {% set count = count_label(item['count']) if count_label else ('%d' % item['count']) %}
+
+
+ {{ label_truncated }}
+ -
+ {{ count }}
+ {% if item.active %}x {% endif %}
+
+
+ {% endfor %}
+
+
-
- {% else %}
- {{ _('There are no {facet_type} that match this search').format(facet_type=title) }}
- {% endif %}
- {% endwith %}
- {% endblock %}
-
- {% endblock %}
- {% endif %}
- {% endwith %}
+
+ {% else %}
+ {{ _('There are no {facet_type} that match this search').format(facet_type=title) }}
+ {% endif %}
+ {% endwith %}
+ {% endblock %}
+
+ {% endblock %}
+ {% endif %}
+ {% endwith %}
{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/follow_button.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/follow_button.html
new file mode 100644
index 0000000..a3b6f06
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/follow_button.html
@@ -0,0 +1,11 @@
+{% if following %}
+
+
+{% else %}
+
+
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/group.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/group.html
new file mode 100644
index 0000000..e06f0d9
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/group.html
@@ -0,0 +1,25 @@
+{#
+Embeds a group within the sidebar of a page.
+
+group - The group dict.
+
+Example:
+
+ {% snippet 'snippets/group, group=c.group_dict %}
+
+#}
+{% with url=h.url_for('group.read', id=group.name) %}
+
+
+
+
+
+
+ {% if group.description %}
+
{{ h.markdown_extract(group.description, 120) }}
+ {% endif %}
+
+
+{% endwith %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/group_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/group_item.html
new file mode 100644
index 0000000..8fc5bc1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/group_item.html
@@ -0,0 +1,27 @@
+{% block group_item %}
+
+ {% block group_item_header %}
+
+ {% endblock %}
+ {% block group_item_content %}
+ {% set list_class = "list-unstyled dataset-list" %}
+ {% set item_class = "dataset-item module-content" %}
+ {% snippet 'snippets/package_list.html', packages=group.packages, list_class=list_class, item_class=item_class %}
+ {% endblock %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/home_breadcrumb_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/home_breadcrumb_item.html
new file mode 100644
index 0000000..09ee14f
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/home_breadcrumb_item.html
@@ -0,0 +1,2 @@
+{# Used to insert the home icon into a breadcrumb #}
+
{{ _('Home') }}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/language_selector.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/language_selector.html
new file mode 100644
index 0000000..16c3877
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/language_selector.html
@@ -0,0 +1,17 @@
+{% set current_lang = request.environ.CKAN_LANG %}
+
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/license.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/license.html
new file mode 100644
index 0000000..397d0a9
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/license.html
@@ -0,0 +1,37 @@
+{% macro license_string(pkg_dict) %}
+ {% if 'license_url' in pkg_dict %}
+
{{ pkg_dict.license_title }}
+ {% else %}
+ {% if pkg_dict.license_id %}
+
{{ pkg_dict.license_title }}
+ {% else %}
+
{{ _('No License Provided') }}
+ {% endif %}
+ {% endif %}
+{% endmacro %}
+
+{% block license %}
+ {% if text_only %}
+ {{ license_string(pkg_dict) }}
+ {% else %}
+ {% block license_wrapper %}
+
+ {% block license_title %}
+
+ {% block license_content_inner %}
+ {{ license_string(pkg_dict) }}
+ {% if pkg_dict.isopen %}
+
+
+ {% endif %}
+ {% endblock %}
+
+ {% endblock %}
+
+ {% endblock %}
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/local_friendly_datetime.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/local_friendly_datetime.html
new file mode 100644
index 0000000..912cf0b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/local_friendly_datetime.html
@@ -0,0 +1,14 @@
+{#
+Displays a datetime that can be converted to a users timezone using JavaScript.
+In the data-datetime attribute, the date is rendered in ISO 8601 format.
+
+datetime_obj - the datetime object to display
+
+Example:
+
+ {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_created %}
+
+#}
+
+ {{ h.render_datetime(datetime_obj, with_hours=True) }}
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/organization.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/organization.html
new file mode 100644
index 0000000..755a9be
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/organization.html
@@ -0,0 +1,73 @@
+{#
+ Embeds a organization within the sidebar of a page.
+
+ organization - The organization dict.
+ has_context_title - Boolean for whether to display a module-heading for when
+ this snippet appears within a non-organization template
+ (e.g. dataset page sidebar)
+
+ Example:
+
+ {% snippet 'snippets/organization, organization=c.group_dict %}
+
+ #}
+
+{% set url = h.url_for(organization.type + '.read', id=organization.name, ) %}
+{% set dataset_type = h.default_package_type() %}
+
+{% block info %}
+
+ {% if has_context_title %}
+
+ {% endif %}
+
+ {% block inner %}
+ {% block image %}
+
+ {% endblock %}
+ {% block heading %}
+ {{ organization.title or organization.name }}
+ {% if organization.state == 'deleted' %}
+ [{{ _('Deleted') }}]
+ {% endif %}
+
+ {% endblock %}
+ {% block description %}
+ {% if organization.description %}
+
+ {{ h.markdown_extract(organization.description, 180) }}
+
+
+ {% link_for _('read more'), named_route=organization.type ~ '.about', id=organization.name %}
+
+ {% else %}
+ {{ h.humanize_entity_type('organization', organization.type, 'no description') or _('There is no description for this organization') }}
+ {% endif %}
+ {% endblock %}
+ {% if show_nums %}
+ {% block nums %}
+
+
+ {{ _('Followers') }}
+ {{ h.SI_number_span(organization.num_followers) }}
+
+
+ {{ h.humanize_entity_type('package', dataset_type, 'facet label') or _('Datasets') }}
+ {{ h.SI_number_span(organization.package_count) }}
+
+
+ {{ h.follow_button('group', organization.id) }}
+
+ {% endblock %}
+ {% endif %}
+ {% endblock %}
+
+
+ {% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/organization_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/organization_item.html
new file mode 100644
index 0000000..9bd7e22
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/organization_item.html
@@ -0,0 +1,27 @@
+{% set url=h.url_for(organization.type ~ '.read', id=organization.name) %}
+{% block organization_item %}
+
+ {% block organization_item_header %}
+
+ {% endblock %}
+ {% block organization_item_content %}
+ {% set list_class = "list-unstyled dataset-list" %}
+ {% set item_class = "dataset-item module-content" %}
+ {% snippet 'snippets/package_list.html', packages=organization.packages, list_class=list_class, item_class=item_class %}
+ {% endblock %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_item.html
index ca2d067..801d9c8 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_item.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_item.html
@@ -4,130 +4,67 @@ Displays a single of dataset.
package - A package to display.
item_class - The class name to use on the list item.
hide_resources - If true hides the resources (default: false).
-banner - If true displays a popular banner (default: false).
-truncate - The length to trucate the description to (default: 180)
-truncate_title - The length to truncate the title to (default: 80).
Example:
{% snippet 'snippets/package_item.html', package=c.datasets[0] %}
#}
-
-{% set truncate = truncate or 180 %}
-{% set truncate_title = truncate_title or 80 %}
{% set title = package.title or package.name %}
-{% set notes = h.markdown_extract(package.notes, extract_length=truncate) %}
-{% set acquired = h.is_dataset_acquired(package) %}
-{% set owner = h.is_owner(package) %}
+{% set notes = h.markdown_extract(package.notes, extract_length=180) %}
-{#
-{% resource 'd4science_theme/custom.css' %}
-CHANGED BY FRANCESCO.MANGIACRAPA
-#}
-
-{% block package_item_content %}
-
-{% if package.private and not h.can_read(package) %}
-
-
-
- {% if package.private and not h.can_read(package) %}
-
-
+{% block package_item %}
+
+ {% block content %}
+
+ {% block heading %}
+
+ {% block heading_private %}
+ {% if package.private %}
+
+
+ {% endif %}
+ {% endblock %}
+ {% block heading_title %}
+
+ {{title|truncate(80)}}
+
+ {% endblock %}
+ {% block heading_meta %}
+ {% if package.get('state', '').startswith('draft') %}
+ {{ _('Draft') }}
+ {% elif package.get('state', '').startswith('deleted') %}
+ {{ _('Deleted') }}
+ {% endif %}
+ {{ h.popular('recent views', package.tracking_summary.recent, min=10) if package.tracking_summary }}
+ {% endblock %}
+
+ {% endblock %}
+ {% block notes %}
+ {% if notes %}
+
{{ notes|urlize }}
+ {% else %}
+
{{ h.humanize_entity_type('package', package.type, 'no description') or _("There is no description for this dataset") }}
+ {% endif %}
+ {% endblock %}
+
+ {% block resources_inner %}
+ {% for resource in h.dict_list_reduce(package.resources, 'format') %}
+
+ {{ resource }}
+
+ {% endfor %}
+ {% endblock %}
+
+ {% endblock %}
{% endif %}
- {% if acquired and not owner %}
-
-
- {% endif %}
-
-
- {% if package.private and not h.can_read(package) %}
- {# {{ _(h.truncate(title, truncate_title)) }} #}
- Dataset
-
- {{ h.acquire_button(package) }}
- {% else %}
- {{ h.link_to(h.truncate(title, truncate_title), h.url_for(controller='dataset', action='read', id=package.name)) }}
- {% endif %}
-
-
- {% if package.get('state', '').startswith('draft') %}
- {{ _('Draft') }}
- {% elif package.get('state', '').startswith('deleted') %}
- {{ _('Deleted') }}
- {% endif %}
- {{ h.popular('recent views', package.tracking_summary.recent, min=10) if package.tracking_summary }}
-
- {% if banner %}
- {{ _('Popular') }}
- {% endif %}
- {% if notes %}
-
- {% endif %}
-
-{% else %}
-
-
-
- {% if package.private and not h.can_read(package) %}
-
-
- {% endif %}
- {% if acquired and not owner %}
-
-
- {% endif %}
- {% if owner %}
-
-
- {% endif %}
-
-
- {% if package.private and not h.can_read(package) %}
- {{ _(h.truncate(title, truncate_title)) }}
-
- {{ h.acquire_button(package) }}
- {% else %}
- {{ h.link_to(h.truncate(title, truncate_title), h.url_for(controller='dataset', action='read', id=package.name)) }}
- {% endif %}
-
-
- {% if package.get('state', '').startswith('draft') %}
- {{ _('Draft') }}
- {% elif package.get('state', '').startswith('deleted') %}
- {{ _('Deleted') }}
- {% endif %}
- {{ h.popular('recent views', package.tracking_summary.recent, min=10) if package.tracking_summary }}
-
- {% if banner %}
-
{{ _('Popular') }}
- {% endif %}
- {% if notes %}
-
{{ notes|urlize }}
- {% endif %}
-
- {% for resource in h.dict_list_reduce(package.resources, 'format') %}
-
- {{ resource }}
-
- {% endfor %}
-
- {% endif %}
-
-
-{% endif %}
+ {% endblock %}
+ {% endblock %}
+
{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_list.html
new file mode 100644
index 0000000..634b1a1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/package_list.html
@@ -0,0 +1,24 @@
+{#
+Displays a list of datasets.
+
+packages - A list of packages to display.
+list_class - The class name for the list item.
+item_class - The class name to use on each item.
+hide_resources - If true hides the resources (default: false).
+
+Example:
+
+ {% snippet 'snippets/package_list.html', packages=c.datasets %}
+
+#}
+{% block package_list %}
+ {% if packages %}
+
+ {% block package_list_inner %}
+ {% for package in packages %}
+ {% snippet 'snippets/package_item.html', package=package, item_class=item_class, hide_resources=hide_resources %}
+ {% endfor %}
+ {% endblock %}
+
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/popular.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/popular.html
new file mode 100644
index 0000000..1663fad
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/popular.html
@@ -0,0 +1,4 @@
+{# show the popularity icon #}
+{% if number >= min %}
+
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/private.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/private.html
new file mode 100644
index 0000000..70c86ae
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/private.html
@@ -0,0 +1,3 @@
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/search_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/search_form.html
new file mode 100644
index 0000000..9d755c2
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/search_form.html
@@ -0,0 +1,90 @@
+{% import 'macros/form.html' as form %}
+
+{% set placeholder = placeholder if placeholder else _('Search datasets...') %}
+{% set sorting = sorting if sorting else [(_('Name Ascending'), 'name asc'), (_('Name Descending'), 'name desc')] %}
+{% set search_class = search_class if search_class else 'search-giant' %}
+{% set no_bottom_border = no_bottom_border if no_bottom_border else false %}
+{% set form_id = form_id if form_id else false %}
+
+
+
+{% if show_empty and count == 0 and not error %}
+ {% trans %}
+
+ {% endtrans %}
+{% endif %}
+
+{% if error %}
+ {% trans %}
+
There was an error while searching. Please try again.
+ {% endtrans %}
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/search_result_text.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/search_result_text.html
new file mode 100644
index 0000000..ffc6794
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/search_result_text.html
@@ -0,0 +1,58 @@
+{#
+
+Displays a test for results of a search.
+
+query - The text that was searched for
+count - The number of results for the search
+type - Search result type (dataset, group, organization)
+
+Example:
+
+ {% snippet 'snippets/search_result_text.html', query=query, count=count, type='dataset' %}
+
+#}
+{% if type == 'dataset' %}
+ {% set text_query = ungettext('{number} dataset found for "{query}"', '{number} datasets found for "{query}"', count) %}
+ {% set text_query_none = _('No datasets found for "{query}"') %}
+ {% set text_no_query = ungettext('{number} dataset found', '{number} datasets found', count) %}
+ {% set text_no_query_none = _('No datasets found') %}
+
+{% elif type == 'group' %}
+ {% set text_query = ungettext('{number} group found for "{query}"', '{number} groups found for "{query}"', count) %}
+ {% set text_query_none = _('No groups found for "{query}"') %}
+ {% set text_no_query = ungettext('{number} group found', '{number} groups found', count) %}
+ {% set text_no_query_none = _('No groups found') %}
+
+{% elif type == 'organization' %}
+ {% set text_query = ungettext('{number} organization found for "{query}"', '{number} organizations found for "{query}"', count) %}
+ {% set text_query_none = _('No organizations found for "{query}"') %}
+ {% set text_no_query = ungettext('{number} organization found', '{number} organizations found', count) %}
+ {% set text_no_query_none = _('No organizations found') %}
+
+{% else %}
+ {% set text_query_singular = '{number} ' + type + ' found for "{query}"' %}
+ {% set text_query_plural = '{number} ' + type + 's found for "{query}"' %}
+ {% set text_query_none_plural = 'No ' + type + 's found for "{query}"' %}
+ {% set text_no_query_singular = '{number} ' + type + ' found' %}
+ {% set text_no_query_plural = '{number} ' + type + 's found' %}
+ {% set text_no_query_none_plural = 'No ' + type + 's found' %}
+
+ {% set text_query = ungettext(text_query_singular, text_query_plural, count) %}
+ {% set text_query_none = _(text_query_none_plural) %}
+ {% set text_no_query = ungettext(text_no_query_singular, text_no_query_plural, count) %}
+ {% set text_no_query_none = _(text_no_query_none_plural) %}
+{%- endif -%}
+
+{% if query %}
+ {%- if count -%}
+ {{ text_query.format(number=h.localised_number(count), query=query, type=type) }}
+ {%- else -%}
+ {{ text_query_none.format(query=query, type=type) }}
+ {%- endif -%}
+{%- else -%}
+ {%- if count -%}
+ {{ text_no_query.format(number=h.localised_number(count), type=type) }}
+ {%- else -%}
+ {{ text_no_query_none.format(type=type) }}
+ {%- endif -%}
+{%- endif -%}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/simple_search.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/simple_search.html
new file mode 100644
index 0000000..6bc6a53
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/simple_search.html
@@ -0,0 +1,17 @@
+{% set input_class = input_class or 'search-giant' %}
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/social.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/social.html
new file mode 100644
index 0000000..16da6b6
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/social.html
@@ -0,0 +1,14 @@
+{% set current_url = h.full_current_url() %}
+{% block social %}
+
+ {% block social_title %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/tag_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/tag_list.html
index c14f6d1..77f753e 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/tag_list.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/snippets/tag_list.html
@@ -1,17 +1,19 @@
{#
-render a list of tags linking to the dataset search page
-tags: list of tags
+ Render a list of tags linking to the dataset search page
+
+ tags - list of tag dicts
+
+ Example:
+
+ {% snippet 'snippets/tag_list.html', tags=tags, _class='tag-list' %}
-Removed truncate function by Francesco Mangiacrapa
- {{ h.truncate(tag.display_name, 22) }}
-
#}
{% set _class = _class or 'tag-list' %}
{% block tag_list %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/broken_helper_as_attribute.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/broken_helper_as_attribute.html
new file mode 100644
index 0000000..b592887
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/broken_helper_as_attribute.html
@@ -0,0 +1,5 @@
+{% extends 'page.html' %}
+
+{% block page %}
+ {{ h.not_here() }}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/broken_helper_as_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/broken_helper_as_item.html
new file mode 100644
index 0000000..d19d2e0
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/broken_helper_as_item.html
@@ -0,0 +1,5 @@
+{% extends 'page.html' %}
+
+{% block page %}
+ {{ h['not_here']() }}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/flash_messages.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/flash_messages.html
new file mode 100644
index 0000000..95c5315
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/flash_messages.html
@@ -0,0 +1,14 @@
+
+
+
+
Hello
+
+
+ Flash messages:
+ {% for category, message in h.get_flashed_messages(with_categories=true) %}
+
+ {{ category }}: {{ h.literal(message) }}
+
+ {% endfor %}
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/helper_as_attribute.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/helper_as_attribute.html
new file mode 100644
index 0000000..8e22f36
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/helper_as_attribute.html
@@ -0,0 +1,5 @@
+{% extends 'page.html' %}
+
+{% block page %}
+ My lang is: {{ h.lang() }}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/helper_as_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/helper_as_item.html
new file mode 100644
index 0000000..4907461
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/helper_as_item.html
@@ -0,0 +1,5 @@
+{% extends 'page.html' %}
+
+{% block page %}
+ My lang is: {{ h['lang']() }}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/mock_json_resource_preview_template.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/mock_json_resource_preview_template.html
new file mode 100644
index 0000000..20630de
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/mock_json_resource_preview_template.html
@@ -0,0 +1,17 @@
+{% extends 'dataviewer/base.html' %}
+
+{% block page %}
+
+
+
+ {{ _('Loading...') }}
+
+
+
+{% endblock %}
+
+{% block scripts %}
+ {{ super() }}
+
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/mock_resource_preview_template.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/mock_resource_preview_template.html
new file mode 100644
index 0000000..9bea46d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/tests/mock_resource_preview_template.html
@@ -0,0 +1,17 @@
+{% extends 'dataviewer/base.html' %}
+
+{% block page %}
+
+
+
+ {{ _('Loading...') }}
+
+
+
+{% endblock %}
+
+{% block scripts %}
+ {{ super() }}
+
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/api_tokens.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/api_tokens.html
new file mode 100644
index 0000000..0aa5bc1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/api_tokens.html
@@ -0,0 +1,42 @@
+{% extends "user/read_base.html" %}
+
+{% import 'macros/form.html' as form %}
+
+{% block page_primary_action %}
+ {% if h.check_access('api_token_create', {'user': user['name']}) %}
+
+ {% endif %}
+{% endblock page_primary_action %}
+
+
+{% block primary_content_inner %}
+
+ {% block page_heading %}{{ _('API Tokens') }}{% endblock %}
+
+
+ {% block token_list %}
+ {% if tokens %}
+ {% snippet 'user/snippets/api_token_list.html', tokens=tokens, user=user %}
+ {% else %}
+
+ {{ _('You haven\'t created any API Tokens.') }}
+
+ {% endif %}
+ {% endblock token_list %}
+{% endblock primary_content_inner %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/confirm_delete.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/confirm_delete.html
new file mode 100644
index 0000000..f7d7112
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/confirm_delete.html
@@ -0,0 +1,22 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _("Confirm Delete") }}{% endblock %}
+
+{% block maintag %}
{% endblock %}
+
+{% block main_content %}
+
+
+ {% block form %}
+
{{ _('Are you sure you want to delete user - {name}?').format(name=user_dict.name) }}
+
+
+
+ {% endblock %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard.html
index 7a8ecbd..25a5eda 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard.html
@@ -1,9 +1,13 @@
-{% ckan_extends %}
+{% extends "user/edit_base.html" %}
+
+{% set user = current_user %}
+{% set org_type = h.default_group_type('organization') %}
+{% set group_type = h.default_group_type('group') %}
+{% set dataset_type = h.default_package_type() %}
-{% set user = c.userobj %}
{% block breadcrumb_content %}
-
{{ _('Dashboard') }}
+
{{ _('Dashboard') }}
{% endblock %}
{% block secondary %}{% endblock %}
@@ -11,23 +15,20 @@
{% block primary %}
{% block page_header %}
- {# CODE ADDED BY Francesco Mangiacrapa #}
- {% set hide = h.get_cookie_value('ckan_hide_header') %}
- {% if hide=='true' %}
- {# NOT SHOW 'Edit settings' #}
- {% else %}
- {% endif %}
{% endblock %}
{% if self.page_primary_action() | trim %}
@@ -36,19 +37,7 @@
{% endif %}
{% block primary_content_inner %}
-
- {% snippet 'user/snippets/followee_dropdown.html', context=c.dashboard_activity_stream_context, followees=c.followee_list %}
-
- {% block page_heading %}
- {{ _('News feed') }}
- {% endblock %}
- {{ _("Activity from items that I'm following") }}
-
- {% block activity_stream %}
- {{ c.dashboard_activity_stream }}
- {% endblock %}
-
{% endblock %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_datasets.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_datasets.html
index f2810e5..f2f8ae5 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_datasets.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_datasets.html
@@ -1,65 +1,22 @@
-{#{% extends "user/dashboard.html" %}#}
-{# CREATED BY FRANCESCO MANGIACRAPA #}
+{% extends "user/dashboard.html" %}
+{% set dataset_type = h.default_package_type() %}
-{% ckan_extends %}
-{% set user = c.userobj %}
-
-{% block breadcrumb_content %}
+{% block page_primary_action %}
+ {% if h.check_access('package_create') %}
+ {% snippet 'snippets/add_dataset.html' %}
+ {% endif %}
{% endblock %}
-{% block primary %}
-
- {% block page_header %}
- {# CODE ADDED BY Francesco Mangiacrapa #}
- {% set hide = h.get_cookie_value('ckan_hide_header') %}
- {% if hide=='true' %}
- {# NOT SHOW 'Edit settings' #}
- {% else %}
-
- {% endif %}
- {% endblock %}
-
- {% if self.page_primary_action() | trim %}
-
- {% block page_primary_action %}
- {% set hide = h.get_cookie_value('ckan_hide_header') %}
- {% if hide=='true' %}
- {# NOT SHOW 'Edit settings' #}
- {% else %}
- {% if h.check_access('package_create') %}
- {% link_for _('Add Dataset'), controller='dataset', action='new', class_="btn btn-primary", icon="plus-sign-alt" %}
- {% endif %}
- {% endif %}
- {% endblock %}
-
- {% endif %}
- {% block primary_content_inner %}
-
{{ _('My Datasets') }}
- {% if c.user_dict.datasets %}
- {% snippet 'snippets/package_list.html', packages=c.user_dict.datasets %}
- {% else %}
-
+{% block primary_content_inner %}
+
{{ h.humanize_entity_type('package', dataset_type, 'my label') or _('My Datasets') }}
+ {% if user_dict.datasets %}
+ {% snippet 'snippets/package_list.html', packages=user_dict.datasets %}
+ {% else %}
+
{{ _('You haven\'t created any datasets.') }}
{% if h.check_access('package_create') %}
- {#{% link_for _('Create one now?'), controller='dataset', action='new' %}#}
+ {% link_for _('Create one now?'), named_route=dataset_type ~ '.new' %}
{% endif %}
-
- {% endif %}
- {% endblock %}
-
+
+ {% endif %}
{% endblock %}
-
-{% block dashboard_activity_stream_context %}{% endblock %}
-
-{% block secondary %}{% endblock %}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_groups.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_groups.html
index ecbd041..e5fb24f 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_groups.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_groups.html
@@ -1,65 +1,26 @@
-{#{% extends "user/dashboard.html" %}#}
+{% extends "user/dashboard.html" %}
-{#{% extends "user/dashboard.html" %}#}
-{# CREATED BY FRANCESCO MANGIACRAPA #}
+{% set group_type = h.default_group_type('group') %}
-{% ckan_extends %}
-{% set user = c.userobj %}
-
-{% block breadcrumb_content %}
-{% endblock %}
-
-{% block primary %}
-
- {% block page_header %}
- {# CODE ADDED BY Francesco Mangiacrapa #}
- {% set hide = h.get_cookie_value('ckan_hide_header') %}
- {% if hide=='true' %}
- {# NOT SHOW 'Edit settings' #}
- {% else %}
-
- {% endif %}
- {% endblock %}
-
- {% if self.page_primary_action() | trim %}
-
- {% block page_primary_action %}
+{% block page_primary_action %}
{% if h.check_access('group_create') %}
- {% link_for _('Add Group'), controller='group', action='new', class_="btn btn-primary", icon="plus-sign-alt" %}
+ {% link_for h.humanize_entity_type('group', group_type, 'add link') or _('Add Group'), named_route=group_type ~ '.new', class_="btn btn-primary", icon="plus-square" %}
{% endif %}
{% endblock %}
-
- {% endif %}
- {% block primary_content_inner %}
-
{{ _('My Groups') }}
- {% set groups = h.groups_available(am_member=True) %}
- {% if groups %}
-
- {% snippet "group/snippets/group_list_simple.html", groups=groups %}
+
+{% block primary_content_inner %}
+
{{ h.humanize_entity_type('group', group_type, 'my label') or _('My Groups') }}
+ {% set groups = h.groups_available(am_member=True) %}
+ {% if groups %}
+
+ {% snippet "group/snippets/group_list.html", groups=groups %}
- {% else %}
+ {% else %}
- {{ _('You are not a member of any groups.') }}
+ {{ h.humanize_entity_type('group', group_type, 'you not member') or _('You are not a member of any groups.') }}
{% if h.check_access('group_create') %}
- {% link_for _('Create one now?'), controller='group', action='new' %}
+ {% link_for _('Create one now?'), named_route=group_type ~ '.new' %}
{% endif %}
{% endif %}
{% endblock %}
-
-
-{% endblock %}
-
-{% block dashboard_activity_stream_context %}{% endblock %}
-
-{% block secondary %}{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_organizations.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_organizations.html
index 1d2f8d7..187566e 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_organizations.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/dashboard_organizations.html
@@ -1,66 +1,27 @@
-{#{% extends "user/dashboard.html" %}#}
+{% extends "user/dashboard.html" %}
-{#{% extends "user/dashboard.html" %}#}
-{# CREATED BY FRANCESCO MANGIACRAPA #}
+{% set group_type = h.default_group_type('organization') %}
-{% ckan_extends %}
-{% set user = c.userobj %}
-
-{% block breadcrumb_content %}
-{% endblock %}
-
-{% block primary %}
-
- {% block page_header %}
- {# CODE ADDED BY Francesco Mangiacrapa #}
- {% set hide = h.get_cookie_value('ckan_hide_header') %}
- {% if hide=='true' %}
- {# NOT SHOW 'Edit settings' #}
- {% else %}
-
- {% endif %}
- {% endblock %}
-
- {% if self.page_primary_action() | trim %}
-
{% block page_primary_action %}
{% if h.check_access('organization_create') %}
- {% link_for _('Add Organization'), controller='organization', action='new', class_="btn btn-primary", icon="plus-sign-alt" %}
+ {% link_for h.humanize_entity_type('organization', group_type, 'add link') or _('Add Organization'), named_route=group_type ~ '.new', class_="btn btn-primary", icon="plus-square" %}
{% endif %}
{% endblock %}
-
- {% endif %}
{% block primary_content_inner %}
-
{{ _('My Organizations') }}
- {% set organizations = h.organizations_available() %}
+
{{ h.humanize_entity_type('organization', group_type, 'my label') or _('My Organizations') }}
+ {% set organizations = h.organizations_available(permission='manage_group',
+ include_dataset_count=True) %}
{% if organizations %}
- {% snippet "organization/snippets/organization_list.html", organizations=organizations %}
+ {% snippet "organization/snippets/organization_list.html", organizations=organizations, show_capacity=True %}
{% else %}
- {{ _('You are not a member of any organizations.') }}
+ {{ h.humanize_entity_type('organization', group_type, 'you not member') or _('You are not a member of any organizations.') }}
{% if h.check_access('organization_create') %}
- {% link_for _('Create one now?'), controller='organization', action='new' %}
+ {% link_for _('Create one now?'), named_route=group_type ~ '.new' %}
{% endif %}
{% endif %}
{% endblock %}
-
-{% endblock %}
-
-{% block dashboard_activity_stream_context %}{% endblock %}
-
-{% block secondary %}{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit.html
new file mode 100644
index 0000000..5771589
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit.html
@@ -0,0 +1,25 @@
+{% extends 'user/edit_base.html' %}
+
+{% block actions_content %}{% endblock %}
+
+{% block breadcrumb_content %}
+
{{ _('Users') }}
+
{{ user_dict.display_name }}
+
{{ _('Manage') }}
+{% endblock %}
+
+{% block primary_content_inner %}
+ {{ form | safe }}
+{% endblock %}
+
+{% block secondary_content %}
+
+
+ {% trans %}
+ Your profile lets other CKAN users know about who you are and what you
+ do.
+ {% endtrans %}
+
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit_base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit_base.html
new file mode 100644
index 0000000..76c6f45
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit_base.html
@@ -0,0 +1,11 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _('Manage') }} {{ g.template_title_delimiter }} {{ user_dict.display_name }} {{ g.template_title_delimiter }} {{ _('Users') }}{% endblock %}
+
+{% block primary_content %}
+
+
+ {% block primary_content_inner %}{% endblock %}
+
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit_user_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit_user_form.html
new file mode 100644
index 0000000..1241728
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/edit_user_form.html
@@ -0,0 +1,96 @@
+{% import 'macros/form.html' as form %}
+
+{% block form %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/followers.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/followers.html
new file mode 100644
index 0000000..d0c1b9a
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/followers.html
@@ -0,0 +1,12 @@
+{% extends "user/read.html" %}
+
+{% block subtitle %}{{ _('Followers') }}{% endblock %}
+
+{% block primary_content_inner %}
+
+ {% block page_heading %}{{ _('Followers') }}{% endblock %}
+
+ {% block followers_list %}
+ {% snippet "user/snippets/followers.html", followers=followers %}
+ {% endblock %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/list.html
new file mode 100644
index 0000000..08296fb
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/list.html
@@ -0,0 +1,33 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _('All Users') }}{% endblock %}
+
+{% block breadcrumb_content %}
+
{{ h.nav_link(_('Users'), named_route='user.index') }}
+{% endblock %}
+
+{% block primary_content %}
+
+
+
+ {% block page_heading %}{{ _('Users') }}{% endblock %}
+
+ {% block users_list %}
+
+ {% block users_list_inner %}
+ {% for user in page.items %}
+ {{ h.linked_user(user['name'], maxlength=20) }}
+ {% endfor %}
+ {% endblock %}
+
+ {% endblock %}
+
+{% endblock %}
+
+{% block secondary_content %}
+ {% snippet 'user/snippets/user_search.html', q=q %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/login.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/login.html
new file mode 100644
index 0000000..432505e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/login.html
@@ -0,0 +1,54 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _('Login') }}{% endblock %}
+
+{% block breadcrumb_content %}
+
{{ h.nav_link(_('Login'), named_route='user.login') }}
+{% endblock %}
+
+{% block primary_content %}
+
+
+
{% block page_heading %}{{ _('Login') }}{% endblock %}
+ {% block form %}
+ {% snippet "user/snippets/login_form.html", error_summary=error_summary %}
+ {% endblock %}
+
+
+{% endblock %}
+
+{% block secondary_content %}
+ {% if h.check_access('user_create') %}
+ {% block help_register %}
+
+ {% block help_register_inner %}
+ {{ _('Need an Account?') }}
+
+
{% trans %}Then sign right up, it only takes a minute.{% endtrans %}
+
+ {% block help_register_button %}
+ {{ _('Create an Account') }}
+ {% endblock %}
+
+
+ {% endblock %}
+ {% endif %}
+
+ {% block help_forgotten %}
+
+ {% block help_forgotten_inner %}
+ {{ _('Forgotten your password?') }}
+
+
{% trans %}No problem, use our password recovery form to reset it.{% endtrans %}
+
+ {% block help_forgotten_button %}
+ {{ _('Forgot your password?') }}
+ {% endblock %}
+
+
+ {% endblock %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/logout.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/logout.html
new file mode 100644
index 0000000..9d1460a
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/logout.html
@@ -0,0 +1,14 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _('Logged Out') }}{% endblock %}
+
+{% block primary_content %}
+
+
+
+ {% block page_heading %}{{ _('Logged Out') }}{% endblock %}
+
+
{% trans %}You are now logged out.{% endtrans %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/logout_first.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/logout_first.html
new file mode 100644
index 0000000..1aacd6f
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/logout_first.html
@@ -0,0 +1,26 @@
+{% import 'macros/form.html' as form %}
+{% extends "user/login.html" %}
+
+{% set logout_url = h.url_for('user.logout') %}
+
+{% block actions %}{% endblock %}
+
+{% block form %}
+
{{ _("You're already logged in as {user}.").format(user=current_user.name) }}
{{ _('Logout') }} ?
+ {{ form.input('login', label=_('Username'), id='field-login', value="", error="", classes=["control-full"], attrs={"disabled": "disabled"}) }}
+ {{ form.input('password', label=_('Password'), id='field-password', type="password", value="", error="", classes=["control-full"], attrs={"disabled": "disabled"}) }}
+ {{ form.checkbox('remember', label=_('Remember me'), id='field-remember', checked=true, attrs={"disabled": "disabled"}) }}
+
+ {{ _('Log in') }}
+
+{% endblock %}
+
+{% block secondary_content %}
+
+ {{ _("You're already logged in") }}
+
+
{{ _("You need to log out before you can log in with another account.") }}
+
{{ _("Log out now") }}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/new.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/new.html
new file mode 100644
index 0000000..62d8de9
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/new.html
@@ -0,0 +1,33 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _('Register') }}{% endblock %}
+
+{% block breadcrumb_content %}
+
{{ h.nav_link(_('Registration'), named_route='user.register') }}
+{% endblock %}
+
+{% block primary_content %}
+
+
+ {% block primary_content_inner %}
+
+ {% block page_heading %}{{ _('Register for an Account') }}{% endblock %}
+
+ {{ form | safe }}
+ {% endblock %}
+
+
+{% endblock %}
+
+{% block secondary_content %}
+ {% block help %}
+
+ {% block help_inner %}
+ {{ _('Why Sign Up?') }}
+
+
{% trans %}Create datasets, groups and other exciting things{% endtrans %}
+
+ {% endblock %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/new_user_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/new_user_form.html
new file mode 100644
index 0000000..f28304a
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/new_user_form.html
@@ -0,0 +1,41 @@
+{% import "macros/form.html" as form %}
+
+{% block form %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/perform_reset.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/perform_reset.html
new file mode 100644
index 0000000..a7f9a27
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/perform_reset.html
@@ -0,0 +1,51 @@
+{% extends "page.html" %}
+{% import "macros/form.html" as form %}
+
+{% block subtitle %}{{ _('Reset Your Password') }}{% endblock %}
+
+{% block breadcrumb_content %}
+
{{ _('Password Reset') }}
+{% endblock %}
+
+{% block primary_content %}
+
+ {% block primary_content_inner %}
+
+
+ {% block page_heading %}{{ _('Reset Your Password') }}{% endblock %}
+
+ {% block form %}
+
+ {% endblock %}
+
+{% endblock %}
+
+{% block secondary_content %}
+ {% block help %}
+
+ {% block help_inner %}
+ {{ _('How does this work?') }}
+
+
{% trans %}Simply enter a new password and we'll update your account{% endtrans %}
+
+ {% endblock %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/read.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/read.html
new file mode 100644
index 0000000..b5203e1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/read.html
@@ -0,0 +1,32 @@
+{% extends "user/read_base.html" %}
+
+{% block page_primary_action %}
+ {% if h.check_access('package_create') and user.datasets %}
+ {% snippet 'snippets/add_dataset.html' %}
+ {% endif %}
+{% endblock %}
+
+{% block primary_content_inner %}
+
+ {% block page_heading %}{{ _('Datasets') }}{% endblock %}
+
+ {% block package_list %}
+ {% if user.datasets %}
+ {% snippet 'snippets/package_list.html', packages=user.datasets %}
+ {% else %}
+
+ {% if is_myself %}
+
+ {{ _('You haven\'t created any datasets.') }}
+ {% if h.check_access('package_create') %}
+ {% link_for _('Create one now?'), named_route='dataset.new' %}
+ {% endif %}
+
+ {% else %}
+
+ {{ _('User hasn\'t created any datasets.') }}
+
+ {% endif %}
+ {% endif %}
+ {% endblock %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/read_base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/read_base.html
new file mode 100644
index 0000000..b764d74
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/read_base.html
@@ -0,0 +1,110 @@
+{% extends "page.html" %}
+
+{% set user = user_dict %}
+{% set dataset_type = h.default_package_type() %}
+
+{% block subtitle %}{{ user.display_name }} {{ g.template_title_delimiter }} {{ _('Users') }}{% endblock %}
+
+{% block breadcrumb_content %}
+ {{ h.build_nav('user.index', _('Users')) }}
+ {{ h.build_nav('user.read', user.display_name|truncate(35), id=user.name) }}
+{% endblock %}
+
+{% block content_action %}
+ {% if h.check_access('user_update', user) %}
+ {% link_for _('Manage'), named_route='user.edit', id=user.name, class_='btn btn-default', icon='wrench' %}
+ {% endif %}
+{% endblock %}
+
+{% block content_primary_nav %}
+ {{ h.build_nav_icon('user.read', h.humanize_entity_type('package', dataset_type, 'content tab') or _('Datasets'), id=user.name, icon='sitemap') }}
+ {% if h.check_access('api_token_list', {'user': user['name']}) %}
+ {{ h.build_nav_icon('user.api_tokens', _('API Tokens'), id=user.name, icon='key') }}
+ {% endif %}
+{% endblock %}
+
+{% block secondary_content %}
+
+
+ {% block secondary_content_inner %}
+ {% block user_image %}
+ {{ h.user_image(user.id, size=270) }}
+ {% endblock %}
+ {% block user_heading %}
+ {{ user.display_name }}
+ {% endblock %}
+ {% block user_about %}
+ {% if about_formatted %}
+ {{ about_formatted }}
+ {% else %}
+
+ {% if is_myself %}
+ {% trans %}You have not provided a biography.{% endtrans %}
+ {% else %}
+ {% trans %}This user has no biography.{% endtrans %}
+ {% endif %}
+
+ {% endif %}
+ {% endblock %}
+ {% block user_nums %}
+
+
+ {{ _('Followers') }}
+ {{ h.SI_number_span(user.num_followers) }}
+
+
+ {{ h.humanize_entity_type('package', dataset_type, 'facet label') or _('Datasets') }}
+ {{ h.SI_number_span(user.number_created_packages) }}
+
+
+ {{ h.follow_button('user', user.id) }}
+
+ {% endblock %}
+ {% endif %}
+ {% block user_info %}
+
+
+ {% if user.name.startswith('http://') or user.name.startswith('https://') %}
+ {{ _('Open ID') }}
+ {{ user.name|urlize(25) }}{# Be great if this just showed the domain #}
+ {% else %}
+ {{ _('Username') }}
+ {{ user.name }}
+ {% endif %}
+
+ {% if is_myself %}
+
+ {{ _('Email') }} {{ _('Private') }}
+ {{ user.email }}
+
+ {% endif %}
+
+ {{ _('Member Since') }}
+ {{ h.render_datetime(user.created) }}
+
+ {% if is_sysadmin %}
+
+ {{_('Last Active') }}
+ {{ h.time_ago_from_timestamp(user.last_active) }}
+
+ {% endif %}
+
+ {{ _('State') }}
+ {{ _(user.state) }}
+
+ {% if is_myself %}
+
+ {{ _('API Key') }} {{ _('Private') }}
+ {{ user.apikey }}
+ {% endif %}
+
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/request_reset.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/request_reset.html
new file mode 100644
index 0000000..329265d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/request_reset.html
@@ -0,0 +1,46 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ _('Reset your password') }}{% endblock %}
+
+{% block breadcrumb_content %}
+
{% link_for _('Password Reset'), named_route='user.request_reset' %}
+{% endblock %}
+
+{% block primary_content %}
+
+
+ {% block primary_content_inner %}
+
{{ _('Reset your password') }}
+ {% block form %}
+
+ {% endblock %}
+ {% endblock %}
+
+{% endblock %}
+
+{% block secondary_content %}
+ {% block help %}
+
+ {% block help_inner %}
+ {{ _('How does this work?') }}
+
+
{% trans %}Enter your email address or username into the box and we
+ will send you an email with a link to enter a new password.
+ {% endtrans %}
+
+ {% endblock %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/api_token_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/api_token_list.html
new file mode 100644
index 0000000..7e4d050
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/api_token_list.html
@@ -0,0 +1,52 @@
+{#
+ Template variables:
+
+ user - currently viewed user
+ tokens - list of ApiToken entities
+
+ #}
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/followers.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/followers.html
new file mode 100644
index 0000000..742faec
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/followers.html
@@ -0,0 +1,10 @@
+{% if followers %}
+
+ {% for follower in followers %}
+ {# HACK: (for group/admins) follower can be the id of the user or a user_dict #}
+ {{ h.linked_user(follower.name or follower, maxlength=20) }}
+ {% endfor %}
+
+{% else %}
+
{{ _('No followers') }}
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/login_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/login_form.html
new file mode 100644
index 0000000..7426031
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/login_form.html
@@ -0,0 +1,32 @@
+{#
+Renders the login form.
+
+action - The url that the form should be submitted to.
+error_summary - A tuple/list of form errors.
+
+Example:
+
+ {% snippet "user/snippets/login_form.html", action=g.login_handler, error_summary=error_summary %}
+
+#}
+{% import 'macros/form.html' as form %}
+
+{% set username_error = true if error_summary %}
+{% set password_error = true if error_summary %}
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/placeholder.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/placeholder.html
new file mode 100644
index 0000000..c881d6b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/placeholder.html
@@ -0,0 +1,2 @@
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/recaptcha.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/recaptcha.html
new file mode 100644
index 0000000..0f0f7e7
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/recaptcha.html
@@ -0,0 +1,18 @@
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/user_search.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/user_search.html
new file mode 100644
index 0000000..c9d268b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/user/snippets/user_search.html
@@ -0,0 +1,13 @@
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/.gitignore b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/.gitignore
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/base.html
new file mode 100644
index 0000000..85848b7
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/base.html
@@ -0,0 +1,121 @@
+{# Allows the DOCTYPE to be set on a page by page basis #}
+{%- block doctype %}{% endblock -%}
+
+{# Allows custom attributes to be added to the tag #}
+{%- block htmltag -%}
+{% set lang = h.lang() %}
+
+
+{%- endblock -%}
+
+ {# Allows custom attributes to be added to the tag #}
+
+ {#
+ Add custom meta tags to the page. Call super() to get the default tags
+ such as charset, viewport and generator.
+
+ Example:
+
+ {% block meta %}
+ {{ super() }}
+
+ {%- block title -%}
+ {%- block subtitle %}{% endblock -%}
+ {%- if self.subtitle()|trim %} {{ g.template_title_delimiter }} {% endif -%}
+ {{ g.site_title }}
+ {%- endblock -%}
+
+
+ {#
+ The links block allows you to add additonal content before the stylesheets
+ such as rss feeds and favicons in the same way as the meta block.
+ #}
+ {% block links -%}
+
+
+ {#
+ The page block allows you to add content to the page. Most of the time it is
+ recommended that you extend one of the page.html templates in order to get
+ the site header and footer. If you need a clean page then this is the
+ block to use.
+
+ Example:
+
+ {% block page %}
+ Some other page content
+ {% endblock %}
+ #}
+ {%- block page %}{% endblock -%}
+
+ {#
+ DO NOT USE THIS BLOCK FOR ADDING SCRIPTS
+ Scripts should be loaded by the {% assets %} tag except in very special
+ circumstances
+ #}
+ {%- block scripts %}
+ {% endblock -%}
+
+ {# render all assets included in scripts block and everywhere else #}
+ {# make sure there are no calls to `asset` tag after this point #}
+ {{ h.render_assets('style') }}
+ {{ h.render_assets('script') }}
+
+
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/footer.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/footer.html
new file mode 100644
index 0000000..f149c21
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/footer.html
@@ -0,0 +1,20 @@
+{% block header_hide_container_content %}
+{% set hideParameter = h.get_request_param('hh', None) %}
+{% set hide = h.get_cookie_value('ckan_hide_header') %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/activity_stream.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/activity_stream.html
new file mode 100644
index 0000000..0a7484e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/activity_stream.html
@@ -0,0 +1,15 @@
+{% extends "group/read_base.html" %}
+
+{% block subtitle %}{{ _('Activity Stream') }} {{ g.template_title_delimiter }} {{ super() }}{% endblock %}
+
+{% block primary_content_inner %}
+
+ {% if activity_types is defined %}
+ {% snippet 'snippets/activity_type_selector.html', id=id, activity_type=activity_type, activity_types=activity_types, blueprint='activity.group_activity' %}
+ {% endif %}
+
+ {% snippet 'snippets/stream.html', activity_stream=activity_stream, id=id, object_type='group' %}
+
+ {% snippet 'snippets/pagination.html', newer_activities_url=newer_activities_url, older_activities_url=older_activities_url %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/changes.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/changes.html
new file mode 100644
index 0000000..00b7307
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/changes.html
@@ -0,0 +1,64 @@
+{% extends "group/read_base.html" %}
+
+{% block subtitle %}{{ group_dict.name }} {{ g.template_title_delimiter }} Changes {{ g.template_title_delimiter }} {{ super() }}{% endblock %}
+
+{% block breadcrumb_content_selected %}{% endblock %}
+
+{% block breadcrumb_content %}
+ {{ super() }}
+ {% link_for _('Changes'), named_route='activity.group_activity', id=group_dict.name %}
+ {% link_for activity_diffs[0].activities[1].id|truncate(30), named_route='activity.group_changes', id=activity_diffs[0].activities[1].id %}
+{% endblock %}
+
+{% block primary %}
+
+
+ {% block group_changes_header %}
+
{{ _('Changes') }}
+ {% endblock %}
+
+ {% set select_list1 = h.activity_list_select(group_activity_list, activity_diffs[-1].activities[0].id) %}
+ {% set select_list2 = h.activity_list_select(group_activity_list, activity_diffs[0].activities[1].id) %}
+
+
+
+
+ {# iterate through the list of activity diffs #}
+
+ {% for i in range(activity_diffs|length) %}
+ {% snippet "group/snippets/item_group.html", activity_diff=activity_diffs[i], group_dict=group_dict %}
+
+ {# TODO: display metadata for more than most recent change #}
+ {% if i == 0 %}
+ {# button to show JSON metadata diff for the most recent change - not shown by default #}
+
+
+ {% endif %}
+
+
+ {% endfor %}
+
+{% endblock %}
+
+{% block secondary %}{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/read_base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/read_base.html
new file mode 100644
index 0000000..03ce862
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/read_base.html
@@ -0,0 +1,6 @@
+{% ckan_extends %}
+
+{% block content_primary_nav %}
+ {{ super() }}
+ {{ h.build_nav_icon('activity.group_activity', _('Activity Stream'), id=group_dict.name, offset=0, icon='clock') }}
+{% endblock content_primary_nav %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/snippets/item_group.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/snippets/item_group.html
new file mode 100644
index 0000000..915fabe
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group/snippets/item_group.html
@@ -0,0 +1,10 @@
+{{ gettext('On %(timestamp)s, %(username)s:', timestamp=h.render_datetime(activity_diff.activities[1].timestamp, with_hours=True, with_seconds=True), username=h.linked_user(activity_diff.activities[1].user_id)) }}
+
+{% set changes = h.compare_group_dicts(activity_diff.activities[0].data.group, activity_diff.activities[1].data.group, activity_diff.activities[0].id) %}
+
+ {% for change in changes %}
+ {% snippet "snippets/group_changes/{}.html".format(
+ change.type), change=change %}
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/read_base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/read_base.html
new file mode 100644
index 0000000..23014cd
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/read_base.html
@@ -0,0 +1,36 @@
+{% extends "page.html" %}
+
+{% block subtitle %}{{ c.group_dict.display_name }} - {{ _('Groups') }}{% endblock %}
+
+{% block breadcrumb_content %}
+ {% link_for _('Groups'), controller='group', action='index' %}
+ {% link_for c.group_dict.display_name|truncate(35), controller='group', action='read', id=c.group_dict.name %}
+{% endblock %}
+
+{% block content_action %}
+ {% if h.check_access('group_update', {'id': c.group_dict.id}) %}
+ {% link_for _('Manage'), controller='group', action='edit', id=c.group_dict.name, class_='btn', icon='wrench' %}
+ {% endif %}
+{% endblock %}
+
+{% block content_primary_nav %}
+ {{ h.build_nav_icon('group.read', _('Datasets'), id=c.group_dict.name) }}
+
+ {% if h.check_access('group_update', {'id': c.group_dict.id}) %}
+ {% set role_for_user = h.d4science_theme_get_user_role_for_group_or_org(c.group_dict.id, c.userobj.name) %}
+ {% if (role_for_user and role_for_user == 'admin') or c.userobj.sysadmin %}
+
+ {#{{ h.build_nav_icon('group.activity', _('Activity Stream'), id=c.group_dict.name, offset=0) }}#}
+ {% endif %}
+ {% endif %}
+ {{ h.build_nav_icon('group.about', _('About'), id=c.group_dict.name) }}
+{% endblock %}
+
+{% block secondary_content %}
+ {% snippet "group/snippets/info.html", group=c.group_dict, show_nums=true %}
+{% endblock %}
+
+{% block links %}
+ {{ super() }}
+ {% include "group/snippets/feeds.html" %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/snippets/group_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/snippets/group_form.html
new file mode 100644
index 0000000..fffd192
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/snippets/group_form.html
@@ -0,0 +1,108 @@
+{% import 'macros/form.html' as form %}
+
+{# SCRIPT ADDED BY FRACESCO MANGIACRAPA, SEE 20425 #}
+
+
+
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/snippets/group_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/snippets/group_item.html
new file mode 100644
index 0000000..3f9e5ad
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/group2.6/snippets/group_item.html
@@ -0,0 +1,65 @@
+{#
+Renders a media item for a group. This should be used in a list.
+
+group - A group dict.
+
+Example:
+
+
+#}
+
+
+{# Updated By Francesco Mangiacrapa, related to breadcrumb for Group #}
+{% set type = group.type or 'group' %}
+{% set url = h.url_for(type ~ '_read', action='read', id=group.name) %}
+{% block item %}
+
+ {% block breadcrumb %}
+
+ {% set parents = h.d4science_theme_get_parents_for_group(group.name) %}
+ {% for parent in parents recursive %}
+
+ {% link_for parent.title, controller='group', action='read', id=parent.name %}
+ {% if loop.last == False %}
+ {% endif %}
+
+ {% endfor %}
+
+ {% endblock %}
+ {% block item_inner %}
+ {% block image %}
+ {{ h.d4science_theme_markdown_extract_html(group.description, extract_length=80, allow_html=True) }}
+ {% endif %}
+ {% endblock %}
+ {% block datasets %}
+ {% if group.package_count %}
+ {{ ungettext('{num} Dataset', '{num} Datasets', group.package_count).format(num=group.package_count) }}
+ {% elif group.package_count == 0 %}
+
+ {% endif %}
+ {% endblock %}
+ {% if group.user_member %}
+ {# Added By Francesco Mangiacrapa, related to Task #5196 #}
+ {% set username_id = c.userobj.id %}
+ {% set role_for_user = h.d4science_theme_get_user_role_for_group_or_org(group.id, username_id) %}
+ {% if role_for_user and role_for_user == 'admin' %}
+
+{% endblock %}
+{% if position is divisibleby 3 %}
+
+
Group Hierarchy
+
+ {% snippet 'group/snippets/group_tree.html', top_nodes=top_nodes %}
+
+
+
+
+ {% block promoted %}
+ {% snippet 'home/snippets/promoted.html' %}
+ {% endblock %}
+
+
+
+
+
+
+ {% block search %}
+ {% snippet 'home/snippets/search.html' %}
+ {% endblock %}
+
+
+ {% block stats %}
+ {% snippet 'home/snippets/stats.html' %}
+ {% endblock %}
+
+
+
+
+
+
+ {% block popular_formats %}
+ {% snippet 'home/snippets/popular_formats.html' %}
+ {% endblock %}
+
+
+ {% block popular_tags %}
+ {% snippet 'home/snippets/popular_tags.html' %}
+ {% endblock %}
+
+ {% block recent_activity %}
+
+ {% if c.userobj and c.userobj.sysadmin %}
+
+
+
+ {{ h.recently_changed_packages_activity_stream(limit=4) }}
+
+
+ {%endif%}
+ {% endblock %}
+
+
+ {% snippet 'snippets/organization_item.html', organization=organization, truncate=50, truncate_title=35 %}
+
+{% endfor %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_formats.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_formats.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_formats.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_formats.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_groups.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_groups.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_groups.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_groups.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_metadatatypes.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_metadatatypes.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_metadatatypes.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_metadatatypes.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_tags.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_tags.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/home/snippets/popular_tags.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/popular_tags.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/promoted.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/promoted.html
new file mode 100644
index 0000000..ab4dcd3
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/home/snippets/promoted.html
@@ -0,0 +1,27 @@
+{% set intro = g.site_intro_text %}
+
+
+
+ {% if intro %}
+ {{ h.render_markdown(intro) }}
+ {% else %}
+ {% block home_image %}
+
+ {% endblock %}
+
+
Welcome to the {{g.site_title}}!
+
+Here you will find data and other resources hosted by the D4Science.org infrastructure.
+
The catalogue contains a wealth of resources resulting from several activities, projects and communities including BlueBRIDGE (www.bluebridge-vres.eu ), i-Marine (www.i-marine.eu ), SoBigData.eu (www.sobigdata.eu ), and (FAO www.fao.org ).
+
+
All the products are accompanied with rich descriptions capturing general attributes, e.g. title and creator(s), as well as usage policies and licences.
+
+
+
+
{{ _('{0} statistics').format(g.site_title) }}
+
+
+
{% link_for _('Organizations'), controller='organization', action='index' %}
+ {% link_for c.group_dict.display_name|truncate(35), controller='organization', action='read', id=c.group_dict.name %}
+{% endblock %}
+
+{% block content_action %}
+ {% if h.check_access('organization_update', {'id': c.group_dict.id}) %}
+ {# CODE ADDED BY Francesco Mangiacrapa #}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Manage' #}
+ {% else %}
+ {% link_for _('Manage'), controller='organization', action='edit', id=c.group_dict.name, class_='btn', icon='wrench' %}
+ {% endif %}
+ {% endif %}
+{% endblock %}
+
+{% block secondary_content %}
+ {% snippet 'snippets/organization.html', organization=c.group_dict, show_nums=true %}
+ {# ADDED BY FRANCESCO MANGIACRAPA SEE #12651 #}
+ {% set alternative_url = h.url_for(controller='organization', action='read', id=c.group_dict.name) %}
+ {% snippet 'spatial/snippets/spatial_query.html', alternative_url=alternative_url %}
+ {# END #}
+ {% block organization_facets %}{% endblock %}
+{% endblock %}
+
+{% block links %}
+ {{ super() }}
+ {% include "organization/snippets/feeds.html" %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/organization/snippets/organization_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/organization/snippets/organization_item.html
new file mode 100644
index 0000000..2f1da6e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/organization/snippets/organization_item.html
@@ -0,0 +1,44 @@
+{#
+Renders a media item for a organization. This should be used in a list.
+
+organization - A organization dict.
+
+Example:
+
+
+#}
+{% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
+{% block item %}
+
+ {% block item_inner %}
+ {% block image %}
+ {{ h.markdown_extract(organization.description, extract_length=80) }}
+ {% endif %}
+ {% endblock %}
+ {% block datasets %}
+ {% if organization.package_count %}
+ {{ ungettext('{num} Dataset', '{num} Datasets', organization.package_count).format(num=organization.package_count) }}
+ {% endif %}
+ {% endblock %}
+ {% block link %}
+
+ {{ _('View {organization_name}').format(organization_name=organization.display_name) }}
+
+ {% endblock %}
+ {% endblock %}
+
+{% endblock %}
+{% if position is divisibleby 3 %}
+ {% link_for _('Organizations'), controller='organization', action='index' %}
+{% endblock %}
+
+{% block page_header %}{% endblock %}
+
+{% block page_primary_action %}
+ {% if h.check_access('organization_create') %}
+ {% link_for _('Add Organization'), controller='organization', action='new', class_='btn btn-primary', icon='plus-sign-alt' %}
+ {% endif %}
+{% endblock %}
+
+{% block primary_content_inner %}
+ {% block page_heading %}{{ _('Organizations') }}{% endblock %}
+ {% block organizations_search_form %}
+ {% snippet 'snippets/search_form.html', form_id='organization-search-form', type='organization', query=c.q, sorting_selected=c.sort_by_selected, count=c.page.item_count, placeholder=_('Search organizations...'), show_empty=request.args, no_bottom_border=true if c.page.items %}
+ {% endblock %}
+ {% block organizations_list %}
+ {% if c.page.items or request.args %}
+ {% if c.page.items %}
+ {% snippet "organization/snippets/organization_list.html", organizations=c.page.items %}
+ {% endif %}
+ {% else %}
+
+ {{ _('There are currently no organizations for this site') }}.
+ {% if h.check_access('organization_create') %}
+ {% link_for _('How about creating one?'), controller='organization', action='new' %}.
+ {% endif %}
+
+ {% endif %}
+ {% endblock %}
+ {% block page_pagination %}
+ {{ c.page.pager(q=c.q or '', sort=c.sort_by_selected or '') }}
+ {% endblock %}
+{% endblock %}
+
+{% block secondary_content %}
+ {% snippet "organization/snippets/helper.html" %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/organization_vre/read.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/organization_vre/read.html
new file mode 100644
index 0000000..60ae76d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/organization_vre/read.html
@@ -0,0 +1,61 @@
+{% extends "organization/read_base.html" %}
+
+{% block page_primary_action %}
+ {#Added by Francesco Mangiacrapa; see: 8964 #}
+
+
+ {% if h.check_access('package_create', {'owner_org': c.group_dict.id}) %}
+ {# CODE ADDED BY Francesco Mangiacrapa #}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Manage' #}
+ {% else %}
+ {% link_for _('Add Dataset'), controller='dataset', action='new', group=c.group_dict.id, class_='btn btn-primary', icon='plus-sign-alt' %}
+ {% endif %}
+ {% endif %}
+{% endblock %}
+
+{% block primary_content_inner %}
+ {% block groups_search_form %}
+ {% set facets = {
+ 'fields': c.fields_grouped,
+ 'search': c.search_facets,
+ 'titles': c.facet_titles,
+ 'translated_fields': c.translated_fields,
+ 'remove_field': c.remove_field }
+ %}
+ {% set sorting = [
+ (_('Relevance'), 'score desc, metadata_modified desc'),
+ (_('Name Ascending'), 'title_string asc'),
+ (_('Name Descending'), 'title_string desc'),
+ (_('Last Modified'), 'metadata_modified desc'),
+ (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false) ]
+ %}
+ {% snippet 'snippets/search_form.html', form_id='organization-datasets-search-form', type='dataset', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, placeholder=_('Search datasets...'), show_empty=request.args, fields=c.fields %}
+ {% endblock %}
+ {% block packages_list %}
+ {% if c.page.items %}
+ {{ h.snippet('snippets/package_list.html', packages=c.page.items) }}
+ {% endif %}
+ {% endblock %}
+ {% block page_pagination %}
+ {{ c.page.pager(q=c.q) }}
+ {% endblock %}
+{% endblock %}
+
+{% block organization_facets %}
+ {% for facet in c.facet_titles %}
+ {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, extras={'id':c.group_dict.id}) }}
+ {% endfor %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/base.html
new file mode 100644
index 0000000..f48121e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/base.html
@@ -0,0 +1,24 @@
+{% extends "page.html" %}
+
+{% set pkg = c.pkg_dict or pkg_dict %}
+
+{% block breadcrumb_content_selected %} class="active"{% endblock %}
+
+{% block subtitle %}{{ _('Datasets') }}{% endblock %}
+
+{% block breadcrumb_content %}
+ {% if pkg %}
+ {% set dataset = pkg.title or pkg.name %}
+ {% if pkg.organization %}
+ {% set organization = pkg.organization.title or pkg.organization.name %}
+ {% link_for _('Organizations'), controller='organization', action='index' %}
+ {% link_for organization|truncate(30), controller='organization', action='read', id=pkg.organization.name %}
+ {% else %}
+ {% link_for _('Datasets'), controller='dataset', action='search' %}
+ {% endif %}
+ {% link_for dataset|truncate(30), controller='dataset', action='read', id=pkg.name %}
+ {% else %}
+ {% link_for _('Datasets'), controller='dataset', action='search' %}
+ {{ _('Create Dataset') }} {{ _('Groups') }}
+ {% if c.group_dropdown %}
+ {# ADDED BY FRANCESCO MANGIACRAPA, related to TASK #5196 #}
+ {% if c.userobj %}
+ {% set username_id = c.userobj.id %}
+ {% set whereAdmin = [] %}
+ {% for option in c.group_dropdown %}
+ {% set role_for_user = h.d4science_theme_get_user_role_for_group_or_org(option[0], username_id) %}
+ {% if role_for_user and role_for_user == 'admin' %}
+ {% do whereAdmin.append(option) %}
+ {% else %}
+ {# SYSADMIN #}
+ {% if c.userobj.sysadmin %}
+ {% do whereAdmin.append(option) %}
+ {% endif %}
+ {% endif %}
+ {% endfor %}
+ {% if whereAdmin %}
+
+ {% endif %}
+ {% endif %}
+ {% endif %}
+
+
+ {% if c.pkg_dict.groups %}
+
+ {% else %}
+ {{ _('There are no groups associated with this dataset') }}
+ {% endif %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/read.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/read.html
new file mode 100644
index 0000000..a851298
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/read.html
@@ -0,0 +1,59 @@
+{% ckan_extends %}
+{% set dataset_extent = h.get_pkg_dict_extra(c.pkg_dict, 'spatial', '') %}
+{% set d4science_cms_obj_placeholders = h.d4science_get_content_moderator_system_placeholder() %}
+{% set moderation_item_status = h.get_pkg_dict_extra(c.pkg_dict,d4science_cms_obj_placeholders.item_status,'') %}
+
+{% block primary_content_inner %}
+{% if moderation_item_status %}
+
+ {{ moderation_item_status }}
+
+{% endif %}
+ {% block package_description %}
+ {% if pkg.private %}
+
+
+ {% endif %}
+
+ {% block page_heading %}
+ {{ pkg.title or pkg.name }}
+ {% if pkg.state.startswith('draft') %}
+ [{{ _('Draft') }}]
+ {% endif %}
+ {% if pkg.state == 'deleted' %}
+ [{{ _('Deleted') }}]
+ {% endif %}
+ {% endblock %}
+
+ {% block package_notes %}
+ {% if pkg.notes %}
+
+ {{ h.render_markdown(pkg.notes) }}
+
+ {% endif %}
+ {% endblock %}
+ {# FIXME why is this here? seems wrong #}
+
+ {% endblock %}
+
+
+ {% if dataset_extent %}
+ {% snippet "spatial/snippets/dataset_map.html", extent=dataset_extent %}
+ {% endif %}
+
+ {% block package_tags %}
+ {{_('Tags')}}
+ {% snippet "package/snippets/tags.html", tags=pkg.tags %}
+ {% endblock %}
+
+ {% block package_resources %}
+ {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources %}
+ {% endblock %}
+
+ {% block package_additional_info %}
+ {% snippet "package/snippets/additional_info.html", pkg_dict=pkg %}
+ {% endblock %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/read_base.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/read_base.html
new file mode 100644
index 0000000..a723063
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/read_base.html
@@ -0,0 +1,116 @@
+{# CODE UPDATED BY Francesco Mangiacrapa
+ USE TO DEBUG
+ {% print "role for user: "+role_for_user %}
+ {% print "role for user:" +role_for_user %}
+ {% print "username_id = "+username_id %}
+ {% print "pkg_creator_user_id= "+pkg_creator_user_id %}
+#}
+
+
+{% extends "package/base.html" %}
+
+{% block subtitle %}{{ pkg.title or pkg.name }} - {{ super() }}{% endblock %}
+
+{% block head_extras -%}
+ {{ super() }}
+ {% set description = h.d4science_theme_markdown_extract_html(pkg.notes, extract_length=200)|forceescape %}
+
+
+ {% set timestamp = h.render_datetime(c.revision_date, with_hours=True) %}
+ {% set url = h.url_for(controller='dataset', action='read', id=pkg.name) %}
+
+ {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the current revision .{% endtrans %}
+
+
{% link_for h.resource_display_name(res)|truncate(30), controller='dataset', action='resource_read', id=pkg.name, resource_id=res.id %}
+ {{ _('Edit') }}
+ {% endif %}
+{% endblock %}
+
+{% block content_action %}
+ {% link_for _('All resources'), controller='dataset', action='resources', id=pkg.name, class_='btn', icon='arrow-left' %}
+ {% if res %}
+ {% link_for _('View resource'), controller='dataset', action='resource_read', id=pkg.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+ {% endif %}
+{% endblock %}
+
+{% block content_primary_nav %}
+ {{ h.build_nav_icon('resource_edit', _('Edit resource'), id=pkg.name, resource_id=res.id) }}
+ {# Updated by Francesco Mangiacrapa #11303 #}
+ {# if 'datapusher' in g.plugins #}
+ {# {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} #}
+ {# {% endif %} #}
+ {# {{ h.build_nav_icon('views', _('Views'), id=pkg.name, resource_id=res.id) }} #}
+ {% set is_sys_admin = true if (c.userobj and c.userobj.sysadmin) else false %}
+ {% if is_sys_admin %}
+ {% if 'datapusher' in g.plugins %}
+ {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
+ {% endif %}
+ {{ h.build_nav_icon('views', _('Views'), id=pkg.name, resource_id=res.id) }}
+ {% endif %}
+{% endblock %}
+
+{% block primary_content_inner %}
+ {% block form_title %}{{ _('Edit resource') }}{% endblock %}
+ {% block form %}{% endblock %}
+{% endblock %}
+
+{% block secondary_content %}
+ {% snippet 'package/snippets/resource_info.html', res=res %}
+{% endblock %}
+
+{% block scripts %}
+ {{ super() }}
+ {% resource 'vendor/fileupload' %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/resource_read.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/resource_read.html
new file mode 100644
index 0000000..066624e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/resource_read.html
@@ -0,0 +1,187 @@
+{% extends "package/base.html" %}
+
+{% set res = c.resource %}
+
+{% block head_extras -%}
+ {{ super() }}
+ {% set description = h.markdown_extract(res.description, extract_length=200) if res.description else h.markdown_extract(c.package.notes, extract_length=200) %}
+ {{ h.resource_display_name(res)|truncate(30) }}
+ {% block resource_inner %}
+
+
+ {% block resource_actions %}
+
+ {% block resource_actions_inner %}
+ {% if h.check_access('package_update', {'id':pkg.id }) %}
+ {% link_for _('Manage'), controller='dataset', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
+ {% endif %}
+ {% if res.url and h.is_url(res.url) %}
+
+
+ {% if res.resource_type in ('listing', 'service') %}
+
+
+ {% endif %}
+ {% if 'datastore' in g.plugins %}
+ {% snippet 'package/snippets/data_api_button.html', resource=res, datastore_root_url=c.datastore_api %}
+ {% endif %}
+ {% endblock %}
+
+ {% endblock %}
+
+ {% block resource_content %}
+ {% block resource_read_title %}
{{ h.resource_display_name(res) | truncate(50) }} {% endblock %}
+ {% block resource_read_url %}
+ {% if res.url and h.is_url(res.url) %}
+
{{ _('URL:') }} {{ res.url }}
+ {% elif res.url %}
+
{{ _('URL:') }} {{ res.url }}
+ {% endif %}
+ {% endblock %}
+
+ {% if res.description %}
+ {{ h.render_markdown(res.description) }}
+ {% endif %}
+ {% if not res.description and c.package.notes %}
+
{{ _('From the dataset abstract') }}
+
{{ h.d4science_theme_markdown_extract_html(c.package.get('notes')) }}
+
{% trans dataset=c.package.title, url=h.url_for(controller='dataset', action='read', id=c.package['name']) %}Source: {{ dataset }} {% endtrans %}
+ {% endif %}
+
+ {% endblock %}
+
+ {% block resource_view_content %}
+
+ {% set resource_preview = h.resource_preview(c.resource, c.package) %}
+ {% set views_created = res.has_views or resource_preview %}
+ {% if views_created %}
+ {% if resource_preview and not current_resource_view %}
+ {{ h.resource_preview(c.resource, c.package) }}
+ {% else %}
+ {% for resource_view in resource_views %}
+ {% if resource_view == current_resource_view %}
+ {% snippet 'package/snippets/resource_view.html',
+ resource_view=resource_view,
+ resource=c.resource,
+ package=c.package
+ %}
+ {% endif %}
+ {% endfor %}
+ {% endif %}
+ {# REMOVED View not created cases by Francesco Mangiacrapa #}
+ {% endif %}
+
+ {% endblock %}
+
+ {% endblock %}
+{% endblock %}
+
+{% block primary_content %}
+ {% block resource_additional_information %}
+ {% if res %}
+
+ {% block resource_additional_information_inner %}
+
+
{{ _('Additional Information') }}
+
+
+
+ {{ _('Field') }}
+ {{ _('Value') }}
+
+
+
+
+ {{ _('Last updated') }}
+ {# {{ h.render_datetime(res.last_modified) or h.render_datetime(res.revision_timestamp) or h.render_datetime(res.created) or _('unknown') }} #}
+
+ {{ h.render_datetime(res.last_modified) or h.render_datetime(res.revision_timestamp) or h.render_datetime(res.Created) or h.render_datetime(res.created) or _('unknown') }}
+
+
+ {{ _('Created') }}
+ {# {{ h.render_datetime(res.Created) or h.render_datetime(res.created) or _('unknown') }} #}
+
+ {{ h.render_datetime(res.Created) or h.render_datetime(res.created) or _('unknown') }}
+
+
+ {{ _('Format') }}
+ {{ res.mimetype_inner or res.mimetype or res.format or _('unknown') }}
+
+
+ {{ _('License') }}
+ {% snippet "snippets/license.html", pkg_dict=pkg, text_only=True %}
+
+ {% for key, value in h.format_resource_items(res.items()) %}
+ {{ key }} {{ value }}
+
+
+ {% endif %}
+ {% endblock %}
+{% endblock %}
+
+{% block secondary_content %}
+
+ {% block resources_list %}
+ {% snippet "package/snippets/resources.html", pkg=pkg, active=res.id %}
+ {% endblock %}
+
+ {% block resource_license %}
+ {# See: #7055
+ {% snippet "snippets/social.html" %}
+ #}
+ {% endblock %}
+{% endblock %}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/search.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/search.html
new file mode 100644
index 0000000..c508bcf
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/search.html
@@ -0,0 +1,90 @@
+{% extends "page.html" %}
+{% import 'macros/form.html' as form %}
+
+{% block subtitle %}{{ _("Datasets") }}{% endblock %}
+
+{% block breadcrumb_content %}
+ {{ h.nav_link(_('Datasets'), controller='dataset', action='search', highlight_actions = 'new index') }}
+{% endblock %}
+
+{% block primary_content %}
+
+
+ {% block page_primary_action %}
+ {% if h.check_access('package_create') %}
+ {# ADDED BY FRANCESCO MANGIACRAPA, 'ADD DATASET' IS SHOWN ONLY TO USER SYSADMIN' #7188 #}
+ {% if c.userobj and c.userobj.sysadmin %}
+
+ {% link_for _('Add Dataset'), controller='dataset', action='new', class_='btn btn-primary', icon='plus-sign-alt' %}
+
+ {% endif %}
+ {% endif %}
+ {% endblock %}
+ {% block form %}
+ {% set facets = {
+ 'fields': c.fields_grouped,
+ 'search': c.search_facets,
+ 'titles': c.facet_titles,
+ 'translated_fields': c.translated_fields,
+ 'remove_field': c.remove_field }
+ %}
+ {% set sorting = [
+ (_('Relevance'), 'score desc, metadata_modified desc'),
+ (_('Name Ascending'), 'title_string asc'),
+ (_('Name Descending'), 'title_string desc'),
+ (_('Last Modified'), 'metadata_modified desc'),
+ (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false) ]
+ %}
+ {% snippet 'snippets/search_form.html', form_id='dataset-search-form', type='dataset', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, show_empty=request.args, error=c.query_error, fields=c.fields %}
+ {% endblock %}
+ {% block package_search_results_list %}
+ {{ h.snippet('snippets/package_list.html', packages=c.page.items) }}
+ {% endblock %}
+
+
+ {% block package_search_results_api %}
+
+
+ {% block package_search_results_api_inner %}
+
+ {# Commented by Francesco, see Support #18126 #}
+ {#
+ {% set api_link = h.link_to(_('API'), h.url_for(controller='api', action='get_api', ver=3)) %}
+ {% set api_doc_link = h.link_to(_('API Docs'), 'http://docs.ckan.org/en/{0}/api/'.format(g.ckan_doc_version)) %}
+ {% if g.dumps_url -%}
+ {% set dump_link = h.link_to(_('full {format} dump').format(format=g.dumps_format), g.dumps_url) %}
+ {% trans %}
+ You can also access this registry using the {{ api_link }} (see {{ api_doc_link }}) or download a {{ dump_link }}.
+ {% endtrans %}
+ {% else %}
+ {% trans %}
+ You can also access this registry using the {{ api_link }} (see {{ api_doc_link}}).
+ {% endtrans %}
+ {%- endif %}
+ #}
+
+ {% endblock %}
+
+
+ {% endblock %}
+{% endblock %}
+
+
+{% block secondary_content %}
+
+
+
+ {% snippet "spatial/snippets/spatial_query.html" %}
+
+ {% for facet in c.facet_titles %}
+ {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet) }}
+ {% endfor %}
+
+
close
+
+ {% block extras scoped %}
+ {#
+ This performs a sort
+ {% for extra in h.sorted_extras(pkg_dict.extras) %}
+ #}
+ {% if pkg_dict.extras %}
+ {# Added by Francesco Mangiacrapa, see 17901 #}
+ {% set extra_item_url = h.get_pkg_dict_extra(pkg_dict,key_item_url) %}
+ {% if extra_item_url %}
+ {{ key_item_url }}
+
+
+ {{ extra_item_url }} {% snippet "package/snippets/qrcode_show.html", package_url=extra_item_url %}
+
+
{{category_idx.category.title}}
+ {% if category_idx.category.description %}
+
Description: {{category_idx.category.description}}
+ {% endif %}
+
+
+
+ {{ _('Field') }}
+ {{ _('Value') }}
+
+
+
+ {% set my_extras = h.d4science_get_extra_for_category(extras_indexed_for_categories, k_cat) %}
+ {% snippet "package/snippets/extras_table.html", my_extras=my_extras, key_item_url=key_item_url %}
+
+
+
{{ _('Additional Info') }}
+
+
+
+ {{ _('Field') }}
+ {{ _('Value') }}
+
+
+
+ {% snippet "package/snippets/extras_table.html", my_extras=my_extras, key_item_url=key_item_url %}
+
+
+
{{ _('Management Info') }}
+
+
+
+ {{ _('Field') }}
+ {{ _('Value') }}
+
+
+
+ {% block package_additional_info %}
+ {% if pkg_dict.url %}
+
+ {{ _('Source') }}
+ {% if h.is_url(pkg_dict.url) %}
+ {{ h.link_to(pkg_dict.url, pkg_dict.url, rel='foaf:homepage', target='_blank') }}
+ {% else %}
+ {{ pkg_dict.url }}
+ {% endif %}
+
+ {% endif %}
+
+ {% if pkg_dict.author_email %}
+
+ {{ _("Author") }}
+ {{ h.mail_to(email_address=pkg_dict.author_email, name=pkg_dict.author) }}
+
+ {% elif pkg_dict.author %}
+
+ {{ _("Author") }}
+ {{ pkg_dict.author }}
+
+ {% endif %}
+ {# Added by Francesco Mangiacrapa #}
+ {% set user_maintainer = h.d4science_get_user_info(pkg_dict.maintainer) %}
+ {% if pkg_dict.maintainer_email %}
+
+ {{ _('Maintainer') }}
+ {{ h.mail_to(email_address=pkg_dict.maintainer_email, name=user_maintainer.fullname if (user_maintainer and user_maintainer.fullname) else pkg_dict.maintainer) }}
+
+ {% elif pkg_dict.maintainer %}
+
+ {{ _('Maintainer') }}
+ {{ user_maintainer.fullname if (user_maintainer and user_maintainer.fullname) else pkg_dict.maintainer }}
+
+ {% endif %}
+
+ {% if pkg_dict.version %}
+
+ {{ _("Version") }}
+ {{ pkg_dict.version }}
+
+ {% endif %}
+
+ {% if h.check_access('package_update',{'id':pkg_dict.id}) %}
+
+ {{ _("State") }}
+ {{ _(pkg_dict.state) }}
+
+ {% endif %}
+ {% if pkg_dict.metadata_modified %}
+
+ {{ _("Last Updated") }}
+
+ {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_modified %}
+
+
+ {% endif %}
+ {% if pkg_dict.metadata_created %}
+
+ {{ _("Created") }}
+
+
+ {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_created %}
+
+
+ {% endif %}
+
+
+ {% endblock %}
+
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/package/snippets/extras_table.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/extras_table.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/package/snippets/extras_table.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/extras_table.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/info.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/info.html
new file mode 100644
index 0000000..2658c11
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/info.html
@@ -0,0 +1,47 @@
+{#
+Displays a sidebard module with information for given package
+
+pkg - The package dict that owns the resources.
+
+Example:
+
+ {% snippet "package/snippets/info.html", pkg=pkg %}
+
+#}
+{% block package_info %}
+ {% if pkg %}
+
+
+
+ {% block package_info_inner %}
+ {% block heading %}
+
{{ h.dataset_display_name(pkg) }}
+ {% endblock %}
+ {% if pkg.extras %}
+ {% for extra in pkg.extras if extra['key'] == 'graphic-preview-file' %}
+
+ {% endfor %}
+ {% endif %}
+ {% block nums %}
+
+
+ {{ _('Followers') }}
+ {{ h.SI_number_span(h.follow_count('dataset', pkg.id)) }}
+
+
+ {% endblock %}
+ {% block follow_button %}
+ {% if not hide_follow_button %}
+
+ {{ h.follow_button('dataset', pkg.name) }}
+
+ {% endif %}
+ {% endblock %}
+ {% endblock %}
+
+
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/package/snippets/qrcode_show.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/qrcode_show.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/package/snippets/qrcode_show.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/qrcode_show.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resource_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resource_form.html
new file mode 100644
index 0000000..2fab14e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resource_form.html
@@ -0,0 +1,96 @@
+{% import 'macros/form.html' as form %}
+
+{% set data = data or {} %}
+{% set errors = errors or {} %}
+{% set action = form_action or h.url_for(controller='dataset', action='new_resource', id=pkg_name) %}
+
+
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resource_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resource_item.html
new file mode 100644
index 0000000..9d1c462
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resource_item.html
@@ -0,0 +1,106 @@
+{% set can_edit = h.check_access('package_update', {'id':pkg.id }) %}
+{% set url_action = 'resource_edit' if url_is_edit and can_edit else 'resource_read' %}
+{% set url = h.url_for(controller='dataset', action=url_action, id=pkg.name, resource_id=res.id) %}
+
+{# Added by Francesco Mangiacrapa block custom_view_on_resources see:4851 #}
+{% block custom_view_on_resources %}
+{% set user = c.user %}
+{% if user %}
+ {% block resource_item_title %}
+
+ {{ h.resource_display_name(res) | truncate(50) }}{{ res.format }}
+ {{ h.popular('views', res.tracking_summary.total, min=10) }}
+
+ {% endblock %}
+ {% block resource_item_description %}
+
+ {% if res.description %}
+ {{ h.markdown_extract(res.description, extract_length=80) }}
+ {% endif %}
+
+ {% endblock %}
+ {% block resource_item_explore %}
+ {% if not url_is_edit %}
+ {# Only if can edit, explorer button is shown with several facility #}
+ {% if can_edit %}
+
+ {% else %}
+
+ {{ _('Go to resource') }}
+
+ {% endif %}
+ {% endif %}
+ {% endblock %}
+{% else %}
+ {% block resource_item_title2 %}
+ {# Updated by Francesco Mangiacrapa, see: #10056 #}
+
+ {{ h.resource_display_name(res) | truncate(50) }}{{ res.format }}
+ {{ h.popular('views', res.tracking_summary.total, min=10) }}
+
+ {% endblock %}
+
+{% endif %}
+{% endblock %}
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resources.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resources.html
new file mode 100644
index 0000000..63bd8f2
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resources.html
@@ -0,0 +1,34 @@
+{#
+Displays a sidebard module with navigation containing the provided resources.
+If no resources are provided then the module will not be displayed.
+
+pkg - The package dict that owns the resources.
+active - The id of the currently displayed resource.
+action - The controller action to use (default: 'resource_read').
+
+Example:
+
+ {% snippet "package/snippets/resources.html", pkg=pkg, active=res.id %}
+
+#}
+{% set resources = pkg.resources or [] %}
+{% if resources %}
+ {% block resources %}
+
+ {% block resources_inner %}
+ {% block resources_title %}
+
+ {% for resource in resources %}
+
+ {% link_for h.resource_display_name(resource)|truncate(25), controller='dataset', action=action or 'resource_read', id=pkg.name, resource_id=resource.id, inner_span=true %}
+
+ {% endfor %}
+
+ {% endblock %}
+ {% endblock %}
+
+ {% endblock %}
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resources_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resources_list.html
new file mode 100644
index 0000000..8b3e23d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/package/snippets/resources_list.html
@@ -0,0 +1,76 @@
+{#
+Renders a list of resources with icons and view links.
+
+resources - A list of resources to render
+pkg - A package object that the resources belong to.
+
+Example:
+
+ {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources %}
+
+#}
+
+
+
+
+ {{ _('Data and Resources') }}
+ {% set user = c.user %}
+ {# Added by Francesco Mangiacrapa #10389 #}
+ {% if not user %}
+ To access the resources you must log in
+ {% endif %}
+ {# end #}
+ {% block resource_list %}
+ {% if resources %}
+
+ {% block resource_list_inner %}
+ {% for resource in resources %}
+ {% snippet 'package/snippets/resource_item.html', pkg=pkg, res=resource %}
+ {% endfor %}
+ {% endblock %}
+
+ {% else %}
+ {% if h.check_access('resource_create', {'package_id': pkg['id']}) %}
+ {% trans url=h.url_for(controller='dataset', action='new_resource', id=pkg.name) %}
+ This dataset has no data, why not add some?
+ {% endtrans %}
+ {% else %}
+ {{ _('This dataset has no data') }}
+ {% endif %}
+ {% endif %}
+ {% endblock %}
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/page.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/page.html
new file mode 100644
index 0000000..ed706cc
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/page.html
@@ -0,0 +1,134 @@
+{% extends "base.html" %}
+
+{%- block page -%}
+
+ {% block skip %}
+
+ {% endblock %}
+
+ {#
+ Override the header on a page by page basis by extending this block. If
+ making sitewide header changes it is preferable to override the header.html
+ file.
+ #}
+ {%- block header %}
+ {% include "header.html" %}
+ {% endblock -%}
+
+ {# The content block allows you to replace the content of the page if needed #}
+ {%- block content %}
+ {% block maintag %}{% endblock %}
+
+ {% block main_content %}
+ {% block flash %}
+
+ {% block flash_inner %}
+ {% for category, message in h.get_flashed_messages(with_categories=true) %}
+
+ {{ h.literal(message) }}
+
+ {% endfor %}
+ {% endblock %}
+
+ {% endblock %}
+
+ {% block toolbar %}
+
+ {% endblock %}
+
+
+ {#
+ The pre_primary block can be used to add content to before the
+ rendering of the main content columns of the page.
+ #}
+ {% block pre_primary %}
+ {% endblock %}
+
+ {% block secondary %}
+
+ {% endblock %}
+
+ {% block primary %}
+
+ {#
+ The primary_content block can be used to add content to the page.
+ This is the main block that is likely to be used within a template.
+
+ Example:
+
+ {% block primary_content %}
+
My page content
+
Some content for the page
+ {% endblock %}
+ #}
+ {% block primary_content %}
+
+ {% block page_header %}
+
+ {% endblock %}
+
+ {% if self.page_primary_action() | trim %}
+
+ {% block page_primary_action %}{% endblock %}
+
+ {% endif %}
+ {% block primary_content_inner %}
+ {% endblock %}
+
+ {% endblock %}
+
+ {% endblock %}
+
+ {% endblock %}
+
+
+
+ {% for extra in extras %}
+ {% set is_system_field = extra.key.startswith(d4science_cms_obj_placeholders.prefix) or extra.key.startswith(d4science_system_type_obj.id) %}
+ {% set apply_classes = [] %}
+ {% if is_system_field %}
+ {% set apply_classes = ['disabled-div'] %}
+ {% endif %}
+ {% set prefix = 'extras__%d__' % loop.index0 %}
+ {{ form.custom(
+ names=(prefix ~ 'key', prefix ~ 'value', prefix ~ 'deleted'),
+ id='field-extras-%d' % loop.index,
+ label=_('Custom Field'),
+ values=(extra.key, extra.value, extra.deleted),
+ classes=apply_classes,
+ error=errors[prefix ~ 'key'] or errors[prefix ~ 'value']
+ ) }}
+ {% endfor %}
+
+ {# Add a max of 3 empty columns #}
+ {% set total_extras = extras|count %}
+ {% set empty_extras = (limit or 3) - total_extras %}
+ {% if empty_extras <= 0 %}{% set empty_extras = 1 %}{% endif %}
+
+ {% for extra in range(total_extras, total_extras + empty_extras) %}
+ {% set index = loop.index0 + (extras|count) %}
+ {% set prefix = 'extras__%d__' % index %}
+ {{ form.custom(
+ names=(prefix ~ 'key', prefix ~ 'value', prefix ~ 'deleted'),
+ id='field-extras-%d' % index,
+ label=_('Custom Field'),
+ values=(extra.key, extra.value, extra.deleted),
+ error=errors[prefix ~ 'key'] or errors[prefix ~ 'value']
+ ) }}
+ {% endfor %}
+
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/facet_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/facet_list.html
new file mode 100644
index 0000000..e985503
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/facet_list.html
@@ -0,0 +1,99 @@
+{#
+Construct a facet module populated with links to filtered results.
+
+name
+ The field name identifying the facet field, eg. "tags"
+
+title
+ The title of the facet, eg. "Tags", or "Tag Cloud"
+
+label_function
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a `facet_item`.
+ eg. lambda facet_item: facet_item.display_name.upper()
+ By default it displays the facet item's display name, which should
+ usually be good enough
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+extras
+ Extra info passed into the add/remove params to make the url
+
+alternative_url
+ URL to use when building the necessary URLs, instead of the default
+ ones returned by url_for. Useful eg for dataset types.
+
+hide_empty
+ Do not show facet if there are none, Default: false.
+
+within_tertiary
+ Boolean for when a facet list should appear in the the right column of the
+ page and not the left column.
+
+#}
+{% block facet_list %}
+ {% set hide_empty = hide_empty or false %}
+ {% with items = items or h.get_facet_items_dict(name) %}
+ {% if items or not hide_empty %}
+ {% if within_tertiary %}
+ {% set nav_class = 'nav nav-pills nav-stacked' %}
+ {% set nav_item_class = ' ' %}
+ {% set wrapper_class = 'nav-facet nav-facet-tertiary' %}
+ {% endif %}
+ {% block facet_list_item %}
+
+ {% block facet_list_heading %}
+
+
+ {% endblock %}
+ {% block facet_list_items %}
+ {% with items = items or h.get_facet_items_dict(name) %}
+ {% if items %}
+
+
+ {% for item in items %}
+
+ {% set href = h.remove_url_param(name, item.name, extras=extras, alternative_url=alternative_url) if item.active else h.add_url_param(new_params={name: item.name}, extras=extras, alternative_url=alternative_url) %}
+ {% set label = label_function(item) if label_function else item.display_name %}
+ {# Updated by Francesco Mangiacrapa, see #11747 #}
+ {#
+ {% set label_truncated = h.truncate(label, 22) if not label_function else label %}
+ #}
+ {% set count = count_label(item['count']) if count_label else ('(%d)' % item['count']) %}
+
+
+ {{ label }} {{ count }}
+
+
+ {% endfor %}
+
+
+
+
+ {% else %}
+ {{ _('There are no {facet_type} that match this search').format(facet_type=title) }}
+ {% endif %}
+ {% endwith %}
+ {% endblock %}
+
+ {% endblock %}
+ {% endif %}
+ {% endwith %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/package_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/package_item.html
new file mode 100644
index 0000000..ca2d067
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/package_item.html
@@ -0,0 +1,133 @@
+{#
+Displays a single of dataset.
+
+package - A package to display.
+item_class - The class name to use on the list item.
+hide_resources - If true hides the resources (default: false).
+banner - If true displays a popular banner (default: false).
+truncate - The length to trucate the description to (default: 180)
+truncate_title - The length to truncate the title to (default: 80).
+
+Example:
+
+ {% snippet 'snippets/package_item.html', package=c.datasets[0] %}
+
+#}
+
+{% set truncate = truncate or 180 %}
+{% set truncate_title = truncate_title or 80 %}
+{% set title = package.title or package.name %}
+{% set notes = h.markdown_extract(package.notes, extract_length=truncate) %}
+{% set acquired = h.is_dataset_acquired(package) %}
+{% set owner = h.is_owner(package) %}
+
+{#
+{% resource 'd4science_theme/custom.css' %}
+CHANGED BY FRANCESCO.MANGIACRAPA
+#}
+
+{% block package_item_content %}
+
+{% if package.private and not h.can_read(package) %}
+
+
+
+ {% if package.private and not h.can_read(package) %}
+
+
+ {% endif %}
+ {% if acquired and not owner %}
+
+
+ {% endif %}
+
+
+ {% if package.private and not h.can_read(package) %}
+ {# {{ _(h.truncate(title, truncate_title)) }} #}
+ Dataset
+
+ {{ h.acquire_button(package) }}
+ {% else %}
+ {{ h.link_to(h.truncate(title, truncate_title), h.url_for(controller='dataset', action='read', id=package.name)) }}
+ {% endif %}
+
+
+ {% if package.get('state', '').startswith('draft') %}
+ {{ _('Draft') }}
+ {% elif package.get('state', '').startswith('deleted') %}
+ {{ _('Deleted') }}
+ {% endif %}
+ {{ h.popular('recent views', package.tracking_summary.recent, min=10) if package.tracking_summary }}
+
+ {% if banner %}
+
{{ _('Popular') }}
+ {% endif %}
+ {% if notes %}
+
+ {% endif %}
+
+{% else %}
+
+
+
+ {% if package.private and not h.can_read(package) %}
+
+
+ {% endif %}
+ {% if acquired and not owner %}
+
+
+ {% endif %}
+ {% if owner %}
+
+
+ {% endif %}
+
+
+ {% if package.private and not h.can_read(package) %}
+ {{ _(h.truncate(title, truncate_title)) }}
+
+ {{ h.acquire_button(package) }}
+ {% else %}
+ {{ h.link_to(h.truncate(title, truncate_title), h.url_for(controller='dataset', action='read', id=package.name)) }}
+ {% endif %}
+
+
+ {% if package.get('state', '').startswith('draft') %}
+ {{ _('Draft') }}
+ {% elif package.get('state', '').startswith('deleted') %}
+ {{ _('Deleted') }}
+ {% endif %}
+ {{ h.popular('recent views', package.tracking_summary.recent, min=10) if package.tracking_summary }}
+
+ {% if banner %}
+
{{ _('Popular') }}
+ {% endif %}
+ {% if notes %}
+
{{ notes|urlize }}
+ {% endif %}
+
+ {% for resource in h.dict_list_reduce(package.resources, 'format') %}
+
+ {{ resource }}
+
+ {% endfor %}
+
+ {% endif %}
+
+
+{% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/tag_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/tag_list.html
new file mode 100644
index 0000000..c14f6d1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/snippets/tag_list.html
@@ -0,0 +1,18 @@
+{#
+render a list of tags linking to the dataset search page
+tags: list of tags
+
+Removed truncate function by Francesco Mangiacrapa
+ {{ h.truncate(tag.display_name, 22) }}
+
+#}
+{% set _class = _class or 'tag-list' %}
+{% block tag_list %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/index.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/index.html
similarity index 90%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/index.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/index.html
index bb27080..a1e1d72 100644
--- a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/index.html
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/index.html
@@ -3,7 +3,7 @@
{% block subtitle %}{{ _('Types') }}{% endblock %}
{% block breadcrumb_content %}
- {% link_for _('Types'), controller='d4STypeController', action='index' %}
+ {% link_for _('Types'), controller='d4s', action='index' %}
{% endblock %}
{% block page_header %}{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/helper.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/helper.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/helper.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/helper.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/type_form.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/type_form.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/type_form.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/type_form.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/type_item.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/type_item.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/type_item.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/type_item.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/type_list.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/type_list.html
similarity index 100%
rename from src/ckanext-d4science_theme/ckanext/d4science_theme/templates/type/snippets/type_list.html
rename to src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/type/snippets/type_list.html
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard.html
new file mode 100644
index 0000000..7a8ecbd
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard.html
@@ -0,0 +1,54 @@
+{% ckan_extends %}
+
+{% set user = c.userobj %}
+
+{% block breadcrumb_content %}
+ {{ _('Dashboard') }}
+ {% block page_header %}
+ {# CODE ADDED BY Francesco Mangiacrapa #}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Edit settings' #}
+ {% else %}
+
+ {% endif %}
+ {% endblock %}
+
+ {% if self.page_primary_action() | trim %}
+
+ {% block page_primary_action %}{% endblock %}
+
+ {% endif %}
+ {% block primary_content_inner %}
+
+ {% snippet 'user/snippets/followee_dropdown.html', context=c.dashboard_activity_stream_context, followees=c.followee_list %}
+
+ {% block page_heading %}
+ {{ _('News feed') }}
+ {% endblock %}
+ {{ _("Activity from items that I'm following") }}
+
+ {% block activity_stream %}
+ {{ c.dashboard_activity_stream }}
+ {% endblock %}
+
+ {% endblock %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_datasets.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_datasets.html
new file mode 100644
index 0000000..f2810e5
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_datasets.html
@@ -0,0 +1,65 @@
+{#{% extends "user/dashboard.html" %}#}
+{# CREATED BY FRANCESCO MANGIACRAPA #}
+
+{% ckan_extends %}
+{% set user = c.userobj %}
+
+{% block breadcrumb_content %}
+{% endblock %}
+
+{% block primary %}
+
+ {% block page_header %}
+ {# CODE ADDED BY Francesco Mangiacrapa #}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Edit settings' #}
+ {% else %}
+
+ {% endif %}
+ {% endblock %}
+
+ {% if self.page_primary_action() | trim %}
+
+ {% block page_primary_action %}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Edit settings' #}
+ {% else %}
+ {% if h.check_access('package_create') %}
+ {% link_for _('Add Dataset'), controller='dataset', action='new', class_="btn btn-primary", icon="plus-sign-alt" %}
+ {% endif %}
+ {% endif %}
+ {% endblock %}
+
+ {% endif %}
+ {% block primary_content_inner %}
+
{{ _('My Datasets') }}
+ {% if c.user_dict.datasets %}
+ {% snippet 'snippets/package_list.html', packages=c.user_dict.datasets %}
+ {% else %}
+
+ {{ _('You haven\'t created any datasets.') }}
+ {% if h.check_access('package_create') %}
+ {#{% link_for _('Create one now?'), controller='dataset', action='new' %}#}
+ {% endif %}
+
+ {% endif %}
+ {% endblock %}
+
+{% endblock %}
+
+{% block dashboard_activity_stream_context %}{% endblock %}
+
+{% block secondary %}{% endblock %}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_groups.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_groups.html
new file mode 100644
index 0000000..ecbd041
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_groups.html
@@ -0,0 +1,65 @@
+{#{% extends "user/dashboard.html" %}#}
+
+{#{% extends "user/dashboard.html" %}#}
+{# CREATED BY FRANCESCO MANGIACRAPA #}
+
+{% ckan_extends %}
+{% set user = c.userobj %}
+
+{% block breadcrumb_content %}
+{% endblock %}
+
+{% block primary %}
+
+ {% block page_header %}
+ {# CODE ADDED BY Francesco Mangiacrapa #}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Edit settings' #}
+ {% else %}
+
+ {% endif %}
+ {% endblock %}
+
+ {% if self.page_primary_action() | trim %}
+
+ {% block page_primary_action %}
+ {% if h.check_access('group_create') %}
+ {% link_for _('Add Group'), controller='group', action='new', class_="btn btn-primary", icon="plus-sign-alt" %}
+ {% endif %}
+{% endblock %}
+
+ {% endif %}
+ {% block primary_content_inner %}
+
{{ _('My Groups') }}
+ {% set groups = h.groups_available(am_member=True) %}
+ {% if groups %}
+
+ {% snippet "group/snippets/group_list_simple.html", groups=groups %}
+
+ {% else %}
+
+ {{ _('You are not a member of any groups.') }}
+ {% if h.check_access('group_create') %}
+ {% link_for _('Create one now?'), controller='group', action='new' %}
+ {% endif %}
+
+ {% endif %}
+{% endblock %}
+
+{% endblock %}
+
+{% block dashboard_activity_stream_context %}{% endblock %}
+
+{% block secondary %}{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_organizations.html b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_organizations.html
new file mode 100644
index 0000000..1d2f8d7
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/d4science_theme/templates2.6/user/dashboard_organizations.html
@@ -0,0 +1,66 @@
+{#{% extends "user/dashboard.html" %}#}
+
+{#{% extends "user/dashboard.html" %}#}
+{# CREATED BY FRANCESCO MANGIACRAPA #}
+
+{% ckan_extends %}
+{% set user = c.userobj %}
+
+{% block breadcrumb_content %}
+{% endblock %}
+
+{% block primary %}
+
+ {% block page_header %}
+ {# CODE ADDED BY Francesco Mangiacrapa #}
+ {% set hide = h.get_cookie_value('ckan_hide_header') %}
+ {% if hide=='true' %}
+ {# NOT SHOW 'Edit settings' #}
+ {% else %}
+
+ {% endif %}
+ {% endblock %}
+
+ {% if self.page_primary_action() | trim %}
+
+{% block page_primary_action %}
+ {% if h.check_access('organization_create') %}
+ {% link_for _('Add Organization'), controller='organization', action='new', class_="btn btn-primary", icon="plus-sign-alt" %}
+ {% endif %}
+{% endblock %}
+
+ {% endif %}
+
+{% block primary_content_inner %}
+
{{ _('My Organizations') }}
+ {% set organizations = h.organizations_available() %}
+ {% if organizations %}
+
+ {% snippet "organization/snippets/organization_list.html", organizations=organizations %}
+
+ {% else %}
+
+ {{ _('You are not a member of any organizations.') }}
+ {% if h.check_access('organization_create') %}
+ {% link_for _('Create one now?'), controller='organization', action='new' %}
+ {% endif %}
+
+ {% endif %}
+{% endblock %}
+
+{% endblock %}
+
+{% block dashboard_activity_stream_context %}{% endblock %}
+
+{% block secondary %}{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/__init__.py b/src/ckanext-d4science_theme/ckanext/datapusher/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css
new file mode 100644
index 0000000..e2d61a0
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css
@@ -0,0 +1,33 @@
+.datapusher-activity {
+ padding: 0;
+ list-style-type: none;
+ background: transparent url("/dotted.png") 21px 0 repeat-y; }
+ .datapusher-activity .item {
+ position: relative;
+ margin: 0 0 15px 0;
+ padding: 0; }
+ .datapusher-activity .item .date {
+ color: #999;
+ font-size: 12px;
+ white-space: nowrap;
+ margin: 5px 0 0 80px; }
+ .datapusher-activity .item.no-avatar p {
+ margin-left: 40px; }
+
+.popover {
+ width: 300px; }
+ .popover .popover-header {
+ font-weight: bold;
+ margin-bottom: 0; }
+ .popover .popover-close {
+ float: right;
+ text-decoration: none; }
+
+.datapusher-activity .item .icon {
+ color: #999999; }
+.datapusher-activity .item.failure .icon {
+ color: #B95252; }
+.datapusher-activity .item.success .icon {
+ color: #69A67A; }
+
+/*# sourceMappingURL=datapusher.css.map */
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map
new file mode 100644
index 0000000..d5aad5f
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map
@@ -0,0 +1,7 @@
+{
+"version": 3,
+"mappings": "AAAA,oBAAqB;EACjB,OAAO,EAAE,CAAC;EACV,eAAe,EAAE,IAAI;EACrB,UAAU,EAAE,8CAA+C;EAC3D,0BAAM;IACF,QAAQ,EAAE,QAAQ;IAClB,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,CAAC;IACV,gCAAM;MACF,KAAK,EAAE,IAAI;MACX,SAAS,EAAE,IAAI;MACf,WAAW,EAAE,MAAM;MACnB,MAAM,EAAE,YAAY;IAExB,sCAAc;MACV,WAAW,EAAE,IAAI;;AAM7B,QAAS;EACL,KAAK,EAAE,KAAK;EACZ,wBAAgB;IACZ,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,CAAC;EAEpB,uBAAe;IACX,KAAK,EAAE,KAAK;IACZ,eAAe,EAAE,IAAI;;AAUzB,gCAAM;EACF,KAAK,EANQ,OAAO;AAQxB,wCAAgB;EACZ,KAAK,EAPS,OAAO;AASzB,wCAAgB;EACZ,KAAK,EAXM,OAAO",
+"sources": ["datapusher.scss"],
+"names": [],
+"file": "datapusher.css"
+}
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss
new file mode 100644
index 0000000..44617be
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss
@@ -0,0 +1,49 @@
+.datapusher-activity {
+ padding: 0;
+ list-style-type: none;
+ background: transparent url('/dotted.png') 21px 0 repeat-y;
+ .item {
+ position: relative;
+ margin: 0 0 15px 0;
+ padding: 0;
+ .date {
+ color: #999;
+ font-size: 12px;
+ white-space: nowrap;
+ margin: 5px 0 0 80px;
+ }
+ &.no-avatar p {
+ margin-left: 40px;
+ }
+ }
+}
+
+// popover
+.popover {
+ width: 300px;
+ .popover-header {
+ font-weight: bold;
+ margin-bottom: 0;
+ }
+ .popover-close {
+ float: right;
+ text-decoration: none;
+ }
+}
+
+// base colors
+$activityColorBlank: #999999;
+$activityColorNew: #69A67A;
+$activityColorDelete: #B95252;
+
+.datapusher-activity .item {
+ .icon {
+ color: $activityColorBlank ;
+ }
+ &.failure .icon {
+ color: $activityColorDelete;
+ }
+ &.success .icon {
+ color: $activityColorNew;
+ }
+}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js
new file mode 100644
index 0000000..47d9887
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js
@@ -0,0 +1,27 @@
+"use strict";
+
+/* Datapusher popover
+ * Handle the popup display in 'resource view page' activity stream
+ *
+ * Options
+ * - title: title of the popover
+ * - content: content for the popover
+ */
+ckan.module('datapusher_popover', function($) {
+ return {
+ initialize: function() {
+ $.proxyAll(this, /_on/);
+ this.popover = this.el;
+ this.popover.popover({
+ html: true,
+ title: this.options.title,
+ content: this.options.content
+ });
+ $(document).on('click', '.popover-close', this._onClose);
+ },
+
+ _onClose: function() {
+ this.popover.popover('hide');
+ }
+ };
+});
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml b/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml
new file mode 100644
index 0000000..f7f18bb
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml
@@ -0,0 +1,13 @@
+datapusher:
+ filters: rjsmin
+ output: datapusher/%(version)s_datapusher.js
+ extra:
+ preload:
+ - base/main
+ contents:
+ - datapusher_popover.js
+
+datapusher-css:
+ output: ckanext-datapusher/%(version)s_datapusher.css
+ contents:
+ - datapusher.css
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/cli.py b/src/ckanext-d4science_theme/ckanext/datapusher/cli.py
new file mode 100644
index 0000000..bfb33d3
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/cli.py
@@ -0,0 +1,105 @@
+# encoding: utf-8
+
+from __future__ import annotations
+
+from ckan.types import Context
+
+import logging
+from typing import cast
+
+import click
+
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+import ckanext.datastore.backend as datastore_backend
+from ckan.cli import error_shout
+
+log = logging.getLogger(__name__)
+
+question = (
+ u"Data in any datastore resource that isn't in their source files "
+ u"(e.g. data added using the datastore API) will be permanently "
+ u"lost. Are you sure you want to proceed?"
+)
+requires_confirmation = click.option(
+ u'--yes', u'-y', is_flag=True, help=u'Always answer yes to questions'
+)
+
+
+def confirm(yes: bool):
+ if yes:
+ return
+ click.confirm(question, abort=True)
+
+
+@click.group(short_help=u"Perform commands in the datapusher.")
+def datapusher():
+ """Perform commands in the datapusher.
+ """
+ pass
+
+
+@datapusher.command()
+@requires_confirmation
+def resubmit(yes: bool):
+ u'''Resubmit updated datastore resources.
+ '''
+ confirm(yes)
+
+ resource_ids = datastore_backend.get_all_resources_ids_in_datastore()
+ _submit(resource_ids)
+
+
+@datapusher.command()
+@click.argument(u'package', required=False)
+@requires_confirmation
+def submit(package: str, yes: bool):
+ u'''Submits resources from package.
+
+ If no package ID/name specified, submits all resources from all
+ packages.
+ '''
+ confirm(yes)
+
+ if not package:
+ ids = tk.get_action(u'package_list')(cast(Context, {
+ u'model': model,
+ u'ignore_auth': True
+ }), {})
+ else:
+ ids = [package]
+
+ for id in ids:
+ package_show = tk.get_action(u'package_show')
+ try:
+ pkg = package_show(cast(Context, {
+ u'model': model,
+ u'ignore_auth': True
+ }), {u'id': id})
+ except Exception as e:
+ error_shout(e)
+ error_shout(u"Package '{}' was not found".format(package))
+ raise click.Abort()
+ if not pkg[u'resources']:
+ continue
+ resource_ids = [r[u'id'] for r in pkg[u'resources']]
+ _submit(resource_ids)
+
+
+def _submit(resources: list[str]):
+ click.echo(u'Submitting {} datastore resources'.format(len(resources)))
+ user = tk.get_action(u'get_site_user')(cast(Context, {
+ u'model': model,
+ u'ignore_auth': True
+ }), {})
+ datapusher_submit = tk.get_action(u'datapusher_submit')
+ for id in resources:
+ click.echo(u'Submitting {}...'.format(id), nl=False)
+ data_dict = {
+ u'resource_id': id,
+ u'ignore_hash': True,
+ }
+ if datapusher_submit({u'user': user[u'name']}, data_dict):
+ click.echo(u'OK')
+ else:
+ click.echo(u'Fail')
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml b/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml
new file mode 100644
index 0000000..e5d440d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml
@@ -0,0 +1,51 @@
+version: 1
+groups:
+- annotation: Datapusher settings
+ options:
+ - key: ckan.datapusher.formats
+ default:
+ - csv
+ - xls
+ - xlsx
+ - tsv
+ - application/csv
+ - application/vnd.ms-excel
+ - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+ - ods
+ - application/vnd.oasis.opendocument.spreadsheet
+ type: list
+ example: csv xls
+ description: >-
+ File formats that will be pushed to the DataStore by the DataPusher. When
+ adding or editing a resource which links to a file in one of these formats,
+ the DataPusher will automatically try to import its contents to the DataStore.
+
+ - key: ckan.datapusher.url
+ example: http://127.0.0.1:8800/
+ description: >-
+ DataPusher endpoint to use when enabling the ``datapusher`` extension. If you
+ installed CKAN via :doc:`/maintaining/installing/install-from-package`, the DataPusher was installed for you
+ running on port 8800. If you want to manually install the DataPusher, follow
+ the installation `instructions `_.
+
+ - key: ckan.datapusher.api_token
+ description: >-
+ Starting from CKAN 2.10, DataPusher requires a valid API token to operate (see :ref:`api authentication`), and
+ will fail to start if this option is not set.
+
+ - key: ckan.datapusher.callback_url_base
+ example: http://ckan:5000/
+ placeholder: '%(ckan.site_url)s'
+ description: >-
+ Alternative callback URL for DataPusher when performing a request to CKAN. This is
+ useful on scenarios where the host where DataPusher is running can not access the
+ public CKAN site URL.
+
+ - key: ckan.datapusher.assume_task_stale_after
+ default: 3600
+ type: int
+ example: 86400
+ description: >-
+ In case a DataPusher task gets stuck and fails to recover, this is the minimum
+ amount of time (in seconds) after a resource is submitted to DataPusher that the
+ resource can be submitted again.
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py b/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py
new file mode 100644
index 0000000..551c644
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py
@@ -0,0 +1,31 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from typing import Any
+import ckan.plugins.toolkit as toolkit
+
+
+def datapusher_status(resource_id: str):
+ try:
+ return toolkit.get_action('datapusher_status')(
+ {}, {'resource_id': resource_id})
+ except toolkit.ObjectNotFound:
+ return {
+ 'status': 'unknown'
+ }
+
+
+def datapusher_status_description(status: dict[str, Any]):
+ _ = toolkit._
+
+ if status.get('status'):
+ captions = {
+ 'complete': _('Complete'),
+ 'pending': _('Pending'),
+ 'submitting': _('Submitting'),
+ 'error': _('Error'),
+ }
+
+ return captions.get(status['status'], status['status'].capitalize())
+ else:
+ return _('Not Uploaded Yet')
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py b/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py
new file mode 100644
index 0000000..2a7e5d2
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py
@@ -0,0 +1,55 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from ckan.plugins.interfaces import Interface
+from typing import Any
+
+
+class IDataPusher(Interface):
+ """
+ The IDataPusher interface allows plugin authors to receive notifications
+ before and after a resource is submitted to the datapusher service, as
+ well as determining whether a resource should be submitted in can_upload
+
+ The before_submit function, when implemented
+ """
+
+ def can_upload(self, resource_id: str) -> bool:
+ """ This call when implemented can be used to stop the processing of
+ the datapusher submit function. This method will not be called if
+ the resource format does not match those defined in the
+ ckan.datapusher.formats config option or the default formats.
+
+ If this function returns False then processing will be aborted,
+ whilst returning True will submit the resource to the datapusher
+ service
+
+ Note that before reaching this hook there is a prior check on the
+ resource format, which depends on the value of
+ the :ref:`ckan.datapusher.formats` configuration option (and requires
+ the resource to have a format defined).
+
+ :param resource_id: The ID of the resource that is to be
+ pushed to the datapusher service.
+
+ Returns ``True`` if the job should be submitted and ``False`` if
+ the job should be aborted
+
+ :rtype: bool
+ """
+ return True
+
+ def after_upload(
+ self, context: dict[str, Any], resource_dict: dict[str, Any],
+ dataset_dict: dict[str, Any]) -> None:
+ """ After a resource has been successfully upload to the datastore
+ this method will be called with the resource dictionary and the
+ package dictionary for this resource.
+
+ :param context: The context within which the upload happened
+ :param resource_dict: The dict represenstaion of the resource that was
+ successfully uploaded to the datastore
+ :param dataset_dict: The dict represenstation of the dataset containing
+ the resource that was uploaded
+ """
+ pass
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/logic/__init__.py b/src/ckanext-d4science_theme/ckanext/datapusher/logic/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py b/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py
new file mode 100644
index 0000000..1c172e7
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py
@@ -0,0 +1,338 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from ckan.types import Context
+import logging
+import json
+import datetime
+import time
+from typing import Any, cast
+
+from urllib.parse import urljoin
+from dateutil.parser import parse as parse_date
+
+import requests
+
+import ckan.lib.helpers as h
+import ckan.lib.navl.dictization_functions
+import ckan.logic as logic
+import ckan.plugins as p
+from ckan.common import config
+import ckanext.datapusher.logic.schema as dpschema
+import ckanext.datapusher.interfaces as interfaces
+
+log = logging.getLogger(__name__)
+_get_or_bust = logic.get_or_bust
+_validate = ckan.lib.navl.dictization_functions.validate
+
+
+def datapusher_submit(context: Context, data_dict: dict[str, Any]):
+ ''' Submit a job to the datapusher. The datapusher is a service that
+ imports tabular data into the datastore.
+
+ :param resource_id: The resource id of the resource that the data
+ should be imported in. The resource's URL will be used to get the data.
+ :type resource_id: string
+ :param set_url_type: If set to True, the ``url_type`` of the resource will
+ be set to ``datastore`` and the resource URL will automatically point
+ to the :ref:`datastore dump ` URL. (optional, default: False)
+ :type set_url_type: bool
+ :param ignore_hash: If set to True, the datapusher will reload the file
+ even if it haven't changed. (optional, default: False)
+ :type ignore_hash: bool
+
+ Returns ``True`` if the job has been submitted and ``False`` if the job
+ has not been submitted, i.e. when the datapusher is not configured.
+
+ :rtype: bool
+ '''
+ schema = context.get('schema', dpschema.datapusher_submit_schema())
+ data_dict, errors = _validate(data_dict, schema, context)
+ if errors:
+ raise p.toolkit.ValidationError(errors)
+
+ res_id = data_dict['resource_id']
+
+ p.toolkit.check_access('datapusher_submit', context, data_dict)
+
+ try:
+ resource_dict = p.toolkit.get_action('resource_show')(context, {
+ 'id': res_id,
+ })
+ except logic.NotFound:
+ return False
+
+ datapusher_url: str = config.get('ckan.datapusher.url')
+
+ callback_url_base = config.get(
+ 'ckan.datapusher.callback_url_base'
+ ) or config.get("ckan.site_url")
+ if callback_url_base:
+ site_url = callback_url_base
+ callback_url = urljoin(
+ callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook')
+ else:
+ site_url = h.url_for('home.index', qualified=True)
+ callback_url = h.url_for(
+ '/api/3/action/datapusher_hook', qualified=True)
+
+ for plugin in p.PluginImplementations(interfaces.IDataPusher):
+ upload = plugin.can_upload(res_id)
+ if not upload:
+ msg = "Plugin {0} rejected resource {1}"\
+ .format(plugin.__class__.__name__, res_id)
+ log.info(msg)
+ return False
+
+ task = {
+ 'entity_id': res_id,
+ 'entity_type': 'resource',
+ 'task_type': 'datapusher',
+ 'last_updated': str(datetime.datetime.utcnow()),
+ 'state': 'submitting',
+ 'key': 'datapusher',
+ 'value': '{}',
+ 'error': '{}',
+ }
+ try:
+ existing_task = p.toolkit.get_action('task_status_show')(context, {
+ 'entity_id': res_id,
+ 'task_type': 'datapusher',
+ 'key': 'datapusher'
+ })
+ assume_task_stale_after = datetime.timedelta(
+ seconds=config.get(
+ 'ckan.datapusher.assume_task_stale_after'))
+ if existing_task.get('state') == 'pending':
+ updated = datetime.datetime.strptime(
+ existing_task['last_updated'], '%Y-%m-%dT%H:%M:%S.%f')
+ time_since_last_updated = datetime.datetime.utcnow() - updated
+ if time_since_last_updated > assume_task_stale_after:
+ # it's been a while since the job was last updated - it's more
+ # likely something went wrong with it and the state wasn't
+ # updated than its still in progress. Let it be restarted.
+ log.info('A pending task was found %r, but it is only %s hours'
+ ' old', existing_task['id'], time_since_last_updated)
+ else:
+ log.info('A pending task was found %s for this resource, so '
+ 'skipping this duplicate task', existing_task['id'])
+ return False
+
+ task['id'] = existing_task['id']
+ except logic.NotFound:
+ pass
+
+ context['ignore_auth'] = True
+ # Use local session for task_status_update, so it can commit its own
+ # results without messing up with the parent session that contains pending
+ # updats of dataset/resource/etc.
+ context['session'] = cast(
+ Any, context['model'].meta.create_local_session())
+ p.toolkit.get_action('task_status_update')(context, task)
+
+ timeout = config.get('ckan.requests.timeout')
+
+ # This setting is checked on startup
+ api_token = p.toolkit.config.get("ckan.datapusher.api_token")
+ try:
+ r = requests.post(
+ urljoin(datapusher_url, 'job'),
+ headers={
+ 'Content-Type': 'application/json'
+ },
+ timeout=timeout,
+ data=json.dumps({
+ 'api_key': api_token,
+ 'job_type': 'push_to_datastore',
+ 'result_url': callback_url,
+ 'metadata': {
+ 'ignore_hash': data_dict.get('ignore_hash', False),
+ 'ckan_url': site_url,
+ 'resource_id': res_id,
+ 'set_url_type': data_dict.get('set_url_type', False),
+ 'task_created': task['last_updated'],
+ 'original_url': resource_dict.get('url'),
+ }
+ }))
+ except requests.exceptions.ConnectionError as e:
+ error: dict[str, Any] = {'message': 'Could not connect to DataPusher.',
+ 'details': str(e)}
+ task['error'] = json.dumps(error)
+ task['state'] = 'error'
+ task['last_updated'] = str(datetime.datetime.utcnow()),
+ p.toolkit.get_action('task_status_update')(context, task)
+ raise p.toolkit.ValidationError(error)
+ try:
+ r.raise_for_status()
+ except requests.exceptions.HTTPError as e:
+ m = 'An Error occurred while sending the job: {0}'.format(str(e))
+ try:
+ body = e.response.json()
+ if body.get('error'):
+ m += ' ' + body['error']
+ except ValueError:
+ body = e.response.text
+ error = {'message': m,
+ 'details': body,
+ 'status_code': r.status_code}
+ task['error'] = json.dumps(error)
+ task['state'] = 'error'
+ task['last_updated'] = str(datetime.datetime.utcnow()),
+ p.toolkit.get_action('task_status_update')(context, task)
+ raise p.toolkit.ValidationError(error)
+
+ value = json.dumps({'job_id': r.json()['job_id'],
+ 'job_key': r.json()['job_key']})
+
+ task['value'] = value
+ task['state'] = 'pending'
+ task['last_updated'] = str(datetime.datetime.utcnow()),
+ p.toolkit.get_action('task_status_update')(context, task)
+
+ return True
+
+
+def datapusher_hook(context: Context, data_dict: dict[str, Any]):
+ ''' Update datapusher task. This action is typically called by the
+ datapusher whenever the status of a job changes.
+
+ :param metadata: metadata produced by datapuser service must have
+ resource_id property.
+ :type metadata: dict
+ :param status: status of the job from the datapusher service
+ :type status: string
+ '''
+
+ metadata, status = _get_or_bust(data_dict, ['metadata', 'status'])
+
+ res_id = _get_or_bust(metadata, 'resource_id')
+
+ # Pass metadata, not data_dict, as it contains the resource id needed
+ # on the auth checks
+ p.toolkit.check_access('datapusher_submit', context, metadata)
+
+ task = p.toolkit.get_action('task_status_show')(context, {
+ 'entity_id': res_id,
+ 'task_type': 'datapusher',
+ 'key': 'datapusher'
+ })
+
+ task['state'] = status
+ task['last_updated'] = str(datetime.datetime.utcnow())
+
+ resubmit = False
+
+ if status == 'complete':
+ # Create default views for resource if necessary (only the ones that
+ # require data to be in the DataStore)
+ resource_dict = p.toolkit.get_action('resource_show')(
+ context, {'id': res_id})
+
+ dataset_dict = p.toolkit.get_action('package_show')(
+ context, {'id': resource_dict['package_id']})
+
+ for plugin in p.PluginImplementations(interfaces.IDataPusher):
+ plugin.after_upload(cast("dict[str, Any]", context),
+ resource_dict, dataset_dict)
+
+ logic.get_action('resource_create_default_resource_views')(
+ context,
+ {
+ 'resource': resource_dict,
+ 'package': dataset_dict,
+ 'create_datastore_views': True,
+ })
+
+ # Check if the uploaded file has been modified in the meantime
+ if (resource_dict.get('last_modified') and
+ metadata.get('task_created')):
+ try:
+ last_modified_datetime = parse_date(
+ resource_dict['last_modified'])
+ task_created_datetime = parse_date(metadata['task_created'])
+ if last_modified_datetime > task_created_datetime:
+ log.debug('Uploaded file more recent: {0} > {1}'.format(
+ last_modified_datetime, task_created_datetime))
+ resubmit = True
+ except ValueError:
+ pass
+ # Check if the URL of the file has been modified in the meantime
+ elif (resource_dict.get('url') and
+ metadata.get('original_url') and
+ resource_dict['url'] != metadata['original_url']):
+ log.debug('URLs are different: {0} != {1}'.format(
+ resource_dict['url'], metadata['original_url']))
+ resubmit = True
+
+ context['ignore_auth'] = True
+ p.toolkit.get_action('task_status_update')(context, task)
+
+ if resubmit:
+ log.debug('Resource {0} has been modified, '
+ 'resubmitting to DataPusher'.format(res_id))
+ p.toolkit.get_action('datapusher_submit')(
+ context, {'resource_id': res_id})
+
+
+def datapusher_status(
+ context: Context, data_dict: dict[str, Any]) -> dict[str, Any]:
+ ''' Get the status of a datapusher job for a certain resource.
+
+ :param resource_id: The resource id of the resource that you want the
+ datapusher status for.
+ :type resource_id: string
+ '''
+
+ p.toolkit.check_access('datapusher_status', context, data_dict)
+
+ if 'id' in data_dict:
+ data_dict['resource_id'] = data_dict['id']
+ res_id = _get_or_bust(data_dict, 'resource_id')
+
+ task = p.toolkit.get_action('task_status_show')(context, {
+ 'entity_id': res_id,
+ 'task_type': 'datapusher',
+ 'key': 'datapusher'
+ })
+
+ datapusher_url = config.get('ckan.datapusher.url')
+ if not datapusher_url:
+ raise p.toolkit.ValidationError(
+ {'configuration': ['ckan.datapusher.url not in config file']})
+
+ value = json.loads(task['value'])
+ job_key = value.get('job_key')
+ job_id = value.get('job_id')
+ url = None
+ job_detail = None
+
+ if job_id:
+ url = urljoin(datapusher_url, 'job' + '/' + job_id)
+ try:
+ timeout = config.get('ckan.requests.timeout')
+ r = requests.get(url,
+ timeout=timeout,
+ headers={'Content-Type': 'application/json',
+ 'Authorization': job_key})
+ r.raise_for_status()
+ job_detail = r.json()
+ for log in job_detail['logs']:
+ if 'timestamp' in log:
+ date = time.strptime(
+ log['timestamp'], "%Y-%m-%dT%H:%M:%S.%f")
+ date = datetime.datetime.utcfromtimestamp(
+ time.mktime(date))
+ log['timestamp'] = date
+ except (requests.exceptions.ConnectionError,
+ requests.exceptions.HTTPError):
+ job_detail = {'error': 'cannot connect to datapusher'}
+
+ return {
+ 'status': task['state'],
+ 'job_id': job_id,
+ 'job_url': url,
+ 'last_updated': task['last_updated'],
+ 'job_key': job_key,
+ 'task_info': job_detail,
+ 'error': json.loads(task['error'])
+ }
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py b/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py
new file mode 100644
index 0000000..7d179ec
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py
@@ -0,0 +1,16 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from typing import Any
+from ckan.types import AuthResult, Context
+import ckanext.datastore.logic.auth as auth
+
+
+def datapusher_submit(
+ context: Context, data_dict: dict[str, Any]) -> AuthResult:
+ return auth.datastore_auth(context, data_dict)
+
+
+def datapusher_status(
+ context: Context, data_dict: dict[str, Any]) -> AuthResult:
+ return auth.datastore_auth(context, data_dict)
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py b/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py
new file mode 100644
index 0000000..4ab41d0
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py
@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+from ckan.types import Schema
+
+import ckan.plugins as p
+import ckanext.datastore.logic.schema as dsschema
+
+get_validator = p.toolkit.get_validator
+
+not_missing = get_validator('not_missing')
+not_empty = get_validator('not_empty')
+ignore_missing = get_validator('ignore_missing')
+empty = get_validator('empty')
+boolean_validator = get_validator('boolean_validator')
+unicode_safe = get_validator('unicode_safe')
+
+
+def datapusher_submit_schema() -> Schema:
+ schema = {
+ 'resource_id': [not_missing, not_empty, unicode_safe],
+ 'id': [ignore_missing],
+ 'set_url_type': [ignore_missing, boolean_validator],
+ 'ignore_hash': [ignore_missing, boolean_validator],
+ '__junk': [empty],
+ '__before': [dsschema.rename('id', 'resource_id')]
+ }
+ return schema
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py b/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py
new file mode 100644
index 0000000..c7cc476
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py
@@ -0,0 +1,163 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from ckan.common import CKANConfig
+from ckan.types import Action, AuthFunction, Context
+import logging
+from typing import Any, Callable, cast
+
+import ckan.model as model
+import ckan.plugins as p
+import ckanext.datapusher.views as views
+import ckanext.datapusher.helpers as helpers
+import ckanext.datapusher.logic.action as action
+import ckanext.datapusher.logic.auth as auth
+
+log = logging.getLogger(__name__)
+
+
+class DatastoreException(Exception):
+ pass
+
+
+@p.toolkit.blanket.config_declarations
+class DatapusherPlugin(p.SingletonPlugin):
+ p.implements(p.IConfigurer, inherit=True)
+ p.implements(p.IConfigurable, inherit=True)
+ p.implements(p.IActions)
+ p.implements(p.IAuthFunctions)
+ p.implements(p.IResourceUrlChange)
+ p.implements(p.IResourceController, inherit=True)
+ p.implements(p.ITemplateHelpers)
+ p.implements(p.IBlueprint)
+
+ legacy_mode = False
+ resource_show_action = None
+
+ def update_config(self, config: CKANConfig):
+ templates_base = config.get(u'ckan.base_templates_folder')
+ p.toolkit.add_template_directory(config, templates_base)
+ p.toolkit.add_public_directory(config, 'public')
+ p.toolkit.add_resource('assets', 'ckanext-datapusher')
+
+ def configure(self, config: CKANConfig):
+ self.config = config
+
+ for config_option in (
+ "ckan.site_url",
+ "ckan.datapusher.url",
+ "ckan.datapusher.api_token",
+ ):
+ if not config.get(config_option):
+ raise Exception(
+ u'Config option `{0}` must be set to use the DataPusher.'.
+ format(config_option)
+ )
+
+ # IResourceUrlChange
+
+ def notify(self, resource: model.Resource):
+ context = cast(Context, {
+ u'model': model,
+ u'ignore_auth': True,
+ })
+ resource_dict = p.toolkit.get_action(u'resource_show')(
+ context, {
+ u'id': resource.id,
+ }
+ )
+ self._submit_to_datapusher(resource_dict)
+
+ # IResourceController
+
+ def after_resource_create(
+ self, context: Context, resource_dict: dict[str, Any]):
+
+ self._submit_to_datapusher(resource_dict)
+
+ def after_update(
+ self, context: Context, resource_dict: dict[str, Any]):
+
+ self._submit_to_datapusher(resource_dict)
+
+ def _submit_to_datapusher(self, resource_dict: dict[str, Any]):
+ context = cast(Context, {
+ u'model': model,
+ u'ignore_auth': True,
+ u'defer_commit': True
+ })
+
+ resource_format = resource_dict.get('format')
+ supported_formats = p.toolkit.config.get(
+ 'ckan.datapusher.formats'
+ )
+
+ submit = (
+ resource_format
+ and resource_format.lower() in supported_formats
+ and resource_dict.get('url_type') != u'datapusher'
+ )
+
+ if not submit:
+ return
+
+ try:
+ task = p.toolkit.get_action(u'task_status_show')(
+ context, {
+ u'entity_id': resource_dict['id'],
+ u'task_type': u'datapusher',
+ u'key': u'datapusher'
+ }
+ )
+
+ if task.get(u'state') in (u'pending', u'submitting'):
+ # There already is a pending DataPusher submission,
+ # skip this one ...
+ log.debug(
+ u'Skipping DataPusher submission for '
+ u'resource {0}'.format(resource_dict['id'])
+ )
+ return
+ except p.toolkit.ObjectNotFound:
+ pass
+
+ try:
+ log.debug(
+ u'Submitting resource {0}'.format(resource_dict['id']) +
+ u' to DataPusher'
+ )
+ p.toolkit.get_action(u'datapusher_submit')(
+ context, {
+ u'resource_id': resource_dict['id']
+ }
+ )
+ except p.toolkit.ValidationError as e:
+ # If datapusher is offline want to catch error instead
+ # of raising otherwise resource save will fail with 500
+ log.critical(e)
+ pass
+
+ def get_actions(self) -> dict[str, Action]:
+ return {
+ u'datapusher_submit': action.datapusher_submit,
+ u'datapusher_hook': action.datapusher_hook,
+ u'datapusher_status': action.datapusher_status
+ }
+
+ def get_auth_functions(self) -> dict[str, AuthFunction]:
+ return {
+ u'datapusher_submit': auth.datapusher_submit,
+ u'datapusher_status': auth.datapusher_status
+ }
+
+ def get_helpers(self) -> dict[str, Callable[..., Any]]:
+ return {
+ u'datapusher_status': helpers.datapusher_status,
+ u'datapusher_status_description': helpers.
+ datapusher_status_description,
+ }
+
+ # IBlueprint
+
+ def get_blueprint(self):
+ return views.get_blueprints()
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/public/dotted.png b/src/ckanext-d4science_theme/ckanext/datapusher/public/dotted.png
new file mode 100644
index 0000000..fa0ae80
Binary files /dev/null and b/src/ckanext-d4science_theme/ckanext/datapusher/public/dotted.png differ
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html b/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html
new file mode 100644
index 0000000..293a9ce
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html
@@ -0,0 +1,88 @@
+{% extends "package/resource_edit_base.html" %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block primary_content_inner %}
+
+ {% set action = h.url_for('datapusher.resource_data', id=pkg.name, resource_id=res.id) %}
+ {% set show_table = true %}
+
+
+
+ {% if status.error and status.error.message %}
+ {% set show_table = false %}
+
+ {{ _('Upload error:') }} {{ status.error.message }}
+
+ {% elif status.task_info and status.task_info.error %}
+
+ {% if status.task_info.error is string %}
+ {# DataPusher < 0.0.3 #}
+
{{ _('Error:') }} {{ status.task_info.error }}
+ {% elif status.task_info.error is mapping %}
+
{{ _('Error:') }} {{ status.task_info.error.message }}
+ {% for error_key, error_value in status.task_info.error.items() %}
+ {% if error_key != "message" and error_value %}
+
+
{{ error_key }} :
+ {{ error_value }}
+ {% endif %}
+ {% endfor %}
+ {% elif status.task_info.error is iterable %}
+
{{ _('Error traceback:') }}
+
{{ ''.join(status.task_info.error) }}
+ {% endif %}
+
+
+
+
+ {{ _('Status') }}
+ {{ h.datapusher_status_description(status) }}
+
+
+ {{ _('Last updated') }}
+ {% if status.status %}
+ {{ h.time_ago_from_timestamp(status.last_updated) }} {{ _('Never') }}
+ {% endif %}
+
+
+
+ {% if status.status and status.task_info and show_table %}
+ {{ _('Upload Log') }}
+
+ {% endif %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html b/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html
new file mode 100644
index 0000000..7d08984
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html
@@ -0,0 +1,16 @@
+{% ckan_extends %}
+
+{% block scripts %}
+ {{ super() }}
+ {% asset 'ckanext-datapusher/datapusher' %}
+{% endblock scripts %}
+
+{% block styles %}
+ {{ super() }}
+ {% asset 'ckanext-datapusher/datapusher-css' %}
+{% endblock %}
+
+{% block inner_primary_nav %}
+ {{ super() }}
+ {{ h.build_nav_icon('datapusher.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html b/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html
new file mode 100644
index 0000000..3598e7b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html
@@ -0,0 +1,95 @@
+{% extends "package/resource_edit_base.html" %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block primary_content_inner %}
+
+ {% set action = h.url_for('datapusher.resource_data', id=pkg.name, resource_id=res.id) %}
+ {% set show_table = true %}
+
+
+
+ {% if status.error and status.error.message %}
+ {% set show_table = false %}
+
+ {{ _('Upload error:') }} {{ status.error.message }}
+
+ {% elif status.task_info and status.task_info.error %}
+
+ {% if status.task_info.error is string %}
+ {# DataPusher < 0.0.3 #}
+
{{ _('Error:') }} {{ status.task_info.error }}
+ {% elif status.task_info.error is mapping %}
+
{{ _('Error:') }} {{ status.task_info.error.message }}
+ {% for error_key, error_value in status.task_info.error.items() %}
+ {% if error_key != "message" and error_value %}
+
+
{{ error_key }} :
+ {{ error_value }}
+ {% endif %}
+ {% endfor %}
+ {% elif status.task_info.error is iterable %}
+
{{ _('Error traceback:') }}
+
{{ ''.join(status.task_info.error) }}
+ {% endif %}
+
+
+
+
+ {{ _('Status') }}
+ {{ h.datapusher_status_description(status) }}
+
+
+ {{ _('Last updated') }}
+ {% if status.status %}
+ {{ h.time_ago_from_timestamp(status.last_updated) }} {{ _('Never') }}
+ {% endif %}
+
+
+
+ {% if status.status and status.task_info and show_table %}
+ {{ _('Upload Log') }}
+
+ {% for item in status.task_info.logs|sort(attribute='timestamp') %}
+ {% set icon = 'check' if item.level == 'INFO' else 'exclamation' %}
+ {% set class = ' failure' if icon == 'exclamation' else ' success' %}
+ {% set popover_content = 'test' %}
+
+
+
+ {% for line in item.message.strip().split('\n') %}
+ {{ line | urlize }}
+ {{ h.time_ago_from_timestamp(item.timestamp) }}
+
+ {{ _('Details') }}
+
+
+
+ {% endfor %}
+
+
+
+ {{ _('End of log') }}
+
+
+ {% endif %}
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html b/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html
new file mode 100644
index 0000000..7d08984
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html
@@ -0,0 +1,16 @@
+{% ckan_extends %}
+
+{% block scripts %}
+ {{ super() }}
+ {% asset 'ckanext-datapusher/datapusher' %}
+{% endblock scripts %}
+
+{% block styles %}
+ {{ super() }}
+ {% asset 'ckanext-datapusher/datapusher-css' %}
+{% endblock %}
+
+{% block inner_primary_nav %}
+ {{ super() }}
+ {{ h.build_nav_icon('datapusher.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py b/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py
new file mode 100644
index 0000000..737a9be
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py
@@ -0,0 +1,6 @@
+# encoding: utf-8
+from ckan.tests import factories
+
+
+def get_api_token() -> str:
+ return factories.SysadminWithToken()["sysadmin"]
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py
new file mode 100644
index 0000000..9f6397d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py
@@ -0,0 +1,322 @@
+# encoding: utf-8
+
+import datetime
+
+import json
+import pytest
+import responses
+
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.plugins as p
+from ckan.tests import factories
+from ckan.tests.helpers import call_action
+from ckan.common import config
+from ckanext.datastore.tests.helpers import set_url_type
+from ckanext.datapusher.tests import get_api_token
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore datapusher")
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures("with_plugins", "non_clean_db")
+class TestDatastoreNew:
+ def test_create_ckan_resource_in_package(self, app, api_token):
+ package = factories.Dataset.model()
+ data = {"resource": {"package_id": package.id}}
+ auth = {"Authorization": api_token["token"]}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=200,
+ )
+ res_dict = json.loads(res.body)
+
+ assert "resource_id" in res_dict["result"]
+ assert len(package.resources) == 1
+
+ res = call_action("resource_show",
+ id=res_dict["result"]["resource_id"])
+ assert res["url"].endswith("/datastore/dump/" + res["id"]), res
+
+ @responses.activate
+ def test_providing_res_with_url_calls_datapusher_correctly(self, app):
+ config["datapusher.url"] = "http://datapusher.ckan.org"
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "bar"}),
+ )
+ responses.add_passthru(config["solr_url"])
+
+ package = factories.Dataset.model()
+
+ call_action("datastore_create",
+ resource=dict(package_id=package.id, url="demo.ckan.org"))
+
+ assert len(package.resources) == 1
+ resource = package.resources[0]
+ data = json.loads(responses.calls[-1].request.body)
+ assert data["metadata"]["resource_id"] == resource.id, data
+ assert not data["metadata"].get("ignore_hash"), data
+ assert data["result_url"].endswith("/action/datapusher_hook"), data
+ assert data["result_url"].startswith("http://"), data
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore datapusher")
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures("with_plugins")
+class TestDatastoreCreate(object):
+ sysadmin_user = None
+ normal_user = None
+
+ @pytest.fixture(autouse=True)
+ def initial_data(self, clean_db):
+ ctd.CreateTestData.create()
+ self.sysadmin_user = factories.Sysadmin()
+ self.sysadmin_token = factories.APIToken(
+ user=self.sysadmin_user["id"])["token"]
+ self.normal_user = factories.User()
+ self.normal_user_token = factories.APIToken(
+ user=self.normal_user["id"])["token"]
+ set_url_type(
+ model.Package.get("annakarenina").resources, self.sysadmin_user
+ )
+
+ @responses.activate
+ def test_pass_the_received_ignore_hash_param_to_the_datapusher(self, app):
+ config["datapusher.url"] = "http://datapusher.ckan.org"
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "bar"}),
+ )
+
+ package = model.Package.get("annakarenina")
+ resource = package.resources[0]
+
+ call_action(
+ "datapusher_submit",
+ resource_id=resource.id,
+ ignore_hash=True,
+ )
+
+ data = json.loads(responses.calls[-1].request.body)
+ assert data["metadata"]["ignore_hash"], data
+
+ def test_cant_provide_resource_and_resource_id(self, app):
+ package = model.Package.get("annakarenina")
+ resource = package.resources[0]
+ data = {
+ "resource_id": resource.id,
+ "resource": {"package_id": package.id},
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.body)
+
+ assert res_dict["error"]["__type"] == "Validation Error"
+
+ @responses.activate
+ def test_send_datapusher_creates_task(self, test_request_context):
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "bar"}),
+ )
+
+ package = model.Package.get("annakarenina")
+ resource = package.resources[0]
+
+ context = {"ignore_auth": True, "user": self.sysadmin_user["name"]}
+ with test_request_context():
+ p.toolkit.get_action("datapusher_submit")(
+ context, {"resource_id": resource.id}
+ )
+
+ context.pop("task_status", None)
+
+ task = p.toolkit.get_action("task_status_show")(
+ context,
+ {
+ "entity_id": resource.id,
+ "task_type": "datapusher",
+ "key": "datapusher",
+ },
+ )
+
+ assert task["state"] == "pending", task
+
+ def _call_datapusher_hook(self, user, app):
+ package = model.Package.get("annakarenina")
+ resource = package.resources[0]
+
+ context = {"user": self.sysadmin_user["name"]}
+
+ p.toolkit.get_action("task_status_update")(
+ context,
+ {
+ "entity_id": resource.id,
+ "entity_type": "resource",
+ "task_type": "datapusher",
+ "key": "datapusher",
+ "value": '{"job_id": "my_id", "job_key":"my_key"}',
+ "last_updated": str(datetime.datetime.now()),
+ "state": "pending",
+ },
+ )
+
+ data = {"status": "success", "metadata": {"resource_id": resource.id}}
+
+ if user["sysadmin"]:
+ auth = {"Authorization": self.sysadmin_token}
+ else:
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datapusher_hook",
+ json=data,
+ extra_environ=auth,
+ status=200,
+ )
+ res_dict = json.loads(res.body)
+
+ assert res_dict["success"] is True
+
+ task = call_action(
+ "task_status_show",
+ entity_id=resource.id,
+ task_type="datapusher",
+ key="datapusher",
+ )
+
+ assert task["state"] == "success", task
+
+ task = call_action(
+ "task_status_show",
+ entity_id=resource.id,
+ task_type="datapusher",
+ key="datapusher",
+ )
+
+ assert task["state"] == "success", task
+
+ def test_datapusher_hook_sysadmin(self, app, test_request_context):
+ with test_request_context():
+ self._call_datapusher_hook(self.sysadmin_user, app)
+
+ def test_datapusher_hook_normal_user(self, app, test_request_context):
+ with test_request_context():
+ self._call_datapusher_hook(self.normal_user, app)
+
+ def test_datapusher_hook_no_metadata(self, app):
+ data = {"status": "success"}
+
+ app.post("/api/action/datapusher_hook", json=data, status=409)
+
+ def test_datapusher_hook_no_status(self, app):
+ data = {"metadata": {"resource_id": "res_id"}}
+
+ app.post("/api/action/datapusher_hook", json=data, status=409)
+
+ def test_datapusher_hook_no_resource_id_in_metadata(self, app):
+ data = {"status": "success", "metadata": {}}
+
+ app.post("/api/action/datapusher_hook", json=data, status=409)
+
+ @responses.activate
+ @pytest.mark.ckan_config(
+ "ckan.datapusher.callback_url_base", "https://ckan.example.com"
+ )
+ @pytest.mark.ckan_config(
+ "ckan.datapusher.url", "http://datapusher.ckan.org"
+ )
+ def test_custom_callback_url_base(self, app):
+
+ package = model.Package.get("annakarenina")
+ resource = package.resources[0]
+
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "barloco"}),
+ )
+ responses.add_passthru(config["solr_url"])
+
+ call_action(
+ "datapusher_submit",
+ resource_id=resource.id,
+ ignore_hash=True,
+ )
+
+ data = json.loads(responses.calls[-1].request.body)
+ assert (
+ data["result_url"]
+ == "https://ckan.example.com/api/3/action/datapusher_hook"
+ )
+
+ @responses.activate
+ @pytest.mark.ckan_config(
+ "ckan.datapusher.callback_url_base", "https://ckan.example.com"
+ )
+ @pytest.mark.ckan_config(
+ "ckan.datapusher.url", "http://datapusher.ckan.org"
+ )
+ def test_create_resource_hooks(self, app):
+
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "barloco"}),
+ )
+ responses.add_passthru(config["solr_url"])
+
+ dataset = factories.Dataset()
+ call_action(
+ "resource_create",
+ package_id=dataset['id'],
+ format='CSV',
+ )
+
+ @responses.activate
+ @pytest.mark.ckan_config(
+ "ckan.datapusher.callback_url_base", "https://ckan.example.com"
+ )
+ @pytest.mark.ckan_config(
+ "ckan.datapusher.url", "http://datapusher.ckan.org"
+ )
+ def test_update_resource_url_hooks(self, app):
+
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "barloco"}),
+ )
+ responses.add_passthru(config["solr_url"])
+
+ dataset = factories.Dataset()
+ resource = call_action(
+ "resource_create",
+ package_id=dataset['id'],
+ url='http://example.com/old.csv',
+ format='CSV',
+ )
+
+ resource = call_action(
+ "resource_update",
+ id=resource['id'],
+ url='http://example.com/new.csv',
+ format='CSV',
+ )
+ assert resource
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py
new file mode 100644
index 0000000..0ab187c
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py
@@ -0,0 +1,347 @@
+# encoding: utf-8
+
+import datetime
+
+import unittest.mock as mock
+import pytest
+from ckan.logic import _actions
+
+from ckan.tests import helpers, factories
+from ckanext.datapusher.tests import get_api_token
+
+
+def _pending_task(resource_id):
+ return {
+ "entity_id": resource_id,
+ "entity_type": "resource",
+ "task_type": "datapusher",
+ "last_updated": str(datetime.datetime.utcnow()),
+ "state": "pending",
+ "key": "datapusher",
+ "value": "{}",
+ "error": "{}",
+ }
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datapusher datastore")
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures("non_clean_db", "with_plugins")
+class TestSubmit:
+ def test_submit(self, monkeypatch):
+ """Auto-submit when creating a resource with supported format.
+
+ """
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+ func.assert_not_called()
+
+ helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ )
+
+ func.assert_called()
+
+ @pytest.mark.ckan_config("ckan.views.default_views", "")
+ @pytest.mark.flaky(reruns=2)
+ def test_submit_when_url_changes(self, monkeypatch):
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+
+ resource = helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.pdf",
+ )
+
+ func.assert_not_called()
+
+ helpers.call_action(
+ "resource_update",
+ {},
+ id=resource["id"],
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ )
+
+ func.assert_called()
+
+ def test_does_not_submit_while_ongoing_job(self, monkeypatch):
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+
+ resource = helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.CSV",
+ format="CSV",
+ )
+
+ func.assert_called()
+ func.reset_mock()
+ # Create a task with a state pending to mimic an ongoing job
+ # on the DataPusher
+ helpers.call_action(
+ "task_status_update", {}, **_pending_task(resource["id"])
+ )
+
+ # Update the resource
+ helpers.call_action(
+ "resource_update",
+ {},
+ id=resource["id"],
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ description="Test",
+ )
+ # Not called
+ func.assert_not_called()
+
+ def test_resubmits_if_url_changes_in_the_meantime(self, monkeypatch):
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+
+ resource = helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ )
+
+ func.assert_called()
+ func.reset_mock()
+ # Create a task with a state pending to mimic an ongoing job
+ # on the DataPusher
+ task = helpers.call_action(
+ "task_status_update", {}, **_pending_task(resource["id"])
+ )
+
+ # Update the resource, set a new URL
+ helpers.call_action(
+ "resource_update",
+ {},
+ id=resource["id"],
+ package_id=dataset["id"],
+ url="http://example.com/another.file.csv",
+ format="CSV",
+ )
+ # Not called
+ func.assert_not_called()
+
+ # Call datapusher_hook with state complete, to mock the DataPusher
+ # finishing the job and telling CKAN
+ data_dict = {
+ "metadata": {
+ "resource_id": resource["id"],
+ "original_url": "http://example.com/file.csv",
+ "task_created": task["last_updated"],
+ },
+ "status": "complete",
+ }
+
+ helpers.call_action("datapusher_hook", {}, **data_dict)
+
+ # datapusher_submit was called again
+ func.assert_called()
+
+ def test_resubmits_if_upload_changes_in_the_meantime(self, monkeypatch):
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+
+ resource = helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ )
+
+ func.assert_called()
+ func.reset_mock()
+ # Create a task with a state pending to mimic an ongoing job
+ # on the DataPusher
+ task = helpers.call_action(
+ "task_status_update", {}, **_pending_task(resource["id"])
+ )
+
+ # Update the resource, set a new last_modified (changes on file upload)
+ helpers.call_action(
+ "resource_update",
+ {},
+ id=resource["id"],
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ last_modified=datetime.datetime.utcnow().isoformat(),
+ )
+ # Not called
+ func.assert_not_called()
+
+ # Call datapusher_hook with state complete, to mock the DataPusher
+ # finishing the job and telling CKAN
+ data_dict = {
+ "metadata": {
+ "resource_id": resource["id"],
+ "original_url": "http://example.com/file.csv",
+ "task_created": task["last_updated"],
+ },
+ "status": "complete",
+ }
+ helpers.call_action("datapusher_hook", {}, **data_dict)
+
+ # datapusher_submit was called again
+ func.assert_called()
+
+ def test_does_not_resubmit_if_a_resource_field_changes_in_the_meantime(
+ self, monkeypatch
+ ):
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+
+ resource = helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ )
+
+ func.assert_called()
+ func.reset_mock()
+
+ # Create a task with a state pending to mimic an ongoing job
+ # on the DataPusher
+ task = helpers.call_action(
+ "task_status_update", {}, **_pending_task(resource["id"])
+ )
+
+ # Update the resource, set a new description
+ helpers.call_action(
+ "resource_update",
+ {},
+ id=resource["id"],
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ description="Test",
+ )
+ # Not called
+ func.assert_not_called()
+
+ # Call datapusher_hook with state complete, to mock the DataPusher
+ # finishing the job and telling CKAN
+ data_dict = {
+ "metadata": {
+ "resource_id": resource["id"],
+ "original_url": "http://example.com/file.csv",
+ "task_created": task["last_updated"],
+ },
+ "status": "complete",
+ }
+ helpers.call_action("datapusher_hook", {}, **data_dict)
+
+ # Not called
+ func.assert_not_called()
+
+ def test_does_not_resubmit_if_a_dataset_field_changes_in_the_meantime(
+ self, monkeypatch
+ ):
+ dataset = factories.Dataset()
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+
+ resource = helpers.call_action(
+ "resource_create",
+ {},
+ package_id=dataset["id"],
+ url="http://example.com/file.csv",
+ format="CSV",
+ )
+
+ func.assert_called()
+ func.reset_mock()
+ # Create a task with a state pending to mimic an ongoing job
+ # on the DataPusher
+ task = helpers.call_action(
+ "task_status_update", {}, **_pending_task(resource["id"])
+ )
+
+ # Update the parent dataset
+ helpers.call_action(
+ "package_update",
+ {},
+ id=dataset["id"],
+ notes="Test notes",
+ resources=[resource],
+ )
+ # Not called
+ func.assert_not_called()
+ # Call datapusher_hook with state complete, to mock the DataPusher
+ # finishing the job and telling CKAN
+ data_dict = {
+ "metadata": {
+ "resource_id": resource["id"],
+ "original_url": "http://example.com/file.csv",
+ "task_created": task["last_updated"],
+ },
+ "status": "complete",
+ }
+ helpers.call_action("datapusher_hook", {}, **data_dict)
+
+ # Not called
+ func.assert_not_called()
+
+ def test_duplicated_tasks(self, app):
+ def submit(res, user):
+ return helpers.call_action(
+ "datapusher_submit",
+ context=dict(user=user["name"]),
+ resource_id=res["id"],
+ )
+
+ user = factories.User()
+ res = factories.Resource(user=user)
+
+ with app.flask_app.test_request_context():
+ with mock.patch("requests.post") as r_mock:
+ r_mock().json = mock.Mock(
+ side_effect=lambda: dict.fromkeys(["job_id", "job_key"])
+ )
+ r_mock.reset_mock()
+ submit(res, user)
+ submit(res, user)
+
+ assert r_mock.call_count == 1
+
+ @pytest.mark.usefixtures("with_request_context")
+ def test_task_status_changes(self, create_with_upload):
+ """While updating task status, datapusher commits changes to database.
+
+ Make sure that changes to task_status won't close session that is still
+ used on higher levels of API for resource updates.
+
+ """
+ user = factories.User()
+ dataset = factories.Dataset()
+ context = {"user": user["name"]}
+ resource = create_with_upload(
+ "id,name\n1,2\n3,4", "file.csv", package_id=dataset["id"],
+ context=context)
+
+ create_with_upload(
+ "id,name\n1,2\n3,4", "file.csv", package_id=dataset["id"],
+ id=resource["id"], context=context, action="resource_update")
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py
new file mode 100644
index 0000000..2e7126c
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py
@@ -0,0 +1,102 @@
+# encoding: utf-8
+
+import datetime
+
+import pytest
+
+from ckan.tests import helpers, factories
+from ckanext.datapusher.tests import get_api_token
+
+
+@pytest.mark.ckan_config("ckan.views.default_views", "recline_grid_view")
+@pytest.mark.ckan_config(
+ "ckan.plugins", "datapusher datastore recline_grid_view"
+)
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures("non_clean_db", "with_plugins")
+def test_datapusher_creates_default_views_on_complete():
+
+ dataset = factories.Dataset()
+
+ resource = factories.Resource(package_id=dataset["id"])
+
+ # Push data directly to the DataStore for the resource to be marked as
+ # `datastore_active=True`, so the grid view can be created
+ data = {
+ "resource_id": resource["id"],
+ "fields": [{"id": "a", "type": "text"}, {"id": "b", "type": "text"}],
+ "records": [{"a": "1", "b": "2"}],
+ "force": True,
+ }
+ helpers.call_action("datastore_create", **data)
+
+ # Create a task for `datapusher_hook` to update
+ task_dict = {
+ "entity_id": resource["id"],
+ "entity_type": "resource",
+ "task_type": "datapusher",
+ "key": "datapusher",
+ "value": '{"job_id": "my_id", "job_key":"my_key"}',
+ "last_updated": str(datetime.datetime.now()),
+ "state": "pending",
+ }
+ helpers.call_action("task_status_update", context={}, **task_dict)
+
+ # Call datapusher_hook with a status of complete to trigger the
+ # default views creation
+ params = {
+ "status": "complete",
+ "metadata": {"resource_id": resource["id"]},
+ }
+
+ helpers.call_action("datapusher_hook", context={}, **params)
+
+ views = helpers.call_action("resource_view_list", id=resource["id"])
+
+ assert len(views) == 1
+ assert views[0]["view_type"] == "recline_grid_view"
+
+
+@pytest.mark.ckan_config("ckan.views.default_views", "recline_grid_view")
+@pytest.mark.ckan_config(
+ "ckan.plugins", "datapusher datastore recline_grid_view"
+)
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures("non_clean_db", "with_plugins")
+def test_datapusher_does_not_create_default_views_on_pending():
+
+ dataset = factories.Dataset()
+
+ resource = factories.Resource(package_id=dataset["id"])
+
+ # Push data directly to the DataStore for the resource to be marked as
+ # `datastore_active=True`, so the grid view can be created
+ data = {
+ "resource_id": resource["id"],
+ "fields": [{"id": "a", "type": "text"}, {"id": "b", "type": "text"}],
+ "records": [{"a": "1", "b": "2"}],
+ "force": True,
+ }
+ helpers.call_action("datastore_create", **data)
+
+ # Create a task for `datapusher_hook` to update
+ task_dict = {
+ "entity_id": resource["id"],
+ "entity_type": "resource",
+ "task_type": "datapusher",
+ "key": "datapusher",
+ "value": '{"job_id": "my_id", "job_key":"my_key"}',
+ "last_updated": str(datetime.datetime.now()),
+ "state": "pending",
+ }
+ helpers.call_action("task_status_update", context={}, **task_dict)
+
+ # Call datapusher_hook with a status of complete to trigger the
+ # default views creation
+ params = {"status": "pending", "metadata": {"resource_id": resource["id"]}}
+
+ helpers.call_action("datapusher_hook", context={}, **params)
+
+ views = helpers.call_action("resource_view_list", id=resource["id"])
+
+ assert len(views) == 0
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py
new file mode 100644
index 0000000..6f126b2
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py
@@ -0,0 +1,121 @@
+# encoding: utf-8
+
+import datetime
+
+import json
+import pytest
+import responses
+
+import ckan.plugins as p
+import ckanext.datapusher.interfaces as interfaces
+from ckanext.datapusher.tests import get_api_token
+
+from ckan.tests import helpers, factories
+
+
+class FakeDataPusherPlugin(p.SingletonPlugin):
+ p.implements(p.IConfigurable, inherit=True)
+ p.implements(interfaces.IDataPusher, inherit=True)
+
+ def configure(self, _config):
+ self.after_upload_calls = 0
+
+ def can_upload(self, resource_id):
+ return False
+
+ def after_upload(self, context, resource_dict, package_dict):
+ self.after_upload_calls += 1
+
+
+@pytest.mark.ckan_config(
+ "ckan.plugins", "datastore datapusher test_datapusher_plugin"
+)
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures("non_clean_db", "with_plugins")
+class TestInterace(object):
+
+ @responses.activate
+ @pytest.mark.parametrize("resource__url_type", ["datastore"])
+ def test_send_datapusher_creates_task(self, test_request_context, resource):
+ sysadmin = factories.Sysadmin()
+
+ responses.add(
+ responses.POST,
+ "http://datapusher.ckan.org/job",
+ content_type="application/json",
+ body=json.dumps({"job_id": "foo", "job_key": "bar"}),
+ )
+
+ context = {"ignore_auth": True, "user": sysadmin["name"]}
+ with test_request_context():
+ result = p.toolkit.get_action("datapusher_submit")(
+ context, {"resource_id": resource["id"]}
+ )
+ assert not result
+
+ context.pop("task_status", None)
+
+ with pytest.raises(p.toolkit.ObjectNotFound):
+ p.toolkit.get_action("task_status_show")(
+ context,
+ {
+ "entity_id": resource["id"],
+ "task_type": "datapusher",
+ "key": "datapusher",
+ },
+ )
+
+ def test_after_upload_called(self):
+ dataset = factories.Dataset()
+ resource = factories.Resource(package_id=dataset["id"])
+
+ # Push data directly to the DataStore for the resource to be marked as
+ # `datastore_active=True`, so the grid view can be created
+ data = {
+ "resource_id": resource["id"],
+ "fields": [
+ {"id": "a", "type": "text"},
+ {"id": "b", "type": "text"},
+ ],
+ "records": [{"a": "1", "b": "2"}],
+ "force": True,
+ }
+ helpers.call_action("datastore_create", **data)
+
+ # Create a task for `datapusher_hook` to update
+ task_dict = {
+ "entity_id": resource["id"],
+ "entity_type": "resource",
+ "task_type": "datapusher",
+ "key": "datapusher",
+ "value": '{"job_id": "my_id", "job_key":"my_key"}',
+ "last_updated": str(datetime.datetime.now()),
+ "state": "pending",
+ }
+ helpers.call_action("task_status_update", context={}, **task_dict)
+
+ # Call datapusher_hook with a status of complete to trigger the
+ # default views creation
+ params = {
+ "status": "complete",
+ "metadata": {"resource_id": resource["id"]},
+ }
+ helpers.call_action("datapusher_hook", context={}, **params)
+
+ total = sum(
+ plugin.after_upload_calls
+ for plugin in p.PluginImplementations(interfaces.IDataPusher)
+ )
+ assert total == 1, total
+
+ params = {
+ "status": "complete",
+ "metadata": {"resource_id": resource["id"]},
+ }
+ helpers.call_action("datapusher_hook", context={}, **params)
+
+ total = sum(
+ plugin.after_upload_calls
+ for plugin in p.PluginImplementations(interfaces.IDataPusher)
+ )
+ assert total == 2, total
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py
new file mode 100644
index 0000000..bba6d89
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py
@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+from unittest import mock
+import pytest
+
+import ckan.tests.factories as factories
+import ckan.model as model
+from ckan.logic import _actions
+from ckanext.datapusher.tests import get_api_token
+
+
+@mock.patch("flask_login.utils._get_user")
+@pytest.mark.ckan_config(u"ckan.plugins", u"datapusher datastore")
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures(u"non_clean_db", u"with_plugins", u"with_request_context")
+def test_resource_data(current_user, app, monkeypatch):
+ user = factories.User()
+ user_obj = model.User.get(user["name"])
+ # mock current_user
+ current_user.return_value = user_obj
+
+ dataset = factories.Dataset(creator_user_id=user["id"])
+ resource = factories.Resource(
+ package_id=dataset["id"], creator_user_id=user["id"]
+ )
+
+ url = u"/dataset/{id}/resource_data/{resource_id}".format(
+ id=str(dataset["name"]), resource_id=str(resource["id"])
+ )
+
+ func = mock.Mock()
+ monkeypatch.setitem(_actions, 'datapusher_submit', func)
+ app.post(url=url, status=200)
+ func.assert_called()
+ func.reset_mock()
+
+ app.get(url=url, status=200)
+ func.assert_not_called()
diff --git a/src/ckanext-d4science_theme/ckanext/datapusher/views.py b/src/ckanext-d4science_theme/ckanext/datapusher/views.py
new file mode 100644
index 0000000..650189b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datapusher/views.py
@@ -0,0 +1,75 @@
+# encoding: utf-8
+
+from flask import Blueprint
+from flask.views import MethodView
+
+import ckan.plugins.toolkit as toolkit
+import ckan.logic as logic
+import ckan.lib.helpers as core_helpers
+import ckan.lib.base as base
+
+from ckan.common import _
+
+datapusher = Blueprint(u'datapusher', __name__)
+
+
+def get_blueprints():
+ return [datapusher]
+
+
+class ResourceDataView(MethodView):
+
+ def post(self, id: str, resource_id: str):
+ try:
+ toolkit.get_action(u'datapusher_submit')(
+ {}, {
+ u'resource_id': resource_id
+ }
+ )
+ except logic.ValidationError:
+ pass
+
+ return core_helpers.redirect_to(
+ u'datapusher.resource_data', id=id, resource_id=resource_id
+ )
+
+ def get(self, id: str, resource_id: str):
+ try:
+ pkg_dict = toolkit.get_action(u'package_show')({}, {u'id': id})
+ resource = toolkit.get_action(u'resource_show'
+ )({}, {
+ u'id': resource_id
+ })
+
+ # backward compatibility with old templates
+ toolkit.g.pkg_dict = pkg_dict
+ toolkit.g.resource = resource
+
+ except (logic.NotFound, logic.NotAuthorized):
+ base.abort(404, _(u'Resource not found'))
+
+ try:
+ datapusher_status = toolkit.get_action(u'datapusher_status')(
+ {}, {
+ u'resource_id': resource_id
+ }
+ )
+ except logic.NotFound:
+ datapusher_status = {}
+ except logic.NotAuthorized:
+ base.abort(403, _(u'Not authorized to see this page'))
+
+ return base.render(
+ u'datapusher/resource_data.html',
+ extra_vars={
+ u'status': datapusher_status,
+ u'pkg_dict': pkg_dict,
+ u'resource': resource,
+ }
+ )
+
+
+datapusher.add_url_rule(
+ u'/dataset//resource_data/',
+ view_func=ResourceDataView.as_view(str(u'resource_data'))
+)
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/__init__.py b/src/ckanext-d4science_theme/ckanext/datastore/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt b/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt
new file mode 100644
index 0000000..dd3840f
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt
@@ -0,0 +1,283 @@
+abbrev
+abs
+abstime
+abstimeeq
+abstimege
+abstimegt
+abstimein
+abstimele
+abstimelt
+abstimene
+abstimeout
+abstimerecv
+abstimesend
+acos
+acosd
+age
+area
+array_agg
+array_append
+array_cat
+array_fill
+array_length
+array_lower
+array_ndims
+array_position
+array_positions
+array_prepend
+array_remove
+array_replace
+array_to_json
+array_to_string
+array_to_tsvector
+array_upper
+avg
+bit_and
+bit_or
+bool_and
+bool_or
+bound_box
+box
+broadcast
+btrim
+cardinality
+cbrt
+ceil
+ceiling
+center
+chr
+circle
+clock_timestamp
+concat
+concat_ws
+convert
+convert_from
+convert_to
+corr
+cos
+cosd
+cot
+cotd
+count
+covar_pop
+covar_samp
+cume_dist
+date_part
+date_pl_interval
+date_pli
+date_recv
+date_send
+date_smaller
+date_sortsupport
+date_trunc
+decode
+degrees
+dense_rank
+diagonal
+diameter
+div
+encode
+enum_cmp
+enum_eq
+enum_first
+enum_ge
+enum_gt
+enum_in
+enum_larger
+enum_last
+enum_le
+enum_lt
+enum_ne
+enum_range
+enum_smaller
+exp
+factorial
+family
+first_value
+floor
+format
+generate_series
+get_bit
+get_byte
+get_current_ts_config
+height
+host
+hostmask
+inet_merge
+inet_same_family
+initcap
+isclosed
+isempty
+isfinite
+isopen
+json_agg
+json_array_element
+json_array_element_text
+json_array_elements
+json_array_elements_text
+json_array_length
+json_build_array
+json_build_object
+json_each
+json_each_text
+json_extract_path
+json_extract_path_text
+json_object_agg
+json_object_keys
+json_populate_record
+json_populate_recordset
+jsonb_agg
+jsonb_array_element
+jsonb_array_element_text
+jsonb_array_elements
+jsonb_array_elements_text
+jsonb_array_length
+jsonb_build_array
+jsonb_build_object
+jsonb_extract_path
+jsonb_extract_path_text
+jsonb_insert
+jsonb_pretty
+jsonb_set
+jsonb_strip_nulls
+jsonb_typeof
+justify_days
+justify_hours
+justify_interval
+lag
+last_value
+lead
+left
+length
+line
+ln
+log
+lower
+lower_inc
+lower_inf
+lpad
+lseg
+ltrim
+macaddr8_set7bit
+make_date
+make_interval
+make_time
+make_timestamp
+make_timestamptz
+masklen
+max
+md5
+min
+mod
+mode
+network
+now
+npoints
+nth_value
+ntile
+num_nonnulls
+num_nulls
+numnode
+parse_ident
+path
+pclose
+percent_rank
+percentile_cont
+percentile_disc
+phraseto_tsquery
+pi
+plainto_tsquery
+point
+polygon
+popen
+pow
+power
+querytree
+quote_ident
+quote_literal
+quote_nullable
+radians
+radius
+random
+range_merge
+rank
+regexp_match
+regexp_matches
+regexp_replace
+regexp_split_to_array
+regexp_split_to_table
+regr_avgx
+regr_avgy
+regr_count
+regr_intercept
+regr_r2
+regr_slope
+regr_sxx
+regr_sxy
+regr_syy
+repeat
+replace
+reverse
+right
+round
+row_number
+row_to_json
+rpad
+rtrim
+scale
+set_bit
+set_byte
+set_masklen
+setweight
+sign
+sin
+sind
+slope
+split_part
+sqrt
+statement_timestamp
+stddev
+stddev_pop
+stddev_samp
+string_agg
+string_to_array
+strip
+strpos
+substr
+substring
+sum
+tan
+tand
+text
+to_ascii
+to_char
+to_date
+to_hex
+to_json
+to_jsonb
+to_timestamp
+to_tsquery
+to_tsvector
+transaction_timestamp
+translate
+trunc
+ts_delete
+ts_filter
+ts_headline
+ts_rank
+ts_rewrite
+tsvector_to_array
+unnest
+upper
+upper_inc
+upper_inf
+var_pop
+var_samp
+variance
+width
+width_bucket
+xmlagg
+xmlcomment
+xmlconcat
+xpath
+xpath_exists
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py b/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py
new file mode 100644
index 0000000..7710ad8
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py
@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+from ckan.types import Context
+import re
+import logging
+from typing import Any, Container
+
+import ckan.plugins as plugins
+from ckan.common import CKANConfig, config
+from ckanext.datastore.interfaces import IDatastoreBackend
+
+log = logging.getLogger(__name__)
+
+
+def get_all_resources_ids_in_datastore() -> list[str]:
+ """
+ Helper for getting id of all resources in datastore.
+
+ Uses `get_all_ids` of active datastore backend.
+ """
+ DatastoreBackend.register_backends()
+ DatastoreBackend.set_active_backend(config)
+ backend = DatastoreBackend.get_active_backend()
+ backend.configure(config)
+
+ return backend.get_all_ids()
+
+
+def _parse_sort_clause( # type: ignore
+ clause: str, fields_types: Container[str]):
+ clause_match = re.match(
+ u'^(.+?)( +(asc|desc))?( nulls +(first|last) *)?$', clause, re.I
+ )
+
+ if not clause_match:
+ return False
+
+ field = clause_match.group(1)
+ if field[0] == field[-1] == u'"':
+ field = field[1:-1]
+ sort = (clause_match.group(3) or u'asc').lower()
+ if clause_match.group(4):
+ sort += (clause_match.group(4)).lower()
+
+ if field not in fields_types:
+ return False
+
+ return field, sort
+
+
+class DatastoreException(Exception):
+ pass
+
+
+class InvalidDataError(Exception):
+ """Exception that's raised if you try to add invalid data to the datastore.
+
+ For example if you have a column with type "numeric" and then you try to
+ add a non-numeric value like "foo" to it, this exception should be raised.
+
+ """
+ pass
+
+
+class DatastoreBackend:
+ """Base class for all datastore backends.
+
+ Very simple example of implementation based on SQLite can be found in
+ `ckanext.example_idatastorebackend`. In order to use it, set
+ datastore.write_url to
+ 'example-sqlite:////tmp/database-name-on-your-choice'
+
+ :prop _backend: mapping(schema, class) of all registered backends
+ :type _backend: dictonary
+ :prop _active_backend: current active backend
+ :type _active_backend: DatastoreBackend
+ """
+
+ _backends = {}
+ _active_backend: "DatastoreBackend"
+
+ @classmethod
+ def register_backends(cls):
+ """Register all backend implementations inside extensions.
+ """
+ for plugin in plugins.PluginImplementations(IDatastoreBackend):
+ cls._backends.update(plugin.register_backends())
+
+ @classmethod
+ def set_active_backend(cls, config: CKANConfig):
+ """Choose most suitable backend depending on configuration
+
+ :param config: configuration object
+ :rtype: ckan.common.CKANConfig
+
+ """
+ schema = config.get(u'ckan.datastore.write_url').split(u':')[0]
+ read_schema = config.get(
+ u'ckan.datastore.read_url').split(u':')[0]
+ assert read_schema == schema, u'Read and write engines are different'
+ cls._active_backend = cls._backends[schema]()
+
+ @classmethod
+ def get_active_backend(cls):
+ """Return currently used backend
+ """
+ return cls._active_backend
+
+ def configure(self, config: CKANConfig):
+ """Configure backend, set inner variables, make some initial setup.
+
+ :param config: configuration object
+ :returns: config
+ :rtype: CKANConfig
+
+ """
+
+ return config
+
+ def create(self, context: Context, data_dict: dict[str, Any]) -> Any:
+ """Create new resourct inside datastore.
+
+ Called by `datastore_create`.
+
+ :param data_dict: See `ckanext.datastore.logic.action.datastore_create`
+ :returns: The newly created data object
+ :rtype: dictonary
+ """
+ raise NotImplementedError()
+
+ def upsert(self, context: Context, data_dict: dict[str, Any]) -> Any:
+ """Update or create resource depending on data_dict param.
+
+ Called by `datastore_upsert`.
+
+ :param data_dict: See `ckanext.datastore.logic.action.datastore_upsert`
+ :returns: The modified data object
+ :rtype: dictonary
+ """
+ raise NotImplementedError()
+
+ def delete(self, context: Context, data_dict: dict[str, Any]) -> Any:
+ """Remove resource from datastore.
+
+ Called by `datastore_delete`.
+
+ :param data_dict: See `ckanext.datastore.logic.action.datastore_delete`
+ :returns: Original filters sent.
+ :rtype: dictonary
+ """
+ raise NotImplementedError()
+
+ def search(self, context: Context, data_dict: dict[str, Any]) -> Any:
+ """Base search.
+
+ Called by `datastore_search`.
+
+ :param data_dict: See `ckanext.datastore.logic.action.datastore_search`
+ :rtype: dictonary with following keys
+
+ :param fields: fields/columns and their extra metadata
+ :type fields: list of dictionaries
+ :param offset: query offset value
+ :type offset: int
+ :param limit: query limit value
+ :type limit: int
+ :param filters: query filters
+ :type filters: list of dictionaries
+ :param total: number of total matching records
+ :type total: int
+ :param records: list of matching results
+ :type records: list of dictionaries
+
+ """
+ raise NotImplementedError()
+
+ def search_sql(self, context: Context, data_dict: dict[str, Any]) -> Any:
+ """Advanced search.
+
+ Called by `datastore_search_sql`.
+ :param sql: a single seach statement
+ :type sql: string
+
+ :rtype: dictonary
+ :param fields: fields/columns and their extra metadata
+ :type fields: list of dictionaries
+ :param records: list of matching results
+ :type records: list of dictionaries
+ """
+ raise NotImplementedError()
+
+ def resource_exists(self, id: str) -> bool:
+ """Define whether resource exists in datastore.
+ """
+ raise NotImplementedError()
+
+ def resource_fields(self, id: str) -> Any:
+ """Return dictonary with resource description.
+
+ Called by `datastore_info`.
+ :returns: A dictionary describing the columns and their types.
+ """
+ raise NotImplementedError()
+
+ def resource_info(self, id: str) -> Any:
+ """Return DataDictonary with resource's info - #3414
+ """
+ raise NotImplementedError()
+
+ def resource_id_from_alias(self, alias: str) -> Any:
+ """Convert resource's alias to real id.
+
+ :param alias: resource's alias or id
+ :type alias: string
+ :returns: real id of resource
+ :rtype: string
+
+ """
+ raise NotImplementedError()
+
+ def get_all_ids(self) -> list[str]:
+ """Return id of all resource registered in datastore.
+
+ :returns: all resources ids
+ :rtype: list of strings
+ """
+ raise NotImplementedError()
+
+ def create_function(self, *args: Any, **kwargs: Any) -> Any:
+ """Called by `datastore_function_create` action.
+ """
+ raise NotImplementedError()
+
+ def drop_function(self, *args: Any, **kwargs: Any) -> Any:
+ """Called by `datastore_function_delete` action.
+ """
+ raise NotImplementedError()
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py b/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py
new file mode 100644
index 0000000..aaa951d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py
@@ -0,0 +1,2323 @@
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+from typing_extensions import TypeAlias
+
+import sqlalchemy.exc
+from sqlalchemy.engine.base import Engine
+from ckan.types import Context, ErrorDict
+import copy
+import logging
+import sys
+from typing import (
+ Any, Callable, Container, Dict, Iterable, Optional, Set, Union,
+ cast)
+import sqlalchemy
+import os
+import pprint
+import sqlalchemy.engine.url as sa_url
+import datetime
+import hashlib
+import json
+from collections import OrderedDict
+
+import six
+from urllib.parse import (
+ urlencode, urlunparse, parse_qsl, urlparse
+)
+from io import StringIO
+
+import ckan.plugins as p
+import ckan.plugins.toolkit as toolkit
+from ckan.lib.lazyjson import LazyJSONObject
+
+import ckanext.datastore.helpers as datastore_helpers
+import ckanext.datastore.interfaces as interfaces
+
+from psycopg2.extras import register_default_json, register_composite
+import distutils.version
+from sqlalchemy.exc import (ProgrammingError, IntegrityError,
+ DBAPIError, DataError)
+
+import ckan.plugins as plugins
+from ckan.common import CKANConfig, config
+
+from ckanext.datastore.backend import (
+ DatastoreBackend,
+ DatastoreException,
+ _parse_sort_clause
+)
+from ckanext.datastore.backend import InvalidDataError
+
+log = logging.getLogger(__name__)
+
+_pg_types: dict[str, str] = {}
+_type_names: Set[str] = set()
+_engines: Dict[str, Engine] = {}
+
+_TIMEOUT = 60000 # milliseconds
+
+# See http://www.postgresql.org/docs/9.2/static/errcodes-appendix.html
+_PG_ERR_CODE = {
+ 'unique_violation': '23505',
+ 'query_canceled': '57014',
+ 'undefined_object': '42704',
+ 'syntax_error': '42601',
+ 'permission_denied': '42501',
+ 'duplicate_table': '42P07',
+ 'duplicate_alias': '42712',
+}
+
+_DATE_FORMATS = ['%Y-%m-%d',
+ '%Y-%m-%d %H:%M:%S',
+ '%Y-%m-%dT%H:%M:%S',
+ '%Y-%m-%dT%H:%M:%SZ',
+ '%d/%m/%Y',
+ '%m/%d/%Y',
+ '%d-%m-%Y',
+ '%m-%d-%Y']
+
+_INSERT = 'insert'
+_UPSERT = 'upsert'
+_UPDATE = 'update'
+
+
+if not os.environ.get('DATASTORE_LOAD'):
+ ValidationError = toolkit.ValidationError # type: ignore
+else:
+ log.warn("Running datastore without CKAN")
+
+ class ValidationError(Exception):
+ def __init__(self, error_dict: ErrorDict):
+ pprint.pprint(error_dict)
+
+is_single_statement = datastore_helpers.is_single_statement
+
+_engines = {}
+
+
+def literal_string(s: str):
+ """
+ Return s as a postgres literal string
+ """
+ return u"'" + s.replace(u"'", u"''").replace(u'\0', '') + u"'"
+
+
+def identifier(s: str):
+ """
+ Return s as a quoted postgres identifier
+ """
+ return u'"' + s.replace(u'"', u'""').replace(u'\0', '') + u'"'
+
+
+def get_read_engine():
+ return _get_engine_from_url(
+ config['ckan.datastore.read_url'],
+ isolation_level='READ_UNCOMMITTED')
+
+
+def get_write_engine():
+ return _get_engine_from_url(config['ckan.datastore.write_url'])
+
+
+def _get_engine_from_url(connection_url: str, **kwargs: Any) -> Engine:
+ '''Get either read or write engine.'''
+ engine = _engines.get(connection_url)
+ if not engine:
+ extras = {'url': connection_url}
+ config.setdefault('ckan.datastore.sqlalchemy.pool_pre_ping', True)
+ for key, value in kwargs.items():
+ config.setdefault(key, value)
+ engine = sqlalchemy.engine_from_config(config,
+ 'ckan.datastore.sqlalchemy.',
+ **extras)
+ _engines[connection_url] = engine
+
+ # don't automatically convert to python objects
+ # when using native json types in 9.2+
+ # http://initd.org/psycopg/docs/extras.html#adapt-json
+ _loads: Callable[[Any], Any] = lambda x: x
+ register_default_json(
+ conn_or_curs=engine.raw_connection().connection,
+ globally=False,
+ loads=_loads)
+
+ return engine
+
+
+def _dispose_engines():
+ '''Dispose all database engines.'''
+ global _engines
+ for _, engine in _engines.items():
+ engine.dispose()
+ _engines = {}
+
+
+def _pluck(field: str, arr: list[dict[str, Any]]):
+ return [x[field] for x in arr]
+
+
+def _rename_json_field(data_dict: dict[str, Any]):
+ '''Rename json type to a corresponding type for the datastore since
+ pre 9.2 postgres versions do not support native json'''
+ return _rename_field(data_dict, 'json', 'nested')
+
+
+def _unrename_json_field(data_dict: dict[str, Any]):
+ return _rename_field(data_dict, 'nested', 'json')
+
+
+def _rename_field(data_dict: dict[str, Any], term: Any, replace: Any):
+ fields = data_dict.get('fields', [])
+ for i, field in enumerate(fields):
+ if 'type' in field and field['type'] == term:
+ data_dict['fields'][i]['type'] = replace
+ return data_dict
+
+
+def _get_fields_types(
+ connection: Any, resource_id: str) -> 'OrderedDict[str, str]':
+ u'''
+ return a OrderedDict([(column_name, column_type)...]) for the passed
+ resource_id including '_id' but excluding other '_'-prefixed columns.
+ '''
+ all_fields = _get_fields(connection, resource_id)
+ all_fields.insert(0, {'id': '_id', 'type': 'int'})
+ field_types = OrderedDict([(f['id'], f['type']) for f in all_fields])
+ return field_types
+
+
+def _result_fields(fields_types: 'OrderedDict[str, str]',
+ field_info: dict[str, Any], fields: Optional[list[str]]
+ ) -> list[dict[str, Any]]:
+ u'''
+ return a list of field information based on the fields present,
+ passed and query passed.
+
+ :param fields_types: OrderedDict returned from _get_fields_types(..)
+ with rank column types added by plugins' datastore_search
+ :param field_info: dict returned from _get_field_info(..)
+ :param fields: list of field names passed to datastore_search
+ or None for all
+ '''
+ result_fields: list[dict[str, str]] = []
+
+ if fields is None:
+ fields = list(fields_types)
+
+ for field_id in fields:
+ f = {u'id': field_id, u'type': fields_types[field_id]}
+ if field_id in field_info:
+ f[u'info'] = field_info[f['id']]
+ result_fields.append(f)
+ return result_fields
+
+
+def _get_type(connection: Any, oid: str) -> str:
+ _cache_types(connection)
+ return _pg_types[oid]
+
+
+def _guess_type(field: Any):
+ '''Simple guess type of field, only allowed are
+ integer, numeric and text'''
+ data_types = set([int, float])
+ if isinstance(field, (dict, list)):
+ return 'nested'
+ if isinstance(field, int):
+ return 'int'
+ if isinstance(field, float):
+ return 'float'
+ for data_type in list(data_types):
+ try:
+ data_type(field)
+ except (TypeError, ValueError):
+ data_types.discard(data_type)
+ if not data_types:
+ break
+ if int in data_types:
+ return 'integer'
+ elif float in data_types:
+ return 'numeric'
+
+ # try iso dates
+ for format in _DATE_FORMATS:
+ try:
+ datetime.datetime.strptime(field, format)
+ return 'timestamp'
+ except (ValueError, TypeError):
+ continue
+ return 'text'
+
+
+def _get_unique_key(context: Context, data_dict: dict[str, Any]) -> list[str]:
+ sql_get_unique_key = '''
+ SELECT
+ a.attname AS column_names
+ FROM
+ pg_class t,
+ pg_index idx,
+ pg_attribute a
+ WHERE
+ t.oid = idx.indrelid
+ AND a.attrelid = t.oid
+ AND a.attnum = ANY(idx.indkey)
+ AND t.relkind = 'r'
+ AND idx.indisunique = true
+ AND idx.indisprimary = false
+ AND t.relname = %s
+ '''
+ key_parts = context['connection'].execute(sql_get_unique_key,
+ data_dict['resource_id'])
+ return [x[0] for x in key_parts]
+
+
+def _get_field_info(connection: Any, resource_id: str) -> dict[str, Any]:
+ u'''return a dictionary mapping column names to their info data,
+ when present'''
+ qtext = sqlalchemy.text(u'''
+ select pa.attname as name, pd.description as info
+ from pg_class pc, pg_attribute pa, pg_description pd
+ where pa.attrelid = pc.oid and pd.objoid = pc.oid
+ and pd.objsubid = pa.attnum and pc.relname = :res_id
+ and pa.attnum > 0
+ ''')
+ try:
+ return dict(
+ (n, json.loads(v)) for (n, v) in
+ connection.execute(qtext, res_id=resource_id).fetchall())
+ except ValueError: # don't die on non-json comments
+ return {}
+
+
+def _get_fields(connection: Any, resource_id: str):
+ u'''
+ return a list of {'id': column_name, 'type': column_type} dicts
+ for the passed resource_id, excluding '_'-prefixed columns.
+ '''
+ fields: list[dict[str, Any]] = []
+ all_fields = connection.execute(
+ u'SELECT * FROM "{0}" LIMIT 1'.format(resource_id)
+ )
+ for field in all_fields.cursor.description:
+ if not field[0].startswith('_'):
+ fields.append({
+ 'id': six.ensure_text(field[0]),
+ 'type': _get_type(connection, field[1])
+ })
+ return fields
+
+
+def _cache_types(connection: Any) -> None:
+ if not _pg_types:
+ results = connection.execute(
+ 'SELECT oid, typname FROM pg_type;'
+ )
+ for result in results:
+ _pg_types[result[0]] = result[1]
+ _type_names.add(result[1])
+ if 'nested' not in _type_names:
+ native_json = _pg_version_is_at_least(connection, '9.2')
+
+ log.info("Create nested type. Native JSON: {0!r}".format(
+ native_json))
+
+ backend = DatastorePostgresqlBackend.get_active_backend()
+ engine: Engine = backend._get_write_engine() # type: ignore
+ with cast(Any, engine.begin()) as write_connection:
+ write_connection.execute(
+ 'CREATE TYPE "nested" AS (json {0}, extra text)'.format(
+ 'json' if native_json else 'text'))
+ _pg_types.clear()
+
+ # redo cache types with json now available.
+ return _cache_types(connection)
+
+ register_composite('nested', connection.connection.connection, True)
+
+
+def _pg_version_is_at_least(connection: Any, version: Any):
+ try:
+ v = distutils.version.LooseVersion(version)
+ pg_version = connection.execute('select version();').fetchone()
+ pg_version_number = pg_version[0].split()[1]
+ pv = distutils.version.LooseVersion(pg_version_number)
+ return v <= pv
+ except ValueError:
+ return False
+
+
+def _is_array_type(field_type: str):
+ return field_type.startswith('_')
+
+
+def _validate_record(record: Any, num: int, field_names: Iterable[str]):
+ # check record for sanity
+ if not isinstance(record, dict):
+ raise ValidationError({
+ 'records': [u'row "{0}" is not a json object'.format(num)]
+ })
+ # check for extra fields in data
+ extra_keys: set[str] = set(record.keys()) - set(field_names)
+
+ if extra_keys:
+ raise ValidationError({
+ 'records': [u'row "{0}" has extra keys "{1}"'.format(
+ num + 1,
+ ', '.join(list(extra_keys))
+ )]
+ })
+
+
+def _where_clauses(
+ data_dict: dict[str, Any], fields_types: dict[str, Any]
+) -> list[Any]:
+ filters = data_dict.get('filters', {})
+ clauses: list[Any] = []
+
+ for field, value in filters.items():
+ if field not in fields_types:
+ continue
+ field_array_type = _is_array_type(fields_types[field])
+ # "%" needs to be escaped as "%%" in any query going to
+ # connection.execute, otherwise it will think the "%" is for
+ # substituting a bind parameter
+ field = field.replace('%', '%%')
+ if isinstance(value, list) and not field_array_type:
+ clause_str = (u'"{0}" in ({1})'.format(field,
+ ','.join(['%s'] * len(value))))
+ clause = (clause_str,) + tuple(value)
+ else:
+ clause: tuple[Any, ...] = (u'"{0}" = %s'.format(field), value)
+ clauses.append(clause)
+
+ # add full-text search where clause
+ q: Union[dict[str, str], str, Any] = data_dict.get('q')
+ full_text = data_dict.get('full_text')
+ if q and not full_text:
+ if isinstance(q, str):
+ ts_query_alias = _ts_query_alias()
+ clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
+ clauses.append((clause_str,))
+ elif isinstance(q, dict):
+ lang = _fts_lang(data_dict.get('language'))
+ for field, value in q.items():
+ if field not in fields_types:
+ continue
+ query_field = _ts_query_alias(field)
+
+ ftyp = fields_types[field]
+ if not datastore_helpers.should_fts_index_field_type(ftyp):
+ clause_str = u'_full_text @@ {0}'.format(query_field)
+ clauses.append((clause_str,))
+
+ clause_str = (
+ u'to_tsvector({0}, cast({1} as text)) @@ {2}').format(
+ literal_string(lang),
+ identifier(field),
+ query_field)
+ clauses.append((clause_str,))
+ elif (full_text and not q):
+ ts_query_alias = _ts_query_alias()
+ clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
+ clauses.append((clause_str,))
+
+ elif full_text and isinstance(q, dict):
+ ts_query_alias = _ts_query_alias()
+ clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
+ clauses.append((clause_str,))
+ # update clauses with q dict
+ _update_where_clauses_on_q_dict(data_dict, fields_types, q, clauses)
+
+ elif full_text and isinstance(q, str):
+ ts_query_alias = _ts_query_alias()
+ clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
+ clauses.append((clause_str,))
+
+ return clauses
+
+
+def _update_where_clauses_on_q_dict(
+ data_dict: dict[str, str], fields_types: dict[str, str],
+ q: dict[str, str], clauses: list[tuple[str]]) -> None:
+ lang = _fts_lang(data_dict.get('language'))
+ for field, _ in q.items():
+ if field not in fields_types:
+ continue
+ query_field = _ts_query_alias(field)
+
+ ftyp = fields_types[field]
+ if not datastore_helpers.should_fts_index_field_type(ftyp):
+ clause_str = u'_full_text @@ {0}'.format(query_field)
+ clauses.append((clause_str,))
+
+ clause_str = (
+ u'to_tsvector({0}, cast({1} as text)) @@ {2}').format(
+ literal_string(lang),
+ identifier(field),
+ query_field)
+ clauses.append((clause_str,))
+
+
+def _textsearch_query(
+ lang: str, q: Optional[Union[str, dict[str, str], Any]], plain: bool,
+ full_text: Optional[str]) -> tuple[str, dict[str, str]]:
+ u'''
+ :param lang: language for to_tsvector
+ :param q: string to search _full_text or dict to search columns
+ :param plain: True to use plainto_tsquery, False for to_tsquery
+ :param full_text: string to search _full_text
+
+ return (query, rank_columns) based on passed text/dict query
+ rank_columns is a {alias: statement} dict where alias is "rank" for
+ _full_text queries, and "rank " for column search
+ '''
+ if not (q or full_text):
+ return '', {}
+
+ statements: list[str] = []
+ rank_columns: dict[str, str] = {}
+ if q and not full_text:
+ if isinstance(q, str):
+ query, rank = _build_query_and_rank_statements(
+ lang, q, plain)
+ statements.append(query)
+ rank_columns[u'rank'] = rank
+ elif isinstance(q, dict):
+ for field, value in q.items():
+ query, rank = _build_query_and_rank_statements(
+ lang, value, plain, field)
+ statements.append(query)
+ rank_columns[u'rank ' + field] = rank
+ elif full_text and not q:
+ _update_rank_statements_and_columns(
+ statements, rank_columns, lang, full_text, plain
+ )
+ elif full_text and isinstance(q, dict):
+ _update_rank_statements_and_columns(
+ statements, rank_columns, lang, full_text, plain)
+ for field, value in q.items():
+ _update_rank_statements_and_columns(
+ statements, rank_columns, lang, value, plain, field
+ )
+ elif full_text and isinstance(q, str):
+ _update_rank_statements_and_columns(
+ statements, rank_columns, lang, full_text, plain
+ )
+
+ statements_str = ', ' + ', '.join(statements)
+ return statements_str, rank_columns
+
+
+def _update_rank_statements_and_columns(
+ statements: list[str], rank_columns: dict[str, str], lang: str,
+ query: str, plain: bool, field: Optional[str] = None):
+ query, rank = _build_query_and_rank_statements(
+ lang, query, plain, field)
+ statements.append(query)
+ if field:
+ rank_columns[u'rank ' + field] = rank
+ else:
+ rank_columns[u'rank'] = rank
+
+
+def _build_query_and_rank_statements(
+ lang: str, query: str, plain: bool, field: Optional[str] = None):
+ query_alias = _ts_query_alias(field)
+ lang_literal = literal_string(lang)
+ query_literal = literal_string(query)
+ if plain:
+ statement = u"plainto_tsquery({lang_literal}, {literal}) {alias}"
+ else:
+ statement = u"to_tsquery({lang_literal}, {literal}) {alias}"
+ statement = statement.format(
+ lang_literal=lang_literal,
+ literal=query_literal, alias=query_alias)
+ if field is None:
+ rank_field = u'_full_text'
+ else:
+ rank_field = u'to_tsvector({0}, cast({1} as text))'.format(
+ lang_literal, identifier(field))
+ rank_statement = u'ts_rank({0}, {1}, 32)'.format(
+ rank_field, query_alias)
+ return statement, rank_statement
+
+
+def _fts_lang(lang: Optional[str] = None) -> str:
+ return lang or config.get('ckan.datastore.default_fts_lang')
+
+
+def _sort(sort: Union[None, str, list[str]], fields_types: Container[str],
+ rank_columns: dict[str, Any]) -> list[str]:
+ u'''
+ :param sort: string or list sort parameter passed to datastore_search,
+ use None if not given
+ :param fields_types: OrderedDict returned from _get_fields_types(..)
+ :param rank_columns: rank_columns returned from _ts_query(..)
+
+ returns sort expression as a string. When sort is None use rank_columns
+ to order by best text search match
+ '''
+ if not sort:
+ rank_sorting: list[str] = []
+ for column in rank_columns.values():
+ rank_sorting.append(u'{0} DESC'.format(column))
+ return rank_sorting
+
+ clauses = datastore_helpers.get_list(sort, False) or []
+
+ clause_parsed: list[str] = []
+
+ for clause in clauses:
+ parsed = _parse_sort_clause(clause, fields_types)
+ if parsed:
+ field, sort = parsed
+ clause_parsed.append(
+ u'{0} {1}'.format(identifier(field), sort))
+ return clause_parsed
+
+
+def _ts_query_alias(field: Optional[str] = None):
+ query_alias = u'query'
+ if field:
+ query_alias += u' ' + field
+ return identifier(query_alias)
+
+
+def _get_aliases(context: Context, data_dict: dict[str, Any]):
+ '''Get a list of aliases for a resource.'''
+ res_id = data_dict['resource_id']
+ alias_sql = sqlalchemy.text(
+ u'SELECT name FROM "_table_metadata" WHERE alias_of = :id')
+ results = context['connection'].execute(alias_sql, id=res_id).fetchall()
+ return [x[0] for x in results]
+
+
+def _get_resources(context: Context, alias: str):
+ '''Get a list of resources for an alias. There could be more than one alias
+ in a resource_dict.'''
+ alias_sql = sqlalchemy.text(
+ u'''SELECT alias_of FROM "_table_metadata"
+ WHERE name = :alias AND alias_of IS NOT NULL''')
+ results = context['connection'].execute(alias_sql, alias=alias).fetchall()
+ return [x[0] for x in results]
+
+
+def create_alias(context: Context, data_dict: dict[str, Any]):
+ values: Optional[str] = data_dict.get('aliases')
+ aliases: Any = datastore_helpers.get_list(values)
+ alias = None
+ if aliases is not None:
+ # delete previous aliases
+ previous_aliases = _get_aliases(context, data_dict)
+ for alias in previous_aliases:
+ sql_alias_drop_string = u'DROP VIEW "{0}"'.format(alias)
+ context['connection'].execute(sql_alias_drop_string)
+
+ try:
+ for alias in aliases:
+ sql_alias_string = u'''CREATE VIEW "{alias}"
+ AS SELECT * FROM "{main}"'''.format(
+ alias=alias,
+ main=data_dict['resource_id']
+ )
+
+ res_ids = _get_resources(context, alias)
+ if res_ids:
+ raise ValidationError({
+ 'alias': [(u'The alias "{0}" already exists.').format(
+ alias)]
+ })
+
+ context['connection'].execute(sql_alias_string)
+ except DBAPIError as e:
+ if e.orig.pgcode in [_PG_ERR_CODE['duplicate_table'],
+ _PG_ERR_CODE['duplicate_alias']]:
+ raise ValidationError({
+ 'alias': [u'"{0}" already exists'.format(alias)]
+ })
+
+
+def _generate_index_name(resource_id: str, field: str):
+ value = (resource_id + field).encode('utf-8')
+ return hashlib.sha1(value).hexdigest()
+
+
+def _get_fts_index_method() -> str:
+ return config.get('ckan.datastore.default_fts_index_method')
+
+
+def _build_fts_indexes(
+ data_dict: dict[str, Any], # noqa
+ sql_index_str_method: str, fields: list[dict[str, Any]]):
+ fts_indexes: list[str] = []
+ resource_id = data_dict['resource_id']
+ fts_lang = data_dict.get(
+ 'language', config.get('ckan.datastore.default_fts_lang'))
+
+ # create full-text search indexes
+ def to_tsvector(x: str):
+ return u"to_tsvector('{0}', {1})".format(fts_lang, x)
+
+ def cast_as_text(x: str):
+ return u'cast("{0}" AS text)'.format(x)
+
+ full_text_field = {'type': 'tsvector', 'id': '_full_text'}
+ for field in [full_text_field] + fields:
+ if not datastore_helpers.should_fts_index_field_type(field['type']):
+ continue
+
+ field_str = field['id']
+ if field['type'] not in ['text', 'tsvector']:
+ field_str = cast_as_text(field_str)
+ else:
+ field_str = u'"{0}"'.format(field_str)
+ if field['type'] != 'tsvector':
+ field_str = to_tsvector(field_str)
+ fts_indexes.append(sql_index_str_method.format(
+ res_id=resource_id,
+ unique='',
+ name=_generate_index_name(resource_id, field_str),
+ method=_get_fts_index_method(), fields=field_str))
+
+ return fts_indexes
+
+
+def _drop_indexes(context: Context, data_dict: dict[str, Any],
+ unique: bool = False):
+ sql_drop_index = u'DROP INDEX "{0}" CASCADE'
+ sql_get_index_string = u"""
+ SELECT
+ i.relname AS index_name
+ FROM
+ pg_class t,
+ pg_class i,
+ pg_index idx
+ WHERE
+ t.oid = idx.indrelid
+ AND i.oid = idx.indexrelid
+ AND t.relkind = 'r'
+ AND idx.indisunique = {unique}
+ AND idx.indisprimary = false
+ AND t.relname = %s
+ """.format(unique='true' if unique else 'false')
+ indexes_to_drop = context['connection'].execute(
+ sql_get_index_string, data_dict['resource_id']).fetchall()
+ for index in indexes_to_drop:
+ context['connection'].execute(
+ sql_drop_index.format(index[0]).replace('%', '%%'))
+
+
+def _get_index_names(connection: Any, resource_id: str):
+ sql = u"""
+ SELECT
+ i.relname AS index_name
+ FROM
+ pg_class t,
+ pg_class i,
+ pg_index idx
+ WHERE
+ t.oid = idx.indrelid
+ AND i.oid = idx.indexrelid
+ AND t.relkind = 'r'
+ AND t.relname = %s
+ """
+ results = connection.execute(sql, resource_id).fetchall()
+ return [result[0] for result in results]
+
+
+def _is_valid_pg_type(context: Context, type_name: str):
+ if type_name in _type_names:
+ return True
+ else:
+ connection = context['connection']
+ try:
+ connection.execute('SELECT %s::regtype', type_name)
+ except ProgrammingError as e:
+ if e.orig.pgcode in [_PG_ERR_CODE['undefined_object'],
+ _PG_ERR_CODE['syntax_error']]:
+ return False
+ raise
+ else:
+ return True
+
+
+def _execute_single_statement(
+ context: Context, sql_string: str, where_values: Any):
+ if not datastore_helpers.is_single_statement(sql_string):
+ raise ValidationError({
+ 'query': ['Query is not a single statement.']
+ })
+
+ results = context['connection'].execute(sql_string, [where_values])
+
+ return results
+
+
+def _insert_links(data_dict: dict[str, Any], limit: int, offset: int):
+ '''Adds link to the next/prev part (same limit, offset=offset+limit)
+ and the resource page.'''
+ data_dict['_links'] = {}
+
+ # get the url from the request
+ try:
+ urlstring = toolkit.request.environ['CKAN_CURRENT_URL']
+ except (KeyError, TypeError, RuntimeError):
+ return # no links required for local actions
+
+ # change the offset in the url
+ parsed = list(urlparse(urlstring))
+ query = parsed[4]
+
+ arguments = dict(parse_qsl(query))
+ arguments_start = dict(arguments)
+ arguments_prev: dict[str, Any] = dict(arguments)
+ arguments_next: dict[str, Any] = dict(arguments)
+ if 'offset' in arguments_start:
+ arguments_start.pop('offset')
+ arguments_next['offset'] = int(offset) + int(limit)
+ arguments_prev['offset'] = int(offset) - int(limit)
+
+ parsed_start = parsed[:]
+ parsed_prev = parsed[:]
+ parsed_next = parsed[:]
+ parsed_start[4] = urlencode(arguments_start)
+ parsed_next[4] = urlencode(arguments_next)
+ parsed_prev[4] = urlencode(arguments_prev)
+
+ # add the links to the data dict
+ data_dict['_links']['start'] = urlunparse(parsed_start)
+ data_dict['_links']['next'] = urlunparse(parsed_next)
+ if int(offset) - int(limit) > 0:
+ data_dict['_links']['prev'] = urlunparse(parsed_prev)
+
+
+def _where(
+ where_clauses_and_values: list[tuple[Any, ...]]
+) -> tuple[str, list[Any]]:
+ '''Return a SQL WHERE clause from list with clauses and values
+
+ :param where_clauses_and_values: list of tuples with format
+ (where_clause, param1, ...)
+ :type where_clauses_and_values: list of tuples
+
+ :returns: SQL WHERE string with placeholders for the parameters, and list
+ of parameters
+ :rtype: string
+ '''
+ where_clauses = []
+ values: list[Any] = []
+
+ for clause_and_values in where_clauses_and_values:
+ where_clauses.append('(' + clause_and_values[0] + ')')
+ values += clause_and_values[1:]
+
+ where_clause = u' AND '.join(where_clauses)
+ if where_clause:
+ where_clause = u'WHERE ' + where_clause
+
+ return where_clause, values
+
+
+def convert(data: Any, type_name: str) -> Any:
+ if data is None:
+ return None
+ if type_name == 'nested':
+ return json.loads(data[0])
+ # array type
+ if type_name.startswith('_'):
+ sub_type = type_name[1:]
+ return [convert(item, sub_type) for item in data]
+ if type_name == 'tsvector':
+ return six.ensure_text(data)
+ if isinstance(data, datetime.datetime):
+ return data.isoformat()
+ if isinstance(data, (int, float)):
+ return data
+ return str(data)
+
+
+def check_fields(context: Context, fields: Iterable[dict[str, Any]]):
+ '''Check if field types are valid.'''
+ for field in fields:
+ if field.get('type') and not _is_valid_pg_type(context, field['type']):
+ raise ValidationError({
+ 'fields': [u'"{0}" is not a valid field type'.format(
+ field['type'])]
+ })
+ elif not datastore_helpers.is_valid_field_name(field['id']):
+ raise ValidationError({
+ 'fields': [u'"{0}" is not a valid field name'.format(
+ field['id'])]
+ })
+
+
+Indexes: TypeAlias = "Optional[list[Union[str, list[str]]]]"
+
+
+def create_indexes(context: Context, data_dict: dict[str, Any]):
+ connection = context['connection']
+
+ indexes: Indexes = cast(Indexes, datastore_helpers.get_list(
+ data_dict.get('indexes', None)))
+ # primary key is not a real primary key
+ # it's just a unique key
+ primary_key: Any = datastore_helpers.get_list(data_dict.get('primary_key'))
+
+ sql_index_tmpl = u'CREATE {unique} INDEX "{name}" ON "{res_id}"'
+ sql_index_string_method = sql_index_tmpl + u' USING {method}({fields})'
+ sql_index_string = sql_index_tmpl + u' ({fields})'
+ sql_index_strings: list[str] = []
+
+ fields = _get_fields(connection, data_dict['resource_id'])
+ field_ids = _pluck('id', fields)
+ json_fields = [x['id'] for x in fields if x['type'] == 'nested']
+
+ fts_indexes = _build_fts_indexes(data_dict,
+ sql_index_string_method,
+ fields)
+ sql_index_strings = sql_index_strings + fts_indexes
+
+ if indexes is not None:
+ _drop_indexes(context, data_dict, False)
+ else:
+ indexes = []
+
+ if primary_key is not None:
+ unique_keys = _get_unique_key(context, data_dict)
+ if sorted(unique_keys) != sorted(primary_key):
+ _drop_indexes(context, data_dict, True)
+ indexes.append(primary_key)
+
+ for index in indexes:
+ if not index:
+ continue
+
+ index_fields = datastore_helpers.get_list(index)
+ assert index_fields is not None
+ for field in index_fields:
+ if field not in field_ids:
+ raise ValidationError({
+ 'index': [
+ u'The field "{0}" is not a valid column name.'.format(
+ index)]
+ })
+ fields_string = u', '.join(
+ ['(("{0}").json::text)'.format(field)
+ if field in json_fields else
+ '"%s"' % field
+ for field in index_fields])
+ sql_index_strings.append(sql_index_string.format(
+ res_id=data_dict['resource_id'],
+ unique='unique' if index == primary_key else '',
+ name=_generate_index_name(data_dict['resource_id'], fields_string),
+ fields=fields_string))
+
+ sql_index_strings = [x.replace('%', '%%') for x in sql_index_strings]
+ current_indexes = _get_index_names(context['connection'],
+ data_dict['resource_id'])
+ for sql_index_string in sql_index_strings:
+ has_index = [c for c in current_indexes
+ if sql_index_string.find(c) != -1]
+ if not has_index:
+ connection.execute(sql_index_string)
+
+
+def create_table(context: Context, data_dict: dict[str, Any]):
+ '''Creates table, columns and column info (stored as comments).
+
+ :param resource_id: The resource ID (i.e. postgres table name)
+ :type resource_id: string
+ :param fields: details of each field/column, each with properties:
+ id - field/column name
+ type - optional, otherwise it is guessed from the first record
+ info - some field/column properties, saved as a JSON string in postgres
+ as a column comment. e.g. "type_override", "label", "notes"
+ :type fields: list of dicts
+ :param records: records, of which the first is used when a field type needs
+ guessing.
+ :type records: list of dicts
+ '''
+
+ datastore_fields = [
+ {'id': '_id', 'type': 'serial primary key'},
+ {'id': '_full_text', 'type': 'tsvector'},
+ ]
+
+ # check first row of data for additional fields
+ extra_fields = []
+ supplied_fields = data_dict.get('fields', [])
+ check_fields(context, supplied_fields)
+ field_ids = _pluck('id', supplied_fields)
+ records = data_dict.get('records')
+
+ fields_errors = []
+
+ for field_id in field_ids:
+ # Postgres has a limit of 63 characters for a column name
+ if len(field_id) > 63:
+ message = 'Column heading "{0}" exceeds limit of 63 '\
+ 'characters.'.format(field_id)
+ fields_errors.append(message)
+
+ if fields_errors:
+ raise ValidationError({
+ 'fields': fields_errors
+ })
+
+ # if type is field is not given try and guess or throw an error
+ for field in supplied_fields:
+ if 'type' not in field:
+ if not records or field['id'] not in records[0]:
+ raise ValidationError({
+ 'fields': [u'"{0}" type not guessable'.format(field['id'])]
+ })
+ field['type'] = _guess_type(records[0][field['id']])
+
+ # Check for duplicate fields
+ unique_fields = {f['id'] for f in supplied_fields}
+ if not len(unique_fields) == len(supplied_fields):
+ raise ValidationError({
+ 'field': ['Duplicate column names are not supported']
+ })
+
+ if records:
+ # check record for sanity
+ if not isinstance(records[0], dict):
+ raise ValidationError({
+ 'records': ['The first row is not a json object']
+ })
+ supplied_field_ids = records[0].keys()
+ for field_id in supplied_field_ids:
+ if field_id not in field_ids:
+ extra_fields.append({
+ 'id': field_id,
+ 'type': _guess_type(records[0][field_id])
+ })
+
+ fields = datastore_fields + supplied_fields + extra_fields
+ sql_fields = u", ".join([u'{0} {1}'.format(
+ identifier(f['id']), f['type']) for f in fields])
+
+ sql_string = u'CREATE TABLE {0} ({1});'.format(
+ identifier(data_dict['resource_id']),
+ sql_fields
+ )
+
+ info_sql = []
+ for f in supplied_fields:
+ info = f.get(u'info')
+ if isinstance(info, dict):
+ info_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format(
+ identifier(data_dict['resource_id']),
+ identifier(f['id']),
+ literal_string(
+ json.dumps(info, ensure_ascii=False))))
+
+ context['connection'].execute(
+ (sql_string + u';'.join(info_sql)).replace(u'%', u'%%'))
+
+
+def alter_table(context: Context, data_dict: dict[str, Any]):
+ '''Adds new columns and updates column info (stored as comments).
+
+ :param resource_id: The resource ID (i.e. postgres table name)
+ :type resource_id: string
+ :param fields: details of each field/column, each with properties:
+ id - field/column name
+ type - optional, otherwise it is guessed from the first record
+ info - some field/column properties, saved as a JSON string in postgres
+ as a column comment. e.g. "type_override", "label", "notes"
+ :type fields: list of dicts
+ :param records: records, of which the first is used when a field type needs
+ guessing.
+ :type records: list of dicts
+ '''
+ supplied_fields = data_dict.get('fields', [])
+ current_fields = _get_fields(
+ context['connection'], data_dict['resource_id'])
+ if not supplied_fields:
+ supplied_fields = current_fields
+ check_fields(context, supplied_fields)
+ field_ids = _pluck('id', supplied_fields)
+ records = data_dict.get('records')
+ new_fields: list[dict[str, Any]] = []
+
+ for num, field in enumerate(supplied_fields):
+ # check to see if field definition is the same or and
+ # extension of current fields
+ if num < len(current_fields):
+ if field['id'] != current_fields[num]['id']:
+ raise ValidationError({
+ 'fields': [(u'Supplied field "{0}" not '
+ u'present or in wrong order').format(
+ field['id'])]
+ })
+ # no need to check type as field already defined.
+ continue
+
+ if 'type' not in field:
+ if not records or field['id'] not in records[0]:
+ raise ValidationError({
+ 'fields': [u'"{0}" type not guessable'.format(field['id'])]
+ })
+ field['type'] = _guess_type(records[0][field['id']])
+ new_fields.append(field)
+
+ if records:
+ # check record for sanity as they have not been
+ # checked during validation
+ if not isinstance(records, list):
+ raise ValidationError({
+ 'records': ['Records has to be a list of dicts']
+ })
+ if not isinstance(records[0], dict):
+ raise ValidationError({
+ 'records': ['The first row is not a json object']
+ })
+ supplied_field_ids = cast(Dict[str, Any], records[0]).keys()
+ for field_id in supplied_field_ids:
+ if field_id not in field_ids:
+ new_fields.append({
+ 'id': field_id,
+ 'type': _guess_type(records[0][field_id])
+ })
+
+ alter_sql = []
+ for f in new_fields:
+ alter_sql.append(u'ALTER TABLE {0} ADD {1} {2};'.format(
+ identifier(data_dict['resource_id']),
+ identifier(f['id']),
+ f['type']))
+
+ for f in supplied_fields:
+ if u'info' in f:
+ info = f.get(u'info')
+ if isinstance(info, dict):
+ info_sql = literal_string(
+ json.dumps(info, ensure_ascii=False))
+ else:
+ info_sql = 'NULL'
+ alter_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format(
+ identifier(data_dict['resource_id']),
+ identifier(f['id']),
+ info_sql))
+
+ if alter_sql:
+ context['connection'].execute(
+ u';'.join(alter_sql).replace(u'%', u'%%'))
+
+
+def insert_data(context: Context, data_dict: dict[str, Any]):
+ """
+
+ :raises InvalidDataError: if there is an invalid value in the given data
+
+ """
+ data_dict['method'] = _INSERT
+ result = upsert_data(context, data_dict)
+ return result
+
+
+def upsert_data(context: Context, data_dict: dict[str, Any]):
+ '''insert all data from records'''
+ if not data_dict.get('records'):
+ return
+
+ method = data_dict.get('method', _UPSERT)
+
+ fields = _get_fields(context['connection'], data_dict['resource_id'])
+ field_names = _pluck('id', fields)
+ records = data_dict['records']
+ sql_columns = ", ".join(
+ identifier(name) for name in field_names)
+
+ if method == _INSERT:
+ rows = []
+ for num, record in enumerate(records):
+ _validate_record(record, num, field_names)
+
+ row = []
+ for field in fields:
+ value = record.get(field['id'])
+ if value is not None and field['type'].lower() == 'nested':
+ # a tuple with an empty second value
+ value = (json.dumps(value), '')
+ row.append(value)
+ rows.append(row)
+
+ sql_string = u'''INSERT INTO {res_id} ({columns})
+ VALUES ({values});'''.format(
+ res_id=identifier(data_dict['resource_id']),
+ columns=sql_columns.replace('%', '%%'),
+ values=', '.join(['%s' for _ in field_names])
+ )
+
+ try:
+ context['connection'].execute(sql_string, rows)
+ except sqlalchemy.exc.DataError as err:
+ raise InvalidDataError(
+ toolkit._("The data was invalid: {}"
+ ).format(_programming_error_summary(err)))
+ except sqlalchemy.exc.DatabaseError as err:
+ raise ValidationError(
+ {u'records': [_programming_error_summary(err)]})
+
+ elif method in [_UPDATE, _UPSERT]:
+ unique_keys = _get_unique_key(context, data_dict)
+
+ for num, record in enumerate(records):
+ if not unique_keys and '_id' not in record:
+ raise ValidationError({
+ 'table': [u'unique key must be passed for update/upsert']
+ })
+
+ elif '_id' not in record:
+ # all key columns have to be defined
+ missing_fields = [field for field in unique_keys
+ if field not in record]
+ if missing_fields:
+ raise ValidationError({
+ 'key': [u'''fields "{fields}" are missing
+ but needed as key'''.format(
+ fields=', '.join(missing_fields))]
+ })
+
+ for field in fields:
+ value = record.get(field['id'])
+ if value is not None and field['type'].lower() == 'nested':
+ # a tuple with an empty second value
+ record[field['id']] = (json.dumps(value), '')
+
+ non_existing_field_names = [
+ field for field in record
+ if field not in field_names and field != '_id'
+ ]
+ if non_existing_field_names:
+ raise ValidationError({
+ 'fields': [u'fields "{0}" do not exist'.format(
+ ', '.join(non_existing_field_names))]
+ })
+
+ if '_id' in record:
+ unique_values = [record['_id']]
+ pk_sql = '"_id"'
+ pk_values_sql = '%s'
+ else:
+ unique_values = [record[key] for key in unique_keys]
+ pk_sql = ','.join([identifier(part) for part in unique_keys])
+ pk_values_sql = ','.join(['%s'] * len(unique_keys))
+
+ used_fields = [field for field in fields
+ if field['id'] in record]
+
+ used_field_names = _pluck('id', used_fields)
+
+ used_values = [record[field] for field in used_field_names]
+
+ if method == _UPDATE:
+ sql_string = u'''
+ UPDATE {res_id}
+ SET ({columns}, "_full_text") = ({values}, NULL)
+ WHERE ({primary_key}) = ({primary_value});
+ '''.format(
+ res_id=identifier(data_dict['resource_id']),
+ columns=u', '.join(
+ [identifier(field)
+ for field in used_field_names]).replace('%', '%%'),
+ values=u', '.join(
+ ['%s' for _ in used_field_names]),
+ primary_key=pk_sql.replace('%', '%%'),
+ primary_value=pk_values_sql,
+ )
+ try:
+ results = context['connection'].execute(
+ sql_string, used_values + unique_values)
+ except sqlalchemy.exc.DatabaseError as err:
+ raise ValidationError({
+ u'records': [_programming_error_summary(err)],
+ u'_records_row': num})
+
+ # validate that exactly one row has been updated
+ if results.rowcount != 1:
+ raise ValidationError({
+ 'key': [u'key "{0}" not found'.format(unique_values)]
+ })
+
+ elif method == _UPSERT:
+ sql_string = u'''
+ UPDATE {res_id}
+ SET ({columns}, "_full_text") = ({values}, NULL)
+ WHERE ({primary_key}) = ({primary_value});
+ INSERT INTO {res_id} ({columns})
+ SELECT {values}
+ WHERE NOT EXISTS (SELECT 1 FROM {res_id}
+ WHERE ({primary_key}) = ({primary_value}));
+ '''.format(
+ res_id=identifier(data_dict['resource_id']),
+ columns=u', '.join(
+ [identifier(field)
+ for field in used_field_names]).replace('%', '%%'),
+ values=u', '.join(['%s::nested'
+ if field['type'] == 'nested' else '%s'
+ for field in used_fields]),
+ primary_key=pk_sql.replace('%', '%%'),
+ primary_value=pk_values_sql,
+ )
+ try:
+ context['connection'].execute(
+ sql_string,
+ (used_values + unique_values) * 2)
+ except sqlalchemy.exc.DatabaseError as err:
+ raise ValidationError({
+ u'records': [_programming_error_summary(err)],
+ u'_records_row': num})
+
+
+def validate(context: Context, data_dict: dict[str, Any]):
+ fields_types = _get_fields_types(
+ context['connection'], data_dict['resource_id'])
+ data_dict_copy = copy.deepcopy(data_dict)
+
+ # TODO: Convert all attributes that can be a comma-separated string to
+ # lists
+ if 'fields' in data_dict_copy:
+ fields = datastore_helpers.get_list(data_dict_copy['fields'])
+ data_dict_copy['fields'] = fields
+ if 'sort' in data_dict_copy:
+ fields = datastore_helpers.get_list(data_dict_copy['sort'], False)
+ data_dict_copy['sort'] = fields
+
+ for plugin in plugins.PluginImplementations(interfaces.IDatastore):
+ data_dict_copy = plugin.datastore_validate(context,
+ data_dict_copy,
+ fields_types)
+
+ # Remove default elements in data_dict
+ data_dict_copy.pop('connection_url', None)
+ data_dict_copy.pop('resource_id', None)
+
+ data_dict_copy.pop('id', None)
+ data_dict_copy.pop('include_total', None)
+ data_dict_copy.pop('total_estimation_threshold', None)
+ data_dict_copy.pop('records_format', None)
+ data_dict_copy.pop('calculate_record_count', None)
+
+ for key, values in data_dict_copy.items():
+ if not values:
+ continue
+ if isinstance(values, str):
+ value = values
+ elif isinstance(values, (list, tuple)):
+ value: Any = values[0]
+ elif isinstance(values, dict):
+ value: Any = list(values.keys())[0]
+ else:
+ value = values
+
+ raise ValidationError({
+ key: [u'invalid value "{0}"'.format(value)]
+ })
+
+ return True
+
+
+def search_data(context: Context, data_dict: dict[str, Any]):
+ validate(context, data_dict)
+ fields_types = _get_fields_types(
+ context['connection'], data_dict['resource_id'])
+
+ query_dict: dict[str, Any] = {
+ 'select': [],
+ 'sort': [],
+ 'where': []
+ }
+
+ for plugin in p.PluginImplementations(interfaces.IDatastore):
+ query_dict = plugin.datastore_search(context, data_dict,
+ fields_types, query_dict)
+
+ where_clause, where_values = _where(query_dict['where'])
+
+ # FIXME: Remove duplicates on select columns
+ select_columns = ', '.join(query_dict['select']).replace('%', '%%')
+ ts_query = cast(str, query_dict['ts_query']).replace('%', '%%')
+ resource_id = data_dict['resource_id'].replace('%', '%%')
+ sort = query_dict['sort']
+ limit = query_dict['limit']
+ offset = query_dict['offset']
+
+ if query_dict.get('distinct'):
+ distinct = 'DISTINCT'
+ else:
+ distinct = ''
+
+ if not sort and not distinct:
+ sort = ['_id']
+
+ if sort:
+ sort_clause = 'ORDER BY %s' % (', '.join(sort)).replace('%', '%%')
+ else:
+ sort_clause = ''
+
+ records_format = data_dict['records_format']
+ if records_format == u'objects':
+ sql_fmt = u'''
+ SELECT array_to_json(array_agg(j))::text FROM (
+ SELECT {distinct} {select}
+ FROM "{resource}" {ts_query}
+ {where} {sort} LIMIT {limit} OFFSET {offset}
+ ) AS j'''
+ elif records_format == u'lists':
+ select_columns = u" || ',' || ".join(
+ s for s in query_dict['select']
+ ).replace('%', '%%')
+ sql_fmt = u'''
+ SELECT '[' || array_to_string(array_agg(j.v), ',') || ']' FROM (
+ SELECT {distinct} '[' || {select} || ']' v
+ FROM (
+ SELECT * FROM "{resource}" {ts_query}
+ {where} {sort} LIMIT {limit} OFFSET {offset}) as z
+ ) AS j'''
+ elif records_format == u'csv':
+ sql_fmt = u'''
+ COPY (
+ SELECT {distinct} {select}
+ FROM "{resource}" {ts_query}
+ {where} {sort} LIMIT {limit} OFFSET {offset}
+ ) TO STDOUT csv DELIMITER ',' '''
+ elif records_format == u'tsv':
+ sql_fmt = u'''
+ COPY (
+ SELECT {distinct} {select}
+ FROM "{resource}" {ts_query}
+ {where} {sort} LIMIT {limit} OFFSET {offset}
+ ) TO STDOUT csv DELIMITER '\t' '''
+ else:
+ sql_fmt = u''
+ sql_string = sql_fmt.format(
+ distinct=distinct,
+ select=select_columns,
+ resource=resource_id,
+ ts_query=ts_query,
+ where=where_clause,
+ sort=sort_clause,
+ limit=limit,
+ offset=offset)
+ if records_format == u'csv' or records_format == u'tsv':
+ buf = StringIO()
+ _execute_single_statement_copy_to(
+ context, sql_string, where_values, buf)
+ records = buf.getvalue()
+ else:
+ v = list(_execute_single_statement(
+ context, sql_string, where_values))[0][0]
+ if v is None or v == '[]':
+ records = []
+ else:
+ records = LazyJSONObject(v)
+ data_dict['records'] = records
+
+ data_dict['fields'] = _result_fields(
+ fields_types,
+ _get_field_info(context['connection'], data_dict['resource_id']),
+ datastore_helpers.get_list(data_dict.get('fields')))
+
+ _unrename_json_field(data_dict)
+
+ _insert_links(data_dict, limit, offset)
+
+ if data_dict.get('include_total', True):
+ total_estimation_threshold = \
+ data_dict.get('total_estimation_threshold')
+ estimated_total = None
+ if total_estimation_threshold is not None and \
+ not (where_clause or distinct):
+ # there are no filters, so we can try to use the estimated table
+ # row count from pg stats
+ # See: https://wiki.postgresql.org/wiki/Count_estimate
+ # (We also tried using the EXPLAIN to estimate filtered queries but
+ # it didn't estimate well in tests)
+ analyze_count_sql = sqlalchemy.text('''
+ SELECT reltuples::BIGINT AS approximate_row_count
+ FROM pg_class
+ WHERE relname=:resource;
+ ''')
+ count_result = context['connection'].execute(analyze_count_sql,
+ resource=resource_id)
+ try:
+ estimated_total = count_result.fetchall()[0][0]
+ except ValueError:
+ # the table doesn't have the stats calculated yet. (This should
+ # be done by xloader/datapusher at the end of loading.)
+ # We could provoke their creation with an ANALYZE, but that
+ # takes 10x the time to run, compared to SELECT COUNT(*) so
+ # we'll just revert to the latter. At some point the autovacuum
+ # will run and create the stats so we can use an estimate in
+ # future.
+ pass
+
+ if estimated_total is not None \
+ and estimated_total >= total_estimation_threshold:
+ data_dict['total'] = estimated_total
+ data_dict['total_was_estimated'] = True
+ else:
+ # this is slow for large results
+ count_sql_string = u'''SELECT count(*) FROM (
+ SELECT {distinct} {select}
+ FROM "{resource}" {ts_query} {where}) as t;'''.format(
+ distinct=distinct,
+ select=select_columns,
+ resource=resource_id,
+ ts_query=ts_query,
+ where=where_clause)
+ count_result = _execute_single_statement(
+ context, count_sql_string, where_values)
+ data_dict['total'] = count_result.fetchall()[0][0]
+ data_dict['total_was_estimated'] = False
+
+ return data_dict
+
+
+def _execute_single_statement_copy_to(
+ context: Context, sql_string: str,
+ where_values: Any, buf: Any):
+ if not datastore_helpers.is_single_statement(sql_string):
+ raise ValidationError({
+ 'query': ['Query is not a single statement.']
+ })
+
+ cursor = context['connection'].connection.cursor()
+ cursor.copy_expert(cursor.mogrify(sql_string, where_values), buf)
+ cursor.close()
+
+
+def format_results(context: Context, results: Any, data_dict: dict[str, Any]):
+ result_fields: list[dict[str, Any]] = []
+ for field in results.cursor.description:
+ result_fields.append({
+ 'id': six.ensure_text(field[0]),
+ 'type': _get_type(context['connection'], field[1])
+ })
+
+ records = []
+ for row in results:
+ converted_row = {}
+ for field in result_fields:
+ converted_row[field['id']] = convert(row[field['id']],
+ field['type'])
+ records.append(converted_row)
+ data_dict['records'] = records
+ if data_dict.get('records_truncated', False):
+ data_dict['records'].pop()
+ data_dict['fields'] = result_fields
+
+ return _unrename_json_field(data_dict)
+
+
+def delete_data(context: Context, data_dict: dict[str, Any]):
+ validate(context, data_dict)
+ fields_types = _get_fields_types(
+ context['connection'], data_dict['resource_id'])
+
+ query_dict: dict[str, Any] = {
+ 'where': []
+ }
+
+ for plugin in plugins.PluginImplementations(interfaces.IDatastore):
+ query_dict = plugin.datastore_delete(context, data_dict,
+ fields_types, query_dict)
+
+ where_clause, where_values = _where(query_dict['where'])
+ sql_string = u'DELETE FROM "{0}" {1}'.format(
+ data_dict['resource_id'],
+ where_clause
+ )
+
+ _execute_single_statement(context, sql_string, where_values)
+
+
+def _create_triggers(connection: Any, resource_id: str,
+ triggers: Iterable[dict[str, Any]]):
+ u'''
+ Delete existing triggers on table then create triggers
+
+ Currently our schema requires "before insert or update"
+ triggers run on each row, so we're not reading "when"
+ or "for_each" parameters from triggers list.
+ '''
+ existing = connection.execute(
+ u"""SELECT tgname FROM pg_trigger
+ WHERE tgrelid = %s::regclass AND tgname LIKE 't___'""",
+ resource_id)
+ sql_list = (
+ [u'DROP TRIGGER {name} ON {table}'.format(
+ name=identifier(r[0]),
+ table=identifier(resource_id))
+ for r in existing] +
+ [u'''CREATE TRIGGER {name}
+ BEFORE INSERT OR UPDATE ON {table}
+ FOR EACH ROW EXECUTE PROCEDURE {function}()'''.format(
+ # 1000 triggers per table should be plenty
+ name=identifier(u't%03d' % i),
+ table=identifier(resource_id),
+ function=identifier(t['function']))
+ for i, t in enumerate(triggers)])
+ try:
+ if sql_list:
+ connection.execute(u';\n'.join(sql_list))
+ except ProgrammingError as pe:
+ raise ValidationError({u'triggers': [_programming_error_summary(pe)]})
+
+
+def _create_fulltext_trigger(connection: Any, resource_id: str):
+ connection.execute(
+ u'''CREATE TRIGGER zfulltext
+ BEFORE INSERT OR UPDATE ON {table}
+ FOR EACH ROW EXECUTE PROCEDURE populate_full_text_trigger()'''.format(
+ table=identifier(resource_id)))
+
+
+def upsert(context: Context, data_dict: dict[str, Any]):
+ '''
+ This method combines upsert insert and update on the datastore. The method
+ that will be used is defined in the mehtod variable.
+
+ Any error results in total failure! For now pass back the actual error.
+ Should be transactional.
+ '''
+ backend = DatastorePostgresqlBackend.get_active_backend()
+ engine = backend._get_write_engine() # type: ignore
+ context['connection'] = engine.connect()
+ timeout = context.get('query_timeout', _TIMEOUT)
+
+ trans: Any = context['connection'].begin()
+ try:
+ # check if table already existes
+ context['connection'].execute(
+ u'SET LOCAL statement_timeout TO {0}'.format(timeout))
+ upsert_data(context, data_dict)
+ if data_dict.get(u'dry_run', False):
+ trans.rollback()
+ else:
+ trans.commit()
+ return _unrename_json_field(data_dict)
+ except IntegrityError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['unique_violation']:
+ raise ValidationError(cast(ErrorDict, {
+ 'constraints': ['Cannot insert records or create index because'
+ ' of uniqueness constraint'],
+ 'info': {
+ 'orig': str(e.orig),
+ 'pgcode': e.orig.pgcode
+ }
+ }))
+ raise
+ except DataError as e:
+ raise ValidationError(cast(ErrorDict, {
+ 'data': str(e),
+ 'info': {
+ 'orig': [str(e.orig)]
+ }}))
+ except DBAPIError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['query_canceled']:
+ raise ValidationError({
+ 'query': ['Query took too long']
+ })
+ raise
+ except Exception:
+ trans.rollback()
+ raise
+ finally:
+ context['connection'].close()
+
+
+def search(context: Context, data_dict: dict[str, Any]):
+ backend = DatastorePostgresqlBackend.get_active_backend()
+ engine = backend._get_read_engine() # type: ignore
+ context['connection'] = engine.connect()
+ timeout = context.get('query_timeout', _TIMEOUT)
+ _cache_types(context['connection'])
+
+ try:
+ context['connection'].execute(
+ u'SET LOCAL statement_timeout TO {0}'.format(timeout))
+ return search_data(context, data_dict)
+ except DBAPIError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['query_canceled']:
+ raise ValidationError({
+ 'query': ['Search took too long']
+ })
+ raise ValidationError(cast(ErrorDict, {
+ 'query': ['Invalid query'],
+ 'info': {
+ 'statement': [e.statement],
+ 'params': [e.params],
+ 'orig': [str(e.orig)]
+ }
+ }))
+ finally:
+ context['connection'].close()
+
+
+def search_sql(context: Context, data_dict: dict[str, Any]):
+ backend = DatastorePostgresqlBackend.get_active_backend()
+ engine = backend._get_read_engine() # type: ignore
+
+ context['connection'] = engine.connect()
+ timeout = context.get('query_timeout', _TIMEOUT)
+ _cache_types(context['connection'])
+
+ sql = data_dict['sql'].replace('%', '%%')
+
+ # limit the number of results to ckan.datastore.search.rows_max + 1
+ # (the +1 is so that we know if the results went over the limit or not)
+ rows_max = config.get('ckan.datastore.search.rows_max')
+ sql = 'SELECT * FROM ({0}) AS blah LIMIT {1} ;'.format(sql, rows_max + 1)
+
+ try:
+
+ context['connection'].execute(
+ u'SET LOCAL statement_timeout TO {0}'.format(timeout))
+
+ get_names = datastore_helpers.get_table_and_function_names_from_sql
+ table_names, function_names = get_names(context, sql)
+ log.debug('Tables involved in input SQL: {0!r}'.format(table_names))
+ log.debug('Functions involved in input SQL: {0!r}'.format(
+ function_names))
+
+ if any(t.startswith('pg_') for t in table_names):
+ raise toolkit.NotAuthorized(
+ 'Not authorized to access system tables'
+ )
+ context['check_access'](table_names)
+
+ for f in function_names:
+ for name_variant in [f.lower(), '"{}"'.format(f)]:
+ if name_variant in \
+ backend.allowed_sql_functions: # type: ignore
+ break
+ else:
+ raise toolkit.NotAuthorized(
+ 'Not authorized to call function {}'.format(f)
+ )
+
+ results: Any = context['connection'].execute(sql)
+
+ if results.rowcount == rows_max + 1:
+ data_dict['records_truncated'] = True
+
+ return format_results(context, results, data_dict)
+
+ except ProgrammingError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['permission_denied']:
+ raise toolkit.NotAuthorized('Not authorized to read resource.')
+
+ def _remove_explain(msg: str):
+ return (msg.replace('EXPLAIN (VERBOSE, FORMAT JSON) ', '')
+ .replace('EXPLAIN ', ''))
+
+ raise ValidationError(cast(ErrorDict, {
+ 'query': [_remove_explain(str(e))],
+ 'info': {
+ 'statement': [_remove_explain(e.statement)],
+ 'params': [e.params],
+ 'orig': [_remove_explain(str(e.orig))]
+ }
+ }))
+ except DBAPIError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['query_canceled']:
+ raise ValidationError({
+ 'query': ['Query took too long']
+ })
+ raise
+ finally:
+ context['connection'].close()
+
+
+class DatastorePostgresqlBackend(DatastoreBackend):
+
+ def _get_write_engine(self):
+ return _get_engine_from_url(self.write_url)
+
+ def _get_read_engine(self):
+ return _get_engine_from_url(self.read_url)
+
+ def _log_or_raise(self, message: str):
+ if self.config.get('debug'):
+ log.critical(message)
+ else:
+ raise DatastoreException(message)
+
+ def _check_urls_and_permissions(self):
+ # Make sure that the right permissions are set
+ # so that no harmful queries can be made
+
+ if self._same_ckan_and_datastore_db():
+ self._log_or_raise(
+ 'CKAN and DataStore database cannot be the same.')
+
+ if self._same_read_and_write_url():
+ self._log_or_raise('The write and read-only database '
+ 'connection urls are the same.')
+
+ if not self._read_connection_has_correct_privileges():
+ self._log_or_raise('The read-only user has write privileges.')
+
+ def _is_postgresql_engine(self):
+ ''' Returns True if the read engine is a Postgresql Database.
+
+ According to
+ http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
+ all Postgres driver names start with `postgres`.
+ '''
+ drivername = self._get_read_engine().engine.url.drivername
+ return drivername.startswith('postgres')
+
+ def _is_read_only_database(self):
+ ''' Returns True if no connection has CREATE privileges on the public
+ schema. This is the case if replication is enabled.'''
+ for url in [self.ckan_url, self.write_url, self.read_url]:
+ connection = _get_engine_from_url(url).connect()
+ try:
+ sql = u"SELECT has_schema_privilege('public', 'CREATE')"
+ is_writable: bool = connection.execute(sql).one()[0]
+ finally:
+ connection.close()
+ if is_writable:
+ return False
+ return True
+
+ def _same_ckan_and_datastore_db(self):
+ '''Returns True if the CKAN and DataStore db are the same'''
+ return self._get_db_from_url(self.ckan_url) == self._get_db_from_url(
+ self.read_url)
+
+ def _get_db_from_url(self, url: str):
+ db_url = sa_url.make_url(url)
+ return db_url.host, db_url.port, db_url.database
+
+ def _same_read_and_write_url(self) -> bool:
+ return self.write_url == self.read_url
+
+ def _read_connection_has_correct_privileges(self):
+ ''' Returns True if the right permissions are set for the read
+ only user. A table is created by the write user to test the
+ read only user.
+ '''
+ write_connection = self._get_write_engine().connect()
+ read_connection_user = sa_url.make_url(self.read_url).username
+
+ drop_foo_sql = u'DROP TABLE IF EXISTS _foo'
+
+ write_connection.execute(drop_foo_sql)
+
+ try:
+ write_connection.execute(u'CREATE TEMP TABLE _foo ()')
+ for privilege in ['INSERT', 'UPDATE', 'DELETE']:
+ privilege_sql = u"SELECT has_table_privilege(%s, '_foo', %s)"
+ have_privilege: bool = write_connection.execute(
+ privilege_sql,
+ (read_connection_user, privilege)
+ ).one()[0]
+ if have_privilege:
+ return False
+ finally:
+ write_connection.execute(drop_foo_sql)
+ write_connection.close()
+ return True
+
+ def configure(self, config: CKANConfig):
+ self.config = config
+ # check for ckan.datastore.write_url and ckan.datastore.read_url
+ if ('ckan.datastore.write_url' not in config):
+ error_msg = 'ckan.datastore.write_url not found in config'
+ raise DatastoreException(error_msg)
+ if ('ckan.datastore.read_url' not in config):
+ error_msg = 'ckan.datastore.read_url not found in config'
+ raise DatastoreException(error_msg)
+
+ # Check whether users have disabled datastore_search_sql
+ self.enable_sql_search = self.config.get(
+ 'ckan.datastore.sqlsearch.enabled')
+
+ if self.enable_sql_search:
+ allowed_sql_functions_file = self.config.get(
+ 'ckan.datastore.sqlsearch.allowed_functions_file'
+ )
+
+ def format_entry(line: str):
+ '''Prepare an entry from the 'allowed_functions' file
+ to be used in the whitelist.
+
+ Leading and trailing whitespace is removed, and the
+ entry is lowercased unless enclosed in "double quotes".
+ '''
+ entry = line.strip()
+ if not entry.startswith('"'):
+ entry = entry.lower()
+ return entry
+
+ with open(allowed_sql_functions_file, 'r') as f:
+ self.allowed_sql_functions = set(format_entry(line)
+ for line in f)
+
+ # Check whether we are running one of the paster commands which means
+ # that we should ignore the following tests.
+ args = sys.argv
+ if args[0].split('/')[-1] == 'paster' and 'datastore' in args[1:]:
+ log.warn('Omitting permission checks because you are '
+ 'running paster commands.')
+ return
+
+ self.ckan_url = self.config['sqlalchemy.url']
+ self.write_url = self.config['ckan.datastore.write_url']
+ self.read_url = self.config['ckan.datastore.read_url']
+
+ if not self._is_postgresql_engine():
+ log.warn('We detected that you do not use a PostgreSQL '
+ 'database. The DataStore will NOT work and DataStore '
+ 'tests will be skipped.')
+ return
+
+ if self._is_read_only_database():
+ log.warn('We detected that CKAN is running on a read '
+ 'only database. Permission checks and the creation '
+ 'of _table_metadata are skipped.')
+ else:
+ self._check_urls_and_permissions()
+
+ def datastore_delete(
+ self, context: Context, data_dict: dict[str, Any], # noqa
+ fields_types: dict[str, Any], query_dict: dict[str, Any]):
+ query_dict['where'] += _where_clauses(data_dict, fields_types)
+ return query_dict
+
+ def datastore_search(
+ self, context: Context, data_dict: dict[str, Any], # noqa
+ fields_types: dict[str, Any], query_dict: dict[str, Any]):
+
+ fields: str = data_dict.get('fields', '')
+
+ ts_query, rank_columns = _textsearch_query(
+ _fts_lang(data_dict.get('language')),
+ data_dict.get('q'),
+ data_dict.get('plain', True),
+ data_dict.get('full_text'))
+ # mutate parameter to add rank columns for _result_fields
+ for rank_alias in rank_columns:
+ fields_types[rank_alias] = u'float'
+ fts_q = data_dict.get('full_text')
+ if fields and not fts_q:
+ field_ids = datastore_helpers.get_list(fields)
+ elif fields and fts_q:
+ field_ids = datastore_helpers.get_list(fields)
+ all_field_ids = list(fields_types.keys())
+ if "rank" not in fields:
+ all_field_ids.remove("rank")
+ field_intersect = [x for x in field_ids
+ if x not in all_field_ids]
+ field_ids = all_field_ids + field_intersect
+ else:
+ field_ids = fields_types.keys()
+
+ # add default limit here just in case - already defaulted in the schema
+ limit = data_dict.get('limit', 100)
+ offset = data_dict.get('offset', 0)
+
+ sort = _sort(
+ data_dict.get('sort'),
+ fields_types,
+ rank_columns)
+ where = _where_clauses(data_dict, fields_types)
+ select_cols = []
+ records_format = data_dict.get('records_format')
+ for field_id in field_ids:
+ fmt = '{0}'
+ if records_format == 'lists':
+ fmt = "coalesce(to_json({0}),'null')"
+ typ = fields_types.get(field_id, '')
+ if typ == 'nested':
+ fmt = "coalesce(({0}).json,'null')"
+ elif typ == 'timestamp':
+ fmt = "to_char({0}, 'YYYY-MM-DD\"T\"HH24:MI:SS')"
+ if records_format == 'lists':
+ fmt = f"coalesce(to_json({fmt}), 'null')"
+ elif typ.startswith('_') or typ.endswith('[]'):
+ fmt = "coalesce(array_to_json({0}),'null')"
+
+ if field_id in rank_columns:
+ select_cols.append((fmt + ' as {1}').format(
+ rank_columns[field_id], identifier(field_id)))
+ continue
+
+ if records_format == 'objects':
+ fmt += ' as {0}'
+ select_cols.append(fmt.format(identifier(field_id)))
+
+ query_dict['distinct'] = data_dict.get('distinct', False)
+ query_dict['select'] += select_cols
+ query_dict['ts_query'] = ts_query
+ query_dict['sort'] += sort
+ query_dict['where'] += where
+ query_dict['limit'] = limit
+ query_dict['offset'] = offset
+
+ return query_dict
+
+ def delete(self, context: Context, data_dict: dict[str, Any]):
+ engine = self._get_write_engine()
+ context['connection'] = engine.connect()
+ _cache_types(context['connection'])
+
+ trans = context['connection'].begin()
+ try:
+ # check if table exists
+ if 'filters' not in data_dict:
+ context['connection'].execute(
+ u'DROP TABLE "{0}" CASCADE'.format(
+ data_dict['resource_id'])
+ )
+ else:
+ delete_data(context, data_dict)
+
+ trans.commit()
+ return _unrename_json_field(data_dict)
+ except Exception:
+ trans.rollback()
+ raise
+ finally:
+ context['connection'].close()
+
+ def create(self, context: Context, data_dict: dict[str, Any]):
+ '''
+ The first row will be used to guess types not in the fields and the
+ guessed types will be added to the headers permanently.
+ Consecutive rows have to conform to the field definitions.
+ rows can be empty so that you can just set the fields.
+ fields are optional but needed if you want to do type hinting or
+ add extra information for certain columns or to explicitly
+ define ordering.
+ eg: [{"id": "dob", "type": "timestamp"},
+ {"id": "name", "type": "text"}]
+ A header items values can not be changed after it has been defined
+ nor can the ordering of them be changed. They can be extended though.
+ Any error results in total failure! For now pass back the actual error.
+ Should be transactional.
+ :raises InvalidDataError: if there is an invalid value in the given
+ data
+ '''
+ engine = get_write_engine()
+ context['connection'] = engine.connect()
+ timeout = context.get('query_timeout', _TIMEOUT)
+ _cache_types(context['connection'])
+
+ _rename_json_field(data_dict)
+
+ trans = context['connection'].begin()
+ try:
+ # check if table already exists
+ context['connection'].execute(
+ u'SET LOCAL statement_timeout TO {0}'.format(timeout))
+ result = context['connection'].execute(
+ u'SELECT * FROM pg_tables WHERE tablename = %s',
+ data_dict['resource_id']
+ ).fetchone()
+ if not result:
+ create_table(context, data_dict)
+ _create_fulltext_trigger(
+ context['connection'],
+ data_dict['resource_id'])
+ else:
+ alter_table(context, data_dict)
+ if 'triggers' in data_dict:
+ _create_triggers(
+ context['connection'],
+ data_dict['resource_id'],
+ data_dict['triggers'])
+ insert_data(context, data_dict)
+ create_indexes(context, data_dict)
+ create_alias(context, data_dict)
+ trans.commit()
+ return _unrename_json_field(data_dict)
+ except IntegrityError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['unique_violation']:
+ raise ValidationError(cast(ErrorDict, {
+ 'constraints': ['Cannot insert records or create index'
+ 'because of uniqueness constraint'],
+ 'info': {
+ 'orig': str(e.orig),
+ 'pgcode': e.orig.pgcode
+ }
+ }))
+ raise
+ except DataError as e:
+ raise ValidationError(cast(ErrorDict, {
+ 'data': str(e),
+ 'info': {
+ 'orig': [str(e.orig)]
+ }}))
+ except DBAPIError as e:
+ if e.orig.pgcode == _PG_ERR_CODE['query_canceled']:
+ raise ValidationError({
+ 'query': ['Query took too long']
+ })
+ raise
+ except Exception:
+ trans.rollback()
+ raise
+ finally:
+ context['connection'].close()
+
+ def upsert(self, context: Context, data_dict: dict[str, Any]):
+ data_dict['connection_url'] = self.write_url
+ return upsert(context, data_dict)
+
+ def search(self, context: Context, data_dict: dict[str, Any]):
+ data_dict['connection_url'] = self.write_url
+ return search(context, data_dict)
+
+ def search_sql(self, context: Context, data_dict: dict[str, Any]):
+ sql = toolkit.get_or_bust(data_dict, 'sql')
+ data_dict['connection_url'] = self.read_url
+
+ if not is_single_statement(sql):
+ raise toolkit.ValidationError({
+ 'query': ['Query is not a single statement.']
+ })
+ return search_sql(context, data_dict)
+
+ def resource_exists(self, id: str) -> bool:
+ resources_sql = sqlalchemy.text(
+ u'''SELECT 1 FROM "_table_metadata"
+ WHERE name = :id AND alias_of IS NULL''')
+ results = self._get_read_engine().execute(resources_sql, id=id)
+ res_exists = results.rowcount > 0
+ return res_exists
+
+ def resource_id_from_alias(self, alias: str) -> tuple[bool, Optional[str]]:
+ real_id: Optional[str] = None
+ resources_sql = sqlalchemy.text(
+ u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''')
+ results = self._get_read_engine().execute(resources_sql, id=alias)
+
+ res_exists = results.rowcount > 0
+ if res_exists:
+ real_id = results.fetchone()[0] # type: ignore
+ return res_exists, real_id
+
+ # def resource_info(self, id):
+ # pass
+
+ def resource_fields(self, id: str) -> dict[str, Any]:
+
+ info: dict[str, Any] = {'meta': {}, 'fields': []}
+
+ try:
+ engine = self._get_read_engine()
+
+ # resource id for deferencing aliases
+ info['meta']['id'] = id
+
+ # count of rows in table
+ meta_sql = sqlalchemy.text(
+ u'SELECT count(_id) FROM "{0}"'.format(id))
+ meta_results = engine.execute(meta_sql)
+ info['meta']['count'] = meta_results.fetchone()[0] # type: ignore
+
+ # table_type - BASE TABLE, VIEW, FOREIGN TABLE, MATVIEW
+ tabletype_sql = sqlalchemy.text(u'''
+ SELECT table_type FROM INFORMATION_SCHEMA.TABLES
+ WHERE table_name = '{0}'
+ '''.format(id))
+ tabletype_results = engine.execute(tabletype_sql)
+ info['meta']['table_type'] = \
+ tabletype_results.fetchone()[0] # type: ignore
+ # MATERIALIZED VIEWS show as BASE TABLE, so
+ # we check pg_matviews
+ matview_sql = sqlalchemy.text(u'''
+ SELECT count(*) FROM pg_matviews
+ WHERE matviewname = '{0}'
+ '''.format(id))
+ matview_results = engine.execute(matview_sql)
+ if matview_results.fetchone()[0]: # type: ignore
+ info['meta']['table_type'] = 'MATERIALIZED VIEW'
+
+ # SIZE - size of table in bytes
+ size_sql = sqlalchemy.text(
+ u"SELECT pg_relation_size('{0}')".format(id))
+ size_results = engine.execute(size_sql)
+ info['meta']['size'] = size_results.fetchone()[0] # type: ignore
+
+ # DB_SIZE - size of database in bytes
+ dbsize_sql = sqlalchemy.text(
+ u"SELECT pg_database_size(current_database())")
+ dbsize_results = engine.execute(dbsize_sql)
+ info['meta']['db_size'] = \
+ dbsize_results.fetchone()[0] # type: ignore
+
+ # IDXSIZE - size of all indices for table in bytes
+ idxsize_sql = sqlalchemy.text(
+ u"SELECT pg_indexes_size('{0}')".format(id))
+ idxsize_results = engine.execute(idxsize_sql)
+ info['meta']['idx_size'] = \
+ idxsize_results.fetchone()[0] # type: ignore
+
+ # all the aliases for this resource
+ alias_sql = sqlalchemy.text(u'''
+ SELECT name FROM "_table_metadata" WHERE alias_of = '{0}'
+ '''.format(id))
+ alias_results = engine.execute(alias_sql)
+ aliases = []
+ for alias in alias_results.fetchall():
+ aliases.append(alias[0])
+ info['meta']['aliases'] = aliases
+
+ # get the data dictionary for the resource
+ data_dictionary = datastore_helpers.datastore_dictionary(id)
+
+ schema_sql = sqlalchemy.text(u'''
+ SELECT
+ f.attname AS column_name,
+ pg_catalog.format_type(f.atttypid,f.atttypmod) AS native_type,
+ f.attnotnull AS notnull,
+ i.relname as index_name,
+ CASE
+ WHEN i.oid<>0 THEN True
+ ELSE False
+ END AS is_index,
+ CASE
+ WHEN p.contype = 'u' THEN True
+ WHEN p.contype = 'p' THEN True
+ ELSE False
+ END AS uniquekey
+ FROM pg_attribute f
+ JOIN pg_class c ON c.oid = f.attrelid
+ JOIN pg_type t ON t.oid = f.atttypid
+ LEFT JOIN pg_constraint p ON p.conrelid = c.oid
+ AND f.attnum = ANY (p.conkey)
+ LEFT JOIN pg_index AS ix ON f.attnum = ANY(ix.indkey)
+ AND c.oid = f.attrelid AND c.oid = ix.indrelid
+ LEFT JOIN pg_class AS i ON ix.indexrelid = i.oid
+ WHERE c.relkind = 'r'::char
+ AND c.relname = '{0}'
+ AND f.attnum > 0
+ ORDER BY c.relname,f.attnum;
+ '''.format(id))
+ schema_results = engine.execute(schema_sql)
+ schemainfo = {}
+ for row in schema_results.fetchall():
+ row: Any # Row has incomplete type definition
+ colname: str = row.column_name
+ if colname.startswith('_'): # Skip internal rows
+ continue
+ colinfo: dict[str, Any] = {'native_type': row.native_type,
+ 'notnull': row.notnull,
+ 'index_name': row.index_name,
+ 'is_index': row.is_index,
+ 'uniquekey': row.uniquekey}
+ schemainfo[colname] = colinfo
+
+ for field in data_dictionary:
+ field.update({'schema': schemainfo[field['id']]})
+ info['fields'].append(field)
+
+ except Exception:
+ pass
+ return info
+
+ def get_all_ids(self):
+ resources_sql = sqlalchemy.text(
+ u'''SELECT name FROM "_table_metadata"
+ WHERE alias_of IS NULL''')
+ query = self._get_read_engine().execute(resources_sql)
+ return [q[0] for q in query.fetchall()]
+
+ def create_function(self, *args: Any, **kwargs: Any):
+ return create_function(*args, **kwargs)
+
+ def drop_function(self, *args: Any, **kwargs: Any):
+ return drop_function(*args, **kwargs)
+
+ def before_fork(self):
+ # Called by DatastorePlugin.before_fork. Dispose SQLAlchemy engines
+ # to avoid sharing them between parent and child processes.
+ _dispose_engines()
+
+ def calculate_record_count(self, resource_id: str):
+ '''
+ Calculate an estimate of the record/row count and store it in
+ Postgresql's pg_stat_user_tables. This number will be used when
+ specifying `total_estimation_threshold`
+ '''
+ connection = get_write_engine().connect()
+ sql = 'ANALYZE "{}"'.format(resource_id)
+ try:
+ connection.execute(sql)
+ except sqlalchemy.exc.DatabaseError as err:
+ raise DatastoreException(err)
+
+
+def create_function(name: str, arguments: Iterable[dict[str, Any]],
+ rettype: Any, definition: str, or_replace: bool):
+ sql = u'''
+ CREATE {or_replace} FUNCTION
+ {name}({args}) RETURNS {rettype} AS {definition}
+ LANGUAGE plpgsql;'''.format(
+ or_replace=u'OR REPLACE' if or_replace else u'',
+ name=identifier(name),
+ args=u', '.join(
+ u'{argname} {argtype}'.format(
+ argname=identifier(a['argname']),
+ argtype=identifier(a['argtype']))
+ for a in arguments),
+ rettype=identifier(rettype),
+ definition=literal_string(definition))
+
+ try:
+ _write_engine_execute(sql)
+ except ProgrammingError as pe:
+ already_exists = (
+ u'function "{}" already exists with same argument types'
+ .format(name)
+ in pe.args[0])
+ key = u'name' if already_exists else u'definition'
+ raise ValidationError({key: [_programming_error_summary(pe)]})
+
+
+def drop_function(name: str, if_exists: bool):
+ sql = u'''
+ DROP FUNCTION {if_exists} {name}();
+ '''.format(
+ if_exists=u'IF EXISTS' if if_exists else u'',
+ name=identifier(name))
+
+ try:
+ _write_engine_execute(sql)
+ except ProgrammingError as pe:
+ raise ValidationError({u'name': [_programming_error_summary(pe)]})
+
+
+def _write_engine_execute(sql: str):
+ connection = get_write_engine().connect()
+ # No special meaning for '%' in sql parameter:
+ connection: Any = connection.execution_options(no_parameters=True)
+ trans = connection.begin()
+ try:
+ connection.execute(sql)
+ trans.commit()
+ except Exception:
+ trans.rollback()
+ raise
+ finally:
+ connection.close()
+
+
+def _programming_error_summary(pe: Any):
+ u'''
+ return the text description of a sqlalchemy DatabaseError
+ without the actual SQL included, for raising as a
+ ValidationError to send back to API users
+ '''
+ # first line only, after the '(ProgrammingError)' text
+ message = six.ensure_text(pe.args[0].split('\n')[0])
+ return message.split(u') ', 1)[-1]
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py b/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py
new file mode 100644
index 0000000..ba10a69
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py
@@ -0,0 +1,290 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from typing import Any, Optional, cast, Union
+from itertools import zip_longest
+
+from flask import Blueprint
+from flask.wrappers import Response
+from flask.views import MethodView
+
+import ckan.lib.navl.dictization_functions as dict_fns
+from ckan.logic import (
+ tuplize_dict,
+ parse_params,
+)
+from ckan.plugins.toolkit import (
+ ObjectNotFound, NotAuthorized, get_action, get_validator, _, request,
+ abort, render, g, h
+)
+from ckan.types import Schema, ValidatorFactory
+from ckanext.datastore.logic.schema import (
+ list_of_strings_or_string,
+ json_validator,
+ unicode_or_json_validator,
+)
+from ckanext.datastore.writer import (
+ csv_writer,
+ tsv_writer,
+ json_writer,
+ xml_writer,
+)
+
+int_validator = get_validator(u'int_validator')
+boolean_validator = get_validator(u'boolean_validator')
+ignore_missing = get_validator(u'ignore_missing')
+one_of = cast(ValidatorFactory, get_validator(u'one_of'))
+default = cast(ValidatorFactory, get_validator(u'default'))
+unicode_only = get_validator(u'unicode_only')
+
+DUMP_FORMATS = u'csv', u'tsv', u'json', u'xml'
+PAGINATE_BY = 32000
+
+datastore = Blueprint(u'datastore', __name__)
+
+
+def dump_schema() -> Schema:
+ return {
+ u'offset': [default(0), int_validator],
+ u'limit': [ignore_missing, int_validator],
+ u'format': [default(u'csv'), one_of(DUMP_FORMATS)],
+ u'bom': [default(False), boolean_validator],
+ u'filters': [ignore_missing, json_validator],
+ u'q': [ignore_missing, unicode_or_json_validator],
+ u'distinct': [ignore_missing, boolean_validator],
+ u'plain': [ignore_missing, boolean_validator],
+ u'language': [ignore_missing, unicode_only],
+ u'fields': [ignore_missing, list_of_strings_or_string],
+ u'sort': [default(u'_id'), list_of_strings_or_string],
+ }
+
+
+def dump(resource_id: str):
+ try:
+ get_action('datastore_search')({}, {'resource_id': resource_id,
+ 'limit': 0})
+ except ObjectNotFound:
+ abort(404, _('DataStore resource not found'))
+
+ data, errors = dict_fns.validate(request.args.to_dict(), dump_schema())
+ if errors:
+ abort(
+ 400, '\n'.join(
+ '{0}: {1}'.format(k, ' '.join(e)) for k, e in errors.items()
+ )
+ )
+
+ fmt = data['format']
+ offset = data['offset']
+ limit = data.get('limit')
+ options = {'bom': data['bom']}
+ sort = data['sort']
+ search_params = {
+ k: v
+ for k, v in data.items()
+ if k in [
+ 'filters', 'q', 'distinct', 'plain', 'language',
+ 'fields'
+ ]
+ }
+
+ user_context = g.user
+
+ content_type = None
+ content_disposition = None
+
+ if fmt == 'csv':
+ content_disposition = 'attachment; filename="{name}.csv"'.format(
+ name=resource_id)
+ content_type = b'text/csv; charset=utf-8'
+ elif fmt == 'tsv':
+ content_disposition = 'attachment; filename="{name}.tsv"'.format(
+ name=resource_id)
+ content_type = b'text/tab-separated-values; charset=utf-8'
+ elif fmt == 'json':
+ content_disposition = 'attachment; filename="{name}.json"'.format(
+ name=resource_id)
+ content_type = b'application/json; charset=utf-8'
+ elif fmt == 'xml':
+ content_disposition = 'attachment; filename="{name}.xml"'.format(
+ name=resource_id)
+ content_type = b'text/xml; charset=utf-8'
+ else:
+ abort(404, _('Unsupported format'))
+
+ headers = {}
+ if content_type:
+ headers['Content-Type'] = content_type
+ if content_disposition:
+ headers['Content-disposition'] = content_disposition
+
+ try:
+ return Response(dump_to(resource_id,
+ fmt=fmt,
+ offset=offset,
+ limit=limit,
+ options=options,
+ sort=sort,
+ search_params=search_params,
+ user=user_context),
+ mimetype='application/octet-stream',
+ headers=headers)
+ except ObjectNotFound:
+ abort(404, _('DataStore resource not found'))
+
+
+class DictionaryView(MethodView):
+
+ def _prepare(self, id: str, resource_id: str) -> dict[str, Any]:
+ try:
+ # resource_edit_base template uses these
+ pkg_dict = get_action(u'package_show')({}, {u'id': id})
+ resource = get_action(u'resource_show')({}, {u'id': resource_id})
+ rec = get_action(u'datastore_search')(
+ {}, {
+ u'resource_id': resource_id,
+ u'limit': 0
+ }
+ )
+ return {
+ u'pkg_dict': pkg_dict,
+ u'resource': resource,
+ u'fields': [
+ f for f in rec[u'fields'] if not f[u'id'].startswith(u'_')
+ ]
+ }
+
+ except (ObjectNotFound, NotAuthorized):
+ abort(404, _(u'Resource not found'))
+
+ def get(self, id: str, resource_id: str):
+ u'''Data dictionary view: show field labels and descriptions'''
+
+ data_dict = self._prepare(id, resource_id)
+
+ # global variables for backward compatibility
+ g.pkg_dict = data_dict[u'pkg_dict']
+ g.resource = data_dict[u'resource']
+
+ return render(u'datastore/dictionary.html', data_dict)
+
+ def post(self, id: str, resource_id: str):
+ u'''Data dictionary view: edit field labels and descriptions'''
+ data_dict = self._prepare(id, resource_id)
+ fields = data_dict[u'fields']
+ data = dict_fns.unflatten(tuplize_dict(parse_params(request.form)))
+ info = data.get(u'info')
+ if not isinstance(info, list):
+ info = []
+ info = info[:len(fields)]
+
+ get_action(u'datastore_create')(
+ {}, {
+ u'resource_id': resource_id,
+ u'force': True,
+ u'fields': [{
+ u'id': f[u'id'],
+ u'type': f[u'type'],
+ u'info': fi if isinstance(fi, dict) else {}
+ } for f, fi in zip_longest(fields, info)]
+ }
+ )
+
+ h.flash_success(
+ _(
+ u'Data Dictionary saved. Any type overrides will '
+ u'take effect when the resource is next uploaded '
+ u'to DataStore'
+ )
+ )
+ return h.redirect_to(
+ u'datastore.dictionary', id=id, resource_id=resource_id
+ )
+
+
+def dump_to(
+ resource_id: str, fmt: str, offset: int,
+ limit: Optional[int], options: dict[str, Any], sort: str,
+ search_params: dict[str, Any], user: str
+):
+ if fmt == 'csv':
+ writer_factory = csv_writer
+ records_format = 'csv'
+ elif fmt == 'tsv':
+ writer_factory = tsv_writer
+ records_format = 'tsv'
+ elif fmt == 'json':
+ writer_factory = json_writer
+ records_format = 'lists'
+ elif fmt == 'xml':
+ writer_factory = xml_writer
+ records_format = 'objects'
+ else:
+ assert False, 'Unsupported format'
+
+ bom = options.get('bom', False)
+
+ def start_stream_writer(fields: list[dict[str, Any]]):
+ return writer_factory(fields, bom=bom)
+
+ def stream_result_page(offs: int, lim: Union[None, int]):
+ return get_action('datastore_search')(
+ {'user': user},
+ dict({
+ 'resource_id': resource_id,
+ 'limit': PAGINATE_BY
+ if limit is None else min(PAGINATE_BY, lim), # type: ignore
+ 'offset': offs,
+ 'sort': sort,
+ 'records_format': records_format,
+ 'include_total': False,
+ }, **search_params)
+ )
+
+ def stream_dump(offset: int, limit: Union[None, int],
+ paginate_by: int, result: dict[str, Any]):
+ with start_stream_writer(result['fields']) as writer:
+ while True:
+ if limit is not None and limit <= 0:
+ break
+
+ records = result['records']
+
+ yield writer.write_records(records)
+
+ if records_format == 'objects' or records_format == 'lists':
+ if len(records) < paginate_by:
+ break
+ elif not records:
+ break
+
+ offset += paginate_by
+ if limit is not None:
+ limit -= paginate_by
+ if limit <= 0:
+ break
+
+ result = stream_result_page(offset, limit)
+
+ yield writer.end_file()
+
+ result = stream_result_page(offset, limit)
+
+ if result['limit'] != limit:
+ # `limit` (from PAGINATE_BY) must have been more than
+ # ckan.datastore.search.rows_max, so datastore_search responded
+ # with a limit matching ckan.datastore.search.rows_max.
+ # So we need to paginate by that amount instead, otherwise
+ # we'll have gaps in the records.
+ paginate_by = result['limit']
+ else:
+ paginate_by = PAGINATE_BY
+
+ return stream_dump(offset, limit, paginate_by, result)
+
+
+datastore.add_url_rule(u'/datastore/dump/', view_func=dump)
+datastore.add_url_rule(
+ u'/dataset//dictionary/',
+ view_func=DictionaryView.as_view(str(u'dictionary'))
+)
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/cli.py b/src/ckanext-d4science_theme/ckanext/datastore/cli.py
new file mode 100644
index 0000000..8091cba
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/cli.py
@@ -0,0 +1,179 @@
+# encoding: utf-8
+
+from typing import Any
+import logging
+import os
+
+import click
+
+from ckan.model import parse_db_config
+from ckan.common import config
+import ckan.logic as logic
+
+import ckanext.datastore as datastore_module
+from ckanext.datastore.backend.postgres import identifier
+from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to
+
+log = logging.getLogger(__name__)
+
+
+@click.group(short_help=u"Perform commands to set up the datastore.")
+def datastore():
+ """Perform commands to set up the datastore.
+ """
+ pass
+
+
+@datastore.command(
+ u'set-permissions',
+ short_help=u'Generate SQL for permission configuration.'
+)
+def set_permissions():
+ u'''Emit an SQL script that will set the permissions for the datastore
+ users as configured in your configuration file.'''
+
+ write_url = _parse_db_config(u'ckan.datastore.write_url')
+ read_url = _parse_db_config(u'ckan.datastore.read_url')
+ db_url = _parse_db_config(u'sqlalchemy.url')
+
+ # Basic validation that read and write URLs reference the same database.
+ # This obviously doesn't check they're the same database (the hosts/ports
+ # could be different), but it's better than nothing, I guess.
+
+ if write_url[u'db_name'] != read_url[u'db_name']:
+ click.secho(
+ u'The datastore write_url and read_url must refer to the same '
+ u'database!',
+ fg=u'red',
+ bold=True
+ )
+ raise click.Abort()
+
+ sql = permissions_sql(
+ maindb=db_url[u'db_name'],
+ datastoredb=write_url[u'db_name'],
+ mainuser=db_url[u'db_user'],
+ writeuser=write_url[u'db_user'],
+ readuser=read_url[u'db_user']
+ )
+
+ click.echo(sql)
+
+
+def permissions_sql(maindb: str, datastoredb: str, mainuser: str,
+ writeuser: str, readuser: str):
+ template_filename = os.path.join(
+ os.path.dirname(datastore_module.__file__), u'set_permissions.sql'
+ )
+ with open(template_filename) as fp:
+ template = fp.read()
+ return template.format(
+ maindb=identifier(maindb),
+ datastoredb=identifier(datastoredb),
+ mainuser=identifier(mainuser),
+ writeuser=identifier(writeuser),
+ readuser=identifier(readuser)
+ )
+
+
+@datastore.command()
+@click.argument(u'resource-id', nargs=1)
+@click.argument(
+ u'output-file',
+ type=click.File(u'wb'),
+ default=click.get_binary_stream(u'stdout')
+)
+@click.option(u'--format', default=u'csv', type=click.Choice(DUMP_FORMATS))
+@click.option(u'--offset', type=click.IntRange(0, None), default=0)
+@click.option(u'--limit', type=click.IntRange(0))
+@click.option(u'--bom', is_flag=True)
+@click.pass_context
+def dump(ctx: Any, resource_id: str, output_file: Any, format: str,
+ offset: int, limit: int, bom: bool):
+ u'''Dump a datastore resource.
+ '''
+ flask_app = ctx.meta['flask_app']
+ user = logic.get_action('get_site_user')(
+ {'ignore_auth': True}, {})
+ with flask_app.test_request_context():
+ for block in dump_to(resource_id,
+ fmt=format,
+ offset=offset,
+ limit=limit,
+ options={u'bom': bom},
+ sort=u'_id',
+ search_params={},
+ user=user['name']):
+ output_file.write(block)
+
+
+def _parse_db_config(config_key: str = u'sqlalchemy.url'):
+ db_config = parse_db_config(config_key)
+ if not db_config:
+ click.secho(
+ u'Could not extract db details from url: %r' % config[config_key],
+ fg=u'red',
+ bold=True
+ )
+ raise click.Abort()
+ return db_config
+
+
+@datastore.command(
+ 'purge',
+ short_help='purge orphaned resources from the datastore.'
+)
+def purge():
+ '''Purge orphaned resources from the datastore using the datastore_delete
+ action, which drops tables when called without filters.'''
+
+ site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {})
+
+ result = logic.get_action('datastore_search')(
+ {'user': site_user['name']},
+ {'resource_id': '_table_metadata'}
+ )
+
+ resource_id_list = []
+ for record in result['records']:
+ try:
+ # ignore 'alias' records (views) as they are automatically
+ # deleted when the parent resource table is dropped
+ if record['alias_of']:
+ continue
+
+ logic.get_action('resource_show')(
+ {'user': site_user['name']},
+ {'id': record['name']}
+ )
+ except logic.NotFound:
+ resource_id_list.append(record['name'])
+ click.echo("Resource '%s' orphaned - queued for drop" %
+ record[u'name'])
+ except KeyError:
+ continue
+
+ orphaned_table_count = len(resource_id_list)
+ click.echo('%d orphaned tables found.' % orphaned_table_count)
+
+ if not orphaned_table_count:
+ return
+
+ click.confirm('Proceed with purge?', abort=True)
+
+ # Drop the orphaned datastore tables. When datastore_delete is called
+ # without filters, it does a drop table cascade
+ drop_count = 0
+ for resource_id in resource_id_list:
+ logic.get_action('datastore_delete')(
+ {'user': site_user['name']},
+ {'resource_id': resource_id, 'force': True}
+ )
+ click.echo("Table '%s' dropped)" % resource_id)
+ drop_count += 1
+
+ click.echo('Dropped %s tables' % drop_count)
+
+
+def get_commands():
+ return (set_permissions, dump, purge)
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml b/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml
new file mode 100644
index 0000000..5e30803
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml
@@ -0,0 +1,101 @@
+version: 1
+groups:
+- annotation: Datastore settings
+ options:
+ - default: postgresql://ckan_default:pass@localhost/datastore_default
+ key: ckan.datastore.write_url
+ example: postgresql://ckanuser:pass@localhost/datastore
+ required: true
+ description: >-
+ The database connection to use for writing to the datastore (this can be
+ ignored if you're not using the :ref:`datastore`). Note that the database used
+ should not be the same as the normal CKAN database. The format is the same as
+ in :ref:`sqlalchemy.url`.
+
+ - default: postgresql://datastore_default:pass@localhost/datastore_default
+ key: ckan.datastore.read_url
+ required: true
+ example: postgresql://readonlyuser:pass@localhost/datastore
+ description: >-
+ The database connection to use for reading from the datastore (this can be
+ ignored if you're not using the :ref:`datastore`). The database used must be
+ the same used in :ref:`ckan.datastore.write_url`, but the user should be one
+ with read permissions only. The format is the same as in :ref:`sqlalchemy.url`.
+
+ - key: ckan.datastore.sqlsearch.allowed_functions_file
+ default_callable: ckanext.datastore.plugin:sql_functions_allowlist_file
+ example: /path/to/my_allowed_functions.txt
+ description: |
+ Allows to define the path to a text file listing the SQL functions that should be allowed to run
+ on queries sent to the :py:func:`~ckanext.datastore.logic.action.datastore_search_sql` function
+ (if enabled, see :ref:`ckan.datastore.sqlsearch.enabled`). Function names should be listed one on
+ each line, eg::
+
+ abbrev
+ abs
+ abstime
+ ...
+
+ - key: ckan.datastore.sqlsearch.enabled
+ type: bool
+ example: 'true'
+ description: |
+ This option allows you to enable the :py:func:`~ckanext.datastore.logic.action.datastore_search_sql` action function, and corresponding API endpoint.
+
+ This action function has protections from abuse including:
+
+ - parsing of the query to prevent unsafe functions from being called, see :ref:`ckan.datastore.sqlsearch.allowed_functions_file`
+ - parsing of the query to prevent multiple statements
+ - prevention of data modification by using a read-only database role
+ - use of ``explain`` to resolve tables accessed in the query to check against user permissions
+ - use of a statement timeout to prevent queries from running indefinitely
+
+ These protections offer some safety but are not designed to prevent all types of abuse. Depending on the sensitivity of private data in your datastore and the likelihood of abuse of your site you may choose to disable this action function or restrict its use with a :py:class:`~ckan.plugins.interfaces.IAuthFunctions` plugin.
+
+ - default: 100
+ key: ckan.datastore.search.rows_default
+ type: int
+ example: 1000
+ description: |
+ Default number of rows returned by ``datastore_search``, unless the client
+ specifies a different ``limit`` (up to ``ckan.datastore.search.rows_max``).
+
+ NB this setting does not affect ``datastore_search_sql``.
+
+ - default: 32000
+ key: ckan.datastore.search.rows_max
+ example: 1000000
+ type: int
+ description: |
+ Maximum allowed value for the number of rows returned by the datastore.
+
+ Specifically this limits:
+
+ * ``datastore_search``'s ``limit`` parameter.
+ * ``datastore_search_sql`` queries have this limit inserted.
+
+ - key: ckan.datastore.sqlalchemy.
+ type: dynamic
+ description: |
+ Custom sqlalchemy config parameters used to establish the DataStore
+ database connection.
+
+ To get the list of all the available properties check the `SQLAlchemy documentation`_
+
+ .. _SQLAlchemy documentation: http://docs.sqlalchemy.org/en/rel_0_9/core/engines.html#engine-creation-api
+
+ - default: english
+ key: ckan.datastore.default_fts_lang
+ example: english
+ description: >
+ The default language used when creating full-text search indexes and querying
+ them. It can be overwritten by the user by passing the "lang" parameter to
+ "datastore_search" and "datastore_create".
+
+ - default: gist
+ key: ckan.datastore.default_fts_index_method
+ example: gist
+ description: >
+ The default method used when creating full-text search indexes. Currently it
+ can be "gin" or "gist". Refer to PostgreSQL's documentation to understand the
+ characteristics of each one and pick the best for your instance.
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/helpers.py b/src/ckanext-d4science_theme/ckanext/datastore/helpers.py
new file mode 100644
index 0000000..79731a1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/helpers.py
@@ -0,0 +1,235 @@
+# encoding: utf-8
+from __future__ import annotations
+
+import json
+import logging
+from typing import (
+ Any, Iterable, Optional, Sequence, Union, cast, overload
+)
+from typing_extensions import Literal
+
+import sqlparse
+import six
+
+import ckan.common as converters
+import ckan.plugins.toolkit as tk
+from ckan.types import Context
+
+
+log = logging.getLogger(__name__)
+
+
+def is_single_statement(sql: str):
+ '''Returns True if received SQL string contains at most one statement'''
+ return len(sqlparse.split(sql)) <= 1
+
+
+def is_valid_field_name(name: str):
+ '''
+ Check that field name is valid:
+ * can't start or end with whitespace characters
+ * can't start with underscore
+ * can't contain double quote (")
+ * can't be empty
+ '''
+ return (name and name == name.strip() and
+ not name.startswith('_') and
+ '"' not in name)
+
+
+def is_valid_table_name(name: str):
+ if '%' in name:
+ return False
+ return is_valid_field_name(name)
+
+
+@overload
+def get_list(input: Literal[None], strip_values: bool = ...) -> Literal[None]:
+ ...
+
+
+@overload
+def get_list(input: Union[str, "Sequence[Any]"],
+ strip_values: bool = ...) -> list[str]:
+ ...
+
+
+def get_list(input: Any, strip_values: bool = True) -> Optional[list[str]]:
+ '''Transforms a string or list to a list'''
+ if input is None:
+ return
+ if input == '':
+ return []
+
+ converters_list = converters.aslist(input, ',', True)
+ if strip_values:
+ return [_strip(x) for x in converters_list]
+ else:
+ return converters_list
+
+
+def validate_int(i: Any, non_negative: bool = False):
+ try:
+ i = int(i)
+ except ValueError:
+ return False
+ return i >= 0 or not non_negative
+
+
+def _strip(s: Any):
+ if isinstance(s, str) and len(s) and s[0] == s[-1]:
+ return s.strip().strip('"')
+ return s
+
+
+def should_fts_index_field_type(field_type: str):
+ return field_type.lower() in ['tsvector', 'text', 'number']
+
+
+def get_table_and_function_names_from_sql(context: Context, sql: str):
+ '''Parses the output of EXPLAIN (FORMAT JSON) looking for table and
+ function names
+
+ It performs an EXPLAIN query against the provided SQL, and parses
+ the output recusively.
+
+ Note that this requires Postgres 9.x.
+
+ :param context: a CKAN context dict. It must contain a 'connection' key
+ with the current DB connection.
+ :type context: dict
+ :param sql: the SQL statement to parse for table and function names
+ :type sql: string
+
+ :rtype: a tuple with two list of strings, one for table and one for
+ function names
+ '''
+
+ queries = [sql]
+ table_names: list[str] = []
+ function_names: list[str] = []
+
+ while queries:
+ sql = queries.pop()
+
+ function_names.extend(_get_function_names_from_sql(sql))
+
+ result = context['connection'].execute(
+ 'EXPLAIN (VERBOSE, FORMAT JSON) {0}'.format(
+ six.ensure_str(sql))).fetchone()
+
+ try:
+ query_plan = json.loads(result['QUERY PLAN'])
+ plan = query_plan[0]['Plan']
+
+ t, q, f = _parse_query_plan(plan)
+ table_names.extend(t)
+ queries.extend(q)
+
+ function_names = list(set(function_names) | set(f))
+
+ except ValueError:
+ log.error('Could not parse query plan')
+ raise
+
+ return table_names, function_names
+
+
+def _parse_query_plan(
+ plan: dict[str, Any]) -> tuple[list[str], list[str], list[str]]:
+ '''
+ Given a Postgres Query Plan object (parsed from the output of an EXPLAIN
+ query), returns a tuple with three items:
+
+ * A list of tables involved
+ * A list of remaining queries to parse
+ * A list of function names involved
+ '''
+
+ table_names: list[str] = []
+ queries: list[str] = []
+ functions: list[str] = []
+
+ if plan.get('Relation Name'):
+ table_names.append(plan['Relation Name'])
+ if 'Function Name' in plan:
+ if plan['Function Name'].startswith(
+ 'crosstab'):
+ try:
+ queries.append(_get_subquery_from_crosstab_call(
+ plan['Function Call']))
+ except ValueError:
+ table_names.append('_unknown_crosstab_sql')
+ else:
+ functions.append(plan['Function Name'])
+
+ if 'Plans' in plan:
+ for child_plan in plan['Plans']:
+ t, q, f = _parse_query_plan(child_plan)
+ table_names.extend(t)
+ queries.extend(q)
+ functions.extend(f)
+
+ return table_names, queries, functions
+
+
+def _get_function_names_from_sql(sql: str):
+ function_names: list[str] = []
+
+ def _get_function_names(tokens: Iterable[Any]):
+ for token in tokens:
+ if isinstance(token, sqlparse.sql.Function):
+ function_name = cast(str, token.get_name())
+ if function_name not in function_names:
+ function_names.append(function_name)
+ if hasattr(token, 'tokens'):
+ _get_function_names(token.tokens)
+
+ parsed = sqlparse.parse(sql)[0]
+ _get_function_names(parsed.tokens)
+
+ return function_names
+
+
+def _get_subquery_from_crosstab_call(ct: str):
+ """
+ Crosstabs are a useful feature some sites choose to enable on
+ their datastore databases. To support the sql parameter passed
+ safely we accept only the simple crosstab(text) form where text
+ is a literal SQL string, otherwise raise ValueError
+ """
+ if not ct.startswith("crosstab('") or not ct.endswith("'::text)"):
+ raise ValueError('only simple crosstab calls supported')
+ ct = ct[10:-8]
+ if "'" in ct.replace("''", ""):
+ raise ValueError('only escaped single quotes allowed in query')
+ return ct.replace("''", "'")
+
+
+def datastore_dictionary(resource_id: str):
+ """
+ Return the data dictionary info for a resource
+ """
+ try:
+ return [
+ f for f in tk.get_action('datastore_search')(
+ {}, {
+ u'resource_id': resource_id,
+ u'limit': 0,
+ u'include_total': False})['fields']
+ if not f['id'].startswith(u'_')]
+ except (tk.ObjectNotFound, tk.NotAuthorized):
+ return []
+
+
+def datastore_search_sql_enabled(*args: Any) -> bool:
+ """
+ Return the configuration setting
+ if search sql is enabled as
+ CKAN__DATASTORE__SQLSEARCH__ENABLED
+ """
+ try:
+ config = tk.config.get('ckan.datastore.sqlsearch.enabled', False)
+ return tk.asbool(config)
+ except (tk.ObjectNotFound, tk.NotAuthorized):
+ return False
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/interfaces.py b/src/ckanext-d4science_theme/ckanext/datastore/interfaces.py
new file mode 100644
index 0000000..d706200
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/interfaces.py
@@ -0,0 +1,180 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from ckan.types import Context
+from typing import Any
+import ckan.plugins.interfaces as interfaces
+
+
+class IDatastore(interfaces.Interface):
+ '''Allow modifying Datastore queries'''
+
+ def datastore_validate(self, context: Context, data_dict: dict[str, Any],
+ fields_types: dict[str, str]):
+ '''Validates the ``data_dict`` sent by the user
+
+ This is the first method that's called. It's used to guarantee that
+ there aren't any unrecognized parameters, so other methods don't need
+ to worry about that.
+
+ You'll need to go through the received ``data_dict`` and remove
+ everything that you understand as valid. For example, if your extension
+ supports an ``age_between`` filter, you have to remove this filter from
+ the filters on the ``data_dict``.
+
+ The same ``data_dict`` will be passed to every IDatastore extension in
+ the order they've been loaded (the ``datastore`` plugin will always
+ come first). One extension will get the resulting ``data_dict`` from
+ the previous extensions. In the end, if the ``data_dict`` is empty, it
+ means that it's valid. If not, it's invalid and we throw an error.
+
+ Attributes on the ``data_dict`` that can be comma-separated strings
+ (e.g. fields) will already be converted to lists.
+
+ :param context: the context
+ :type context: dictionary
+ :param data_dict: the parameters received from the user
+ :type data_dict: dictionary
+ :param fields_types: the current resource's fields as dict keys and
+ their types as values
+ :type fields_types: dictionary
+ '''
+ return data_dict
+
+ def datastore_search(self, context: Context, data_dict: dict[str, Any],
+ fields_types: dict[str, str],
+ query_dict: dict[str, Any]):
+ '''Modify queries made on datastore_search
+
+ The overall design is that every IDatastore extension will receive the
+ ``query_dict`` with the modifications made by previous extensions, then
+ it can add/remove stuff into it before passing it on. You can think of
+ it as pipes, where the ``query_dict`` is being passed to each
+ IDatastore extension in the order they've been loaded allowing them to
+ change the ``query_dict``. The ``datastore`` extension always comes
+ first.
+
+ The ``query_dict`` is on the form:
+ {
+ 'select': [],
+ 'ts_query': '',
+ 'sort': [],
+ 'where': [],
+ 'limit': 100,
+ 'offset': 0
+ }
+
+ As extensions can both add and remove those keys, it's not guaranteed
+ that any of them will exist when you receive the ``query_dict``, so
+ you're supposed to test for its existence before, for example, adding a
+ new column to the ``select`` key.
+
+ The ``where`` key is a special case. It's elements are on the form:
+
+ (format_string, param1, param2, ...)
+
+ The ``format_string`` isn't escaped for SQL Injection attacks, so
+ everything coming from the user should be in the params list. With this
+ format, you could do something like:
+
+ ('"age" BETWEEN %s AND %s', age_between[0], age_between[1])
+
+ This escapes the ``age_between[0]`` and ``age_between[1]`` making sure
+ we're not vulnerable.
+
+ After finishing this, you should return your modified ``query_dict``.
+
+ :param context: the context
+ :type context: dictionary
+ :param data_dict: the parameters received from the user
+ :type data_dict: dictionary
+ :param fields_types: the current resource's fields as dict keys and
+ their types as values
+ :type fields_types: dictionary
+ :param query_dict: the current query_dict, as changed by the IDatastore
+ extensions that ran before yours
+ :type query_dict: dictionary
+
+ :returns: the query_dict with your modifications
+ :rtype: dictionary
+ '''
+ return query_dict
+
+ def datastore_delete(self, context: Context, data_dict: dict[str, Any],
+ fields_types: dict[str, str],
+ query_dict: dict[str, Any]):
+ '''Modify queries made on datastore_delete
+
+ The overall design is that every IDatastore extension will receive the
+ ``query_dict`` with the modifications made by previous extensions, then
+ it can add/remove stuff into it before passing it on. You can think of
+ it as pipes, where the ``query_dict`` is being passed to each
+ IDatastore extension in the order they've been loaded allowing them to
+ change the ``query_dict``. The ``datastore`` extension always comes
+ first.
+
+ The ``query_dict`` is on the form:
+ {
+ 'where': []
+ }
+
+ As extensions can both add and remove those keys, it's not guaranteed
+ that any of them will exist when you receive the ``query_dict``, so
+ you're supposed to test the existence of any keys before modifying
+ them.
+
+ The ``where`` elements are on the form:
+
+ (format_string, param1, param2, ...)
+
+ The ``format_string`` isn't escaped for SQL Injection attacks, so
+ everything coming from the user should be in the params list. With this
+ format, you could do something like:
+
+ ('"age" BETWEEN %s AND %s', age_between[0], age_between[1])
+
+ This escapes the ``age_between[0]`` and ``age_between[1]`` making sure
+ we're not vulnerable.
+
+ After finishing this, you should return your modified ``query_dict``.
+
+ :param context: the context
+ :type context: dictionary
+ :param data_dict: the parameters received from the user
+ :type data_dict: dictionary
+ :param fields_types: the current resource's fields as dict keys and
+ their types as values
+ :type fields_types: dictionary
+ :param query_dict: the current query_dict, as changed by the IDatastore
+ extensions that ran before yours
+ :type query_dict: dictionary
+
+ :returns: the query_dict with your modifications
+ :rtype: dictionary
+ '''
+ return query_dict
+
+
+class IDatastoreBackend(interfaces.Interface):
+ """Allow custom implementations of datastore backend"""
+ def register_backends(self) -> dict[str, Any]:
+ """
+ Register classes that inherits from DatastoreBackend.
+
+ Every registered class provides implementations of DatastoreBackend
+ and, depending on `datastore.write_url`, one of them will be used
+ inside actions.
+
+ `ckanext.datastore.DatastoreBackend` has method `set_active_backend`
+ which will define most suitable backend depending on schema of
+ `ckan.datastore.write_url` config directive. eg. 'postgresql://a:b@x'
+ will use 'postgresql' backend, but 'mongodb://a:b@c' will try to use
+ 'mongodb' backend(if such backend has been registered, of course).
+ If read and write urls use different engines, `set_active_backend`
+ will raise assertion error.
+
+
+ :returns: the dictonary with backend name as key and backend class as
+ value
+ """
+ return {}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/logic/__init__.py b/src/ckanext-d4science_theme/ckanext/datastore/logic/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/logic/action.py b/src/ckanext-d4science_theme/ckanext/datastore/logic/action.py
new file mode 100644
index 0000000..539a7d5
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/logic/action.py
@@ -0,0 +1,724 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from ckan.types import Context
+import logging
+from typing import Any, cast
+import json
+
+import sqlalchemy
+import sqlalchemy.exc
+import six
+from sqlalchemy.dialects.postgresql import TEXT, JSONB
+from sqlalchemy.sql.expression import func
+from sqlalchemy.sql.functions import coalesce
+
+import ckan.lib.search as search
+import ckan.lib.navl.dictization_functions
+import ckan.logic as logic
+import ckan.plugins as p
+import ckanext.datastore.logic.schema as dsschema
+import ckanext.datastore.helpers as datastore_helpers
+from ckanext.datastore.backend import (
+ DatastoreBackend, InvalidDataError
+)
+from ckanext.datastore.backend.postgres import identifier
+
+log = logging.getLogger(__name__)
+_get_or_bust = logic.get_or_bust
+_validate = ckan.lib.navl.dictization_functions.validate
+
+WHITELISTED_RESOURCES = ['_table_metadata']
+
+
+def datastore_create(context: Context, data_dict: dict[str, Any]):
+ '''Adds a new table to the DataStore.
+
+ The datastore_create action allows you to post JSON data to be
+ stored against a resource. This endpoint also supports altering tables,
+ aliases and indexes and bulk insertion. This endpoint can be called
+ multiple times to initially insert more data, add fields, change the
+ aliases or indexes as well as the primary keys.
+
+ To create an empty datastore resource and a CKAN resource at the same time,
+ provide ``resource`` with a valid ``package_id`` and omit the
+ ``resource_id``.
+
+ If you want to create a datastore resource from the content of a file,
+ provide ``resource`` with a valid ``url``.
+
+ See :ref:`fields` and :ref:`records` for details on how to lay out records.
+
+ :param resource_id: resource id that the data is going to be stored
+ against.
+ :type resource_id: string
+ :param force: set to True to edit a read-only resource
+ :type force: bool (optional, default: False)
+ :param resource: resource dictionary that is passed to
+ :meth:`~ckan.logic.action.create.resource_create`.
+ Use instead of ``resource_id`` (optional)
+ :type resource: dictionary
+ :param aliases: names for read only aliases of the resource. (optional)
+ :type aliases: list or comma separated string
+ :param fields: fields/columns and their extra metadata. (optional)
+ :type fields: list of dictionaries
+ :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a", "b"]}]
+ (optional)
+ :type records: list of dictionaries
+ :param primary_key: fields that represent a unique key (optional)
+ :type primary_key: list or comma separated string
+ :param indexes: indexes on table (optional)
+ :type indexes: list or comma separated string
+ :param triggers: trigger functions to apply to this table on update/insert.
+ functions may be created with
+ :meth:`~ckanext.datastore.logic.action.datastore_function_create`.
+ eg: [
+ {"function": "trigger_clean_reference"},
+ {"function": "trigger_check_codes"}]
+ :type triggers: list of dictionaries
+ :param calculate_record_count: updates the stored count of records, used to
+ optimize datastore_search in combination with the
+ `total_estimation_threshold` parameter. If doing a series of requests
+ to change a resource, you only need to set this to True on the last
+ request.
+ :type calculate_record_count: bool (optional, default: False)
+
+ Please note that setting the ``aliases``, ``indexes`` or ``primary_key``
+ replaces the existing aliases or constraints. Setting ``records`` appends
+ the provided records to the resource.
+
+ **Results:**
+
+ :returns: The newly created data object, excluding ``records`` passed.
+ :rtype: dictionary
+
+ See :ref:`fields` and :ref:`records` for details on how to lay out records.
+
+ '''
+ backend = DatastoreBackend.get_active_backend()
+ schema = context.get('schema', dsschema.datastore_create_schema())
+ records = data_dict.pop('records', None)
+ resource = data_dict.pop('resource', None)
+ data_dict, errors = _validate(data_dict, schema, context)
+ resource_dict = None
+ if records:
+ data_dict['records'] = records
+ if resource:
+ data_dict['resource'] = resource
+ if errors:
+ raise p.toolkit.ValidationError(errors)
+
+ p.toolkit.check_access('datastore_create', context, data_dict)
+
+ if 'resource' in data_dict and 'resource_id' in data_dict:
+ raise p.toolkit.ValidationError({
+ 'resource': ['resource cannot be used with resource_id']
+ })
+
+ if 'resource' not in data_dict and 'resource_id' not in data_dict:
+ raise p.toolkit.ValidationError({
+ 'resource_id': ['resource_id or resource required']
+ })
+
+ if 'resource' in data_dict:
+ has_url = 'url' in data_dict['resource']
+ # A datastore only resource does not have a url in the db
+ data_dict['resource'].setdefault('url', '_datastore_only_resource')
+ resource_dict = p.toolkit.get_action('resource_create')(
+ context, data_dict['resource'])
+ data_dict['resource_id'] = resource_dict['id']
+ # create resource from file
+ if has_url:
+ if not p.plugin_loaded('datapusher'):
+ raise p.toolkit.ValidationError({'resource': [
+ 'The datapusher has to be enabled.']})
+ p.toolkit.get_action('datapusher_submit')(context, {
+ 'resource_id': resource_dict['id'],
+ 'set_url_type': True
+ })
+ # since we'll overwrite the datastore resource anyway, we
+ # don't need to create it here
+ return
+
+ # create empty resource
+ else:
+ # no need to set the full url because it will be set
+ # in before_resource_show
+ resource_dict['url_type'] = 'datastore'
+ p.toolkit.get_action('resource_update')(context, resource_dict)
+ else:
+ if not data_dict.pop('force', False):
+ resource_id = data_dict['resource_id']
+ _check_read_only(context, resource_id)
+
+ # validate aliases
+ aliases = datastore_helpers.get_list(data_dict.get('aliases', [])) or []
+ for alias in aliases:
+ if not datastore_helpers.is_valid_table_name(alias):
+ raise p.toolkit.ValidationError({
+ 'alias': [u'"{0}" is not a valid alias name'.format(alias)]
+ })
+
+ try:
+ result = backend.create(context, data_dict)
+ except InvalidDataError as err:
+ raise p.toolkit.ValidationError({'message': str(err)})
+
+ if data_dict.get('calculate_record_count', False):
+ backend.calculate_record_count(data_dict['resource_id']) # type: ignore
+
+ # Set the datastore_active flag on the resource if necessary
+ model = _get_or_bust(cast("dict[str, Any]", context), 'model')
+ resobj = model.Resource.get(data_dict['resource_id'])
+ if resobj.extras.get('datastore_active') is not True:
+ log.debug(
+ 'Setting datastore_active=True on resource {0}'.format(resobj.id)
+ )
+ set_datastore_active_flag(context, data_dict, True)
+
+ result.pop('id', None)
+ result.pop('connection_url', None)
+ result.pop('records', None)
+ return result
+
+
+def datastore_run_triggers(context: Context, data_dict: dict[str, Any]) -> int:
+ ''' update each record with trigger
+
+ The datastore_run_triggers API action allows you to re-apply existing
+ triggers to an existing DataStore resource.
+
+ :param resource_id: resource id that the data is going to be stored under.
+ :type resource_id: string
+
+ **Results:**
+
+ :returns: The rowcount in the table.
+ :rtype: int
+
+ '''
+ res_id = data_dict['resource_id']
+ p.toolkit.check_access('datastore_run_triggers', context, data_dict)
+ backend = DatastoreBackend.get_active_backend()
+ connection = backend._get_write_engine().connect() # type: ignore
+
+ sql = sqlalchemy.text(u'''update {0} set _id=_id '''.format(
+ identifier(res_id)))
+ try:
+ results: Any = connection.execute(sql)
+ except sqlalchemy.exc.DatabaseError as err:
+ message = six.ensure_text(err.args[0].split('\n')[0])
+ raise p.toolkit.ValidationError({
+ u'records': [message.split(u') ', 1)[-1]]})
+ return results.rowcount
+
+
+def datastore_upsert(context: Context, data_dict: dict[str, Any]):
+ '''Updates or inserts into a table in the DataStore
+
+ The datastore_upsert API action allows you to add or edit records to
+ an existing DataStore resource. In order for the *upsert* and *update*
+ methods to work, a unique key has to be defined via the datastore_create
+ action. The available methods are:
+
+ *upsert*
+ Update if record with same key already exists, otherwise insert.
+ Requires unique key or _id field.
+ *insert*
+ Insert only. This method is faster that upsert, but will fail if any
+ inserted record matches an existing one. Does *not* require a unique
+ key.
+ *update*
+ Update only. An exception will occur if the key that should be updated
+ does not exist. Requires unique key or _id field.
+
+
+ :param resource_id: resource id that the data is going to be stored under.
+ :type resource_id: string
+ :param force: set to True to edit a read-only resource
+ :type force: bool (optional, default: False)
+ :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}]
+ (optional)
+ :type records: list of dictionaries
+ :param method: the method to use to put the data into the datastore.
+ Possible options are: upsert, insert, update
+ (optional, default: upsert)
+ :type method: string
+ :param calculate_record_count: updates the stored count of records, used to
+ optimize datastore_search in combination with the
+ `total_estimation_threshold` parameter. If doing a series of requests
+ to change a resource, you only need to set this to True on the last
+ request.
+ :type calculate_record_count: bool (optional, default: False)
+ :param dry_run: set to True to abort transaction instead of committing,
+ e.g. to check for validation or type errors.
+ :type dry_run: bool (optional, default: False)
+
+ **Results:**
+
+ :returns: The modified data object.
+ :rtype: dictionary
+
+ '''
+ backend = DatastoreBackend.get_active_backend()
+ schema = context.get('schema', dsschema.datastore_upsert_schema())
+ records = data_dict.pop('records', None)
+ data_dict, errors = _validate(data_dict, schema, context)
+ if records:
+ data_dict['records'] = records
+ if errors:
+ raise p.toolkit.ValidationError(errors)
+
+ p.toolkit.check_access('datastore_upsert', context, data_dict)
+
+ resource_id = data_dict['resource_id']
+
+ if not data_dict.pop('force', False):
+ _check_read_only(context, resource_id)
+
+ res_exists = backend.resource_exists(resource_id)
+ if not res_exists:
+ raise p.toolkit.ObjectNotFound(p.toolkit._(
+ u'Resource "{0}" was not found.'.format(resource_id)
+ ))
+
+ result = backend.upsert(context, data_dict)
+ p.toolkit.signals.datastore_upsert.send(resource_id, records=records)
+
+ result.pop('id', None)
+ result.pop('connection_url', None)
+
+ if data_dict.get('calculate_record_count', False):
+ backend.calculate_record_count(data_dict['resource_id']) # type: ignore
+
+ return result
+
+
+def datastore_info(context: Context, data_dict: dict[str, Any]):
+ '''
+ Returns detailed metadata about a resource.
+
+ :param resource_id: id or alias of the resource we want info about.
+ :type resource_id: string
+
+ **Results:**
+
+ :rtype: dictionary
+ :returns:
+ **meta**: resource metadata dictionary with the following keys:
+
+ - aliases - aliases (views) for the resource
+ - count - row count
+ - db_size - size of the datastore database (bytes)
+ - id - resource id (useful for dereferencing aliases)
+ - idx_size - size of all indices for the resource (bytes)
+ - size - size of resource (bytes)
+ - table_type - BASE TABLE, VIEW, FOREIGN TABLE or MATERIALIZED VIEW
+
+ **fields**: A list of dictionaries based on :ref:`fields`, with an
+ additional nested dictionary per field called **schema**, with the
+ following keys:
+
+ - native_type - native database data type
+ - index_name
+ - is_index
+ - notnull
+ - uniquekey
+
+ '''
+ backend = DatastoreBackend.get_active_backend()
+
+ resource_id = _get_or_bust(data_dict, 'id')
+ res_exists = backend.resource_exists(resource_id)
+ if not res_exists:
+ alias_exists, real_id = backend.resource_id_from_alias(resource_id)
+ if not alias_exists:
+ raise p.toolkit.ObjectNotFound(p.toolkit._(
+ u'Resource/Alias "{0}" was not found.'.format(resource_id)
+ ))
+ else:
+ id = real_id
+ else:
+ id = resource_id
+
+ data_dict['id'] = id
+ p.toolkit.check_access('datastore_info', context, data_dict)
+
+ p.toolkit.get_action('resource_show')(context, {'id': id})
+
+ info = backend.resource_fields(id)
+
+ return info
+
+
+def datastore_delete(context: Context, data_dict: dict[str, Any]):
+ '''Deletes a table or a set of records from the DataStore.
+
+ :param resource_id: resource id that the data will be deleted from.
+ (optional)
+ :type resource_id: string
+ :param force: set to True to edit a read-only resource
+ :type force: bool (optional, default: False)
+ :param filters: filters to apply before deleting (eg {"name": "fred"}).
+ If missing delete whole table and all dependent views.
+ (optional)
+ :type filters: dictionary
+ :param calculate_record_count: updates the stored count of records, used to
+ optimize datastore_search in combination with the
+ `total_estimation_threshold` parameter. If doing a series of requests
+ to change a resource, you only need to set this to True on the last
+ request.
+ :type calculate_record_count: bool (optional, default: False)
+
+ **Results:**
+
+ :returns: Original filters sent.
+ :rtype: dictionary
+
+ '''
+ schema = context.get('schema', dsschema.datastore_delete_schema())
+ backend = DatastoreBackend.get_active_backend()
+
+ # Remove any applied filters before running validation.
+ filters = data_dict.pop('filters', None)
+ data_dict, errors = _validate(data_dict, schema, context)
+
+ if filters is not None:
+ if not isinstance(filters, dict):
+ raise p.toolkit.ValidationError({
+ 'filters': [
+ 'filters must be either a dict or null.'
+ ]
+ })
+ data_dict['filters'] = filters
+
+ if errors:
+ raise p.toolkit.ValidationError(errors)
+
+ p.toolkit.check_access('datastore_delete', context, data_dict)
+
+ if not data_dict.pop('force', False):
+ resource_id = data_dict['resource_id']
+ _check_read_only(context, resource_id)
+
+ res_id = data_dict['resource_id']
+
+ res_exists = backend.resource_exists(res_id)
+
+ if not res_exists:
+ raise p.toolkit.ObjectNotFound(p.toolkit._(
+ u'Resource "{0}" was not found.'.format(res_id)
+ ))
+
+ result = backend.delete(context, data_dict)
+ p.toolkit.signals.datastore_delete.send(
+ res_id, result=result, data_dict=data_dict)
+ if data_dict.get('calculate_record_count', False):
+ backend.calculate_record_count(data_dict['resource_id']) # type: ignore
+
+ # Set the datastore_active flag on the resource if necessary
+ model = _get_or_bust(cast("dict[str, Any]", context), 'model')
+ resource = model.Resource.get(data_dict['resource_id'])
+
+ if (not data_dict.get('filters') and
+ resource is not None and
+ resource.extras.get('datastore_active') is True):
+ log.debug(
+ 'Setting datastore_active=False on resource {0}'.format(
+ resource.id)
+ )
+ set_datastore_active_flag(context, data_dict, False)
+
+ result.pop('id', None)
+ result.pop('connection_url', None)
+ return result
+
+
+@logic.side_effect_free
+def datastore_search(context: Context, data_dict: dict[str, Any]):
+ '''Search a DataStore resource.
+
+ The datastore_search action allows you to search data in a resource. By
+ default 100 rows are returned - see the `limit` parameter for more info.
+
+ A DataStore resource that belongs to a private CKAN resource can only be
+ read by you if you have access to the CKAN resource and send the
+ appropriate authorization.
+
+ :param resource_id: id or alias of the resource to be searched against
+ :type resource_id: string
+ :param filters: matching conditions to select, e.g
+ {"key1": "a", "key2": "b"} (optional)
+ :type filters: dictionary
+ :param q: full text query. If it's a string, it'll search on all fields on
+ each row. If it's a dictionary as {"key1": "a", "key2": "b"},
+ it'll search on each specific field (optional)
+ :type q: string or dictionary
+ :param full_text: full text query. It search on all fields on each row.
+ This should be used in replace of ``q`` when performing
+ string search accross all fields
+ :type full_text: string
+ :param distinct: return only distinct rows (optional, default: false)
+ :type distinct: bool
+ :param plain: treat as plain text query (optional, default: true)
+ :type plain: bool
+ :param language: language of the full text query
+ (optional, default: english)
+ :type language: string
+ :param limit: maximum number of rows to return
+ (optional, default: ``100``, unless set in the site's configuration
+ ``ckan.datastore.search.rows_default``, upper limit: ``32000`` unless
+ set in site's configuration ``ckan.datastore.search.rows_max``)
+ :type limit: int
+ :param offset: offset this number of rows (optional)
+ :type offset: int
+ :param fields: fields to return
+ (optional, default: all fields in original order)
+ :type fields: list or comma separated string
+ :param sort: comma separated field names with ordering
+ e.g.: "fieldname1, fieldname2 desc"
+ :type sort: string
+ :param include_total: True to return total matching record count
+ (optional, default: true)
+ :type include_total: bool
+ :param total_estimation_threshold: If "include_total" is True and
+ "total_estimation_threshold" is not None and the estimated total
+ (matching record count) is above the "total_estimation_threshold" then
+ this datastore_search will return an *estimate* of the total, rather
+ than a precise one. This is often good enough, and saves
+ computationally expensive row counting for larger results (e.g. >100000
+ rows). The estimated total comes from the PostgreSQL table statistics,
+ generated when Express Loader or DataPusher finishes a load, or by
+ autovacuum. NB Currently estimation can't be done if the user specifies
+ 'filters' or 'distinct' options. (optional, default: None)
+ :type total_estimation_threshold: int or None
+ :param records_format: the format for the records return value:
+ 'objects' (default) list of {fieldname1: value1, ...} dicts,
+ 'lists' list of [value1, value2, ...] lists,
+ 'csv' string containing comma-separated values with no header,
+ 'tsv' string containing tab-separated values with no header
+ :type records_format: controlled list
+
+
+ Setting the ``plain`` flag to false enables the entire PostgreSQL
+ `full text search query language`_.
+
+ A listing of all available resources can be found at the
+ alias ``_table_metadata``.
+
+ .. _full text search query language: http://www.postgresql.org/docs/9.1/static/datatype-textsearch.html#DATATYPE-TSQUERY
+
+ If you need to download the full resource, read :ref:`dump`.
+
+ **Results:**
+
+ The result of this action is a dictionary with the following keys:
+
+ :rtype: A dictionary with the following keys
+ :param fields: fields/columns and their extra metadata
+ :type fields: list of dictionaries
+ :param offset: query offset value
+ :type offset: int
+ :param limit: queried limit value (if the requested ``limit`` was above the
+ ``ckan.datastore.search.rows_max`` value then this response ``limit``
+ will be set to the value of ``ckan.datastore.search.rows_max``)
+ :type limit: int
+ :param filters: query filters
+ :type filters: list of dictionaries
+ :param total: number of total matching records
+ :type total: int
+ :param total_was_estimated: whether or not the total was estimated
+ :type total_was_estimated: bool
+ :param records: list of matching results
+ :type records: depends on records_format value passed
+
+ '''
+ backend = DatastoreBackend.get_active_backend()
+ schema = context.get('schema', dsschema.datastore_search_schema())
+ data_dict, errors = _validate(data_dict, schema, context)
+ if errors:
+ raise p.toolkit.ValidationError(errors)
+
+ res_id = data_dict['resource_id']
+
+ if data_dict['resource_id'] not in WHITELISTED_RESOURCES:
+ res_exists, real_id = backend.resource_id_from_alias(res_id)
+ # Resource only has to exist in the datastore (because it could be an
+ # alias)
+
+ if not res_exists:
+ raise p.toolkit.ObjectNotFound(p.toolkit._(
+ 'Resource "{0}" was not found.'.format(res_id)
+ ))
+
+ # Replace potential alias with real id to simplify access checks
+ if real_id:
+ data_dict['resource_id'] = real_id
+
+ p.toolkit.check_access('datastore_search', context, data_dict)
+
+ result = backend.search(context, data_dict)
+ result.pop('id', None)
+ result.pop('connection_url', None)
+ return result
+
+
+@logic.side_effect_free
+def datastore_search_sql(context: Context, data_dict: dict[str, Any]):
+ '''Execute SQL queries on the DataStore.
+
+ The datastore_search_sql action allows a user to search data in a resource
+ or connect multiple resources with join expressions. The underlying SQL
+ engine is the
+ `PostgreSQL engine
+
+
+
+
+ {{ _('Access resource data via a web API with powerful query support') }} .
+ {% trans %}
+ Further information in the main
+ CKAN Data API and DataStore documentation .
+ {% endtrans %}
+
+
+
+
+
+
{{ _('The Data API can be accessed via the following actions of the CKAN action API.') }}
+
+
+
+ {{ _('Create') }}
+ {{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_create', qualified=True) }}
+
+ {{ _('Update / Insert') }}
+ {{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_upsert', qualified=True) }}
+
+ {{ _('Query') }}
+ {{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', qualified=True) }}
+
+ {{ _('Query (via SQL)') }}
+ {{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search_sql', qualified=True) }}
+
+
+
+
+
+
+
+
+
+
+
+
{{ _('Query example (first 5 results)') }}
+
+ {{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5, qualified=True) }}
+
+
{{ _('Query example (results containing \'jones\')') }}
+
+ {{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, q='jones', qualified=True) }}
+
+
{{ _('Query example (via SQL statement)') }}
+
+ {{ sql_example_url }}
+
+
+
+
+
+
+
+
+
+
{{ _('A simple ajax (JSONP) request to the data API using jQuery.') }}
+
+ var data = {
+ resource_id: '{{resource_id}}', // the resource id
+ limit: 5, // get 5 results
+ q: 'jones' // query for 'jones'
+ };
+ $.ajax({
+ url: '{{ h.url_for(controller='api', action='action', ver=3, logic_function='datastore_search', qualified=True) }}',
+ data: data,
+ dataType: 'jsonp',
+ success: function(data) {
+ alert('Total results found: ' + data.result.total)
+ }
+ });
+
+
+
+
+
+
+
+
+
+ import urllib
+ url = '{{ h.url_for(qualified=True, controller='api', action='action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5) + '&q=title:jones' }}' {# not urlencoding the ":" because its clearer #}
+ fileobj = urllib.urlopen(url)
+ print fileobj.read()
+
+
+
+
+
+
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html
new file mode 100644
index 0000000..939bf0b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html
@@ -0,0 +1,40 @@
+{% call register_example('curl', 'button_label') %}
+curl
+{% endcall %}
+
+
+{% call register_example('curl', 'request_limit') %}
+
curl {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }} \
+ -H"Authorization:$API_TOKEN" -d '
+{
+ "resource_id": "{{ resource_id }}",
+ "limit": 5,
+ "q": "jones"
+}'
+{% endcall %}
+
+
+{% call register_example('curl', 'request_filter') %}
+
curl {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }} \
+-H"Authorization:$API_TOKEN" -d '
+{
+"resource_id": "{{ resource_id }}",
+ "filters": {
+ "subject": ["watershed", "survey"],
+ "stage": "active"
+ }
+}'
+{% endcall %}
+
+
+{% call register_example('curl', 'request_sql') %}
+
curl {{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }} \
+ -H"Authorization:$API_TOKEN" -d @- <<END
+{
+ "sql": "SELECT * FROM \"{{ resource_id }}\" WHERE title LIKE 'jones'"
+}
+END
+{% endcall %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html
new file mode 100644
index 0000000..d0a065d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html
@@ -0,0 +1,55 @@
+{% call register_example('javascript', 'button_label') %}
+JavaScript
+{% endcall %}
+
+{% call register_example('javascript', 'request_limit') %}
+
const resp = await fetch(`{{
+ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ authorization: API_TOKEN
+ },
+ body: JSON.stringify({
+ resource_id: '{{ resource_id }}',
+ limit: 5,
+ q: 'jones'
+ })
+})
+await resp.json()
+{% endcall %}
+
+
+{% call register_example('javascript', 'request_filter') %}
+
const resp = await fetch(`{{
+ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ authorization: API_TOKEN
+ },
+ body: JSON.stringify({resource_id: '{{ resource_id }}', filters: {
+ subject: ['watershed', 'survey'],
+ stage: 'active'
+ }})})
+await resp.json()
+{% endcall %}
+
+
+{% call register_example('javascript', 'request_sql') %}
+
const resp = await fetch(`{{
+ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ authorization: API_TOKEN
+ },
+ body: JSON.stringify({
+ sql: `SELECT * FROM "{{ resource_id }}" WHERE title LIKE 'jones'`
+ })
+})
+await resp.json()
+{% endcall %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html
new file mode 100644
index 0000000..c008d0a
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html
@@ -0,0 +1,48 @@
+{% call register_example('powershell', 'button_label') %}
+PowerShell
+{% endcall %}
+
+
+{% call register_example('powershell', 'request_limit') %}
+
$json = @'
+{
+ "resource_id": "{{resource_id}}",
+ "limit": 5,
+ "q": "jones"
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`
+ -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+{% endcall %}
+
+
+{% call register_example('powershell', 'request_filter') %}
+
$json = @'
+{
+ "resource_id": "{{resource_id}}",
+ "filters": {
+ "subject": ["watershed", "survey"],
+ "stage": "active"
+ }
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`
+ -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+{% endcall %}
+
+
+{% call register_example('powershell', 'request_sql') %}
+
$json = @'
+{
+ "sql": "SELECT * from \"{{resource_id}}\" WHERE title LIKE 'jones'"
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}`
+ -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+{% endcall %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html
new file mode 100644
index 0000000..9cd2045
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html
@@ -0,0 +1,52 @@
+{% call register_example('python', 'button_label') %}
+Python
+{% endcall %}
+
+
+{% call register_example('python', 'request_limit') %}
+
+
+ {% trans url='https://github.com/ckan/ckanapi/blob/master/README.md#remoteckan' -%}
+ (using the ckanapi client library)
+ {%- endtrans %}
+
+
from ckanapi import RemoteCKAN
+
+rc = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = rc.action.datastore_search(
+ resource_id="{{ resource_id }}",
+ limit=5,
+ q="jones",
+)
+print(result['records'])
+{% endcall %}
+
+
+{% call register_example('python', 'request_filter') %}
+
from ckanapi import RemoteCKAN
+
+rc = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = rc.action.datastore_search(
+ resource_id="{{ resource_id }}",
+ filters={
+ "subject": ["watershed", "survey"],
+ "stage": "active",
+ },
+)
+print(result['records'])
+{% endcall %}
+
+
+{% call register_example('python', 'request_sql') %}
+
from ckanapi import RemoteCKAN
+
+rc = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = rc.action.datastore_search_sql(
+ sql="""SELECT * from "{{resource_id}}" WHERE title LIKE 'jones'"""
+)
+print(result['records'])
+{% endcall %}
+
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html
new file mode 100644
index 0000000..4ba2d60
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html
@@ -0,0 +1,51 @@
+{% call register_example('r', 'button_label') %}
+R
+{% endcall %}
+
+
+{% call register_example('r', 'request_limit') %}
+
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}")
+result <- req %>%
+ req_headers(Authorization = API_TOKEN) %>%
+ req_body_json(list(
+ resource_id = '{{ resource_id }}',
+ limit = 5,
+ q = 'jones'))
+ req_perform %>%
+ resp_body_json
+{% endcall %}
+
+
+{% call register_example('r', 'request_filter') %}
+
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}")
+result <- req %>%
+ req_headers(Authorization = API_TOKEN) %>%
+ req_body_json(list(
+ resource_id='{{ resource_id }}',
+ filters = list(
+ subject = list("watershed", "survey"),
+ stage = "active")))
+ req_perform %>%
+ resp_body_json
+{% endcall %}
+
+
+{% call register_example('r', 'request_sql') %}
+
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}")
+result <- req %>%
+ req_headers(Authorization = API_TOKEN) %>%
+ req_body_json(list(
+ sql = "SELECT * FROM \"{{resource_id}}\" WHERE title LIKE 'jones'"))
+ req_perform %>%
+ resp_body_json
+
+{% endcall %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html
new file mode 100644
index 0000000..ac46294
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html
@@ -0,0 +1,22 @@
+{% extends "package/resource_edit_base.html" %}
+
+{% import 'macros/form.html' as form %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block primary_content_inner %}
+
+ {% set action = h.url_for('datastore.dictionary', id=pkg.name, resource_id=res.id) %}
+
+
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html
new file mode 100644
index 0000000..41266c9
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html
@@ -0,0 +1,25 @@
+{% import 'macros/form.html' as form %}
+
+
{{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})
+
+{#
+ Data Dictionary fields may be added this snippet. New fields following
+ the 'info__' ~ position ~ '__namegoeshere' convention will be saved
+ as part of the "info" object on the column.
+#}
+
+{{ form.select('info__' ~ position ~ '__type_override',
+ label=_('Type Override'), options=[
+ {'name': '', 'value': ''},
+ {'name': 'text', 'value': 'text'},
+ {'name': 'numeric', 'value': 'numeric'},
+ {'name': 'timestamp', 'value': 'timestamp'},
+ ], selected=field.get('info', {}).get('type_override', '')) }}
+
+{{ form.input('info__' ~ position ~ '__label',
+ label=_('Label'), id='field-f' ~ position ~ 'label',
+ value=field.get('info', {}).get('label', ''), classes=['control-full']) }}
+
+{{ form.markdown('info__' ~ position ~ '__notes',
+ label=_('Description'), id='field-d' ~ position ~ 'notes',
+ value=field.get('info', {}).get('notes', '')) }}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html
new file mode 100644
index 0000000..1152def
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html
@@ -0,0 +1,9 @@
+{% ckan_extends %}
+
+{% block inner_primary_nav %}
+ {{ super() }}
+ {% if res.datastore_active %}
+ {{ h.build_nav_icon('datastore.dictionary', _('Data Dictionary'),
+ id=pkg.name, resource_id=res.id, icon='code') }}
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html
new file mode 100644
index 0000000..67edefb
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html
@@ -0,0 +1,49 @@
+{% ckan_extends %}
+
+{% block resource_actions_inner %}
+ {{ super() }}
+ {% if res.datastore_active %}
+
{% snippet 'package/snippets/data_api_button.html', resource=res %}
+ {% endif %}
+{% endblock %}
+
+{% block resource_additional_information_inner %}
+ {% if res.datastore_active %}
+ {% block resource_data_dictionary %}
+
+
{{ _('Data Dictionary') }}
+
+
+ {% block resouce_data_dictionary_headers %}
+
+ {{ _('Column') }}
+ {{ _('Type') }}
+ {{ _('Label') }}
+ {{ _('Description') }}
+
+ {% endblock %}
+
+ {% block resource_data_dictionary_data %}
+ {% set dict=h.datastore_dictionary(res.id) %}
+ {% for field in dict %}
+ {% snippet "package/snippets/dictionary_table.html", field=field %}
+ {% endfor %}
+ {% endblock %}
+
+
+ {% endblock %}
+ {% endif %}
+ {{ super() }}
+{% endblock %}
+
+{% block action_manage_inner %}
+ {{ super() }}
+ {% if res.datastore_active %}
+
{% link_for _('Data Dictionary'), named_route='datastore.dictionary', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='code' %}
+ {% endif %}
+{% endblock %}
+
+{% block scripts %}
+ {{ super() }}
+ {% asset "ckanext_datastore/datastore" %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html
new file mode 100644
index 0000000..7320529
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html
@@ -0,0 +1,10 @@
+{# Data API Help Button
+
+resource: the resource
+
+#}
+{% if resource.datastore_active %}
+ {% set loading_text = _('Loading...') %}
+ {% set api_info_url = h.url_for('api.snippet', ver=1, snippet_path='api_info.html', resource_id=resource.id) %}
+
+ {{ field.id }}
+ {{ field.type }}
+ {{ h.get_translated(field.get('info', {}), 'label') }}
+ {{ h.render_markdown(
+ h.get_translated(field.get('info', {}), 'notes')) }}
+
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html
new file mode 100644
index 0000000..51daa78
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html
@@ -0,0 +1,8 @@
+{% ckan_extends %}
+
+{% block resource_item_explore_inner %}
+ {{ super() }}
+ {% if res.datastore_active %}
+ {% link_for _('Data Dictionary'), named_route='datastore.dictionary', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='code' %}
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html
new file mode 100644
index 0000000..31fbae8
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html
@@ -0,0 +1,8 @@
+{% ckan_extends %}
+
+{% block resources_list_edit_dropdown_inner %}
+ {{ super() }}
+ {% if resource.datastore_active %}
+ {% link_for _('Data Dictionary'), named_route='datastore.dictionary', id=pkg.name, resource_id=resource.id, class_='dropdown-item', icon='code' %}
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html b/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html
new file mode 100644
index 0000000..77cfc29
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html
@@ -0,0 +1,137 @@
+{#
+Displays information about accessing a resource via the API.
+
+resource_id - The resource id
+embedded - If true will not include the "modal" classes on the snippet.
+
+Example
+
+ {% snippet 'ajax_snippets/api_info.html', resource_id=resource_id, embedded=true %}
+
+#}
+
+{% set resource_id = h.sanitize_id(resource_id) %}
+{% set sql_example_url = h.url_for('api.action', ver=3, logic_function='datastore_search_sql', qualified=True) + '?sql=SELECT * from "' + resource_id + '" WHERE title LIKE \'jones\'' %}
+{# not urlencoding the sql because its clearer #}
+
+
+
+
+
+
{{ _('Access resource data via a web API with powerful query support') }} .
+ {% trans %}
+ Further information in the main
+ CKAN Data API and DataStore documentation .
+ {% endtrans %}
+
+
+
+ {{ _('Endpoints') }} »
+
+
+
+
{{ _('The Data API can be accessed via the following actions of the CKAN action API.') }}
+
+
+
+ {{ _('Create') }}
+ {{ h.url_for('api.action', ver=3, logic_function='datastore_create', qualified=True) }}
+
+ {{ _('Update / Insert') }}
+ {{ h.url_for('api.action', ver=3, logic_function='datastore_upsert', qualified=True) }}
+
+ {{ _('Query') }}
+ {{ h.url_for('api.action', ver=3, logic_function='datastore_search', qualified=True) }}
+ {% if h.datastore_search_sql_enabled() %}
+
+ {{ _('Query (via SQL)') }}
+ {{ h.url_for('api.action', ver=3, logic_function='datastore_search_sql', qualified=True) }}
+ {% endif %}
+
+
+
+
+
+
+
+
+ {{ _('Querying') }} »
+
+
+
+
{{ _('Query example (first 5 results)') }}
+
+ {{ h.url_for('api.action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5, qualified=True) }}
+
+
{{ _('Query example (results containing \'jones\')') }}
+
+ {{ h.url_for('api.action', ver=3, logic_function='datastore_search', resource_id=resource_id, q='jones', qualified=True) }}
+ {% if h.datastore_search_sql_enabled() %}
+
{{ _('Query example (via SQL statement)') }}
+
+ {{ sql_example_url }}
+ {% endif %}
+
+
+
+
+
+
+ {{ _('Example: Javascript') }} »
+
+
+
+
{{ _('A simple ajax (JSONP) request to the data API using jQuery.') }}
+
+ var data = {
+ resource_id: '{{resource_id}}', // the resource id
+ limit: 5, // get 5 results
+ q: 'jones' // query for 'jones'
+ };
+ $.ajax({
+ url: '{{ h.url_for('api.action', ver=3, logic_function='datastore_search', qualified=True) }}',
+ data: data,
+ dataType: 'jsonp',
+ success: function(data) {
+ alert('Total results found: ' + data.result.total)
+ }
+ });
+
+
+
+
+
+
+ {{ _('Example: Python') }} »
+
+
+
+
+ import urllib.request
+ url = '{{ h.url_for('api.action', qualified=True, ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5) + '&q=title:jones' }}' {# not urlencoding the ":" because its clearer #}
+ fileobj = urllib.request.urlopen(url)
+ print(fileobj.read())
+
+
+
+
+
+
+
+
+
+
+
{{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})
+
+{#
+ Data Dictionary fields may be added this snippet. New fields following
+ the 'info__' ~ position ~ '__namegoeshere' convention will be saved
+ as part of the "info" object on the column.
+#}
+
+{{ form.select('info__' ~ position ~ '__type_override',
+ label=_('Type Override'), options=[
+ {'name': '', 'value': ''},
+ {'name': 'text', 'value': 'text'},
+ {'name': 'numeric', 'value': 'numeric'},
+ {'name': 'timestamp', 'value': 'timestamp'},
+ ], selected=field.get('info', {}).get('type_override', '')) }}
+
+{{ form.input('info__' ~ position ~ '__label',
+ label=_('Label'), id='field-f' ~ position ~ 'label',
+ value=field.get('info', {}).get('label', ''), classes=['control-full']) }}
+
+{{ form.markdown('info__' ~ position ~ '__notes',
+ label=_('Description'), id='field-d' ~ position ~ 'notes',
+ value=field.get('info', {}).get('notes', '')) }}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html
new file mode 100644
index 0000000..a5dddbd
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html
@@ -0,0 +1,8 @@
+{% ckan_extends %}
+
+{% block inner_primary_nav %}
+ {{ super() }}
+ {% if res.datastore_active %}
+ {{ h.build_nav_icon('datastore.dictionary', _('Data Dictionary'), id=pkg.name, resource_id=res.id) }}
+ {% endif %}
+{% endblock %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html
new file mode 100644
index 0000000..2e1fdef
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html
@@ -0,0 +1,37 @@
+{% ckan_extends %}
+
+{% block resource_actions_inner %}
+ {{ super() }}
+ {% if res.datastore_active %}
+ {% snippet 'package/snippets/data_api_button.html', resource=res %}
+ {% endif %}
+{% endblock %}
+
+{% block resource_additional_information_inner %}
+ {% if res.datastore_active %}
+ {% block resource_data_dictionary %}
+
+
{{ _('Data Dictionary') }}
+
+
+ {% block resouce_data_dictionary_headers %}
+
+ {{ _('Column') }}
+ {{ _('Type') }}
+ {{ _('Label') }}
+ {{ _('Description') }}
+
+ {% endblock %}
+
+ {% block resource_data_dictionary_data %}
+ {% set dict=h.datastore_dictionary(res.id) %}
+ {% for field in dict %}
+ {% snippet "package/snippets/dictionary_table.html", field=field %}
+ {% endfor %}
+ {% endblock %}
+
+
+ {% endblock %}
+ {% endif %}
+ {{ super() }}
+{% endblock %}
\ No newline at end of file
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html
new file mode 100644
index 0000000..7320529
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html
@@ -0,0 +1,10 @@
+{# Data API Help Button
+
+resource: the resource
+
+#}
+{% if resource.datastore_active %}
+ {% set loading_text = _('Loading...') %}
+ {% set api_info_url = h.url_for('api.snippet', ver=1, snippet_path='api_info.html', resource_id=resource.id) %}
+
+{% endif %}
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html
new file mode 100644
index 0000000..adb07f3
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html
@@ -0,0 +1,7 @@
+
+ {{ field.id }}
+ {{ field.type }}
+ {{ h.get_translated(field.get('info', {}), 'label') }}
+ {{ h.render_markdown(
+ h.get_translated(field.get('info', {}), 'notes')) }}
+
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/__init__.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt b/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt
new file mode 100644
index 0000000..265e577
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt
@@ -0,0 +1,2 @@
+upper
+"count"
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py
new file mode 100644
index 0000000..e2b13c6
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+import ckanext.datastore.tests.helpers as test_helpers
+
+
+@pytest.fixture
+def clean_datastore(clean_db, clean_index):
+ engine = test_helpers.db.get_write_engine()
+ test_helpers.rebuild_all_dbs(
+ test_helpers.orm.scoped_session(
+ test_helpers.orm.sessionmaker(bind=engine)
+ )
+ )
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py
new file mode 100644
index 0000000..a8f1a05
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py
@@ -0,0 +1,107 @@
+# encoding: utf-8
+
+from sqlalchemy import orm
+
+import ckan.model as model
+from ckan.lib import search
+
+import ckan.plugins as p
+from ckan.tests.helpers import FunctionalTestBase, reset_db
+import ckanext.datastore.backend.postgres as db
+
+
+def extract(d, keys):
+ return dict((k, d[k]) for k in keys if k in d)
+
+
+def clear_db(Session): # noqa
+ drop_tables = u"""select 'drop table "' || tablename || '" cascade;'
+ from pg_tables where schemaname = 'public' """
+ c = Session.connection()
+ results = c.execute(drop_tables)
+ for result in results:
+ c.execute(result[0])
+
+ drop_functions_sql = u"""
+ SELECT 'drop function if exists ' || quote_ident(proname) || '();'
+ FROM pg_proc
+ INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid)
+ WHERE ns.nspname = 'public' AND proname != 'populate_full_text_trigger'
+ """
+ drop_functions = u"".join(r[0] for r in c.execute(drop_functions_sql))
+ if drop_functions:
+ c.execute(drop_functions)
+
+ Session.commit()
+ Session.remove()
+
+
+def rebuild_all_dbs(Session): # noqa
+ """ If the tests are running on the same db, we have to make sure that
+ the ckan tables are recrated.
+ """
+ db_read_url_parts = model.parse_db_config('ckan.datastore.write_url')
+ db_ckan_url_parts = model.parse_db_config('sqlalchemy.url')
+ same_db = db_read_url_parts["db_name"] == db_ckan_url_parts["db_name"]
+
+ if same_db:
+ model.repo.tables_created_and_initialised = False
+ clear_db(Session)
+ model.repo.rebuild_db()
+
+
+def set_url_type(resources, user):
+ context = {"user": user["name"]}
+ for resource in resources:
+ resource = p.toolkit.get_action("resource_show")(
+ context, {"id": resource.id}
+ )
+ resource["url_type"] = "datastore"
+ p.toolkit.get_action("resource_update")(context, resource)
+
+
+def execute_sql(sql, *args):
+ engine = db.get_write_engine()
+ session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ return session.connection().execute(sql, *args)
+
+
+def when_was_last_analyze(resource_id):
+ results = execute_sql(
+ """SELECT last_analyze
+ FROM pg_stat_user_tables
+ WHERE relname=%s;
+ """,
+ resource_id,
+ ).fetchall()
+ return results[0][0]
+
+
+class DatastoreFunctionalTestBase(FunctionalTestBase):
+ _load_plugins = (u"datastore",)
+
+ @classmethod
+ def setup_class(cls):
+ engine = db.get_write_engine()
+ rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine)))
+ super(DatastoreFunctionalTestBase, cls).setup_class()
+
+
+class DatastoreLegacyTestBase(object):
+ u"""
+ Tests that rely on data created in setup_class. No cleanup done between
+ each test method. Not recommended for new tests.
+ """
+
+ @classmethod
+ def setup_class(cls):
+ if not p.plugin_loaded(u"datastore"):
+ p.load(u"datastore")
+ reset_db()
+ search.clear_all()
+ engine = db.get_write_engine()
+ rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine)))
+
+ @classmethod
+ def teardown_class(cls):
+ p.unload(u"datastore")
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py
new file mode 100644
index 0000000..d82b813
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py
@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+import ckan.plugins as p
+
+import ckanext.datastore.interfaces as interfaces
+
+
+class SampleDataStorePlugin(p.SingletonPlugin):
+ p.implements(interfaces.IDatastore, inherit=True)
+
+ def datastore_validate(self, context, data_dict, column_names):
+ valid_filters = ("age_between", "age_not_between", "insecure_filter")
+ filters = data_dict.get("filters", {})
+ for key in list(filters.keys()):
+ if key in valid_filters:
+ del filters[key]
+
+ return data_dict
+
+ def datastore_search(self, context, data_dict, column_names, query_dict):
+ query_dict["where"] += self._where(data_dict)
+ return query_dict
+
+ def datastore_delete(self, context, data_dict, column_names, query_dict):
+ query_dict["where"] += self._where(data_dict)
+ return query_dict
+
+ def _where(self, data_dict):
+ filters = data_dict.get("filters", {})
+ where_clauses = []
+
+ if "age_between" in filters:
+ age_between = filters["age_between"]
+
+ clause = (
+ '"age" >= %s AND "age" <= %s',
+ age_between[0],
+ age_between[1],
+ )
+ where_clauses.append(clause)
+ if "age_not_between" in filters:
+ age_not_between = filters["age_not_between"]
+
+ clause = (
+ '"age" < %s OR "age" > %s',
+ age_not_between[0],
+ age_not_between[1],
+ )
+ where_clauses.append(clause)
+ if "insecure_filter" in filters:
+ insecure_filter = filters["insecure_filter"]
+
+ clause = (insecure_filter,)
+ where_clauses.append(clause)
+
+ return where_clauses
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py
new file mode 100644
index 0000000..dcb931d
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py
@@ -0,0 +1,278 @@
+# encoding: utf-8
+
+import pytest
+
+from ckan import model, logic
+from ckan.tests import helpers, factories
+
+
+@pytest.mark.ckan_config(u'ckan.plugins', u'datastore')
+@pytest.mark.usefixtures(u'clean_db', u'with_plugins')
+@pytest.mark.ckan_config(u'ckan.auth.allow_dataset_collaborators', True)
+class TestCollaboratorsDataStore():
+
+ def _get_context(self, user):
+
+ return {
+ u'model': model,
+ u'user': user if isinstance(user, str) else user.get(u'name')
+ }
+
+ def test_datastore_search_private_editor(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_search',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor')
+
+ assert helpers.call_auth(
+ u'datastore_search',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_search_private_member(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_search',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'member')
+
+ assert helpers.call_auth(
+ u'datastore_search',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_info_private_editor(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_info',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor')
+
+ assert helpers.call_auth(
+ u'datastore_info',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_info_private_member(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_info',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'member')
+
+ assert helpers.call_auth(
+ u'datastore_info',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_search_sql_private_editor(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ context[u'table_names'] = [resource[u'id']]
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_search_sql',
+ context=context, sql=u'SELECT * FROM "{}"'.format(
+ resource[u'id']))
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor')
+
+ assert helpers.call_auth(
+ u'datastore_search_sql',
+ context=context, sql=u'SELECT * FROM "{}"'.format(resource[u'id']))
+
+ def test_datastore_search_sql_private_member(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ context[u'table_names'] = [resource[u'id']]
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_search_sql',
+ context=context, sql=u'SELECT * FROM "{}"'.format(
+ resource[u'id']))
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'member')
+
+ assert helpers.call_auth(
+ u'datastore_search_sql',
+ context=context, sql=u'SELECT * FROM "{}"'.format(resource[u'id']))
+
+ def test_datastore_create_private_editor(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_create',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor')
+
+ assert helpers.call_auth(
+ u'datastore_create',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_create_private_member(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_create',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'member')
+
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_create',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_upsert_private_editor(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_upsert',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor')
+
+ assert helpers.call_auth(
+ u'datastore_upsert',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_upsert_private_member(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_upsert',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'member')
+
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_upsert',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_delete_private_editor(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_delete',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor')
+
+ assert helpers.call_auth(
+ u'datastore_delete',
+ context=context, resource_id=resource[u'id'])
+
+ def test_datastore_delete_private_member(self):
+
+ org = factories.Organization()
+ dataset = factories.Dataset(private=True, owner_org=org[u'id'])
+ resource = factories.Resource(package_id=dataset[u'id'])
+ user = factories.User()
+
+ context = self._get_context(user)
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_delete',
+ context=context, resource_id=resource[u'id'])
+
+ helpers.call_action(
+ u'package_collaborator_create',
+ id=dataset[u'id'], user_id=user[u'id'], capacity=u'member')
+
+ with pytest.raises(logic.NotAuthorized):
+ helpers.call_auth(
+ u'datastore_delete',
+ context=context, resource_id=resource[u'id'])
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py
new file mode 100644
index 0000000..e3ed8a1
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+
+import ckan.plugins as p
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+from ckan.logic.action.get import package_list as core_package_list
+
+
+package_list_message = u"The content of this message is largely irrelevant"
+
+
+class ActionTestException(Exception):
+ pass
+
+
+@p.toolkit.chained_action
+def datastore_delete(up_func, context, data_dict):
+ res = helpers.call_action(
+ u"datastore_search",
+ resource_id=data_dict[u"resource_id"],
+ filters=data_dict[u"filters"],
+ limit=10,
+ )
+ result = up_func(context, data_dict)
+ result["deleted_count"] = res.get(u"total", 0)
+ return result
+
+
+@p.toolkit.chained_action
+def package_list(next_func, context, data_dict):
+ # check it's received the core function as the first arg
+ assert next_func == core_package_list
+ raise ActionTestException(package_list_message)
+
+
+class ExampleDataStoreDeletedWithCountPlugin(p.SingletonPlugin):
+ p.implements(p.IActions)
+
+ def get_actions(self):
+ return {
+ u"datastore_delete": datastore_delete,
+ u"package_list": package_list,
+ }
+
+
+@pytest.mark.usefixtures(u"with_request_context")
+class TestChainedAction(object):
+ @pytest.mark.ckan_config(
+ u"ckan.plugins",
+ u"datastore example_datastore_deleted_with_count_plugin",
+ )
+ @pytest.mark.usefixtures(u"with_plugins", u"clean_db")
+ def test_datastore_delete_filters(self):
+ records = [{u"age": 20}, {u"age": 30}, {u"age": 40}]
+ resource = self._create_datastore_resource(records)
+ filters = {u"age": 30}
+
+ response = helpers.call_action(
+ u"datastore_delete",
+ resource_id=resource[u"id"],
+ force=True,
+ filters=filters,
+ )
+
+ result = helpers.call_action(
+ u"datastore_search", resource_id=resource[u"id"]
+ )
+
+ new_records_ages = [r[u"age"] for r in result[u"records"]]
+ new_records_ages.sort()
+ assert new_records_ages == [20, 40]
+ assert response["deleted_count"] == 1
+
+ def _create_datastore_resource(self, records):
+ dataset = factories.Dataset()
+ resource = factories.Resource(package=dataset)
+
+ data = {
+ u"resource_id": resource[u"id"],
+ u"force": True,
+ u"records": records,
+ }
+
+ helpers.call_action(u"datastore_create", **data)
+
+ return resource
+
+ @pytest.mark.ckan_config(
+ u"ckan.plugins",
+ u"datastore example_datastore_deleted_with_count_plugin",
+ )
+ @pytest.mark.usefixtures(u"with_plugins", u"clean_db")
+ def test_chain_core_action(self):
+ with pytest.raises(ActionTestException) as raise_context:
+ helpers.call_action(u"package_list", {})
+ assert raise_context.value.args == (package_list_message, )
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py
new file mode 100644
index 0000000..9c911a8
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import pytest
+
+import ckan.lib.create_test_data as ctd
+import ckan.plugins as p
+import ckan.tests.factories as factories
+from ckan.logic import check_access, NotAuthorized
+from ckan.logic.auth.get import user_list as core_user_list
+
+
+auth_message = u"No search for you"
+user_list_message = u"Nothing to see here"
+
+
+class AuthTestException(Exception):
+ pass
+
+
+@p.toolkit.chained_auth_function
+def datastore_search_sql_auth(up_func, context, data_dict):
+ # simple checks to confirm that the up_func we've received is the original
+ # sql search auth function
+ assert up_func.auth_allow_anonymous_access
+ assert up_func(context, data_dict) == {u"success": True}
+ raise AuthTestException(auth_message)
+
+
+@p.toolkit.chained_auth_function
+def user_list(next_auth, context, data_dict):
+ # check it's received the core function as the first arg
+ assert next_auth == core_user_list
+ raise AuthTestException(user_list_message)
+
+
+@p.toolkit.chained_auth_function
+def user_create(next_auth, context, data_dict):
+ return next_auth(context, data_dict)
+
+
+class ExampleDataStoreSearchSQLPlugin(p.SingletonPlugin):
+ p.implements(p.IAuthFunctions)
+
+ def get_auth_functions(self):
+ return {
+ u"datastore_search_sql": datastore_search_sql_auth,
+ u"user_list": user_list,
+ }
+
+
+@pytest.mark.ckan_config(
+ u"ckan.plugins", u"datastore example_data_store_search_sql_plugin"
+)
+@pytest.mark.usefixtures(u"with_request_context", u"with_plugins", u"clean_db")
+class TestChainedAuth(object):
+ def test_datastore_search_sql_auth(self):
+ ctd.CreateTestData.create()
+ with pytest.raises(AuthTestException) as raise_context:
+ # checking access should call to our chained version defined above
+ # first, thus this should throw an exception
+ check_access(
+ u"datastore_search_sql",
+ {u"user": u"annafan", u"table_names": []},
+ {},
+ )
+ # check that exception returned has the message from our auth function
+ assert raise_context.value.args == (auth_message, )
+
+ def test_chain_core_auth_functions(self):
+ user = factories.User()
+ context = {u"user": user[u"name"]}
+ with pytest.raises(AuthTestException) as raise_context:
+ check_access(u"user_list", context, {})
+ assert raise_context.value.args == (user_list_message, )
+ # check that the 'auth failed' msg doesn't fail because it's a partial
+ with pytest.raises(NotAuthorized):
+ check_access(
+ u"user_list",
+ {u"ignore_auth": False, u"user": u"not_a_real_user"},
+ {},
+ )
+
+
+class ExampleExternalProviderPlugin(p.SingletonPlugin):
+ p.implements(p.IAuthFunctions)
+
+ def get_auth_functions(self):
+ return {u"user_create": user_create}
+
+
+@pytest.mark.ckan_config(
+ u"ckan.plugins", u"datastore example_data_store_search_sql_plugin"
+)
+@pytest.mark.usefixtures(u"with_plugins", u"clean_db", u"with_request_context")
+class TestChainedAuthBuiltInFallback(object):
+
+ @pytest.mark.ckan_config("ckan.auth.create_user_via_web", True)
+ def test_user_create_chained_auth(self):
+ ctd.CreateTestData.create()
+ # check if chained auth fallbacks to built-in user_create
+ check_access(u"user_create", {u"user": u"annafan"}, {})
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py
new file mode 100644
index 0000000..31ddf92
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py
@@ -0,0 +1,1449 @@
+# encoding: utf-8
+
+import json
+import pytest
+import sqlalchemy.orm as orm
+
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.plugins as p
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+import ckanext.datastore.backend.postgres as db
+from ckan.common import config
+from ckan.plugins.toolkit import ValidationError
+from ckanext.datastore.tests.helpers import (
+ set_url_type,
+ execute_sql,
+ when_was_last_analyze,
+)
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreCreateNewTests(object):
+ def _has_index_on_field(self, resource_id, field):
+ sql = u"""
+ SELECT
+ relname
+ FROM
+ pg_class
+ WHERE
+ pg_class.relname = %s
+ """
+ index_name = db._generate_index_name(resource_id, field)
+ results = execute_sql(sql, index_name).fetchone()
+ return bool(results)
+
+ def _get_index_names(self, resource_id):
+ sql = u"""
+ SELECT
+ i.relname AS index_name
+ FROM
+ pg_class t,
+ pg_class i,
+ pg_index idx
+ WHERE
+ t.oid = idx.indrelid
+ AND i.oid = idx.indexrelid
+ AND t.relkind = 'r'
+ AND t.relname = %s
+ """
+ results = execute_sql(sql, resource_id).fetchall()
+ return [result[0] for result in results]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_works_with_empty_array_in_json_field(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {"package_id": package["id"]},
+ "fields": [
+ {"id": "movie", "type": "text"},
+ {"id": "directors", "type": "json"},
+ ],
+ "records": [{"movie": "sideways", "directors": []}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ assert result["resource_id"] is not None
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_works_with_empty_object_in_json_field(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {"package_id": package["id"]},
+ "fields": [
+ {"id": "movie", "type": "text"},
+ {"id": "director", "type": "json"},
+ ],
+ "records": [{"movie": "sideways", "director": {}}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ assert result["resource_id"] is not None
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_creates_index_on_primary_key(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "boo%k": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ }
+ }
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ index_names = self._get_index_names(resource_id)
+ assert resource_id + "_pkey" in index_names
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_creates_url_with_site_name(self):
+ package = factories.Dataset()
+ data = {"resource": {"boo%k": "crime", "package_id": package["id"]}}
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ resource = helpers.call_action("resource_show", id=resource_id)
+ url = resource["url"]
+ assert url.startswith(config.get("ckan.site_url"))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_index_on_specific_fields(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "boo%k": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ },
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "indexes": ["author"],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ assert self._has_index_on_field(resource_id, '"author"')
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_adds_index_on_full_text_search_when_creating_other_indexes(
+ self
+ ):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "boo%k": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ },
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "indexes": ["author"],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ assert self._has_index_on_field(resource_id, '"_full_text"')
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_adds_index_on_full_text_search_when_not_creating_other_indexes(
+ self
+ ):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "boo%k": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ },
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ assert self._has_index_on_field(resource_id, '"_full_text"')
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_add_full_text_search_indexes_on_every_text_field(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "book": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ },
+ "fields": [
+ {"id": "boo%k", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "lang": "english",
+ }
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ assert self._has_index_on_field(
+ resource_id, "to_tsvector('english', \"boo%k\")"
+ )
+ assert self._has_index_on_field(
+ resource_id, "to_tsvector('english', \"author\")"
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_doesnt_add_more_indexes_when_updating_data(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"book": "annakarenina", "author": "tolstoy"}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ previous_index_names = self._get_index_names(resource["id"])
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"book": "warandpeace", "author": "tolstoy"}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ current_index_names = self._get_index_names(resource["id"])
+ assert previous_index_names == current_index_names
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_duplicate_fields(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "book": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ },
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "book", "type": "text"},
+ ],
+ }
+ with pytest.raises(p.toolkit.ValidationError):
+ helpers.call_action("datastore_create", **data)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_sets_datastore_active_on_resource_on_create(self):
+ resource = factories.Resource()
+
+ assert not (resource["datastore_active"])
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"book": "annakarenina", "author": "tolstoy"}],
+ }
+
+ helpers.call_action("datastore_create", **data)
+
+ resource = helpers.call_action("resource_show", id=resource["id"])
+
+ assert resource["datastore_active"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_sets_datastore_active_on_resource_on_delete(self):
+ resource = factories.Resource(datastore_active=True)
+
+ assert resource["datastore_active"]
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"book": "annakarenina", "author": "tolstoy"}],
+ }
+
+ helpers.call_action("datastore_create", **data)
+
+ helpers.call_action(
+ "datastore_delete", resource_id=resource["id"], force=True
+ )
+
+ resource = helpers.call_action("resource_show", id=resource["id"])
+
+ assert not (resource["datastore_active"])
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_exceeds_column_name_limit(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {"package_id": package["id"]},
+ "fields": [
+ {
+ "id": "This is a really long name for a column. Column names "
+ "in Postgres have a limit of 63 characters",
+ "type": "text",
+ }
+ ],
+ }
+ with pytest.raises(p.toolkit.ValidationError):
+ helpers.call_action("datastore_create", **data)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_calculate_record_count_is_false(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "fields": [
+ {"id": "name", "type": "text"},
+ {"id": "age", "type": "text"},
+ ],
+ "records": [
+ {"name": "Sunita", "age": "51"},
+ {"name": "Bowan", "age": "68"},
+ ],
+ "force": True,
+ }
+ helpers.call_action("datastore_create", **data)
+ last_analyze = when_was_last_analyze(resource["id"])
+ assert last_analyze is None
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.flaky(reruns=2) # because analyze is sometimes delayed
+ def test_calculate_record_count(self):
+ # how datapusher loads data (send_resource_to_datastore)
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "fields": [
+ {"id": "name", "type": "text"},
+ {"id": "age", "type": "text"},
+ ],
+ "records": [
+ {"name": "Sunita", "age": "51"},
+ {"name": "Bowan", "age": "68"},
+ ],
+ "calculate_record_count": True,
+ "force": True,
+ }
+ helpers.call_action("datastore_create", **data)
+ last_analyze = when_was_last_analyze(resource["id"])
+ assert last_analyze is not None
+
+
+
+class TestDatastoreCreate(object):
+ sysadmin_user = None
+ normal_user = None
+
+ @pytest.fixture(autouse=True)
+ def create_test_data(self, clean_datastore, test_request_context):
+ ctd.CreateTestData.create()
+ self.sysadmin_user = factories.Sysadmin()
+ self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"])
+ self.sysadmin_token = self.sysadmin_token["token"]
+ self.normal_user = factories.User()
+ self.normal_user_token = factories.APIToken(user=self.normal_user["id"])
+ self.normal_user_token = self.normal_user_token["token"]
+ engine = db.get_write_engine()
+ self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ with test_request_context():
+ set_url_type(
+ model.Package.get("annakarenina").resources, self.sysadmin_user
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_requires_auth(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {"resource_id": resource.id}
+
+ res = app.post(
+ "/api/action/datastore_create", data=data
+ )
+ assert res.status_code == 403
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_empty_fails(self, app):
+ data = {}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ data=data,
+ extra_environ=auth,
+ )
+ assert res.status_code == 409
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_invalid_alias_name(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "aliases": u'foo"bar',
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ data = {
+ "resource_id": resource.id,
+ "aliases": u"fo%25bar", # alias with percent
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_duplicate_alias_name(self, app):
+ resource = factories.Resource(url_type = 'datastore')
+ data = {"resource_id": resource['id'], "aliases": u"myalias"}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ data=data,
+ extra_environ=auth,
+ )
+ assert res.status_code == 200
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ # try to create another table with the same alias
+ resource_2 = factories.Resource(url_type = 'datastore')
+ data = {"resource_id": resource_2['id'], "aliases": u"myalias"}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ data=data,
+ extra_environ=auth,
+ )
+ assert res.status_code == 409
+
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ # try to create an alias that is a resource id
+ data = {
+ "resource_id": resource_2['id'],
+ "aliases": resource_2['id'],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ data=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_invalid_field_type(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "book", "type": "int["}, # this is invalid
+ {"id": "author", "type": "INVALID"},
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_invalid_field_name(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ auth = {"Authorization": self.sysadmin_token}
+ invalid_names = [
+ "_author",
+ '"author',
+ "",
+ " author",
+ "author ",
+ "\tauthor",
+ "author\n",
+ ]
+
+ for field_name in invalid_names:
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": field_name, "type": "text"},
+ ],
+ }
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_invalid_record_field(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [
+ {"book": "annakarenina", "author": "tolstoy"},
+ {"book": "warandpeace", "published": "1869"},
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_bad_records(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": ["bad"], # treat author as null
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is False
+ assert res_dict["error"]["__type"] == "Validation Error"
+
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [
+ {"book": "annakarenina", "author": "tolstoy"},
+ [],
+ {"book": "warandpeace"},
+ ], # treat author as null
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is False
+ assert res_dict["error"]["__type"] == "Validation Error"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_invalid_index(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "indexes": "book, dummy",
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_invalid_unique_index(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "primary_key": "dummy",
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_alias_twice(self, app):
+ resource = model.Package.get("annakarenina").resources[1]
+ data = {
+ "resource_id": resource.id,
+ "aliases": "new_alias",
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True, res_dict
+
+ resource = model.Package.get("annakarenina").resources[0]
+ data = {
+ "resource_id": resource.id,
+ "aliases": "new_alias",
+ "fields": [{"id": "more books", "type": "text"}],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False, res_dict
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_basic(self, app):
+ resource = model.Package.get("annakarenina").resources[0]
+ aliases = [u"great_list_of_books", u"another_list_of_b\xfcks"]
+ ### Firstly test to see whether resource has no datastore table yet
+ data = {"id": resource.id}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/resource_show", data=data, extra_environ=auth
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["datastore_active"] is False
+ data = {
+ "resource_id": resource.id,
+ "aliases": aliases,
+ "fields": [
+ {"id": "boo%k", "type": "text"}, # column with percent
+ {"id": "author", "type": "json"},
+ ],
+ "indexes": [["boo%k", "author"], "author"],
+ "records": [
+ {"boo%k": "crime", "author": ["tolstoy", "dostoevsky"]},
+ {"boo%k": "annakarenina", "author": ["tolstoy", "putin"]},
+ {"boo%k": "warandpeace"},
+ ], # treat author as null
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True
+ res = res_dict["result"]
+ assert res["resource_id"] == data["resource_id"]
+ assert res["fields"] == data["fields"], res["fields"]
+
+ c = self.Session.connection()
+ results = c.execute('select * from "{0}"'.format(resource.id))
+
+ assert results.rowcount == 3
+ for i, row in enumerate(results):
+ assert data["records"][i].get("boo%k") == row["boo%k"]
+ assert data["records"][i].get("author") == (
+ json.loads(row["author"][0]) if row["author"] else None
+ )
+
+ results = c.execute(
+ """
+ select * from "{0}" where _full_text @@ to_tsquery('warandpeace')
+ """.format(
+ resource.id
+ )
+ )
+ assert results.rowcount == 1, results.rowcount
+
+ results = c.execute(
+ """
+ select * from "{0}" where _full_text @@ to_tsquery('tolstoy')
+ """.format(
+ resource.id
+ )
+ )
+ assert results.rowcount == 2
+ self.Session.remove()
+
+ # check aliases for resource
+ c = self.Session.connection()
+ for alias in aliases:
+
+ results = [
+ row
+ for row in c.execute(
+ u'select * from "{0}"'.format(resource.id)
+ )
+ ]
+ results_alias = [
+ row for row in c.execute(u'select * from "{0}"'.format(alias))
+ ]
+
+ assert results == results_alias
+
+ sql = (
+ u"select * from _table_metadata "
+ "where alias_of=%s and name=%s"
+ )
+ results = c.execute(sql, resource.id, alias)
+ assert results.rowcount == 1
+ self.Session.remove()
+
+ # check to test to see if resource now has a datastore table
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/resource_show", data={"id": resource.id}, extra_environ=auth
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["datastore_active"]
+
+ ####### insert again simple
+ data2 = {
+ "resource_id": resource.id,
+ "records": [{"boo%k": "hagji murat", "author": ["tolstoy"]}],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data2,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True
+
+ c = self.Session.connection()
+ results = c.execute('select * from "{0}"'.format(resource.id))
+ self.Session.remove()
+
+ assert results.rowcount == 4
+
+ all_data = data["records"] + data2["records"]
+ for i, row in enumerate(results):
+ assert all_data[i].get("boo%k") == row["boo%k"]
+ assert all_data[i].get("author") == (
+ json.loads(row["author"][0]) if row["author"] else None
+ )
+
+ c = self.Session.connection()
+ results = c.execute(
+ """
+ select * from "{0}" where _full_text @@ 'tolstoy'
+ """.format(
+ resource.id
+ )
+ )
+ self.Session.remove()
+ assert results.rowcount == 3
+
+ ####### insert again extra field
+ data3 = {
+ "resource_id": resource.id,
+ "records": [
+ {
+ "boo%k": "crime and punsihment",
+ "author": ["dostoevsky"],
+ "rating": 2,
+ }
+ ],
+ "indexes": ["rating"],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data3,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True
+
+ c = self.Session.connection()
+ results = c.execute('select * from "{0}"'.format(resource.id))
+
+ assert results.rowcount == 5
+
+ all_data = data["records"] + data2["records"] + data3["records"]
+ for i, row in enumerate(results):
+ assert all_data[i].get("boo%k") == row["boo%k"], (
+ i,
+ all_data[i].get("boo%k"),
+ row["boo%k"],
+ )
+ assert all_data[i].get("author") == (
+ json.loads(row["author"][0]) if row["author"] else None
+ )
+
+ results = c.execute(
+ """select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') """.format(
+ resource.id
+ )
+ )
+ self.Session.remove()
+ assert results.rowcount == 2
+
+ ####### insert again which will fail because of unique book name
+ data4 = {
+ "resource_id": resource.id,
+ "records": [{"boo%k": "warandpeace"}],
+ "primary_key": "boo%k",
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data4,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is False
+ assert "constraints" in res_dict["error"], res_dict
+
+ ####### insert again which should not fail because constraint is removed
+ data5 = {
+ "resource_id": resource.id,
+ "aliases": "another_alias", # replaces aliases
+ "records": [{"boo%k": "warandpeace"}],
+ "primary_key": "",
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data5,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True, res_dict
+
+ # new aliases should replace old aliases
+ c = self.Session.connection()
+ for alias in aliases:
+ sql = (
+ "select * from _table_metadata "
+ "where alias_of=%s and name=%s"
+ )
+ results = c.execute(sql, resource.id, alias)
+ assert results.rowcount == 0
+
+ sql = "select * from _table_metadata " "where alias_of=%s and name=%s"
+ results = c.execute(sql, resource.id, "another_alias")
+ assert results.rowcount == 1
+ self.Session.remove()
+
+ ####### insert array type
+ data6 = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "boo%k", "type": "text"},
+ {"id": "author", "type": "json"},
+ {"id": "rating", "type": "int"},
+ {"id": "characters", "type": "_text"},
+ ], # this is an array of strings
+ "records": [
+ {
+ "boo%k": "the hobbit",
+ "author": ["tolkien"],
+ "characters": ["Bilbo", "Gandalf"],
+ }
+ ],
+ "indexes": ["characters"],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data6,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True, res_dict
+
+ ####### insert type that requires additional lookup
+ data7 = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "boo%k", "type": "text"},
+ {"id": "author", "type": "json"},
+ {"id": "rating", "type": "int"},
+ {"id": "characters", "type": "_text"},
+ {"id": "location", "type": "int[2]"},
+ ],
+ "records": [
+ {
+ "boo%k": "lord of the rings",
+ "author": ["tolkien"],
+ "location": [3, -42],
+ }
+ ],
+ "indexes": ["characters"],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data7,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True, res_dict
+
+ ####### insert with parameter id rather than resource_id which is a shortcut
+ data8 = {
+ "id": resource.id,
+ # insert with percent
+ "records": [{"boo%k": "warandpeace", "author": "99% good"}],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data8,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True, res_dict
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_create_datastore_resource_on_dataset(self, app):
+ pkg = model.Package.get("annakarenina")
+
+ data = {
+ "resource": {"package_id": pkg.id},
+ "fields": [
+ {"id": "boo%k", "type": "text"}, # column with percent
+ {"id": "author", "type": "json"},
+ ],
+ "indexes": [["boo%k", "author"], "author"],
+ "records": [
+ {"boo%k": "crime", "author": ["tolstoy", "dostoevsky"]},
+ {"boo%k": "annakarenina", "author": ["tolstoy", "putin"]},
+ {"boo%k": "warandpeace"},
+ ], # treat author as null
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is True
+ res = res_dict["result"]
+ assert res["fields"] == data["fields"], res["fields"]
+
+ # Get resource details
+ data = {"id": res["resource_id"]}
+ res = app.post("/api/action/resource_show", data=data)
+ res_dict = json.loads(res.data)
+
+ assert res_dict["result"]["datastore_active"] is True
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_guess_types(self, app):
+ resource = model.Package.get("annakarenina").resources[1]
+
+ data = {"resource_id": resource.id}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ data=data,
+ extra_environ=auth,
+ ) # ignore status
+ res_dict = json.loads(res.data)
+
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "author", "type": "json"},
+ {"id": "count"},
+ {"id": "book"},
+ {"id": "date"},
+ ],
+ "records": [
+ {
+ "book": "annakarenina",
+ "author": "tolstoy",
+ "count": 1,
+ "date": "2005-12-01",
+ "count2": 0.5,
+ },
+ {"book": "crime", "author": ["tolstoy", "dostoevsky"]},
+ {"book": "warandpeace"},
+ ], # treat author as null
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ c = self.Session.connection()
+ results = c.execute("""select * from "{0}" """.format(resource.id))
+
+ types = [
+ db._pg_types[field[1]] for field in results.cursor.description
+ ]
+
+ assert types == [
+ u"int4",
+ u"tsvector",
+ u"nested",
+ u"int4",
+ u"text",
+ u"timestamp",
+ u"float8",
+ ], types
+
+ assert results.rowcount == 3
+ for i, row in enumerate(results):
+ assert data["records"][i].get("book") == row["book"]
+ assert data["records"][i].get("author") == (
+ json.loads(row["author"][0]) if row["author"] else None
+ )
+ self.Session.remove()
+
+ ### extend types
+
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "author", "type": "text"},
+ {"id": "count"},
+ {"id": "book"},
+ {"id": "date"},
+ {"id": "count2"},
+ {"id": "extra", "type": "text"},
+ {"id": "date2"},
+ ],
+ "records": [
+ {
+ "book": "annakarenina",
+ "author": "tolstoy",
+ "count": 1,
+ "date": "2005-12-01",
+ "count2": 2,
+ "nested": [1, 2],
+ "date2": "2005-12-01",
+ }
+ ],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+
+ c = self.Session.connection()
+ results = c.execute("""select * from "{0}" """.format(resource.id))
+ self.Session.remove()
+
+ types = [
+ db._pg_types[field[1]] for field in results.cursor.description
+ ]
+
+ assert types == [
+ u"int4", # id
+ u"tsvector", # fulltext
+ u"nested", # author
+ u"int4", # count
+ u"text", # book
+ u"timestamp", # date
+ u"float8", # count2
+ u"text", # extra
+ u"timestamp", # date2
+ u"nested", # count3
+ ], types
+
+ ### fields resupplied in wrong order
+
+ data = {
+ "resource_id": resource.id,
+ "fields": [
+ {"id": "author", "type": "text"},
+ {"id": "count"},
+ {"id": "date"}, # date and book in wrong order
+ {"id": "book"},
+ {"id": "count2"},
+ {"id": "extra", "type": "text"},
+ {"id": "date2"},
+ ],
+ "records": [
+ {
+ "book": "annakarenina",
+ "author": "tolstoy",
+ "count": 1,
+ "date": "2005-12-01",
+ "count2": 2,
+ "count3": 432,
+ "date2": "2005-12-01",
+ }
+ ],
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins")
+ def test_datastore_create_with_invalid_data_value(self, app):
+ """datastore_create() should return an error for invalid data."""
+ resource = factories.Resource(url_type="datastore")
+ data_dict = {
+ "resource_id": resource["id"],
+ "fields": [{"id": "value", "type": "numeric"}],
+ "records": [
+ {"value": 0},
+ {"value": 1},
+ {"value": 2},
+ {"value": 3},
+ {"value": " "}, # Invalid numeric value.
+ {"value": 5},
+ {"value": 6},
+ {"value": 7},
+ ],
+ "method": "insert",
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create",
+ json=data_dict,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+
+ assert res_dict["success"] is False
+ assert res_dict["error"]["__type"] == "Validation Error"
+ assert res_dict["error"]["message"].startswith("The data was invalid")
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreFunctionCreate(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_nop_trigger(self):
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_nop",
+ rettype=u"trigger",
+ definition=u"BEGIN RETURN NEW; END;",
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_invalid_definition(self):
+ with pytest.raises(ValidationError) as error:
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_invalid_def",
+ rettype=u"trigger",
+ definition=u"HELLO WORLD",
+ )
+ assert error.value.error_dict == {
+ u"definition": [u'syntax error at or near "HELLO"']
+ }
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_redefined_trigger(self):
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_redefined",
+ rettype=u"trigger",
+ definition=u"BEGIN RETURN NEW; END;",
+ )
+ with pytest.raises(ValidationError) as error:
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_redefined",
+ rettype=u"trigger",
+ definition=u"BEGIN RETURN NEW; END;",
+ )
+ assert error.value.error_dict == {
+ u"name": [
+ u'function "test_redefined" already exists '
+ u"with same argument types"
+ ]
+ }
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_redefined_with_or_replace_trigger(self):
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_replaceme",
+ rettype=u"trigger",
+ definition=u"BEGIN RETURN NEW; END;",
+ )
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_replaceme",
+ or_replace=True,
+ rettype=u"trigger",
+ definition=u"BEGIN RETURN NEW; END;",
+ )
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreCreateTriggers(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_with_missing_trigger(self, app):
+ ds = factories.Dataset()
+
+ with pytest.raises(ValidationError) as error:
+ with app.flask_app.test_request_context():
+ helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}],
+ triggers=[{u"function": u"no_such_trigger_function"}],
+ )
+ assert error.value.error_dict == {
+ u"triggers": [
+ u"function no_such_trigger_function() does not exist"
+ ]
+ }
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_trigger_applies_to_records(self, app):
+ ds = factories.Dataset()
+
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"spamify_trigger",
+ rettype=u"trigger",
+ definition=u"""
+ BEGIN
+ NEW.spam := 'spam spam ' || NEW.spam || ' spam';
+ RETURN NEW;
+ END;""",
+ )
+
+ with app.flask_app.test_request_context():
+ res = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}],
+ triggers=[{u"function": u"spamify_trigger"}],
+ )
+ assert helpers.call_action(
+ u"datastore_search",
+ fields=[u"spam"],
+ resource_id=res["resource_id"],
+ )["records"] == [
+ {u"spam": u"spam spam SPAM spam"},
+ {u"spam": u"spam spam EGGS spam"},
+ ]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_trigger_applies_to_records(self, app):
+ ds = factories.Dataset()
+
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"more_spam_trigger",
+ rettype=u"trigger",
+ definition=u"""
+ BEGIN
+ NEW.spam := 'spam spam ' || NEW.spam || ' spam';
+ RETURN NEW;
+ END;""",
+ )
+
+ with app.flask_app.test_request_context():
+ res = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ triggers=[{u"function": u"more_spam_trigger"}],
+ )
+ helpers.call_action(
+ u"datastore_upsert",
+ method=u"insert",
+ resource_id=res["resource_id"],
+ records=[{u"spam": u"BEANS"}, {u"spam": u"SPAM"}],
+ )
+ assert helpers.call_action(
+ u"datastore_search",
+ fields=[u"spam"],
+ resource_id=res["resource_id"],
+ )["records"] == [
+ {u"spam": u"spam spam BEANS spam"},
+ {u"spam": u"spam spam SPAM spam"},
+ ]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_trigger_exception(self, app):
+ ds = factories.Dataset()
+
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"spamexception_trigger",
+ rettype=u"trigger",
+ definition=u"""
+ BEGIN
+ IF NEW.spam != 'spam' THEN
+ RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam;
+ END IF;
+ RETURN NEW;
+ END;""",
+ )
+ with pytest.raises(ValidationError) as error:
+ with app.flask_app.test_request_context():
+ helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ records=[{u"spam": u"spam"}, {u"spam": u"EGGS"}],
+ triggers=[{u"function": u"spamexception_trigger"}],
+ )
+ assert error.value.error_dict == {u"records": [u'"EGGS"? Yeeeeccch!']}
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_trigger_exception(self, app):
+ ds = factories.Dataset()
+
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"spamonly_trigger",
+ rettype=u"trigger",
+ definition=u"""
+ BEGIN
+ IF NEW.spam != 'spam' THEN
+ RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam;
+ END IF;
+ RETURN NEW;
+ END;""",
+ )
+ with app.flask_app.test_request_context():
+ res = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ triggers=[{u"function": u"spamonly_trigger"}],
+ )
+ with pytest.raises(ValidationError) as error:
+ helpers.call_action(
+ u"datastore_upsert",
+ method=u"insert",
+ resource_id=res["resource_id"],
+ records=[{u"spam": u"spam"}, {u"spam": u"BEANS"}],
+ )
+ assert error.value.error_dict == {
+ u"records": [u'"BEANS"? Yeeeeccch!']
+ }
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py
new file mode 100644
index 0000000..01cc702
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py
@@ -0,0 +1,225 @@
+# encoding: utf-8
+
+import unittest.mock as mock
+import pytest
+import sqlalchemy.exc
+
+import ckan.lib.jobs as jobs
+import ckan.plugins as p
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+import ckanext.datastore.backend as backend
+import ckanext.datastore.backend.postgres as db
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("with_plugins")
+class TestCreateIndexes(object):
+ def test_creates_fts_index_using_gist_by_default(self):
+ connection = mock.MagicMock()
+ context = {"connection": connection}
+ resource_id = "resource_id"
+ data_dict = {"resource_id": resource_id}
+
+ db.create_indexes(context, data_dict)
+
+ self._assert_created_index_on(
+ "_full_text", connection, resource_id, method="gist"
+ )
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_index_method", "gin")
+ def test_default_fts_index_method_can_be_overwritten_by_config_var(self):
+ connection = mock.MagicMock()
+ context = {"connection": connection}
+ resource_id = "resource_id"
+ data_dict = {"resource_id": resource_id}
+
+ db.create_indexes(context, data_dict)
+
+ self._assert_created_index_on(
+ "_full_text", connection, resource_id, method="gin"
+ )
+
+ @mock.patch("ckanext.datastore.backend.postgres._get_fields")
+ def test_creates_fts_index_on_all_fields_except_dates_nested_and_arrays_with_english_as_default(
+ self, _get_fields
+ ):
+ _get_fields.return_value = [
+ {"id": "text", "type": "text"},
+ {"id": "number", "type": "number"},
+ {"id": "nested", "type": "nested"},
+ {"id": "date", "type": "date"},
+ {"id": "text array", "type": "text[]"},
+ {"id": "timestamp", "type": "timestamp"},
+ ]
+ connection = mock.MagicMock()
+ context = {"connection": connection}
+ resource_id = "resource_id"
+ data_dict = {"resource_id": resource_id}
+
+ db.create_indexes(context, data_dict)
+
+ self._assert_created_index_on(
+ "text", connection, resource_id, "english"
+ )
+ self._assert_created_index_on(
+ "number", connection, resource_id, "english", cast=True
+ )
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple")
+ @mock.patch("ckanext.datastore.backend.postgres._get_fields")
+ def test_creates_fts_index_on_textual_fields_can_overwrite_lang_with_config_var(
+ self, _get_fields
+ ):
+ _get_fields.return_value = [{"id": "foo", "type": "text"}]
+ connection = mock.MagicMock()
+ context = {"connection": connection}
+ resource_id = "resource_id"
+ data_dict = {"resource_id": resource_id}
+
+ db.create_indexes(context, data_dict)
+
+ self._assert_created_index_on("foo", connection, resource_id, "simple")
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple")
+ @mock.patch("ckanext.datastore.backend.postgres._get_fields")
+ def test_creates_fts_index_on_textual_fields_can_overwrite_lang_using_lang_param(
+ self, _get_fields
+ ):
+ _get_fields.return_value = [{"id": "foo", "type": "text"}]
+ connection = mock.MagicMock()
+ context = {"connection": connection}
+ resource_id = "resource_id"
+ data_dict = {"resource_id": resource_id, "language": "french"}
+
+ db.create_indexes(context, data_dict)
+
+ self._assert_created_index_on("foo", connection, resource_id, "french")
+
+ def _assert_created_index_on(
+ self,
+ field,
+ connection,
+ resource_id,
+ lang=None,
+ cast=False,
+ method="gist",
+ ):
+ field = u'"{0}"'.format(field)
+ if cast:
+ field = u"cast({0} AS text)".format(field)
+ if lang is not None:
+ sql_str = (
+ u'ON "resource_id" '
+ u"USING {method}(to_tsvector('{lang}', {field}))"
+ )
+ sql_str = sql_str.format(method=method, lang=lang, field=field)
+ else:
+ sql_str = u"USING {method}({field})".format(
+ method=method, field=field
+ )
+
+ calls = connection.execute.call_args_list
+ was_called = [call for call in calls if call[0][0].find(sql_str) != -1]
+
+ assert was_called, (
+ "Expected 'connection.execute' to have been "
+ "called with a string containing '%s'" % sql_str
+ )
+
+
+@mock.patch("ckanext.datastore.backend.postgres._get_fields")
+def test_upsert_with_insert_method_and_invalid_data(mock_get_fields_function):
+ """upsert_data() should raise InvalidDataError if given invalid data.
+
+ If the type of a field is numeric and upsert_data() is given a whitespace
+ value like " ", it should raise DataError.
+
+ In this case we're testing with "method": "insert" in the data_dict.
+
+ """
+ mock_connection = mock.Mock()
+ mock_connection.execute.side_effect = sqlalchemy.exc.DataError(
+ "statement", "params", "orig", connection_invalidated=False
+ )
+
+ context = {"connection": mock_connection}
+ data_dict = {
+ "fields": [{"id": "value", "type": "numeric"}],
+ "records": [
+ {"value": 0},
+ {"value": 1},
+ {"value": 2},
+ {"value": 3},
+ {"value": " "}, # Invalid numeric value.
+ {"value": 5},
+ {"value": 6},
+ {"value": 7},
+ ],
+ "method": "insert",
+ "resource_id": "fake-resource-id",
+ }
+
+ mock_get_fields_function.return_value = data_dict["fields"]
+
+ with pytest.raises(backend.InvalidDataError):
+ db.upsert_data(context, data_dict)
+
+
+class TestGetAllResourcesIdsInDatastore(object):
+ @pytest.mark.ckan_config(u"ckan.plugins", u"datastore")
+ @pytest.mark.usefixtures(u"with_plugins", u"clean_db")
+ def test_get_all_resources_ids_in_datastore(self):
+ resource_in_datastore = factories.Resource()
+ resource_not_in_datastore = factories.Resource()
+ data = {"resource_id": resource_in_datastore["id"], "force": True}
+ helpers.call_action("datastore_create", **data)
+
+ resource_ids = backend.get_all_resources_ids_in_datastore()
+
+ assert resource_in_datastore["id"] in resource_ids
+ assert resource_not_in_datastore["id"] not in resource_ids
+
+
+def datastore_job(res_id, value):
+ """
+ A background job that uses the Datastore.
+ """
+ app = helpers._get_test_app()
+ if not p.plugin_loaded(u"datastore"):
+ p.load("datastore")
+ data = {
+ "resource_id": res_id,
+ "method": "insert",
+ "records": [{"value": value}],
+ }
+
+ with app.flask_app.test_request_context():
+ helpers.call_action("datastore_upsert", **data)
+
+
+class TestBackgroundJobs(helpers.RQTestBase):
+ """
+ Test correct interaction with the background jobs system.
+ """
+ @pytest.mark.ckan_config(u"ckan.plugins", u"datastore")
+ @pytest.mark.usefixtures(u"with_plugins", u"clean_db", u"with_request_context")
+ def test_worker_datastore_access(self, app):
+ """
+ Test DataStore access from within a worker.
+ """
+ pkg = factories.Dataset()
+ data = {
+ "resource": {"package_id": pkg["id"]},
+ "fields": [{"id": "value", "type": "int"}],
+ }
+
+ table = helpers.call_action("datastore_create", **data)
+ res_id = table["resource_id"]
+ for i in range(3):
+ self.enqueue(datastore_job, args=[res_id, i])
+ jobs.Worker().work(burst=True)
+ # Aside from ensuring that the job succeeded, this also checks
+ # that accessing the Datastore still works in the main process.
+ result = helpers.call_action("datastore_search", resource_id=res_id)
+ assert [0, 1, 2] == [r["value"] for r in result["records"]]
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py
new file mode 100644
index 0000000..f12bc67
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py
@@ -0,0 +1,447 @@
+# encoding: utf-8
+
+import json
+import pytest
+
+import sqlalchemy.orm as orm
+
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+from ckan.tests import helpers
+from ckan.plugins.toolkit import ValidationError
+import ckan.tests.factories as factories
+from ckan.logic import NotFound
+import ckanext.datastore.backend.postgres as db
+from ckanext.datastore.tests.helpers import (
+ set_url_type,
+ when_was_last_analyze,
+ execute_sql,
+)
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreDelete(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_basic(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "aliases": u"b\xfck2",
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "rating with %", "type": "text"},
+ ],
+ "records": [
+ {
+ "book": "annakarenina",
+ "author": "tolstoy",
+ "rating with %": "90%",
+ },
+ {
+ "book": "warandpeace",
+ "author": "tolstoy",
+ "rating with %": "42%",
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {"resource_id": resource["id"], "force": True}
+ helpers.call_action("datastore_delete", **data)
+
+ # regression test for #7832
+ resobj = model.Resource.get(data["resource_id"])
+ assert resobj.extras.get('datastore_active') is False
+
+ results = execute_sql(
+ u"select 1 from pg_views where viewname = %s", u"b\xfck2"
+ )
+ assert results.rowcount == 0
+
+ # check the table is gone
+ results = execute_sql(
+ u"""SELECT table_name
+ FROM information_schema.tables
+ WHERE table_name=%s;""",
+ resource["id"],
+ )
+ assert results.rowcount == 0
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_calculate_record_count_is_false(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": "name", "type": "text"},
+ {"id": "age", "type": "text"},
+ ],
+ "records": [
+ {"name": "Sunita", "age": "51"},
+ {"name": "Bowan", "age": "68"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {
+ "resource_id": resource["id"],
+ "filters": {"name": "Bowan"},
+ "force": True,
+ }
+ helpers.call_action("datastore_delete", **data)
+ last_analyze = when_was_last_analyze(resource["id"])
+ assert last_analyze is None
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.flaky(reruns=2) # because analyze is sometimes delayed
+ def test_calculate_record_count(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": "name", "type": "text"},
+ {"id": "age", "type": "text"},
+ ],
+ "records": [
+ {"name": "Sunita", "age": "51"},
+ {"name": "Bowan", "age": "68"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {
+ "resource_id": resource["id"],
+ "filters": {"name": "Bowan"},
+ "calculate_record_count": True,
+ "force": True,
+ }
+ helpers.call_action("datastore_delete", **data)
+ last_analyze = when_was_last_analyze(resource["id"])
+ assert last_analyze is not None
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreDeleteLegacy(object):
+ sysadmin_user = None
+ normal_user = None
+ Session = None
+
+ @pytest.fixture(autouse=True)
+ def initial_data(self, clean_datastore, app, test_request_context):
+ self.app = app
+ ctd.CreateTestData.create()
+ self.sysadmin_user = factories.Sysadmin()
+ self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"])
+ self.sysadmin_token = self.sysadmin_token["token"]
+ self.normal_user = factories.User()
+ self.normal_user_token = factories.APIToken(user=self.normal_user["id"])
+ self.normal_user_token = self.normal_user_token["token"]
+ resource = model.Package.get("annakarenina").resources[0]
+ self.data = {
+ "resource_id": resource.id,
+ "aliases": u"b\xfck2",
+ "fields": [
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "rating with %", "type": "text"},
+ ],
+ "records": [
+ {
+ "book": "annakarenina",
+ "author": "tolstoy",
+ "rating with %": "90%",
+ },
+ {
+ "book": "warandpeace",
+ "author": "tolstoy",
+ "rating with %": "42%",
+ },
+ ],
+ }
+
+ engine = db.get_write_engine()
+
+ self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ with test_request_context():
+ set_url_type(
+ model.Package.get("annakarenina").resources, self.sysadmin_user
+ )
+
+ def _create(self):
+ auth = {"Authorization": self.sysadmin_token}
+ res = self.app.post(
+ "/api/action/datastore_create",
+ json=self.data,
+ environ_overrides=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ return res_dict
+
+ def _delete(self):
+ data = {"resource_id": self.data["resource_id"]}
+ auth = {"Authorization": self.sysadmin_token}
+ res = self.app.post(
+ "/api/action/datastore_delete",
+ data=data,
+ environ_overrides=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ assert res_dict["result"] == data
+ return res_dict
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("with_plugins", "with_request_context")
+ def test_datastore_deleted_during_resource_deletion(self):
+ package = factories.Dataset()
+ data = {
+ "resource": {
+ "boo%k": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ }
+ }
+
+ result = helpers.call_action("datastore_create", **data)
+ resource_id = result["resource_id"]
+ helpers.call_action("resource_delete", id=resource_id)
+
+ with pytest.raises(NotFound):
+ helpers.call_action("datastore_search", resource_id=resource_id)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_datastore_deleted_during_resource_only_for_deleted_resource(self):
+ package = factories.Dataset()
+ data = {
+ "boo%k": "crime",
+ "author": ["tolstoy", "dostoevsky"],
+ "package_id": package["id"],
+ }
+
+ result_1 = helpers.call_action(
+ "datastore_create", resource=data.copy()
+ )
+ resource_id_1 = result_1["resource_id"]
+
+ result_2 = helpers.call_action(
+ "datastore_create", resource=data.copy()
+ )
+ resource_id_2 = result_2["resource_id"]
+
+ res_1 = model.Resource.get(resource_id_1)
+ res_2 = model.Resource.get(resource_id_2)
+
+ # `synchronize_session=False` and session cache requires
+ # refreshing objects
+ model.Session.refresh(res_1)
+ model.Session.refresh(res_2)
+ assert res_1.extras["datastore_active"]
+ assert res_2.extras["datastore_active"]
+
+ helpers.call_action("resource_delete", id=resource_id_1)
+
+ with pytest.raises(NotFound):
+ helpers.call_action("datastore_search", resource_id=resource_id_1)
+ with pytest.raises(NotFound):
+ helpers.call_action("resource_show", id=resource_id_1)
+ model.Session.refresh(res_1)
+ model.Session.refresh(res_2)
+ assert not res_1.extras["datastore_active"]
+ assert res_2.extras["datastore_active"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_invalid_resource_id(self, app):
+ data = {"resource_id": "bad"}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ data=data,
+ environ_overrides=auth,
+ )
+ assert res.status_code == 404
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_filters(self, app):
+ self._create()
+ resource_id = self.data["resource_id"]
+
+ # try and delete just the 'warandpeace' row
+ data = {"resource_id": resource_id, "filters": {"book": "warandpeace"}}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ json=data,
+ environ_overrides=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ c = self.Session.connection()
+ result = c.execute(u'select * from "{0}";'.format(resource_id))
+ results = [r for r in result]
+ assert len(results) == 1
+ assert results[0].book == "annakarenina"
+ self.Session.remove()
+
+ # shouldn't delete anything
+ data = {
+ "resource_id": resource_id,
+ "filters": {"book": "annakarenina", "author": "bad"},
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ json=data,
+ environ_overrides=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ c = self.Session.connection()
+ result = c.execute(u'select * from "{0}";'.format(resource_id))
+ results = [r for r in result]
+ assert len(results) == 1
+ assert results[0].book == "annakarenina"
+ self.Session.remove()
+
+ # delete the 'annakarenina' row and also only use id
+ data = {
+ "id": resource_id,
+ "filters": {"book": "annakarenina", "author": "tolstoy"},
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ json=data,
+ environ_overrides=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ c = self.Session.connection()
+ result = c.execute(u'select * from "{0}";'.format(resource_id))
+ results = [r for r in result]
+ assert len(results) == 0
+ self.Session.remove()
+
+ self._delete()
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_is_unsuccessful_when_called_with_invalid_filters(
+ self, app
+ ):
+ self._create()
+
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": {"invalid-column-name": "value"},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ json=data,
+ environ_overrides=auth,
+ )
+ assert res.status_code == 409
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"].get("filters") is not None, res_dict["error"]
+
+ self._delete()
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_is_unsuccessful_when_called_with_filters_not_as_dict(
+ self, app
+ ):
+ self._create()
+
+ data = {"resource_id": self.data["resource_id"], "filters": []}
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_delete",
+ json=data,
+ environ_overrides=auth,
+ )
+ assert res.status_code == 409
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"].get("filters") is not None, res_dict["error"]
+
+ self._delete()
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_with_blank_filters(self, app):
+ self._create()
+
+ res = app.post(
+ "/api/action/datastore_delete",
+ json={"resource_id": self.data["resource_id"], "filters": {}},
+ environ_overrides={"Authorization": self.normal_user_token},
+ )
+ assert res.status_code == 200
+ results = json.loads(res.data)
+ assert results["success"] is True
+
+ res = app.post(
+ "/api/action/datastore_search",
+ json={"resource_id": self.data["resource_id"]},
+ environ_overrides={"Authorization": self.normal_user_token},
+ )
+ assert res.status_code == 200
+ results = json.loads(res.data)
+ assert results["success"] is True
+ assert len(results["result"]["records"]) == 0
+
+ self._delete()
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreFunctionDelete(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_create_delete(self):
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"test_nop",
+ rettype=u"trigger",
+ definition=u"BEGIN RETURN NEW; END;",
+ )
+ helpers.call_action(u"datastore_function_delete", name=u"test_nop")
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_nonexistant(self):
+ try:
+ helpers.call_action(
+ u"datastore_function_delete", name=u"test_not_there"
+ )
+ except ValidationError as ve:
+ assert ve.error_dict == {
+ u"name": [u"function test_not_there() does not exist"]
+ }
+ else:
+ assert 0, u"no validation error"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_if_exists(self):
+ helpers.call_action(
+ u"datastore_function_delete",
+ name=u"test_not_there_either",
+ if_exists=True,
+ )
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py
new file mode 100644
index 0000000..7de3e1b
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py
@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+import pytest
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+
+from ckanext.datapusher.tests import get_api_token
+
+
+@pytest.mark.ckan_config(u"ckan.plugins", u"datastore datapusher")
+@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token())
+@pytest.mark.usefixtures(u"clean_datastore", u"with_plugins", u"with_request_context")
+def test_read(app):
+ user = factories.User()
+ user_token = factories.APIToken(user=user["id"])
+ dataset = factories.Dataset(creator_user_id=user["id"])
+ resource = factories.Resource(
+ package_id=dataset["id"], creator_user_id=user["id"]
+ )
+ data = {
+ u"resource_id": resource["id"],
+ u"force": True,
+ u"records": [
+ {u"from": u"Brazil", u"to": u"Brazil", u"num": 2},
+ {u"from": u"Brazil", u"to": u"Italy", u"num": 22},
+ ],
+ }
+ helpers.call_action(u"datastore_create", **data)
+ auth = {u"Authorization": user_token["token"]}
+ app.get(
+ url=u"/dataset/{id}/dictionary/{resource_id}".format(
+ id=str(dataset["name"]), resource_id=str(resource["id"])
+ ),
+ extra_environ=auth,
+ )
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py
new file mode 100644
index 0000000..966e3f3
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py
@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+import pytest
+import ckan.plugins as p
+
+
+@pytest.mark.ckan_config("ckan.datastore.sqlsearch.enabled", None)
+def test_disabled_by_default():
+ with p.use_plugin("datastore"):
+ with pytest.raises(
+ KeyError, match=u"Action 'datastore_search_sql' not found"
+ ):
+ p.toolkit.get_action("datastore_search_sql")
+
+
+@pytest.mark.ckan_config("ckan.datastore.sqlsearch.enabled", False)
+def test_disable_sql_search():
+ with p.use_plugin("datastore"):
+ with pytest.raises(
+ KeyError, match=u"Action 'datastore_search_sql' not found"
+ ):
+ p.toolkit.get_action("datastore_search_sql")
+
+
+@pytest.mark.ckan_config("ckan.datastore.sqlsearch.enabled", True)
+def test_enabled_sql_search():
+ with p.use_plugin("datastore"):
+ p.toolkit.get_action("datastore_search_sql")
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py
new file mode 100644
index 0000000..6c0aa3e
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py
@@ -0,0 +1,669 @@
+# encoding: utf-8
+
+import unittest.mock as mock
+import json
+import pytest
+import six
+import ckan.tests.helpers as helpers
+import ckan.tests.factories as factories
+
+
+class TestDatastoreDump(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_basic(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"book": "annakarenina"}, {u"book": "warandpeace"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get("/datastore/dump/{0}".format(str(resource["id"])))
+ assert (
+ "_id,book\r\n"
+ "1,annakarenina\n"
+ "2,warandpeace\n" == six.ensure_text(response.data)
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_all_fields_types(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get("/datastore/dump/{0}".format(str(resource["id"])))
+ content = six.ensure_text(response.data)
+ expected = (
+ u"_id,b\xfck,author,published" u",characters,random_letters,nested"
+ )
+ assert content[: len(expected)] == expected
+ assert "warandpeace" in content
+ assert '"[""Princess Anna"",""Sergius""]"' in content
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_alias(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "aliases": "books",
+ "records": [{u"book": "annakarenina"}, {u"book": "warandpeace"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ # get with alias instead of id
+ response = app.get("/datastore/dump/books")
+ assert (
+ "_id,book\r\n"
+ "1,annakarenina\n"
+ "2,warandpeace\n" == six.ensure_text(response.data)
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_does_not_exist_raises_404(self, app):
+
+ app.get("/datastore/dump/does-not-exist", status=404)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_limit(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"book": "annakarenina"}, {u"book": "warandpeace"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=1".format(str(resource["id"]))
+ )
+ content = six.ensure_text(response.data)
+ expected_content = u"_id,book\r\n" u"1,annakarenina\n"
+ assert content == expected_content
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_q_and_fields(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ u"/datastore/dump/{0}?q=warandpeace&fields=nested,author".format(
+ resource["id"]
+ )
+ )
+ content = six.ensure_text(response.data)
+
+ expected_content = u"nested,author\r\n" u'"{""a"": ""b""}",tolstoy\n'
+ assert content == expected_content
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_csv_file_extension(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ u"/datastore/dump/{0}?limit=1&format=csv".format(
+ resource["id"]
+ )
+ )
+
+ attachment_filename = res.headers['Content-disposition']
+
+ expected_attch_filename = 'attachment; filename="{0}.csv"'.format(
+ resource['id'])
+
+ assert attachment_filename == expected_attch_filename
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_tsv(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ "/datastore/dump/{0}?limit=1&format=tsv".format(
+ str(resource["id"])
+ )
+ )
+ content = six.ensure_text(res.data)
+
+ expected_content = (
+ u"_id\tb\xfck\tauthor\tpublished\tcharacters\trandom_letters\t"
+ u"nested\r\n1\tannakarenina\ttolstoy\t2005-03-01T00:00:00\t"
+ u'"[""Princess Anna"",""Sergius""]"\t'
+ u'"[""a"",""e"",""x""]"\t"[""b"", '
+ u'{""moo"": ""moo""}]"\n'
+ )
+ assert content == expected_content
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_tsv_file_extension(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ "/datastore/dump/{0}?limit=1&format=tsv".format(
+ str(resource["id"])
+ )
+ )
+
+ attachment_filename = res.headers['Content-disposition']
+
+ expected_attch_filename = 'attachment; filename="{0}.tsv"'.format(
+ resource['id'])
+
+ assert attachment_filename == expected_attch_filename
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_json(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ "/datastore/dump/{0}?limit=1&format=json".format(
+ resource["id"]
+ )
+ )
+
+ content = json.loads(six.ensure_text(res.data))
+ expected_content = {
+ u'fields': [
+ {u'id': u'_id', u'type': u'int'},
+ {u'id': u'bük', u'type': u'text'},
+ {u'id': u'author', u'type': u'text'},
+ {u'id': u'published', u'type': u'timestamp'},
+ {u'id': u'characters', u'type': u'_text'},
+ {u'id': u'random_letters', u'type': u'_text'},
+ {u'id': u'nested', u'type': u'json'}
+ ],
+ u'records': [
+ [
+ 1, u'annakarenina', u'tolstoy', u'2005-03-01T00:00:00',
+ [u'Princess Anna', u'Sergius'],
+ [u'a', u'e', u'x'],
+ [u'b', {u'moo': u'moo'}]
+ ]
+ ]
+ }
+ assert content == expected_content
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_json_file_extension(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ "/datastore/dump/{0}?limit=1&format=json".format(
+ resource["id"]
+ )
+ )
+
+ attachment_filename = res.headers['Content-disposition']
+
+ expected_attch_filename = 'attachment; filename="{0}.json"'.format(
+ resource['id'])
+
+ assert attachment_filename == expected_attch_filename
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_xml(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ "/datastore/dump/{0}?limit=1&format=xml".format(
+ str(resource["id"])
+ )
+ )
+ content = six.ensure_text(res.data)
+ expected_content = (
+ u'
\n'
+ r''
+ u"annakarenina "
+ u"tolstoy "
+ u"2005-03-01T00:00:00 "
+ u""
+ u'Princess Anna '
+ u'Sergius '
+ u" "
+ u""
+ u'a '
+ u'e '
+ u'x '
+ u" "
+ u""
+ u'b '
+ u''
+ u'moo '
+ u" "
+ u" "
+ u"
\n"
+ u" \n"
+ )
+ assert content == expected_content
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_xml_file_extension(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "random_letters", "type": "text[]"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "random_letters": ["a", "e", "x"],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "random_letters": [],
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ res = app.get(
+ "/datastore/dump/{0}?limit=1&format=xml".format(
+ str(resource["id"])
+ )
+ )
+
+ attachment_filename = res.headers['Content-disposition']
+
+ expected_attch_filename = 'attachment; filename="{0}.xml"'.format(
+ resource['id'])
+
+ assert attachment_filename == expected_attch_filename
+
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "3")
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dump_with_low_rows_max(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get("/datastore/dump/{0}".format(str(resource["id"])))
+ assert get_csv_record_values(response.data) == list(range(12))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5)
+ def test_dump_pagination(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get("/datastore/dump/{0}".format(str(resource["id"])))
+ assert get_csv_record_values(response.data) == list(range(12))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "7")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5)
+ def test_dump_pagination_with_limit(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=11".format(str(resource["id"]))
+ )
+ assert get_csv_record_values(response.data) == list(range(11))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "7")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 6)
+ def test_dump_pagination_csv_with_limit_same_as_paginate(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=6".format(str(resource["id"]))
+ )
+ assert get_csv_record_values(response.data) == list(range(6))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "6")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5)
+ def test_dump_pagination_with_rows_max(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=7".format(str(resource["id"]))
+ )
+ assert get_csv_record_values(response.data) == list(range(7))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "6")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 6)
+ def test_dump_pagination_with_rows_max_same_as_paginate(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=7".format(str(resource["id"]))
+ )
+ assert get_csv_record_values(response.data) == list(range(7))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "7")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5)
+ def test_dump_pagination_json_with_limit(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=6&format=json".format(
+ str(resource["id"])
+ )
+ )
+ assert get_json_record_values(response.data) == list(range(6))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "6")
+ @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5)
+ def test_dump_pagination_json_with_rows_max(self, app):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{u"record": str(num)} for num in list(range(12))],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ response = app.get(
+ "/datastore/dump/{0}?limit=7&format=json".format(
+ str(resource["id"])
+ )
+ )
+ assert get_json_record_values(response.data) == list(range(7))
+
+
+def get_csv_record_values(response_body):
+ return [
+ int(record.split(",")[1]) for record in six.ensure_text(
+ response_body).split()[1:]
+ ]
+
+
+def get_json_record_values(response_body):
+ return [record[1] for record in json.loads(response_body)["records"]]
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py
new file mode 100644
index 0000000..201efec
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py
@@ -0,0 +1,214 @@
+# encoding: utf-8
+import re
+
+import pytest
+import sqlalchemy.orm as orm
+from sqlalchemy.exc import ProgrammingError
+
+import ckanext.datastore.backend.postgres as db
+import ckanext.datastore.backend.postgres as postgres_backend
+import ckanext.datastore.helpers as datastore_helpers
+
+
+class TestTypeGetters(object):
+ def test_get_list(self):
+ get_list = datastore_helpers.get_list
+ assert get_list(None) is None
+ assert get_list([]) == []
+ assert get_list("") == []
+ assert get_list("foo") == ["foo"]
+ assert get_list("foo, bar") == ["foo", "bar"]
+ assert get_list('foo_"bar, baz') == ['foo_"bar', "baz"]
+ assert get_list('"foo", "bar"') == ["foo", "bar"]
+ assert get_list(u"foo, bar") == ["foo", "bar"]
+ assert get_list(["foo", "bar"]) == ["foo", "bar"]
+ assert get_list([u"foo", u"bar"]) == ["foo", "bar"]
+ assert get_list(["foo", ["bar", "baz"]]) == ["foo", ["bar", "baz"]]
+
+ def test_is_single_statement(self):
+ singles = [
+ "SELECT * FROM footable",
+ 'SELECT * FROM "bartable"',
+ 'SELECT * FROM "bartable";',
+ 'SELECT * FROM "bart;able";',
+ "select 'foo'||chr(59)||'bar'",
+ ]
+
+ multiples = [
+ "SELECT * FROM abc; SET LOCAL statement_timeout to"
+ "SET LOCAL statement_timeout to; SELECT * FROM abc",
+ 'SELECT * FROM "foo"; SELECT * FROM "abc"',
+ ]
+
+ for single in singles:
+ assert postgres_backend.is_single_statement(single) is True
+
+ for multiple in multiples:
+ assert postgres_backend.is_single_statement(multiple) is False
+
+ def test_should_fts_index_field_type(self):
+ indexable_field_types = ["tsvector", "text", "number"]
+
+ non_indexable_field_types = [
+ "nested",
+ "timestamp",
+ "date",
+ "_text",
+ "text[]",
+ ]
+
+ for indexable in indexable_field_types:
+ assert (
+ datastore_helpers.should_fts_index_field_type(indexable)
+ is True
+ )
+
+ for non_indexable in non_indexable_field_types:
+ assert (
+ datastore_helpers.should_fts_index_field_type(non_indexable)
+ is False
+ )
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins")
+class TestGetTables(object):
+ def test_get_table_names(self):
+ engine = db.get_write_engine()
+ session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ create_tables = [
+ u"CREATE TABLE test_a (id_a text)",
+ u"CREATE TABLE test_b (id_b text)",
+ u'CREATE TABLE "TEST_C" (id_c text)',
+ u'CREATE TABLE test_d ("α/α" integer)',
+ ]
+ for create_table_sql in create_tables:
+ session.execute(create_table_sql)
+
+ test_cases = [
+ (u"SELECT * FROM test_a", ["test_a"]),
+ (u"SELECT * FROM public.test_a", ["test_a"]),
+ (u'SELECT * FROM "TEST_C"', ["TEST_C"]),
+ (u'SELECT * FROM public."TEST_C"', ["TEST_C"]),
+ (u"SELECT * FROM pg_catalog.pg_database", ["pg_database"]),
+ (u"SELECT rolpassword FROM pg_roles", ["pg_authid"]),
+ (
+ u"""SELECT p.rolpassword
+ FROM pg_roles p
+ JOIN test_b b
+ ON p.rolpassword = b.id_b""",
+ ["pg_authid", "test_b"],
+ ),
+ (
+ u"""SELECT id_a, id_b, id_c
+ FROM (
+ SELECT *
+ FROM (
+ SELECT *
+ FROM "TEST_C") AS c,
+ test_b) AS b,
+ test_a AS a""",
+ ["test_a", "test_b", "TEST_C"],
+ ),
+ (u"INSERT INTO test_a VALUES ('a')", ["test_a"]),
+ (u'SELECT "α/α" FROM test_d', ["test_d"]),
+ (u'SELECT "α/α" FROM test_d WHERE "α/α" > 1000', ["test_d"]),
+ ]
+
+ context = {"connection": session.connection()}
+ for case in test_cases:
+ assert sorted(
+ datastore_helpers.get_table_and_function_names_from_sql(context, case[0])[0]
+ ) == sorted(case[1])
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins")
+class TestGetFunctions(object):
+ def test_get_function_names(self):
+
+ engine = db.get_write_engine()
+ session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ create_tables = [
+ u"CREATE TABLE test_a (id int, period date, subject_id text, result decimal)",
+ u"CREATE TABLE test_b (name text, subject_id text)",
+ ]
+ for create_table_sql in create_tables:
+ session.execute(create_table_sql)
+
+ test_cases = [
+ (u"SELECT max(id) from test_a", ["max"]),
+ (u"SELECT count(distinct(id)) FROM test_a", ["count", "distinct"]),
+ (u"SELECT trunc(avg(result),2) FROM test_a", ["trunc", "avg"]),
+ (u"SELECT trunc(avg(result),2), avg(result) FROM test_a", ["trunc", "avg"]),
+ (u"SELECT * from pg_settings", ["pg_show_all_settings"]),
+ (u"SELECT * from pg_settings UNION SELECT * from pg_settings", ["pg_show_all_settings"]),
+ (u"SELECT * from (SELECT * FROM pg_settings) AS tmp", ["pg_show_all_settings"]),
+ (u"SELECT query_to_xml('SELECT max(id) FROM test_a', true, true , '')", ["query_to_xml"]),
+ (u"select $$'$$, query_to_xml($X$SELECT table_name FROM information_schema.tables$X$,true,true,$X$$X$), $$'$$", ["query_to_xml"])
+ ]
+
+ context = {"connection": session.connection()}
+ for case in test_cases:
+ assert sorted(
+ datastore_helpers.get_table_and_function_names_from_sql(context, case[0])[1]
+ ) == sorted(case[1])
+
+ def test_get_function_names_custom_function(self):
+
+ engine = db.get_write_engine()
+ session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ create_tables = [
+ u"""CREATE FUNCTION add(integer, integer) RETURNS integer
+ AS 'select $1 + $2;'
+ LANGUAGE SQL
+ IMMUTABLE
+ RETURNS NULL ON NULL INPUT;
+ """
+ ]
+ for create_table_sql in create_tables:
+ session.execute(create_table_sql)
+
+ context = {"connection": session.connection()}
+
+ sql = "SELECT add(1, 2);"
+
+ assert datastore_helpers.get_table_and_function_names_from_sql(context, sql)[1] == ["add"]
+
+ def test_get_function_names_crosstab(self):
+ """
+ Crosstab functions need to be enabled in the database by executing the following using
+ a super user:
+
+ CREATE extension tablefunc;
+
+ """
+
+ engine = db.get_write_engine()
+ session = orm.scoped_session(orm.sessionmaker(bind=engine))
+ create_tables = [
+ u"CREATE TABLE test_a (id int, period date, subject_id text, result decimal)",
+ u"CREATE TABLE test_b (name text, subject_id text)",
+ ]
+ for create_table_sql in create_tables:
+ session.execute(create_table_sql)
+
+ test_cases = [
+ (u"""SELECT *
+ FROM crosstab(
+ 'SELECT extract(month from period)::text, test_b.name, trunc(avg(result),2)
+ FROM test_a, test_b
+ WHERE test_a.subject_id = test_b.subject_id')
+ AS final_result(month text, subject_1 numeric,subject_2 numeric);""",
+ ['crosstab', 'final_result', 'extract', 'trunc', 'avg']),
+ ]
+
+ context = {"connection": session.connection()}
+ try:
+ for case in test_cases:
+ assert sorted(
+ datastore_helpers.get_table_and_function_names_from_sql(context, case[0])[1]
+ ) == sorted(case[1])
+ except ProgrammingError as e:
+ if bool(re.search("function crosstab(.*) does not exist", str(e))):
+ pytest.skip("crosstab functions not enabled in DataStore database")
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py
new file mode 100644
index 0000000..cb5ec51
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py
@@ -0,0 +1,88 @@
+# encoding: utf-8
+
+import pytest
+import six
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+from ckan.lib import helpers as template_helpers
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins")
+def test_info_success():
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"from": "Brazil", "to": "Brazil", "num": 2},
+ {"from": "Brazil", "to": "Italy", "num": 22},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ # aliases can only be created against an existing resource
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "aliases": "testalias1, testview2",
+
+ }
+ helpers.call_action("datastore_create", **data)
+
+ info = helpers.call_action("datastore_info", id=resource["id"])
+
+ assert len(info["meta"]) == 7, info["meta"]
+ assert info["meta"]["count"] == 2
+ assert info["meta"]["table_type"] == "BASE TABLE"
+ assert len(info["meta"]["aliases"]) == 2
+ assert info["meta"]["aliases"] == ["testview2", "testalias1"]
+ assert len(info["fields"]) == 3, info["fields"]
+ assert info["fields"][0]["id"] == "from"
+ assert info["fields"][0]["type"] == "text"
+ assert info["fields"][0]["schema"]["native_type"] == "text"
+ assert not info["fields"][0]["schema"]["is_index"]
+ assert info["fields"][2]["id"] == "num"
+ assert info["fields"][2]["schema"]["native_type"] == "integer"
+
+ # check datastore_info with alias
+ info = helpers.call_action("datastore_info", id='testalias1')
+
+ assert len(info["meta"]) == 7, info["meta"]
+ assert info["meta"]["count"] == 2
+ assert info["meta"]["id"] == resource["id"]
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins")
+def test_api_info(app):
+ dataset = factories.Dataset()
+ resource = factories.Resource(
+ id="588dfa82-760c-45a2-b78a-e3bc314a4a9b",
+ package_id=dataset["id"],
+ datastore_active=True,
+ )
+
+ # the 'API info' is seen on the resource_read page, a snippet loaded by
+ # javascript via data_api_button.html
+ url = template_helpers.url_for(
+ "api.snippet",
+ ver=1,
+ snippet_path="api_info.html",
+ resource_id=resource["id"],
+ )
+
+ page = app.get(url, status=200)
+
+ # check we built all the urls ok
+ expected_urls = (
+ "http://test.ckan.net/api/3/action/datastore_create",
+ "http://test.ckan.net/api/3/action/datastore_upsert",
+ "
http://test.ckan.net/api/3/action/datastore_search",
+ "http://test.ckan.net/api/3/action/datastore_search_sql",
+ "url: 'http://test.ckan.net/api/3/action/datastore_search'",
+ "http://test.ckan.net/api/3/action/datastore_search"
+ )
+ content = six.ensure_text(page.data)
+ for url in expected_urls:
+ assert url in content
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py
new file mode 100644
index 0000000..2c6fa2f
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py
@@ -0,0 +1,169 @@
+# encoding: utf-8
+
+import pytest
+import ckan.plugins as p
+import ckan.tests.helpers as helpers
+import ckan.tests.factories as factories
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_search_can_create_custom_filters():
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ filters = {"age_between": [25, 35]}
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"], filters=filters
+ )
+
+ assert result["total"] == 1, result
+ assert result["records"][0]["age"] == 30, result
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_search_filters_sent_arent_modified():
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ filters = {"age_between": [25, 35]}
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"], filters=filters.copy()
+ )
+
+ assert result["filters"] == filters
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_search_custom_filters_have_the_correct_operator_precedence():
+ """
+ We're testing that the WHERE clause becomes:
+ (age < 50 OR age > 60) AND age = 30
+ And not:
+ age < 50 OR age > 60 AND age = 30
+ """
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ filters = {"age_not_between": [50, 60], "age": 30}
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"], filters=filters
+ )
+
+ assert result["total"] == 1, result
+ assert result["records"][0]["age"] == 30, result
+ assert result["filters"] == filters
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_search_insecure_filter():
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ sql_inject = '1=1); DELETE FROM "%s"; COMMIT; SELECT * FROM "%s";--' % (
+ resource["id"],
+ resource["id"],
+ )
+ filters = {"insecure_filter": sql_inject}
+
+ with pytest.raises(p.toolkit.ValidationError):
+ helpers.call_action(
+ "datastore_search", resource_id=resource["id"], filters=filters
+ )
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"]
+ )
+
+ assert result["total"] == 3, result
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_delete_can_create_custom_filters():
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ filters = {"age_between": [25, 35]}
+
+ helpers.call_action(
+ "datastore_delete",
+ resource_id=resource["id"],
+ force=True,
+ filters=filters,
+ )
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"]
+ )
+
+ new_records_ages = [r["age"] for r in result["records"]]
+ new_records_ages.sort()
+ assert new_records_ages == [20, 40]
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_delete_custom_filters_have_the_correct_operator_precedence():
+ """
+ We're testing that the WHERE clause becomes:
+ (age < 50 OR age > 60) AND age = 30
+ And not:
+ age < 50 OR age > 60 AND age = 30
+ """
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ filters = {"age_not_between": [50, 60], "age": 30}
+
+ helpers.call_action(
+ "datastore_delete",
+ resource_id=resource["id"],
+ force=True,
+ filters=filters,
+ )
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"]
+ )
+
+ new_records_ages = [r["age"] for r in result["records"]]
+ new_records_ages.sort()
+ assert new_records_ages == [20, 40]
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin")
+@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context")
+def test_datastore_delete_insecure_filter():
+ records = [{"age": 20}, {"age": 30}, {"age": 40}]
+ resource = _create_datastore_resource(records)
+ sql_inject = '1=1); DELETE FROM "%s"; SELECT * FROM "%s";--' % (
+ resource["id"],
+ resource["id"],
+ )
+ filters = {"age": 20, "insecure_filter": sql_inject}
+
+ with pytest.raises(p.toolkit.ValidationError):
+ helpers.call_action(
+ "datastore_delete",
+ resource_id=resource["id"],
+ force=True,
+ filters=filters,
+ )
+
+ result = helpers.call_action(
+ "datastore_search", resource_id=resource["id"]
+ )
+
+ assert result["total"] == 3, result
+
+
+def _create_datastore_resource(records):
+ dataset = factories.Dataset()
+ resource = factories.Resource(package=dataset)
+
+ data = {"resource_id": resource["id"], "force": True, "records": records}
+
+ helpers.call_action("datastore_create", **data)
+
+ return resource
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py
new file mode 100644
index 0000000..c08f473
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py
@@ -0,0 +1,195 @@
+# encoding: utf-8
+
+import unittest.mock as mock
+import pytest
+
+import ckan.plugins as p
+import ckanext.datastore.interfaces as interfaces
+import ckanext.datastore.plugin as plugin
+
+
+DatastorePlugin = plugin.DatastorePlugin
+
+
+class TestPluginLoadingOrder(object):
+ def setup(self):
+ if p.plugin_loaded("datastore"):
+ p.unload("datastore")
+ if p.plugin_loaded("sample_datastore_plugin"):
+ p.unload("sample_datastore_plugin")
+
+ def teardown(self):
+ if p.plugin_loaded("sample_datastore_plugin"):
+ p.unload("sample_datastore_plugin")
+ if p.plugin_loaded("datastore"):
+ p.unload("datastore")
+
+ def test_loading_datastore_first_works(self):
+ p.load("datastore")
+ p.load("sample_datastore_plugin")
+ p.unload("sample_datastore_plugin")
+ p.unload("datastore")
+
+ def test_loading_datastore_last_doesnt_work(self):
+ # This test is complicated because we can't import
+ # ckanext.datastore.plugin before running it. If we did so, the
+ # DatastorePlugin class would be parsed which breaks the reason of our
+ # test.
+ p.load("sample_datastore_plugin")
+ thrown_exception = None
+ try:
+ p.load("datastore")
+ except Exception as e:
+ thrown_exception = e
+ idatastores = [
+ x.__class__.__name__
+ for x in p.PluginImplementations(interfaces.IDatastore)
+ ]
+ p.unload("sample_datastore_plugin")
+
+ assert thrown_exception is not None, (
+ 'Loading "datastore" after another IDatastore plugin was'
+ "loaded should raise DatastoreException"
+ )
+ assert (
+ thrown_exception.__class__.__name__
+ == plugin.DatastoreException.__name__
+ )
+ assert plugin.DatastorePlugin.__name__ not in idatastores, (
+ 'You shouldn\'t be able to load the "datastore" plugin after'
+ "another IDatastore plugin was loaded"
+ )
+
+
+@pytest.mark.ckan_config("ckan.plugins", "datastore")
+@pytest.mark.usefixtures("clean_index", "with_plugins", "with_request_context")
+class TestPluginDatastoreSearch(object):
+ def test_english_is_default_fts_language(self):
+ expected_ts_query = ", plainto_tsquery('english', 'foo') \"query\""
+ data_dict = {"q": "foo"}
+
+ result = self._datastore_search(data_dict=data_dict)
+
+ assert result["ts_query"] == expected_ts_query
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple")
+ def test_can_overwrite_default_fts_lang_using_config_variable(self):
+ expected_ts_query = ", plainto_tsquery('simple', 'foo') \"query\""
+ data_dict = {"q": "foo"}
+
+ result = self._datastore_search(data_dict=data_dict)
+
+ assert result["ts_query"] == expected_ts_query
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple")
+ def test_lang_parameter_overwrites_default_fts_lang(self):
+ expected_ts_query = ", plainto_tsquery('french', 'foo') \"query\""
+ data_dict = {"q": "foo", "language": "french"}
+
+ result = self._datastore_search(data_dict=data_dict)
+
+ assert result["ts_query"] == expected_ts_query
+
+ def test_fts_rank_column_uses_lang_when_casting_to_tsvector(self):
+ expected_select_content = (
+ u"to_tsvector('french', cast(\"country\" as text))"
+ )
+ data_dict = {"q": {"country": "Brazil"}, "language": "french"}
+ result = self._datastore_search(data_dict=data_dict, fields_types={})
+ assert expected_select_content in result["select"][0]
+
+ def test_adds_fts_on_full_text_field_when_q_is_a_string(self):
+ expected_where = [(u'_full_text @@ "query"',)]
+ data_dict = {"q": "foo"}
+
+ result = self._datastore_search(data_dict=data_dict)
+
+ assert result["where"] == expected_where
+
+ def test_ignores_fts_searches_on_inexistent_fields(self):
+ data_dict = {"q": {"inexistent-field": "value"}}
+
+ result = self._datastore_search(data_dict=data_dict, fields_types={})
+
+ assert result["where"] == []
+
+ def test_fts_where_clause_lang_uses_english_by_default(self):
+ expected_where = [
+ (
+ u"to_tsvector('english', cast(\"country\" as text))"
+ u' @@ "query country"',
+ )
+ ]
+ data_dict = {"q": {"country": "Brazil"}}
+ fields_types = {"country": "text"}
+
+ result = self._datastore_search(
+ data_dict=data_dict, fields_types=fields_types
+ )
+
+ assert result["where"] == expected_where
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple")
+ def test_fts_where_clause_lang_can_be_overwritten_by_config(self):
+ expected_where = [
+ (
+ u"to_tsvector('simple', cast(\"country\" as text))"
+ u' @@ "query country"',
+ )
+ ]
+ data_dict = {"q": {"country": "Brazil"}}
+ fields_types = {"country": "text"}
+
+ result = self._datastore_search(
+ data_dict=data_dict, fields_types=fields_types
+ )
+
+ assert result["where"] == expected_where
+
+ @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple")
+ def test_fts_where_clause_lang_can_be_overwritten_using_lang_param(self):
+ expected_where = [
+ (
+ u"to_tsvector('french', cast(\"country\" as text))"
+ u' @@ "query country"',
+ )
+ ]
+ data_dict = {"q": {"country": "Brazil"}, "language": "french"}
+ fields_types = {"country": "text"}
+
+ result = self._datastore_search(
+ data_dict=data_dict, fields_types=fields_types
+ )
+
+ assert result["where"] == expected_where
+
+ @mock.patch("ckanext.datastore.helpers.should_fts_index_field_type")
+ def test_fts_adds_where_clause_on_full_text_when_querying_non_indexed_fields(
+ self, should_fts_index_field_type
+ ):
+ should_fts_index_field_type.return_value = False
+ expected_where = [
+ ('_full_text @@ "query country"',),
+ (
+ u"to_tsvector('english', cast(\"country\" as text))"
+ u' @@ "query country"',
+ ),
+ ]
+ data_dict = {"q": {"country": "Brazil"}, "language": "english"}
+ fields_types = {"country": "non-indexed field type"}
+
+ result = self._datastore_search(
+ data_dict=data_dict, fields_types=fields_types
+ )
+
+ assert result["where"] == expected_where
+
+ def _datastore_search(
+ self, context={}, data_dict={}, fields_types={}, query_dict={}
+ ):
+ _query_dict = {"select": [], "sort": [], "where": []}
+ _query_dict.update(query_dict)
+
+ return DatastorePlugin().datastore_search(
+ context, data_dict, fields_types, _query_dict
+ )
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py
new file mode 100644
index 0000000..7f41ce3
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py
@@ -0,0 +1,2181 @@
+# encoding: utf-8
+
+import json
+import pytest
+import sqlalchemy.orm as orm
+
+import ckan.lib.create_test_data as ctd
+import ckan.logic as logic
+import ckan.model as model
+import ckan.plugins as p
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+import ckanext.datastore.backend.postgres as db
+from ckanext.datastore.tests.helpers import extract
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreSearch(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_fts_on_field_calculates_ranks_only_on_that_specific_field(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"from": "Brazil", "to": "Brazil"},
+ {"from": "Brazil", "to": "Italy"},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "fields": "from, rank from",
+ "q": {"from": "Brazil"},
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ ranks = [r["rank from"] for r in result["records"]]
+ assert len(result["records"]) == 2
+ assert len(set(ranks)) == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_fts_works_on_non_textual_fields(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"from": "Brazil", "year": {"foo": 2014}},
+ {"from": "Brazil", "year": {"foo": 1986}},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+
+ search_data = {
+ "resource_id": resource["id"],
+ "fields": "year",
+ "plain": False,
+ "q": {"year": "20:*"},
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert len(result["records"]) == 1
+ assert result["records"][0]["year"] == {"foo": 2014}
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_all_params_work_with_fields_with_whitespaces(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "fields": "the year",
+ "sort": "the year",
+ "filters": {"the year": 2013},
+ "q": {"the year": "2013"},
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ result_years = [r["the year"] for r in result["records"]]
+ assert result_years == [2013]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_total(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {"resource_id": resource["id"], "include_total": True}
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 2
+ assert not (result.get("total_was_estimated"))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_without_total(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {"resource_id": resource["id"], "include_total": False}
+ result = helpers.call_action("datastore_search", **search_data)
+ assert "total" not in result
+ assert "total_was_estimated" not in result
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(100)],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ analyze_sql = """
+ ANALYZE "{resource}";
+ """.format(
+ resource=resource["id"]
+ )
+ db.get_write_engine().execute(analyze_sql)
+ search_data = {
+ "resource_id": resource["id"],
+ "total_estimation_threshold": 50,
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result.get("total_was_estimated")
+ assert 95 < result["total"] < 105, result["total"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total_with_filters(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(3)] * 10,
+ }
+ result = helpers.call_action("datastore_create", **data)
+ analyze_sql = """
+ ANALYZE "{resource}";
+ """.format(
+ resource=resource["id"]
+ )
+ db.get_write_engine().execute(analyze_sql)
+ search_data = {
+ "resource_id": resource["id"],
+ "filters": {u"the year": 1901},
+ "total_estimation_threshold": 5,
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 10
+ # estimation is not compatible with filters
+ assert not (result.get("total_was_estimated"))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total_with_distinct(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(3)] * 10,
+ }
+ result = helpers.call_action("datastore_create", **data)
+ analyze_sql = """
+ ANALYZE "{resource}";
+ """.format(
+ resource=resource["id"]
+ )
+ db.get_write_engine().execute(analyze_sql)
+ search_data = {
+ "resource_id": resource["id"],
+ "fields": ["the year"],
+ "distinct": True,
+ "total_estimation_threshold": 1,
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 3
+ # estimation is not compatible with distinct
+ assert not (result.get("total_was_estimated"))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total_where_analyze_is_not_already_done(self):
+ # ANALYSE is done by latest datapusher/xloader, but need to cope in
+ # if tables created in other ways which may not have had an ANALYSE
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(100)],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "total_estimation_threshold": 50,
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result.get("total_was_estimated")
+ assert 95 < result["total"] < 105, result["total"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total_with_zero_threshold(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(100)],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ analyze_sql = """
+ ANALYZE "{resource}";
+ """.format(
+ resource=resource["id"]
+ )
+ db.get_write_engine().execute(analyze_sql)
+ search_data = {
+ "resource_id": resource["id"],
+ "total_estimation_threshold": 0,
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ # threshold of 0 means always estimate
+ assert result.get("total_was_estimated")
+ assert 95 < result["total"] < 105, result["total"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total_off(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(100)],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ analyze_sql = """
+ ANALYZE "{resource}";
+ """.format(
+ resource=resource["id"]
+ )
+ db.get_write_engine().execute(analyze_sql)
+ search_data = {
+ "resource_id": resource["id"],
+ "total_estimation_threshold": None,
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ # threshold of None means don't estimate
+ assert not (result.get("total_was_estimated"))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_estimate_total_default_off(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 1900 + i} for i in range(100)],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ analyze_sql = """
+ ANALYZE "{resource}";
+ """.format(
+ resource=resource["id"]
+ )
+ db.get_write_engine().execute(analyze_sql)
+ search_data = {
+ "resource_id": resource["id"],
+ # don't specify total_estimation_threshold
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ # default threshold is None, meaning don't estimate
+ assert not (result.get("total_was_estimated"))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_limit(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {"resource_id": resource["id"], "limit": 1}
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 2
+ assert result["records"] == [{u"the year": 2014, u"_id": 1}]
+ assert result["limit"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_limit_invalid(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ helpers.call_action("datastore_create", **data)
+ search_data = {"resource_id": resource["id"], "limit": "bad"}
+ with pytest.raises(logic.ValidationError, match="Invalid integer"):
+ helpers.call_action("datastore_search", **search_data)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_limit_invalid_negative(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ helpers.call_action("datastore_create", **data)
+ search_data = {"resource_id": resource["id"], "limit": -1}
+ with pytest.raises(
+ logic.ValidationError, match="Must be a natural number"
+ ):
+ helpers.call_action("datastore_search", **search_data)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_default", "1")
+ def test_search_limit_config_default(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ # limit not specified - leaving to the configured default of 1
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 2
+ assert result["records"] == [{u"the year": 2014, u"_id": 1}]
+ assert result["limit"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_default", "1")
+ def test_search_limit_config(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"the year": 2015},
+ {"the year": 2014},
+ {"the year": 2013},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "limit": 2, # specified limit overrides the rows_default
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 3
+ assert result["records"] == [
+ {u"the year": 2015, u"_id": 1},
+ {u"the year": 2014, u"_id": 2},
+ ]
+ assert result["limit"] == 2
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "1")
+ def test_search_limit_config_max(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ # limit not specified - leaving to the configured default of 1
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 2
+ assert result["records"] == [{u"the year": 2014, u"_id": 1}]
+ assert result["limit"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_default", "1")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "2")
+ def test_search_limit_config_combination(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"the year": 2016},
+ {"the year": 2015},
+ {"the year": 2014},
+ {"the year": 2013},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "limit": 3, # ignored because it is above rows_max
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 4
+ # returns 2 records,
+ # ignoring the rows_default because we specified limit
+ # but limit is more than rows_max so rows_max=2 wins
+ assert result["records"] == [
+ {u"the year": 2016, u"_id": 1},
+ {u"the year": 2015, u"_id": 2},
+ ]
+ assert result["limit"] == 2
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_filter_with_percent_in_column_name(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "bo%ok", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1%", "bo%ok": u"El Nino", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ search_data = {
+ "resource_id": resource["id"],
+ "filters": {u"bo%ok": "El Nino"},
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ assert result["total"] == 1
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreSearchLegacyTests(object):
+ sysadmin_user = None
+ normal_user = None
+
+ @pytest.fixture(autouse=True)
+ def initial_data(self, clean_datastore, app):
+ ctd.CreateTestData.create()
+ self.sysadmin_user = factories.Sysadmin()
+ self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"])
+ self.sysadmin_token = self.sysadmin_token["token"]
+ self.normal_user = factories.User()
+ self.normal_user_token = factories.APIToken(user=self.normal_user["id"])
+ self.normal_user_token = self.normal_user_token["token"]
+ self.dataset = model.Package.get("annakarenina")
+ self.resource = self.dataset.resources[0]
+ self.data = {
+ "resource_id": self.resource.id,
+ "force": True,
+ "aliases": "books3",
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ {"id": u"characters", u"type": u"_text"},
+ {"id": "rating with %"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ u"characters": [u"Princess Anna", u"Sergius"],
+ "rating with %": "60%",
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ "rating with %": "99%",
+ },
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create", json=self.data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ # Make an organization, because private datasets must belong to one.
+ self.organization = helpers.call_action(
+ "organization_create",
+ {"user": self.sysadmin_user["name"]},
+ name="test_org",
+ )
+
+ self.expected_records = [
+ {
+ u"published": u"2005-03-01T00:00:00",
+ u"_id": 1,
+ u"nested": [u"b", {u"moo": u"moo"}],
+ u"b\xfck": u"annakarenina",
+ u"author": u"tolstoy",
+ u"characters": [u"Princess Anna", u"Sergius"],
+ u"rating with %": u"60%",
+ },
+ {
+ u"published": None,
+ u"_id": 2,
+ u"nested": {u"a": u"b"},
+ u"b\xfck": u"warandpeace",
+ u"author": u"tolstoy",
+ u"characters": None,
+ u"rating with %": u"99%",
+ },
+ ]
+
+ engine = db.get_write_engine()
+ self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_basic(self, app):
+ data = {"resource_id": self.data["resource_id"]}
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == len(self.data["records"])
+ assert result["records"] == self.expected_records, result["records"]
+
+ # search with parameter id should yield the same results
+ data = {"id": self.data["resource_id"]}
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == len(self.data["records"])
+ assert result["records"] == self.expected_records, result["records"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_private_dataset(self, app):
+ group = self.dataset.get_groups()[0]
+ context = {
+ "user": self.sysadmin_user["name"],
+ "ignore_auth": True,
+ "model": model,
+ }
+ package = p.toolkit.get_action("package_create")(
+ context,
+ {
+ "name": "privatedataset",
+ "private": True,
+ "owner_org": self.organization["id"],
+ "groups": [{"id": group.id}],
+ },
+ )
+ resource = p.toolkit.get_action("resource_create")(
+ context,
+ {
+ "name": "privateresource",
+ "url": "https://www.example.com/",
+ "package_id": package["id"],
+ },
+ )
+ helpers.call_action("datastore_create", resource_id=resource["id"], force=True)
+ data = {"resource_id": resource["id"]}
+ auth = {"Authorization": self.normal_user_token}
+
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_alias(self, app):
+ data = {"resource_id": self.data["aliases"]}
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict_alias = json.loads(res.data)
+ result = res_dict_alias["result"]
+ assert result["total"] == len(self.data["records"])
+ assert result["records"] == self.expected_records, result["records"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_invalid_field(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "fields": [{"id": "bad"}],
+ }
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_fields(self, app):
+ data = {"resource_id": self.data["resource_id"], "fields": [u"b\xfck"]}
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == len(self.data["records"])
+ assert result["records"] == [
+ {u"b\xfck": "annakarenina"},
+ {u"b\xfck": "warandpeace"},
+ ], result["records"]
+
+ data = {
+ "resource_id": self.data["resource_id"],
+ "fields": u"b\xfck, author",
+ }
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == len(self.data["records"])
+ assert result["records"] == [
+ {u"b\xfck": "annakarenina", "author": "tolstoy"},
+ {u"b\xfck": "warandpeace", "author": "tolstoy"},
+ ], result["records"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_distinct(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "fields": [u"author"],
+ "distinct": True,
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 1
+ assert result["records"] == [{u"author": "tolstoy"}], result["records"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_filters(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": {u"b\xfck": "annakarenina"},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 1
+ assert result["records"] == [self.expected_records[0]]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_filter_array_field(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": {u"characters": [u"Princess Anna", u"Sergius"]},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 1
+ assert result["records"] == [self.expected_records[0]]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_multiple_filters_on_same_field(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": {u"b\xfck": [u"annakarenina", u"warandpeace"]},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 2
+ assert result["records"] == self.expected_records
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_filter_normal_field_passing_multiple_values_in_array(
+ self, app
+ ):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": {u"b\xfck": [u"annakarenina", u"warandpeace"]},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 2
+ assert result["records"] == self.expected_records, result["records"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_filters_get(self, app):
+ filters = {u"b\xfck": "annakarenina"}
+ res = app.get(
+ "/api/action/datastore_search?resource_id={0}&filters={1}".format(
+ self.data["resource_id"], json.dumps(filters)
+ )
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 1
+ assert result["records"] == [self.expected_records[0]]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_invalid_filter(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ # invalid because author is not a numeric field
+ "filters": {u"author": 42},
+ }
+
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_sort(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "sort": u"b\xfck asc, author desc",
+ }
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 2
+
+ assert result["records"] == self.expected_records, result["records"]
+
+ data = {
+ "resource_id": self.data["resource_id"],
+ "sort": [u"b\xfck desc", '"author" asc'],
+ }
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 2
+
+ assert result["records"] == self.expected_records[::-1]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_invalid(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "sort": u"f\xfc\xfc asc",
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ error_msg = res_dict["error"]["sort"][0]
+ assert (
+ u"f\xfc\xfc" in error_msg
+ ), 'Expected "{0}" to contain "{1}"'.format(error_msg, u"f\xfc\xfc")
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_offset(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "limit": 1,
+ "offset": 1,
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 2
+ assert result["records"] == [self.expected_records[1]]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_invalid_offset(self, app):
+ data = {"resource_id": self.data["resource_id"], "offset": "bad"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ data = {"resource_id": self.data["resource_id"], "offset": -1}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_full_text(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "annakarenina"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 1
+
+ results = [
+ extract(
+ result["records"][0],
+ [
+ u"_id",
+ u"author",
+ u"b\xfck",
+ u"nested",
+ u"published",
+ u"characters",
+ u"rating with %",
+ ],
+ )
+ ]
+ assert results == [self.expected_records[0]], results["records"]
+
+ data = {"resource_id": self.data["resource_id"], "q": "tolstoy"}
+
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 2
+ results = [
+ extract(
+ record,
+ [
+ u"_id",
+ u"author",
+ u"b\xfck",
+ u"nested",
+ u"published",
+ u"characters",
+ u"rating with %",
+ ],
+ )
+ for record in result["records"]
+ ]
+ assert results == self.expected_records, result["records"]
+
+ expected_fields = [
+ {u"type": u"int", u"id": u"_id"},
+ {u"type": u"text", u"id": u"b\xfck"},
+ {u"type": u"text", u"id": u"author"},
+ {u"type": u"timestamp", u"id": u"published"},
+ {u"type": u"json", u"id": u"nested"},
+ ]
+ for field in expected_fields:
+ assert field in result["fields"]
+
+ # test multiple word queries (connected with and)
+ data = {
+ "resource_id": self.data["resource_id"],
+ "plain": True,
+ "q": "tolstoy annakarenina",
+ }
+
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["total"] == 1
+ results = [
+ extract(
+ result["records"][0],
+ [
+ u"_id",
+ u"author",
+ u"b\xfck",
+ u"nested",
+ u"published",
+ u"characters",
+ u"rating with %",
+ ],
+ )
+ ]
+ assert results == [self.expected_records[0]], results["records"]
+
+ for field in expected_fields:
+ assert field in result["fields"], field
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_full_text_on_specific_column(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "q": {u"b\xfck": "annakarenina"},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ assert len(res_dict["result"]["records"]) == 1
+ assert (
+ res_dict["result"]["records"][0]["_id"]
+ == self.expected_records[0]["_id"]
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_full_text_on_specific_column_even_if_q_is_a_json_string(
+ self, app
+ ):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "q": u'{"b\xfck": "annakarenina"}',
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ assert len(res_dict["result"]["records"]) == 1
+ assert (
+ res_dict["result"]["records"][0]["_id"]
+ == self.expected_records[0]["_id"]
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_full_text_invalid_field_name(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "q": {"invalid_field_name": "value"},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_full_text_invalid_field_value(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "q": {"author": ["invalid", "value"]},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_table_metadata(self, app):
+ data = {"resource_id": "_table_metadata"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_is_unsuccessful_when_called_with_filters_not_as_dict(
+ self, app
+ ):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": "the-filter",
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"].get("filters") is not None, res_dict["error"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_is_unsuccessful_when_called_with_invalid_filters(
+ self, app
+ ):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "filters": {"invalid-column-name": "value"},
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"].get("filters") is not None, res_dict["error"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_is_unsuccessful_when_called_with_invalid_fields(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "fields": ["invalid-column-name"],
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search",
+ json=data,
+ extra_environ=auth,
+ status=409,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"].get("fields") is not None, res_dict["error"]
+
+
+class TestDatastoreFullTextSearchLegacyTests(object):
+ @pytest.fixture(autouse=True)
+ def initial_data(self, clean_datastore, app):
+ ctd.CreateTestData.create()
+ self.sysadmin_user = factories.Sysadmin()
+ self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"])
+ self.sysadmin_token = self.sysadmin_token["token"]
+ self.normal_user = factories.User()
+ self.normal_user_token = factories.APIToken(user=self.normal_user["id"])
+ self.normal_user_token = self.normal_user_token["token"]
+ resource = model.Package.get("annakarenina").resources[0]
+ self.data = dict(
+ resource_id=resource.id,
+ force=True,
+ fields=[
+ {"id": "id"},
+ {"id": "date", "type": "date"},
+ {"id": "x"},
+ {"id": "y"},
+ {"id": "z"},
+ {"id": "country"},
+ {"id": "title"},
+ {"id": "lat"},
+ {"id": "lon"},
+ ],
+ records=[
+ {
+ "id": 0,
+ "date": "2011-01-01",
+ "x": 1,
+ "y": 2,
+ "z": 3,
+ "country": "DE",
+ "title": "first 99",
+ "lat": 52.56,
+ "lon": 13.40,
+ },
+ {
+ "id": 1,
+ "date": "2011-02-02",
+ "x": 2,
+ "y": 4,
+ "z": 24,
+ "country": "UK",
+ "title": "second",
+ "lat": 54.97,
+ "lon": -1.60,
+ },
+ {
+ "id": 2,
+ "date": "2011-03-03",
+ "x": 3,
+ "y": 6,
+ "z": 9,
+ "country": "US",
+ "title": "third",
+ "lat": 40.00,
+ "lon": -75.5,
+ },
+ {
+ "id": 3,
+ "date": "2011-04-04",
+ "x": 4,
+ "y": 8,
+ "z": 6,
+ "country": "UK",
+ "title": "fourth",
+ "lat": 57.27,
+ "lon": -6.20,
+ },
+ {
+ "id": 4,
+ "date": "2011-05-04",
+ "x": 5,
+ "y": 10,
+ "z": 15,
+ "country": "UK",
+ "title": "fifth",
+ "lat": 51.58,
+ "lon": 0,
+ },
+ {
+ "id": 5,
+ "date": "2011-06-02",
+ "x": 6,
+ "y": 12,
+ "z": 18,
+ "country": "DE",
+ "title": "sixth 53.56",
+ "lat": 51.04,
+ "lon": 7.9,
+ },
+ ],
+ )
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_create", json=self.data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_search_full_text(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "DE"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 2
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_advanced_search_full_text(self, app):
+ data = {
+ "resource_id": self.data["resource_id"],
+ "plain": "False",
+ "q": "DE | UK",
+ }
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 5
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_full_text_search_on_integers_within_text_strings(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "99"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_full_text_search_on_integers(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "4"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 3
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_full_text_search_on_decimal_within_text_strings(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "53.56"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_full_text_search_on_decimal(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "52.56"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_full_text_search_on_date(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": "2011-01-01"}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["result"]["total"] == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_full_text_search_on_json_like_string_succeeds(self, app):
+ data = {"resource_id": self.data["resource_id"], "q": '"{}"'}
+
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"]
+
+
+class TestDatastoreSQLLegacyTests(object):
+ sysadmin_user = None
+ normal_user = None
+
+ @pytest.fixture(autouse=True)
+ def initial_data(self, clean_datastore, app):
+ ctd.CreateTestData.create()
+ self.sysadmin_user = factories.Sysadmin()
+ self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"])
+ self.sysadmin_token = self.sysadmin_token["token"]
+ self.normal_user = factories.User()
+ self.normal_user_token = factories.APIToken(user=self.normal_user["id"])
+ self.normal_user_token = self.normal_user_token["token"]
+ self.dataset = model.Package.get("annakarenina")
+ resource = self.dataset.resources[0]
+ self.data = {
+ "resource_id": resource.id,
+ "force": True,
+ "aliases": "books4",
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "published"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ },
+ ],
+ }
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_create", json=self.data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+
+ # Make an organization, because private datasets must belong to one.
+ self.organization = helpers.call_action(
+ "organization_create",
+ {"user": self.sysadmin_user["name"]},
+ name="test_org",
+ )
+
+ self.expected_records = [
+ {
+ u"_full_text": [
+ u"'annakarenina'",
+ u"'b'",
+ u"'moo'",
+ u"'tolstoy'",
+ u"'2005'",
+ ],
+ u"_id": 1,
+ u"author": u"tolstoy",
+ u"b\xfck": u"annakarenina",
+ u"nested": [u"b", {u"moo": u"moo"}],
+ u"published": u"2005-03-01T00:00:00",
+ },
+ {
+ u"_full_text": [u"'tolstoy'", u"'warandpeac'", u"'b'"],
+ u"_id": 2,
+ u"author": u"tolstoy",
+ u"b\xfck": u"warandpeace",
+ u"nested": {u"a": u"b"},
+ u"published": None,
+ },
+ ]
+ self.expected_join_results = [
+ {u"first": 1, u"second": 1},
+ {u"first": 1, u"second": 2},
+ ]
+
+ engine = db.get_write_engine()
+ self.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_select_where_like_with_percent(self, app):
+ query = 'SELECT * FROM public."{0}" WHERE "author" LIKE \'tol%\''.format(
+ self.data["resource_id"]
+ )
+ data = {"sql": query}
+ auth = {"Authorization": self.sysadmin_token}
+ res = app.post(
+ "/api/action/datastore_search_sql", json=data, extra_environ=auth,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert len(result["records"]) == len(self.expected_records)
+ for (row_index, row) in enumerate(result["records"]):
+ expected_row = self.expected_records[row_index]
+ assert set(row.keys()) == set(expected_row.keys())
+ for field in row:
+ if field == "_full_text":
+ for ft_value in expected_row["_full_text"]:
+ assert ft_value in row["_full_text"]
+ else:
+ assert row[field] == expected_row[field]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_self_join(self, app):
+ query = """
+ select a._id as first, b._id as second
+ from "{0}" AS a,
+ "{0}" AS b
+ where a.author = b.author
+ limit 2
+ """.format(
+ self.data["resource_id"]
+ )
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search_sql",
+ json={"sql": query},
+ extra_environ=auth,
+ )
+
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is True
+ result = res_dict["result"]
+ assert result["records"] == self.expected_join_results
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures(
+ "clean_datastore", "with_plugins", "with_request_context"
+ )
+ def test_new_datastore_table_from_private_resource(self, app):
+ # make a private CKAN resource
+ group = self.dataset.get_groups()[0]
+ context = {
+ "user": self.sysadmin_user["name"],
+ "ignore_auth": True,
+ "model": model,
+ }
+ package = p.toolkit.get_action("package_create")(
+ context,
+ {
+ "name": "privatedataset",
+ "private": True,
+ "owner_org": self.organization["id"],
+ "groups": [{"id": group.id}],
+ },
+ )
+ resource = p.toolkit.get_action("resource_create")(
+ context,
+ {
+ "name": "privateresource",
+ "url": "https://www.example.com/",
+ "package_id": package["id"],
+ },
+ )
+
+ auth = {"Authorization": self.sysadmin_token}
+ helpers.call_action(
+ "datastore_create", resource_id=resource["id"], force=True
+ )
+
+ # new resource should be private
+ query = 'SELECT * FROM "{0}"'.format(resource["id"])
+ data = {"sql": query}
+ auth = {"Authorization": self.normal_user_token}
+ res = app.post(
+ "/api/action/datastore_search_sql",
+ json=data,
+ extra_environ=auth,
+ status=403,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"]["__type"] == "Authorization Error"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_not_authorized_to_access_system_tables(self, app):
+ test_cases = [
+ "SELECT * FROM pg_roles",
+ "SELECT * FROM pg_catalog.pg_database",
+ "SELECT rolpassword FROM pg_roles",
+ """SELECT p.rolpassword
+ FROM pg_roles p
+ JOIN "{0}" r
+ ON p.rolpassword = r.author""".format(
+ self.data["resource_id"]
+ ),
+ ]
+ for query in test_cases:
+ data = {"sql": query.replace("\n", "")}
+ res = app.post(
+ "/api/action/datastore_search_sql", json=data, status=403,
+ )
+ res_dict = json.loads(res.data)
+ assert res_dict["success"] is False
+ assert res_dict["error"]["__type"] == "Authorization Error"
+
+
+class TestDatastoreSQLFunctional(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures(
+ "clean_datastore", "with_plugins", "with_request_context"
+ )
+ def test_search_sql_enforces_private(self):
+ user1 = factories.User()
+ user2 = factories.User()
+ user3 = factories.User()
+ ctx1 = {u"user": user1["name"], u"ignore_auth": False}
+ ctx2 = {u"user": user2["name"], u"ignore_auth": False}
+ ctx3 = {u"user": user3["name"], u"ignore_auth": False}
+
+ org1 = factories.Organization(
+ user=user1,
+ users=[{u"name": user3["name"], u"capacity": u"member"}],
+ )
+ org2 = factories.Organization(
+ user=user2,
+ users=[{u"name": user3["name"], u"capacity": u"member"}],
+ )
+ ds1 = factories.Dataset(owner_org=org1["id"], private=True)
+ ds2 = factories.Dataset(owner_org=org2["id"], private=True)
+ r1 = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds1["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ )
+ r2 = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds2["id"]},
+ fields=[{u"id": u"ham", u"type": u"text"}],
+ )
+
+ sql1 = 'SELECT spam FROM "{0}"'.format(r1["resource_id"])
+ sql2 = 'SELECT ham FROM "{0}"'.format(r2["resource_id"])
+ sql3 = 'SELECT spam, ham FROM "{0}", "{1}"'.format(
+ r1["resource_id"], r2["resource_id"]
+ )
+
+ with pytest.raises(p.toolkit.NotAuthorized):
+ helpers.call_action("datastore_search_sql", context=ctx2, sql=sql1)
+ with pytest.raises(p.toolkit.NotAuthorized):
+ helpers.call_action("datastore_search_sql", context=ctx1, sql=sql2)
+ with pytest.raises(p.toolkit.NotAuthorized):
+ helpers.call_action("datastore_search_sql", context=ctx1, sql=sql3)
+ with pytest.raises(p.toolkit.NotAuthorized):
+ helpers.call_action("datastore_search_sql", context=ctx2, sql=sql3)
+ helpers.call_action("datastore_search_sql", context=ctx1, sql=sql1)
+ helpers.call_action("datastore_search_sql", context=ctx2, sql=sql2)
+ helpers.call_action("datastore_search_sql", context=ctx3, sql=sql3)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_validates_sql_has_a_single_statement(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"the year": 2014}, {"the year": 2013}],
+ }
+ helpers.call_action("datastore_create", **data)
+ sql = 'SELECT * FROM public."{0}"; SELECT * FROM public."{0}";'.format(
+ resource["id"]
+ )
+ with pytest.raises(
+ p.toolkit.ValidationError, match="Query is not a single statement"
+ ):
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_works_with_semicolons_inside_strings(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"author": "bob"}, {"author": "jane"}],
+ }
+ helpers.call_action("datastore_create", **data)
+ sql = 'SELECT * FROM public."{0}" WHERE "author" = \'foo; bar\''.format(
+ resource["id"]
+ )
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_works_with_allowed_functions(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"author": "bob"}, {"author": "jane"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ sql = 'SELECT upper(author) from "{}"'.format(
+ resource["id"]
+ )
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_not_authorized_with_disallowed_functions(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"author": "bob"}, {"author": "jane"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ sql = "SELECT query_to_xml('SELECT upper(author) from \"{}\"', true, true, '')".format(
+ resource["id"]
+ )
+ with pytest.raises(p.toolkit.NotAuthorized):
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_allowed_functions_are_case_insensitive(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"author": "bob"}, {"author": "jane"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ sql = 'SELECT UpPeR(author) from "{}"'.format(
+ resource["id"]
+ )
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_quoted_allowed_functions_are_case_sensitive(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [{"author": "bob"}, {"author": "jane"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ sql = 'SELECT count(*) from "{}"'.format(
+ resource["id"]
+ )
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ sql = 'SELECT CoUnT(*) from "{}"'.format(
+ resource["id"]
+ )
+ with pytest.raises(p.toolkit.NotAuthorized):
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_invalid_statement(self):
+ sql = "SELECT ** FROM foobar"
+ with pytest.raises(
+ logic.ValidationError, match='syntax error at or near "FROM"'
+ ):
+ helpers.call_action("datastore_search_sql", sql=sql)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_select_basic(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ },
+ ],
+ }
+ expected_records = [
+ {
+ u"_full_text": [
+ u"'annakarenina'",
+ u"'b'",
+ u"'moo'",
+ u"'tolstoy'",
+ u"'2005'",
+ ],
+ u"_id": 1,
+ u"author": u"tolstoy",
+ u"b\xfck": u"annakarenina",
+ u"nested": [u"b", {u"moo": u"moo"}],
+ u"published": u"2005-03-01T00:00:00",
+ },
+ {
+ u"_full_text": [u"'tolstoy'", u"'warandpeac'", u"'b'"],
+ u"_id": 2,
+ u"author": u"tolstoy",
+ u"b\xfck": u"warandpeace",
+ u"nested": {u"a": u"b"},
+ u"published": None,
+ },
+ ]
+ helpers.call_action("datastore_create", **data)
+ sql = 'SELECT * FROM "{0}"'.format(resource["id"])
+ result = helpers.call_action("datastore_search_sql", sql=sql)
+ assert len(result["records"]) == 2
+ for (row_index, row) in enumerate(result["records"]):
+ expected_row = expected_records[row_index]
+ assert set(row.keys()) == set(expected_row.keys())
+ for field in row:
+ if field == "_full_text":
+ for ft_value in expected_row["_full_text"]:
+ assert ft_value in row["_full_text"]
+ else:
+ assert row[field] == expected_row[field]
+ assert u"records_truncated" not in result
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_alias_search(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "aliases": "books4",
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ sql = 'SELECT * FROM "{0}"'.format(resource["id"])
+ result = helpers.call_action("datastore_search_sql", sql=sql)
+ sql = 'SELECT * FROM "books4"'
+ result_with_alias = helpers.call_action(
+ "datastore_search_sql", sql=sql
+ )
+ assert result["records"] == result_with_alias["records"]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "2")
+ def test_search_limit(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"the year": 2014},
+ {"the year": 2013},
+ {"the year": 2015},
+ {"the year": 2016},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ sql = 'SELECT * FROM "{0}"'.format(resource["id"])
+ result = helpers.call_action("datastore_search_sql", sql=sql)
+ assert len(result["records"]) == 2
+ assert [res[u"the year"] for res in result["records"]] == [2014, 2013]
+ assert result[u"records_truncated"]
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreSearchRecordsFormat(object):
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_sort_results_objects(self):
+ ds = factories.Dataset()
+ r = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[
+ {u"id": u"num", u"type": u"numeric"},
+ {u"id": u"dt", u"type": u"timestamp"},
+ {u"id": u"txt", u"type": u"text"},
+ ],
+ records=[
+ {u"num": 10, u"dt": u"2020-01-01", u"txt": "aaab"},
+ {u"num": 9, u"dt": u"2020-01-02", u"txt": "aaab"},
+ {u"num": 9, u"dt": u"2020-01-01", u"txt": "aaac"},
+ ],
+ )
+ assert helpers.call_action(
+ "datastore_search", resource_id=r["resource_id"], sort=u"num, dt"
+ )["records"] == [
+ {
+ u"_id": 3,
+ u"num": 9,
+ u"dt": u"2020-01-01T00:00:00",
+ u"txt": u"aaac",
+ },
+ {
+ u"_id": 2,
+ u"num": 9,
+ u"dt": u"2020-01-02T00:00:00",
+ u"txt": u"aaab",
+ },
+ {
+ u"_id": 1,
+ u"num": 10,
+ u"dt": u"2020-01-01T00:00:00",
+ u"txt": u"aaab",
+ },
+ ]
+ assert helpers.call_action(
+ "datastore_search", resource_id=r["resource_id"], sort=u"dt, txt"
+ )["records"] == [
+ {
+ u"_id": 1,
+ u"num": 10,
+ u"dt": u"2020-01-01T00:00:00",
+ u"txt": u"aaab",
+ },
+ {
+ u"_id": 3,
+ u"num": 9,
+ u"dt": u"2020-01-01T00:00:00",
+ u"txt": u"aaac",
+ },
+ {
+ u"_id": 2,
+ u"num": 9,
+ u"dt": u"2020-01-02T00:00:00",
+ u"txt": u"aaab",
+ },
+ ]
+ assert helpers.call_action(
+ "datastore_search", resource_id=r["resource_id"], sort=u"txt, num"
+ )["records"] == [
+ {
+ u"_id": 2,
+ u"num": 9,
+ u"dt": u"2020-01-02T00:00:00",
+ u"txt": u"aaab",
+ },
+ {
+ u"_id": 1,
+ u"num": 10,
+ u"dt": u"2020-01-01T00:00:00",
+ u"txt": u"aaab",
+ },
+ {
+ u"_id": 3,
+ u"num": 9,
+ u"dt": u"2020-01-01T00:00:00",
+ u"txt": u"aaac",
+ },
+ ]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_sort_results_lists(self):
+ ds = factories.Dataset()
+ r = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[
+ {u"id": u"num", u"type": u"numeric"},
+ {u"id": u"dt", u"type": u"timestamp"},
+ {u"id": u"txt", u"type": u"text"},
+ ],
+ records=[
+ {u"num": 10, u"dt": u"2020-01-01", u"txt": u"aaab"},
+ {u"num": 9, u"dt": u"2020-01-02", u"txt": u"aaab"},
+ {u"num": 9, u"dt": u"2020-01-01", u"txt": u"aaac"},
+ ],
+ )
+ assert helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"lists",
+ sort=u"num, dt",
+ )["records"] == [
+ [3, 9, u"2020-01-01T00:00:00", u"aaac"],
+ [2, 9, u"2020-01-02T00:00:00", u"aaab"],
+ [1, 10, u"2020-01-01T00:00:00", u"aaab"],
+ ]
+ assert helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"lists",
+ sort=u"dt, txt",
+ )["records"] == [
+ [1, 10, u"2020-01-01T00:00:00", u"aaab"],
+ [3, 9, u"2020-01-01T00:00:00", u"aaac"],
+ [2, 9, u"2020-01-02T00:00:00", u"aaab"],
+ ]
+ assert helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"lists",
+ sort=u"txt, num",
+ )["records"] == [
+ [2, 9, u"2020-01-02T00:00:00", u"aaab"],
+ [1, 10, u"2020-01-01T00:00:00", u"aaab"],
+ [3, 9, u"2020-01-01T00:00:00", u"aaac"],
+ ]
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_sort_results_csv(self):
+ ds = factories.Dataset()
+ r = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[
+ {u"id": u"num", u"type": u"numeric"},
+ {u"id": u"dt", u"type": u"timestamp"},
+ {u"id": u"txt", u"type": u"text"},
+ ],
+ records=[
+ {u"num": 10, u"dt": u"2020-01-01", u"txt": u"aaab"},
+ {u"num": 9, u"dt": u"2020-01-02", u"txt": u"aaab"},
+ {u"num": 9, u"dt": u"2020-01-01", u"txt": u"aaac"},
+ ],
+ )
+ assert (
+ helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"csv",
+ sort=u"num, dt",
+ )["records"]
+ == u"3,9,2020-01-01T00:00:00,aaac\n"
+ u"2,9,2020-01-02T00:00:00,aaab\n"
+ u"1,10,2020-01-01T00:00:00,aaab\n"
+ )
+ assert (
+ helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"csv",
+ sort=u"dt, txt",
+ )["records"]
+ == u"1,10,2020-01-01T00:00:00,aaab\n"
+ u"3,9,2020-01-01T00:00:00,aaac\n"
+ u"2,9,2020-01-02T00:00:00,aaab\n"
+ )
+ assert (
+ helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"csv",
+ sort=u"txt, num",
+ )["records"]
+ == u"2,9,2020-01-02T00:00:00,aaab\n"
+ u"1,10,2020-01-01T00:00:00,aaab\n"
+ u"3,9,2020-01-01T00:00:00,aaac\n"
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_fields_results_csv(self):
+ ds = factories.Dataset()
+ r = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[
+ {u"id": u"num", u"type": u"numeric"},
+ {u"id": u"dt", u"type": u"timestamp"},
+ {u"id": u"txt", u"type": u"text"},
+ ],
+ records=[
+ {u"num": 9, u"dt": u"2020-01-02", u"txt": u"aaab"},
+ {u"num": 9, u"dt": u"2020-01-01", u"txt": u"aaac"},
+ ],
+ )
+ r = helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"csv",
+ fields=u"dt, num, txt",
+ )
+ assert r["fields"] == [
+ {u"id": u"dt", u"type": u"timestamp"},
+ {u"id": u"num", u"type": u"numeric"},
+ {u"id": u"txt", u"type": u"text"},
+ ]
+ assert (
+ r["records"] == u"2020-01-02T00:00:00,9,aaab\n"
+ u"2020-01-01T00:00:00,9,aaac\n"
+ )
+ r = helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"csv",
+ fields=u"dt",
+ q=u"aaac",
+ )
+ assert r["fields"] == [{u"id": u"dt", u"type": u"timestamp"}]
+ assert r["records"] == u"2020-01-01T00:00:00\n"
+ r = helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"csv",
+ fields=u"txt, rank txt",
+ q={u"txt": u"aaac"},
+ )
+ assert r["fields"] == [
+ {u"id": u"txt", u"type": u"text"},
+ {u"id": u"rank txt", u"type": u"float"},
+ ]
+ assert r["records"][:7] == u"aaac,0."
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_fts_on_field_calculates_ranks_specific_field_and_all_fields(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"from": "Brazil", "to": "Brazil"},
+ {"from": "Brazil", "to": "Italy"},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "fields": "from, rank from",
+ "full_text": "Brazil",
+ "q": {"from": "Brazil"},
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ ranks_from = [r["rank from"] for r in result["records"]]
+ assert len(result["records"]) == 2
+ assert len(set(ranks_from)) == 1
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_fts_on_field_calculates_ranks_when_q_string_and_fulltext_is_given(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"from": "Brazil", "to": "Brazil"},
+ {"from": "Brazil", "to": "Italy"},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "full_text": "Brazil",
+ "q": "Brazil",
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ ranks = [r["rank"] for r in result["records"]]
+ assert len(result["records"]) == 2
+ assert len(set(ranks)) == 2
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_fts_on_field_calculates_ranks_when_full_text_is_given(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "records": [
+ {"from": "Brazil", "to": "Brazil"},
+ {"from": "Brazil", "to": "Italy"},
+ ],
+ }
+ result = helpers.call_action("datastore_create", **data)
+ search_data = {
+ "resource_id": resource["id"],
+ "full_text": "Brazil",
+ }
+ result = helpers.call_action("datastore_search", **search_data)
+ ranks = [r["rank"] for r in result["records"]]
+ assert len(result["records"]) == 2
+ assert len(set(ranks)) == 2
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_results_with_nulls(self):
+ ds = factories.Dataset()
+ r = helpers.call_action(
+ "datastore_create",
+ resource={"package_id": ds["id"]},
+ fields=[
+ {"id": "num", "type": "numeric"},
+ {"id": "dt", "type": "timestamp"},
+ {"id": "txt", "type": "text"},
+ {"id": "lst", "type": "_text"},
+ ],
+ records=[
+ {"num": 10, "dt": "2020-01-01", "txt": "aaab", "lst": ["one", "two"]},
+ {"num": 9, "dt": "2020-01-02", "txt": "aaab"},
+ {"num": 9, "txt": "aaac", "lst": ["one", "two"]},
+ {}, # all nulls
+ ],
+ )
+ assert helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"lists",
+ sort=u"num nulls last, dt nulls last",
+ )["records"] == [
+ [2, 9, "2020-01-02T00:00:00", "aaab", None],
+ [3, 9, None, "aaac", ["one", "two"]],
+ [1, 10, "2020-01-01T00:00:00", "aaab", ["one", "two"]],
+ [4, None, None, None, None],
+ ]
+ assert helpers.call_action(
+ "datastore_search",
+ resource_id=r["resource_id"],
+ records_format=u"objects",
+ sort=u"num nulls last, dt nulls last",
+ )["records"] == [
+ {"_id": 2, "num": 9, "dt": "2020-01-02T00:00:00", "txt": "aaab", "lst": None},
+ {"_id": 3, "num": 9, "dt": None, "txt": "aaac", "lst": ["one", "two"]},
+ {"_id": 1, "num": 10, "dt": "2020-01-01T00:00:00", "txt": "aaab", "lst": ["one", "two"]},
+ {"_id": 4, "num": None, "dt": None, "txt": None, "lst": None},
+ ]
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py
new file mode 100644
index 0000000..9c25795
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+import ckanext.datastore.backend.postgres as backend
+import ckanext.datastore.backend.postgres as db
+import ckanext.datastore.helpers as helpers
+from ckan.common import config
+
+
+postgres_backend = backend.DatastorePostgresqlBackend()
+postgres_backend.configure(config)
+
+
+def test_is_valid_field_name():
+ assert helpers.is_valid_field_name("foo")
+ assert helpers.is_valid_field_name("foo bar")
+ assert helpers.is_valid_field_name("42")
+ assert not helpers.is_valid_field_name('foo"bar')
+ assert not helpers.is_valid_field_name('"')
+ assert helpers.is_valid_field_name("'")
+ assert not helpers.is_valid_field_name("")
+ assert helpers.is_valid_field_name("foo%bar")
+
+
+def test_is_valid_table_name():
+ assert helpers.is_valid_table_name("foo")
+ assert helpers.is_valid_table_name("foo bar")
+ assert helpers.is_valid_table_name("42")
+ assert not helpers.is_valid_table_name('foo"bar')
+ assert not helpers.is_valid_table_name('"')
+ assert helpers.is_valid_table_name("'")
+ assert not helpers.is_valid_table_name("")
+ assert not helpers.is_valid_table_name("foo%bar")
+
+
+def test_pg_version_check():
+ engine = db._get_engine_from_url(config["sqlalchemy.url"])
+ connection = engine.connect()
+ assert db._pg_version_is_at_least(connection, "8.0")
+ assert not db._pg_version_is_at_least(connection, "20.0")
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py
new file mode 100644
index 0000000..32a07e3
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py
@@ -0,0 +1,953 @@
+# encoding: utf-8
+
+import pytest
+
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+from ckan.plugins.toolkit import ValidationError, NotAuthorized
+from ckanext.datastore.tests.helpers import when_was_last_analyze
+
+
+def _search(resource_id):
+ return helpers.call_action(u"datastore_search", resource_id=resource_id)
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreUpsert(object):
+ # Test action 'datastore_upsert' with 'method': 'upsert'
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_requires_auth(self):
+ resource = factories.Resource(url_type=u"datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ ],
+ "records": [],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {"resource_id": resource["id"]}
+ with pytest.raises(NotAuthorized) as context:
+ helpers.call_action(
+ "datastore_upsert",
+ context={"user": "", "ignore_auth": False},
+ **data
+ )
+ assert (
+ u"Action datastore_upsert requires an authenticated user"
+ in str(context.value)
+ )
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_empty_fails(self):
+ resource = factories.Resource(url_type=u"datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ ],
+ "records": [],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {} # empty
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u"'Missing value'" in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_basic_as_update(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [
+ {"id": "1", "book": u"The boy", "author": u"F Torres"}
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["book"] == "The boy"
+ assert search_result["records"][0]["author"] == "F Torres"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_basic_as_insert(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [
+ {"id": "2", "book": u"The boy", "author": u"F Torres"}
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 2
+ assert search_result["records"][0]["book"] == u"El Niño"
+ assert search_result["records"][1]["book"] == u"The boy"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_only_one_field(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [
+ {"id": "1", "book": u"The boy"}
+ ], # not changing the author
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["book"] == "The boy"
+ assert search_result["records"][0]["author"] == "Torres"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_field_types(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"b\xfck",
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "nested", "type": "json"},
+ {"id": "characters", "type": "text[]"},
+ {"id": "published"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ },
+ {
+ "author": "adams",
+ "characters": ["Arthur", "Marvin"],
+ "nested": {"foo": "bar"},
+ u"b\xfck": u"guide to the galaxy",
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "upsert",
+ "records": [
+ {
+ "author": "adams",
+ "characters": ["Bob", "Marvin"],
+ "nested": {"baz": 3},
+ u"b\xfck": u"guide to the galaxy",
+ }
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 3
+ assert (
+ search_result["records"][0]["published"] == u"2005-03-01T00:00:00"
+ ) # i.e. stored in db as datetime
+ assert search_result["records"][2]["author"] == "adams"
+ assert search_result["records"][2]["characters"] == ["Bob", "Marvin"]
+ assert search_result["records"][2]["nested"] == {"baz": 3}
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_percent(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "bo%ok", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1%", "bo%ok": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [
+ {"id": "1%", "bo%ok": u"The % boy", "author": u"F Torres"},
+ {"id": "2%", "bo%ok": u"Gu%ide", "author": u"Adams"},
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 2
+ assert search_result["records"][0]["bo%ok"] == "The % boy"
+ assert search_result["records"][1]["bo%ok"] == "Gu%ide"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_missing_key(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": "guide", "author": "adams"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [{"book": u"El Niño", "author": "Torres"}], # no key
+ }
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u'fields "id" are missing' in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_non_existing_field(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "upsert",
+ "records": [{"id": "1", "dummy": "tolkien"}], # key not known
+ }
+
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u'fields "dummy" do not exist' in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_works_with_empty_list_in_json_field(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "nested", "type": "json"},
+ ],
+ "records": [{"id": "1", "nested": {"foo": "bar"}}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [{"id": "1", "nested": []}], # empty list
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["nested"] == []
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_delete_field_value(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [{"id": "1", "book": None}],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["book"] is None
+ assert search_result["records"][0]["author"] == "Torres"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_doesnt_crash_with_json_field(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "json"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "insert",
+ "records": [
+ {
+ "id": "1",
+ "book": {"code": "A", "title": u"ñ"},
+ "author": "tolstoy",
+ }
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_upsert_doesnt_crash_with_json_field_with_string_value(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "json"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "insert",
+ "records": [{"id": "1", "book": u"ñ", "author": "tolstoy"}],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dry_run(self):
+ ds = factories.Dataset()
+ table = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ primary_key=u"spam",
+ )
+ helpers.call_action(
+ u"datastore_upsert",
+ resource_id=table["resource_id"],
+ records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}],
+ dry_run=True,
+ )
+ result = helpers.call_action(
+ u"datastore_search", resource_id=table["resource_id"]
+ )
+ assert result["records"] == []
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dry_run_type_error(self):
+ ds = factories.Dataset()
+ table = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"numeric"}],
+ primary_key=u"spam",
+ )
+ try:
+ helpers.call_action(
+ u"datastore_upsert",
+ resource_id=table["resource_id"],
+ records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}],
+ dry_run=True,
+ )
+ except ValidationError as ve:
+ assert ve.error_dict["records"] == [
+ u'invalid input syntax for type numeric: "SPAM"'
+ ]
+ else:
+ assert 0, "error not raised"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_dry_run_trigger_error(self):
+ ds = factories.Dataset()
+ helpers.call_action(
+ u"datastore_function_create",
+ name=u"spamexception_trigger",
+ rettype=u"trigger",
+ definition=u"""
+ BEGIN
+ IF NEW.spam != 'spam' THEN
+ RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam;
+ END IF;
+ RETURN NEW;
+ END;""",
+ )
+ table = helpers.call_action(
+ u"datastore_create",
+ resource={u"package_id": ds["id"]},
+ fields=[{u"id": u"spam", u"type": u"text"}],
+ primary_key=u"spam",
+ triggers=[{u"function": u"spamexception_trigger"}],
+ )
+ try:
+ helpers.call_action(
+ u"datastore_upsert",
+ resource_id=table["resource_id"],
+ records=[{u"spam": u"EGGS"}],
+ dry_run=True,
+ )
+ except ValidationError as ve:
+ assert ve.error_dict["records"] == [u'"EGGS"? Yeeeeccch!']
+ else:
+ assert 0, "error not raised"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_calculate_record_count_is_false(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": "name", "type": "text"},
+ {"id": "age", "type": "text"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "insert",
+ "records": [
+ {"name": "Sunita", "age": "51"},
+ {"name": "Bowan", "age": "68"},
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+ last_analyze = when_was_last_analyze(resource["id"])
+ assert last_analyze is None
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ @pytest.mark.flaky(reruns=2) # because analyze is sometimes delayed
+ def test_calculate_record_count(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": "name", "type": "text"},
+ {"id": "age", "type": "text"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "insert",
+ "records": [
+ {"name": "Sunita", "age": "51"},
+ {"name": "Bowan", "age": "68"},
+ ],
+ "calculate_record_count": True,
+ }
+ helpers.call_action("datastore_upsert", **data)
+ last_analyze = when_was_last_analyze(resource["id"])
+ assert last_analyze is not None
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_no_pk_update(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": "book", "type": "text"},
+ ],
+ "records": [{"book": u"El Niño"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [
+ {"_id": "1", "book": u"The boy"}
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["book"] == "The boy"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_id_instead_of_pk_update(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "pk",
+ "fields": [
+ {"id": "pk", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"pk": "1000", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "upsert",
+ "records": [
+ {"_id": "1", "book": u"The boy", "author": u"F Torres"}
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["pk"] == "1000"
+ assert search_result["records"][0]["book"] == "The boy"
+ assert search_result["records"][0]["author"] == "F Torres"
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreInsert(object):
+ # Test action 'datastore_upsert' with 'method': 'insert'
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_basic_insert(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "insert",
+ "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["fields"] == [
+ {u"id": "_id", u"type": "int"},
+ {u"id": u"id", u"type": u"text"},
+ {u"id": u"book", u"type": u"text"},
+ {u"id": u"author", u"type": u"text"},
+ ]
+ assert search_result["records"][0] == {
+ u"book": u"El Ni\xf1o",
+ u"_id": 1,
+ u"id": u"1",
+ u"author": u"Torres",
+ }
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_non_existing_field(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "insert",
+ "records": [{"id": "1", "dummy": "tolkien"}], # key not known
+ }
+
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u'row "1" has extra keys "dummy"' in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_key_already_exists(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": "guide", "author": "adams"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "insert",
+ "records": [
+ {
+ "id": "1", # already exists
+ "book": u"El Niño",
+ "author": "Torres",
+ }
+ ],
+ }
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u"duplicate key value violates unique constraint" in str(
+ context.value
+ )
+
+
+@pytest.mark.usefixtures("with_request_context")
+class TestDatastoreUpdate(object):
+ # Test action 'datastore_upsert' with 'method': 'update'
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_basic(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "update",
+ "records": [{"id": "1", "book": u"The boy"}],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["book"] == "The boy"
+ assert search_result["records"][0]["author"] == "Torres"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_field_types(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"b\xfck",
+ "fields": [
+ {"id": u"b\xfck", "type": "text"},
+ {"id": "author", "type": "text"},
+ {"id": "nested", "type": "json"},
+ {"id": "characters", "type": "text[]"},
+ {"id": "published"},
+ ],
+ "records": [
+ {
+ u"b\xfck": "annakarenina",
+ "author": "tolstoy",
+ "published": "2005-03-01",
+ "nested": ["b", {"moo": "moo"}],
+ },
+ {
+ u"b\xfck": "warandpeace",
+ "author": "tolstoy",
+ "nested": {"a": "b"},
+ },
+ {
+ "author": "adams",
+ "characters": ["Arthur", "Marvin"],
+ "nested": {"foo": "bar"},
+ u"b\xfck": u"guide to the galaxy",
+ },
+ ],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "update",
+ "records": [
+ {
+ "author": "adams",
+ "characters": ["Bob", "Marvin"],
+ "nested": {"baz": 3},
+ u"b\xfck": u"guide to the galaxy",
+ }
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 3
+ assert search_result["records"][2]["author"] == "adams"
+ assert search_result["records"][2]["characters"] == ["Bob", "Marvin"]
+ assert search_result["records"][2]["nested"] == {"baz": 3}
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_update_unspecified_key(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "update",
+ "records": [{"author": "tolkien"}], # no id
+ }
+
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u'fields "id" are missing' in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_update_unknown_key(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "update",
+ "records": [{"id": "1", "author": "tolkien"}], # unknown
+ }
+
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u"key \"[\\'1\\']\" not found" in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_update_non_existing_field(self):
+ resource = factories.Resource(url_type="datastore")
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": u"id",
+ "fields": [
+ {"id": "id", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"id": "1", "book": "guide"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "method": "update",
+ "records": [{"id": "1", "dummy": "tolkien"}], # key not known
+ }
+
+ with pytest.raises(ValidationError) as context:
+ helpers.call_action("datastore_upsert", **data)
+ assert u'fields "dummy" do not exist' in str(context.value)
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_no_pk_update(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "fields": [
+ {"id": "book", "type": "text"},
+ ],
+ "records": [{"book": u"El Niño"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "update",
+ "records": [
+ {"_id": "1", "book": u"The boy"}
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["book"] == "The boy"
+
+ @pytest.mark.ckan_config("ckan.plugins", "datastore")
+ @pytest.mark.usefixtures("clean_datastore", "with_plugins")
+ def test_id_instead_of_pk_update(self):
+ resource = factories.Resource()
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "primary_key": "pk",
+ "fields": [
+ {"id": "pk", "type": "text"},
+ {"id": "book", "type": "text"},
+ {"id": "author", "type": "text"},
+ ],
+ "records": [{"pk": "1000", "book": u"El Niño", "author": "Torres"}],
+ }
+ helpers.call_action("datastore_create", **data)
+
+ data = {
+ "resource_id": resource["id"],
+ "force": True,
+ "method": "update",
+ "records": [
+ {"_id": "1", "book": u"The boy", "author": u"F Torres"}
+ ],
+ }
+ helpers.call_action("datastore_upsert", **data)
+
+ search_result = _search(resource["id"])
+ assert search_result["total"] == 1
+ assert search_result["records"][0]["pk"] == "1000"
+ assert search_result["records"][0]["book"] == "The boy"
+ assert search_result["records"][0]["author"] == "F Torres"
diff --git a/src/ckanext-d4science_theme/ckanext/datastore/writer.py b/src/ckanext-d4science_theme/ckanext/datastore/writer.py
new file mode 100644
index 0000000..b46c42c
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datastore/writer.py
@@ -0,0 +1,179 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from io import StringIO, BytesIO
+
+from contextlib import contextmanager
+from typing import Any, Optional
+from simplejson import dumps
+
+from xml.etree.cElementTree import Element, SubElement, ElementTree
+
+import csv
+
+from codecs import BOM_UTF8
+
+
+BOM = "\N{bom}"
+
+
+@contextmanager
+def csv_writer(fields: list[dict[str, Any]], bom: bool = False):
+ '''Context manager for writing UTF-8 CSV data to file
+
+ :param fields: list of datastore fields
+ :param bom: True to include a UTF-8 BOM at the start of the file
+ '''
+ output = StringIO()
+
+ if bom:
+ output.write(BOM)
+
+ csv.writer(output).writerow(
+ f['id'] for f in fields)
+ yield TextWriter(output)
+
+
+@contextmanager
+def tsv_writer(fields: list[dict[str, Any]], bom: bool = False):
+ '''Context manager for writing UTF-8 TSV data to file
+
+ :param fields: list of datastore fields
+ :param bom: True to include a UTF-8 BOM at the start of the file
+ '''
+ output = StringIO()
+
+ if bom:
+ output.write(BOM)
+
+ csv.writer(
+ output,
+ dialect='excel-tab').writerow(
+ f['id'] for f in fields)
+ yield TextWriter(output)
+
+
+class TextWriter(object):
+ u'text in, text out'
+ def __init__(self, output: StringIO):
+ self.output = output
+
+ def write_records(self, records: list[Any]) -> bytes:
+ self.output.write(records) # type: ignore
+ self.output.seek(0)
+ output = self.output.read().encode('utf-8')
+ self.output.truncate(0)
+ self.output.seek(0)
+ return output
+
+ def end_file(self) -> bytes:
+ return b''
+
+
+@contextmanager
+def json_writer(fields: list[dict[str, Any]], bom: bool = False):
+ '''Context manager for writing UTF-8 JSON data to file
+
+ :param fields: list of datastore fields
+ :param bom: True to include a UTF-8 BOM at the start of the file
+ '''
+ output = StringIO()
+
+ if bom:
+ output.write(BOM)
+
+ output.write(
+ '{\n "fields": %s,\n "records": [' % dumps(
+ fields, ensure_ascii=False, separators=(u',', u':')))
+ yield JSONWriter(output)
+
+
+class JSONWriter(object):
+ def __init__(self, output: StringIO):
+ self.output = output
+ self.first = True
+
+ def write_records(self, records: list[Any]) -> bytes:
+ for r in records:
+ if self.first:
+ self.first = False
+ self.output.write('\n ')
+ else:
+ self.output.write(',\n ')
+
+ self.output.write(dumps(
+ r, ensure_ascii=False, separators=(',', ':')))
+
+ self.output.seek(0)
+ output = self.output.read().encode('utf-8')
+ self.output.truncate(0)
+ self.output.seek(0)
+ return output
+
+ def end_file(self) -> bytes:
+ return b'\n]}\n'
+
+
+@contextmanager
+def xml_writer(fields: list[dict[str, Any]], bom: bool = False):
+ '''Context manager for writing UTF-8 XML data to file
+
+ :param fields: list of datastore fields
+ :param bom: True to include a UTF-8 BOM at the start of the file
+ '''
+ output = BytesIO()
+
+ if bom:
+ output.write(BOM_UTF8)
+
+ output.write(
+ b'\n')
+ yield XMLWriter(output, [f[u'id'] for f in fields])
+
+
+class XMLWriter(object):
+ _key_attr = 'key'
+ _value_tag = 'value'
+
+ def __init__(self, output: BytesIO, columns: list[str]):
+ self.output = output
+ self.id_col = columns[0] == '_id'
+ if self.id_col:
+ columns = columns[1:]
+ self.columns = columns
+
+ def _insert_node(self, root: Any, k: str, v: Any,
+ key_attr: Optional[Any] = None):
+ element = SubElement(root, k)
+ if v is None:
+ element.attrib['xsi:nil'] = 'true'
+ elif not isinstance(v, (list, dict)):
+ element.text = str(v)
+ else:
+ if isinstance(v, list):
+ it = enumerate(v)
+ else:
+ it = v.items()
+ for key, value in it:
+ self._insert_node(element, self._value_tag, value, key)
+
+ if key_attr is not None:
+ element.attrib[self._key_attr] = str(key_attr)
+
+ def write_records(self, records: list[Any]) -> bytes:
+ for r in records:
+ root = Element('row')
+ if self.id_col:
+ root.attrib['_id'] = str(r['_id'])
+ for c in self.columns:
+ self._insert_node(root, c, r[c])
+ ElementTree(root).write(self.output, encoding='utf-8')
+ self.output.write(b'\n')
+ self.output.seek(0)
+ output = self.output.read()
+ self.output.truncate(0)
+ self.output.seek(0)
+ return output
+
+ def end_file(self) -> bytes:
+ return b' \n'
diff --git a/src/ckanext-d4science_theme/ckanext/datatablesview/__init__.py b/src/ckanext-d4science_theme/ckanext/datatablesview/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py b/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py
new file mode 100644
index 0000000..45e2298
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py
@@ -0,0 +1,215 @@
+# encoding: utf-8
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urlencode
+
+from flask import Blueprint
+
+
+from ckan.common import json
+from ckan.lib.helpers import decode_view_request_filters
+from ckan.plugins.toolkit import get_action, request, h
+import re
+
+datatablesview = Blueprint(u'datatablesview', __name__)
+
+
+def merge_filters(view_filters: dict[str, Any],
+ user_filters: dict[str, Any] | None) -> dict[str, Any]:
+ u'''
+ view filters are built as part of the view, user filters
+ are selected by the user interacting with the view. Any filters
+ selected by user may only tighten filters set in the view,
+ others are ignored.
+
+ >>> merge_filters({
+ ... u'Department': [u'BTDT'], u'OnTime_Status': [u'ONTIME']},
+ ... u'CASE_STATUS:Open|CASE_STATUS:Closed|Department:INFO')
+ {u'Department': [u'BTDT'],
+ u'OnTime_Status': [u'ONTIME'],
+ u'CASE_STATUS': [u'Open', u'Closed']}
+ '''
+ filters = dict(view_filters)
+ if not user_filters:
+ return filters
+ combined_user_filters = {}
+ for k in user_filters:
+ if k not in view_filters or user_filters[k] in view_filters[k]:
+ combined_user_filters[k] = user_filters[k]
+ else:
+ combined_user_filters[k] = user_filters[k] + view_filters[k]
+ for k in combined_user_filters:
+ filters[k] = combined_user_filters[k]
+ return filters
+
+
+def ajax(resource_view_id: str):
+ resource_view = get_action(u'resource_view_show'
+ )({}, {
+ u'id': resource_view_id
+ })
+
+ draw = int(request.form[u'draw'])
+ search_text = str(request.form[u'search[value]'])
+ offset = int(request.form[u'start'])
+ limit = int(request.form[u'length'])
+ view_filters = resource_view.get(u'filters', {})
+ user_filters = decode_view_request_filters()
+ filters = merge_filters(view_filters, user_filters)
+
+ datastore_search = get_action(u'datastore_search')
+ unfiltered_response = datastore_search(
+ {}, {
+ u"resource_id": resource_view[u'resource_id'],
+ u"limit": 0,
+ u"filters": view_filters,
+ }
+ )
+
+ cols = [f[u'id'] for f in unfiltered_response[u'fields']]
+ if u'show_fields' in resource_view:
+ cols = [c for c in cols if c in resource_view[u'show_fields']]
+
+ sort_list = []
+ i = 0
+ while True:
+ if u'order[%d][column]' % i not in request.form:
+ break
+ sort_by_num = int(request.form[u'order[%d][column]' % i])
+ sort_order = (
+ u'desc' if request.form[u'order[%d][dir]' %
+ i] == u'desc' else u'asc'
+ )
+ sort_list.append(cols[sort_by_num] + u' ' + sort_order)
+ i += 1
+
+ colsearch_dict = {}
+ i = 0
+ while True:
+ if u'columns[%d][search][value]' % i not in request.form:
+ break
+ v = str(request.form[u'columns[%d][search][value]' % i])
+ if v:
+ k = str(request.form[u'columns[%d][name]' % i])
+ # replace non-alphanumeric characters with FTS wildcard (_)
+ v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v)
+ # append ':*' so we can do partial FTS searches
+ colsearch_dict[k] = v + u':*'
+ i += 1
+
+ if colsearch_dict:
+ search_text = json.dumps(colsearch_dict)
+ else:
+ search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_',
+ search_text) + u':*' if search_text else u''
+
+ try:
+ response = datastore_search(
+ {}, {
+ u"q": search_text,
+ u"resource_id": resource_view[u'resource_id'],
+ u'plain': False,
+ u'language': u'simple',
+ u"offset": offset,
+ u"limit": limit,
+ u"sort": u', '.join(sort_list),
+ u"filters": filters,
+ }
+ )
+ except Exception:
+ query_error = u'Invalid search query... ' + search_text
+ dtdata = {u'error': query_error}
+ else:
+ data = []
+ for row in response[u'records']:
+ record = {colname: str(row.get(colname, u''))
+ for colname in cols}
+ # the DT_RowId is used in DT to set an element id for each record
+ record['DT_RowId'] = 'row' + str(row.get(u'_id', u''))
+ data.append(record)
+
+ dtdata = {
+ u'draw': draw,
+ u'recordsTotal': unfiltered_response.get(u'total', 0),
+ u'recordsFiltered': response.get(u'total', 0),
+ u'data': data
+ }
+
+ return json.dumps(dtdata)
+
+
+def filtered_download(resource_view_id: str):
+ params = json.loads(request.form[u'params'])
+ resource_view = get_action(u'resource_view_show'
+ )({}, {
+ u'id': resource_view_id
+ })
+
+ search_text = str(params[u'search'][u'value'])
+ view_filters = resource_view.get(u'filters', {})
+ user_filters = decode_view_request_filters()
+ filters = merge_filters(view_filters, user_filters)
+
+ datastore_search = get_action(u'datastore_search')
+ unfiltered_response = datastore_search(
+ {}, {
+ u"resource_id": resource_view[u'resource_id'],
+ u"limit": 0,
+ u"filters": view_filters,
+ }
+ )
+
+ cols = [f[u'id'] for f in unfiltered_response[u'fields']]
+ if u'show_fields' in resource_view:
+ cols = [c for c in cols if c in resource_view[u'show_fields']]
+
+ sort_list = []
+ for order in params[u'order']:
+ sort_by_num = int(order[u'column'])
+ sort_order = (u'desc' if order[u'dir'] == u'desc' else u'asc')
+ sort_list.append(cols[sort_by_num] + u' ' + sort_order)
+
+ cols = [c for (c, v) in zip(cols, params[u'visible']) if v]
+
+ colsearch_dict = {}
+ columns = params[u'columns']
+ for column in columns:
+ if column[u'search'][u'value']:
+ v = column[u'search'][u'value']
+ if v:
+ k = column[u'name']
+ # replace non-alphanumeric characters with FTS wildcard (_)
+ v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v)
+ # append ':*' so we can do partial FTS searches
+ colsearch_dict[k] = v + u':*'
+
+ if colsearch_dict:
+ search_text = json.dumps(colsearch_dict)
+ else:
+ search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_',
+ search_text) + u':*' if search_text else ''
+
+ return h.redirect_to(
+ h.url_for(
+ u'datastore.dump',
+ resource_id=resource_view[u'resource_id']) + u'?' + urlencode(
+ {
+ u'q': search_text,
+ u'plain': False,
+ u'language': u'simple',
+ u'sort': u','.join(sort_list),
+ u'filters': json.dumps(filters),
+ u'format': request.form[u'format'],
+ u'fields': u','.join(cols),
+ }))
+
+
+datatablesview.add_url_rule(
+ u'/datatables/ajax/', view_func=ajax, methods=[u'POST']
+)
+
+datatablesview.add_url_rule(
+ u'/datatables/filtered-download/',
+ view_func=filtered_download, methods=[u'POST']
+)
diff --git a/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml b/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml
new file mode 100644
index 0000000..fac8438
--- /dev/null
+++ b/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml
@@ -0,0 +1,95 @@
+version: 1
+groups:
+- annotation: datatables_view settings
+ options:
+ - key: ckan.datatables.page_length_choices
+ default: 20 50 100 500 1000
+ description: https://datatables.net/examples/advanced_init/length_menu.html
+ type: list
+ example: 20 50 100 500 1000 5000
+ description: |
+ Space-delimited list of the choices for the number of rows per page, with
+ the lowest value being the default initial value.
+
+ .. note:: On larger screens, DataTables view will attempt to fill the
+ table with as many rows that can fit using the lowest closest choice.
+
+ - key: ckan.datatables.state_saving
+ default: true
+ type: bool
+ example: 'false'
+ description: |
+ Enable or disable state saving. When enabled, DataTables view will store
+ state information such as pagination position, page length, row
+ selection/s, column visibility/ordering, filtering and sorting using the
+ browser's localStorage. When the end user reloads the page, the table's
+ state will be altered to match what they had previously set up.
+
+ This also enables/disables the "Reset" and "Share current view"
+ buttons. "Reset" discards the saved state. "Share current view" base-64
+ encodes the state and passes it as a url parameter, acting like a "saved
+ search" that can be used for embedding and sharing table searches.
+
+ - key: ckan.datatables.state_duration
+ default: 7200
+ type: int
+ example: 86400
+ description: |
+ Duration (in seconds) for which the saved state information is considered
+ valid. After this period has elapsed, the table's state will be returned
+ to the default, and the state cleared from the browser's localStorage.
+
+ .. note:: The value ``0`` is a special value as it indicates that the
+ state can be stored and retrieved indefinitely with no time limit.
+
+ - key: ckan.datatables.data_dictionary_labels
+ default: true
+ type: bool
+ example: 'false'
+ description: >
+ Enable or disable data dictionary integration. When enabled, a column's
+ data dictionary label will be used in the table header. A tooltip for
+ each column with data dictionary information will also be integrated into
+ the header.
+
+ - key: ckan.datatables.ellipsis_length
+ default: 100
+ type: int
+ example: 100
+ description: |
+ The maximum number of characters to show in a cell before it is
+ truncated. An ellipsis (...) will be added at the truncation point and
+ the full text of the cell will be available as a tooltip. This value can
+ be overridden at the resource level when configuring a DataTables
+ resource view.
+
+ .. note:: The value ``0`` is a special value as it indicates that the
+ column's width will be determined by the column name, and cell content
+ will word-wrap.
+
+ - key: ckan.datatables.date_format
+ default: llll
+ description: see Moment.js cheatsheet https://devhints.io/moment
+ example: YYYY-MM-DD dd ww
+ description: |
+ The `moment.js date format
+ ' + title + ' ')
+ prtWindow.document.write(contents)
+ prtWindow.document.write('
')
+ prtWindow.print()
+ prtWindow.close()
+}
+
+// helper for modal clipboard copy
+function copyModal (title) {
+ const el = document.querySelector('.dtr-details')
+ const body = document.body
+ let range
+ let sel
+ if (document.createRange && window.getSelection) {
+ range = document.createRange()
+ sel = window.getSelection()
+ sel.removeAllRanges()
+ try {
+ range.selectNodeContents(el)
+ sel.addRange(range)
+ } catch (e) {
+ range.selectNode(el)
+ sel.addRange(range)
+ }
+ } else if (body.createTextRange) {
+ range = body.createTextRange()
+ range.moveToElementText(el)
+ range.select()
+ }
+ document.execCommand('copy')
+ window.getSelection().removeAllRanges()
+}
+
+// force column auto width adjustment to kick in
+// used by "Autofit columns" button
+function fitColText () {
+ const dt = $('#dtprv').DataTable({ retrieve: true })
+ if (gcurrentView === 'list') {
+ dt.responsive.recalc()
+ }
+ dt.columns.adjust().draw(false)
+}
+
+// ensure element id is valid
+function validateId (id) {
+ id = id.toLowerCase()
+ // Make alphanumeric (removes all other characters)
+ id = id.replace(/[^a-z0-9_\s-]/g, '')
+ // Convert whitespaces and underscore to #
+ id = id.replace(/[\s_]/g, '#')
+ // Convert multiple # to hyphen
+ id = id.replace(/[#]+/g, '-')
+ return id
+}
+
+// compile sort & active filters for display in print, clipboard copy & search tooltip
+function filterInfo (dt, noHtml = false, justFilterInfo = false, wrapped = false) {
+ let filtermsg = justFilterInfo ? '' : document.getElementById('dtprv_info').innerText
+
+ const selinfo = document.getElementsByClassName('select-info')[0]
+
+ if (selinfo !== undefined) {
+ filtermsg = filtermsg.replace(selinfo.innerText, ', ' + selinfo.innerText)
+ }
+
+ // add active filter info to messageTop
+ if (gsearchMode === 'table') {
+ filtermsg = filtermsg + '' + gtableSearchText + ': ' + dt.search()
+ } else if (gsearchMode === 'column') {
+ let colsearchflag = false
+ let colsearchmsg = ''
+ dt.columns().every(function () {
+ const colsearch = this.search()
+ const colname = this.name()
+
+ if (colsearch) {
+ colsearchflag = true
+ colsearchmsg = colsearchmsg + ' ' + colname + ': ' + gcolFilterText + ': ' + colsearchmsg.slice(0, -2)
+ }
+ }
+ filtermsg = justFilterInfo ? filtermsg : filtermsg + '
')
+ .css({
+ height: 1,
+ width: 1,
+ overflow: 'hidden',
+ position: 'fixed',
+ top: 0,
+ left: 0
+ })
+
+ const textarea = $('')
+ .val(deeplink)
+ .appendTo(hiddenDiv)
+
+ // save & deselect rows, so we copy the link, not the rows
+ const selectedRows = dt.rows({ selected: true })[0]
+ dt.rows().deselect()
+
+ hiddenDiv.appendTo(dt.table().container())
+ textarea[0].focus()
+ textarea[0].select()
+
+ hiddenDiv.appendTo(dt.table().container())
+ textarea[0].focus()
+ textarea[0].select()
+ // use copy execCommand to copy link to clipboard
+ const successful = document.execCommand('copy')
+ hiddenDiv.remove()
+
+ if (successful) {
+ dt.buttons.info(shareText, sharemsgText, 2000)
+ }
+ dt.rows(selectedRows).select()
+}
+
+// helper for hiding search inputs for list/responsive mode
+function hideSearchInputs (columns) {
+ for (let i = 0; i < columns.length; i++) {
+ if (columns[i]) {
+ $('#cdx' + i).show()
+ } else {
+ $('#cdx' + i).hide()
+ }
+ }
+ $('#_colspacerfilter').hide()
+}
+
+// helper for setting up filterObserver
+function initFilterObserver () {
+ // if no filter is active, toggle filter tooltip as required
+ // this is less expensive than querying the DT api to check global filter and each column
+ // separately for filter status. Here, we're checking if an open parenthesis is in the filter info,
+ // which indicates that there is a filter active, regardless of language
+ // (e.g. "4 of 1000 entries (filtered from...)")
+ const filterObserver = new MutationObserver(function (e) {
+ const infoText = document.getElementById('dtprv_info').innerText
+ if (!infoText.includes('(')) {
+ document.getElementById('filterinfoicon').style.visibility = 'hidden'
+ } else {
+ document.getElementById('filterinfoicon').style.visibility = 'visible'
+ }
+ })
+ try {
+ filterObserver.observe(document.getElementById('dtprv_info'), { characterData: true, subtree: true, childList: true })
+ } catch (e) {}
+}
+
+// helper for converting links in text into clickable links
+const linkify = (input) => {
+ let text = input
+ const linksFound = text.match(/(\b(https?)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig)
+ const links = []
+ if (linksFound != null) {
+ if (linksFound.length === 1 && input.match(/\.(jpeg|jpg|gif|png|svg|apng|webp|avif)$/)) {
+ // the whole text is just one link and its a picture, create a thumbnail
+ text = ''
+ return { text: text, links: linksFound }
+ }
+ for (let i = 0; i < linksFound.length; i++) {
+ links.push('' + linksFound[i] + ' ')
+ text = text.split(linksFound[i]).map(item => { return item }).join(links[i])
+ }
+ return { text: text, links: linksFound }
+ } else {
+ return { text: input, links: [] }
+ }
+}
+
+// helper to protect against uncontrolled HTML input
+const esc = function (t) {
+ return t
+ .replace(/&/g, '&')
+ .replace(//g, '>')
+ .replace(/"/g, '"')
+}
+
+// MAIN
+this.ckan.module('datatables_view', function (jQuery) {
+ return {
+ initialize: function () {
+ const that = this
+
+ // fetch parameters from template data attributes
+ const dtprv = $('#dtprv')
+ const resourcename = dtprv.data('resource-name')
+ const languagecode = dtprv.data('languagecode')
+ const languagefile = dtprv.data('languagefile')
+ const statesaveflag = dtprv.data('state-save-flag')
+ const stateduration = parseInt(dtprv.data('state-duration'))
+ const ellipsislength = parseInt(dtprv.data('ellipsis-length'))
+ const dateformat = dtprv.data('date-format').trim()
+ const formatdateflag = dateformat.toUpperCase() !== 'NONE'
+ const packagename = dtprv.data('package-name')
+ const responsiveflag = dtprv.data('responsive-flag')
+ const pagelengthchoices = dtprv.data('page-length-choices')
+ const ajaxurl = dtprv.data('ajaxurl')
+ const ckanfilters = dtprv.data('ckanfilters')
+ const resourceurl = dtprv.data('resource-url')
+ const defaultview = dtprv.data('default-view')
+
+ // get view mode setting from localstorage (table or list/responsive])
+ const lastView = getWithExpiry('lastView')
+ if (!lastView) {
+ if (responsiveflag) {
+ gcurrentView = 'list' // aka responsive
+ } else {
+ gcurrentView = defaultview
+ }
+ setWithExpiry('lastView', gcurrentView, 0)
+ } else {
+ gcurrentView = lastView
+ }
+
+ // get column definitions dynamically from data dictionary,
+ // init data structure with _id column definition
+ const dynamicCols = [{
+ data: '_id',
+ searchable: false,
+ type: 'num',
+ className: 'dt-body-right',
+ width: gcurrentView === 'table' ? '28px' : '50px'
+ }]
+
+ gdataDict.forEach((colDefn, idx) => {
+ const colDict = { name: colDefn.id, data: colDefn.id, contentPadding: 'MM' }
+ switch (colDefn.type) {
+ case 'numeric':
+ colDict.type = 'num'
+ break
+ case 'timestamp':
+ case 'timestamptz':
+ colDict.type = 'date'
+ if (formatdateflag) {
+ colDict.render = $.fn.dataTable.render.moment(window.moment.ISO_8601, dateformat, languagecode)
+ }
+ break
+ default:
+ colDict.type = 'html'
+ if (!ellipsislength) {
+ colDict.className = 'wrapcell'
+ }
+ colDict.render = function (data, type, row, meta) {
+ // Order, search and type get the original data
+ if (type !== 'display') {
+ return data
+ }
+ if (typeof data !== 'number' && typeof data !== 'string') {
+ return data
+ }
+ data = data.toString() // cast numbers
+
+ const linkifiedData = linkify(data)
+ // if there are no http links, see if we need to apply ellipsis truncation logic
+ if (!linkifiedData.links) {
+ // no links, just do simple truncation if ellipsislength is defined
+ if (!ellipsislength || data.length <= ellipsislength) {
+ return data
+ }
+ const shortened = data.substr(0, ellipsislength - 1).trimEnd()
+ return '' + shortened + '… '
+ } else {
+ // there are links
+ const strippedData = linkifiedData.text.replace(/(<([^>]+)>)/gi, '')
+ if (!ellipsislength || strippedData.length <= ellipsislength) {
+ return linkifiedData.text
+ }
+ let linkpos = ellipsislength
+ let lastpos = ellipsislength
+ let lastlink = ''
+ let addLen = 0
+ // check if truncation point is in the middle of a link
+ for (const aLink of linkifiedData.links) {
+ linkpos = data.indexOf(aLink)
+ if (linkpos + aLink.length >= ellipsislength) {
+ // truncation point is in the middle of a link, truncate to where the link started
+ break
+ } else {
+ addLen = addLen + lastlink.length ? (lastlink.length) + 31 : 0 // 31 is the number of other chars in the full anchor tag
+ lastpos = linkpos
+ lastlink = aLink
+ }
+ }
+ const shortened = linkifiedData.text.substr(0, lastpos + addLen).trimEnd()
+ return '' + shortened + '… '
+ }
+ }
+ }
+ dynamicCols.push(colDict)
+ })
+
+ // labels for showing active filters in clipboard copy & print
+ gtableSearchText = that._('TABLE FILTER')
+ gcolFilterText = that._('COLUMN FILTER/S')
+
+ let activelanguage = languagefile
+ // en is the default language, no need to load i18n file
+ if (languagefile === '/vendor/DataTables/i18n/en.json') {
+ activelanguage = ''
+ }
+
+ // settings if gcurrentView === table
+ let fixedColumnSetting = true
+ let scrollXflag = true
+ let responsiveSettings = false
+
+ if (gcurrentView === 'list') {
+ // we're in list view mode (aka responsive mode)
+ // not compatible with scrollX
+ fixedColumnSetting = false
+ scrollXflag = false
+
+ // create _colspacer column to ensure display of green record detail button
+ dynamicCols.push({
+ data: '',
+ searchable: false,
+ className: 'none',
+ defaultContent: ''
+ })
+
+ // initialize settings for responsive mode (list view)
+ responsiveSettings = {
+ details: {
+ display: $.fn.dataTable.Responsive.display.modal({
+ header: function (row) {
+ // add clipboard and print buttons to modal record display
+ var data = row.data();
+ return 'Details: ' +
+ '
'
+ }
+ }),
+ // render the Record Details in a modal dialog box
+ // do not render the _colspacer column, which has the 'none' class
+ // the none class in responsive mode forces the _colspacer column to be hidden
+ // guaranteeing the blue record details button is always displayed, even for narrow tables
+ // also, when a column's content has been truncated with an ellipsis, show the untruncated content
+ renderer: function (api, rowIdx, columns) {
+ const data = $.map(columns, function (col, i) {
+ return col.className !== ' none'
+ ? '' +
+ '' + col.title + ':' + ' ' +
+ (col.data.startsWith('') - 30) : col.data) +
+ ' '
+ : ''
+ }).join('')
+ return data ? $('').append(data) : false
+ }
+ }
+ }
+ } else {
+ // we're in table view mode
+ // remove _colspacer column/filter if it exists
+ $('#_colspacer').remove()
+ $('#_colspacerfilter').remove()
+ }
+
+ // create column filters
+ $('.fhead').each(function (i) {
+ const thecol = this
+ const colname = thecol.textContent
+ const colid = 'dtcol-' + validateId(colname) + '-' + i
+ const coltype = $(thecol).data('type')
+ const placeholderText = formatdateflag && coltype.substr(0, 9) === 'timestamp' ? ' placeholder="yyyy-mm-dd"' : ''
+ $(' ')
+
+ // on mouseenter on Search info icon, update tooltip with filterinfo
+ $('#filterinfoicon').mouseenter(function () {
+ document.getElementById('filterinfoicon').title = filterInfo(datatable, true, true, true) +
+ '\n' + (gelapsedTime / 1000).toFixed(2) + ' ' + that._('seconds') + '\n' +
+ that._('Double-click to reset filters')
+ })
+
+ // on dblclick on Search info icon, clear all filters
+ $('#filterinfoicon').dblclick(function () {
+ datatable.search('')
+ .columns().search('')
+ .draw(false)
+ $('th.fhead input').val('')
+ })
+
+ // add resourceinfo in footer, very useful if this view is embedded
+ const resourceInfo = document.getElementById('dtv-resource-info').innerText
+ $('div.resourceinfo').html('' +
+ packagename + '—' + resourcename +
+ ' ')
+
+ // if in list/responsive mode, hide search inputs for hidden columns
+ if (gcurrentView === 'list') {
+ hideSearchInputs(api.columns().responsiveHidden().toArray())
+ }
+
+ // only do table search on enter key, or clearing of input
+ const tableSearchInput = $('#dtprv_filter label input')
+ tableSearchInput.unbind()
+ tableSearchInput.bind('keyup search', function (event) {
+ // Firefox doesn't do clearing of input when ESC is pressed
+ if (gisFirefox && event.keyCode === 27) {
+ this.value = ''
+ }
+ if (event.keyCode === 13 || (tableSearchInput.val() === '' && datatable.search() !== '')) {
+ datatable
+ .search(this.value)
+ .draw()
+ gsearchMode = 'table'
+ }
+ })
+
+ // start showing page once everything is just about rendered
+ // we need to make it visible now so smartsize works if needed
+ document.getElementsByClassName('dt-view')[0].style.visibility = 'visible'
+
+ const url = new URL(window.location.href)
+ const state = url.searchParams.get('state')
+ // if there is a state url parm, its a deeplink share
+ if (state) {
+ // we need to reload to get the deeplink active
+ // to init localstorage
+ if (!getWithExpiry('deeplink_firsttime')) {
+ setWithExpiry('deeplink_firsttime', true, 4)
+ setTimeout(function () {
+ window.location.reload()
+ }, 200)
+ }
+ } else {
+ // otherwise, do a smartsize check to fill up screen
+ // if default pagelen is too low and there is available space
+ const currPageLen = api.page.len()
+ if (json.recordsTotal > currPageLen) {
+ const scrollBodyHeight = $('#resize_wrapper').height() - ($('.dataTables_scrollHead').height() * 2.75)
+ const rowHeight = $('tbody tr').first().height()
+ // find nearest pagelen to fill display
+ const minPageLen = Math.floor(scrollBodyHeight / rowHeight)
+ if (currPageLen < minPageLen) {
+ for (const pageLen of pagelengthchoices) {
+ if (pageLen >= minPageLen) {
+ api.page.len(pageLen)
+ api.ajax.reload()
+ api.columns.adjust()
+ window.localStorage.removeItem('loadctr-' + gresviewId)
+ console.log('smart sized >' + minPageLen)
+ setTimeout(function () {
+ const api = $('#dtprv').DataTable({ retrieve: true })
+ api.draw(false)
+ fitColText()
+ }, 100)
+ break
+ }
+ }
+ }
+ }
+ }
+ }, // end InitComplete
+ buttons: [{
+ name: 'viewToggleButton',
+ text: gcurrentView === 'table' ? ' ' + that._('Sort') + ' : '
+ sortOrder.forEach((sortcol, idx) => {
+ const colText = datatable.column(sortcol[0]).name()
+ gsortInfo = gsortInfo + colText +
+ (sortcol[1] === 'asc'
+ ? '