From 5243725e393f5374b9c2f54da583a7c0c55d626b Mon Sep 17 00:00:00 2001 From: Elia Bellavista Date: Wed, 17 Jul 2024 16:44:57 +0200 Subject: [PATCH] deleted unused folders --- .../ckanext/activity/__init__.py | 0 .../activity/assets/activity-stream.js | 34 - .../ckanext/activity/assets/activity.css | 103 - .../ckanext/activity/assets/activity.css.map | 7 - .../ckanext/activity/assets/activity.scss | 154 - .../ckanext/activity/assets/dashboard.js | 95 - .../ckanext/activity/assets/dashboard.spec.js | 64 - .../ckanext/activity/assets/webassets.yml | 14 - .../ckanext/activity/changes.py | 1255 - .../ckanext/activity/email_notifications.py | 278 - .../ckanext/activity/helpers.py | 190 - .../ckanext/activity/logic/__init__.py | 0 .../ckanext/activity/logic/action.py | 647 - .../ckanext/activity/logic/auth.py | 184 - .../ckanext/activity/logic/schema.py | 102 - .../ckanext/activity/logic/validators.py | 97 - .../ckanext/activity/model/__init__.py | 28 - .../ckanext/activity/model/activity.py | 791 - .../ckanext/activity/plugin.py | 30 - .../ckanext/activity/public/dotted.png | Bin 74 -> 0 bytes .../ckanext/activity/subscriptions.py | 177 - .../activity_stream_email_notifications.text | 7 - .../templates/ajax_snippets/dashboard.html | 4 - .../ckanext/activity/templates/base.html | 6 - .../templates/group/activity_stream.html | 15 - .../activity/templates/group/changes.html | 64 - .../activity/templates/group/read_base.html | 6 - .../templates/group/snippets/item_group.html | 10 - .../ckanext/activity/templates/header.html | 15 - .../organization/activity_stream.html | 14 - .../templates/organization/changes.html | 64 - .../templates/organization/read_base.html | 6 - .../snippets/item_organization.html | 10 - .../templates/package/activity_stream.html | 24 - .../activity/templates/package/changes.html | 64 - .../activity/templates/package/history.html | 23 - .../activity/templates/package/read_base.html | 6 - .../templates/package/resource_history.html | 26 - .../package/snippets/change_item.html | 10 - .../package/snippets/resource_item.html | 16 - .../templates/package/snippets/resources.html | 12 - .../package/snippets/resources_list.html | 15 - .../ckanext/activity/templates/page.html | 6 - .../snippets/activities/added_tag.html | 15 - .../snippets/activities/changed_group.html | 20 - .../activities/changed_organization.html | 20 - .../snippets/activities/changed_package.html | 26 - .../snippets/activities/changed_resource.html | 15 - .../snippets/activities/changed_user.html | 13 - .../snippets/activities/deleted_group.html | 14 - .../activities/deleted_organization.html | 14 - .../snippets/activities/deleted_package.html | 17 - .../snippets/activities/deleted_resource.html | 15 - .../snippets/activities/fallback.html | 38 - .../snippets/activities/follow_dataset.html | 14 - .../snippets/activities/follow_group.html | 14 - .../snippets/activities/follow_user.html | 14 - .../snippets/activities/new_group.html | 14 - .../snippets/activities/new_organization.html | 14 - .../snippets/activities/new_package.html | 22 - .../snippets/activities/new_resource.html | 15 - .../snippets/activities/new_user.html | 13 - .../snippets/activities/removed_tag.html | 15 - .../snippets/activity_type_selector.html | 25 - .../templates/snippets/changes/author.html | 30 - .../snippets/changes/author_email.html | 47 - .../snippets/changes/delete_resource.html | 10 - .../snippets/changes/extension_fields.html | 9 - .../snippets/changes/extra_fields.html | 85 - .../templates/snippets/changes/license.html | 67 - .../snippets/changes/maintainer.html | 30 - .../snippets/changes/maintainer_email.html | 47 - .../templates/snippets/changes/name.html | 23 - .../templates/snippets/changes/new_file.html | 10 - .../snippets/changes/new_resource.html | 17 - .../templates/snippets/changes/no_change.html | 5 - .../templates/snippets/changes/notes.html | 30 - .../templates/snippets/changes/org.html | 30 - .../templates/snippets/changes/private.html | 8 - .../snippets/changes/resource_desc.html | 39 - .../snippets/changes/resource_extras.html | 112 - .../snippets/changes/resource_format.html | 54 - .../snippets/changes/resource_name.html | 13 - .../templates/snippets/changes/tags.html | 59 - .../templates/snippets/changes/title.html | 8 - .../templates/snippets/changes/url.html | 46 - .../templates/snippets/changes/version.html | 30 - .../snippets/group_changes/description.html | 30 - .../snippets/group_changes/image_url.html | 46 - .../snippets/group_changes/no_change.html | 5 - .../snippets/group_changes/title.html | 8 - .../organization_changes/description.html | 30 - .../organization_changes/image_url.html | 46 - .../organization_changes/no_change.html | 5 - .../snippets/organization_changes/title.html | 8 - .../templates/snippets/pagination.html | 7 - .../activity/templates/snippets/stream.html | 63 - .../templates/user/activity_stream.html | 14 - .../activity/templates/user/dashboard.html | 27 - .../templates/user/edit_user_form.html | 12 - .../activity/templates/user/read_base.html | 6 - .../user/snippets/followee_dropdown.html | 52 - .../ckanext/activity/tests/__init__.py | 0 .../ckanext/activity/tests/conftest.py | 15 - .../ckanext/activity/tests/logic/__init__.py | 0 .../activity/tests/logic/test_action.py | 2178 -- .../ckanext/activity/tests/logic/test_auth.py | 46 - .../activity/tests/logic/test_functional.py | 40 - .../activity/tests/logic/test_pagination.py | 201 - .../ckanext/activity/tests/model/__init__.py | 0 .../activity/tests/model/test_activity.py | 56 - .../ckanext/activity/tests/test_changes.py | 1229 - .../tests/test_email_notifications.py | 66 - .../ckanext/activity/tests/test_helpers.py | 55 - .../ckanext/activity/tests/test_views.py | 958 - .../ckanext/activity/views.py | 943 - .../ckanext/audioview/__init__.py | 0 .../ckanext/audioview/plugin.py | 50 - .../ckanext/audioview/tests/__init__.py | 0 .../ckanext/audioview/tests/test_view.py | 28 - .../audioview/theme/templates/audio_form.html | 3 - .../audioview/theme/templates/audio_view.html | 9 - .../ckanext/chained_functions/__init__.py | 0 .../ckanext/chained_functions/plugin.py | 55 - .../chained_functions/tests/__init__.py | 0 .../chained_functions/tests/test_plugin.py | 22 - .../d4science_theme/controllers/home.py | 2 +- .../d4science_theme/controllers/systemtype.py | 2 +- .../ckanext/datapusher/__init__.py | 0 .../ckanext/datapusher/assets/datapusher.css | 33 - .../datapusher/assets/datapusher.css.map | 7 - .../ckanext/datapusher/assets/datapusher.scss | 49 - .../datapusher/assets/datapusher_popover.js | 27 - .../ckanext/datapusher/assets/webassets.yml | 13 - .../ckanext/datapusher/cli.py | 105 - .../datapusher/config_declaration.yaml | 51 - .../ckanext/datapusher/helpers.py | 31 - .../ckanext/datapusher/interfaces.py | 55 - .../ckanext/datapusher/logic/__init__.py | 0 .../ckanext/datapusher/logic/action.py | 338 - .../ckanext/datapusher/logic/auth.py | 16 - .../ckanext/datapusher/logic/schema.py | 27 - .../ckanext/datapusher/plugin.py | 163 - .../ckanext/datapusher/public/dotted.png | Bin 74 -> 0 bytes .../datapusher/resource_data.html | 88 - .../package/resource_edit_base.html | 16 - .../templates/datapusher/resource_data.html | 95 - .../templates/package/resource_edit_base.html | 16 - .../ckanext/datapusher/tests/__init__.py | 6 - .../ckanext/datapusher/tests/test.py | 322 - .../ckanext/datapusher/tests/test_action.py | 347 - .../datapusher/tests/test_default_views.py | 102 - .../datapusher/tests/test_interfaces.py | 121 - .../ckanext/datapusher/tests/test_views.py | 38 - .../ckanext/datapusher/views.py | 75 - .../ckanext/datastore/__init__.py | 0 .../ckanext/datastore/allowed_functions.txt | 283 - .../ckanext/datastore/backend/__init__.py | 238 - .../ckanext/datastore/backend/postgres.py | 2323 -- .../ckanext/datastore/blueprint.py | 290 - .../ckanext/datastore/cli.py | 179 - .../ckanext/datastore/config_declaration.yaml | 101 - .../ckanext/datastore/helpers.py | 235 - .../ckanext/datastore/interfaces.py | 180 - .../ckanext/datastore/logic/__init__.py | 0 .../ckanext/datastore/logic/action.py | 724 - .../ckanext/datastore/logic/auth.py | 95 - .../ckanext/datastore/logic/schema.py | 225 - .../ckanext/datastore/plugin.py | 279 - .../ckanext/datastore/set_permissions.sql | 108 - .../templates-bs3/ajax_snippets/api_info.html | 137 - .../datastore/api_examples/curl.html | 40 - .../datastore/api_examples/javascript.html | 55 - .../datastore/api_examples/powershell.html | 48 - .../datastore/api_examples/python.html | 52 - .../datastore/api_examples/r.html | 51 - .../templates-bs3/datastore/dictionary.html | 22 - .../datastore/snippets/dictionary_form.html | 25 - .../package/resource_edit_base.html | 9 - .../templates-bs3/package/resource_read.html | 49 - .../package/snippets/data_api_button.html | 10 - .../package/snippets/dictionary_table.html | 7 - .../package/snippets/resource_item.html | 8 - .../package/snippets/resources.html | 8 - .../templates/ajax_snippets/api_info.html | 137 - .../templates/datastore/dictionary.html | 22 - .../datastore/snippets/dictionary_form.html | 25 - .../templates/package/resource_edit_base.html | 8 - .../templates/package/resource_read.html | 37 - .../package/snippets/data_api_button.html | 10 - .../package/snippets/dictionary_table.html | 7 - .../ckanext/datastore/tests/__init__.py | 0 .../datastore/tests/allowed_functions.txt | 2 - .../ckanext/datastore/tests/conftest.py | 14 - .../ckanext/datastore/tests/helpers.py | 107 - .../tests/sample_datastore_plugin.py | 56 - .../ckanext/datastore/tests/test_auth.py | 278 - .../datastore/tests/test_chained_action.py | 98 - .../tests/test_chained_auth_functions.py | 100 - .../ckanext/datastore/tests/test_create.py | 1449 - .../ckanext/datastore/tests/test_db.py | 225 - .../ckanext/datastore/tests/test_delete.py | 447 - .../datastore/tests/test_dictionary.py | 35 - .../ckanext/datastore/tests/test_disable.py | 28 - .../ckanext/datastore/tests/test_dump.py | 669 - .../ckanext/datastore/tests/test_helpers.py | 214 - .../ckanext/datastore/tests/test_info.py | 88 - .../ckanext/datastore/tests/test_interface.py | 169 - .../ckanext/datastore/tests/test_plugin.py | 195 - .../ckanext/datastore/tests/test_search.py | 2181 -- .../ckanext/datastore/tests/test_unit.py | 39 - .../ckanext/datastore/tests/test_upsert.py | 953 - .../ckanext/datastore/writer.py | 179 - .../ckanext/datatablesview/__init__.py | 0 .../ckanext/datatablesview/blueprint.py | 215 - .../datatablesview/config_declaration.yaml | 95 - .../ckanext/datatablesview/plugin.py | 99 - .../datatablesview/public/datatables_view.css | 220 - .../datatablesview/public/datatablesview.js | 944 - .../datatablesview/public/resource.config | 3 - .../DataTables/dataTables.scrollResize.js | 195 - .../public/vendor/DataTables/datatables.css | 1165 - .../public/vendor/DataTables/datatables.js | 25227 ---------------- .../vendor/DataTables/datatables.mark.css | 4 - .../vendor/DataTables/datatables.mark.js | 119 - .../public/vendor/DataTables/datetime.js | 126 - .../public/vendor/DataTables/i18n/af.json | 22 - .../public/vendor/DataTables/i18n/am.json | 22 - .../public/vendor/DataTables/i18n/ar.json | 43 - .../public/vendor/DataTables/i18n/az.json | 22 - .../public/vendor/DataTables/i18n/az_az.json | 22 - .../public/vendor/DataTables/i18n/be.json | 19 - .../public/vendor/DataTables/i18n/bg.json | 15 - .../public/vendor/DataTables/i18n/bn.json | 15 - .../public/vendor/DataTables/i18n/bs.json | 76 - .../public/vendor/DataTables/i18n/ca.json | 44 - .../public/vendor/DataTables/i18n/cs.json | 22 - .../public/vendor/DataTables/i18n/cs_CZ.json | 134 - .../public/vendor/DataTables/i18n/cy.json | 22 - .../public/vendor/DataTables/i18n/da.json | 15 - .../public/vendor/DataTables/i18n/de.json | 43 - .../public/vendor/DataTables/i18n/el.json | 25 - .../public/vendor/DataTables/i18n/en-gb.json | 137 - .../public/vendor/DataTables/i18n/en_GB.json | 178 - .../public/vendor/DataTables/i18n/eo.json | 22 - .../public/vendor/DataTables/i18n/es.json | 26 - .../public/vendor/DataTables/i18n/et.json | 15 - .../public/vendor/DataTables/i18n/eu.json | 22 - .../public/vendor/DataTables/i18n/fa.json | 22 - .../public/vendor/DataTables/i18n/fi.json | 38 - .../public/vendor/DataTables/i18n/fil.json | 15 - .../public/vendor/DataTables/i18n/fr.json | 29 - .../public/vendor/DataTables/i18n/ga.json | 15 - .../public/vendor/DataTables/i18n/gl.json | 22 - .../public/vendor/DataTables/i18n/gu.json | 22 - .../public/vendor/DataTables/i18n/he.json | 16 - .../public/vendor/DataTables/i18n/hi.json | 15 - .../public/vendor/DataTables/i18n/hr.json | 22 - .../public/vendor/DataTables/i18n/hu.json | 38 - .../public/vendor/DataTables/i18n/hy.json | 22 - .../public/vendor/DataTables/i18n/id.json | 82 - .../public/vendor/DataTables/i18n/id_alt.json | 15 - .../public/vendor/DataTables/i18n/is.json | 22 - .../public/vendor/DataTables/i18n/it.json | 22 - .../public/vendor/DataTables/i18n/ja.json | 22 - .../public/vendor/DataTables/i18n/ka.json | 22 - .../public/vendor/DataTables/i18n/kk.json | 21 - .../public/vendor/DataTables/i18n/km.json | 22 - .../public/vendor/DataTables/i18n/kn.json | 22 - .../public/vendor/DataTables/i18n/ko.json | 22 - .../public/vendor/DataTables/i18n/ko_KR.json | 68 - .../public/vendor/DataTables/i18n/ku.json | 21 - .../public/vendor/DataTables/i18n/ky.json | 22 - .../public/vendor/DataTables/i18n/lo.json | 22 - .../public/vendor/DataTables/i18n/lt.json | 19 - .../public/vendor/DataTables/i18n/lv.json | 21 - .../public/vendor/DataTables/i18n/mk.json | 17 - .../public/vendor/DataTables/i18n/mn.json | 22 - .../public/vendor/DataTables/i18n/ms.json | 22 - .../public/vendor/DataTables/i18n/ne.json | 22 - .../public/vendor/DataTables/i18n/nl.json | 22 - .../public/vendor/DataTables/i18n/no.json | 22 - .../public/vendor/DataTables/i18n/no_nb.json | 22 - .../public/vendor/DataTables/i18n/pa.json | 22 - .../public/vendor/DataTables/i18n/pl.json | 21 - .../public/vendor/DataTables/i18n/ps.json | 22 - .../public/vendor/DataTables/i18n/pt.json | 37 - .../public/vendor/DataTables/i18n/pt_BR.json | 131 - .../public/vendor/DataTables/i18n/pt_pt.json | 21 - .../public/vendor/DataTables/i18n/rm.json | 43 - .../public/vendor/DataTables/i18n/ro.json | 15 - .../public/vendor/DataTables/i18n/ru.json | 28 - .../public/vendor/DataTables/i18n/si.json | 22 - .../public/vendor/DataTables/i18n/sk.json | 22 - .../public/vendor/DataTables/i18n/sl.json | 22 - .../public/vendor/DataTables/i18n/snd.json | 30 - .../public/vendor/DataTables/i18n/sq.json | 22 - .../public/vendor/DataTables/i18n/sr.json | 22 - .../vendor/DataTables/i18n/sr@latin.json | 22 - .../public/vendor/DataTables/i18n/sr_sp.json | 22 - .../public/vendor/DataTables/i18n/sv.json | 22 - .../public/vendor/DataTables/i18n/sw.json | 22 - .../public/vendor/DataTables/i18n/ta.json | 22 - .../public/vendor/DataTables/i18n/te.json | 22 - .../public/vendor/DataTables/i18n/tg.json | 28 - .../public/vendor/DataTables/i18n/th.json | 22 - .../public/vendor/DataTables/i18n/tl.json | 15 - .../public/vendor/DataTables/i18n/tr.json | 29 - .../public/vendor/DataTables/i18n/uk.json | 19 - .../public/vendor/DataTables/i18n/ur.json | 15 - .../public/vendor/DataTables/i18n/uz.json | 21 - .../public/vendor/DataTables/i18n/vi.json | 15 - .../public/vendor/DataTables/i18n/zh.json | 22 - .../public/vendor/DataTables/i18n/zh_CN.json | 22 - .../vendor/DataTables/i18n/zh_Hant.json | 20 - .../public/vendor/DataTables/jquery.mark.js | 1228 - .../FontAwesome/images/times-circle-solid.svg | 1 - .../datatablesview/public/webassets.yml | 18 - .../templates/datatables/datatables_form.html | 68 - .../templates/datatables/datatables_view.html | 103 - .../ckanext/expire_api_token/__init__.py | 0 .../ckanext/expire_api_token/plugin.py | 74 - .../templates/user/api_tokens.html | 18 - .../user/snippets/api_token_list.html | 16 - .../expire_api_token/tests/__init__.py | 0 .../expire_api_token/tests/test_plugin.py | 45 - .../ckanext/imageview/__init__.py | 0 .../ckanext/imageview/config_declaration.yaml | 8 - .../ckanext/imageview/plugin.py | 45 - .../ckanext/imageview/tests/__init__.py | 0 .../ckanext/imageview/tests/test_view.py | 28 - .../imageview/theme/templates/image_form.html | 3 - .../imageview/theme/templates/image_view.html | 1 - .../ckanext/multilingual/__init__.py | 0 .../ckanext/multilingual/plugin.py | 413 - .../ckanext/multilingual/solr/dutch_stop.txt | 117 - .../multilingual/solr/english_stop.txt | 317 - .../ckanext/multilingual/solr/fr_elision.txt | 8 - .../ckanext/multilingual/solr/french_stop.txt | 183 - .../ckanext/multilingual/solr/german_stop.txt | 292 - .../multilingual/solr/greek_stopwords.txt | 76 - .../multilingual/solr/italian_stop.txt | 301 - .../ckanext/multilingual/solr/polish_stop.txt | 186 - .../multilingual/solr/portuguese_stop.txt | 251 - .../multilingual/solr/romanian_stop.txt | 233 - .../ckanext/multilingual/solr/schema.xml | 486 - .../multilingual/solr/spanish_stop.txt | 354 - .../tests/test_multilingual_plugin.py | 239 - .../ckanext/reclineview/__init__.py | 0 .../reclineview/config_declaration.yaml | 14 - .../ckanext/reclineview/plugin.py | 306 - .../ckanext/reclineview/tests/__init__.py | 0 .../ckanext/reclineview/tests/test_view.py | 152 - .../reclineview/theme/public/css/recline.css | 380 - .../theme/public/css/recline.min.css | 1 - .../theme/public/img/ajaxload-circle.gif | Bin 4176 -> 0 bytes .../reclineview/theme/public/recline_view.js | 247 - .../theme/public/recline_view.min.js | 15 - .../reclineview/theme/public/resource.config | 15 - .../public/vendor/backbone/1.0.0/backbone.js | 1571 - .../bootstrap/3.2.0/css/bootstrap-theme.css | 442 - .../vendor/bootstrap/3.2.0/css/bootstrap.css | 6203 ---- .../fonts/glyphicons-halflings-regular.eot | Bin 20335 -> 0 bytes .../fonts/glyphicons-halflings-regular.svg | 229 - .../fonts/glyphicons-halflings-regular.ttf | Bin 41280 -> 0 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 23320 -> 0 bytes .../vendor/bootstrap/3.2.0/js/bootstrap.js | 2114 -- .../theme/public/vendor/ckan.js/ckan.js | 262 - .../theme/public/vendor/flot/excanvas.js | 1428 - .../theme/public/vendor/flot/excanvas.min.js | 1 - .../theme/public/vendor/flot/jquery.flot.js | 3061 -- .../public/vendor/flot/jquery.flot.time.js | 431 - .../theme/public/vendor/flotr2/flotr2.js | 6128 ---- .../theme/public/vendor/flotr2/flotr2.min.js | 489 - .../public/vendor/jquery/1.7.1/jquery.js | 9266 ------ .../public/vendor/jquery/1.7.1/jquery.min.js | 4 - .../theme/public/vendor/json/json2.js | 486 - .../theme/public/vendor/json/json2.min.js | 25 - .../MarkerCluster.Default.css | 60 - .../leaflet.markercluster/MarkerCluster.css | 6 - .../leaflet.markercluster-src.js | 2163 -- .../leaflet.markercluster.js | 6 - .../vendor/leaflet/0.7.7/images/layers-2x.png | Bin 1585 -> 0 bytes .../vendor/leaflet/0.7.7/images/layers.png | Bin 913 -> 0 bytes .../leaflet/0.7.7/images/marker-icon-2x.png | Bin 4032 -> 0 bytes .../leaflet/0.7.7/images/marker-icon.png | Bin 1747 -> 0 bytes .../leaflet/0.7.7/images/marker-shadow.png | Bin 681 -> 0 bytes .../vendor/leaflet/0.7.7/leaflet-src.js | 9168 ------ .../public/vendor/leaflet/0.7.7/leaflet.css | 479 - .../public/vendor/leaflet/0.7.7/leaflet.js | 9 - .../public/vendor/moment/2.0.0/moment.js | 1400 - .../vendor/mustache/0.5.0-dev/mustache.js | 536 - .../vendor/mustache/0.5.0-dev/mustache.min.js | 36 - .../theme/public/vendor/recline/flot.css | 26 - .../theme/public/vendor/recline/map.css | 28 - .../theme/public/vendor/recline/recline.css | 644 - .../public/vendor/recline/recline.dataset.js | 896 - .../theme/public/vendor/recline/recline.js | 4463 --- .../theme/public/vendor/recline/slickgrid.css | 188 - .../vendor/showdown/20120615/showdown.js | 1341 - .../vendor/showdown/20120615/showdown.min.js | 48 - .../vendor/slickgrid/2.2/MIT-LICENSE.txt | 20 - .../public/vendor/slickgrid/2.2/README.md | 25 - .../2.2/controls/slick.columnpicker.css | 31 - .../2.2/controls/slick.columnpicker.js | 152 - .../slickgrid/2.2/controls/slick.pager.css | 41 - .../slickgrid/2.2/controls/slick.pager.js | 154 - .../images/ui-bg_flat_0_aaaaaa_40x100.png | Bin 86 -> 0 bytes .../images/ui-bg_flat_75_ffffff_40x100.png | Bin 74 -> 0 bytes .../images/ui-bg_glass_55_fbf9ee_1x400.png | Bin 111 -> 0 bytes .../images/ui-bg_glass_65_ffffff_1x400.png | Bin 90 -> 0 bytes .../images/ui-bg_glass_75_dadada_1x400.png | Bin 102 -> 0 bytes .../images/ui-bg_glass_75_e6e6e6_1x400.png | Bin 102 -> 0 bytes .../images/ui-bg_glass_95_fef1ec_1x400.png | Bin 115 -> 0 bytes .../ui-bg_highlight-soft_75_cccccc_1x100.png | Bin 86 -> 0 bytes .../images/ui-icons_222222_256x240.png | Bin 3687 -> 0 bytes .../images/ui-icons_2e83ff_256x240.png | Bin 3687 -> 0 bytes .../images/ui-icons_454545_256x240.png | Bin 3687 -> 0 bytes .../images/ui-icons_888888_256x240.png | Bin 3687 -> 0 bytes .../images/ui-icons_cd0a0a_256x240.png | Bin 3687 -> 0 bytes .../smoothness/jquery-ui-1.8.16.custom.css | 409 - .../vendor/slickgrid/2.2/images/actions.gif | Bin 170 -> 0 bytes .../2.2/images/ajax-loader-small.gif | Bin 1849 -> 0 bytes .../slickgrid/2.2/images/arrow_redo.png | Bin 550 -> 0 bytes .../2.2/images/arrow_right_peppermint.png | Bin 126 -> 0 bytes .../2.2/images/arrow_right_spearmint.png | Bin 126 -> 0 bytes .../slickgrid/2.2/images/arrow_undo.png | Bin 555 -> 0 bytes .../slickgrid/2.2/images/bullet_blue.png | Bin 231 -> 0 bytes .../slickgrid/2.2/images/bullet_star.png | Bin 273 -> 0 bytes .../2.2/images/bullet_toggle_minus.png | Bin 154 -> 0 bytes .../2.2/images/bullet_toggle_plus.png | Bin 156 -> 0 bytes .../vendor/slickgrid/2.2/images/calendar.gif | Bin 1035 -> 0 bytes .../vendor/slickgrid/2.2/images/collapse.gif | Bin 846 -> 0 bytes .../slickgrid/2.2/images/comment_yellow.gif | Bin 257 -> 0 bytes .../vendor/slickgrid/2.2/images/down.gif | Bin 59 -> 0 bytes .../slickgrid/2.2/images/drag-handle.png | Bin 98 -> 0 bytes .../slickgrid/2.2/images/editor-helper-bg.gif | Bin 1164 -> 0 bytes .../vendor/slickgrid/2.2/images/expand.gif | Bin 851 -> 0 bytes .../vendor/slickgrid/2.2/images/header-bg.gif | Bin 872 -> 0 bytes .../2.2/images/header-columns-bg.gif | Bin 836 -> 0 bytes .../2.2/images/header-columns-over-bg.gif | Bin 823 -> 0 bytes .../vendor/slickgrid/2.2/images/help.png | Bin 328 -> 0 bytes .../vendor/slickgrid/2.2/images/info.gif | Bin 80 -> 0 bytes .../vendor/slickgrid/2.2/images/listview.gif | Bin 2380 -> 0 bytes .../vendor/slickgrid/2.2/images/pencil.gif | Bin 914 -> 0 bytes .../slickgrid/2.2/images/row-over-bg.gif | Bin 823 -> 0 bytes .../vendor/slickgrid/2.2/images/sort-asc.gif | Bin 830 -> 0 bytes .../vendor/slickgrid/2.2/images/sort-asc.png | Bin 104 -> 0 bytes .../vendor/slickgrid/2.2/images/sort-desc.gif | Bin 833 -> 0 bytes .../vendor/slickgrid/2.2/images/sort-desc.png | Bin 106 -> 0 bytes .../vendor/slickgrid/2.2/images/stripes.png | Bin 94 -> 0 bytes .../vendor/slickgrid/2.2/images/tag_red.png | Bin 529 -> 0 bytes .../vendor/slickgrid/2.2/images/tick.png | Bin 465 -> 0 bytes .../slickgrid/2.2/images/user_identity.gif | Bin 905 -> 0 bytes .../2.2/images/user_identity_plus.gif | Bin 546 -> 0 bytes .../vendor/slickgrid/2.2/jquery-1.7.min.js | 4 - .../slickgrid/2.2/jquery-ui-1.8.16.custom.js | 611 - .../slickgrid/2.2/jquery.event.drag-2.2.js | 402 - .../slickgrid/2.2/jquery.event.drop-2.2.js | 302 - .../2.2/plugins/slick.autotooltips.js | 83 - .../2.2/plugins/slick.cellcopymanager.js | 86 - .../2.2/plugins/slick.cellrangedecorator.js | 66 - .../2.2/plugins/slick.cellrangeselector.js | 113 - .../2.2/plugins/slick.cellselectionmodel.js | 154 - .../2.2/plugins/slick.checkboxselectcolumn.js | 153 - .../2.2/plugins/slick.headerbuttons.css | 39 - .../2.2/plugins/slick.headerbuttons.js | 177 - .../2.2/plugins/slick.headermenu.css | 59 - .../slickgrid/2.2/plugins/slick.headermenu.js | 275 - .../2.2/plugins/slick.rowmovemanager.js | 138 - .../2.2/plugins/slick.rowselectionmodel.js | 187 - .../slickgrid/2.2/slick-default-theme.css | 118 - .../public/vendor/slickgrid/2.2/slick.core.js | 467 - .../vendor/slickgrid/2.2/slick.dataview.js | 1126 - .../vendor/slickgrid/2.2/slick.editors.js | 512 - .../vendor/slickgrid/2.2/slick.formatters.js | 59 - .../vendor/slickgrid/2.2/slick.grid.css | 157 - .../public/vendor/slickgrid/2.2/slick.grid.js | 3422 --- .../2.2/slick.groupitemmetadataprovider.js | 158 - .../vendor/slickgrid/2.2/slick.remotemodel.js | 173 - .../theme/public/vendor/timeline/LICENSE | 365 - .../theme/public/vendor/timeline/README | 1 - .../public/vendor/timeline/css/loading.gif | Bin 6909 -> 0 bytes .../public/vendor/timeline/css/timeline.css | 284 - .../public/vendor/timeline/css/timeline.png | Bin 14048 -> 0 bytes .../vendor/timeline/css/timeline@2x.png | Bin 36430 -> 0 bytes .../public/vendor/timeline/js/timeline.js | 10015 ------ .../0.4.0/underscore.deferred.js | 445 - .../0.4.0/underscore.deferred.min.js | 17 - .../vendor/underscore/1.4.4/underscore.js | 1227 - .../reclineview/theme/public/webassets.yml | 47 - .../theme/public/widget.recordcount.js | 34 - .../theme/public/widget.recordcount.min.js | 3 - .../theme/templates/recline_graph_form.html | 8 - .../theme/templates/recline_map_form.html | 11 - .../theme/templates/recline_view.html | 24 - .../ckanext/resourceproxy/__init__.py | 0 .../ckanext/resourceproxy/blueprint.py | 115 - .../resourceproxy/config_declaration.yaml | 22 - .../ckanext/resourceproxy/plugin.py | 76 - .../ckanext/resourceproxy/tests/__init__.py | 0 .../resourceproxy/tests/static/huge.json | 3 - .../resourceproxy/tests/static/test.json | 5 - .../ckanext/resourceproxy/tests/test_proxy.py | 178 - .../ckanext/stats/__init__.py | 3 - .../ckanext/stats/blueprint.py | 55 - .../ckanext/stats/plugin.py | 24 - .../ckanext/stats/public/.gitignore | 2 - .../ckanext/stats/public/__init__.py | 9 - .../ckanext/stats/public/ckanext/__init__.py | 9 - .../stats/public/ckanext/stats/__init__.py | 9 - .../stats/public/ckanext/stats/css/stats.css | 16 - .../ckanext/stats/javascript/modules/plot.js | 209 - .../stats/javascript/modules/stats-nav.js | 39 - .../public/ckanext/stats/resource.config | 12 - .../ckanext/stats/test/fixtures/table.html | 30 - .../public/ckanext/stats/test/index.html | 59 - .../stats/test/spec/modules/plot.spec.js | 136 - .../stats/test/spec/modules/stats-nav.spec.js | 44 - .../public/ckanext/stats/vendor/excanvas.js | 1427 - .../ckanext/stats/vendor/jquery.flot.js | 2599 -- .../stats/public/ckanext/stats/webassets.yml | 16 - .../ckanext/stats/stats.py | 405 - .../stats/templates/ckanext/stats/index.html | 178 - .../ckanext/stats/tests/__init__.py | 0 .../ckanext/stats/tests/test_stats_lib.py | 207 - .../ckanext/stats/tests/test_stats_plugin.py | 11 - .../ckanext/videoview/__init__.py | 0 .../ckanext/videoview/plugin.py | 52 - .../ckanext/videoview/tests/__init__.py | 0 .../ckanext/videoview/tests/test_view.py | 29 - .../videoview/theme/templates/video_form.html | 4 - .../videoview/theme/templates/video_view.html | 12 - .../ckanext/webpageview/__init__.py | 0 .../ckanext/webpageview/plugin.py | 45 - .../ckanext/webpageview/tests/__init__.py | 0 .../ckanext/webpageview/tests/test_view.py | 34 - .../theme/templates/webpage_form.html | 3 - .../theme/templates/webpage_view.html | 3 - 539 files changed, 2 insertions(+), 147254 deletions(-) delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity-stream.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css.map delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.scss delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.spec.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/webassets.yml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/changes.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/email_notifications.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/helpers.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/action.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/auth.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/schema.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/validators.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/activity.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/public/dotted.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/subscriptions.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/activity_streams/activity_stream_email_notifications.text delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/ajax_snippets/dashboard.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/activity_stream.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/changes.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/read_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/snippets/item_group.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/header.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/activity_stream.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/changes.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/read_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/snippets/item_organization.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/activity_stream.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/changes.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/history.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/read_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/resource_history.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/change_item.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resource_item.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources_list.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/page.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/added_tag.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_group.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_organization.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_package.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_resource.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_user.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_group.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_organization.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_package.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_resource.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/fallback.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_dataset.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_group.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_user.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_group.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_organization.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_package.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_resource.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_user.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/removed_tag.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activity_type_selector.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author_email.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/delete_resource.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extension_fields.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extra_fields.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/license.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer_email.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/name.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_file.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_resource.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/no_change.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/notes.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/org.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/private.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_desc.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_extras.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_format.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_name.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/tags.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/title.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/url.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/version.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/description.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/image_url.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/no_change.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/title.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/description.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/image_url.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/no_change.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/title.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/pagination.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/stream.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/activity_stream.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/dashboard.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/edit_user_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/read_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/snippets/followee_dropdown.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/conftest.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_action.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_auth.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_functional.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_pagination.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/model/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/model/test_activity.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/test_changes.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/test_email_notifications.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/test_helpers.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/test_views.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/activity/views.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/tests/test_view.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/theme/templates/audio_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/theme/templates/audio_view.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/test_plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/cli.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/public/dotted.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/views.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/cli.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/helpers.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/interfaces.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/logic/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/logic/action.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/logic/auth.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/logic/schema.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/set_permissions.sql delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/ajax_snippets/api_info.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/dictionary_table.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/dictionary.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/snippets/dictionary_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/writer.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatables_view.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatablesview.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/resource.config delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/dataTables.scrollResize.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/datatables.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/datatables.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/datatables.mark.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/datatables.mark.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/datetime.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/af.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/am.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ar.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/az.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/az_az.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/be.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/bg.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/bn.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/bs.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ca.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/cs.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/cs_CZ.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/cy.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/da.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/de.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/el.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/en-gb.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/en_GB.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/eo.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/es.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/et.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/eu.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/fa.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/fi.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/fil.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/fr.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ga.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/gl.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/gu.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/he.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/hi.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/hr.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/hu.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/hy.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/id.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/id_alt.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/is.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/it.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ja.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ka.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/kk.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/km.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/kn.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ko.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ko_KR.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ku.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ky.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/lo.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/lt.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/lv.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/mk.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/mn.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ms.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ne.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/nl.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/no.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/no_nb.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/pa.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/pl.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ps.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/pt.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/pt_BR.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/pt_pt.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/rm.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ro.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ru.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/si.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sk.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sl.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/snd.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sq.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sr.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sr@latin.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sr_sp.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sv.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/sw.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ta.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/te.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/tg.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/th.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/tl.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/tr.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/uk.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/ur.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/uz.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/vi.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/zh.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/zh_CN.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/i18n/zh_Hant.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/DataTables/jquery.mark.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/vendor/FontAwesome/images/times-circle-solid.svg delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/webassets.yml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/templates/datatables/datatables_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/templates/datatables/datatables_view.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/expire_api_token/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/expire_api_token/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/expire_api_token/templates/user/api_tokens.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/expire_api_token/templates/user/snippets/api_token_list.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/expire_api_token/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/expire_api_token/tests/test_plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/config_declaration.yaml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/tests/test_view.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/theme/templates/image_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/imageview/theme/templates/image_view.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/dutch_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/english_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/fr_elision.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/french_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/german_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/greek_stopwords.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/italian_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/polish_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/portuguese_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/romanian_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/schema.xml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/solr/spanish_stop.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/multilingual/tests/test_multilingual_plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/config_declaration.yaml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/tests/test_view.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/css/recline.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/css/recline.min.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/img/ajaxload-circle.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/recline_view.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/recline_view.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/resource.config delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/backbone/1.0.0/backbone.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/css/bootstrap-theme.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/css/bootstrap.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.eot delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.svg delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.ttf delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.woff delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/js/bootstrap.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/ckan.js/ckan.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/flot/excanvas.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/flot/excanvas.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/flot/jquery.flot.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/flot/jquery.flot.time.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/flotr2/flotr2.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/flotr2/flotr2.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/jquery/1.7.1/jquery.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/jquery/1.7.1/jquery.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/json/json2.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/json/json2.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet.markercluster/MarkerCluster.Default.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet.markercluster/MarkerCluster.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet.markercluster/leaflet.markercluster-src.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet.markercluster/leaflet.markercluster.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/images/layers-2x.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/images/layers.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/images/marker-icon-2x.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/images/marker-icon.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/images/marker-shadow.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/leaflet-src.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/leaflet.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/leaflet/0.7.7/leaflet.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/moment/2.0.0/moment.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/mustache/0.5.0-dev/mustache.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/mustache/0.5.0-dev/mustache.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/recline/flot.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/recline/map.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/recline/recline.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/recline/recline.dataset.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/recline/recline.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/recline/slickgrid.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/showdown/20120615/showdown.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/showdown/20120615/showdown.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/MIT-LICENSE.txt delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/README.md delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.columnpicker.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.columnpicker.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.pager.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.pager.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_222222_256x240.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_2e83ff_256x240.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_454545_256x240.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_888888_256x240.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_cd0a0a_256x240.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/jquery-ui-1.8.16.custom.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/actions.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/ajax-loader-small.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_redo.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_right_peppermint.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_right_spearmint.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_undo.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_blue.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_star.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_toggle_minus.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_toggle_plus.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/calendar.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/collapse.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/comment_yellow.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/down.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/drag-handle.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/editor-helper-bg.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/expand.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/header-bg.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/header-columns-bg.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/header-columns-over-bg.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/help.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/info.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/listview.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/pencil.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/row-over-bg.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/sort-asc.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/sort-asc.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/sort-desc.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/sort-desc.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/stripes.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/tag_red.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/tick.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/user_identity.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/user_identity_plus.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery-1.7.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery-ui-1.8.16.custom.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drag-2.2.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drop-2.2.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.autotooltips.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellcopymanager.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangedecorator.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangeselector.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellselectionmodel.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.checkboxselectcolumn.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowmovemanager.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowselectionmodel.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick-default-theme.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.core.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.dataview.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.editors.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.formatters.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.grid.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.grid.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.groupitemmetadataprovider.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.remotemodel.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/LICENSE delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/README delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/css/loading.gif delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/css/timeline.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/css/timeline.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/css/timeline@2x.png delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/timeline/js/timeline.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/underscore.deferred/0.4.0/underscore.deferred.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/underscore.deferred/0.4.0/underscore.deferred.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/underscore/1.4.4/underscore.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/webassets.yml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/widget.recordcount.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/widget.recordcount.min.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/templates/recline_graph_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/templates/recline_map_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/templates/recline_view.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/blueprint.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/config_declaration.yaml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/tests/static/huge.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/tests/static/test.json delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/resourceproxy/tests/test_proxy.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/blueprint.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/.gitignore delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/css/stats.css delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/javascript/modules/plot.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/javascript/modules/stats-nav.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/resource.config delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/test/fixtures/table.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/test/index.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/test/spec/modules/plot.spec.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/test/spec/modules/stats-nav.spec.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/vendor/excanvas.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/vendor/jquery.flot.js delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/public/ckanext/stats/webassets.yml delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/stats.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/templates/ckanext/stats/index.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/tests/test_stats_lib.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/stats/tests/test_stats_plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/videoview/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/videoview/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/videoview/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/videoview/tests/test_view.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/videoview/theme/templates/video_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/videoview/theme/templates/video_view.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/webpageview/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/webpageview/plugin.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/webpageview/tests/__init__.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/webpageview/tests/test_view.py delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/webpageview/theme/templates/webpage_form.html delete mode 100644 compose/2.10/src/ckanext-d4science_theme/ckanext/webpageview/theme/templates/webpage_view.html diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity-stream.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity-stream.js deleted file mode 100644 index 69874ba..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity-stream.js +++ /dev/null @@ -1,34 +0,0 @@ -/* Activity stream - * Handle the pagination for activity list - * - * Options - * - page: current page number - */ -this.ckan.module('activity-stream', function($) { - return { - - /* Initialises the module setting up elements and event listeners. - * - * Returns nothing. - */ - initialize: function () { - $('#activity_types_filter_select').on( - 'change', - this._onChangeActivityType - ); - }, - - - /* Filter using the selected - * activity type - * - * Returns nothing - */ - _onChangeActivityType: function (event) { - // event.preventDefault(); - url = $("#activity_types_filter_select option:selected" ).data('url'); - window.location = url; - }, - - }; -}); diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css deleted file mode 100644 index e24a4ae..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css +++ /dev/null @@ -1,103 +0,0 @@ -.activity { - padding: 0; - list-style-type: none; - background: transparent url("/dotted.png") 21px 0 repeat-y; } - .activity .item { - position: relative; - margin: 0 0 15px 0; - padding: 0; } - .activity .item .user-image { - border-radius: 100px; } - .activity .item .date { - color: #999; - font-size: 12px; - white-space: nowrap; - margin: 5px 0 0 80px; } - .activity .item.no-avatar p { - margin-left: 40px; } - -.activity_buttons > a.btn.disabled { - color: inherit; - opacity: 0.70 !important; } - -.popover { - width: 300px; } - .popover .popover-title { - font-weight: bold; - margin-bottom: 0; } - .popover p.about { - margin: 0 0 10px 0; } - .popover .popover-close { - float: right; - text-decoration: none; } - .popover .empty { - padding: 10px; - color: #6e6e6e; - font-style: italic; } - -.activity .item .icon { - color: #999999; } -.activity .item.failure .icon { - color: #B95252; } -.activity .item.success .icon { - color: #69A67A; } -.activity .item.added-tag .icon { - color: #6995a6; } -.activity .item.changed-group .icon { - color: #767DCE; } -.activity .item.changed-package .icon { - color: #8c76ce; } -.activity .item.changed-package_extra .icon { - color: #769ace; } -.activity .item.changed-resource .icon { - color: #aa76ce; } -.activity .item.changed-user .icon { - color: #76b8ce; } -.activity .item.changed-organization .icon { - color: #699fa6; } -.activity .item.deleted-group .icon { - color: #B95252; } -.activity .item.deleted-package .icon { - color: #b97452; } -.activity .item.deleted-package_extra .icon { - color: #b95274; } -.activity .item.deleted-resource .icon { - color: #b99752; } -.activity .item.deleted-organization .icon { - color: #b95297; } -.activity .item.new-group .icon { - color: #69A67A; } -.activity .item.new-package .icon { - color: #69a68e; } -.activity .item.new-package_extra .icon { - color: #6ca669; } -.activity .item.new-resource .icon { - color: #81a669; } -.activity .item.new-user .icon { - color: #69a6a3; } -.activity .item.new-organization .icon { - color: #81a669; } -.activity .item.removed-tag .icon { - color: #b95297; } -.activity .item.deleted-related-item .icon { - color: #b9b952; } -.activity .item.follow-dataset .icon { - color: #767DCE; } -.activity .item.follow-user .icon { - color: #8c76ce; } -.activity .item.new-related-item .icon { - color: #95a669; } -.activity .item.follow-group .icon { - color: #8ba669; } - -.select-time { - width: 250px; - display: inline; } - -br.line-height2 { - line-height: 2; } - -.pull-right { - float: right; } - -/*# sourceMappingURL=activity.css.map */ diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css.map b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css.map deleted file mode 100644 index 76482e9..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.css.map +++ /dev/null @@ -1,7 +0,0 @@ -{ -"version": 3, -"mappings": "AAAA,SAAU;EACN,OAAO,EAAE,CAAC;EACV,eAAe,EAAE,IAAI;EACrB,UAAU,EAAE,8CAA+C;EAC3D,eAAM;IACF,QAAQ,EAAE,QAAQ;IAClB,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,CAAC;IACV,2BAAY;MACR,aAAa,EAAE,KAAK;IAExB,qBAAM;MACF,KAAK,EAAE,IAAI;MACX,SAAS,EAAE,IAAI;MACf,WAAW,EAAE,MAAM;MACnB,MAAM,EAAE,YAAY;IAExB,2BAAc;MAEV,WAAW,EAAE,IAAI;;AAK7B,kCAAmC;EAC/B,KAAK,EAAE,OAAO;EACd,OAAO,EAAE,eAAe;;AAG5B,QAAS;EACL,KAAK,EAAE,KAAK;EACZ,uBAAe;IACX,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,CAAC;EAEpB,gBAAQ;IACJ,MAAM,EAAE,UAAU;EAEtB,uBAAe;IACX,KAAK,EAAE,KAAK;IACZ,eAAe,EAAE,IAAI;EAEzB,eAAO;IACH,OAAO,EAAE,IAAI;IACb,KAAK,EAAE,OAAO;IACd,UAAU,EAAE,MAAM;;AAatB,qBAAM;EACF,KAAK,EARQ,OAAO;AAUxB,6BAAgB;EACZ,KAAK,EAPS,OAAO;AASzB,6BAAgB;EACZ,KAAK,EAbM,OAAO;AAetB,+BAAkB;EACd,KAAK,EAAE,OAAiC;AAE5C,mCAAsB;EAClB,KAAK,EAjBS,OAAO;AAmBzB,qCAAwB;EACpB,KAAK,EAAE,OAAoC;AAE/C,2CAA8B;EAC1B,KAAK,EAAE,OAAqC;AAEhD,sCAAyB;EACrB,KAAK,EAAE,OAAoC;AAE/C,kCAAqB;EACjB,KAAK,EAAE,OAAqC;AAEhD,0CAA6B;EACzB,KAAK,EAAE,OAAiC;AAE5C,mCAAsB;EAClB,KAAK,EAlCS,OAAO;AAoCzB,qCAAwB;EACpB,KAAK,EAAE,OAAoC;AAE/C,2CAA8B;EAC1B,KAAK,EAAE,OAAqC;AAEhD,sCAAyB;EACrB,KAAK,EAAE,OAAoC;AAE/C,0CAA6B;EACzB,KAAK,EAAE,OAAqC;AAEhD,+BAAkB;EACd,KAAK,EApDM,OAAO;AAsDtB,iCAAoB;EAChB,KAAK,EAAE,OAAiC;AAE5C,uCAA0B;EACtB,KAAK,EAAE,OAAkC;AAE7C,kCAAqB;EACjB,KAAK,EAAE,OAAkC;AAE7C,8BAAiB;EACb,KAAK,EAAE,OAAiC;AAE5C,sCAAyB;EACrB,KAAK,EAAE,OAAkC;AAE7C,iCAAoB;EAChB,KAAK,EAAE,OAAqC;AAEhD,0CAA6B;EACzB,KAAK,EAAE,OAAoC;AAE/C,oCAAuB;EACnB,KAAK,EA3EU,OAAO;AA6E1B,iCAAoB;EAChB,KAAK,EAAE,OAAqC;AAEhD,sCAAyB;EACrB,KAAK,EAAE,OAAkC;AAE7C,kCAAqB;EACjB,KAAK,EAAE,OAAkC;;AAIjD,YAAa;EACX,KAAK,EAAE,KAAK;EACZ,OAAO,EAAE,MAAM;;AAGjB,eAAgB;EACd,WAAW,EAAE,CAAC;;AAGhB,WAAY;EACR,KAAK,EAAE,KAAK", -"sources": ["activity.scss"], -"names": [], -"file": "activity.css" -} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.scss b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.scss deleted file mode 100644 index 1cf52dc..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/activity.scss +++ /dev/null @@ -1,154 +0,0 @@ -.activity { - padding: 0; - list-style-type: none; - background: transparent url('/dotted.png') 21px 0 repeat-y; - .item { - position: relative; - margin: 0 0 15px 0; - padding: 0; - .user-image { - border-radius: 100px; - } - .date { - color: #999; - font-size: 12px; - white-space: nowrap; - margin: 5px 0 0 80px; - } - &.no-avatar p { - // Use by datapusher - margin-left: 40px; - } - } -} - -.activity_buttons > a.btn.disabled { - color: inherit; - opacity: 0.70 !important; -} - -// For profile information that appears in the popover -.popover { - width: 300px; - .popover-title { - font-weight: bold; - margin-bottom: 0; - } - p.about { - margin: 0 0 10px 0; - } - .popover-close { - float: right; - text-decoration: none; - } - .empty { - padding: 10px; - color: #6e6e6e; - font-style: italic; - } -} - -// Activity Stream base colors -$activityColorText: #FFFFFF; -$activityColorBlank: #999999; -$activityColorNew: #69A67A; -$activityColorNeutral: #767DCE; -$activityColorModify: #767DCE; -$activityColorDelete: #B95252; - -.activity .item { - .icon { - color: $activityColorBlank ; - } // Non defined - &.failure .icon { - color: $activityColorDelete; - } - &.success .icon { - color: $activityColorNew; - } - &.added-tag .icon { - color: adjust-hue($activityColorNew, 60); - } - &.changed-group .icon { - color: $activityColorModify; - } - &.changed-package .icon { - color: adjust-hue($activityColorModify, 20); - } - &.changed-package_extra .icon { - color: adjust-hue($activityColorModify, -20); - } - &.changed-resource .icon { - color: adjust-hue($activityColorModify, 40); - } - &.changed-user .icon { - color: adjust-hue($activityColorModify, -40); - } - &.changed-organization .icon { - color: adjust-hue($activityColorNew, 50); - } - &.deleted-group .icon { - color: $activityColorDelete; - } - &.deleted-package .icon { - color: adjust-hue($activityColorDelete, 20); - } - &.deleted-package_extra .icon { - color: adjust-hue($activityColorDelete, -20); - } - &.deleted-resource .icon { - color: adjust-hue($activityColorDelete, 40); - } - &.deleted-organization .icon { - color: adjust-hue($activityColorDelete, -40); - } - &.new-group .icon { - color: $activityColorNew; - } - &.new-package .icon { - color: adjust-hue($activityColorNew, 20); - } - &.new-package_extra .icon { - color: adjust-hue($activityColorNew, -20); - } - &.new-resource .icon { - color: adjust-hue($activityColorNew, -40); - } - &.new-user .icon { - color: adjust-hue($activityColorNew, 40); - } - &.new-organization .icon { - color: adjust-hue($activityColorNew, -40); - } - &.removed-tag .icon { - color: adjust-hue($activityColorDelete, -40); - } - &.deleted-related-item .icon { - color: adjust-hue($activityColorDelete, 60); - } - &.follow-dataset .icon { - color: $activityColorNeutral; - } - &.follow-user .icon { - color: adjust-hue($activityColorNeutral, 20); - } - &.new-related-item .icon { - color: adjust-hue($activityColorNew, -60); - } - &.follow-group .icon { - color: adjust-hue($activityColorNew, -50); - } -} - -.select-time { - width: 250px; - display: inline; -} - -br.line-height2 { - line-height: 2; -} - -.pull-right { - float: right; -} \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.js deleted file mode 100644 index 69b97c7..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.js +++ /dev/null @@ -1,95 +0,0 @@ -/* User Dashboard - * Handles the filter dropdown menu and the reduction of the notifications number - * within the header to zero - * - * Examples - * - *
- * - */ -this.ckan.module('dashboard', function ($) { - return { - button: null, - popover: null, - searchTimeout: null, - - /* Initialises the module setting up elements and event listeners. - * - * Returns nothing. - */ - initialize: function () { - $.proxyAll(this, /_on/); - this.button = $('#followee-filter .btn'). - on('click', this._onShowFolloweeDropdown); - var title = this.button.prop('title'); - - this.button.popover = new bootstrap.Popover(document.querySelector('#followee-filter .btn'), { - placement: 'bottom', - html: true, - template: '', - customClass: 'popover-followee', - sanitizeFn: function (content) { - return DOMPurify.sanitize(content, { ALLOWED_TAGS: [ - "form", "div", "input", "footer", "header", "h1", "h2", "h3", "h4", - "small", "span", "strong", "i", 'a', 'li', 'ul','p' - - ]}); - }, - content: $('#followee-content').html() - }); - this.button.prop('title', title); - }, - - /* Handles click event on the 'show me:' dropdown button - * - * Returns nothing. - */ - _onShowFolloweeDropdown: function() { - this.button.toggleClass('active'); - if (this.button.hasClass('active')) { - setTimeout(this._onInitSearch, 100); - } - return false; - }, - - /* Handles focusing on the input and making sure that the keyup - * even is applied to the input - * - * Returns nothing. - */ - _onInitSearch: function() { - var input = $('input', this.popover); - if (!input.hasClass('inited')) { - input. - on('keyup', this._onSearchKeyUp). - addClass('inited'); - } - input.focus(); - }, - - /* Handles the keyup event - * - * Returns nothing. - */ - _onSearchKeyUp: function() { - clearTimeout(this.searchTimeout); - this.searchTimeout = setTimeout(this._onSearchKeyUpTimeout, 300); - }, - - /* Handles the actual filtering of search results - * - * Returns nothing. - */ - _onSearchKeyUpTimeout: function() { - this.popover = this.button.popover.tip; - var input = $('input', this.popover); - var q = input.val().toLowerCase(); - if (q) { - $('li', this.popover).hide(); - $('li.everything, [data-search^="' + q + '"]', this.popover).show(); - } else { - $('li', this.popover).show(); - } - } - }; -}); diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.spec.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.spec.js deleted file mode 100644 index 95ef2f2..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/dashboard.spec.js +++ /dev/null @@ -1,64 +0,0 @@ -describe('ckan.modules.DashboardModule()', function () { - before(() => { - cy.visit('/'); - cy.window().then(win => { - cy.wrap(win.ckan.module.registry['dashboard']).as('dashboard'); - win.jQuery('
').appendTo(win.document.body) - cy.loadFixture('dashboard.html').then((template) => { - cy.wrap(template).as('template'); - }); - }) - }); - - beforeEach(function () { - cy.window().then(win => { - win.jQuery('#fixture').html(this.template); - this.el = document.createElement('button'); - this.sandbox = win.ckan.sandbox(); - this.sandbox.body = win.jQuery('#fixture'); - cy.wrap(this.sandbox.body).as('fixture'); - this.module = new this.dashboard(this.el, {}, this.sandbox); - }) - }); - - afterEach(function () { - //this.fixture.empty(); - }); - - describe('.initialize()', function () { - it('should bind callback methods to the module', function () { - cy.window().then(win => { - let target = cy.stub(win.jQuery, 'proxyAll'); - - this.module.initialize(); - - expect(target).to.be.called; - expect(target).to.be.calledWith(this.module, /_on/); - - target.restore(); - }) - }) - }) - - describe('.show()', function () { - it('should append the popover to the document body', function () { - this.module.initialize(); - this.module.button.click(); - assert.equal(this.fixture.children().length, 1); - assert.equal(this.fixture.find('#followee-filter').length, 1); - assert.equal(this.fixture.find('#followee-filter .input-group input').length, 1); - }); - }) - - describe(".search", function(){ - it('should filter based on query', function() { - this.module.initialize(); - this.module.button.click(); - cy.get('#fixture #followee-filter #followee-content').invoke('removeAttr', 'style'); - cy.get('#fixture #followee-filter .nav li').should('have.length', 3); - cy.get('#fixture #followee-filter .input-group input.inited').type('text'); - cy.get('#fixture #followee-filter .nav li[data-search="not valid"]').should('be.visible'); - cy.get('#fixture #followee-filter .nav li[data-search="test followee"]').should('be.visible'); - }) - }) -}) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/webassets.yml b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/webassets.yml deleted file mode 100644 index e762585..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/assets/webassets.yml +++ /dev/null @@ -1,14 +0,0 @@ -activity: - filters: rjsmin - output: activity/%(version)s_activity.js - extra: - preload: - - base/main - contents: - - dashboard.js - - activity-stream.js - -activity-css: - output: ckanext-activity/%(version)s_activity.css - contents: - - activity.css diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/changes.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/changes.py deleted file mode 100644 index 81f090f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/changes.py +++ /dev/null @@ -1,1255 +0,0 @@ -# encoding: utf-8 -""" -Functions for generating a list of differences between two versions of a -dataset -""" -from __future__ import annotations - -import logging -from typing import Any -from typing_extensions import TypeAlias, TypedDict - -log = logging.getLogger(__name__) - -Data: TypeAlias = "dict[str, Any]" -ChangeList: TypeAlias = "list[Data]" - - -class Extra(TypedDict): - key: str - value: Any - - -def _extras_to_dict(extras_list: list[Extra]) -> Data: - """ - Takes a list of dictionaries with the following format: - [ - { - "key": , - "value": - }, - ..., - { - "key": , - "value": - } - ] - and converts it into a single dictionary with the following - format: - { - key_0: value_0, - ..., - key_n: value_n - - } - """ - ret_dict = {} - # the extras_list is a list of dictionaries - for dict in extras_list: - ret_dict[dict["key"]] = dict["value"] - - return ret_dict - - -def check_resource_changes( - change_list: ChangeList, old: Data, new: Data, old_activity_id: str -) -> None: - """ - Compares two versions of a dataset and records the changes between them - (just the resources) in change_list. e.g. resources that are added, changed - or deleted. For existing resources, checks whether their names, formats, - and/or descriptions have changed, as well as whether the url changed (e.g. - a new file has been uploaded for the resource). - """ - - # list of default fields in a resource's metadata dictionary - used - # later to ensure that we don't count changes to default fields as changes - # to extra fields - fields = [ - "package_id", - "url", - "revision_id", - "description", - "format", - "hash", - "name", - "resource_type", - "mimetype", - "mimetype_inner", - "cache_url", - "size", - "created", - "last_modified", - "metadata_modified", - "cache_last_updated", - "upload", - "position", - ] - default_fields_set = set(fields) - - # make a set of the resource IDs present in old and new - old_resource_set = set() - old_resource_dict = {} - new_resource_set = set() - new_resource_dict = {} - - for resource in old.get("resources", []): - old_resource_set.add(resource["id"]) - old_resource_dict[resource["id"]] = { - key: value for (key, value) in resource.items() if key != "id" - } - - for resource in new.get("resources", []): - new_resource_set.add(resource["id"]) - new_resource_dict[resource["id"]] = { - key: value for (key, value) in resource.items() if key != "id" - } - - # get the IDs of the resources that have been added between the versions - new_resources = list(new_resource_set - old_resource_set) - for resource_id in new_resources: - change_list.append( - { - "type": "new_resource", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_name": new_resource_dict[resource_id].get("name"), - "resource_id": resource_id, - } - ) - - # get the IDs of resources that have been deleted between versions - deleted_resources = list(old_resource_set - new_resource_set) - for resource_id in deleted_resources: - change_list.append( - { - "type": "delete_resource", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": old_resource_dict[resource_id].get("name"), - "old_activity_id": old_activity_id, - } - ) - - # now check the resources that are in both and see if any - # have been changed - resources = new_resource_set.intersection(old_resource_set) - for resource_id in resources: - old_metadata = old_resource_dict[resource_id] - new_metadata = new_resource_dict[resource_id] - - if old_metadata.get("name") != new_metadata.get("name"): - change_list.append( - { - "type": "resource_name", - "title": new.get("title"), - "old_pkg_id": old["id"], - "new_pkg_id": new["id"], - "resource_id": resource_id, - "old_resource_name": old_resource_dict[resource_id].get( - "name" - ), - "new_resource_name": new_resource_dict[resource_id].get( - "name" - ), - "old_activity_id": old_activity_id, - } - ) - - # you can't remove a format, but if a resource's format isn't - # recognized, it won't have one set - - # if a format was not originally set and the user set one - if not old_metadata.get("format") and new_metadata.get("format"): - change_list.append( - { - "type": "resource_format", - "method": "add", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_resource_dict[resource_id].get( - "name" - ), - "org_id": new["organization"]["id"] - if new.get("organization") - else "", - "format": new_metadata.get("format"), - } - ) - - # if both versions have a format but the format changed - elif old_metadata.get("format") != new_metadata.get("format"): - change_list.append( - { - "type": "resource_format", - "method": "change", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_resource_dict[resource_id].get( - "name" - ), - "org_id": new["organization"]["id"] - if new.get("organization") - else "", - "old_format": old_metadata.get("format"), - "new_format": new_metadata.get("format"), - } - ) - - # if the description changed - if not old_metadata.get("description") and new_metadata.get( - "description" - ): - change_list.append( - { - "type": "resource_desc", - "method": "add", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_resource_dict[resource_id].get( - "name" - ), - "new_desc": new_metadata.get("description"), - } - ) - - # if there was a description but the user removed it - elif old_metadata.get("description") and not new_metadata.get( - "description" - ): - change_list.append( - { - "type": "resource_desc", - "method": "remove", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_resource_dict[resource_id].get( - "name" - ), - } - ) - - # if both have descriptions but they are different - elif old_metadata.get("description") != new_metadata.get( - "description" - ): - change_list.append( - { - "type": "resource_desc", - "method": "change", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_resource_dict[resource_id].get( - "name" - ), - "new_desc": new_metadata.get("description"), - "old_desc": old_metadata.get("description"), - } - ) - - # check if the url changes (e.g. user uploaded a new file) - # TODO: use regular expressions to determine the actual name of the - # new and old files - if old_metadata.get("url") != new_metadata.get("url"): - change_list.append( - { - "type": "new_file", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - } - ) - - # check any extra fields in the resource - # remove default fields from these sets to make sure we only check - # for changes to extra fields - old_fields_set = set(old_metadata.keys()) - old_fields_set = old_fields_set - default_fields_set - new_fields_set = set(new_metadata.keys()) - new_fields_set = new_fields_set - default_fields_set - - # determine if any new extra fields have been added - new_fields = list(new_fields_set - old_fields_set) - if len(new_fields) == 1: - if new_metadata[new_fields[0]]: - change_list.append( - { - "type": "resource_extras", - "method": "add_one_value", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key": new_fields[0], - "value": new_metadata[new_fields[0]], - } - ) - else: - change_list.append( - { - "type": "resource_extras", - "method": "add_one_no_value", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key": new_fields[0], - } - ) - elif len(new_fields) > 1: - change_list.append( - { - "type": "resource_extras", - "method": "add_multiple", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key_list": new_fields, - "value_list": [ - new_metadata[field] for field in new_fields - ], - } - ) - - # determine if any extra fields have been removed - deleted_fields = list(old_fields_set - new_fields_set) - if len(deleted_fields) == 1: - change_list.append( - { - "type": "resource_extras", - "method": "remove_one", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key": deleted_fields[0], - } - ) - elif len(deleted_fields) > 1: - change_list.append( - { - "type": "resource_extras", - "method": "remove_multiple", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key_list": deleted_fields, - } - ) - - # determine if any extra fields have been changed - # changed_fields is only a set of POTENTIALLY changed fields - we - # still have to check if any of the values associated with the fields - # have actually changed - changed_fields = list(new_fields_set.intersection(old_fields_set)) - for field in changed_fields: - if new_metadata[field] != old_metadata[field]: - if new_metadata[field] and old_metadata[field]: - change_list.append( - { - "type": "resource_extras", - "method": "change_value_with_old", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key": field, - "old_value": old_metadata[field], - "new_value": new_metadata[field], - } - ) - elif not old_metadata[field]: - change_list.append( - { - "type": "resource_extras", - "method": "change_value_no_old", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key": field, - "new_value": new_metadata[field], - } - ) - elif not new_metadata[field]: - change_list.append( - { - "type": "resource_extras", - "method": "change_value_no_new", - "pkg_id": new["id"], - "title": new.get("title"), - "resource_id": resource_id, - "resource_name": new_metadata.get("name"), - "key": field, - } - ) - - -def check_metadata_changes( - change_list: ChangeList, old: Data, new: Data -) -> None: - """ - Compares two versions of a dataset and records the changes between them - (excluding resources) in change_list. - """ - # if the title has changed - if old.get("title") != new.get("title"): - _title_change(change_list, old, new) - - # if the owner organization changed - if old.get("owner_org") != new.get("owner_org"): - _org_change(change_list, old, new) - - # if the maintainer of the dataset changed - if old.get("maintainer") != new.get("maintainer"): - _maintainer_change(change_list, old, new) - - # if the maintainer email of the dataset changed - if old.get("maintainer_email") != new.get("maintainer_email"): - _maintainer_email_change(change_list, old, new) - - # if the author of the dataset changed - if old.get("author") != new.get("author"): - _author_change(change_list, old, new) - - # if the author email of the dataset changed - if old.get("author_email") != new.get("author_email"): - _author_email_change(change_list, old, new) - - # if the visibility of the dataset changed - if old.get("private") != new.get("private"): - change_list.append( - { - "type": "private", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new": "Private" if bool(new.get("private")) else "Public", - } - ) - - # if the description of the dataset changed - if old.get("notes") != new.get("notes"): - _notes_change(change_list, old, new) - - # make sets out of the tags for each dataset - old_tags = {tag.get("name") for tag in old.get("tags", [])} - new_tags = {tag.get("name") for tag in new.get("tags", [])} - # if the tags have changed - if old_tags != new_tags: - _tag_change(change_list, new_tags, old_tags, new) - - # if the license has changed - if old.get("license_title") != new.get("license_title"): - _license_change(change_list, old, new) - - # if the name of the dataset has changed - # this is only visible to the user via the dataset's URL, - # so display the change using that - if old.get("name") != new.get("name"): - _name_change(change_list, old, new) - - # if the source URL (metadata value, not the actual URL of the dataset) - # has changed - if old.get("url") != new.get("url"): - _url_change(change_list, old, new) - - # if the user-provided version has changed - if old.get("version") != new.get("version"): - _version_change(change_list, old, new) - - # check whether fields added by extensions or custom fields - # (in the "extras" field) have been changed - - _extension_fields(change_list, old, new) - _extra_fields(change_list, old, new) - - -def check_metadata_org_changes(change_list: ChangeList, old: Data, new: Data): - """ - Compares two versions of a organization and records the changes between - them in change_list. - """ - # if the title has changed - if old.get("title") != new.get("title"): - _title_change(change_list, old, new) - - # if the description of the organization changed - if old.get("description") != new.get("description"): - _description_change(change_list, old, new) - - # if the image URL has changed - if old.get("image_url") != new.get("image_url"): - _image_url_change(change_list, old, new) - - -def _title_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's title between two versions - (old and new) to change_list. - """ - change_list.append( - { - "type": "title", - "id": new.get("name"), - "new_title": new.get("title"), - "old_title": old.get("title"), - } - ) - - -def _org_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's organization between - two versions (old and new) to change_list. - """ - - # if both versions belong to an organization - if old.get("owner_org") and new.get("owner_org"): - change_list.append( - { - "type": "org", - "method": "change", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_org_id": old["organization"].get("id"), - "old_org_title": old["organization"].get("title"), - "new_org_id": new["organization"].get("id"), - "new_org_title": new["organization"].get("title"), - } - ) - # if the dataset was not in an organization before and it is now - elif not old.get("owner_org") and new.get("owner_org"): - change_list.append( - { - "type": "org", - "method": "add", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_org_id": new["organization"].get("id"), - "new_org_title": new["organization"].get("title"), - } - ) - # if the user removed the organization - else: - change_list.append( - { - "type": "org", - "method": "remove", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_org_id": old["organization"].get("id"), - "old_org_title": old["organization"].get("title"), - } - ) - - -def _maintainer_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's maintainer field between two - versions (old and new) to change_list. - """ - # if the old dataset had a maintainer - if old.get("maintainer") and new.get("maintainer"): - change_list.append( - { - "type": "maintainer", - "method": "change", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_maintainer": new["maintainer"], - "old_maintainer": old["maintainer"], - } - ) - # if they removed the maintainer - elif not new.get("maintainer"): - change_list.append( - { - "type": "maintainer", - "pkg_id": new.get("id"), - "title": new.get("title"), - "method": "remove", - } - ) - # if there wasn't one there before - else: - change_list.append( - { - "type": "maintainer", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_maintainer": new.get("maintainer"), - "method": "add", - } - ) - - -def _maintainer_email_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's maintainer e-mail address - field between two versions (old and new) to change_list. - """ - # if the old dataset had a maintainer email - if old.get("maintainer_email") and new.get("maintainer_email"): - change_list.append( - { - "type": "maintainer_email", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_maintainer_email": new.get("maintainer_email"), - "old_maintainer_email": old.get("maintainer_email"), - "method": "change", - } - ) - # if they removed the maintainer email - elif not new.get("maintainer_email"): - change_list.append( - { - "type": "maintainer_email", - "pkg_id": new.get("id"), - "title": new.get("title"), - "method": "remove", - } - ) - # if there wasn't one there before e - else: - change_list.append( - { - "type": "maintainer_email", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_maintainer_email": new.get("maintainer_email"), - "method": "add", - } - ) - - -def _author_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's author field between two - versions (old and new) to change_list. - """ - # if the old dataset had an author - if old.get("author") and new.get("author"): - change_list.append( - { - "type": "author", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_author": new.get("author"), - "old_author": old.get("author"), - "method": "change", - } - ) - # if they removed the author - elif not new.get("author"): - change_list.append( - { - "type": "author", - "pkg_id": new.get("id"), - "title": new.get("title"), - "method": "remove", - } - ) - # if there wasn't one there before - else: - change_list.append( - { - "type": "author", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_author": new.get("author"), - "method": "add", - } - ) - - -def _author_email_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's author e-mail address field - between two versions (old and new) to change_list. - """ - if old.get("author_email") and new.get("author_email"): - change_list.append( - { - "type": "author_email", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_author_email": new.get("author_email"), - "old_author_email": old.get("author_email"), - "method": "change", - } - ) - # if they removed the author - elif not new.get("author_email"): - change_list.append( - { - "type": "author_email", - "pkg_id": new.get("id"), - "title": new.get("title"), - "method": "remove", - } - ) - # if there wasn't one there before - else: - change_list.append( - { - "type": "author_email", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_author_email": new.get("author_email"), - "method": "add", - } - ) - - -def _notes_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's description between two - versions (old and new) to change_list. - """ - # if the old dataset had a description - if old.get("notes") and new.get("notes"): - change_list.append( - { - "type": "notes", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_notes": new.get("notes"), - "old_notes": old.get("notes"), - "method": "change", - } - ) - elif not new.get("notes"): - change_list.append( - { - "type": "notes", - "pkg_id": new.get("id"), - "title": new.get("title"), - "method": "remove", - } - ) - else: - change_list.append( - { - "type": "notes", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_notes": new.get("notes"), - "method": "add", - } - ) - - -def _tag_change( - change_list: ChangeList, new_tags: set[Any], old_tags: set[Any], new: Data -): - """ - Appends a summary of a change to a dataset's tag list between two - versions (old and new) to change_list. - """ - deleted_tags = old_tags - new_tags - deleted_tags_list = list(deleted_tags) - if len(deleted_tags) == 1: - change_list.append( - { - "type": "tags", - "method": "remove_one", - "pkg_id": new.get("id"), - "title": new.get("title"), - "tag": deleted_tags_list[0], - } - ) - elif len(deleted_tags) > 1: - change_list.append( - { - "type": "tags", - "method": "remove_multiple", - "pkg_id": new.get("id"), - "title": new.get("title"), - "tags": deleted_tags_list, - } - ) - - added_tags = new_tags - old_tags - added_tags_list = list(added_tags) - if len(added_tags) == 1: - change_list.append( - { - "type": "tags", - "method": "add_one", - "pkg_id": new.get("id"), - "title": new.get("title"), - "tag": added_tags_list[0], - } - ) - elif len(added_tags) > 1: - change_list.append( - { - "type": "tags", - "method": "add_multiple", - "pkg_id": new.get("id"), - "title": new.get("title"), - "tags": added_tags_list, - } - ) - - -def _license_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's license between two versions - (old and new) to change_list. - """ - old_license_url = "" - new_license_url = "" - # if the license has a URL - if "license_url" in old and old["license_url"]: - old_license_url = old["license_url"] - if "license_url" in new and new["license_url"]: - new_license_url = new["license_url"] - change_list.append( - { - "type": "license", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_url": old_license_url, - "new_url": new_license_url, - "new_title": new.get("license_title"), - "old_title": old.get("license_title"), - } - ) - - -def _name_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's name (and thus the URL it - can be accessed at) between two versions (old and new) to - change_list. - """ - change_list.append( - { - "type": "name", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_name": old.get("name"), - "new_name": new.get("name"), - } - ) - - -def _url_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's source URL (metadata field, - not its actual URL in the datahub) between two versions (old and - new) to change_list. - """ - # if both old and new versions have source URLs - if old.get("url") and new.get("url"): - change_list.append( - { - "type": "url", - "method": "change", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_url": new.get("url"), - "old_url": old.get("url"), - } - ) - # if the user removed the source URL - elif not new.get("url"): - change_list.append( - { - "type": "url", - "method": "remove", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_url": old.get("url"), - } - ) - # if there wasn't one there before - else: - change_list.append( - { - "type": "url", - "method": "add", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_url": new.get("url"), - } - ) - - -def _version_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a dataset's version field (inputted - by the user, not from version control) between two versions (old - and new) to change_list. - """ - # if both old and new versions have version numbers - if old.get("version") and new.get("version"): - change_list.append( - { - "type": "version", - "method": "change", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_version": old.get("version"), - "new_version": new.get("version"), - } - ) - # if the user removed the version number - elif not new.get("version"): - change_list.append( - { - "type": "version", - "method": "remove", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_version": old.get("version"), - } - ) - # if there wasn't one there before - else: - change_list.append( - { - "type": "version", - "method": "add", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_version": new.get("version"), - } - ) - - -def _extension_fields(change_list: ChangeList, old: Data, new: Data): - """ - Checks whether any fields that have been added to the package - dictionaries by CKAN extensions have been changed between versions. - If there have been any changes between the two versions (old and - new), a general summary of the change is appended to change_list. This - function does not produce summaries for fields added or deleted by - extensions, since these changes are not triggered by the user in the web - interface or API. - """ - # list of the default metadata fields for a dataset - # any fields that are not part of this list are custom fields added by a - # user or extension - fields = [ - "owner_org", - "maintainer", - "maintainer_email", - "relationships_as_object", - "private", - "num_tags", - "id", - "metadata_created", - "metadata_modified", - "author", - "author_email", - "state", - "version", - "license_id", - "type", - "resources", - "num_resources", - "tags", - "title", - "groups", - "creator_user_id", - "relationships_as_subject", - "name", - "isopen", - "url", - "notes", - "license_title", - "extras", - "license_url", - "organization", - "revision_id", - ] - fields_set = set(fields) - - # if there are any fields from extensions that are in the new dataset and - # have been updated, print a generic message stating that - old_set = set(old.keys()) - new_set = set(new.keys()) - - # set of additional fields in the new dictionary - addl_fields_new = new_set - fields_set - # set of additional fields in the old dictionary - addl_fields_old = old_set - fields_set - # set of additional fields in both - addl_fields = addl_fields_new.intersection(addl_fields_old) - - # do NOT display a change if any additional fields have been - # added or deleted, since that is not a change made by the user - # from the web interface - - # if additional fields have been changed - addl_fields_list = list(addl_fields) - for field in addl_fields_list: - if old.get(field) != new.get(field): - change_list.append( - { - "type": "extension_fields", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": field, - "value": new.get(field), - } - ) - - -def _extra_fields(change_list: ChangeList, old: Data, new: Data): - """ - Checks whether a user has added, removed, or changed any extra fields - from the web interface (or API?) and appends a summary of each change to - change_list. - """ - if "extras" in new: - extra_fields_new = _extras_to_dict(new.get("extras", [])) - extra_new_set = set(extra_fields_new.keys()) - - # if the old version has extra fields, we need - # to compare the new version's extras to the old ones - if "extras" in old: - extra_fields_old = _extras_to_dict(old.get("extras", [])) - extra_old_set = set(extra_fields_old.keys()) - - # if some fields were added - new_fields = list(extra_new_set - extra_old_set) - if len(new_fields) == 1: - if extra_fields_new[new_fields[0]]: - change_list.append( - { - "type": "extra_fields", - "method": "add_one_value", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": new_fields[0], - "value": extra_fields_new[new_fields[0]], - } - ) - else: - change_list.append( - { - "type": "extra_fields", - "method": "add_one_no_value", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": new_fields[0], - } - ) - elif len(new_fields) > 1: - change_list.append( - { - "type": "extra_fields", - "method": "add_multiple", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key_list": new_fields, - "value_list": extra_fields_new, - } - ) - - # if some fields were deleted - deleted_fields = list(extra_old_set - extra_new_set) - if len(deleted_fields) == 1: - change_list.append( - { - "type": "extra_fields", - "method": "remove_one", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": deleted_fields[0], - } - ) - elif len(deleted_fields) > 1: - change_list.append( - { - "type": "extra_fields", - "method": "remove_multiple", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key_list": deleted_fields, - } - ) - - # if some existing fields were changed - # list of extra fields in both the old and new versions - extra_fields = list(extra_new_set.intersection(extra_old_set)) - for field in extra_fields: - if extra_fields_old[field] != extra_fields_new[field]: - if extra_fields_old[field]: - change_list.append( - { - "type": "extra_fields", - "method": "change_with_old_value", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": field, - "old_value": extra_fields_old[field], - "new_value": extra_fields_new[field], - } - ) - else: - change_list.append( - { - "type": "extra_fields", - "method": "change_no_old_value", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": field, - "new_value": extra_fields_new[field], - } - ) - - # if the old version didn't have an extras field, - # the user could only have added a field (not changed or deleted) - else: - new_fields = list(extra_new_set) - if len(new_fields) == 1: - if extra_fields_new[new_fields[0]]: - change_list.append( - { - "type": "extra_fields", - "method": "add_one_value", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": new_fields[0], - "value": extra_fields_new[new_fields[0]], - } - ) - else: - change_list.append( - { - "type": "extra_fields", - "method": "add_one_no_value", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": new_fields[0], - } - ) - - elif len(new_fields) > 1: - change_list.append( - { - "type": "extra_fields", - "method": "add_multiple", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key_list": new_fields, - "value_list": extra_fields_new, - } - ) - - elif "extras" in old: - deleted_fields = list(_extras_to_dict(old["extras"]).keys()) - if len(deleted_fields) == 1: - change_list.append( - { - "type": "extra_fields", - "method": "remove_one", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key": deleted_fields[0], - } - ) - elif len(deleted_fields) > 1: - change_list.append( - { - "type": "extra_fields", - "method": "remove_multiple", - "pkg_id": new.get("id"), - "title": new.get("title"), - "key_list": deleted_fields, - } - ) - - -def _description_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a organization's description between two - versions (old and new) to change_list. - """ - - # if the old organization had a description - if old.get("description") and new.get("description"): - change_list.append( - { - "type": "description", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_description": new.get("description"), - "old_description": old.get("description"), - "method": "change", - } - ) - elif not new.get("description"): - change_list.append( - { - "type": "description", - "pkg_id": new.get("id"), - "title": new.get("title"), - "method": "remove", - } - ) - else: - change_list.append( - { - "type": "description", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_description": new.get("description"), - "method": "add", - } - ) - - -def _image_url_change(change_list: ChangeList, old: Data, new: Data): - """ - Appends a summary of a change to a organization's image URL between two - versions (old and new) to change_list. - """ - # if both old and new versions have image URLs - if old.get("image_url") and new.get("image_url"): - change_list.append( - { - "type": "image_url", - "method": "change", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_image_url": new.get("image_url"), - "old_image_url": old.get("image_url"), - } - ) - # if the user removed the image URL - elif not new.get("image_url"): - change_list.append( - { - "type": "image_url", - "method": "remove", - "pkg_id": new.get("id"), - "title": new.get("title"), - "old_image_url": old.get("image_url"), - } - ) - # if there wasn't one there before - else: - change_list.append( - { - "type": "image_url", - "method": "add", - "pkg_id": new.get("id"), - "title": new.get("title"), - "new_image_url": new.get("image_url"), - } - ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/email_notifications.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/email_notifications.py deleted file mode 100644 index d7c49d2..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/email_notifications.py +++ /dev/null @@ -1,278 +0,0 @@ -# encoding: utf-8 - -""" -Code for generating email notifications for users (e.g. email notifications for -new activities in your dashboard activity stream) and emailing them to the -users. - -""" -from __future__ import annotations - -import datetime -import re -from typing import Any, cast -from jinja2 import Environment - -import ckan.model as model -import ckan.logic as logic -import ckan.lib.jinja_extensions as jinja_extensions - -from ckan.common import ungettext, ugettext, config -from ckan.types import Context - - -def string_to_timedelta(s: str) -> datetime.timedelta: - """Parse a string s and return a standard datetime.timedelta object. - - Handles days, hours, minutes, seconds, and microseconds. - - Accepts strings in these formats: - - 2 days - 14 days - 4:35:00 (hours, minutes and seconds) - 4:35:12.087465 (hours, minutes, seconds and microseconds) - 7 days, 3:23:34 - 7 days, 3:23:34.087465 - .087465 (microseconds only) - - :raises ckan.logic.ValidationError: if the given string does not match any - of the recognised formats - - """ - patterns = [] - days_only_pattern = r"(?P\d+)\s+day(s)?" - patterns.append(days_only_pattern) - hms_only_pattern = r"(?P\d?\d):(?P\d\d):(?P\d\d)" - patterns.append(hms_only_pattern) - ms_only_pattern = r".(?P\d\d\d)(?P\d\d\d)" - patterns.append(ms_only_pattern) - hms_and_ms_pattern = hms_only_pattern + ms_only_pattern - patterns.append(hms_and_ms_pattern) - days_and_hms_pattern = r"{0},\s+{1}".format( - days_only_pattern, hms_only_pattern - ) - patterns.append(days_and_hms_pattern) - days_and_hms_and_ms_pattern = days_and_hms_pattern + ms_only_pattern - patterns.append(days_and_hms_and_ms_pattern) - - match = None - for pattern in patterns: - match = re.match("^{0}$".format(pattern), s) - if match: - break - - if not match: - raise logic.ValidationError( - {"message": "Not a valid time: {0}".format(s)} - ) - - gd = match.groupdict() - days = int(gd.get("days", "0")) - hours = int(gd.get("hours", "0")) - minutes = int(gd.get("minutes", "0")) - seconds = int(gd.get("seconds", "0")) - milliseconds = int(gd.get("milliseconds", "0")) - microseconds = int(gd.get("microseconds", "0")) - delta = datetime.timedelta( - days=days, - hours=hours, - minutes=minutes, - seconds=seconds, - milliseconds=milliseconds, - microseconds=microseconds, - ) - return delta - - -def render_activity_email(activities: list[dict[str, Any]]) -> str: - globals = {"site_title": config.get("ckan.site_title")} - template_name = "activity_streams/activity_stream_email_notifications.text" - - env = Environment(**jinja_extensions.get_jinja_env_options()) - # Install the given gettext, ngettext callables into the environment - env.install_gettext_callables(ugettext, ungettext) # type: ignore - - template = env.get_template(template_name, globals=globals) - return template.render({"activities": activities}) - - -def _notifications_for_activities( - activities: list[dict[str, Any]], user_dict: dict[str, Any] -) -> list[dict[str, str]]: - """Return one or more email notifications covering the given activities. - - This function handles grouping multiple activities into a single digest - email. - - :param activities: the activities to consider - :type activities: list of activity dicts like those returned by - ckan.logic.action.get.dashboard_activity_list() - - :returns: a list of email notifications - :rtype: list of dicts each with keys 'subject' and 'body' - - """ - if not activities: - return [] - - if not user_dict.get("activity_streams_email_notifications"): - return [] - - # We just group all activities into a single "new activity" email that - # doesn't say anything about _what_ new activities they are. - # TODO: Here we could generate some smarter content for the emails e.g. - # say something about the contents of the activities, or single out - # certain types of activity to be sent in their own individual emails, - # etc. - - subject = ungettext( - "{n} new activity from {site_title}", - "{n} new activities from {site_title}", - len(activities), - ).format(site_title=config.get("ckan.site_title"), n=len(activities)) - - body = render_activity_email(activities) - notifications = [{"subject": subject, "body": body}] - - return notifications - - -def _notifications_from_dashboard_activity_list( - user_dict: dict[str, Any], since: datetime.datetime -) -> list[dict[str, str]]: - """Return any email notifications from the given user's dashboard activity - list since `since`. - - """ - # Get the user's dashboard activity stream. - context = cast( - Context, - {"model": model, "session": model.Session, "user": user_dict["id"]}, - ) - activity_list = logic.get_action("dashboard_activity_list")(context, {}) - - # Filter out the user's own activities., so they don't get an email every - # time they themselves do something (we are not Trac). - activity_list = [ - activity - for activity in activity_list - if activity["user_id"] != user_dict["id"] - ] - - # Filter out the old activities. - strptime = datetime.datetime.strptime - fmt = "%Y-%m-%dT%H:%M:%S.%f" - activity_list = [ - activity - for activity in activity_list - if strptime(activity["timestamp"], fmt) > since - ] - - return _notifications_for_activities(activity_list, user_dict) - - -# A list of functions that provide email notifications for users from different -# sources. Add to this list if you want to implement a new source of email -# notifications. -_notifications_functions = [ - _notifications_from_dashboard_activity_list, -] - - -def get_notifications( - user_dict: dict[str, Any], since: datetime.datetime -) -> list[dict[str, Any]]: - """Return any email notifications for the given user since `since`. - - For example email notifications about activity streams will be returned for - any activities the occurred since `since`. - - :param user_dict: a dictionary representing the user, should contain 'id' - and 'name' - :type user_dict: dictionary - - :param since: datetime after which to return notifications from - :rtype since: datetime.datetime - - :returns: a list of email notifications - :rtype: list of dicts with keys 'subject' and 'body' - - """ - notifications = [] - for function in _notifications_functions: - notifications.extend(function(user_dict, since)) - return notifications - - -def send_notification( - user: dict[str, Any], email_dict: dict[str, Any] -) -> None: - """Email `email_dict` to `user`.""" - import ckan.lib.mailer - - if not user.get("email"): - # FIXME: Raise an exception. - return - - try: - ckan.lib.mailer.mail_recipient( - user["display_name"], - user["email"], - email_dict["subject"], - email_dict["body"], - ) - except ckan.lib.mailer.MailerException: - raise - - -def get_and_send_notifications_for_user(user: dict[str, Any]) -> None: - - # Parse the email_notifications_since config setting, email notifications - # from longer ago than this time will not be sent. - email_notifications_since = config.get( - "ckan.email_notifications_since" - ) - email_notifications_since = string_to_timedelta(email_notifications_since) - email_notifications_since = ( - datetime.datetime.utcnow() - email_notifications_since - ) - - # FIXME: We are accessing model from lib here but I'm not sure what - # else to do unless we add a get_email_last_sent() logic function which - # would only be needed by this lib. - dashboard = model.Dashboard.get(user["id"]) - if dashboard: - email_last_sent = dashboard.email_last_sent - activity_stream_last_viewed = dashboard.activity_stream_last_viewed - since = max( - email_notifications_since, - email_last_sent, - activity_stream_last_viewed, - ) - - notifications = get_notifications(user, since) - # TODO: Handle failures from send_email_notification. - for notification in notifications: - send_notification(user, notification) - - # FIXME: We are accessing model from lib here but I'm not sure what - # else to do unless we add a update_email_last_sent() - # logic function which would only be needed by this lib. - dashboard.email_last_sent = datetime.datetime.utcnow() - model.repo.commit() - - -def get_and_send_notifications_for_all_users() -> None: - context = cast( - Context, - { - "model": model, - "session": model.Session, - "ignore_auth": True, - "keep_email": True, - }, - ) - users = logic.get_action("user_list")(context, {}) - for user in users: - get_and_send_notifications_for_user(user) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/helpers.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/helpers.py deleted file mode 100644 index a6d8953..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/helpers.py +++ /dev/null @@ -1,190 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations -from typing import Any, Optional, cast - -import jinja2 -import datetime -from markupsafe import Markup - -import ckan.model as model -import ckan.plugins.toolkit as tk - -from ckan.types import Context -from . import changes - - -def dashboard_activity_stream( - user_id: str, - filter_type: Optional[str] = None, - filter_id: Optional[str] = None, - offset: int = 0, - limit: int = 0, - before: Optional[datetime.datetime] = None, - after: Optional[datetime.datetime] = None, -) -> list[dict[str, Any]]: - """Return the dashboard activity stream of the current user. - - :param user_id: the id of the user - :type user_id: string - - :param filter_type: the type of thing to filter by - :type filter_type: string - - :param filter_id: the id of item to filter by - :type filter_id: string - - :returns: an activity stream as an HTML snippet - :rtype: string - - """ - context = cast(Context, {"user": tk.g.user}) - if filter_type: - action_functions = { - "dataset": "package_activity_list", - "user": "user_activity_list", - "group": "group_activity_list", - "organization": "organization_activity_list", - } - action_function = tk.get_action(action_functions[filter_type]) - return action_function( - context, { - "id": filter_id, - "limit": limit, - "offset": offset, - "before": before, - "after": after - }) - else: - return tk.get_action("dashboard_activity_list")( - context, { - "offset": offset, - "limit": limit, - "before": before, - "after": after - } - ) - - -def recently_changed_packages_activity_stream( - limit: Optional[int] = None, -) -> list[dict[str, Any]]: - if limit: - data_dict = {"limit": limit} - else: - data_dict = {} - context = cast( - Context, {"model": model, "session": model.Session, "user": tk.g.user} - ) - return tk.get_action("recently_changed_packages_activity_list")( - context, data_dict - ) - - -def new_activities() -> Optional[int]: - """Return the number of activities for the current user. - - See :func:`logic.action.get.dashboard_new_activities_count` for more - details. - - """ - if not tk.g.userobj: - return None - action = tk.get_action("dashboard_new_activities_count") - return action({}, {}) - - -def activity_list_select( - pkg_activity_list: list[dict[str, Any]], current_activity_id: str -) -> list[Markup]: - """ - Builds an HTML formatted list of options for the select lists - on the "Changes" summary page. - """ - select_list = [] - template = jinja2.Template( - '', - autoescape=True, - ) - for activity in pkg_activity_list: - entry = tk.h.render_datetime( - activity["timestamp"], with_hours=True, with_seconds=True - ) - select_list.append( - Markup( - template.render( - activity_id=activity["id"], - timestamp=entry, - selected="selected" - if activity["id"] == current_activity_id - else "", - ) - ) - ) - - return select_list - - -def compare_pkg_dicts( - old: dict[str, Any], new: dict[str, Any], old_activity_id: str -) -> list[dict[str, Any]]: - """ - Takes two package dictionaries that represent consecutive versions of - the same dataset and returns a list of detailed & formatted summaries of - the changes between the two versions. old and new are the two package - dictionaries. The function assumes that both dictionaries will have - all of the default package dictionary keys, and also checks for fields - added by extensions and extra fields added by the user in the web - interface. - - Returns a list of dictionaries, each of which corresponds to a change - to the dataset made in this revision. The dictionaries each contain a - string indicating the type of change made as well as other data necessary - to form a detailed summary of the change. - """ - - change_list: list[dict[str, Any]] = [] - - changes.check_metadata_changes(change_list, old, new) - - changes.check_resource_changes(change_list, old, new, old_activity_id) - - # if the dataset was updated but none of the fields we check were changed, - # display a message stating that - if len(change_list) == 0: - change_list.append({"type": "no_change"}) - - return change_list - - -def compare_group_dicts( - old: dict[str, Any], new: dict[str, Any], old_activity_id: str -): - """ - Takes two package dictionaries that represent consecutive versions of - the same organization and returns a list of detailed & formatted summaries - of the changes between the two versions. old and new are the two package - dictionaries. The function assumes that both dictionaries will have - all of the default package dictionary keys, and also checks for fields - added by extensions and extra fields added by the user in the web - interface. - - Returns a list of dictionaries, each of which corresponds to a change - to the dataset made in this revision. The dictionaries each contain a - string indicating the type of change made as well as other data necessary - to form a detailed summary of the change. - """ - change_list: list[dict[str, Any]] = [] - - changes.check_metadata_org_changes(change_list, old, new) - - # if the organization was updated but none of the fields we check - # were changed, display a message stating that - if len(change_list) == 0: - change_list.append({"type": "no_change"}) - - return change_list - - -def activity_show_email_notifications() -> bool: - return tk.config.get("ckan.activity_streams_email_notifications") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/action.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/action.py deleted file mode 100644 index a42a1c8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/action.py +++ /dev/null @@ -1,647 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations - -import logging -import datetime -import json -from typing import Any, Optional - -import ckan.plugins.toolkit as tk - -from ckan.logic import validate -from ckan.types import Context, DataDict, ActionResult -import ckanext.activity.email_notifications as email_notifications - -from . import schema -from ..model import activity as model_activity, activity_dict_save - -log = logging.getLogger(__name__) - - -def send_email_notifications( - context: Context, data_dict: DataDict -) -> ActionResult.SendEmailNotifications: - """Send any pending activity stream notification emails to users. - - You must provide a sysadmin's API key/token in the Authorization header of - the request, or call this action from the command-line via a `ckan notify - send_emails ...` command. - - """ - tk.check_access("send_email_notifications", context, data_dict) - - if not tk.config.get("ckan.activity_streams_email_notifications"): - raise tk.ValidationError( - { - "message": ( - "ckan.activity_streams_email_notifications" - " is not enabled in config" - ) - } - ) - - email_notifications.get_and_send_notifications_for_all_users() - - -def dashboard_mark_activities_old( - context: Context, data_dict: DataDict -) -> ActionResult.DashboardMarkActivitiesOld: - """Mark all the authorized user's new dashboard activities as old. - - This will reset - :py:func:`~ckan.logic.action.get.dashboard_new_activities_count` to 0. - - """ - tk.check_access("dashboard_mark_activities_old", context, data_dict) - model = context["model"] - user_obj = model.User.get(context["user"]) - assert user_obj - user_id = user_obj.id - dashboard = model.Dashboard.get(user_id) - if dashboard: - dashboard.activity_stream_last_viewed = datetime.datetime.utcnow() - if not context.get("defer_commit"): - model.repo.commit() - - -def activity_create( - context: Context, data_dict: DataDict -) -> Optional[dict[str, Any]]: - """Create a new activity stream activity. - - You must be a sysadmin to create new activities. - - :param user_id: the name or id of the user who carried out the activity, - e.g. ``'seanh'`` - :type user_id: string - :param object_id: the name or id of the object of the activity, e.g. - ``'my_dataset'`` - :param activity_type: the type of the activity, this must be an activity - type that CKAN knows how to render, e.g. ``'new package'``, - ``'changed user'``, ``'deleted group'`` etc. - :type activity_type: string - :param data: any additional data about the activity - :type data: dictionary - - :returns: the newly created activity - :rtype: dictionary - - """ - - tk.check_access("activity_create", context, data_dict) - - if not tk.config.get("ckan.activity_streams_enabled"): - return - - model = context["model"] - - # Any revision_id that the caller attempts to pass in the activity_dict is - # ignored and removed here. - if "revision_id" in data_dict: - del data_dict["revision_id"] - - sch = context.get("schema") or schema.default_create_activity_schema() - - data, errors = tk.navl_validate(data_dict, sch, context) - if errors: - raise tk.ValidationError(errors) - - activity = activity_dict_save(data, context) - - if not context.get("defer_commit"): - model.repo.commit() - - log.debug("Created '%s' activity" % activity.activity_type) - return model_activity.activity_dictize(activity, context) - - -@validate(schema.default_activity_list_schema) -@tk.side_effect_free -def user_activity_list( - context: Context, data_dict: DataDict -) -> list[dict[str, Any]]: - """Return a user's public activity stream. - - You must be authorized to view the user's profile. - - - :param id: the id or name of the user - :type id: string - :param offset: where to start getting activity items from - (optional, default: ``0``) - :type offset: int - :param limit: the maximum number of activities to return - (optional, default: ``31`` unless set in site's configuration - ``ckan.activity_list_limit``, upper limit: ``100`` unless set in - site's configuration ``ckan.activity_list_limit_max``) - :type limit: int - :param after: After timestamp - (optional, default: ``None``) - :type after: int, str - :param before: Before timestamp - (optional, default: ``None``) - :type before: int, str - - :rtype: list of dictionaries - - """ - # FIXME: Filter out activities whose subject or object the user is not - # authorized to read. - tk.check_access("user_activity_list", context, data_dict) - - model = context["model"] - - user_ref = data_dict.get("id") # May be user name or id. - user = model.User.get(user_ref) - if user is None: - raise tk.ObjectNotFound() - - offset = data_dict.get("offset", 0) - limit = data_dict["limit"] # defaulted, limited & made an int by schema - after = data_dict.get("after") - before = data_dict.get("before") - - activity_objects = model_activity.user_activity_list( - user.id, - limit=limit, - offset=offset, - after=after, - before=before, - ) - - return model_activity.activity_list_dictize(activity_objects, context) - - -@validate(schema.default_activity_list_schema) -@tk.side_effect_free -def package_activity_list( - context: Context, data_dict: DataDict -) -> list[dict[str, Any]]: - """Return a package's activity stream (not including detail) - - You must be authorized to view the package. - - :param id: the id or name of the package - :type id: string - :param offset: where to start getting activity items from - (optional, default: ``0``) - :type offset: int - :param limit: the maximum number of activities to return - (optional, default: ``31`` unless set in site's configuration - ``ckan.activity_list_limit``, upper limit: ``100`` unless set in - site's configuration ``ckan.activity_list_limit_max``) - :type limit: int - :param after: After timestamp - (optional, default: ``None``) - :type after: int, str - :param before: Before timestamp - (optional, default: ``None``) - :type before: int, str - :param include_hidden_activity: whether to include 'hidden' activity, which - is not shown in the Activity Stream page. Hidden activity includes - activity done by the site_user, such as harvests, which are not shown - in the activity stream because they can be too numerous, or activity by - other users specified in config option `ckan.hide_activity_from_users`. - NB Only sysadmins may set include_hidden_activity to true. - (default: false) - :type include_hidden_activity: bool - :param activity_types: A list of activity types to include in the response - :type activity_types: list - - :param exclude_activity_types: A list of activity types to exclude from the - response - :type exclude_activity_types: list - - :rtype: list of dictionaries - - """ - # FIXME: Filter out activities whose subject or object the user is not - # authorized to read. - include_hidden_activity = data_dict.get("include_hidden_activity", False) - activity_types = data_dict.pop("activity_types", None) - exclude_activity_types = data_dict.pop("exclude_activity_types", None) - - if activity_types is not None and exclude_activity_types is not None: - raise tk.ValidationError( - { - "activity_types": [ - "Cannot be used together with `exclude_activity_types" - ] - } - ) - - tk.check_access("package_activity_list", context, data_dict) - - model = context["model"] - - package_ref = data_dict.get("id") # May be name or ID. - package = model.Package.get(package_ref) - if package is None: - raise tk.ObjectNotFound() - - offset = int(data_dict.get("offset", 0)) - limit = data_dict["limit"] # defaulted, limited & made an int by schema - after = data_dict.get("after") - before = data_dict.get("before") - - activity_objects = model_activity.package_activity_list( - package.id, - limit=limit, - offset=offset, - after=after, - before=before, - include_hidden_activity=include_hidden_activity, - activity_types=activity_types, - exclude_activity_types=exclude_activity_types, - ) - - return model_activity.activity_list_dictize(activity_objects, context) - - -@validate(schema.default_activity_list_schema) -@tk.side_effect_free -def group_activity_list( - context: Context, data_dict: DataDict -) -> list[dict[str, Any]]: - """Return a group's activity stream. - - You must be authorized to view the group. - - :param id: the id or name of the group - :type id: string - :param offset: where to start getting activity items from - (optional, default: ``0``) - :type offset: int - :param limit: the maximum number of activities to return - (optional, default: ``31`` unless set in site's configuration - ``ckan.activity_list_limit``, upper limit: ``100`` unless set in - site's configuration ``ckan.activity_list_limit_max``) - :type limit: int - :param include_hidden_activity: whether to include 'hidden' activity, which - is not shown in the Activity Stream page. Hidden activity includes - activity done by the site_user, such as harvests, which are not shown - in the activity stream because they can be too numerous, or activity by - other users specified in config option `ckan.hide_activity_from_users`. - NB Only sysadmins may set include_hidden_activity to true. - (default: false) - :type include_hidden_activity: bool - - :rtype: list of dictionaries - - """ - # FIXME: Filter out activities whose subject or object the user is not - # authorized to read. - data_dict = dict(data_dict, include_data=False) - include_hidden_activity = data_dict.get("include_hidden_activity", False) - activity_types = data_dict.pop("activity_types", None) - tk.check_access("group_activity_list", context, data_dict) - - group_id = data_dict.get("id") - offset = data_dict.get("offset", 0) - limit = data_dict["limit"] # defaulted, limited & made an int by schema - - # Convert group_id (could be id or name) into id. - group_show = tk.get_action("group_show") - group_id = group_show(context, {"id": group_id})["id"] - - after = data_dict.get("after") - before = data_dict.get("before") - - activity_objects = model_activity.group_activity_list( - group_id, - limit=limit, - offset=offset, - after=after, - before=before, - include_hidden_activity=include_hidden_activity, - activity_types=activity_types - ) - - return model_activity.activity_list_dictize(activity_objects, context) - - -@validate(schema.default_activity_list_schema) -@tk.side_effect_free -def organization_activity_list( - context: Context, data_dict: DataDict -) -> list[dict[str, Any]]: - """Return a organization's activity stream. - - :param id: the id or name of the organization - :type id: string - :param offset: where to start getting activity items from - (optional, default: ``0``) - :type offset: int - :param limit: the maximum number of activities to return - (optional, default: ``31`` unless set in site's configuration - ``ckan.activity_list_limit``, upper limit: ``100`` unless set in - site's configuration ``ckan.activity_list_limit_max``) - :type limit: int - :param include_hidden_activity: whether to include 'hidden' activity, which - is not shown in the Activity Stream page. Hidden activity includes - activity done by the site_user, such as harvests, which are not shown - in the activity stream because they can be too numerous, or activity by - other users specified in config option `ckan.hide_activity_from_users`. - NB Only sysadmins may set include_hidden_activity to true. - (default: false) - :type include_hidden_activity: bool - - :rtype: list of dictionaries - - """ - # FIXME: Filter out activities whose subject or object the user is not - # authorized to read. - include_hidden_activity = data_dict.get("include_hidden_activity", False) - tk.check_access("organization_activity_list", context, data_dict) - - org_id = data_dict.get("id") - offset = data_dict.get("offset", 0) - limit = data_dict["limit"] # defaulted, limited & made an int by schema - activity_types = data_dict.pop("activity_types", None) - - # Convert org_id (could be id or name) into id. - org_show = tk.get_action("organization_show") - org_id = org_show(context, {"id": org_id})["id"] - - after = data_dict.get("after") - before = data_dict.get("before") - - activity_objects = model_activity.organization_activity_list( - org_id, - limit=limit, - offset=offset, - after=after, - before=before, - include_hidden_activity=include_hidden_activity, - activity_types=activity_types - ) - - return model_activity.activity_list_dictize(activity_objects, context) - - -@validate(schema.default_dashboard_activity_list_schema) -@tk.side_effect_free -def recently_changed_packages_activity_list( - context: Context, data_dict: DataDict -) -> list[dict[str, Any]]: - """Return the activity stream of all recently added or changed packages. - - :param offset: where to start getting activity items from - (optional, default: ``0``) - :type offset: int - :param limit: the maximum number of activities to return - (optional, default: ``31`` unless set in site's configuration - ``ckan.activity_list_limit``, upper limit: ``100`` unless set in - site's configuration ``ckan.activity_list_limit_max``) - :type limit: int - - :rtype: list of dictionaries - - """ - # FIXME: Filter out activities whose subject or object the user is not - # authorized to read. - offset = data_dict.get("offset", 0) - limit = data_dict["limit"] # defaulted, limited & made an int by schema - - activity_objects = model_activity.recently_changed_packages_activity_list( - limit=limit, offset=offset - ) - - return model_activity.activity_list_dictize(activity_objects, context) - - -@validate(schema.default_dashboard_activity_list_schema) -@tk.side_effect_free -def dashboard_activity_list( - context: Context, data_dict: DataDict -) -> list[dict[str, Any]]: - """Return the authorized (via login or API key) user's dashboard activity - stream. - - Unlike the activity dictionaries returned by other ``*_activity_list`` - actions, these activity dictionaries have an extra boolean value with key - ``is_new`` that tells you whether the activity happened since the user last - viewed her dashboard (``'is_new': True``) or not (``'is_new': False``). - - The user's own activities are always marked ``'is_new': False``. - - :param offset: where to start getting activity items from - (optional, default: ``0``) - :type offset: int - :param limit: the maximum number of activities to return - (optional, default: ``31`` unless set in site's configuration - ``ckan.activity_list_limit``, upper limit: ``100`` unless set in - site's configuration ``ckan.activity_list_limit_max``) - :type limit: int - - :rtype: list of activity dictionaries - - """ - tk.check_access("dashboard_activity_list", context, data_dict) - - model = context["model"] - user_obj = model.User.get(context["user"]) - assert user_obj - user_id = user_obj.id - offset = data_dict.get("offset", 0) - limit = data_dict["limit"] # defaulted, limited & made an int by schema - before = data_dict.get("before") - after = data_dict.get("after") - # FIXME: Filter out activities whose subject or object the user is not - # authorized to read. - activity_objects = model_activity.dashboard_activity_list( - user_id, limit=limit, offset=offset, before=before, after=after - ) - - activity_dicts = model_activity.activity_list_dictize( - activity_objects, context - ) - - # Mark the new (not yet seen by user) activities. - strptime = datetime.datetime.strptime - fmt = "%Y-%m-%dT%H:%M:%S.%f" - dashboard = model.Dashboard.get(user_id) - last_viewed = None - if dashboard: - last_viewed = dashboard.activity_stream_last_viewed - for activity in activity_dicts: - if activity["user_id"] == user_id: - # Never mark the user's own activities as new. - activity["is_new"] = False - elif last_viewed: - activity["is_new"] = ( - strptime(activity["timestamp"], fmt) > last_viewed - ) - - return activity_dicts - - -@tk.side_effect_free -def dashboard_new_activities_count( - context: Context, data_dict: DataDict -) -> ActionResult.DashboardNewActivitiesCount: - """Return the number of new activities in the user's dashboard. - - Return the number of new activities in the authorized user's dashboard - activity stream. - - Activities from the user herself are not counted by this function even - though they appear in the dashboard (users don't want to be notified about - things they did themselves). - - :rtype: int - - """ - tk.check_access("dashboard_new_activities_count", context, data_dict) - activities = tk.get_action("dashboard_activity_list")(context, data_dict) - return len([activity for activity in activities if activity["is_new"]]) - - -@tk.side_effect_free -def activity_show(context: Context, data_dict: DataDict) -> dict[str, Any]: - """Show details of an item of 'activity' (part of the activity stream). - - :param id: the id of the activity - :type id: string - - :rtype: dictionary - """ - model = context["model"] - activity_id = tk.get_or_bust(data_dict, "id") - - activity = model.Session.query(model_activity.Activity).get(activity_id) - if activity is None: - raise tk.ObjectNotFound() - context["activity"] = activity - - tk.check_access("activity_show", context, data_dict) - - activity = model_activity.activity_dictize(activity, context) - return activity - - -@tk.side_effect_free -def activity_data_show( - context: Context, data_dict: DataDict -) -> dict[str, Any]: - """Show the data from an item of 'activity' (part of the activity - stream). - - For example for a package update this returns just the dataset dict but - none of the activity stream info of who and when the version was created. - - :param id: the id of the activity - :type id: string - :param object_type: 'package', 'user', 'group' or 'organization' - :type object_type: string - - :rtype: dictionary - """ - model = context["model"] - activity_id = tk.get_or_bust(data_dict, "id") - object_type = data_dict.get("object_type") - - activity = model.Session.query(model_activity.Activity).get(activity_id) - if activity is None: - raise tk.ObjectNotFound() - context["activity"] = activity - - tk.check_access("activity_data_show", context, data_dict) - - activity = model_activity.activity_dictize(activity, context) - try: - activity_data = activity["data"] - except KeyError: - raise tk.ObjectNotFound("Could not find data in the activity") - if object_type: - try: - activity_data = activity_data[object_type] - except KeyError: - raise tk.ObjectNotFound( - "Could not find that object_type in the activity" - ) - return activity_data - - -@tk.side_effect_free -def activity_diff(context: Context, data_dict: DataDict) -> dict[str, Any]: - """Returns a diff of the activity, compared to the previous version of the - object - - :param id: the id of the activity - :type id: string - :param object_type: 'package', 'user', 'group' or 'organization' - :type object_type: string - :param diff_type: 'unified', 'context', 'html' - :type diff_type: string - """ - import difflib - - model = context["model"] - activity_id = tk.get_or_bust(data_dict, "id") - object_type = tk.get_or_bust(data_dict, "object_type") - diff_type = data_dict.get("diff_type", "unified") - - tk.check_access("activity_diff", context, data_dict) - - activity = model.Session.query(model_activity.Activity).get(activity_id) - if activity is None: - raise tk.ObjectNotFound() - prev_activity = ( - model.Session.query(model_activity.Activity) - .filter_by(object_id=activity.object_id) - .filter(model_activity.Activity.timestamp < activity.timestamp) - .order_by( - # type_ignore_reason: incomplete SQLAlchemy types - model_activity.Activity.timestamp.desc() # type: ignore - ) - .first() - ) - if prev_activity is None: - raise tk.ObjectNotFound("Previous activity for this object not found") - activity_objs = [prev_activity, activity] - try: - objs = [ - activity_obj.data[object_type] for activity_obj in activity_objs - ] - except KeyError: - raise tk.ObjectNotFound("Could not find object in the activity data") - # convert each object dict to 'pprint'-style - # and split into lines to suit difflib - obj_lines = [ - json.dumps(obj, indent=2, sort_keys=True).split("\n") for obj in objs - ] - - # do the diff - if diff_type == "unified": - # type_ignore_reason: typechecker can't predict number of items - diff_generator = difflib.unified_diff(*obj_lines) # type: ignore - diff = "\n".join(line for line in diff_generator) - elif diff_type == "context": - # type_ignore_reason: typechecker can't predict number of items - diff_generator = difflib.context_diff(*obj_lines) # type: ignore - diff = "\n".join(line for line in diff_generator) - elif diff_type == "html": - # word-wrap lines. Otherwise you get scroll bars for most datasets. - import re - - for obj_index in (0, 1): - wrapped_obj_lines = [] - for line in obj_lines[obj_index]: - wrapped_obj_lines.extend(re.findall(r".{1,70}(?:\s+|$)", line)) - obj_lines[obj_index] = wrapped_obj_lines - # type_ignore_reason: typechecker can't predict number of items - diff = difflib.HtmlDiff().make_table(*obj_lines) # type: ignore - else: - raise tk.ValidationError({"message": "diff_type not recognized"}) - - activities = [ - model_activity.activity_dictize(activity_obj, context) - for activity_obj in activity_objs - ] - - return { - "diff": diff, - "activities": activities, - } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/auth.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/auth.py deleted file mode 100644 index 0df9fd7..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/auth.py +++ /dev/null @@ -1,184 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations - -from typing import Optional - -import ckan.authz as authz -import ckan.plugins.toolkit as tk -from ckan.types import Context, DataDict, AuthResult - -from ..model import Activity - - -def _get_activity_object( - context: Context, data_dict: Optional[DataDict] = None -) -> Activity: - try: - return context["activity"] - except KeyError: - if not data_dict: - data_dict = {} - id = data_dict.get("id", None) - if not id: - raise tk.ValidationError( - {"message": "Missing id, can not get Activity object"} - ) - obj = Activity.get(id) - if not obj: - raise tk.ObjectNotFound() - # Save in case we need this again during the request - context["activity"] = obj - return obj - - -def send_email_notifications( - context: Context, data_dict: DataDict -) -> AuthResult: - # Only sysadmins are authorized to send email notifications. - return {"success": False} - - -def activity_create(context: Context, data_dict: DataDict) -> AuthResult: - return {"success": False} - - -@tk.auth_allow_anonymous_access -def dashboard_activity_list( - context: Context, data_dict: DataDict -) -> AuthResult: - # FIXME: context['user'] could be an IP address but that case is not - # handled here. Maybe add an auth helper function like is_logged_in(). - if context.get("user"): - return {"success": True} - else: - return { - "success": False, - "msg": tk._("You must be logged in to access your dashboard."), - } - - -@tk.auth_allow_anonymous_access -def dashboard_new_activities_count( - context: Context, data_dict: DataDict -) -> AuthResult: - # FIXME: This should go through check_access() not call is_authorized() - # directly, but wait until 2939-orgs is merged before fixing this. - # This is so a better not authourized message can be sent. - return authz.is_authorized("dashboard_activity_list", context, data_dict) - - -@tk.auth_allow_anonymous_access -def activity_list(context: Context, data_dict: DataDict) -> AuthResult: - """ - :param id: the id or name of the object (e.g. package id) - :type id: string - :param object_type: The type of the object (e.g. 'package', 'organization', - 'group', 'user') - :type object_type: string - :param include_data: include the data field, containing a full object dict - (otherwise the data field is only returned with the object's title) - :type include_data: boolean - """ - if data_dict["object_type"] not in ( - "package", - "organization", - "group", - "user", - ): - return {"success": False, "msg": "object_type not recognized"} - is_public = authz.check_config_permission("public_activity_stream_detail") - if data_dict.get("include_data") and not is_public: - # The 'data' field of the activity is restricted to users who are - # allowed to edit the object - show_or_update = "update" - else: - # the activity for an object (i.e. the activity metadata) can be viewed - # if the user can see the object - show_or_update = "show" - action_on_which_to_base_auth = "{}_{}".format( - data_dict["object_type"], show_or_update - ) # e.g. 'package_update' - return authz.is_authorized( - action_on_which_to_base_auth, context, {"id": data_dict["id"]} - ) - - -@tk.auth_allow_anonymous_access -def user_activity_list(context: Context, data_dict: DataDict) -> AuthResult: - data_dict["object_type"] = "user" - # TODO: use authz.is_authorized in order to allow chained auth functions. - # TODO: fix the issue in other functions as well - return activity_list(context, data_dict) - - -@tk.auth_allow_anonymous_access -def package_activity_list(context: Context, data_dict: DataDict) -> AuthResult: - data_dict["object_type"] = "package" - return activity_list(context, data_dict) - - -@tk.auth_allow_anonymous_access -def group_activity_list(context: Context, data_dict: DataDict) -> AuthResult: - data_dict["object_type"] = "group" - return activity_list(context, data_dict) - - -@tk.auth_allow_anonymous_access -def organization_activity_list( - context: Context, data_dict: DataDict -) -> AuthResult: - data_dict["object_type"] = "organization" - return activity_list(context, data_dict) - - -@tk.auth_allow_anonymous_access -def activity_show(context: Context, data_dict: DataDict) -> AuthResult: - """ - :param id: the id of the activity - :type id: string - :param include_data: include the data field, containing a full object dict - (otherwise the data field is only returned with the object's title) - :type include_data: boolean - """ - activity = _get_activity_object(context, data_dict) - # NB it would be better to have recorded an activity_type against the - # activity - if "package" in activity.activity_type: - object_type = "package" - else: - return {"success": False, "msg": "object_type not recognized"} - return activity_list( - context, - { - "id": activity.object_id, - "include_data": data_dict["include_data"], - "object_type": object_type, - }, - ) - - -@tk.auth_allow_anonymous_access -def activity_data_show(context: Context, data_dict: DataDict) -> AuthResult: - """ - :param id: the id of the activity - :type id: string - """ - data_dict["include_data"] = True - return activity_show(context, data_dict) - - -@tk.auth_allow_anonymous_access -def activity_diff(context: Context, data_dict: DataDict) -> AuthResult: - """ - :param id: the id of the activity - :type id: string - """ - data_dict["include_data"] = True - return activity_show(context, data_dict) - - -def dashboard_mark_activities_old( - context: Context, data_dict: DataDict -) -> AuthResult: - return authz.is_authorized("dashboard_activity_list", context, data_dict) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/schema.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/schema.py deleted file mode 100644 index e01f52f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/schema.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations - -from typing import cast - -from ckan.logic.schema import validator_args, default_pagination_schema -from ckan.types import Schema, Validator, ValidatorFactory - - -@validator_args -def default_create_activity_schema( - ignore: Validator, - not_missing: Validator, - not_empty: Validator, - unicode_safe: Validator, - convert_user_name_or_id_to_id: Validator, - object_id_validator: Validator, - activity_type_exists: Validator, - ignore_empty: Validator, - ignore_missing: Validator, -): - return cast( - Schema, - { - "id": [ignore], - "timestamp": [ignore], - "user_id": [ - not_missing, - not_empty, - unicode_safe, - convert_user_name_or_id_to_id, - ], - "object_id": [ - not_missing, - not_empty, - unicode_safe, - object_id_validator, - ], - "activity_type": [ - not_missing, - not_empty, - unicode_safe, - activity_type_exists, - ], - "data": [ignore_empty, ignore_missing], - }, - ) - - -@validator_args -def default_dashboard_activity_list_schema( - configured_default: ValidatorFactory, - natural_number_validator: Validator, - limit_to_configured_maximum: ValidatorFactory, - ignore_missing: Validator, - datetime_from_timestamp_validator: Validator, - -): - schema = default_pagination_schema() - schema["limit"] = [ - configured_default("ckan.activity_list_limit", 31), - natural_number_validator, - limit_to_configured_maximum("ckan.activity_list_limit_max", 100), - ] - schema["before"] = [ignore_missing, datetime_from_timestamp_validator] - schema["after"] = [ignore_missing, datetime_from_timestamp_validator] - return schema - - -@validator_args -def default_activity_list_schema( - not_missing: Validator, - unicode_safe: Validator, - configured_default: ValidatorFactory, - natural_number_validator: Validator, - limit_to_configured_maximum: ValidatorFactory, - ignore_missing: Validator, - boolean_validator: Validator, - ignore_not_sysadmin: Validator, - list_of_strings: Validator, - datetime_from_timestamp_validator: Validator, -): - - schema = default_pagination_schema() - schema["id"] = [not_missing, unicode_safe] - schema["limit"] = [ - configured_default("ckan.activity_list_limit", 31), - natural_number_validator, - limit_to_configured_maximum("ckan.activity_list_limit_max", 100), - ] - schema["include_hidden_activity"] = [ - ignore_missing, - ignore_not_sysadmin, - boolean_validator, - ] - schema["activity_types"] = [ignore_missing, list_of_strings] - schema["exclude_activity_types"] = [ignore_missing, list_of_strings] - schema["before"] = [ignore_missing, datetime_from_timestamp_validator] - schema["after"] = [ignore_missing, datetime_from_timestamp_validator] - - return schema diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/validators.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/validators.py deleted file mode 100644 index ac3af06..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/logic/validators.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations -from typing import Any, cast - -import ckan.plugins.toolkit as tk - -from ckan.types import ( - FlattenDataDict, - FlattenKey, - Context, - FlattenErrorDict, - ContextValidator, -) - - -def activity_type_exists(activity_type: Any) -> Any: - """Raises Invalid if there is no registered activity renderer for the - given activity_type. Otherwise returns the given activity_type. - - This just uses object_id_validators as a lookup. - very safe. - - """ - if activity_type in object_id_validators: - return activity_type - else: - raise tk.Invalid("%s: %s" % (tk._("Not found"), tk._("Activity type"))) - - -VALIDATORS_PACKAGE_ACTIVITY_TYPES = { - "new package": "package_id_exists", - "changed package": "package_id_exists", - "deleted package": "package_id_exists", - "follow dataset": "package_id_exists", -} - -VALIDATORS_USER_ACTIVITY_TYPES = { - "new user": "user_id_exists", - "changed user": "user_id_exists", - "follow user": "user_id_exists", -} - -VALIDATORS_GROUP_ACTIVITY_TYPES = { - "new group": "group_id_exists", - "changed group": "group_id_exists", - "deleted group": "group_id_exists", - "follow group": "group_id_exists", -} - -VALIDATORS_ORGANIZATION_ACTIVITY_TYPES = { - "new organization": "group_id_exists", - "changed organization": "group_id_exists", - "deleted organization": "group_id_exists", - "follow organization": "group_id_exists", -} - -# A dictionary mapping activity_type values from activity dicts to functions -# for validating the object_id values from those same activity dicts. -object_id_validators = { - **VALIDATORS_PACKAGE_ACTIVITY_TYPES, - **VALIDATORS_USER_ACTIVITY_TYPES, - **VALIDATORS_GROUP_ACTIVITY_TYPES, - **VALIDATORS_ORGANIZATION_ACTIVITY_TYPES -} - - -def object_id_validator( - key: FlattenKey, - activity_dict: FlattenDataDict, - errors: FlattenErrorDict, - context: Context, -) -> Any: - """Validate the 'object_id' value of an activity_dict. - - Uses the object_id_validators dict (above) to find and call an 'object_id' - validator function for the given activity_dict's 'activity_type' value. - - Raises Invalid if the model given in context contains no object of the - correct type (according to the 'activity_type' value of the activity_dict) - with the given ID. - - Raises Invalid if there is no object_id_validator for the activity_dict's - 'activity_type' value. - - """ - activity_type = activity_dict[("activity_type",)] - if activity_type in object_id_validators: - object_id = activity_dict[("object_id",)] - name = object_id_validators[activity_type] - validator = cast(ContextValidator, tk.get_validator(name)) - return validator(object_id, context) - else: - raise tk.Invalid( - 'There is no object_id validator for activity type "%s"' - % activity_type - ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/__init__.py deleted file mode 100644 index 82c3c03..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations -from typing import Any - -from ckan.types import Context - -from .activity import Activity - - -__all__ = ["Activity"] - - -def activity_dict_save( - activity_dict: dict[str, Any], context: Context -) -> "Activity": - - session = context["session"] - user_id = activity_dict["user_id"] - object_id = activity_dict["object_id"] - activity_type = activity_dict["activity_type"] - if "data" in activity_dict: - data = activity_dict["data"] - else: - data = None - activity_obj = Activity(user_id, object_id, activity_type, data) - session.add(activity_obj) - return activity_obj diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/activity.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/activity.py deleted file mode 100644 index 50ed980..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/model/activity.py +++ /dev/null @@ -1,791 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -import datetime -from typing import Any, Optional, Type, TypeVar, cast -from typing_extensions import TypeAlias - -from sqlalchemy.orm import relationship, backref -from sqlalchemy import ( - types, - Column, - ForeignKey, - or_, - and_, - union_all, - text, -) - -from ckan.common import config -import ckan.model as model -import ckan.model.meta as meta -import ckan.model.domain_object as domain_object -import ckan.model.types as _types -from ckan.model.base import BaseModel -from ckan.lib.dictization import table_dictize - -from ckan.types import Context, Query # noqa - - -__all__ = ["Activity", "ActivityDetail"] - -TActivityDetail = TypeVar("TActivityDetail", bound="ActivityDetail") -QActivity: TypeAlias = "Query[Activity]" - - -class Activity(domain_object.DomainObject, BaseModel): # type: ignore - __tablename__ = "activity" - # the line below handles cases when activity table was already loaded into - # metadata state(via stats extension). Can be removed if stats stop using - # Table object. - __table_args__ = {"extend_existing": True} - - id = Column( - "id", types.UnicodeText, primary_key=True, default=_types.make_uuid - ) - timestamp = Column("timestamp", types.DateTime) - user_id = Column("user_id", types.UnicodeText) - object_id = Column("object_id", types.UnicodeText) - # legacy revision_id values are used by migrate_package_activity.py - revision_id = Column("revision_id", types.UnicodeText) - activity_type = Column("activity_type", types.UnicodeText) - data = Column("data", _types.JsonDictType) - - activity_detail: "ActivityDetail" - - def __init__( - self, - user_id: str, - object_id: str, - activity_type: str, - data: Optional[dict[str, Any]] = None, - ) -> None: - self.id = _types.make_uuid() - self.timestamp = datetime.datetime.utcnow() - self.user_id = user_id - self.object_id = object_id - self.activity_type = activity_type - if data is None: - self.data = {} - else: - self.data = data - - @classmethod - def get(cls, id: str) -> Optional["Activity"]: - """Returns an Activity object referenced by its id.""" - if not id: - return None - - return meta.Session.query(cls).get(id) - - @classmethod - def activity_stream_item( - cls, pkg: model.Package, activity_type: str, user_id: str - ) -> Optional["Activity"]: - import ckan.model - import ckan.logic - - assert activity_type in ("new", "changed"), str(activity_type) - - # Handle 'deleted' objects. - # When the user marks a package as deleted this comes through here as - # a 'changed' package activity. We detect this and change it to a - # 'deleted' activity. - if activity_type == "changed" and pkg.state == "deleted": - if ( - meta.Session.query(cls) - .filter_by(object_id=pkg.id, activity_type="deleted") - .all() - ): - # A 'deleted' activity for this object has already been emitted - # FIXME: What if the object was deleted and then activated - # again? - return None - else: - # Emit a 'deleted' activity for this object. - activity_type = "deleted" - - try: - # We save the entire rendered package dict so we can support - # viewing the past packages from the activity feed. - dictized_package = ckan.logic.get_action("package_show")( - cast( - Context, - { - "model": ckan.model, - "session": ckan.model.Session, - # avoid ckanext-multilingual translating it - "for_view": False, - "ignore_auth": True, - }, - ), - {"id": pkg.id, "include_tracking": False}, - ) - except ckan.logic.NotFound: - # This happens if this package is being purged and therefore has no - # current revision. - # TODO: Purge all related activity stream items when a model object - # is purged. - return None - - actor = meta.Session.query(ckan.model.User).get(user_id) - - return cls( - user_id, - pkg.id, - "%s package" % activity_type, - { - "package": dictized_package, - # We keep the acting user name around so that actions can be - # properly displayed even if the user is deleted in the future. - "actor": actor.name if actor else None, - }, - ) - - -def activity_dictize(activity: Activity, context: Context) -> dict[str, Any]: - return table_dictize(activity, context) - - -def activity_list_dictize( - activity_list: list[Activity], context: Context -) -> list[dict[str, Any]]: - return [activity_dictize(activity, context) for activity in activity_list] - - -# deprecated -class ActivityDetail(domain_object.DomainObject): - __tablename__ = "activity_detail" - id = Column( - "id", types.UnicodeText, primary_key=True, default=_types.make_uuid - ) - activity_id = Column( - "activity_id", types.UnicodeText, ForeignKey("activity.id") - ) - object_id = Column("object_id", types.UnicodeText) - object_type = Column("object_type", types.UnicodeText) - activity_type = Column("activity_type", types.UnicodeText) - data = Column("data", _types.JsonDictType) - - activity = relationship( - Activity, - backref=backref("activity_detail", cascade="all, delete-orphan"), - ) - - def __init__( - self, - activity_id: str, - object_id: str, - object_type: str, - activity_type: str, - data: Optional[dict[str, Any]] = None, - ) -> None: - self.activity_id = activity_id - self.object_id = object_id - self.object_type = object_type - self.activity_type = activity_type - if data is None: - self.data = {} - else: - self.data = data - - @classmethod - def by_activity_id( - cls: Type[TActivityDetail], activity_id: str - ) -> list["TActivityDetail"]: - return ( - model.Session.query(cls).filter_by(activity_id=activity_id).all() - ) - - -def _activities_limit( - q: QActivity, - limit: int, - offset: Optional[int] = None, - revese_order: Optional[bool] = False, -) -> QActivity: - """ - Return an SQLAlchemy query for all activities at an offset with a limit. - - revese_order: - if we want the last activities before a date, we must reverse the - order before limiting. - """ - if revese_order: - q = q.order_by(Activity.timestamp) - else: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.order_by(Activity.timestamp.desc()) # type: ignore - - if offset: - q = q.offset(offset) - if limit: - q = q.limit(limit) - return q - - -def _activities_union_all(*qlist: QActivity) -> QActivity: - """ - Return union of two or more activity queries sorted by timestamp, - and remove duplicates - """ - q: QActivity = ( - model.Session.query(Activity) - .select_entity_from(union_all(*[q.subquery().select() for q in qlist])) - .distinct(Activity.timestamp) - ) - return q - - -def _activities_from_user_query(user_id: str) -> QActivity: - """Return an SQLAlchemy query for all activities from user_id.""" - q = model.Session.query(Activity) - q = q.filter(Activity.user_id == user_id) - return q - - -def _activities_about_user_query(user_id: str) -> QActivity: - """Return an SQLAlchemy query for all activities about user_id.""" - q = model.Session.query(Activity) - q = q.filter(Activity.object_id == user_id) - return q - - -def _user_activity_query(user_id: str, limit: int) -> QActivity: - """Return an SQLAlchemy query for all activities from or about user_id.""" - q1 = _activities_limit(_activities_from_user_query(user_id), limit) - q2 = _activities_limit(_activities_about_user_query(user_id), limit) - return _activities_union_all(q1, q2) - - -def user_activity_list( - user_id: str, - limit: int, - offset: int, - after: Optional[datetime.datetime] = None, - before: Optional[datetime.datetime] = None, -) -> list[Activity]: - """Return user_id's public activity stream. - - Return a list of all activities from or about the given user, i.e. where - the given user is the subject or object of the activity, e.g.: - - "{USER} created the dataset {DATASET}" - "{OTHER_USER} started following {USER}" - etc. - - """ - q1 = _activities_from_user_query(user_id) - q2 = _activities_about_user_query(user_id) - - q = _activities_union_all(q1, q2) - - q = _filter_activitites_from_users(q) - - if after: - q = q.filter(Activity.timestamp > after) - if before: - q = q.filter(Activity.timestamp < before) - - # revert sort queries for "only before" queries - revese_order = after and not before - if revese_order: - q = q.order_by(Activity.timestamp) - else: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.order_by(Activity.timestamp.desc()) # type: ignore - - if offset: - q = q.offset(offset) - if limit: - q = q.limit(limit) - - results = q.all() - - # revert result if required - if revese_order: - results.reverse() - - return results - - -def _package_activity_query(package_id: str) -> QActivity: - """Return an SQLAlchemy query for all activities about package_id.""" - q = model.Session.query(Activity).filter_by(object_id=package_id) - return q - - -def package_activity_list( - package_id: str, - limit: int, - offset: Optional[int] = None, - after: Optional[datetime.datetime] = None, - before: Optional[datetime.datetime] = None, - include_hidden_activity: bool = False, - activity_types: Optional[list[str]] = None, - exclude_activity_types: Optional[list[str]] = None, -) -> list[Activity]: - """Return the given dataset (package)'s public activity stream. - - Returns all activities about the given dataset, i.e. where the given - dataset is the object of the activity, e.g.: - - "{USER} created the dataset {DATASET}" - "{USER} updated the dataset {DATASET}" - etc. - - """ - q = _package_activity_query(package_id) - - if not include_hidden_activity: - q = _filter_activitites_from_users(q) - - if activity_types: - q = _filter_activitites_from_type( - q, include=True, types=activity_types - ) - elif exclude_activity_types: - q = _filter_activitites_from_type( - q, include=False, types=exclude_activity_types - ) - - if after: - q = q.filter(Activity.timestamp > after) - if before: - q = q.filter(Activity.timestamp < before) - - # revert sort queries for "only before" queries - revese_order = after and not before - if revese_order: - q = q.order_by(Activity.timestamp) - else: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.order_by(Activity.timestamp.desc()) # type: ignore - - if offset: - q = q.offset(offset) - if limit: - q = q.limit(limit) - - results = q.all() - - # revert result if required - if revese_order: - results.reverse() - - return results - - -def _group_activity_query(group_id: str) -> QActivity: - """Return an SQLAlchemy query for all activities about group_id. - - Returns a query for all activities whose object is either the group itself - or one of the group's datasets. - - """ - group = model.Group.get(group_id) - if not group: - # Return a query with no results. - return model.Session.query(Activity).filter(text("0=1")) - - q: QActivity = ( - model.Session.query(Activity) - .outerjoin(model.Member, Activity.object_id == model.Member.table_id) - .outerjoin( - model.Package, - and_( - model.Package.id == model.Member.table_id, - model.Package.private == False, # noqa - ), - ) - .filter( - # We only care about activity either on the group itself or on - # packages within that group. FIXME: This means that activity that - # occured while a package belonged to a group but was then removed - # will not show up. This may not be desired but is consistent with - # legacy behaviour. - or_( - # active dataset in the group - and_( - model.Member.group_id == group_id, - model.Member.state == "active", - model.Package.state == "active", - ), - # deleted dataset in the group - and_( - model.Member.group_id == group_id, - model.Member.state == "deleted", - model.Package.state == "deleted", - ), - # (we want to avoid showing changes to an active dataset that - # was once in this group) - # activity the the group itself - Activity.object_id == group_id, - ) - ) - ) - - return q - - -def _organization_activity_query(org_id: str) -> QActivity: - """Return an SQLAlchemy query for all activities about org_id. - - Returns a query for all activities whose object is either the org itself - or one of the org's datasets. - - """ - org = model.Group.get(org_id) - if not org or not org.is_organization: - # Return a query with no results. - return model.Session.query(Activity).filter(text("0=1")) - - q: QActivity = ( - model.Session.query(Activity) - .outerjoin( - model.Package, - and_( - model.Package.id == Activity.object_id, - model.Package.private == False, # noqa - ), - ) - .filter( - # We only care about activity either on the the org itself or on - # packages within that org. - # FIXME: This means that activity that occured while a package - # belonged to a org but was then removed will not show up. This may - # not be desired but is consistent with legacy behaviour. - or_( - model.Package.owner_org == org_id, Activity.object_id == org_id - ) - ) - ) - - return q - - -def group_activity_list( - group_id: str, - limit: int, - offset: int, - after: Optional[datetime.datetime] = None, - before: Optional[datetime.datetime] = None, - include_hidden_activity: bool = False, - activity_types: Optional[list[str]] = None -) -> list[Activity]: - - """Return the given group's public activity stream. - - Returns activities where the given group or one of its datasets is the - object of the activity, e.g.: - - "{USER} updated the group {GROUP}" - "{USER} updated the dataset {DATASET}" - etc. - - """ - q = _group_activity_query(group_id) - - if not include_hidden_activity: - q = _filter_activitites_from_users(q) - - if activity_types: - q = _filter_activitites_from_type( - q, include=True, types=activity_types - ) - - if after: - q = q.filter(Activity.timestamp > after) - if before: - q = q.filter(Activity.timestamp < before) - - # revert sort queries for "only before" queries - revese_order = after and not before - if revese_order: - q = q.order_by(Activity.timestamp) - else: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.order_by(Activity.timestamp.desc()) # type: ignore - - if offset: - q = q.offset(offset) - if limit: - q = q.limit(limit) - - results = q.all() - - # revert result if required - if revese_order: - results.reverse() - - return results - - -def organization_activity_list( - group_id: str, - limit: int, - offset: int, - after: Optional[datetime.datetime] = None, - before: Optional[datetime.datetime] = None, - include_hidden_activity: bool = False, - activity_types: Optional[list[str]] = None -) -> list[Activity]: - """Return the given org's public activity stream. - - Returns activities where the given org or one of its datasets is the - object of the activity, e.g.: - - "{USER} updated the organization {ORG}" - "{USER} updated the dataset {DATASET}" - etc. - - """ - q = _organization_activity_query(group_id) - - if not include_hidden_activity: - q = _filter_activitites_from_users(q) - - if activity_types: - q = _filter_activitites_from_type( - q, include=True, types=activity_types - ) - - if after: - q = q.filter(Activity.timestamp > after) - if before: - q = q.filter(Activity.timestamp < before) - - # revert sort queries for "only before" queries - revese_order = after and not before - if revese_order: - q = q.order_by(Activity.timestamp) - else: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.order_by(Activity.timestamp.desc()) # type: ignore - - if offset: - q = q.offset(offset) - if limit: - q = q.limit(limit) - - results = q.all() - - # revert result if required - if revese_order: - results.reverse() - - return results - - -def _activities_from_users_followed_by_user_query( - user_id: str, limit: int -) -> QActivity: - """Return a query for all activities from users that user_id follows.""" - - # Get a list of the users that the given user is following. - follower_objects = model.UserFollowingUser.followee_list(user_id) - if not follower_objects: - # Return a query with no results. - return model.Session.query(Activity).filter(text("0=1")) - - return _activities_union_all( - *[ - _user_activity_query(follower.object_id, limit) - for follower in follower_objects - ] - ) - - -def _activities_from_datasets_followed_by_user_query( - user_id: str, limit: int -) -> QActivity: - """Return a query for all activities from datasets that user_id follows.""" - # Get a list of the datasets that the user is following. - follower_objects = model.UserFollowingDataset.followee_list(user_id) - if not follower_objects: - # Return a query with no results. - return model.Session.query(Activity).filter(text("0=1")) - - return _activities_union_all( - *[ - _activities_limit( - _package_activity_query(follower.object_id), limit - ) - for follower in follower_objects - ] - ) - - -def _activities_from_groups_followed_by_user_query( - user_id: str, limit: int -) -> QActivity: - """Return a query for all activities about groups the given user follows. - - Return a query for all activities about the groups the given user follows, - or about any of the group's datasets. This is the union of - _group_activity_query(group_id) for each of the groups the user follows. - - """ - # Get a list of the group's that the user is following. - follower_objects = model.UserFollowingGroup.followee_list(user_id) - if not follower_objects: - # Return a query with no results. - return model.Session.query(Activity).filter(text("0=1")) - - return _activities_union_all( - *[ - _activities_limit(_group_activity_query(follower.object_id), limit) - for follower in follower_objects - ] - ) - - -def _activities_from_everything_followed_by_user_query( - user_id: str, limit: int = 0 -) -> QActivity: - """Return a query for all activities from everything user_id follows.""" - q1 = _activities_from_users_followed_by_user_query(user_id, limit) - q2 = _activities_from_datasets_followed_by_user_query(user_id, limit) - q3 = _activities_from_groups_followed_by_user_query(user_id, limit) - return _activities_union_all(q1, q2, q3) - - -def activities_from_everything_followed_by_user( - user_id: str, limit: int, offset: int -) -> list[Activity]: - """Return activities from everything that the given user is following. - - Returns all activities where the object of the activity is anything - (user, dataset, group...) that the given user is following. - - """ - q = _activities_from_everything_followed_by_user_query( - user_id, limit + offset - ) - return _activities_limit(q, limit, offset).all() - - -def _dashboard_activity_query(user_id: str, limit: int = 0) -> QActivity: - """Return an SQLAlchemy query for user_id's dashboard activity stream.""" - q1 = _user_activity_query(user_id, limit) - q2 = _activities_from_everything_followed_by_user_query(user_id, limit) - return _activities_union_all(q1, q2) - - -def dashboard_activity_list( - user_id: str, - limit: int, - offset: int, - before: Optional[datetime.datetime] = None, - after: Optional[datetime.datetime] = None, -) -> list[Activity]: - """Return the given user's dashboard activity stream. - - Returns activities from the user's public activity stream, plus - activities from everything that the user is following. - - This is the union of user_activity_list(user_id) and - activities_from_everything_followed_by_user(user_id). - - """ - q = _dashboard_activity_query(user_id) - - q = _filter_activitites_from_users(q) - - if after: - q = q.filter(Activity.timestamp > after) - if before: - q = q.filter(Activity.timestamp < before) - - # revert sort queries for "only before" queries - revese_order = after and not before - if revese_order: - q = q.order_by(Activity.timestamp) - else: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.order_by(Activity.timestamp.desc()) # type: ignore - - if offset: - q = q.offset(offset) - if limit: - q = q.limit(limit) - - results = q.all() - - # revert result if required - if revese_order: - results.reverse() - - return results - - -def _changed_packages_activity_query() -> QActivity: - """Return an SQLAlchemy query for all changed package activities. - - Return a query for all activities with activity_type '*package', e.g. - 'new_package', 'changed_package', 'deleted_package'. - - """ - q = model.Session.query(Activity) - q = q.filter(Activity.activity_type.endswith("package")) - return q - - -def recently_changed_packages_activity_list( - limit: int, offset: int -) -> list[Activity]: - """Return the site-wide stream of recently changed package activities. - - This activity stream includes recent 'new package', 'changed package' and - 'deleted package' activities for the whole site. - - """ - q = _changed_packages_activity_query() - - q = _filter_activitites_from_users(q) - - return _activities_limit(q, limit, offset).all() - - -def _filter_activitites_from_users(q: QActivity) -> QActivity: - """ - Adds a filter to an existing query object to avoid activities from users - defined in :ref:`ckan.hide_activity_from_users` (defaults to the site user) - """ - users_to_avoid = _activity_stream_get_filtered_users() - if users_to_avoid: - # type_ignore_reason: incomplete SQLAlchemy types - q = q.filter(Activity.user_id.notin_(users_to_avoid)) # type: ignore - - return q - - -def _filter_activitites_from_type( - q: QActivity, types: list[str], include: bool = True -): - """Adds a filter to an existing query object to include or exclude - (include=False) activities based on a list of types. - - """ - if include: - q = q.filter(Activity.activity_type.in_(types)) # type: ignore - else: - q = q.filter(Activity.activity_type.notin_(types)) # type: ignore - return q - - -def _activity_stream_get_filtered_users() -> list[str]: - """ - Get the list of users from the :ref:`ckan.hide_activity_from_users` config - option and return a list of their ids. If the config is not specified, - returns the id of the site user. - """ - users_list = config.get("ckan.hide_activity_from_users") - if not users_list: - from ckan.logic import get_action - - context: Context = {"ignore_auth": True} - site_user = get_action("get_site_user")(context, {}) - users_list = [site_user.get("name")] - - return model.User.user_ids_for_name_or_id(users_list) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/plugin.py deleted file mode 100644 index fe610c0..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/plugin.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations - -from ckan.common import CKANConfig - -import ckan.plugins as p -import ckan.plugins.toolkit as tk - -from . import subscriptions - - -@tk.blanket.auth_functions -@tk.blanket.actions -@tk.blanket.helpers -@tk.blanket.blueprints -@tk.blanket.validators -class ActivityPlugin(p.SingletonPlugin): - p.implements(p.IConfigurer) - p.implements(p.ISignal) - - # IConfigurer - def update_config(self, config: CKANConfig): - tk.add_template_directory(config, "templates") - tk.add_public_directory(config, "public") - tk.add_resource("assets", "ckanext-activity") - - # ISignal - def get_signal_subscriptions(self): - return subscriptions.get_subscriptions() diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/public/dotted.png b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/public/dotted.png deleted file mode 100644 index fa0ae8040cbdf64a1a9c3214b062aa5c31a36c23..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 74 zcmeAS@N?(olHy`uVBq!ia0vp^OhC-S0V1_~EAxPqpr?ytNCji^4sJGsSN0MPnhd(h VVu9wjyxM`144$rjF6*2UngGv~4+;PP diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/subscriptions.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/subscriptions.py deleted file mode 100644 index acdcc7a..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/subscriptions.py +++ /dev/null @@ -1,177 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import annotations -import logging -from typing import Any - -import ckan.plugins.toolkit as tk -import ckan.lib.dictization as dictization - -from ckan import types -from .model import Activity - -log = logging.getLogger(__name__) - - -def get_subscriptions() -> types.SignalMapping: - return { - tk.signals.action_succeeded: [ - {"sender": "bulk_update_public", "receiver": bulk_changed}, - {"sender": "bulk_update_private", "receiver": bulk_changed}, - {"sender": "bulk_update_delete", "receiver": bulk_changed}, - {"sender": "package_create", "receiver": package_changed}, - {"sender": "package_update", "receiver": package_changed}, - {"sender": "package_delete", "receiver": package_changed}, - {"sender": "group_create", "receiver": group_or_org_changed}, - {"sender": "group_update", "receiver": group_or_org_changed}, - {"sender": "group_delete", "receiver": group_or_org_changed}, - { - "sender": "organization_create", - "receiver": group_or_org_changed, - }, - { - "sender": "organization_update", - "receiver": group_or_org_changed, - }, - { - "sender": "organization_delete", - "receiver": group_or_org_changed, - }, - {"sender": "user_create", "receiver": user_changed}, - {"sender": "user_update", "receiver": user_changed}, - ] - } - - -# action, context, data_dict, result -def bulk_changed(sender: str, **kwargs: Any): - for key in ("context", "data_dict"): - if key not in kwargs: - log.warning("Activity subscription ignored") - return - - context: types.Context = kwargs["context"] - datasets = kwargs["data_dict"].get("datasets") - model = context["model"] - - user = context["user"] - user_obj = model.User.get(user) - if user_obj: - user_id = user_obj.id - else: - user_id = "not logged in" - for dataset in datasets: - entity = model.Package.get(dataset) - assert entity - - activity = Activity.activity_stream_item(entity, "changed", user_id) - model.Session.add(activity) - - if not context.get("defer_commit"): - model.Session.commit() - - -# action, context, data_dict, result -def package_changed(sender: str, **kwargs: Any): - for key in ("result", "context", "data_dict"): - if key not in kwargs: - log.warning("Activity subscription ignored") - return - - type_ = "new" if sender == "package_create" else "changed" - - context: types.Context = kwargs["context"] - result: types.ActionResult.PackageUpdate = kwargs["result"] - data_dict = kwargs["data_dict"] - - if not result: - id_ = data_dict["id"] - elif isinstance(result, str): - id_ = result - else: - id_ = result["id"] - - pkg = context["model"].Package.get(id_) - assert pkg - - if pkg.private: - return - - user_obj = context["model"].User.get(context["user"]) - if user_obj: - user_id = user_obj.id - else: - user_id = "not logged in" - - activity = Activity.activity_stream_item(pkg, type_, user_id) - context["session"].add(activity) - if not context.get("defer_commit"): - context["session"].commit() - - -# action, context, data_dict, result -def group_or_org_changed(sender: str, **kwargs: Any): - for key in ("result", "context", "data_dict"): - if key not in kwargs: - log.warning("Activity subscription ignored") - return - - context: types.Context = kwargs["context"] - result: types.ActionResult.GroupUpdate = kwargs["result"] - data_dict = kwargs["data_dict"] - - group = context["model"].Group.get( - result["id"] if result else data_dict["id"] - ) - assert group - - type_, action = sender.split("_") - - user_obj = context["model"].User.get(context["user"]) - assert user_obj - - activity_dict: dict[str, Any] = { - "user_id": user_obj.id, - "object_id": group.id, - } - - if group.state == "deleted" or action == "delete": - activity_type = f"deleted {type_}" - elif action == "create": - activity_type = f"new {type_}" - else: - activity_type = f"changed {type_}" - - activity_dict["activity_type"] = activity_type - - activity_dict["data"] = { - "group": dictization.table_dictize(group, context) - } - activity_create_context = tk.fresh_context(context) - activity_create_context['ignore_auth'] = True - tk.get_action("activity_create")(activity_create_context, activity_dict) - - -# action, context, data_dict, result -def user_changed(sender: str, **kwargs: Any): - for key in ("result", "context", "data_dict"): - if key not in kwargs: - log.warning("Activity subscription ignored") - return - - context: types.Context = kwargs["context"] - result: types.ActionResult.UserUpdate = kwargs["result"] - - if sender == "user_create": - activity_type = "new user" - else: - activity_type = "changed user" - - activity_dict = { - "user_id": result["id"], - "object_id": result["id"], - "activity_type": activity_type, - } - activity_create_context = tk.fresh_context(context) - activity_create_context['ignore_auth'] = True - tk.get_action("activity_create")(activity_create_context, activity_dict) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/activity_streams/activity_stream_email_notifications.text b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/activity_streams/activity_stream_email_notifications.text deleted file mode 100644 index 5cafaae..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/activity_streams/activity_stream_email_notifications.text +++ /dev/null @@ -1,7 +0,0 @@ -{% set num = activities|length %}{{ ngettext("You have {num} new activity on your {site_title} dashboard", "You have {num} new activities on your {site_title} dashboard", num).format(site_title=g.site_title if g else site_title, num=num) }} {{ _('To view your dashboard, click on this link:') }} - -{% url_for 'activity.dashboard', _external=True %} - -{{ _('You can turn off these email notifications in your {site_title} preferences. To change your preferences, click on this link:').format(site_title=g.site_title if g else site_title) }} - -{% url_for 'user.edit', _external=True %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/ajax_snippets/dashboard.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/ajax_snippets/dashboard.html deleted file mode 100644 index c245c23..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/ajax_snippets/dashboard.html +++ /dev/null @@ -1,4 +0,0 @@ -{# Snippet for unit testing dashboard.js #} -
- {% snippet 'user/snippets/followee_dropdown.html', context={}, followees=[{"dict": {"id": 1}, "display_name": "Test followee" }, {"dict": {"id": 2}, "display_name": "Not valid" }] %} -
diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/base.html deleted file mode 100644 index be67977..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/base.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block styles %} - {{ super() }} - {% asset 'ckanext-activity/activity-css' %} -{% endblock %} \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/activity_stream.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/activity_stream.html deleted file mode 100644 index 0a7484e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/activity_stream.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "group/read_base.html" %} - -{% block subtitle %}{{ _('Activity Stream') }} {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block primary_content_inner %} - - {% if activity_types is defined %} - {% snippet 'snippets/activity_type_selector.html', id=id, activity_type=activity_type, activity_types=activity_types, blueprint='activity.group_activity' %} - {% endif %} - - {% snippet 'snippets/stream.html', activity_stream=activity_stream, id=id, object_type='group' %} - - {% snippet 'snippets/pagination.html', newer_activities_url=newer_activities_url, older_activities_url=older_activities_url %} - -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/changes.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/changes.html deleted file mode 100644 index 00b7307..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/changes.html +++ /dev/null @@ -1,64 +0,0 @@ -{% extends "group/read_base.html" %} - -{% block subtitle %}{{ group_dict.name }} {{ g.template_title_delimiter }} Changes {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
  • {% link_for _('Changes'), named_route='activity.group_activity', id=group_dict.name %}
    • -
  • {% link_for activity_diffs[0].activities[1].id|truncate(30), named_route='activity.group_changes', id=activity_diffs[0].activities[1].id %}
    • -{% endblock %} - -{% block primary %} -
    -
    - {% block group_changes_header %} -

    {{ _('Changes') }}

    - {% endblock %} - - {% set select_list1 = h.activity_list_select(group_activity_list, activity_diffs[-1].activities[0].id) %} - {% set select_list2 = h.activity_list_select(group_activity_list, activity_diffs[0].activities[1].id) %} -
    - - - View changes from - to - -
    - -
    - - {# iterate through the list of activity diffs #} -
    - {% for i in range(activity_diffs|length) %} - {% snippet "group/snippets/item_group.html", activity_diff=activity_diffs[i], group_dict=group_dict %} - - {# TODO: display metadata for more than most recent change #} - {% if i == 0 %} - {# button to show JSON metadata diff for the most recent change - not shown by default #} - - - {% endif %} - -
    - {% endfor %} -
    -
    -{% endblock %} - -{% block secondary %}{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/read_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/read_base.html deleted file mode 100644 index 03ce862..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/read_base.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block content_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('activity.group_activity', _('Activity Stream'), id=group_dict.name, offset=0, icon='clock') }} -{% endblock content_primary_nav %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/snippets/item_group.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/snippets/item_group.html deleted file mode 100644 index 915fabe..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/group/snippets/item_group.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ gettext('On %(timestamp)s, %(username)s:', timestamp=h.render_datetime(activity_diff.activities[1].timestamp, with_hours=True, with_seconds=True), username=h.linked_user(activity_diff.activities[1].user_id)) }} - -{% set changes = h.compare_group_dicts(activity_diff.activities[0].data.group, activity_diff.activities[1].data.group, activity_diff.activities[0].id) %} -
      - {% for change in changes %} - {% snippet "snippets/group_changes/{}.html".format( - change.type), change=change %} -
      - {% endfor %} -
    diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/header.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/header.html deleted file mode 100644 index 4bd0dff..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/header.html +++ /dev/null @@ -1,15 +0,0 @@ -{% ckan_extends %} - -{% block header_dashboard %} - {% set new_activities = h.new_activities() %} -
  • - {% set notifications_tooltip = ngettext('Dashboard (%(num)d new item)', 'Dashboard (%(num)d new items)', - new_activities) - %} - - - {{ _('Dashboard') }} - {{ new_activities }} - -
    • -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/activity_stream.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/activity_stream.html deleted file mode 100644 index c69b210..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/activity_stream.html +++ /dev/null @@ -1,14 +0,0 @@ -{% extends "organization/read_base.html" %} - -{% block subtitle %}{{ _('Activity Stream') }} {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block primary_content_inner %} - - {% if activity_types is defined %} - {% snippet 'snippets/activity_type_selector.html', id=id, activity_type=activity_type, activity_types=activity_types, blueprint='activity.organization_activity' %} - {% endif %} - - {% snippet 'snippets/stream.html', activity_stream=activity_stream, id=id, object_type='organization', group_type=group_type %} - - {% snippet 'snippets/pagination.html', newer_activities_url=newer_activities_url, older_activities_url=older_activities_url %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/changes.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/changes.html deleted file mode 100644 index 7a85b19..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/changes.html +++ /dev/null @@ -1,64 +0,0 @@ -{% extends "organization/read_base.html" %} - -{% block subtitle %}{{ group_dict.name }} {{ g.template_title_delimiter }} Changes {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
  • {% link_for _('Changes'), named_route='activity.organization_activity', id=group_dict.name %}
    • -
  • {% link_for activity_diffs[0].activities[1].id|truncate(30), named_route='activity.organization_changes', id=activity_diffs[0].activities[1].id %}
    • -{% endblock %} - -{% block primary %} -
    -
    - {% block organization_changes_header %} -

    {{ _('Changes') }}

    - {% endblock %} - - {% set select_list1 = h.activity_list_select(group_activity_list, activity_diffs[-1].activities[0].id) %} - {% set select_list2 = h.activity_list_select(group_activity_list, activity_diffs[0].activities[1].id) %} -
    - - - View changes from - to - -
    - -
    - - {# iterate through the list of activity diffs #} -
    - {% for i in range(activity_diffs|length) %} - {% snippet "organization/snippets/item_organization.html", activity_diff=activity_diffs[i], group_dict=group_dict %} - - {# TODO: display metadata for more than most recent change #} - {% if i == 0 %} - {# button to show JSON metadata diff for the most recent change - not shown by default #} - - - {% endif %} - -
    - {% endfor %} -
    -
    -{% endblock %} - -{% block secondary %}{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/read_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/read_base.html deleted file mode 100644 index 2aa814f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/read_base.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block content_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('activity.organization_activity', _('Activity Stream'), id=group_dict.name, offset=0, icon='clock') }} -{% endblock content_primary_nav %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/snippets/item_organization.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/snippets/item_organization.html deleted file mode 100644 index 5e2ee10..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/organization/snippets/item_organization.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ gettext('On %(timestamp)s, %(username)s:', timestamp=h.render_datetime(activity_diff.activities[1].timestamp, with_hours=True, with_seconds=True), username=h.linked_user(activity_diff.activities[1].user_id)) }} - -{% set changes = h.compare_group_dicts(activity_diff.activities[0].data.group, activity_diff.activities[1].data.group, activity_diff.activities[0].id) %} -
      - {% for change in changes %} - {% snippet "snippets/organization_changes/{}.html".format( - change.type), change=change %} -
      - {% endfor %} -
    diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/activity_stream.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/activity_stream.html deleted file mode 100644 index 40e663e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/activity_stream.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "package/read_base.html" %} - -{% block subtitle %}{{ _('Activity Stream') }} {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block primary_content_inner %} - {% if activity_types is defined %} - {% snippet 'snippets/activity_type_selector.html', id=id, activity_type=activity_type, activity_types=activity_types, blueprint='dataset.activity' %} - {% endif %} - - {% if activity_stream|length > 0 %} - {% snippet 'snippets/stream.html', activity_stream=activity_stream, id=id, object_type='package' %} - {% else %} -

    - {% if activity_type %} - {{ _('No activity found for this type') }} - {% else %} - {{ _('No activity found') }}. - {% endif %} -

    - {% endif %} - - {% snippet 'snippets/pagination.html', newer_activities_url=newer_activities_url, older_activities_url=older_activities_url %} - -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/changes.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/changes.html deleted file mode 100644 index 51052be..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/changes.html +++ /dev/null @@ -1,64 +0,0 @@ -{% extends "package/base.html" %} - -{% block subtitle %}{{ pkg_dict.name }} {{ g.template_title_delimiter }} Changes {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
  • {% link_for _('Changes'), named_route='activity.package_activity', id=pkg_dict.name %}
    • -
  • {% link_for activity_diffs[0].activities[1].id|truncate(30), named_route='activity.package_changes', id=activity_diffs[0].activities[1].id %}
    • -{% endblock %} - -{% block primary %} -
    -
    - {% block package_changes_header %} -

    {{ _('Changes') }}

    - {% endblock %} - - {% set select_list1 = h.activity_list_select(pkg_activity_list, activity_diffs[-1].activities[0].id) %} - {% set select_list2 = h.activity_list_select(pkg_activity_list, activity_diffs[0].activities[1].id) %} -
    - - - View changes from - to - -
    - -
    - - {# iterate through the list of activity diffs #} -
    - {% for i in range(activity_diffs|length) %} - {% snippet "package/snippets/change_item.html", activity_diff=activity_diffs[i], pkg_dict=pkg_dict %} - - {# TODO: display metadata for more than most recent change #} - {% if i == 0 %} - {# button to show JSON metadata diff for the most recent change - not shown by default #} - - - {% endif %} - -
    - {% endfor %} -
    -
    -{% endblock %} - -{% block secondary %}{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/history.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/history.html deleted file mode 100644 index 8a29866..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/history.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "package/read.html" %} - -{% block package_description %} - {% block package_archive_notice %} -
    - {% trans url=h.url_for(pkg.type ~ '.read', id=pkg.id) %} - You're currently viewing an old version of this dataset. - Data files may not match the old version of the metadata. - View the current version. - {% endtrans %} -
    - {% endblock %} - - {{ super() }} -{% endblock package_description %} - - -{% block package_resources %} - {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources, is_activity_archive=true %} -{% endblock %} - -{% block content_action %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/read_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/read_base.html deleted file mode 100644 index c0ec714..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/read_base.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block content_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('activity.package_activity', _('Activity Stream'), id=pkg.id if is_activity_archive else pkg.name, icon='clock') }} -{% endblock content_primary_nav %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/resource_history.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/resource_history.html deleted file mode 100644 index aeee719..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/resource_history.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "package/resource_read.html" %} - -{% block action_manage %} -{% endblock action_manage %} - - -{% block resource_content %} - {% block package_archive_notice %} -
    - {% trans url=h.url_for(pkg.type ~ '.read', id=pkg.id) %} - You're currently viewing an old version of this dataset. - Data files may not match the old version of the metadata. - View the current version. - {% endtrans %} -
    - {% endblock %} - {{ super() }} -{% endblock %} - -{% block data_preview %} -{% endblock %} - - -{% block resources_list %} - {% snippet "package/snippets/resources.html", pkg=pkg, active=res.id, action='read', is_activity_archive=true %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/change_item.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/change_item.html deleted file mode 100644 index df7aacf..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/change_item.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ gettext('On %(timestamp)s, %(username)s:', timestamp=h.render_datetime(activity_diff.activities[1].timestamp, with_hours=True, with_seconds=True), username=h.linked_user(activity_diff.activities[1].user_id)) }} - -{% set changes = h.compare_pkg_dicts(activity_diff.activities[0].data.package, activity_diff.activities[1].data.package, activity_diff.activities[0].id) %} -
      - {% for change in changes %} - {% snippet "snippets/changes/{}.html".format( - change.type), change=change, pkg_dict=pkg_dict %} -
      - {% endfor %} -
    diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resource_item.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resource_item.html deleted file mode 100644 index b1fed10..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resource_item.html +++ /dev/null @@ -1,16 +0,0 @@ -{% ckan_extends %} - -{% block explore_view %} - {% if is_activity_archive %} -
  • - - - {{ _('More information') }} - -
    • - {% else %} - {{ super() }} - {% endif %} - - -{% endblock explore_view %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources.html deleted file mode 100644 index 08a9835..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources.html +++ /dev/null @@ -1,12 +0,0 @@ -{% ckan_extends %} - - -{% block resources_list %} - -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources_list.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources_list.html deleted file mode 100644 index 40afa88..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/package/snippets/resources_list.html +++ /dev/null @@ -1,15 +0,0 @@ -{% ckan_extends %} - -{% block resource_list_inner %} - {% for resource in resources %} - {% if is_activity_archive %} - {% set url = h.url_for("activity.resource_history", id=pkg.id, resource_id=resource.id, activity_id=request.view_args.activity_id) %} - {% endif %} - {% snippet 'package/snippets/resource_item.html', pkg=pkg, res=resource, can_edit=false if is_activity_archive else can_edit, is_activity_archive=is_activity_archive, url=url %} - {% endfor %} -{% endblock %} - - -{% block resource_list_empty %} -

    {{ _('This dataset has no data') }}

    -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/page.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/page.html deleted file mode 100644 index 471757b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/page.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block scripts %} - {{ super() }} - {% asset "ckanext-activity/activity" %} -{% endblock scripts %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/added_tag.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/added_tag.html deleted file mode 100644 index ad07377..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/added_tag.html +++ /dev/null @@ -1,15 +0,0 @@ -
  • - - - - - {{ _('{actor} added the tag {tag} to the dataset {dataset}').format( - actor=ah.actor(activity), - dataset=ah.dataset(activity), - tag=ah.tag(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_group.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_group.html deleted file mode 100644 index 13f0cb7..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_group.html +++ /dev/null @@ -1,20 +0,0 @@ -
  • - - - - - {{ _('{actor} updated the group {group}').format( - actor=ah.actor(activity), - group=ah.group(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - {% if can_show_activity_detail %} -  |  - - {{ _('Changes') }} - - {% endif %} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_organization.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_organization.html deleted file mode 100644 index e63a9bb..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_organization.html +++ /dev/null @@ -1,20 +0,0 @@ -
  • - - - - - {{ _('{actor} updated the organization {organization}').format( - actor=ah.actor(activity), - organization=ah.organization(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - {% if can_show_activity_detail %} -  |  - - {{ _('Changes') }} - - {% endif %} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_package.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_package.html deleted file mode 100644 index f8eee1e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_package.html +++ /dev/null @@ -1,26 +0,0 @@ -{% set dataset_type = activity.data.package.type or 'dataset' %} -
  • - - - - - {{ _('{actor} updated the {dataset_type} {dataset}').format( - actor=ah.actor(activity), - dataset_type=h.humanize_entity_type('package', dataset_type, 'activity_record') or _('dataset'), - dataset=ah.dataset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - {% if can_show_activity_detail %} -  |  - - {{ _('View this version') }} - -  |  - - {{ _('Changes') }} - - {% endif %} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_resource.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_resource.html deleted file mode 100644 index 997f76b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_resource.html +++ /dev/null @@ -1,15 +0,0 @@ -
  • - - - - - {{ _('{actor} updated the resource {resource} in the dataset {dataset}').format( - actor=ah.actor(activity), - resource=ah.resource(activity), - dataset=ah.datset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_user.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_user.html deleted file mode 100644 index 6bc2b0d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/changed_user.html +++ /dev/null @@ -1,13 +0,0 @@ -
  • - - - - - {{ _('{actor} updated their profile').format( - actor=ah.actor(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_group.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_group.html deleted file mode 100644 index 517ce73..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_group.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} deleted the group {group}').format( - actor=ah.actor(activity), - group=ah.group(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_organization.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_organization.html deleted file mode 100644 index 8b09cf2..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_organization.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} deleted the organization {organization}').format( - actor=ah.actor(activity), - organization=ah.organization(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_package.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_package.html deleted file mode 100644 index 0a733e6..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_package.html +++ /dev/null @@ -1,17 +0,0 @@ -{% set dataset_type = activity.data.package.type or 'dataset' %} - -
  • - - - - - {{ _('{actor} deleted the {dataset_type} {dataset}').format( - actor=ah.actor(activity), - dataset_type=h.humanize_entity_type('package', dataset_type, 'activity_record') or _('dataset'), - dataset=ah.dataset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_resource.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_resource.html deleted file mode 100644 index 7334e58..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/deleted_resource.html +++ /dev/null @@ -1,15 +0,0 @@ -
  • - - - - - {{ _('{actor} deleted the resource {resource} from the dataset {dataset}').format( - actor=ah.actor(activity), - resource=ah.resource(activity), - dataset=ah.dataset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/fallback.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/fallback.html deleted file mode 100644 index abaef61..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/fallback.html +++ /dev/null @@ -1,38 +0,0 @@ -{# - Fallback template for displaying an activity. - It's not pretty, but it is better than TemplateNotFound. - - Params: - activity - the Activity dict - can_show_activity_detail - whether we should render detail about the activity (i.e. "as it was" and diff, alternatively will just display the metadata about the activity) - id - the id or current name of the object (e.g. package name, user id) - ah - dict of template macros to render linked: actor, dataset, organization, user, group -#} -
  • - - - - - {{ _('{actor} {activity_type}').format( - actor=ah.actor(activity), - activity_type=activity.activity_type - )|safe }} - {% if activity.data.package %} - {{ ah.dataset(activity) }} - {% endif %} - {% if activity.data.package %} - {{ ah.dataset(activity) }} - {% endif %} - {% if activity.data.group %} - {# do our best to differentiate between org & group #} - {% if 'group' in activity.activity_type %} - {{ ah.group(activity) }} - {% else %} - {{ ah.organization(activity) }} - {% endif %} - {% endif %} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_dataset.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_dataset.html deleted file mode 100644 index 8bd58f0..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_dataset.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} started following {dataset}').format( - actor=ah.actor(activity), - dataset=ah.dataset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_group.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_group.html deleted file mode 100644 index d26a701..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_group.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} started following {group}').format( - actor=ah.actor(activity), - group=ah.group(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_user.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_user.html deleted file mode 100644 index 914be5b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/follow_user.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} started following {user}').format( - actor=ah.actor(activity), - user=ah.user(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_group.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_group.html deleted file mode 100644 index ffa9ab3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_group.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} created the group {group}').format( - actor=ah.actor(activity), - group=ah.group(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_organization.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_organization.html deleted file mode 100644 index 2dc2a93..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_organization.html +++ /dev/null @@ -1,14 +0,0 @@ -
  • - - - - - {{ _('{actor} created the organization {organization}').format( - actor=ah.actor(activity), - organization=ah.organization(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_package.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_package.html deleted file mode 100644 index b607ce8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_package.html +++ /dev/null @@ -1,22 +0,0 @@ -{% set dataset_type = activity.data.package.type or 'dataset' %} -
  • - - - - - {{ _('{actor} created the {dataset_type} {dataset}').format( - actor=ah.actor(activity), - dataset_type=h.humanize_entity_type('package', dataset_type,'activity_record') or _('dataset'), - dataset=ah.dataset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - {% if can_show_activity_detail %} -  |  - - {{ _('View this version') }} - - {% endif %} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_resource.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_resource.html deleted file mode 100644 index 057089e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_resource.html +++ /dev/null @@ -1,15 +0,0 @@ -
  • - - - - - {{ _('{actor} added the resource {resource} to the dataset {dataset}').format( - actor=ah.actor(activity), - resource=ah.resource(activity), - dataset=ah.dataset(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_user.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_user.html deleted file mode 100644 index 25bae1a..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/new_user.html +++ /dev/null @@ -1,13 +0,0 @@ -
  • - - - - - {{ _('{actor} signed up').format( - actor=ah.actor(activity) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/removed_tag.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/removed_tag.html deleted file mode 100644 index 992fcb8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activities/removed_tag.html +++ /dev/null @@ -1,15 +0,0 @@ -
  • - - - - - {{ _('{actor} removed the tag {tag} from the dataset {dataset}').format( - actor=ah.actor(activity), - tag=ah.tag(activity), - dataset=ah.dataset(dataset) - )|safe }} -
    - - {{ h.time_ago_from_timestamp(activity.timestamp) }} - -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activity_type_selector.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activity_type_selector.html deleted file mode 100644 index ba2d5e0..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/activity_type_selector.html +++ /dev/null @@ -1,25 +0,0 @@ -{# - Renders a select element to filter the activity stream - - It uses the activity-stream.js module to dinamically request for a new URl upon selection. - - id - the id or current name of the object (e.g. package name, user id) - activity_type - the current selected activity type - activity_types - the list of activity types the user can filter on - blueprint - blueprint to call when selecting an activity type to filter by (eg: dataset.activity) -#} - -
    - - -
    diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author.html deleted file mode 100644 index 41627bd..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Set author of {pkg_link} to {new_author} (previously {old_author})').format( - pkg_link=h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_author = change.new_author, - old_author = change.old_author - ) }} - - {% elif change.method == "add" %} - - {{ _('Set author of {pkg_link} to {new_author}').format( - pkg_link=h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_author = change.new_author - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed author from {pkg_link}').format( - pkg_link=h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id) - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author_email.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author_email.html deleted file mode 100644 index edc4403..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/author_email.html +++ /dev/null @@ -1,47 +0,0 @@ -
  • -

    - {% set pkg_link = (h.url_for('dataset.read', id=change.pkg_id))%} - {% if change.method == "change" %} - - {{ _('Set author email of ')}} - - {{ change.title }} - - - {{ _('to') }} - - {{change.new_author_email}} - - - {{_('( previously ')}} - - {{change.old_author_email}} - - {{ _(')') }} - - {% elif change.method == "add" %} - - {{_('Set author email of')}} - - {{change.title}} - - - {{ _('to') }} - - {{ change.new_author_email }} - - - {% elif change.method == "remove" %} - - {{ _('Removed author email from')}} - - {{change.title}} - - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/delete_resource.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/delete_resource.html deleted file mode 100644 index 6bf96b3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/delete_resource.html +++ /dev/null @@ -1,10 +0,0 @@ -
  • -

    - {{ _('Deleted resource {resource_link} from {pkg_link}').format( - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', qualified=True, id=change.pkg_id, - resource_id = change.resource_id, activity_id=change.old_activity_id), - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extension_fields.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extension_fields.html deleted file mode 100644 index af1429e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extension_fields.html +++ /dev/null @@ -1,9 +0,0 @@ -
  • -

    - {{ _('Changed value of field {key} to {value} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - key = change.key, - value = change.value - )}} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extra_fields.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extra_fields.html deleted file mode 100644 index e8bdd73..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/extra_fields.html +++ /dev/null @@ -1,85 +0,0 @@ -
  • -

    - {% if change.method == "add_one_value" %} - - {{ _('Added field {key} with value {value} to {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - key = change.key, - value = change.value - )}} - - {% elif change.method == "add_one_no_value" %} - - {{ _('Added field {key} to {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - key = change.key - )}} - - {% elif change.method == "add_multiple" %} - - {{ _('Added the following fields to {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id) - )}} -

      - {% for item in change.key_list %} -
    • - {% if change.value_list[item] != "" %} - {{ _('{key} with value {value}').format( - key = item, - value = change.value_list[item] - )|safe }} - {% else %} - {{ _('{key}').format( - key = item - )|safe }} - {% endif %} -
      • - {% endfor %} -
    - - {% elif change.method == "change_with_old_value" %} - - {{ _('Changed value of field {key} to {new_val} (previously {old_val}) in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - key = change.key, - new_val = change.new_value, - old_val = change.old_value - )}} - - {% elif change.method == "change_no_old_value" %} - - {{ _('Changed value of field {key} to {new_val} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - key = change.key, - new_val = change.new_value, - ) }} - - {% elif change.method == "remove_one" %} - - {{ _('Removed field {key} from {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - key = change.key, - ) }} - - {% elif change.method == "remove_multiple" %} - - {{ _('Removed the following fields from {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id) - ) }} -
      - {% for item in change.key_list %} -
    • - {{ _('{key}').format( - key = item - )| safe }} -
      • - {% endfor %} -
    - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/license.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/license.html deleted file mode 100644 index 923d415..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/license.html +++ /dev/null @@ -1,67 +0,0 @@ -
  • -

    - {% set pkg_link = (h.url_for('dataset.read', id=change.pkg_id)) %} - {# if both of them have URLs #} - {% if change.new_url != "" and change.old_url != "" %} - - {{ _('Changed the license of ') }} - - {{ change.title }} - - - {{ _('to') }} - - {{change.new_title}} - - - {{ _('( previously') }} - - {{change.old_title}} - - {{_(')')}} - - {# if only the new one has a URL #} - {% elif change.new_url != "" and change.old_url == "" %} - - {{ _('Changed the license of') }} - - {{change.title}} - - - {{ _('to') }} - - {{change.new_title}} - - - {{ _('(previously') + change.old_title + ' )'}} - - {# if only the old one has a URL #} - {% elif change.new_url == "" and change.old_url != "" %} - - {{ _('Changed the license of') }} - - {{change.title}} - - - {{ _('to') + change.new_title }} - - {{ _('( previously') }} - - {{change.old_title}} - - {{_(')')}} - - {# otherwise neither has a URL #} - {% else %} - - {{ _('Changed the license of') }} - - {{change.title}} - - - {{ _('to') + change.new_title}} - {{ _('(previously') + change.old_title + _(')')|safe }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer.html deleted file mode 100644 index 6a4ae58..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Set maintainer of {pkg_link} to {new_maintainer} (previously {old_maintainer})').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_maintainer = change.new_maintainer, - old_maintainer = change.old_maintainer - ) }} - - {% elif change.method == "add" %} - - {{ _('Set maintainer of {pkg_link} to {new_maintainer}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_maintainer = change.new_maintainer - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed maintainer from {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer_email.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer_email.html deleted file mode 100644 index a090cee..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/maintainer_email.html +++ /dev/null @@ -1,47 +0,0 @@ -
  • -

    - {% set pkg_link = (h.url_for('dataset.read', id=change.pkg_id)) %} - {% if change.method == "change" %} - - {{ _('Set maintainer email of') }} - - {{change.title}} - - - {{ _('to') }} - - {{change.new_maintainer_email}} - - - {{ _('(previously') }} - - {{change.old_maintainer_email}} - - {{ _(')') }} - - {% elif change.method == "add" %} - - {{ _('Set maintainer email of') }} - - {{change.title}} - - - {{ _('to') }} - - {{change.new_maintainer_email}} - - - {% elif change.method == "remove" %} - - {{ _('Removed maintainer email from') }} - - {{change.title}} - - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/name.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/name.html deleted file mode 100644 index 3ff43bd..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/name.html +++ /dev/null @@ -1,23 +0,0 @@ -
  • -

    - {% set old_url = h.url_for('dataset.read', qualified=True, id=change.old_name) %} - {% set new_url = h.url_for('dataset.read', qualified=True, id=change.new_name) %} - - {{ _('Moved')}} - - {{change.title}} - - - {{ _('from') }} - - {{old_url}} - - - {{ _('to') }} - - {{new_url}} - - - -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_file.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_file.html deleted file mode 100644 index 285d750..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_file.html +++ /dev/null @@ -1,10 +0,0 @@ -
  • -

    - {{ _('Uploaded a new file to resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True) - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_resource.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_resource.html deleted file mode 100644 index 0d29a68..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/new_resource.html +++ /dev/null @@ -1,17 +0,0 @@ -{% set dataset_type = request.view_args.package_type or pkg_dict['type'] or 'dataset' %} -{% set pkg_url = h.url_for(dataset_type ~ '.read', id=change.pkg_id) %} -{% set resource_url = h.url_for(dataset_type ~ '_resource.read', id=change.pkg_id, resource_id = change.resource_id, qualified=True) %} - -{% set pkg_link %} -{{ change.title }} -{% endset %} - -{% set resource_link %} -{{ change.resource_name or _('Unnamed resource') }} -{% endset %} - -
  • -

    - {{ _('Added resource {resource_link} to {pkg_link}').format(pkg_link=pkg_link, resource_link=resource_link) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/no_change.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/no_change.html deleted file mode 100644 index 0d1ae06..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/no_change.html +++ /dev/null @@ -1,5 +0,0 @@ -
  • -

    - {{ _('No fields were updated. See the metadata diff for more details.') }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/notes.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/notes.html deleted file mode 100644 index 693da43..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/notes.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Updated description of {pkg_link} from

    {old_notes}
    to
    {new_notes}
    ').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - old_notes = change.old_notes, - new_notes = change.new_notes - ) }} - - {% elif change.method == "add" %} - - {{ _('Updated description of {pkg_link} to
    {new_notes}
    ').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_notes = change.new_notes - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed description from {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id) - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/org.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/org.html deleted file mode 100644 index f215b30..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/org.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Moved {pkg_link} from organization {old_org_link} to organization {new_org_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - old_org_link = h.nav_link(change.old_org_title, named_route='organization.read', id=change.old_org_id), - new_org_link = h.nav_link(change.new_org_title, named_route='organization.read', id=change.new_org_id) - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed {pkg_link} from organization {old_org_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - old_org_link = h.nav_link(change.old_org_title, named_route='organization.read', id=change.old_org_id) - ) }} - - {% elif change.method == "add" %} - - {{ _('Added {pkg_link} to organization {new_org_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_org_link = h.nav_link(change.new_org_title, named_route='organization.read', id=change.new_org_id) - ) }} - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/private.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/private.html deleted file mode 100644 index 377357a..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/private.html +++ /dev/null @@ -1,8 +0,0 @@ -
  • -

    - {{ _('Set visibility of {pkg_link} to {visibility}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - visibility = change.new - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_desc.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_desc.html deleted file mode 100644 index 62dc3c5..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_desc.html +++ /dev/null @@ -1,39 +0,0 @@ -
  • -

    - {% if change.method == "add" %} - - {{ _('Updated description of resource {resource_link} in {pkg_link} to

    {new_desc}
    ').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - new_desc = change.new_desc - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed description from resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True) - )}} - - {% elif change.method == "change" %} - - {{ _('Updated description of resource {resource_link} in {pkg_link} from
    {old_desc}
    to
    {new_desc}
    ').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - new_desc = change.new_desc, - old_desc = change.old_desc - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_extras.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_extras.html deleted file mode 100644 index 8423361..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_extras.html +++ /dev/null @@ -1,112 +0,0 @@ -
  • -

    - {% if change.method == "add_one_value" %} - - {{ _('Added field {key} with value {value} to resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - key = change.key, - value = change.value - ) }} - - {% elif change.method == "add_one_no_value" %} - - {{ _('Added field {key} to resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - key = change.key, - ) }} - - {% elif change.method == "add_multiple" %} - - {{ _('Added the following fields to resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True) - ) }} -

      - {% for item in change.key_list %} - {% if change.value_list[item] != "" %} - {{ _('{key} with value {value}').format( - key = item, - value = change.value_list[item] - )|safe }} - {% else %} - {{ _('{key}').format( - key = item - )|safe }} - {% endif %} - {% endfor %} -
    - - {% elif change.method == "remove_one" %} - - {{ _('Removed field {key} from resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - key = change.key - ) }} - - {% elif change.method == "remove_multiple" %} - - {{ _('Removed the following fields from resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True) - ) }} -
      - {% for item in change.key_list %} - {{ _('{key}').format( - key = item - )|safe }} - {% endfor %} -
    - - {% elif change.method == "change_value_with_old" %} - - {{ _('Changed value of field {key} of resource {resource_link} to {new_val} (previously {old_val}) in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - key = change.key, - new_val = change.new_value, - old_val = change.old_value - ) }} - - {% elif change.method == "change_value_no_old" %} - - {{ _('Changed value of field {key} to {new_val} in resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - key = change.key, - new_val = change.new_value - ) }} - - {% elif change.method == "change_value_no_new" %} - - {{ _('Removed the value of field {key} in resource {resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - resource_link = h.nav_link( - change.resource_name, named_route='resource.read', id=change.pkg_id, - resource_id=change.resource_id, qualified=True), - key = change.key, - new_val = change.new_value - ) }} - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_format.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_format.html deleted file mode 100644 index d7ac855..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_format.html +++ /dev/null @@ -1,54 +0,0 @@ -
  • -

    - {% set format_search_base_url = ( - h.url_for("organization.read", id=change.org_id) - if change.org_id else - h.url_for("dataset.search")) %} - {% set resource_url = (h.url_for( - "resource.read", - id=change.pkg_id, - resource_id=change.resource_id, - qualified=True)) %} - - {% if change.method == "add" %} - - {{ _('Set format of resource') }} - - {{change.resource_name}} - - - {{ _('to') }} - - {{change.format}} - - - {{ _('in') }} - - {{change.title}} - - - {% elif change.method == "change" %} - - {{ _('Set format of resource') }} - - {{change.resource_name}} - - - {{ _('to') }} - - {{change.new_format}} - - - {{ _('(previously') }} - - {{change.old_format}} - - {{ _(')') }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_name.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_name.html deleted file mode 100644 index a98a6be..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/resource_name.html +++ /dev/null @@ -1,13 +0,0 @@ -
  • -

    - {{ _('Renamed resource {old_resource_link} to {new_resource_link} in {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - old_resource_link = h.nav_link( - change.old_resource_name, named_route='resource.read', id=change.old_pkg_id, - resource_id=change.resource_id, activity_id=change.old_activity_id), - new_resource_link = h.nav_link( - change.new_resource_name, named_route='resource.read', id=change.new_pkg_id, - resource_id=change.resource_id), - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/tags.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/tags.html deleted file mode 100644 index 902a02a..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/tags.html +++ /dev/null @@ -1,59 +0,0 @@ -
  • -

    - {% if change.method == "remove_one" %} - - {{ _('Removed tag {tag_link} from {pkg_link}').format( - tag_link = '{tag}'.format( - tag_url = h.url_for('dataset.search') + - "?tags=" + change.tag, - tag = change.tag - )|safe, - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - ) }} - - {% elif change.method == "remove_multiple" %} - - {{ _('Removed the following tags from {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - )|safe }} - -

      - {% for item in change.tags %} -
    • - {{ _('{tag_link}').format( - tag_link = h.nav_link(item, named_route='dataset.search', tags=item), - )}} -
      • - {% endfor %} -
    - - {% elif change.method == "add_one" %} - - {{ _('Added tag {tag_link} to {pkg_link}').format( - tag_link = h.nav_link(change.tag, named_route='dataset.search', tags=change.tag), - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - ) }} - - {% elif change.method == "add_multiple" %} - - {{ _('Added the following tags to {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - ) }} -
      - {% for item in change.tags %} -
    • - {# TODO: figure out which controller to actually use here #} - {{ _('{tag_link}').format( - tag_link = h.nav_link(item, named_route='dataset.search', tags=item), - )}} -
      • - {% endfor %} -
    - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/title.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/title.html deleted file mode 100644 index 29623bb..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/title.html +++ /dev/null @@ -1,8 +0,0 @@ -
  • -

    - {{ _('Changed title to {title_link} (previously {old_title})').format( - title_link = h.nav_link(change.new_title, named_route='dataset.read', id=change.id), - old_title = change.old_title - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/url.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/url.html deleted file mode 100644 index c194cd0..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/url.html +++ /dev/null @@ -1,46 +0,0 @@ -
  • -

    - {% set pkg_link = (h.url_for('dataset.read', id=change.pkg_id)) %} - {% if change.method == "change" %} - - {{ _('Changed the source URL of') }} - - {{change.title}} - - - {{ _('from') }} - - {{change.old_url}} - - - {{ _('to') }} - - {{change.new_url}} - - - {% elif change.method == "remove" %} - - {{ _('Removed the source URL from') }} - - {{change.title}} - - - {% elif change.method == "add" %} - - {{_('Changed the source URL of') }} - - {{change.title}} - - - {{ _('to') }} - - {{change.new_url}} - - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/version.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/version.html deleted file mode 100644 index 2c93434..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/changes/version.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Changed the version of {pkg_link} to {new_version} (previously {old_version})').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - old_version = change.old_version, - new_version = change.new_version - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed the version from {pkg_link}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - ) }} - - {% elif change.method == "add" %} - - {{ _('Changed the version of {pkg_link} to {new_version}').format( - pkg_link = h.nav_link(change.title, named_route='dataset.read', id=change.pkg_id), - new_version = change.new_version - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/description.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/description.html deleted file mode 100644 index 9210b6a..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/description.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Updated description of {group_link} from

    {old_description}
    to
    {new_description}
    ').format( - group_link = h.nav_link(change.title, named_route='group.read', id=change.pkg_id), - old_description = change.old_description, - new_description = change.new_description - ) }} - - {% elif change.method == "add" %} - - {{ _('Updated description of {group_link} to
    {new_description}
    ').format( - group_link = h.nav_link(change.title, named_route='group.read', id=change.pkg_id), - new_description = change.new_description - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed description from {group_link}').format( - group_link = h.nav_link(change.title, named_route='group.read', id=change.pkg_id) - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/image_url.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/image_url.html deleted file mode 100644 index b74aabf..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/image_url.html +++ /dev/null @@ -1,46 +0,0 @@ -
  • -

    - {% set group_link = (h.url_for('group.read', id=change.pkg_id)) %} - {% if change.method == "change" %} - - {{ _('Changed the image URL of') }} - - {{change.title}} - - - {{ _('from') }} - - {{change.old_image_url}} - - - {{ _('to') }} - - {{change.new_image_url}} - - - {% elif change.method == "remove" %} - - {{ _('Removed the image URL from') }} - - {{change.title}} - - - {% elif change.method == "add" %} - - {{ _('Changed the image URL of') }} - - {{change.title}} - - - {{ _('to') }} - - {{change.new_image_url}} - - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/no_change.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/no_change.html deleted file mode 100644 index 0d1ae06..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/no_change.html +++ /dev/null @@ -1,5 +0,0 @@ -
  • -

    - {{ _('No fields were updated. See the metadata diff for more details.') }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/title.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/title.html deleted file mode 100644 index dc3bbdc..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/group_changes/title.html +++ /dev/null @@ -1,8 +0,0 @@ -
  • -

    - {{ _('Changed title to {title_link} (previously {old_title})').format( - title_link = h.nav_link(change.new_title, named_route='group.read', id=change.id), - old_title = change.old_title - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/description.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/description.html deleted file mode 100644 index 137f186..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/description.html +++ /dev/null @@ -1,30 +0,0 @@ -
  • -

    - {% if change.method == "change" %} - - {{ _('Updated description of {org_link} from

    {old_description}
    to
    {new_description}
    ').format( - org_link = h.nav_link(change.title, named_route='organization.read', id=change.pkg_id), - old_description = change.old_description, - new_description = change.new_description - ) }} - - {% elif change.method == "add" %} - - {{ _('Updated description of {org_link} to
    {new_description}
    ').format( - org_link = h.nav_link(change.title, named_route='organization.read', id=change.pkg_id), - new_description = change.new_description - ) }} - - {% elif change.method == "remove" %} - - {{ _('Removed description from {org_link}').format( - org_link = h.nav_link(change.title, named_route='organization.read', id=change.pkg_id) - ) }} - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/image_url.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/image_url.html deleted file mode 100644 index 5b9ce21..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/image_url.html +++ /dev/null @@ -1,46 +0,0 @@ -
  • -

    - {% set org_link = (h.url_for('organization.read', id=change.pkg_id)) %} - {% if change.method == "change" %} - - {{ _('Changed the image URL of') }} - - {{change.title}} - - - {{ _('from') }} - - {{change.old_image_url}} - - - {{ _('to') }} - - {{change.new_image_url}} - - - {% elif change.method == "remove" %} - - {{ _('Removed the image URL from') }} - - {{change.title}} - - - {% elif change.method == "add" %} - - {{ _('Changed the image URL of') }} - - {{change.title}} - - - {{ _('to') }} - - {{change.new_image_url}} - - - {% else %} - - {{ _('No fields were updated. See the metadata diff for more details.') }} - - {% endif %} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/no_change.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/no_change.html deleted file mode 100644 index 0d1ae06..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/no_change.html +++ /dev/null @@ -1,5 +0,0 @@ -
  • -

    - {{ _('No fields were updated. See the metadata diff for more details.') }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/title.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/title.html deleted file mode 100644 index 444e142..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/organization_changes/title.html +++ /dev/null @@ -1,8 +0,0 @@ -
  • -

    - {{ _('Changed title to {title_link} (previously {old_title})').format( - title_link = h.nav_link(change.new_title, named_route='organization.read', id=change.id), - old_title = change.old_title - ) }} -

    -
    • diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/pagination.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/pagination.html deleted file mode 100644 index 721daf1..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/pagination.html +++ /dev/null @@ -1,7 +0,0 @@ -{% set class_prev = "btn btn-default" if newer_activities_url else "btn disabled" %} -{% set class_next = "btn btn-default" if older_activities_url else "btn disabled" %} - -
    - {{ _('Newer activities') }} - {{ _('Older activities') }} -
    \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/stream.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/stream.html deleted file mode 100644 index 40dd2b4..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/snippets/stream.html +++ /dev/null @@ -1,63 +0,0 @@ -{% macro actor(activity) %} - - {{ h.linked_user(activity.user_id, 0, 30) }} - -{% endmacro %} - -{% macro dataset(activity) %} - {% set dataset_type = activity.data.package.type or 'dataset' %} - - - {{ activity.data.package.title if activity.data.package else _("unknown") }} - - -{% endmacro %} - -{% macro organization(activity) %} - {% set group_type = group_type or (activity.data.group.type if (activity.data.group and activity.data.group.type) else 'organization') %} - - {{ activity.data.group.title if activity.data.group else _('unknown') }} - -{% endmacro %} - -{% macro user(activity) %} - - {{ h.linked_user(activity.object_id, 0, 20) }} - -{% endmacro %} - -{% macro group(activity) %} - - - {{ activity.data.group.title if activity.data.group else _('unknown') }} - - -{% endmacro %} - -{# Renders the actual stream of activities - -activity_stream - the activity data. e.g. the output from package_activity_list -id - the id or current name of the object (e.g. package name, user id) -object_type - 'package', 'organization', 'group', 'user' - -#} -{% block activity_stream %} -
      - {% set can_show_activity_detail = h.check_access('activity_list', {'id': id, 'include_data': True, 'object_type': object_type}) %} - - {% for activity in activity_stream %} - {%- snippet "snippets/activities/{}.html".format(activity.activity_type.replace(' ', '_')), - "snippets/activities/fallback.html", - activity=activity, can_show_activity_detail=can_show_activity_detail, - ah={ - 'actor': actor, - 'dataset': dataset, - 'organization': organization, - 'user': user, - 'group': group, - }, - id=id - -%} - {% endfor %} -
    -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/activity_stream.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/activity_stream.html deleted file mode 100644 index ca6b5f8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/activity_stream.html +++ /dev/null @@ -1,14 +0,0 @@ -{% extends "user/read.html" %} - -{% block subtitle %}{{ _('Activity Stream') }} {{ g.template_title_delimiter }} {{ super() }}{% endblock %} - -{% block primary_content_inner %} -

    - {% block page_heading -%} - {{ _('Activity Stream') }} - {%- endblock %} -

    - {% snippet 'snippets/stream.html', activity_stream=activity_stream, id=id, object_type='user' %} - - {% snippet 'snippets/pagination.html', newer_activities_url=newer_activities_url, older_activities_url=older_activities_url %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/dashboard.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/dashboard.html deleted file mode 100644 index abb2d68..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/dashboard.html +++ /dev/null @@ -1,27 +0,0 @@ -{% ckan_extends %} - -{% block breadcrumb_content %} -
  • {{ _('Dashboard') }}
    • -{% endblock %} - -{% block dashboard_nav_links %} - {{ h.build_nav_icon('activity.dashboard', _('News feed'), icon='list') }} - {{ super() }} -{% endblock %} - - -{% block primary_content_inner %} -
    - {% snippet 'user/snippets/followee_dropdown.html', context=dashboard_activity_stream_context, followees=followee_list %} -

    - {% block page_heading %} - {{ _('News feed') }} - {% endblock %} - {{ _("Activity from items that I'm following") }} -

    - {% snippet 'snippets/stream.html', activity_stream=dashboard_activity_stream %} -
    - - {% snippet 'snippets/pagination.html', newer_activities_url=newer_activities_url, older_activities_url=older_activities_url %} - -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/edit_user_form.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/edit_user_form.html deleted file mode 100644 index 4747ef8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/edit_user_form.html +++ /dev/null @@ -1,12 +0,0 @@ -{% ckan_extends %} - -{% block extra_fields %} - {% if h.activity_show_email_notifications() %} - {% call form.checkbox('activity_streams_email_notifications', label=_('Subscribe to notification emails'), id='field-activity-streams-email-notifications', value=True, checked=g.userobj.activity_streams_email_notifications) %} - {% set helper_text = _("You will receive notification emails from {site_title}, e.g. when you have new activities on your dashboard."|string) %} - {{ form.info(helper_text.format(site_title=g.site_title)) }} - {% endcall %} - {% endif %} - - {{ super() }} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/read_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/read_base.html deleted file mode 100644 index dec036e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/read_base.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block content_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('activity.user_activity', _('Activity Stream'), id=user.name, icon='clock') }} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/snippets/followee_dropdown.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/snippets/followee_dropdown.html deleted file mode 100644 index f3db328..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/templates/user/snippets/followee_dropdown.html +++ /dev/null @@ -1,52 +0,0 @@ -{% macro followee_icon(type) -%} - {% if type == 'dataset' %} - - {% elif type == 'user' %} - - {% elif type == 'group' %} - - {% elif type == 'organization' %} - - {% endif %} -{%- endmacro %} - -
    -
    - - {{ _('Activity from:') }} {{ context.context }} - -
    - -
    -
    -
    - - -
    -
    - {% if followees %} - - {% else %} -

    {{ _('You are not following anything') }}

    - {% endif %} -
    -
    diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/conftest.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/conftest.py deleted file mode 100644 index 11408a3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/conftest.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- - -from pytest_factoryboy import register - -from ckan.tests.factories import CKANFactory -from ckanext.activity.model import Activity - - -@register -class ActivityFactory(CKANFactory): - """A factory class for creating CKAN activity objects.""" - - class Meta: - model = Activity - action = "activity_create" diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_action.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_action.py deleted file mode 100644 index d32ed6b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_action.py +++ /dev/null @@ -1,2178 +0,0 @@ -# -*- coding: utf-8 -*- - -import copy -import datetime -import time - -import pytest - -from ckan import model -import ckan.plugins.toolkit as tk - -import ckan.tests.helpers as helpers -import ckan.tests.factories as factories - -from ckanext.activity.model.activity import Activity, package_activity_list - - -def _clear_activities(): - from ckan import model - - model.Session.query(Activity).delete() - model.Session.flush() - - -def _seconds_since_timestamp(timestamp, format_): - dt = datetime.datetime.strptime(timestamp, format_) - now = datetime.datetime.utcnow() - assert now > dt # we assume timestamp is not in the future - return (now - dt).total_seconds() - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("with_plugins") -class TestLimits: - def test_activity_list_actions(self): - actions = [ - "user_activity_list", - "package_activity_list", - "group_activity_list", - "organization_activity_list", - "recently_changed_packages_activity_list", - "current_package_list_with_resources", - ] - for action in actions: - with pytest.raises(tk.ValidationError): - helpers.call_action( - action, - id="test_user", - limit="not_an_int", - offset="not_an_int", - ) - with pytest.raises(tk.ValidationError): - helpers.call_action( - action, id="test_user", limit=-1, offset=-1 - ) - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestActivityShow: - def test_simple_with_data(self, package, user, activity_factory): - activity = activity_factory( - user_id=user["id"], - object_id=package["id"], - activity_type="new package", - data={"package": copy.deepcopy(package), "actor": "Mr Someone"}, - ) - activity_shown = helpers.call_action( - "activity_show", id=activity["id"] - ) - assert activity_shown["user_id"] == user["id"] - assert ( - _seconds_since_timestamp( - activity_shown["timestamp"], "%Y-%m-%dT%H:%M:%S.%f" - ) - < 10 - ) - assert activity_shown["object_id"] == package["id"] - assert activity_shown["data"] == { - "package": package, - "actor": "Mr Someone", - } - assert activity_shown["activity_type"] == "new package" - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestPackageActivityList(object): - def test_create_dataset(self): - user = factories.User() - dataset = factories.Dataset(user=user) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - assert "extras" in activities[0]["data"]["package"] - - def test_change_dataset(self): - user = factories.User() - _clear_activities() - dataset = factories.Dataset(user=user) - original_title = dataset["title"] - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package", - "new package", - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - assert ( - activities[0]["data"]["package"]["title"] - == "Dataset with changed title" - ) - - # the old dataset still has the old title - assert activities[1]["activity_type"] == "new package" - assert activities[1]["data"]["package"]["title"] == original_title - - def test_change_dataset_add_extra(self): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["extras"].append(dict(key="rating", value="great")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - assert "extras" in activities[0]["data"]["package"] - - def test_change_dataset_change_extra(self): - user = factories.User() - dataset = factories.Dataset( - user=user, extras=[dict(key="rating", value="great")] - ) - _clear_activities() - dataset["extras"][0] = dict(key="rating", value="ok") - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - assert "extras" in activities[0]["data"]["package"] - - def test_change_dataset_delete_extra(self): - user = factories.User() - dataset = factories.Dataset( - user=user, extras=[dict(key="rating", value="great")] - ) - _clear_activities() - dataset["extras"] = [] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - assert "extras" in activities[0]["data"]["package"] - - def test_change_dataset_add_resource(self): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - factories.Resource(package_id=dataset["id"], user=user) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - # NB the detail is not included - that is only added in by - # activity_list_to_html() - - def test_change_dataset_change_resource(self): - user = factories.User() - dataset = factories.Dataset( - user=user, - resources=[dict(url="https://example.com/foo.csv", format="csv")], - ) - _clear_activities() - dataset["resources"][0]["format"] = "pdf" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_delete_resource(self): - user = factories.User() - dataset = factories.Dataset( - user=user, - resources=[dict(url="https://example.com/foo.csv", format="csv")], - ) - _clear_activities() - dataset["resources"] = [] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_add_tag(self): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["tags"].append(dict(name="checked")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_delete_tag_from_dataset(self): - user = factories.User() - dataset = factories.Dataset(user=user, tags=[dict(name="checked")]) - _clear_activities() - dataset["tags"] = [] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_delete_dataset(self): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "deleted package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_private_dataset_has_no_activity(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - dataset = factories.Dataset( - private=True, owner_org=org["id"], user=user - ) - dataset["tags"] = [] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def test_private_dataset_delete_has_no_activity(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - dataset = factories.Dataset( - private=True, owner_org=org["id"], user=user - ) - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def _create_bulk_types_activities(self, types): - dataset = factories.Dataset() - from ckan import model - - user = factories.User() - - objs = [ - Activity( - user_id=user["id"], - object_id=dataset["id"], - activity_type=activity_type, - data=None, - ) - for activity_type in types - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - return dataset["id"] - - def test_error_bad_search(self): - with pytest.raises(tk.ValidationError): - helpers.call_action( - "package_activity_list", - id=id, - activity_types=["new package"], - exclude_activity_types=["deleted package"], - ) - - def test_activity_types_filter(self): - types = [ - "new package", - "changed package", - "deleted package", - "changed package", - "new package", - ] - id = self._create_bulk_types_activities(types) - - activities_new = helpers.call_action( - "package_activity_list", id=id, activity_types=["new package"] - ) - assert len(activities_new) == 2 - - activities_not_new = helpers.call_action( - "package_activity_list", - id=id, - exclude_activity_types=["new package"], - ) - assert len(activities_not_new) == 3 - - activities_delete = helpers.call_action( - "package_activity_list", id=id, activity_types=["deleted package"] - ) - assert len(activities_delete) == 1 - - activities_not_deleted = helpers.call_action( - "package_activity_list", - id=id, - exclude_activity_types=["deleted package"], - ) - assert len(activities_not_deleted) == 4 - - def _create_bulk_package_activities(self, count): - dataset = factories.Dataset() - from ckan import model - - user = factories.User() - - objs = [ - Activity( - user_id=user["id"], - object_id=dataset["id"], - activity_type=None, - data=None, - ) - for _ in range(count) - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - return dataset["id"] - - def test_limit_default(self): - id = self._create_bulk_package_activities(35) - results = helpers.call_action("package_activity_list", id=id) - assert len(results) == 31 # i.e. default value - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_limit_configured(self): - id = self._create_bulk_package_activities(7) - results = helpers.call_action("package_activity_list", id=id) - assert len(results) == 5 # i.e. ckan.activity_list_limit - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - @pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") - def test_limit_hits_max(self): - id = self._create_bulk_package_activities(9) - results = helpers.call_action( - "package_activity_list", id=id, limit="9" - ) - assert len(results) == 7 # i.e. ckan.activity_list_limit_max - - def test_normal_user_doesnt_see_hidden_activities(self): - # activity is 'hidden' because dataset is created by site_user - dataset = factories.Dataset() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def test_sysadmin_user_doesnt_see_hidden_activities_by_default(self): - # activity is 'hidden' because dataset is created by site_user - dataset = factories.Dataset() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def test_sysadmin_user_can_include_hidden_activities(self): - # activity is 'hidden' because dataset is created by site_user - dataset = factories.Dataset() - - activities = helpers.call_action( - "package_activity_list", - include_hidden_activity=True, - id=dataset["id"], - ) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - - def _create_dataset_with_activities(self, updates: int = 3): - user = factories.User() - dataset = factories.Dataset(user=user) - ctx = {"user": user["name"]} - - for c in range(updates): - dataset["title"] = "Dataset v{}".format(c) - helpers.call_action("package_update", context=ctx, **dataset) - - return dataset - - def test_activity_after(self): - """Test activities after timestamp""" - dataset = self._create_dataset_with_activities() - - db_activities = package_activity_list(dataset["id"], limit=10) - pkg_activities = helpers.call_action( - "package_activity_list", - id=dataset["id"], - after=db_activities[2].timestamp.timestamp(), - ) - # we expect just 2 (the first 2) - assert len(pkg_activities) == 2 - # first activity here is the first one. - assert pkg_activities[0]["activity_type"] == "changed package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[0]["timestamp"] - ) - assert pkg_activity_time == db_activities[0].timestamp - - # last activity here is the 2nd one. - assert pkg_activities[1]["activity_type"] == "changed package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[1]["timestamp"] - ) - assert pkg_activity_time == db_activities[1].timestamp - - def test_activity_offset(self): - """Test activities after timestamp""" - dataset = self._create_dataset_with_activities() - - db_activities = package_activity_list(dataset["id"], limit=10) - pkg_activities = helpers.call_action( - "package_activity_list", id=dataset["id"], offset=2 - ) - # we expect just 2 (the last 2) - assert len(pkg_activities) == 2 - # first activity here is the first one. - assert pkg_activities[0]["activity_type"] == "changed package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[0]["timestamp"] - ) - assert pkg_activity_time == db_activities[2].timestamp - - # last activity here is the package creation. - assert pkg_activities[1]["activity_type"] == "new package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[1]["timestamp"] - ) - assert pkg_activity_time == db_activities[3].timestamp - - def test_activity_before(self): - """Test activities before timestamp""" - dataset = self._create_dataset_with_activities() - - db_activities = package_activity_list(dataset["id"], limit=10) - pkg_activities = helpers.call_action( - "package_activity_list", - id=dataset["id"], - before=db_activities[1].timestamp.timestamp(), - ) - # we expect just 2 (the last 2) - assert len(pkg_activities) == 2 - # first activity here is the first one. - assert pkg_activities[0]["activity_type"] == "changed package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[0]["timestamp"] - ) - assert pkg_activity_time == db_activities[2].timestamp - - # last activity here is the package creation. - assert pkg_activities[-1]["activity_type"] == "new package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[-1]["timestamp"] - ) - assert pkg_activity_time == db_activities[3].timestamp - - def test_activity_after_before(self): - """Test activities before timestamp""" - dataset = self._create_dataset_with_activities() - - db_activities = package_activity_list(dataset["id"], limit=10) - pkg_activities = helpers.call_action( - "package_activity_list", - id=dataset["id"], - before=db_activities[1].timestamp.timestamp(), - after=db_activities[3].timestamp.timestamp(), - ) - # we expect just 1 (db_activities[2]) - assert len(pkg_activities) == 1 - # first activity here is the first one. - assert pkg_activities[0]["activity_type"] == "changed package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[0]["timestamp"] - ) - assert pkg_activity_time == db_activities[2].timestamp - - def test_activity_after_before_offset(self): - """Test activities before timestamp""" - dataset = self._create_dataset_with_activities(updates=4) - - db_activities = package_activity_list(dataset["id"], limit=10) - pkg_activities = helpers.call_action( - "package_activity_list", - id=dataset["id"], - before=db_activities[1].timestamp.timestamp(), - after=db_activities[4].timestamp.timestamp(), - offset=1, - ) - # we expect just 1 (db_activities[3]) - assert len(pkg_activities) == 1 - # first activity here is the first one. - assert pkg_activities[0]["activity_type"] == "changed package" - pkg_activity_time = datetime.datetime.fromisoformat( - pkg_activities[0]["timestamp"] - ) - assert pkg_activity_time == db_activities[3].timestamp - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestUserActivityList(object): - def test_create_user(self): - user = factories.User() - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "new user" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == user["id"] - - def test_user_update_activity_stream(self): - """Test that the right activity is emitted when updating a user.""" - - user = factories.User() - before = datetime.datetime.utcnow() - - # FIXME we have to pass the email address and password to user_update - # even though we're not updating those fields, otherwise validation - # fails. - helpers.call_action( - "user_update", - id=user["id"], - name=user["name"], - email=user["email"], - password=factories.User.stub().password, - fullname="updated full name", - ) - - activity_stream = helpers.call_action( - "user_activity_list", id=user["id"] - ) - latest_activity = activity_stream[0] - assert latest_activity["activity_type"] == "changed user" - assert latest_activity["object_id"] == user["id"] - assert latest_activity["user_id"] == user["id"] - after = datetime.datetime.utcnow() - timestamp = datetime.datetime.strptime( - latest_activity["timestamp"], "%Y-%m-%dT%H:%M:%S.%f" - ) - assert timestamp >= before and timestamp <= after - - def test_create_dataset(self): - user = factories.User() - _clear_activities() - dataset = factories.Dataset(user=user) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_dataset_changed_by_another_user(self): - user = factories.User() - another_user = factories.Sysadmin() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["extras"].append(dict(key="rating", value="great")) - helpers.call_action( - "package_update", context={"user": another_user["name"]}, **dataset - ) - - # the user might have created the dataset, but a change by another - # user does not show on the user's activity stream - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [] - - def test_change_dataset_add_extra(self): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["extras"].append(dict(key="rating", value="great")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_add_tag(self): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["tags"].append(dict(name="checked")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_create_group(self): - user = factories.User() - _clear_activities() - group = factories.Group(user=user) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "new group" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == group["id"] - assert activities[0]["data"]["group"]["title"] == group["title"] - - def test_delete_group_using_group_delete(self): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - helpers.call_action( - "group_delete", context={"user": user["name"]}, **group - ) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "deleted group" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == group["id"] - assert activities[0]["data"]["group"]["title"] == group["title"] - - def test_delete_group_by_updating_state(self): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - group["state"] = "deleted" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "deleted group" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == group["id"] - assert activities[0]["data"]["group"]["title"] == group["title"] - - def test_create_organization(self): - user = factories.User() - _clear_activities() - org = factories.Organization(user=user) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "new organization" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert activities[0]["data"]["group"]["title"] == org["title"] - - def test_delete_org_using_organization_delete(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - helpers.call_action( - "organization_delete", context={"user": user["name"]}, **org - ) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "deleted organization" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert activities[0]["data"]["group"]["title"] == org["title"] - - def test_delete_org_by_updating_state(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - org["state"] = "deleted" - helpers.call_action( - "organization_update", context={"user": user["name"]}, **org - ) - - activities = helpers.call_action("user_activity_list", id=user["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "deleted organization" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert activities[0]["data"]["group"]["title"] == org["title"] - - def _create_bulk_user_activities(self, count): - from ckan import model - - user = factories.User() - - objs = [ - Activity( - user_id=user["id"], - object_id=None, - activity_type=None, - data=None, - ) - for _ in range(count) - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - return user["id"] - - def test_limit_default(self): - id = self._create_bulk_user_activities(35) - results = helpers.call_action("user_activity_list", id=id) - assert len(results) == 31 # i.e. default value - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_limit_configured(self): - id = self._create_bulk_user_activities(7) - results = helpers.call_action("user_activity_list", id=id) - assert len(results) == 5 # i.e. ckan.activity_list_limit - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - @pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") - def test_limit_hits_max(self): - id = self._create_bulk_user_activities(9) - results = helpers.call_action("user_activity_list", id=id, limit="9") - assert len(results) == 7 # i.e. ckan.activity_list_limit_max - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestGroupActivityList(object): - def test_create_group(self): - user = factories.User() - group = factories.Group(user=user) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "new group" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == group["id"] - assert activities[0]["data"]["group"]["title"] == group["title"] - - def test_change_group(self): - user = factories.User() - _clear_activities() - group = factories.Group(user=user) - original_title = group["title"] - group["title"] = "Group with changed title" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "changed group", - "new group", - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == group["id"] - assert ( - activities[0]["data"]["group"]["title"] - == "Group with changed title" - ) - - # the old group still has the old title - assert activities[1]["activity_type"] == "new group" - assert activities[1]["data"]["group"]["title"] == original_title - - def test_create_dataset(self): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset(self): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - original_title = dataset["title"] - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "changed package", - "new package", - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - # the old dataset still has the old title - assert activities[1]["activity_type"] == "new package" - assert activities[1]["data"]["package"]["title"] == original_title - - def test_change_dataset_add_extra(self): - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - _clear_activities() - dataset["extras"].append(dict(key="rating", value="great")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_add_tag(self): - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - _clear_activities() - dataset["tags"].append(dict(name="checked")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_delete_dataset(self): - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "deleted package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_that_used_to_be_in_the_group(self): - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - # remove the dataset from the group - dataset["groups"] = [] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - _clear_activities() - # edit the dataset - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - # dataset change should not show up in its former group - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [] - - def test_delete_dataset_that_used_to_be_in_the_group(self): - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - # remove the dataset from the group - dataset["groups"] = [] - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - # NOTE: - # ideally the dataset's deletion would not show up in its old group - # but it can't be helped without _group_activity_query getting very - # complicated - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [ - "deleted package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def _create_bulk_group_activities(self, count): - group = factories.Group() - from ckan import model - - user = factories.User() - - objs = [ - Activity( - user_id=user["id"], - object_id=group["id"], - activity_type=None, - data=None, - ) - for _ in range(count) - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - return group["id"] - - def test_limit_default(self): - id = self._create_bulk_group_activities(35) - results = helpers.call_action("group_activity_list", id=id) - assert len(results) == 31 # i.e. default value - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_limit_configured(self): - id = self._create_bulk_group_activities(7) - results = helpers.call_action("group_activity_list", id=id) - assert len(results) == 5 # i.e. ckan.activity_list_limit - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - @pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") - def test_limit_hits_max(self): - id = self._create_bulk_group_activities(9) - results = helpers.call_action("group_activity_list", id=id, limit="9") - assert len(results) == 7 # i.e. ckan.activity_list_limit_max - - def test_normal_user_doesnt_see_hidden_activities(self): - # activity is 'hidden' because group is created by site_user - group = factories.Group() - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [] - - def test_sysadmin_user_doesnt_see_hidden_activities_by_default(self): - # activity is 'hidden' because group is created by site_user - group = factories.Group() - - activities = helpers.call_action("group_activity_list", id=group["id"]) - assert [activity["activity_type"] for activity in activities] == [] - - def test_sysadmin_user_can_include_hidden_activities(self): - # activity is 'hidden' because group is created by site_user - group = factories.Group() - - activities = helpers.call_action( - "group_activity_list", include_hidden_activity=True, id=group["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "new group" - ] - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestOrganizationActivityList(object): - def test_bulk_make_public(self): - org = factories.Organization() - - dataset1 = factories.Dataset(owner_org=org["id"], private=True) - dataset2 = factories.Dataset(owner_org=org["id"], private=True) - - helpers.call_action( - "bulk_update_public", - {}, - datasets=[dataset1["id"], dataset2["id"]], - org_id=org["id"], - ) - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert activities[0]["activity_type"] == "changed package" - - def test_bulk_delete(self): - org = factories.Organization() - - dataset1 = factories.Dataset(owner_org=org["id"]) - dataset2 = factories.Dataset(owner_org=org["id"]) - - helpers.call_action( - "bulk_update_delete", - {}, - datasets=[dataset1["id"], dataset2["id"]], - org_id=org["id"], - ) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert activities[0]["activity_type"] == "deleted package" - - def test_create_organization(self): - user = factories.User() - org = factories.Organization(user=user) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "new organization" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert activities[0]["data"]["group"]["title"] == org["title"] - - def test_change_organization(self): - user = factories.User() - _clear_activities() - org = factories.Organization(user=user) - original_title = org["title"] - org["title"] = "Organization with changed title" - helpers.call_action( - "organization_update", context={"user": user["name"]}, **org - ) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed organization", - "new organization", - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert ( - activities[0]["data"]["group"]["title"] - == "Organization with changed title" - ) - - # the old org still has the old title - assert activities[1]["activity_type"] == "new organization" - assert activities[1]["data"]["group"]["title"] == original_title - - def test_create_dataset(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - dataset = factories.Dataset(owner_org=org["id"], user=user) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - dataset = factories.Dataset(owner_org=org["id"], user=user) - original_title = dataset["title"] - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package", - "new package", - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - # the old dataset still has the old title - assert activities[1]["activity_type"] == "new package" - assert activities[1]["data"]["package"]["title"] == original_title - - def test_change_dataset_add_tag(self): - user = factories.User() - org = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - dataset["tags"].append(dict(name="checked")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_delete_dataset(self): - user = factories.User() - org = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "deleted package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_that_used_to_be_in_the_org(self): - user = factories.User() - org = factories.Organization(user=user) - org2 = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - # remove the dataset from the org - dataset["owner_org"] = org2["id"] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - _clear_activities() - # edit the dataset - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - # dataset change should not show up in its former group - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def test_delete_dataset_that_used_to_be_in_the_org(self): - user = factories.User() - org = factories.Organization(user=user) - org2 = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - # remove the dataset from the group - dataset["owner_org"] = org2["id"] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - _clear_activities() - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - # dataset deletion should not show up in its former org - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def _create_bulk_org_activities(self, count): - org = factories.Organization() - from ckan import model - - user = factories.User() - - objs = [ - Activity( - user_id=user["id"], - object_id=org["id"], - activity_type=None, - data=None, - ) - for _ in range(count) - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - return org["id"] - - def test_limit_default(self): - id = self._create_bulk_org_activities(35) - results = helpers.call_action("organization_activity_list", id=id) - assert len(results) == 31 # i.e. default value - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_limit_configured(self): - id = self._create_bulk_org_activities(7) - results = helpers.call_action("organization_activity_list", id=id) - assert len(results) == 5 # i.e. ckan.activity_list_limit - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - @pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") - def test_limit_hits_max(self): - id = self._create_bulk_org_activities(9) - results = helpers.call_action( - "organization_activity_list", id=id, limit="9" - ) - assert len(results) == 7 # i.e. ckan.activity_list_limit_max - - def test_normal_user_doesnt_see_hidden_activities(self): - # activity is 'hidden' because org is created by site_user - org = factories.Organization() - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def test_sysadmin_user_doesnt_see_hidden_activities_by_default(self): - # activity is 'hidden' because org is created by site_user - org = factories.Organization() - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [] - - def test_sysadmin_user_can_include_hidden_activities(self): - # activity is 'hidden' because org is created by site_user - org = factories.Organization() - - activities = helpers.call_action( - "organization_activity_list", - include_hidden_activity=True, - id=org["id"], - ) - assert [activity["activity_type"] for activity in activities] == [ - "new organization" - ] - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestRecentlyChangedPackagesActivityList: - def test_create_dataset(self): - user = factories.User() - org = factories.Dataset(user=user) - - activities = helpers.call_action( - "recently_changed_packages_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert activities[0]["data"]["package"]["title"] == org["title"] - - def test_change_dataset(self): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - dataset = factories.Dataset(owner_org=org["id"], user=user) - original_title = dataset["title"] - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "recently_changed_packages_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package", - "new package", - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - # the old dataset still has the old title - assert activities[1]["activity_type"] == "new package" - assert activities[1]["data"]["package"]["title"] == original_title - - def test_change_dataset_add_extra(self): - user = factories.User() - org = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - dataset["extras"].append(dict(key="rating", value="great")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "recently_changed_packages_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_change_dataset_add_tag(self): - user = factories.User() - org = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - dataset["tags"].append(dict(name="checked")) - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "recently_changed_packages_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "changed package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def test_delete_dataset(self): - user = factories.User() - org = factories.Organization(user=user) - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - activities = helpers.call_action( - "organization_activity_list", id=org["id"] - ) - assert [activity["activity_type"] for activity in activities] == [ - "deleted package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - - def _create_bulk_package_activities(self, count): - from ckan import model - - user = factories.User() - - objs = [ - Activity( - user_id=user["id"], - object_id=None, - activity_type="new_package", - data=None, - ) - for _ in range(count) - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - - def test_limit_default(self): - self._create_bulk_package_activities(35) - results = helpers.call_action( - "recently_changed_packages_activity_list" - ) - assert len(results) == 31 # i.e. default value - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_limit_configured(self): - self._create_bulk_package_activities(7) - results = helpers.call_action( - "recently_changed_packages_activity_list" - ) - assert len(results) == 5 # i.e. ckan.activity_list_limit - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - @pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") - def test_limit_hits_max(self): - self._create_bulk_package_activities(9) - results = helpers.call_action( - "recently_changed_packages_activity_list", limit="9" - ) - assert len(results) == 7 # i.e. ckan.activity_list_limit_max - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestDashboardActivityList(object): - def test_create_user(self): - user = factories.User() - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [activity["activity_type"] for activity in activities] == [ - "new user" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == user["id"] - # user's own activities are always marked ``'is_new': False`` - assert not activities[0]["is_new"] - - def test_create_dataset(self): - user = factories.User() - _clear_activities() - dataset = factories.Dataset(user=user) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [activity["activity_type"] for activity in activities] == [ - "new package" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == dataset["id"] - assert activities[0]["data"]["package"]["title"] == dataset["title"] - # user's own activities are always marked ``'is_new': False`` - assert not activities[0]["is_new"] - - def test_create_group(self): - user = factories.User() - _clear_activities() - group = factories.Group(user=user) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [activity["activity_type"] for activity in activities] == [ - "new group" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == group["id"] - assert activities[0]["data"]["group"]["title"] == group["title"] - # user's own activities are always marked ``'is_new': False`` - assert not activities[0]["is_new"] - - def test_create_organization(self): - user = factories.User() - _clear_activities() - org = factories.Organization(user=user) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [activity["activity_type"] for activity in activities] == [ - "new organization" - ] - assert activities[0]["user_id"] == user["id"] - assert activities[0]["object_id"] == org["id"] - assert activities[0]["data"]["group"]["title"] == org["title"] - # user's own activities are always marked ``'is_new': False`` - assert not activities[0]["is_new"] - - def _create_bulk_package_activities(self, count): - user = factories.User() - from ckan import model - - objs = [ - Activity( - user_id=user["id"], - object_id=None, - activity_type=None, - data=None, - ) - for _ in range(count) - ] - model.Session.add_all(objs) - model.repo.commit_and_remove() - return user["id"] - - def test_limit_default(self): - id = self._create_bulk_package_activities(35) - results = helpers.call_action( - "dashboard_activity_list", context={"user": id} - ) - assert len(results) == 31 # i.e. default value - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_limit_configured(self): - id = self._create_bulk_package_activities(7) - results = helpers.call_action( - "dashboard_activity_list", context={"user": id} - ) - assert len(results) == 5 # i.e. ckan.activity_list_limit - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - @pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") - def test_limit_hits_max(self): - id = self._create_bulk_package_activities(9) - results = helpers.call_action( - "dashboard_activity_list", limit="9", context={"user": id} - ) - assert len(results) == 7 # i.e. ckan.activity_list_limit_max - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestDashboardNewActivities(object): - def test_users_own_activities(self): - # a user's own activities are not shown as "new" - user = factories.User() - dataset = factories.Dataset(user=user) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - group = factories.Group(user=user) - group["title"] = "Group with changed title" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - helpers.call_action( - "group_delete", context={"user": user["name"]}, **group - ) - - new_activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [activity["is_new"] for activity in new_activities] == [ - False - ] * 7 - new_activities_count = helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - assert new_activities_count == 0 - - def test_activities_by_a_followed_user(self): - user = factories.User() - followed_user = factories.User() - helpers.call_action( - "follow_user", context={"user": user["name"]}, **followed_user - ) - _clear_activities() - dataset = factories.Dataset(user=followed_user) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", - context={"user": followed_user["name"]}, - **dataset, - ) - helpers.call_action( - "package_delete", - context={"user": followed_user["name"]}, - **dataset, - ) - group = factories.Group(user=followed_user) - group["title"] = "Group with changed title" - helpers.call_action( - "group_update", context={"user": followed_user["name"]}, **group - ) - helpers.call_action( - "group_delete", context={"user": followed_user["name"]}, **group - ) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - activity["activity_type"] for activity in activities[::-1] - ] == [ - "new package", - "changed package", - "deleted package", - "new group", - "changed group", - "deleted group", - ] - assert [activity["is_new"] for activity in activities] == [True] * 6 - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 6 - ) - - def test_activities_on_a_followed_dataset(self): - user = factories.User() - another_user = factories.Sysadmin() - _clear_activities() - dataset = factories.Dataset(user=another_user) - helpers.call_action( - "follow_dataset", context={"user": user["name"]}, **dataset - ) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": another_user["name"]}, **dataset - ) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [ - ("new package", True), - # NB The 'new package' activity is in our activity stream and shows - # as "new" even though it occurred before we followed it. This is - # known & intended design. - ("changed package", True), - ] - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 2 - ) - - def test_activities_on_a_followed_group(self): - user = factories.User() - another_user = factories.Sysadmin() - _clear_activities() - group = factories.Group(user=user) - helpers.call_action( - "follow_group", context={"user": user["name"]}, **group - ) - group["title"] = "Group with changed title" - helpers.call_action( - "group_update", context={"user": another_user["name"]}, **group - ) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [ - ("new group", False), # False because user did this one herself - ("changed group", True), - ] - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 1 - ) - - def test_activities_on_a_dataset_in_a_followed_group(self): - user = factories.User() - another_user = factories.Sysadmin() - group = factories.Group(user=user) - helpers.call_action( - "follow_group", context={"user": user["name"]}, **group - ) - _clear_activities() - dataset = factories.Dataset( - groups=[{"name": group["name"]}], user=another_user - ) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": another_user["name"]}, **dataset - ) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [("new package", True), ("changed package", True)] - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 2 - ) - - def test_activities_on_a_dataset_in_a_followed_org(self): - user = factories.User() - another_user = factories.Sysadmin() - org = factories.Organization(user=user) - helpers.call_action( - "follow_group", context={"user": user["name"]}, **org - ) - _clear_activities() - dataset = factories.Dataset(owner_org=org["id"], user=another_user) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": another_user["name"]}, **dataset - ) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [("new package", True), ("changed package", True)] - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 2 - ) - - def test_activities_that_should_not_show(self): - user = factories.User() - _clear_activities() - # another_user does some activity unconnected with user - another_user = factories.Sysadmin() - group = factories.Group(user=another_user) - dataset = factories.Dataset( - groups=[{"name": group["name"]}], user=another_user - ) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": another_user["name"]}, **dataset - ) - - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [] - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 0 - ) - - @pytest.mark.ckan_config("ckan.activity_list_limit", "5") - def test_maximum_number_of_new_activities(self): - """Test that the new activities count does not go higher than 5, even - if there are more than 5 new activities from the user's followers.""" - user = factories.User() - another_user = factories.Sysadmin() - dataset = factories.Dataset() - helpers.call_action( - "follow_dataset", context={"user": user["name"]}, **dataset - ) - for n in range(0, 7): - dataset["notes"] = "Updated {n} times".format(n=n) - helpers.call_action( - "package_update", - context={"user": another_user["name"]}, - **dataset, - ) - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 5 - ) - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_request_context", "with_plugins") -class TestSendEmailNotifications(object): - # TODO: this action doesn't do much. Maybe it well be better to move tests - # into lib.email_notifications eventually - - def check_email(self, email, address, name, subject): - assert email[1] == "info@test.ckan.net" - assert email[2] == [address] - assert subject in email[3] - # TODO: Check that body contains link to dashboard and email prefs. - - def test_fresh_setupnotifications(self, mail_server): - helpers.call_action("send_email_notifications") - assert ( - len(mail_server.get_smtp_messages()) == 0 - ), "Notification came out of nowhere" - - def test_single_notification(self, mail_server): - pkg = factories.Dataset() - user = factories.User(activity_streams_email_notifications=True) - helpers.call_action( - "follow_dataset", {"user": user["name"]}, id=pkg["id"] - ) - helpers.call_action("package_update", id=pkg["id"], notes="updated") - helpers.call_action("send_email_notifications") - messages = mail_server.get_smtp_messages() - assert len(messages) == 1 - self.check_email( - messages[0], - user["email"], - user["name"], - "1 new activity from CKAN", - ) - - def test_multiple_notifications(self, mail_server): - pkg = factories.Dataset() - user = factories.User(activity_streams_email_notifications=True) - helpers.call_action( - "follow_dataset", {"user": user["name"]}, id=pkg["id"] - ) - for i in range(3): - helpers.call_action( - "package_update", id=pkg["id"], notes=f"updated {i} times" - ) - helpers.call_action("send_email_notifications") - messages = mail_server.get_smtp_messages() - assert len(messages) == 1 - self.check_email( - messages[0], - user["email"], - user["name"], - "3 new activities from CKAN", - ) - - def test_no_notifications_if_dashboard_visited(self, mail_server): - pkg = factories.Dataset() - user = factories.User(activity_streams_email_notifications=True) - helpers.call_action( - "follow_dataset", {"user": user["name"]}, id=pkg["id"] - ) - helpers.call_action("package_update", id=pkg["id"], notes="updated") - new_activities_count = helpers.call_action( - "dashboard_new_activities_count", - {"user": user["name"]}, - id=pkg["id"], - ) - assert new_activities_count == 1 - - helpers.call_action( - "dashboard_mark_activities_old", - {"user": user["name"]}, - id=pkg["id"], - ) - helpers.call_action("send_email_notifications") - messages = mail_server.get_smtp_messages() - assert len(messages) == 0 - - def test_notifications_disabled_by_default(self): - user = factories.User() - assert not user["activity_streams_email_notifications"] - - def test_no_emails_when_notifications_disabled(self, mail_server): - pkg = factories.Dataset() - user = factories.User() - helpers.call_action( - "follow_dataset", {"user": user["name"]}, id=pkg["id"] - ) - helpers.call_action("package_update", id=pkg["id"], notes="updated") - helpers.call_action("send_email_notifications") - messages = mail_server.get_smtp_messages() - assert len(messages) == 0 - new_activities_count = helpers.call_action( - "dashboard_new_activities_count", - {"user": user["name"]}, - id=pkg["id"], - ) - assert new_activities_count == 1 - - @pytest.mark.ckan_config( - "ckan.activity_streams_email_notifications", False - ) - def test_send_email_notifications_feature_disabled(self, mail_server): - with pytest.raises(tk.ValidationError): - helpers.call_action("send_email_notifications") - messages = mail_server.get_smtp_messages() - assert len(messages) == 0 - - @pytest.mark.ckan_config("ckan.email_notifications_since", ".000001") - def test_email_notifications_since(self, mail_server): - pkg = factories.Dataset() - user = factories.User(activity_streams_email_notifications=True) - helpers.call_action( - "follow_dataset", {"user": user["name"]}, id=pkg["id"] - ) - helpers.call_action("package_update", id=pkg["id"], notes="updated") - time.sleep(0.01) - helpers.call_action("send_email_notifications") - messages = mail_server.get_smtp_messages() - assert len(messages) == 0 - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestDashboardMarkActivitiesOld(object): - def test_mark_as_old_some_activities_by_a_followed_user(self): - # do some activity that will show up on user's dashboard - user = factories.User() - # now some activity that is "new" because it is by a followed user - followed_user = factories.User() - helpers.call_action( - "follow_user", context={"user": user["name"]}, **followed_user - ) - dataset = factories.Dataset(user=followed_user) - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", - context={"user": followed_user["name"]}, - **dataset, - ) - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 3 - ) - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [ - ("new user", False), - ("new user", True), - ("new package", True), - ("changed package", True), - ] - - helpers.call_action( - "dashboard_mark_activities_old", context={"user": user["name"]} - ) - - assert ( - helpers.call_action( - "dashboard_new_activities_count", context={"user": user["id"]} - ) - == 0 - ) - activities = helpers.call_action( - "dashboard_activity_list", context={"user": user["id"]} - ) - assert [ - (activity["activity_type"], activity["is_new"]) - for activity in activities[::-1] - ] == [ - ("new user", False), - ("new user", False), - ("new package", False), - ("changed package", False), - ] - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestFollow: - @pytest.mark.usefixtures("app") - def test_follow_dataset_no_activity(self): - user = factories.User() - dataset = factories.Dataset() - _clear_activities() - helpers.call_action( - "follow_dataset", context={"user": user["name"]}, id=dataset["id"] - ) - assert not helpers.call_action("user_activity_list", id=user["id"]) - - @pytest.mark.usefixtures("app") - def test_follow_group_no_activity(self): - user = factories.User() - group = factories.Group() - _clear_activities() - helpers.call_action( - "follow_group", context={"user": user["name"]}, **group - ) - assert not helpers.call_action("user_activity_list", id=user["id"]) - - @pytest.mark.usefixtures("app") - def test_follow_organization_no_activity(self): - user = factories.User() - org = factories.Organization() - _clear_activities() - helpers.call_action( - "follow_group", context={"user": user["name"]}, **org - ) - assert not helpers.call_action("user_activity_list", id=user["id"]) - - @pytest.mark.usefixtures("app") - def test_follow_user_no_activity(self): - user = factories.User() - user2 = factories.User() - _clear_activities() - helpers.call_action( - "follow_user", context={"user": user["name"]}, **user2 - ) - assert not helpers.call_action("user_activity_list", id=user["id"]) - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestDeferCommitOnCreate(object): - - def test_package_create_defer_commit(self): - dataset_dict = { - "name": "test_dataset", - } - context = { - "defer_commit": True, - "user": factories.User()["name"], - } - - helpers.call_action("package_create", context=context, **dataset_dict) - - model.Session.close() - - with pytest.raises(tk.ObjectNotFound): - helpers.call_action("package_show", id=dataset_dict["name"]) - - assert model.Session.query(Activity).filter( - Activity.activity_type != "new user").count() == 0 - - def test_group_create_defer_commit(self): - group_dict = { - "name": "test_group", - } - context = { - "defer_commit": True, - "user": factories.User()["name"], - } - - helpers.call_action("group_create", context=context, **group_dict) - - model.Session.close() - - with pytest.raises(tk.ObjectNotFound): - helpers.call_action("group_show", id=group_dict["name"]) - - assert model.Session.query(Activity).filter( - Activity.activity_type != "new user").count() == 0 - - def test_organization_create_defer_commit(self): - organization_dict = { - "name": "test_org", - } - context = { - "defer_commit": True, - "user": factories.User()["name"], - } - - helpers.call_action("organization_create", context=context, **organization_dict) - - model.Session.close() - - with pytest.raises(tk.ObjectNotFound): - helpers.call_action("organization_show", id=organization_dict["name"]) - - assert model.Session.query(Activity).filter( - Activity.activity_type != "new user").count() == 0 - - def test_user_create_defer_commit(self): - stub = factories.User.stub() - user_dict = { - "name": stub.name, - "email": stub.email, - "password": "test1234", - } - context = {"defer_commit": True} - - helpers.call_action("user_create", context=context, **user_dict) - - model.Session.close() - - with pytest.raises(tk.ObjectNotFound): - helpers.call_action("user_show", id=user_dict["name"]) - - assert model.Session.query(Activity).count() == 0 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_auth.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_auth.py deleted file mode 100644 index eb11bf6..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_auth.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -import pytest - -import ckan.plugins.toolkit as tk -import ckan.tests.helpers as helpers -import ckan.model as model - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("with_plugins") -class TestAuth: - @pytest.mark.ckan_config( - "ckan.auth.public_activity_stream_detail", "false" - ) - def test_config_option_public_activity_stream_detail_denied(self, package): - """Config option says an anon user is not authorized to get activity - stream data/detail. - """ - context = {"user": None, "model": model} - with pytest.raises(tk.NotAuthorized): - helpers.call_auth( - "package_activity_list", - context=context, - id=package["id"], - include_data=True, - ) - - @pytest.mark.ckan_config("ckan.auth.public_activity_stream_detail", "true") - def test_config_option_public_activity_stream_detail(self, package): - """Config option says an anon user is authorized to get activity - stream data/detail. - """ - context = {"user": None, "model": model} - helpers.call_auth( - "package_activity_list", - context=context, - id=package["id"], - include_data=True, - ) - - def test_normal_user_cant_use_it(self, user): - context = {"user": user["name"], "model": model} - - with pytest.raises(tk.NotAuthorized): - helpers.call_auth("activity_create", context=context) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_functional.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_functional.py deleted file mode 100644 index 23f3828..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/tests/logic/test_functional.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- - -import pytest - -import ckan.plugins.toolkit as tk -import ckan.tests.helpers as helpers -import ckan.tests.factories as factories - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins", "reset_index") -@pytest.mark.ckan_config("ckan.activity_list_limit", "5") -@pytest.mark.ckan_config("ckan.activity_list_limit_max", "7") -class TestPagination(): - - def test_pagination(self, app): - user = factories.User() - org = factories.Organization() - dataset = factories.Dataset(owner_org=org["id"]) - - for i in range(0, 8): - dataset["notes"] = f"Update number: {i}" - helpers.call_action( - "package_update", - context={"user": user["name"]}, - **dataset, - ) - - # Test initial pagination buttons are rendered correctly - url = tk.url_for("dataset.activity", id=dataset["id"]) - response = app.get(url) - - assert 'Newer activities' in response.body - assert f'Newer activities' in response.body - assert f'February 1, 2018 at 10:58:59 AM UTC' - "" - ) - assert hasattr(html, "__html__") # shows it is safe Markup - - def test_selected(self): - pkg_activity = { - "id": "id1", - "timestamp": datetime.datetime(2018, 2, 1, 10, 58, 59), - } - - out = tk.h.activity_list_select([pkg_activity], "id1") - - html = out[0] - assert ( - str(html) - == '" - ) - assert hasattr(html, "__html__") # shows it is safe Markup - - def test_escaping(self): - pkg_activity = { - "id": '">', # hacked somehow - "timestamp": datetime.datetime(2018, 2, 1, 10, 58, 59), - } - - out = tk.h.activity_list_select([pkg_activity], "") - - html = out[0] - assert str(html).startswith('{}'.format(user["name"], user["fullname"]) - in response - ) - - -def assert_group_link_in_response(group, response): - assert ( - '{2}'.format(group["type"], group["name"], group["title"]) - in response - ) - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("with_plugins", "clean_db") -class TestOrganization(object): - def test_simple(self, app): - """Checking the template shows the activity stream.""" - user = factories.User() - org = factories.Organization(user=user) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - assert user["fullname"] in response - assert "created the organization" in response - - def test_create_organization(self, app): - user = factories.User() - org = factories.Organization(user=user) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - assert "created the organization" in response - assert_group_link_in_response(org, response) - - def test_change_organization(self, app): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - org["title"] = "Organization with changed title" - helpers.call_action( - "organization_update", context={"user": user["name"]}, **org - ) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - assert "updated the organization" in response - assert_group_link_in_response(org, response) - - def test_delete_org_using_organization_delete(self, app): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - helpers.call_action( - "organization_delete", context={"user": user["name"]}, **org - ) - - url = url_for("activity.organization_activity", id=org["id"]) - env = {"REMOTE_USER": user["name"]} - app.get(url, extra_environ=env, status=404) - # organization_delete causes the Member to state=deleted and then the - # user doesn't have permission to see their own deleted Organization. - # Therefore you can't render the activity stream of that org. You'd - # hope that organization_delete was the same as organization_update - # state=deleted but they are not... - - def test_delete_org_by_updating_state(self, app): - user = factories.User() - org = factories.Organization(user=user) - _clear_activities() - org["state"] = "deleted" - helpers.call_action( - "organization_update", context={"user": user["name"]}, **org - ) - - url = url_for("activity.organization_activity", id=org["id"]) - env = {"REMOTE_USER": user["name"]} - response = app.get(url, extra_environ=env) - assert_user_link_in_response(user, response) - assert "deleted the organization" in response - assert_group_link_in_response(org, response) - - def test_create_dataset(self, app): - user = factories.User() - org = factories.Organization() - _clear_activities() - dataset = factories.Dataset(owner_org=org["id"], user=user) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "created the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_change_dataset(self, app): - user = factories.User() - org = factories.Organization() - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_delete_dataset(self, app): - user = factories.User() - org = factories.Organization() - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "deleted the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestUser: - def test_simple(self, app): - """Checking the template shows the activity stream.""" - - user = factories.User() - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - assert user["fullname"] in response - assert "signed up" in response - - def test_create_user(self, app): - - user = factories.User() - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - assert "signed up" in response - - def test_change_user(self, app): - - user = factories.User() - _clear_activities() - user["fullname"] = "Mr. Changed Name" - helpers.call_action( - "user_update", context={"user": user["name"]}, **user - ) - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - assert "updated their profile" in response - - def test_create_dataset(self, app): - - user = factories.User() - _clear_activities() - dataset = factories.Dataset(user=user) - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "created the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_change_dataset(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_delete_dataset(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.user_activity", id=user["id"]) - env = {"REMOTE_USER": user["name"]} - response = app.get(url, extra_environ=env) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "deleted the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_create_group(self, app): - - user = factories.User() - group = factories.Group(user=user) - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".group") - - assert_user_link_in_response(user, response) - assert "created the group" in response - assert group["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert group["title"] in href.text.strip() - - def test_change_group(self, app): - - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - group["title"] = "Group with changed title" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".group") - assert_user_link_in_response(user, response) - assert "updated the group" in response - assert group["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert group["title"] in href.text.strip() - - def test_delete_group_using_group_delete(self, app): - - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - helpers.call_action( - "group_delete", context={"user": user["name"]}, **group - ) - - url = url_for("activity.user_activity", id=user["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".group") - assert_user_link_in_response(user, response) - assert "deleted the group" in response - assert group["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert group["title"] in href.text.strip() - - def test_delete_group_by_updating_state(self, app): - - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - group["state"] = "deleted" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - - url = url_for("activity.group_activity", id=group["id"]) - env = {"REMOTE_USER": user["name"]} - response = app.get(url, extra_environ=env) - assert_user_link_in_response(user, response) - assert "deleted the group" in response - assert_group_link_in_response(group, response) - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("clean_db", "with_plugins") -class TestPackage: - def test_simple(self, app): - """Checking the template shows the activity stream.""" - user = factories.User() - dataset = factories.Dataset(user=user) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - assert user["fullname"] in response - assert "created the dataset" in response - - def test_create_dataset(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - - assert_user_link_in_response(user, response) - assert "created the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_change_dataset(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_create_tag_directly(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["tags"] = [{"name": "some_tag"}] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - - assert len(activities) == 1 - - def test_create_tag(self, app): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["tags"] = [{"name": "some_tag"}] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - - assert len(activities) == 1 - - def test_create_extra(self, app): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - dataset["extras"] = [{"key": "some", "value": "extra"}] - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - - assert len(activities) == 1 - - def test_create_resource(self, app): - user = factories.User() - dataset = factories.Dataset(user=user) - _clear_activities() - helpers.call_action( - "resource_create", - context={"user": user["name"]}, - name="Test resource", - package_id=dataset["id"], - ) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - - assert len(activities) == 1 - - def test_update_resource(self, app): - user = factories.User() - dataset = factories.Dataset(user=user) - resource = factories.Resource(package_id=dataset["id"]) - _clear_activities() - - helpers.call_action( - "resource_update", - context={"user": user["name"]}, - id=resource["id"], - name="Test resource updated", - package_id=dataset["id"], - ) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - - assert len(activities) == 1 - - def test_delete_dataset(self, app): - user = factories.User() - org = factories.Organization() - dataset = factories.Dataset(owner_org=org["id"], user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.organization_activity", id=org["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - - assert_user_link_in_response(user, response) - assert "deleted the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_admin_can_see_old_versions(self, app): - - user = factories.User() - env = {"REMOTE_USER": user["name"]} - dataset = factories.Dataset(user=user) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url, extra_environ=env) - assert "View this version" in response - - def test_public_cant_see_old_versions(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - assert "View this version" not in response - - def test_admin_can_see_changes(self, app): - - user = factories.User() - env = {"REMOTE_USER": user["name"]} - dataset = factories.Dataset() # activities by system user aren't shown - dataset["title"] = "Changed" - helpers.call_action("package_update", **dataset) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url, extra_environ=env) - assert "Changes" in response - - def test_public_cant_see_changes(self, app): - dataset = factories.Dataset() # activities by system user aren't shown - dataset["title"] = "Changed" - helpers.call_action("package_update", **dataset) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - assert "Changes" not in response - - # ckanext-canada uses their IActivity to add their custom activity to the - # list of validators: https://github.com/open-data/ckanext-canada/blob/6870e5bc38a04aa8cef191b5e9eb361f9560872b/ckanext/canada/plugins.py#L596 - # but it's easier here to just hack patch it in - @mock.patch( - "ckanext.activity.logic.validators.object_id_validators", - dict( - list(object_id_validators.items()) - + [("changed datastore", "package_id_exists")] - ), - ) - def test_custom_activity(self, app): - """Render a custom activity""" - - user = factories.User() - organization = factories.Organization( - users=[{"name": user["id"], "capacity": "admin"}] - ) - dataset = factories.Dataset(owner_org=organization["id"], user=user) - resource = factories.Resource(package_id=dataset["id"]) - _clear_activities() - - # Create a custom Activity object. This one is inspired by: - # https://github.com/open-data/ckanext-canada/blob/master/ckanext/canada/activity.py - activity_dict = { - "user_id": user["id"], - "object_id": dataset["id"], - "activity_type": "changed datastore", - "data": { - "resource_id": resource["id"], - "pkg_type": dataset["type"], - "resource_name": "june-2018", - "owner_org": organization["name"], - "count": 5, - }, - } - helpers.call_action("activity_create", **activity_dict) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - # it renders the activity with fallback.html, since we've not defined - # changed_datastore.html in this case - assert "changed datastore" in response - - def test_redirect_also_with_activity_parameter(self, app): - user = factories.User() - dataset = factories.Dataset(user=user) - activity = activity_model.package_activity_list( - dataset["id"], limit=1, offset=0 - )[0] - # view as an admin because viewing the old versions of a dataset - sysadmin = factories.Sysadmin() - env = {"REMOTE_USER": sysadmin["name"]} - response = app.get( - url_for( - "activity.package_history", - id=dataset["id"], - activity_id=activity.id, - ), - status=302, - extra_environ=env, - follow_redirects=False, - ) - expected_path = url_for( - "activity.package_history", - id=dataset["name"], - _external=True, - activity_id=activity.id, - ) - assert response.headers["location"] == expected_path - - def test_read_dataset_as_it_used_to_be(self, app): - dataset = factories.Dataset(title="Original title") - activity = ( - model.Session.query(Activity) - .filter_by(object_id=dataset["id"]) - .one() - ) - dataset["title"] = "Changed title" - helpers.call_action("package_update", **dataset) - - sysadmin = factories.Sysadmin() - env = {"REMOTE_USER": sysadmin["name"]} - response = app.get( - url_for( - "activity.package_history", - id=dataset["name"], - activity_id=activity.id, - ), - extra_environ=env, - ) - assert helpers.body_contains(response, "Original title") - - def test_read_dataset_as_it_used_to_be_but_is_unmigrated(self, app): - # Renders the dataset using the activity detail, when that Activity was - # created with an earlier version of CKAN, and it has not been migrated - # (with migrate_package_activity.py), which should give a 404 - - user = factories.User() - dataset = factories.Dataset(user=user) - - # delete the modern Activity object that's been automatically created - modern_activity = ( - model.Session.query(Activity) - .filter_by(object_id=dataset["id"]) - .one() - ) - modern_activity.delete() - - # Create an Activity object as it was in earlier versions of CKAN. - # This code is based on: - # https://github.com/ckan/ckan/blob/b348bf2fe68db6704ea0a3e22d533ded3d8d4344/ckan/model/package.py#L508 - activity_type = "changed" - dataset_table_dict = dictization.table_dictize( - model.Package.get(dataset["id"]), context={"model": model} - ) - activity = Activity( - user_id=user["id"], - object_id=dataset["id"], - activity_type="%s package" % activity_type, - data={ - # "actor": a legacy activity had no "actor" - # "package": a legacy activity had just the package table, - # rather than the result of package_show - "package": dataset_table_dict - }, - ) - model.Session.add(activity) - - sysadmin = factories.Sysadmin() - env = {"REMOTE_USER": sysadmin["name"]} - app.get( - url_for( - "activity.package_history", - id=dataset["name"], - activity_id=activity.id, - ), - extra_environ=env, - status=404, - ) - - def test_changes(self, app): - user = factories.User() - dataset = factories.Dataset(title="First title", user=user) - dataset["title"] = "Second title" - helpers.call_action("package_update", **dataset) - - activity = activity_model.package_activity_list( - dataset["id"], limit=1, offset=0 - )[0] - env = {"REMOTE_USER": user["name"]} - response = app.get( - url_for("activity.package_changes", id=activity.id), - extra_environ=env, - ) - assert helpers.body_contains(response, "First") - assert helpers.body_contains(response, "Second") - - @pytest.mark.ckan_config("ckan.activity_list_limit", "3") - def test_invalid_get_params(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url, query_string={"before": "XXX"}, status=400) - assert "Invalid parameters" in response.body - - @pytest.mark.ckan_config("ckan.activity_list_limit", "3") - def test_older_activities_url_button(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - - dataset["title"] = "Second title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "Third title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "Fourth title" - helpers.call_action("package_update", **dataset) - - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get(url) - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - # Last activity in the first page - before_time = datetime.fromisoformat(activities[2]["timestamp"]) - - # Next page button - older_activities_url_url = "/dataset/activity/{}?before={}".format( - dataset["id"], before_time.timestamp() - ) - assert older_activities_url_url in response.body - - # Prev page button is not in the first page - newer_activities_url_url = "/dataset/activity/{}?after=".format(dataset["id"]) - assert newer_activities_url_url not in response.body - - @pytest.mark.ckan_config("ckan.activity_list_limit", "3") - def test_next_before_buttons(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - - dataset["title"] = "Second title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "Third title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "4th title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "5th title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "6th title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "7h title" - helpers.call_action("package_update", **dataset) - - db_activities = activity_model.package_activity_list( - dataset["id"], limit=10 - ) - activities = helpers.call_action( - "package_activity_list", id=dataset["id"] - ) - # Last activity in the first page - last_act_page_1_time = datetime.fromisoformat( - activities[2]["timestamp"] - ) - url = url_for("activity.package_activity", id=dataset["id"]) - response = app.get( - url, query_string={"before": last_act_page_1_time.timestamp()} - ) - - # Next page button exists in page 2 - older_activities_url_url = "/dataset/activity/{}?before={}".format( - dataset["id"], db_activities[5].timestamp.timestamp() - ) - assert older_activities_url_url in response.body - # Prev page button exists in page 2 - newer_activities_url_url = "/dataset/activity/{}?after={}".format( - dataset["id"], db_activities[3].timestamp.timestamp() - ) - assert newer_activities_url_url in response.body - - @pytest.mark.ckan_config("ckan.activity_list_limit", "3") - def test_newer_activities_url_button(self, app): - - user = factories.User() - dataset = factories.Dataset(user=user) - - dataset["title"] = "Second title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "Third title" - helpers.call_action("package_update", **dataset) - dataset["title"] = "Fourth title" - helpers.call_action("package_update", **dataset) - - activities = helpers.call_action( - "package_activity_list", id=dataset["id"], limit=10 - ) - before_time = datetime.fromisoformat(activities[2]["timestamp"]) - - url = url_for("activity.package_activity", id=dataset["id"]) - # url for page 2 - response = app.get( - url, query_string={"before": before_time.timestamp()} - ) - - # There's not a third page - older_activities_url_url = "/dataset/activity/{}?before=".format(dataset["name"]) - assert older_activities_url_url not in response.body - - # previous page exists - after_time = datetime.fromisoformat(activities[3]["timestamp"]) - newer_activities_url_url = "/dataset/activity/{}?after={}".format( - dataset["id"], after_time.timestamp() - ) - assert newer_activities_url_url in response.body - - -@pytest.mark.ckan_config("ckan.plugins", "activity") -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestGroup: - def test_simple(self, app): - """Checking the template shows the activity stream.""" - user = factories.User() - group = factories.Group(user=user) - - url = url_for("activity.group_activity", id=group["id"]) - response = app.get(url) - assert user["fullname"] in response - assert "created the group" in response - - def test_create_group(self, app): - user = factories.User() - group = factories.Group(user=user) - - url = url_for("activity.group_activity", id=group["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - assert "created the group" in response - assert_group_link_in_response(group, response) - - def test_change_group(self, app): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - group["title"] = "Group with changed title" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - - url = url_for("activity.group_activity", id=group["id"]) - response = app.get(url) - assert_user_link_in_response(user, response) - assert "updated the group" in response - assert_group_link_in_response(group, response) - - def test_delete_group_using_group_delete(self, app): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - helpers.call_action( - "group_delete", context={"user": user["name"]}, **group - ) - - url = url_for("activity.group_activity", id=group["id"]) - env = {"REMOTE_USER": user["name"]} - app.get(url, extra_environ=env, status=404) - # group_delete causes the Member to state=deleted and then the user - # doesn't have permission to see their own deleted Group. Therefore you - # can't render the activity stream of that group. You'd hope that - # group_delete was the same as group_update state=deleted but they are - # not... - - def test_delete_group_by_updating_state(self, app): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - group["state"] = "deleted" - helpers.call_action( - "group_update", context={"user": user["name"]}, **group - ) - - url = url_for("activity.group_activity", id=group["id"]) - env = {"REMOTE_USER": user["name"]} - response = app.get(url, extra_environ=env) - assert_user_link_in_response(user, response) - assert "deleted the group" in response - assert_group_link_in_response(group, response) - - def test_create_dataset(self, app): - user = factories.User() - group = factories.Group(user=user) - _clear_activities() - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - - url = url_for("activity.group_activity", id=group["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "created the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_change_dataset(self, app): - - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - _clear_activities() - dataset["title"] = "Dataset with changed title" - helpers.call_action( - "package_update", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.group_activity", id=group["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "updated the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() - - def test_delete_dataset(self, app): - user = factories.User() - group = factories.Group(user=user) - dataset = factories.Dataset(groups=[{"id": group["id"]}], user=user) - _clear_activities() - helpers.call_action( - "package_delete", context={"user": user["name"]}, **dataset - ) - - url = url_for("activity.group_activity", id=group["id"]) - response = app.get(url) - page = BeautifulSoup(response.body) - href = page.select_one(".dataset") - assert_user_link_in_response(user, response) - assert "deleted the dataset" in response - assert dataset["id"] in href.select_one("a")["href"].split("/", 2)[-1] - assert dataset["title"] in href.text.strip() diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/views.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/views.py deleted file mode 100644 index 8374d88..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/activity/views.py +++ /dev/null @@ -1,943 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import annotations -import logging - -from datetime import datetime -from typing import Any, Optional, Union, cast - -from flask import Blueprint - -import ckan.plugins.toolkit as tk -import ckan.model as model -from ckan.views.group import ( - set_org, - # TODO: don't use hidden funcitons - _get_group_dict, - _get_group_template, - _replace_group_org, -) - -# TODO: don't use hidden funcitons -from ckan.views.user import _extra_template_variables - -# TODO: don't use hidden funcitons -from ckan.views.dataset import _setup_template_variables - -from ckan.types import Context, Response -from .model import Activity -from .logic.validators import ( - VALIDATORS_PACKAGE_ACTIVITY_TYPES, - VALIDATORS_GROUP_ACTIVITY_TYPES, - VALIDATORS_ORGANIZATION_ACTIVITY_TYPES -) - - -log = logging.getLogger(__name__) -bp = Blueprint("activity", __name__) - - -def _get_activity_stream_limit() -> int: - base_limit = tk.config.get("ckan.activity_list_limit") - max_limit = tk.config.get("ckan.activity_list_limit_max") - return min(base_limit, max_limit) - - -def _get_older_activities_url( - has_more: bool, - stream: list[dict[str, Any]], - **kwargs: Any -) -> Any: - """ Returns pagination's older activities url. - - If "after", we came from older activities, so we know it exists. - if "before" (or is_first_page), we only show older activities if we know - we have more rows - """ - after = tk.request.args.get("after") - before = tk.request.args.get("before") - is_first_page = after is None and before is None - url = None - if after or (has_more and (before or is_first_page)): - before_time = datetime.fromisoformat( - stream[-1]["timestamp"] - ) - url = tk.h.url_for( - tk.request.endpoint, - before=before_time.timestamp(), - **kwargs - ) - - return url - - -def _get_newer_activities_url( - has_more: bool, - stream: list[dict[str, Any]], - **kwargs: Any -) -> Any: - """ Returns pagination's newer activities url. - - if "before", we came from the newer activities, so it exists. - if "after", we only show newer activities if we know - we have more rows - """ - after = tk.request.args.get("after") - before = tk.request.args.get("before") - url = None - - if before or (has_more and after): - after_time = datetime.fromisoformat( - stream[0]["timestamp"] - ) - url = tk.h.url_for( - tk.request.endpoint, - after=after_time.timestamp(), - **kwargs - ) - return url - - -@bp.route("/dataset//resources//history/") -def resource_history(id: str, resource_id: str, activity_id: str) -> str: - context = cast( - Context, - { - "auth_user_obj": tk.g.userobj, - "for_view": True, - }, - ) - - try: - package = tk.get_action("package_show")(context, {"id": id}) - except (tk.ObjectNotFound, tk.NotAuthorized): - return tk.abort(404, tk._("Dataset not found")) - - # view an 'old' version of the package, as recorded in the - # activity stream - current_pkg = package - try: - activity = context["session"].query(Activity).get(activity_id) - assert activity - package = activity.data["package"] - except AttributeError: - tk.abort(404, tk._("Dataset not found")) - - if package["id"] != current_pkg["id"]: - log.info( - "Mismatch between pkg id in activity and URL {} {}".format( - package["id"], current_pkg["id"] - ) - ) - # the activity is not for the package in the URL - don't allow - # misleading URLs as could be malicious - tk.abort(404, tk._("Activity not found")) - # The name is used lots in the template for links, so fix it to be - # the current one. It's not displayed to the user anyway. - package["name"] = current_pkg["name"] - - # Don't crash on old (unmigrated) activity records, which do not - # include resources or extras. - package.setdefault("resources", []) - - resource = None - for res in package.get("resources", []): - if res["id"] == resource_id: - resource = res - break - if not resource: - return tk.abort(404, tk._("Resource not found")) - - # get package license info - license_id = package.get("license_id") - try: - package["isopen"] = model.Package.get_license_register()[ - license_id - ].isopen() - except KeyError: - package["isopen"] = False - - resource_views = tk.get_action("resource_view_list")( - context, {"id": resource_id} - ) - resource["has_views"] = len(resource_views) > 0 - - current_resource_view = None - view_id = tk.request.args.get("view_id") - if resource["has_views"]: - if view_id: - current_resource_view = [ - rv for rv in resource_views if rv["id"] == view_id - ] - if len(current_resource_view) == 1: - current_resource_view = current_resource_view[0] - else: - return tk.abort(404, tk._("Resource view not found")) - else: - current_resource_view = resource_views[0] - - # required for nav menu - pkg = context["package"] - dataset_type = pkg.type - - # TODO: remove - tk.g.package = package - tk.g.resource = resource - tk.g.pkg = pkg - tk.g.pkg_dict = package - - extra_vars: dict[str, Any] = { - "resource_views": resource_views, - "current_resource_view": current_resource_view, - "dataset_type": dataset_type, - "pkg_dict": package, - "package": package, - "resource": resource, - "pkg": pkg, # NB it is the current version of the dataset, so ignores - # activity_id. Still used though in resource views for - # backward compatibility - } - - return tk.render("package/resource_history.html", extra_vars) - - -@bp.route("/dataset//history/") -def package_history(id: str, activity_id: str) -> Union[Response, str]: - context = cast( - Context, - { - "for_view": True, - "auth_user_obj": tk.g.userobj, - }, - ) - data_dict = {"id": id, "include_tracking": True} - - # check if package exists - try: - pkg_dict = tk.get_action("package_show")(context, data_dict) - pkg = context["package"] - except (tk.ObjectNotFound, tk.NotAuthorized): - return tk.abort(404, tk._("Dataset not found")) - - # if the user specified a package id, redirect to the package name - if ( - data_dict["id"] == pkg_dict["id"] - and data_dict["id"] != pkg_dict["name"] - ): - return tk.h.redirect_to( - "activity.package_history", - id=pkg_dict["name"], - activity_id=activity_id, - ) - - tk.g.pkg_dict = pkg_dict - tk.g.pkg = pkg - # NB templates should not use g.pkg, because it takes no account of - # activity_id - - # view an 'old' version of the package, as recorded in the - # activity stream - try: - activity = tk.get_action("activity_show")( - context, {"id": activity_id, "include_data": True} - ) - except tk.ObjectNotFound: - tk.abort(404, tk._("Activity not found")) - except tk.NotAuthorized: - tk.abort(403, tk._("Unauthorized to view activity data")) - current_pkg = pkg_dict - try: - pkg_dict = activity["data"]["package"] - except KeyError: - tk.abort(404, tk._("Dataset not found")) - if "id" not in pkg_dict or "resources" not in pkg_dict: - log.info( - "Attempt to view unmigrated or badly migrated dataset " - "{} {}".format(id, activity_id) - ) - tk.abort( - 404, tk._("The detail of this dataset activity is not available") - ) - if pkg_dict["id"] != current_pkg["id"]: - log.info( - "Mismatch between pkg id in activity and URL {} {}".format( - pkg_dict["id"], current_pkg["id"] - ) - ) - # the activity is not for the package in the URL - don't allow - # misleading URLs as could be malicious - tk.abort(404, tk._("Activity not found")) - # The name is used lots in the template for links, so fix it to be - # the current one. It's not displayed to the user anyway. - pkg_dict["name"] = current_pkg["name"] - - # Earlier versions of CKAN only stored the package table in the - # activity, so add a placeholder for resources, or the template - # will crash. - pkg_dict.setdefault("resources", []) - - # can the resources be previewed? - for resource in pkg_dict["resources"]: - resource_views = tk.get_action("resource_view_list")( - context, {"id": resource["id"]} - ) - resource["has_views"] = len(resource_views) > 0 - - package_type = pkg_dict["type"] or "dataset" - _setup_template_variables(context, {"id": id}, package_type=package_type) - - return tk.render( - "package/history.html", - { - "dataset_type": package_type, - "pkg_dict": pkg_dict, - "pkg": pkg, - }, - ) - - -@bp.route("/dataset/activity/") -def package_activity(id: str) -> Union[Response, str]: # noqa - """Render this package's public activity stream page.""" - after = tk.request.args.get("after") - before = tk.request.args.get("before") - activity_type = tk.request.args.get("activity_type") - - context = cast( - Context, - { - "for_view": True, - "auth_user_obj": tk.g.userobj, - }, - ) - - data_dict = {"id": id} - limit = _get_activity_stream_limit() - activity_types = [activity_type] if activity_type else None - - try: - pkg_dict = tk.get_action("package_show")(context, data_dict) - activity_dict = { - "id": pkg_dict["id"], - "after": after, - "before": before, - # ask for one more just to know if this query has more results - "limit": limit + 1, - "activity_types": activity_types, - } - activity_stream = tk.get_action("package_activity_list")( - context, activity_dict - ) - dataset_type = pkg_dict["type"] or "dataset" - except tk.ObjectNotFound: - return tk.abort(404, tk._("Dataset not found")) - except tk.NotAuthorized: - return tk.abort(403, tk._("Unauthorized to read dataset %s") % id) - except tk.ValidationError: - return tk.abort(400, tk._("Invalid parameters")) - - has_more = len(activity_stream) > limit - # remove the extra item if exists - if has_more: - if after: - activity_stream.pop(0) - else: - activity_stream.pop() - - older_activities_url = _get_older_activities_url( - has_more, - activity_stream, - id=id, - activity_type=activity_type - ) - - newer_activities_url = _get_newer_activities_url( - has_more, - activity_stream, - id=id, - activity_type=activity_type - ) - - return tk.render( - "package/activity_stream.html", - { - "dataset_type": dataset_type, - "pkg_dict": pkg_dict, - "activity_stream": activity_stream, - "id": id, # i.e. package's current name - "limit": limit, - "has_more": has_more, - "activity_type": activity_type, - "activity_types": VALIDATORS_PACKAGE_ACTIVITY_TYPES.keys(), - "newer_activities_url": newer_activities_url, - "older_activities_url": older_activities_url, - }, - ) - - -@bp.route("/dataset/changes/") -def package_changes(id: str) -> Union[Response, str]: # noqa - """ - Shows the changes to a dataset in one particular activity stream item. - """ - activity_id = id - context = cast(Context, {"auth_user_obj": tk.g.userobj}) - try: - activity_diff = tk.get_action("activity_diff")( - context, - {"id": activity_id, "object_type": "package", "diff_type": "html"}, - ) - except tk.ObjectNotFound as e: - log.info("Activity not found: {} - {}".format(str(e), activity_id)) - return tk.abort(404, tk._("Activity not found")) - except tk.NotAuthorized: - return tk.abort(403, tk._("Unauthorized to view activity data")) - - # 'pkg_dict' needs to go to the templates for page title & breadcrumbs. - # Use the current version of the package, in case the name/title have - # changed, and we need a link to it which works - pkg_id = activity_diff["activities"][1]["data"]["package"]["id"] - current_pkg_dict = tk.get_action("package_show")(context, {"id": pkg_id}) - pkg_activity_list = tk.get_action("package_activity_list")( - context, {"id": pkg_id, "limit": 100} - ) - - return tk.render( - "package/changes.html", - { - "activity_diffs": [activity_diff], - "pkg_dict": current_pkg_dict, - "pkg_activity_list": pkg_activity_list, - "dataset_type": current_pkg_dict["type"], - }, - ) - - -@bp.route("/dataset/changes_multiple") -def package_changes_multiple() -> Union[Response, str]: # noqa - """ - Called when a user specifies a range of versions they want to look at - changes between. Verifies that the range is valid and finds the set of - activity diffs for the changes in the given version range, then - re-renders changes.html with the list. - """ - - new_id = tk.h.get_request_param("new_id") - old_id = tk.h.get_request_param("old_id") - - context = cast(Context, {"auth_user_obj": tk.g.userobj}) - - # check to ensure that the old activity is actually older than - # the new activity - old_activity = tk.get_action("activity_show")( - context, {"id": old_id, "include_data": False} - ) - new_activity = tk.get_action("activity_show")( - context, {"id": new_id, "include_data": False} - ) - - old_timestamp = old_activity["timestamp"] - new_timestamp = new_activity["timestamp"] - - t1 = datetime.strptime(old_timestamp, "%Y-%m-%dT%H:%M:%S.%f") - t2 = datetime.strptime(new_timestamp, "%Y-%m-%dT%H:%M:%S.%f") - - time_diff = t2 - t1 - # if the time difference is negative, just return the change that put us - # at the more recent ID we were just looking at - # TODO: do something better here - go back to the previous page, - # display a warning that the user can't look at a sequence where - # the newest item is older than the oldest one, etc - if time_diff.total_seconds() <= 0: - return package_changes(tk.h.get_request_param("current_new_id")) - - done = False - current_id = new_id - diff_list = [] - - while not done: - try: - activity_diff = tk.get_action("activity_diff")( - context, - { - "id": current_id, - "object_type": "package", - "diff_type": "html", - }, - ) - except tk.ObjectNotFound as e: - log.info("Activity not found: {} - {}".format(str(e), current_id)) - return tk.abort(404, tk._("Activity not found")) - except tk.NotAuthorized: - return tk.abort(403, tk._("Unauthorized to view activity data")) - - diff_list.append(activity_diff) - - if activity_diff["activities"][0]["id"] == old_id: - done = True - else: - current_id = activity_diff["activities"][0]["id"] - - pkg_id: str = diff_list[0]["activities"][1]["data"]["package"]["id"] - current_pkg_dict = tk.get_action("package_show")(context, {"id": pkg_id}) - pkg_activity_list = tk.get_action("package_activity_list")( - context, {"id": pkg_id, "limit": 100} - ) - - return tk.render( - "package/changes.html", - { - "activity_diffs": diff_list, - "pkg_dict": current_pkg_dict, - "pkg_activity_list": pkg_activity_list, - "dataset_type": current_pkg_dict["type"], - }, - ) - - -@bp.route( - "/group/activity/", - endpoint="group_activity", - defaults={"group_type": "group"}, -) -@bp.route( - "/organization/activity/", - endpoint="organization_activity", - defaults={"group_type": "organization"}, -) -def group_activity(id: str, group_type: str) -> str: - """Render this group's public activity stream page.""" - after = tk.request.args.get("after") - before = tk.request.args.get("before") - - if group_type == 'organization': - set_org(True) - - context = cast(Context, {"user": tk.g.user, "for_view": True}) - - try: - group_dict = _get_group_dict(id, group_type) - except (tk.ObjectNotFound, tk.NotAuthorized): - tk.abort(404, tk._("Group not found")) - - real_group_type = group_dict["type"] - action_name = "organization_activity_list" - if not group_dict.get("is_organization"): - action_name = "group_activity_list" - - activity_type = tk.request.args.get("activity_type") - activity_types = [activity_type] if activity_type else None - - limit = _get_activity_stream_limit() - - try: - activity_stream = tk.get_action(action_name)( - context, { - "id": group_dict["id"], - "before": before, - "after": after, - "limit": limit + 1, - "activity_types": activity_types - } - ) - except tk.ValidationError as error: - tk.abort(400, error.message or "") - - filter_types = VALIDATORS_PACKAGE_ACTIVITY_TYPES.copy() - if group_type == 'organization': - filter_types.update(VALIDATORS_ORGANIZATION_ACTIVITY_TYPES) - else: - filter_types.update(VALIDATORS_GROUP_ACTIVITY_TYPES) - - has_more = len(activity_stream) > limit - # remove the extra item if exists - if has_more: - if after: - activity_stream.pop(0) - else: - activity_stream.pop() - - older_activities_url = _get_older_activities_url( - has_more, - activity_stream, - id=id, - activity_type=activity_type - ) - - newer_activities_url = _get_newer_activities_url( - has_more, - activity_stream, - id=id, - activity_type=activity_type - ) - - extra_vars = { - "id": id, - "activity_stream": activity_stream, - "group_type": real_group_type, - "group_dict": group_dict, - "activity_type": activity_type, - "activity_types": filter_types.keys(), - "newer_activities_url": newer_activities_url, - "older_activities_url": older_activities_url - } - - return tk.render( - _get_group_template("activity_template", group_type), extra_vars - ) - - -@bp.route( - "/group/changes/", - defaults={"is_organization": False, "group_type": "group"}, -) -@bp.route( - "/organization/changes/", - endpoint="organization_changes", - defaults={"is_organization": True, "group_type": "organization"}, -) -def group_changes(id: str, group_type: str, is_organization: bool) -> str: - """ - Shows the changes to an organization in one particular activity stream - item. - """ - extra_vars = {} - activity_id = id - context = cast( - Context, - { - "auth_user_obj": tk.g.userobj, - }, - ) - try: - activity_diff = tk.get_action("activity_diff")( - context, - {"id": activity_id, "object_type": "group", "diff_type": "html"}, - ) - except tk.ObjectNotFound as e: - log.info("Activity not found: {} - {}".format(str(e), activity_id)) - return tk.abort(404, tk._("Activity not found")) - except tk.NotAuthorized: - return tk.abort(403, tk._("Unauthorized to view activity data")) - - # 'group_dict' needs to go to the templates for page title & breadcrumbs. - # Use the current version of the package, in case the name/title have - # changed, and we need a link to it which works - group_id = activity_diff["activities"][1]["data"]["group"]["id"] - current_group_dict = tk.get_action(group_type + "_show")( - context, {"id": group_id} - ) - group_activity_list = tk.get_action(group_type + "_activity_list")( - context, {"id": group_id, "limit": 100} - ) - - extra_vars: dict[str, Any] = { - "activity_diffs": [activity_diff], - "group_dict": current_group_dict, - "group_activity_list": group_activity_list, - "group_type": current_group_dict["type"], - } - - return tk.render(_replace_group_org("group/changes.html"), extra_vars) - - -@bp.route( - "/group/changes_multiple", - defaults={"is_organization": False, "group_type": "group"}, -) -@bp.route( - "/organization/changes_multiple", - endpoint="organization_changes_multiple", - defaults={"is_organization": True, "group_type": "organization"}, -) -def group_changes_multiple(is_organization: bool, group_type: str) -> str: - """ - Called when a user specifies a range of versions they want to look at - changes between. Verifies that the range is valid and finds the set of - activity diffs for the changes in the given version range, then - re-renders changes.html with the list. - """ - extra_vars = {} - new_id = tk.h.get_request_param("new_id") - old_id = tk.h.get_request_param("old_id") - - context = cast( - Context, - { - "auth_user_obj": tk.g.userobj, - }, - ) - - # check to ensure that the old activity is actually older than - # the new activity - old_activity = tk.get_action("activity_show")( - context, {"id": old_id, "include_data": False} - ) - new_activity = tk.get_action("activity_show")( - context, {"id": new_id, "include_data": False} - ) - - old_timestamp = old_activity["timestamp"] - new_timestamp = new_activity["timestamp"] - - t1 = datetime.strptime(old_timestamp, "%Y-%m-%dT%H:%M:%S.%f") - t2 = datetime.strptime(new_timestamp, "%Y-%m-%dT%H:%M:%S.%f") - - time_diff = t2 - t1 - # if the time difference is negative, just return the change that put us - # at the more recent ID we were just looking at - # TODO: do something better here - go back to the previous page, - # display a warning that the user can't look at a sequence where - # the newest item is older than the oldest one, etc - if time_diff.total_seconds() < 0: - return group_changes( - tk.h.get_request_param("current_new_id"), - group_type, - is_organization, - ) - - done = False - current_id = new_id - diff_list = [] - - while not done: - try: - activity_diff = tk.get_action("activity_diff")( - context, - { - "id": current_id, - "object_type": "group", - "diff_type": "html", - }, - ) - except tk.ObjectNotFound as e: - log.info("Activity not found: {} - {}".format(str(e), current_id)) - return tk.abort(404, tk._("Activity not found")) - except tk.NotAuthorized: - return tk.abort(403, tk._("Unauthorized to view activity data")) - - diff_list.append(activity_diff) - - if activity_diff["activities"][0]["id"] == old_id: - done = True - else: - current_id = activity_diff["activities"][0]["id"] - - group_id: str = diff_list[0]["activities"][1]["data"]["group"]["id"] - current_group_dict = tk.get_action(group_type + "_show")( - context, {"id": group_id} - ) - group_activity_list = tk.get_action(group_type + "_activity_list")( - context, {"id": group_id, "limit": 100} - ) - - extra_vars: dict[str, Any] = { - "activity_diffs": diff_list, - "group_dict": current_group_dict, - "group_activity_list": group_activity_list, - "group_type": current_group_dict["type"], - } - - return tk.render(_replace_group_org("group/changes.html"), extra_vars) - - -@bp.route("/user/activity/") -def user_activity(id: str) -> str: - """Render this user's public activity stream page.""" - after = tk.request.args.get("after") - before = tk.request.args.get("before") - - context = cast( - Context, - { - "auth_user_obj": tk.g.userobj, - "for_view": True, - }, - ) - data_dict: dict[str, Any] = { - "id": id, - "user_obj": tk.g.userobj, - "include_num_followers": True, - } - try: - tk.check_access("user_show", context, data_dict) - except tk.NotAuthorized: - tk.abort(403, tk._("Not authorized to see this page")) - - extra_vars = _extra_template_variables(context, data_dict) - - limit = _get_activity_stream_limit() - - try: - activity_stream = tk.get_action( - "user_activity_list" - )(context, { - "id": extra_vars["user_dict"]["id"], - "before": before, - "after": after, - "limit": limit + 1, - }) - except tk.ValidationError: - tk.abort(400) - - has_more = len(activity_stream) > limit - # remove the extra item if exists - if has_more: - if after: - activity_stream.pop(0) - else: - activity_stream.pop() - - older_activities_url = _get_older_activities_url( - has_more, - activity_stream, - id=id - ) - - newer_activities_url = _get_newer_activities_url( - has_more, - activity_stream, - id=id - ) - - extra_vars.update({ - "id": id, - "activity_stream": activity_stream, - "newer_activities_url": newer_activities_url, - "older_activities_url": older_activities_url - }) - - return tk.render("user/activity_stream.html", extra_vars) - - -@bp.route("/dashboard/", strict_slashes=False) -def dashboard() -> str: - context = cast( - Context, - { - "auth_user_obj": tk.g.userobj, - "for_view": True, - }, - ) - data_dict: dict[str, Any] = {"user_obj": tk.g.userobj} - extra_vars = _extra_template_variables(context, data_dict) - - q = tk.request.args.get("q", "") - filter_type = tk.request.args.get("type", "") - filter_id = tk.request.args.get("name", "") - before = tk.request.args.get("before") - after = tk.request.args.get("after") - - limit = _get_activity_stream_limit() - - extra_vars["followee_list"] = tk.get_action("followee_list")( - context, {"id": tk.g.userobj.id, "q": q} - ) - extra_vars["dashboard_activity_stream_context"] = _get_dashboard_context( - filter_type, filter_id, q - ) - activity_stream = tk.h.dashboard_activity_stream( - tk.g.userobj.id, - filter_type=filter_type, - filter_id=filter_id, - limit=limit + 1, - before=before, - after=after - ) - - has_more = len(activity_stream) > limit - # remove the extra item if exists - if has_more: - if after: - activity_stream.pop(0) - else: - activity_stream.pop() - - older_activities_url = _get_older_activities_url( - has_more, - activity_stream, - type=filter_type, - name=filter_id - ) - - newer_activities_url = _get_newer_activities_url( - has_more, - activity_stream, - type=filter_type, - name=filter_id - ) - - extra_vars.update({ - "id": id, - "dashboard_activity_stream": activity_stream, - "newer_activities_url": newer_activities_url, - "older_activities_url": older_activities_url - }) - - # Mark the user's new activities as old whenever they view their - # dashboard page. - tk.get_action("dashboard_mark_activities_old")(context, {}) - - return tk.render("user/dashboard.html", extra_vars) - - -def _get_dashboard_context( - filter_type: Optional[str] = None, - filter_id: Optional[str] = None, - q: Optional[str] = None, -) -> dict[str, Any]: - """Return a dict needed by the dashboard view to determine context.""" - - def display_name(followee: dict[str, Any]) -> Optional[str]: - """Return a display name for a user, group or dataset dict.""" - display_name = followee.get("display_name") - fullname = followee.get("fullname") - title = followee.get("title") - name = followee.get("name") - return display_name or fullname or title or name - - if filter_type and filter_id: - context = cast( - Context, - { - "auth_user_obj": tk.g.userobj, - "for_view": True, - }, - ) - data_dict: dict[str, Any] = { - "id": filter_id, - "include_num_followers": True, - } - followee = None - - action_functions = { - "dataset": "package_show", - "user": "user_show", - "group": "group_show", - "organization": "organization_show", - } - action_name = action_functions.get(filter_type) - if action_name is None: - tk.abort(404, tk._("Follow item not found")) - - action_function = tk.get_action(action_name) - try: - followee = action_function(context, data_dict) - except (tk.ObjectNotFound, tk.NotAuthorized): - tk.abort(404, tk._("{0} not found").format(filter_type)) - - if followee is not None: - return { - "filter_type": filter_type, - "q": q, - "context": display_name(followee), - "selected_id": followee.get("id"), - "dict": followee, - } - - return { - "filter_type": filter_type, - "q": q, - "context": tk._("Everything"), - "selected_id": False, - "dict": None, - } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/plugin.py deleted file mode 100644 index e827961..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/audioview/plugin.py +++ /dev/null @@ -1,50 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from typing import Any - -import ckan.plugins as p -from ckan.types import Context, DataDict -from ckan.common import CKANConfig -from ckan.config.declaration import Declaration, Key - -ignore_empty = p.toolkit.get_validator('ignore_empty') -unicode_safe = p.toolkit.get_validator('unicode_safe') - - -class AudioView(p.SingletonPlugin): - '''This plugin makes views of audio resources, using an link to the audio instead. - {% endtrans %} - diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/plugin.py deleted file mode 100644 index 5b0bc57..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/plugin.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import annotations - -from ckan.types import Action, AuthFunction, Context, DataDict -from typing import Any, Callable, Optional -import ckan.plugins as p -from ckan.plugins.toolkit import (auth_allow_anonymous_access, - chained_auth_function, - chained_action, - side_effect_free, - chained_helper - ) - - -class ChainedFunctionsPlugin(p.SingletonPlugin): - p.implements(p.IAuthFunctions) - p.implements(p.IActions) - p.implements(p.ITemplateHelpers) - - def get_auth_functions(self): - return { - "user_show": user_show - } - - def get_actions(self): - return { - "package_search": package_search - } - - def get_helpers(self) -> dict[str, Callable[..., Any]]: - return { - "ckan_version": ckan_version - } - - -@auth_allow_anonymous_access -@chained_auth_function -def user_show(next_auth: AuthFunction, context: Context, - data_dict: Optional[DataDict] = None): - return next_auth(context, data_dict) # type: ignore - - -@side_effect_free -@chained_action -def package_search(original_action: Action, context: Context, - data_dict: DataDict): - return original_action(context, data_dict) - - -@chained_helper -def ckan_version(next_func: Callable[..., Any], **kw: Any): - return next_func(**kw) - - -setattr(ckan_version, "some_attribute", "some_value") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/test_plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/test_plugin.py deleted file mode 100644 index 5e20737..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/chained_functions/tests/test_plugin.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -import pytest - - -@pytest.mark.ckan_config(u"ckan.plugins", u"chained_functions") -@pytest.mark.usefixtures(u"with_plugins") -class TestChainedFunctionsPlugin(object): - def test_auth_attributes_are_retained(self): - from ckan.authz import _AuthFunctions - user_show = _AuthFunctions.get("user_show") - assert hasattr(user_show, 'auth_allow_anonymous_access') is True - - def test_action_attributes_are_retained(self): - from ckan.plugins.toolkit import get_action - package_search = get_action('package_search') - assert hasattr(package_search, 'side_effect_free') is True - - def test_helper_attributes_are_retained(self): - from ckan.lib.helpers import helper_functions - ckan_version = helper_functions.get('ckan_version') - assert hasattr(ckan_version, "some_attribute") is True diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/home.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/home.py index 5192b52..2a9982d 100644 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/home.py +++ b/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/home.py @@ -31,7 +31,7 @@ class d4SHomeController(): default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), - 'tags': _('Tags'), + # 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/systemtype.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/systemtype.py index 37e5593..d951abf 100644 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/systemtype.py +++ b/compose/2.10/src/ckanext-d4science_theme/ckanext/d4science_theme/controllers/systemtype.py @@ -45,7 +45,7 @@ class d4STypeController(): default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), - 'tags': _('Tags'), + # 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css deleted file mode 100644 index e2d61a0..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css +++ /dev/null @@ -1,33 +0,0 @@ -.datapusher-activity { - padding: 0; - list-style-type: none; - background: transparent url("/dotted.png") 21px 0 repeat-y; } - .datapusher-activity .item { - position: relative; - margin: 0 0 15px 0; - padding: 0; } - .datapusher-activity .item .date { - color: #999; - font-size: 12px; - white-space: nowrap; - margin: 5px 0 0 80px; } - .datapusher-activity .item.no-avatar p { - margin-left: 40px; } - -.popover { - width: 300px; } - .popover .popover-header { - font-weight: bold; - margin-bottom: 0; } - .popover .popover-close { - float: right; - text-decoration: none; } - -.datapusher-activity .item .icon { - color: #999999; } -.datapusher-activity .item.failure .icon { - color: #B95252; } -.datapusher-activity .item.success .icon { - color: #69A67A; } - -/*# sourceMappingURL=datapusher.css.map */ diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map deleted file mode 100644 index d5aad5f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.css.map +++ /dev/null @@ -1,7 +0,0 @@ -{ -"version": 3, -"mappings": "AAAA,oBAAqB;EACjB,OAAO,EAAE,CAAC;EACV,eAAe,EAAE,IAAI;EACrB,UAAU,EAAE,8CAA+C;EAC3D,0BAAM;IACF,QAAQ,EAAE,QAAQ;IAClB,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,CAAC;IACV,gCAAM;MACF,KAAK,EAAE,IAAI;MACX,SAAS,EAAE,IAAI;MACf,WAAW,EAAE,MAAM;MACnB,MAAM,EAAE,YAAY;IAExB,sCAAc;MACV,WAAW,EAAE,IAAI;;AAM7B,QAAS;EACL,KAAK,EAAE,KAAK;EACZ,wBAAgB;IACZ,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,CAAC;EAEpB,uBAAe;IACX,KAAK,EAAE,KAAK;IACZ,eAAe,EAAE,IAAI;;AAUzB,gCAAM;EACF,KAAK,EANQ,OAAO;AAQxB,wCAAgB;EACZ,KAAK,EAPS,OAAO;AASzB,wCAAgB;EACZ,KAAK,EAXM,OAAO", -"sources": ["datapusher.scss"], -"names": [], -"file": "datapusher.css" -} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss deleted file mode 100644 index 44617be..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher.scss +++ /dev/null @@ -1,49 +0,0 @@ -.datapusher-activity { - padding: 0; - list-style-type: none; - background: transparent url('/dotted.png') 21px 0 repeat-y; - .item { - position: relative; - margin: 0 0 15px 0; - padding: 0; - .date { - color: #999; - font-size: 12px; - white-space: nowrap; - margin: 5px 0 0 80px; - } - &.no-avatar p { - margin-left: 40px; - } - } -} - -// popover -.popover { - width: 300px; - .popover-header { - font-weight: bold; - margin-bottom: 0; - } - .popover-close { - float: right; - text-decoration: none; - } -} - -// base colors -$activityColorBlank: #999999; -$activityColorNew: #69A67A; -$activityColorDelete: #B95252; - -.datapusher-activity .item { - .icon { - color: $activityColorBlank ; - } - &.failure .icon { - color: $activityColorDelete; - } - &.success .icon { - color: $activityColorNew; - } -} \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js deleted file mode 100644 index 47d9887..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/datapusher_popover.js +++ /dev/null @@ -1,27 +0,0 @@ -"use strict"; - -/* Datapusher popover - * Handle the popup display in 'resource view page' activity stream - * - * Options - * - title: title of the popover - * - content: content for the popover - */ -ckan.module('datapusher_popover', function($) { - return { - initialize: function() { - $.proxyAll(this, /_on/); - this.popover = this.el; - this.popover.popover({ - html: true, - title: this.options.title, - content: this.options.content - }); - $(document).on('click', '.popover-close', this._onClose); - }, - - _onClose: function() { - this.popover.popover('hide'); - } - }; -}); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml deleted file mode 100644 index f7f18bb..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/assets/webassets.yml +++ /dev/null @@ -1,13 +0,0 @@ -datapusher: - filters: rjsmin - output: datapusher/%(version)s_datapusher.js - extra: - preload: - - base/main - contents: - - datapusher_popover.js - -datapusher-css: - output: ckanext-datapusher/%(version)s_datapusher.css - contents: - - datapusher.css diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/cli.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/cli.py deleted file mode 100644 index bfb33d3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/cli.py +++ /dev/null @@ -1,105 +0,0 @@ -# encoding: utf-8 - -from __future__ import annotations - -from ckan.types import Context - -import logging -from typing import cast - -import click - -import ckan.model as model -import ckan.plugins.toolkit as tk -import ckanext.datastore.backend as datastore_backend -from ckan.cli import error_shout - -log = logging.getLogger(__name__) - -question = ( - u"Data in any datastore resource that isn't in their source files " - u"(e.g. data added using the datastore API) will be permanently " - u"lost. Are you sure you want to proceed?" -) -requires_confirmation = click.option( - u'--yes', u'-y', is_flag=True, help=u'Always answer yes to questions' -) - - -def confirm(yes: bool): - if yes: - return - click.confirm(question, abort=True) - - -@click.group(short_help=u"Perform commands in the datapusher.") -def datapusher(): - """Perform commands in the datapusher. - """ - pass - - -@datapusher.command() -@requires_confirmation -def resubmit(yes: bool): - u'''Resubmit updated datastore resources. - ''' - confirm(yes) - - resource_ids = datastore_backend.get_all_resources_ids_in_datastore() - _submit(resource_ids) - - -@datapusher.command() -@click.argument(u'package', required=False) -@requires_confirmation -def submit(package: str, yes: bool): - u'''Submits resources from package. - - If no package ID/name specified, submits all resources from all - packages. - ''' - confirm(yes) - - if not package: - ids = tk.get_action(u'package_list')(cast(Context, { - u'model': model, - u'ignore_auth': True - }), {}) - else: - ids = [package] - - for id in ids: - package_show = tk.get_action(u'package_show') - try: - pkg = package_show(cast(Context, { - u'model': model, - u'ignore_auth': True - }), {u'id': id}) - except Exception as e: - error_shout(e) - error_shout(u"Package '{}' was not found".format(package)) - raise click.Abort() - if not pkg[u'resources']: - continue - resource_ids = [r[u'id'] for r in pkg[u'resources']] - _submit(resource_ids) - - -def _submit(resources: list[str]): - click.echo(u'Submitting {} datastore resources'.format(len(resources))) - user = tk.get_action(u'get_site_user')(cast(Context, { - u'model': model, - u'ignore_auth': True - }), {}) - datapusher_submit = tk.get_action(u'datapusher_submit') - for id in resources: - click.echo(u'Submitting {}...'.format(id), nl=False) - data_dict = { - u'resource_id': id, - u'ignore_hash': True, - } - if datapusher_submit({u'user': user[u'name']}, data_dict): - click.echo(u'OK') - else: - click.echo(u'Fail') diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml deleted file mode 100644 index e5d440d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/config_declaration.yaml +++ /dev/null @@ -1,51 +0,0 @@ -version: 1 -groups: -- annotation: Datapusher settings - options: - - key: ckan.datapusher.formats - default: - - csv - - xls - - xlsx - - tsv - - application/csv - - application/vnd.ms-excel - - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - - ods - - application/vnd.oasis.opendocument.spreadsheet - type: list - example: csv xls - description: >- - File formats that will be pushed to the DataStore by the DataPusher. When - adding or editing a resource which links to a file in one of these formats, - the DataPusher will automatically try to import its contents to the DataStore. - - - key: ckan.datapusher.url - example: http://127.0.0.1:8800/ - description: >- - DataPusher endpoint to use when enabling the ``datapusher`` extension. If you - installed CKAN via :doc:`/maintaining/installing/install-from-package`, the DataPusher was installed for you - running on port 8800. If you want to manually install the DataPusher, follow - the installation `instructions `_. - - - key: ckan.datapusher.api_token - description: >- - Starting from CKAN 2.10, DataPusher requires a valid API token to operate (see :ref:`api authentication`), and - will fail to start if this option is not set. - - - key: ckan.datapusher.callback_url_base - example: http://ckan:5000/ - placeholder: '%(ckan.site_url)s' - description: >- - Alternative callback URL for DataPusher when performing a request to CKAN. This is - useful on scenarios where the host where DataPusher is running can not access the - public CKAN site URL. - - - key: ckan.datapusher.assume_task_stale_after - default: 3600 - type: int - example: 86400 - description: >- - In case a DataPusher task gets stuck and fails to recover, this is the minimum - amount of time (in seconds) after a resource is submitted to DataPusher that the - resource can be submitted again. diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py deleted file mode 100644 index 551c644..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/helpers.py +++ /dev/null @@ -1,31 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from typing import Any -import ckan.plugins.toolkit as toolkit - - -def datapusher_status(resource_id: str): - try: - return toolkit.get_action('datapusher_status')( - {}, {'resource_id': resource_id}) - except toolkit.ObjectNotFound: - return { - 'status': 'unknown' - } - - -def datapusher_status_description(status: dict[str, Any]): - _ = toolkit._ - - if status.get('status'): - captions = { - 'complete': _('Complete'), - 'pending': _('Pending'), - 'submitting': _('Submitting'), - 'error': _('Error'), - } - - return captions.get(status['status'], status['status'].capitalize()) - else: - return _('Not Uploaded Yet') diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py deleted file mode 100644 index 2a7e5d2..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/interfaces.py +++ /dev/null @@ -1,55 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from ckan.plugins.interfaces import Interface -from typing import Any - - -class IDataPusher(Interface): - """ - The IDataPusher interface allows plugin authors to receive notifications - before and after a resource is submitted to the datapusher service, as - well as determining whether a resource should be submitted in can_upload - - The before_submit function, when implemented - """ - - def can_upload(self, resource_id: str) -> bool: - """ This call when implemented can be used to stop the processing of - the datapusher submit function. This method will not be called if - the resource format does not match those defined in the - ckan.datapusher.formats config option or the default formats. - - If this function returns False then processing will be aborted, - whilst returning True will submit the resource to the datapusher - service - - Note that before reaching this hook there is a prior check on the - resource format, which depends on the value of - the :ref:`ckan.datapusher.formats` configuration option (and requires - the resource to have a format defined). - - :param resource_id: The ID of the resource that is to be - pushed to the datapusher service. - - Returns ``True`` if the job should be submitted and ``False`` if - the job should be aborted - - :rtype: bool - """ - return True - - def after_upload( - self, context: dict[str, Any], resource_dict: dict[str, Any], - dataset_dict: dict[str, Any]) -> None: - """ After a resource has been successfully upload to the datastore - this method will be called with the resource dictionary and the - package dictionary for this resource. - - :param context: The context within which the upload happened - :param resource_dict: The dict represenstaion of the resource that was - successfully uploaded to the datastore - :param dataset_dict: The dict represenstation of the dataset containing - the resource that was uploaded - """ - pass diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py deleted file mode 100644 index 1c172e7..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/action.py +++ /dev/null @@ -1,338 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from ckan.types import Context -import logging -import json -import datetime -import time -from typing import Any, cast - -from urllib.parse import urljoin -from dateutil.parser import parse as parse_date - -import requests - -import ckan.lib.helpers as h -import ckan.lib.navl.dictization_functions -import ckan.logic as logic -import ckan.plugins as p -from ckan.common import config -import ckanext.datapusher.logic.schema as dpschema -import ckanext.datapusher.interfaces as interfaces - -log = logging.getLogger(__name__) -_get_or_bust = logic.get_or_bust -_validate = ckan.lib.navl.dictization_functions.validate - - -def datapusher_submit(context: Context, data_dict: dict[str, Any]): - ''' Submit a job to the datapusher. The datapusher is a service that - imports tabular data into the datastore. - - :param resource_id: The resource id of the resource that the data - should be imported in. The resource's URL will be used to get the data. - :type resource_id: string - :param set_url_type: If set to True, the ``url_type`` of the resource will - be set to ``datastore`` and the resource URL will automatically point - to the :ref:`datastore dump ` URL. (optional, default: False) - :type set_url_type: bool - :param ignore_hash: If set to True, the datapusher will reload the file - even if it haven't changed. (optional, default: False) - :type ignore_hash: bool - - Returns ``True`` if the job has been submitted and ``False`` if the job - has not been submitted, i.e. when the datapusher is not configured. - - :rtype: bool - ''' - schema = context.get('schema', dpschema.datapusher_submit_schema()) - data_dict, errors = _validate(data_dict, schema, context) - if errors: - raise p.toolkit.ValidationError(errors) - - res_id = data_dict['resource_id'] - - p.toolkit.check_access('datapusher_submit', context, data_dict) - - try: - resource_dict = p.toolkit.get_action('resource_show')(context, { - 'id': res_id, - }) - except logic.NotFound: - return False - - datapusher_url: str = config.get('ckan.datapusher.url') - - callback_url_base = config.get( - 'ckan.datapusher.callback_url_base' - ) or config.get("ckan.site_url") - if callback_url_base: - site_url = callback_url_base - callback_url = urljoin( - callback_url_base.rstrip('/'), '/api/3/action/datapusher_hook') - else: - site_url = h.url_for('home.index', qualified=True) - callback_url = h.url_for( - '/api/3/action/datapusher_hook', qualified=True) - - for plugin in p.PluginImplementations(interfaces.IDataPusher): - upload = plugin.can_upload(res_id) - if not upload: - msg = "Plugin {0} rejected resource {1}"\ - .format(plugin.__class__.__name__, res_id) - log.info(msg) - return False - - task = { - 'entity_id': res_id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'last_updated': str(datetime.datetime.utcnow()), - 'state': 'submitting', - 'key': 'datapusher', - 'value': '{}', - 'error': '{}', - } - try: - existing_task = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'datapusher' - }) - assume_task_stale_after = datetime.timedelta( - seconds=config.get( - 'ckan.datapusher.assume_task_stale_after')) - if existing_task.get('state') == 'pending': - updated = datetime.datetime.strptime( - existing_task['last_updated'], '%Y-%m-%dT%H:%M:%S.%f') - time_since_last_updated = datetime.datetime.utcnow() - updated - if time_since_last_updated > assume_task_stale_after: - # it's been a while since the job was last updated - it's more - # likely something went wrong with it and the state wasn't - # updated than its still in progress. Let it be restarted. - log.info('A pending task was found %r, but it is only %s hours' - ' old', existing_task['id'], time_since_last_updated) - else: - log.info('A pending task was found %s for this resource, so ' - 'skipping this duplicate task', existing_task['id']) - return False - - task['id'] = existing_task['id'] - except logic.NotFound: - pass - - context['ignore_auth'] = True - # Use local session for task_status_update, so it can commit its own - # results without messing up with the parent session that contains pending - # updats of dataset/resource/etc. - context['session'] = cast( - Any, context['model'].meta.create_local_session()) - p.toolkit.get_action('task_status_update')(context, task) - - timeout = config.get('ckan.requests.timeout') - - # This setting is checked on startup - api_token = p.toolkit.config.get("ckan.datapusher.api_token") - try: - r = requests.post( - urljoin(datapusher_url, 'job'), - headers={ - 'Content-Type': 'application/json' - }, - timeout=timeout, - data=json.dumps({ - 'api_key': api_token, - 'job_type': 'push_to_datastore', - 'result_url': callback_url, - 'metadata': { - 'ignore_hash': data_dict.get('ignore_hash', False), - 'ckan_url': site_url, - 'resource_id': res_id, - 'set_url_type': data_dict.get('set_url_type', False), - 'task_created': task['last_updated'], - 'original_url': resource_dict.get('url'), - } - })) - except requests.exceptions.ConnectionError as e: - error: dict[str, Any] = {'message': 'Could not connect to DataPusher.', - 'details': str(e)} - task['error'] = json.dumps(error) - task['state'] = 'error' - task['last_updated'] = str(datetime.datetime.utcnow()), - p.toolkit.get_action('task_status_update')(context, task) - raise p.toolkit.ValidationError(error) - try: - r.raise_for_status() - except requests.exceptions.HTTPError as e: - m = 'An Error occurred while sending the job: {0}'.format(str(e)) - try: - body = e.response.json() - if body.get('error'): - m += ' ' + body['error'] - except ValueError: - body = e.response.text - error = {'message': m, - 'details': body, - 'status_code': r.status_code} - task['error'] = json.dumps(error) - task['state'] = 'error' - task['last_updated'] = str(datetime.datetime.utcnow()), - p.toolkit.get_action('task_status_update')(context, task) - raise p.toolkit.ValidationError(error) - - value = json.dumps({'job_id': r.json()['job_id'], - 'job_key': r.json()['job_key']}) - - task['value'] = value - task['state'] = 'pending' - task['last_updated'] = str(datetime.datetime.utcnow()), - p.toolkit.get_action('task_status_update')(context, task) - - return True - - -def datapusher_hook(context: Context, data_dict: dict[str, Any]): - ''' Update datapusher task. This action is typically called by the - datapusher whenever the status of a job changes. - - :param metadata: metadata produced by datapuser service must have - resource_id property. - :type metadata: dict - :param status: status of the job from the datapusher service - :type status: string - ''' - - metadata, status = _get_or_bust(data_dict, ['metadata', 'status']) - - res_id = _get_or_bust(metadata, 'resource_id') - - # Pass metadata, not data_dict, as it contains the resource id needed - # on the auth checks - p.toolkit.check_access('datapusher_submit', context, metadata) - - task = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'datapusher' - }) - - task['state'] = status - task['last_updated'] = str(datetime.datetime.utcnow()) - - resubmit = False - - if status == 'complete': - # Create default views for resource if necessary (only the ones that - # require data to be in the DataStore) - resource_dict = p.toolkit.get_action('resource_show')( - context, {'id': res_id}) - - dataset_dict = p.toolkit.get_action('package_show')( - context, {'id': resource_dict['package_id']}) - - for plugin in p.PluginImplementations(interfaces.IDataPusher): - plugin.after_upload(cast("dict[str, Any]", context), - resource_dict, dataset_dict) - - logic.get_action('resource_create_default_resource_views')( - context, - { - 'resource': resource_dict, - 'package': dataset_dict, - 'create_datastore_views': True, - }) - - # Check if the uploaded file has been modified in the meantime - if (resource_dict.get('last_modified') and - metadata.get('task_created')): - try: - last_modified_datetime = parse_date( - resource_dict['last_modified']) - task_created_datetime = parse_date(metadata['task_created']) - if last_modified_datetime > task_created_datetime: - log.debug('Uploaded file more recent: {0} > {1}'.format( - last_modified_datetime, task_created_datetime)) - resubmit = True - except ValueError: - pass - # Check if the URL of the file has been modified in the meantime - elif (resource_dict.get('url') and - metadata.get('original_url') and - resource_dict['url'] != metadata['original_url']): - log.debug('URLs are different: {0} != {1}'.format( - resource_dict['url'], metadata['original_url'])) - resubmit = True - - context['ignore_auth'] = True - p.toolkit.get_action('task_status_update')(context, task) - - if resubmit: - log.debug('Resource {0} has been modified, ' - 'resubmitting to DataPusher'.format(res_id)) - p.toolkit.get_action('datapusher_submit')( - context, {'resource_id': res_id}) - - -def datapusher_status( - context: Context, data_dict: dict[str, Any]) -> dict[str, Any]: - ''' Get the status of a datapusher job for a certain resource. - - :param resource_id: The resource id of the resource that you want the - datapusher status for. - :type resource_id: string - ''' - - p.toolkit.check_access('datapusher_status', context, data_dict) - - if 'id' in data_dict: - data_dict['resource_id'] = data_dict['id'] - res_id = _get_or_bust(data_dict, 'resource_id') - - task = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'datapusher' - }) - - datapusher_url = config.get('ckan.datapusher.url') - if not datapusher_url: - raise p.toolkit.ValidationError( - {'configuration': ['ckan.datapusher.url not in config file']}) - - value = json.loads(task['value']) - job_key = value.get('job_key') - job_id = value.get('job_id') - url = None - job_detail = None - - if job_id: - url = urljoin(datapusher_url, 'job' + '/' + job_id) - try: - timeout = config.get('ckan.requests.timeout') - r = requests.get(url, - timeout=timeout, - headers={'Content-Type': 'application/json', - 'Authorization': job_key}) - r.raise_for_status() - job_detail = r.json() - for log in job_detail['logs']: - if 'timestamp' in log: - date = time.strptime( - log['timestamp'], "%Y-%m-%dT%H:%M:%S.%f") - date = datetime.datetime.utcfromtimestamp( - time.mktime(date)) - log['timestamp'] = date - except (requests.exceptions.ConnectionError, - requests.exceptions.HTTPError): - job_detail = {'error': 'cannot connect to datapusher'} - - return { - 'status': task['state'], - 'job_id': job_id, - 'job_url': url, - 'last_updated': task['last_updated'], - 'job_key': job_key, - 'task_info': job_detail, - 'error': json.loads(task['error']) - } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py deleted file mode 100644 index 7d179ec..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/auth.py +++ /dev/null @@ -1,16 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from typing import Any -from ckan.types import AuthResult, Context -import ckanext.datastore.logic.auth as auth - - -def datapusher_submit( - context: Context, data_dict: dict[str, Any]) -> AuthResult: - return auth.datastore_auth(context, data_dict) - - -def datapusher_status( - context: Context, data_dict: dict[str, Any]) -> AuthResult: - return auth.datastore_auth(context, data_dict) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py deleted file mode 100644 index 4ab41d0..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/logic/schema.py +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -from ckan.types import Schema - -import ckan.plugins as p -import ckanext.datastore.logic.schema as dsschema - -get_validator = p.toolkit.get_validator - -not_missing = get_validator('not_missing') -not_empty = get_validator('not_empty') -ignore_missing = get_validator('ignore_missing') -empty = get_validator('empty') -boolean_validator = get_validator('boolean_validator') -unicode_safe = get_validator('unicode_safe') - - -def datapusher_submit_schema() -> Schema: - schema = { - 'resource_id': [not_missing, not_empty, unicode_safe], - 'id': [ignore_missing], - 'set_url_type': [ignore_missing, boolean_validator], - 'ignore_hash': [ignore_missing, boolean_validator], - '__junk': [empty], - '__before': [dsschema.rename('id', 'resource_id')] - } - return schema diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py deleted file mode 100644 index c7cc476..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/plugin.py +++ /dev/null @@ -1,163 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from ckan.common import CKANConfig -from ckan.types import Action, AuthFunction, Context -import logging -from typing import Any, Callable, cast - -import ckan.model as model -import ckan.plugins as p -import ckanext.datapusher.views as views -import ckanext.datapusher.helpers as helpers -import ckanext.datapusher.logic.action as action -import ckanext.datapusher.logic.auth as auth - -log = logging.getLogger(__name__) - - -class DatastoreException(Exception): - pass - - -@p.toolkit.blanket.config_declarations -class DatapusherPlugin(p.SingletonPlugin): - p.implements(p.IConfigurer, inherit=True) - p.implements(p.IConfigurable, inherit=True) - p.implements(p.IActions) - p.implements(p.IAuthFunctions) - p.implements(p.IResourceUrlChange) - p.implements(p.IResourceController, inherit=True) - p.implements(p.ITemplateHelpers) - p.implements(p.IBlueprint) - - legacy_mode = False - resource_show_action = None - - def update_config(self, config: CKANConfig): - templates_base = config.get(u'ckan.base_templates_folder') - p.toolkit.add_template_directory(config, templates_base) - p.toolkit.add_public_directory(config, 'public') - p.toolkit.add_resource('assets', 'ckanext-datapusher') - - def configure(self, config: CKANConfig): - self.config = config - - for config_option in ( - "ckan.site_url", - "ckan.datapusher.url", - "ckan.datapusher.api_token", - ): - if not config.get(config_option): - raise Exception( - u'Config option `{0}` must be set to use the DataPusher.'. - format(config_option) - ) - - # IResourceUrlChange - - def notify(self, resource: model.Resource): - context = cast(Context, { - u'model': model, - u'ignore_auth': True, - }) - resource_dict = p.toolkit.get_action(u'resource_show')( - context, { - u'id': resource.id, - } - ) - self._submit_to_datapusher(resource_dict) - - # IResourceController - - def after_resource_create( - self, context: Context, resource_dict: dict[str, Any]): - - self._submit_to_datapusher(resource_dict) - - def after_update( - self, context: Context, resource_dict: dict[str, Any]): - - self._submit_to_datapusher(resource_dict) - - def _submit_to_datapusher(self, resource_dict: dict[str, Any]): - context = cast(Context, { - u'model': model, - u'ignore_auth': True, - u'defer_commit': True - }) - - resource_format = resource_dict.get('format') - supported_formats = p.toolkit.config.get( - 'ckan.datapusher.formats' - ) - - submit = ( - resource_format - and resource_format.lower() in supported_formats - and resource_dict.get('url_type') != u'datapusher' - ) - - if not submit: - return - - try: - task = p.toolkit.get_action(u'task_status_show')( - context, { - u'entity_id': resource_dict['id'], - u'task_type': u'datapusher', - u'key': u'datapusher' - } - ) - - if task.get(u'state') in (u'pending', u'submitting'): - # There already is a pending DataPusher submission, - # skip this one ... - log.debug( - u'Skipping DataPusher submission for ' - u'resource {0}'.format(resource_dict['id']) - ) - return - except p.toolkit.ObjectNotFound: - pass - - try: - log.debug( - u'Submitting resource {0}'.format(resource_dict['id']) + - u' to DataPusher' - ) - p.toolkit.get_action(u'datapusher_submit')( - context, { - u'resource_id': resource_dict['id'] - } - ) - except p.toolkit.ValidationError as e: - # If datapusher is offline want to catch error instead - # of raising otherwise resource save will fail with 500 - log.critical(e) - pass - - def get_actions(self) -> dict[str, Action]: - return { - u'datapusher_submit': action.datapusher_submit, - u'datapusher_hook': action.datapusher_hook, - u'datapusher_status': action.datapusher_status - } - - def get_auth_functions(self) -> dict[str, AuthFunction]: - return { - u'datapusher_submit': auth.datapusher_submit, - u'datapusher_status': auth.datapusher_status - } - - def get_helpers(self) -> dict[str, Callable[..., Any]]: - return { - u'datapusher_status': helpers.datapusher_status, - u'datapusher_status_description': helpers. - datapusher_status_description, - } - - # IBlueprint - - def get_blueprint(self): - return views.get_blueprints() diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/public/dotted.png b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/public/dotted.png deleted file mode 100644 index fa0ae8040cbdf64a1a9c3214b062aa5c31a36c23..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 74 zcmeAS@N?(olHy`uVBq!ia0vp^OhC-S0V1_~EAxPqpr?ytNCji^4sJGsSN0MPnhd(h VVu9wjyxM`144$rjF6*2UngGv~4+;PP diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html deleted file mode 100644 index 293a9ce..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/datapusher/resource_data.html +++ /dev/null @@ -1,88 +0,0 @@ -{% extends "package/resource_edit_base.html" %} - -{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block primary_content_inner %} - - {% set action = h.url_for('datapusher.resource_data', id=pkg.name, resource_id=res.id) %} - {% set show_table = true %} - -
    - -
    - - {% if status.error and status.error.message %} - {% set show_table = false %} -
    - {{ _('Upload error:') }} {{ status.error.message }} -
    - {% elif status.task_info and status.task_info.error %} -
    - {% if status.task_info.error is string %} - {# DataPusher < 0.0.3 #} - {{ _('Error:') }} {{ status.task_info.error }} - {% elif status.task_info.error is mapping %} - {{ _('Error:') }} {{ status.task_info.error.message }} - {% for error_key, error_value in status.task_info.error.items() %} - {% if error_key != "message" and error_value %} -
    - {{ error_key }}: - {{ error_value }} - {% endif %} - {% endfor %} - {% elif status.task_info.error is iterable %} - {{ _('Error traceback:') }} - 
    {{ ''.join(status.task_info.error) }}
    - {% endif %} -
    - {% endif %} - - - - - - - - - - - - - {% if status.status %} - - {% else %} - - {% endif %} - -
    {{ _('Status') }}{{ h.datapusher_status_description(status) }}
    {{ _('Last updated') }}{{ h.time_ago_from_timestamp(status.last_updated) }}{{ _('Never') }}
    - - {% if status.status and status.task_info and show_table %} -

    {{ _('Upload Log') }}

    -
      - {% for item in status.task_info.logs|sort(attribute='timestamp') %} - {% set icon = 'ok' if item.level == 'INFO' else 'exclamation' %} - {% set class = ' failure' if icon == 'exclamation' else ' success' %} - {% set popover_content = 'test' %} -
    • - -

      - {% for line in item.message.strip().split('\n') %} - {{ line | urlize }}
      - {% endfor %} - - {{ h.time_ago_from_timestamp(item.timestamp) }} - {{ _('Details') }} - -

      -
      • - {% endfor %} -
    • - -

      {{ _('End of log') }}

      -
      • -
    - {% endif %} - -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html deleted file mode 100644 index 7d08984..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates-bs3/package/resource_edit_base.html +++ /dev/null @@ -1,16 +0,0 @@ -{% ckan_extends %} - -{% block scripts %} - {{ super() }} - {% asset 'ckanext-datapusher/datapusher' %} -{% endblock scripts %} - -{% block styles %} - {{ super() }} - {% asset 'ckanext-datapusher/datapusher-css' %} -{% endblock %} - -{% block inner_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('datapusher.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html deleted file mode 100644 index 3598e7b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/datapusher/resource_data.html +++ /dev/null @@ -1,95 +0,0 @@ -{% extends "package/resource_edit_base.html" %} - -{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block primary_content_inner %} - - {% set action = h.url_for('datapusher.resource_data', id=pkg.name, resource_id=res.id) %} - {% set show_table = true %} - -
    - {{ h.csrf_input() }} - -
    - - {% if status.error and status.error.message %} - {% set show_table = false %} -
    - {{ _('Upload error:') }} {{ status.error.message }} -
    - {% elif status.task_info and status.task_info.error %} -
    - {% if status.task_info.error is string %} - {# DataPusher < 0.0.3 #} - {{ _('Error:') }} {{ status.task_info.error }} - {% elif status.task_info.error is mapping %} - {{ _('Error:') }} {{ status.task_info.error.message }} - {% for error_key, error_value in status.task_info.error.items() %} - {% if error_key != "message" and error_value %} -
    - {{ error_key }}: - {{ error_value }} - {% endif %} - {% endfor %} - {% elif status.task_info.error is iterable %} - {{ _('Error traceback:') }} - 
    {{ ''.join(status.task_info.error) }}
    - {% endif %} -
    - {% endif %} - - - - - - - - - - - - - {% if status.status %} - - {% else %} - - {% endif %} - -
    {{ _('Status') }}{{ h.datapusher_status_description(status) }}
    {{ _('Last updated') }}{{ h.time_ago_from_timestamp(status.last_updated) }}{{ _('Never') }}
    - - {% if status.status and status.task_info and show_table %} -

    {{ _('Upload Log') }}

    -
      - {% for item in status.task_info.logs|sort(attribute='timestamp') %} - {% set icon = 'check' if item.level == 'INFO' else 'exclamation' %} - {% set class = ' failure' if icon == 'exclamation' else ' success' %} - {% set popover_content = 'test' %} -
    • - - - - - {% for line in item.message.strip().split('\n') %} - {{ line | urlize }}
      - {% endfor %} - - {{ h.time_ago_from_timestamp(item.timestamp) }} - - {{ _('Details') }} - - -
      • - {% endfor %} -
    • - - - - - {{ _('End of log') }} -
      • -
    - {% endif %} - -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html deleted file mode 100644 index 7d08984..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/templates/package/resource_edit_base.html +++ /dev/null @@ -1,16 +0,0 @@ -{% ckan_extends %} - -{% block scripts %} - {{ super() }} - {% asset 'ckanext-datapusher/datapusher' %} -{% endblock scripts %} - -{% block styles %} - {{ super() }} - {% asset 'ckanext-datapusher/datapusher-css' %} -{% endblock %} - -{% block inner_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('datapusher.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py deleted file mode 100644 index 737a9be..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# encoding: utf-8 -from ckan.tests import factories - - -def get_api_token() -> str: - return factories.SysadminWithToken()["sysadmin"] diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py deleted file mode 100644 index 9f6397d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test.py +++ /dev/null @@ -1,322 +0,0 @@ -# encoding: utf-8 - -import datetime - -import json -import pytest -import responses - -import ckan.lib.create_test_data as ctd -import ckan.model as model -import ckan.plugins as p -from ckan.tests import factories -from ckan.tests.helpers import call_action -from ckan.common import config -from ckanext.datastore.tests.helpers import set_url_type -from ckanext.datapusher.tests import get_api_token - - -@pytest.mark.ckan_config("ckan.plugins", "datastore datapusher") -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures("with_plugins", "non_clean_db") -class TestDatastoreNew: - def test_create_ckan_resource_in_package(self, app, api_token): - package = factories.Dataset.model() - data = {"resource": {"package_id": package.id}} - auth = {"Authorization": api_token["token"]} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=200, - ) - res_dict = json.loads(res.body) - - assert "resource_id" in res_dict["result"] - assert len(package.resources) == 1 - - res = call_action("resource_show", - id=res_dict["result"]["resource_id"]) - assert res["url"].endswith("/datastore/dump/" + res["id"]), res - - @responses.activate - def test_providing_res_with_url_calls_datapusher_correctly(self, app): - config["datapusher.url"] = "http://datapusher.ckan.org" - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "bar"}), - ) - responses.add_passthru(config["solr_url"]) - - package = factories.Dataset.model() - - call_action("datastore_create", - resource=dict(package_id=package.id, url="demo.ckan.org")) - - assert len(package.resources) == 1 - resource = package.resources[0] - data = json.loads(responses.calls[-1].request.body) - assert data["metadata"]["resource_id"] == resource.id, data - assert not data["metadata"].get("ignore_hash"), data - assert data["result_url"].endswith("/action/datapusher_hook"), data - assert data["result_url"].startswith("http://"), data - - -@pytest.mark.ckan_config("ckan.plugins", "datastore datapusher") -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures("with_plugins") -class TestDatastoreCreate(object): - sysadmin_user = None - normal_user = None - - @pytest.fixture(autouse=True) - def initial_data(self, clean_db): - ctd.CreateTestData.create() - self.sysadmin_user = factories.Sysadmin() - self.sysadmin_token = factories.APIToken( - user=self.sysadmin_user["id"])["token"] - self.normal_user = factories.User() - self.normal_user_token = factories.APIToken( - user=self.normal_user["id"])["token"] - set_url_type( - model.Package.get("annakarenina").resources, self.sysadmin_user - ) - - @responses.activate - def test_pass_the_received_ignore_hash_param_to_the_datapusher(self, app): - config["datapusher.url"] = "http://datapusher.ckan.org" - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "bar"}), - ) - - package = model.Package.get("annakarenina") - resource = package.resources[0] - - call_action( - "datapusher_submit", - resource_id=resource.id, - ignore_hash=True, - ) - - data = json.loads(responses.calls[-1].request.body) - assert data["metadata"]["ignore_hash"], data - - def test_cant_provide_resource_and_resource_id(self, app): - package = model.Package.get("annakarenina") - resource = package.resources[0] - data = { - "resource_id": resource.id, - "resource": {"package_id": package.id}, - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.body) - - assert res_dict["error"]["__type"] == "Validation Error" - - @responses.activate - def test_send_datapusher_creates_task(self, test_request_context): - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "bar"}), - ) - - package = model.Package.get("annakarenina") - resource = package.resources[0] - - context = {"ignore_auth": True, "user": self.sysadmin_user["name"]} - with test_request_context(): - p.toolkit.get_action("datapusher_submit")( - context, {"resource_id": resource.id} - ) - - context.pop("task_status", None) - - task = p.toolkit.get_action("task_status_show")( - context, - { - "entity_id": resource.id, - "task_type": "datapusher", - "key": "datapusher", - }, - ) - - assert task["state"] == "pending", task - - def _call_datapusher_hook(self, user, app): - package = model.Package.get("annakarenina") - resource = package.resources[0] - - context = {"user": self.sysadmin_user["name"]} - - p.toolkit.get_action("task_status_update")( - context, - { - "entity_id": resource.id, - "entity_type": "resource", - "task_type": "datapusher", - "key": "datapusher", - "value": '{"job_id": "my_id", "job_key":"my_key"}', - "last_updated": str(datetime.datetime.now()), - "state": "pending", - }, - ) - - data = {"status": "success", "metadata": {"resource_id": resource.id}} - - if user["sysadmin"]: - auth = {"Authorization": self.sysadmin_token} - else: - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datapusher_hook", - json=data, - extra_environ=auth, - status=200, - ) - res_dict = json.loads(res.body) - - assert res_dict["success"] is True - - task = call_action( - "task_status_show", - entity_id=resource.id, - task_type="datapusher", - key="datapusher", - ) - - assert task["state"] == "success", task - - task = call_action( - "task_status_show", - entity_id=resource.id, - task_type="datapusher", - key="datapusher", - ) - - assert task["state"] == "success", task - - def test_datapusher_hook_sysadmin(self, app, test_request_context): - with test_request_context(): - self._call_datapusher_hook(self.sysadmin_user, app) - - def test_datapusher_hook_normal_user(self, app, test_request_context): - with test_request_context(): - self._call_datapusher_hook(self.normal_user, app) - - def test_datapusher_hook_no_metadata(self, app): - data = {"status": "success"} - - app.post("/api/action/datapusher_hook", json=data, status=409) - - def test_datapusher_hook_no_status(self, app): - data = {"metadata": {"resource_id": "res_id"}} - - app.post("/api/action/datapusher_hook", json=data, status=409) - - def test_datapusher_hook_no_resource_id_in_metadata(self, app): - data = {"status": "success", "metadata": {}} - - app.post("/api/action/datapusher_hook", json=data, status=409) - - @responses.activate - @pytest.mark.ckan_config( - "ckan.datapusher.callback_url_base", "https://ckan.example.com" - ) - @pytest.mark.ckan_config( - "ckan.datapusher.url", "http://datapusher.ckan.org" - ) - def test_custom_callback_url_base(self, app): - - package = model.Package.get("annakarenina") - resource = package.resources[0] - - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "barloco"}), - ) - responses.add_passthru(config["solr_url"]) - - call_action( - "datapusher_submit", - resource_id=resource.id, - ignore_hash=True, - ) - - data = json.loads(responses.calls[-1].request.body) - assert ( - data["result_url"] - == "https://ckan.example.com/api/3/action/datapusher_hook" - ) - - @responses.activate - @pytest.mark.ckan_config( - "ckan.datapusher.callback_url_base", "https://ckan.example.com" - ) - @pytest.mark.ckan_config( - "ckan.datapusher.url", "http://datapusher.ckan.org" - ) - def test_create_resource_hooks(self, app): - - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "barloco"}), - ) - responses.add_passthru(config["solr_url"]) - - dataset = factories.Dataset() - call_action( - "resource_create", - package_id=dataset['id'], - format='CSV', - ) - - @responses.activate - @pytest.mark.ckan_config( - "ckan.datapusher.callback_url_base", "https://ckan.example.com" - ) - @pytest.mark.ckan_config( - "ckan.datapusher.url", "http://datapusher.ckan.org" - ) - def test_update_resource_url_hooks(self, app): - - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "barloco"}), - ) - responses.add_passthru(config["solr_url"]) - - dataset = factories.Dataset() - resource = call_action( - "resource_create", - package_id=dataset['id'], - url='http://example.com/old.csv', - format='CSV', - ) - - resource = call_action( - "resource_update", - id=resource['id'], - url='http://example.com/new.csv', - format='CSV', - ) - assert resource diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py deleted file mode 100644 index 0ab187c..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_action.py +++ /dev/null @@ -1,347 +0,0 @@ -# encoding: utf-8 - -import datetime - -import unittest.mock as mock -import pytest -from ckan.logic import _actions - -from ckan.tests import helpers, factories -from ckanext.datapusher.tests import get_api_token - - -def _pending_task(resource_id): - return { - "entity_id": resource_id, - "entity_type": "resource", - "task_type": "datapusher", - "last_updated": str(datetime.datetime.utcnow()), - "state": "pending", - "key": "datapusher", - "value": "{}", - "error": "{}", - } - - -@pytest.mark.ckan_config("ckan.plugins", "datapusher datastore") -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestSubmit: - def test_submit(self, monkeypatch): - """Auto-submit when creating a resource with supported format. - - """ - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - func.assert_not_called() - - helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - ) - - func.assert_called() - - @pytest.mark.ckan_config("ckan.views.default_views", "") - @pytest.mark.flaky(reruns=2) - def test_submit_when_url_changes(self, monkeypatch): - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - - resource = helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.pdf", - ) - - func.assert_not_called() - - helpers.call_action( - "resource_update", - {}, - id=resource["id"], - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - ) - - func.assert_called() - - def test_does_not_submit_while_ongoing_job(self, monkeypatch): - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - - resource = helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.CSV", - format="CSV", - ) - - func.assert_called() - func.reset_mock() - # Create a task with a state pending to mimic an ongoing job - # on the DataPusher - helpers.call_action( - "task_status_update", {}, **_pending_task(resource["id"]) - ) - - # Update the resource - helpers.call_action( - "resource_update", - {}, - id=resource["id"], - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - description="Test", - ) - # Not called - func.assert_not_called() - - def test_resubmits_if_url_changes_in_the_meantime(self, monkeypatch): - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - - resource = helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - ) - - func.assert_called() - func.reset_mock() - # Create a task with a state pending to mimic an ongoing job - # on the DataPusher - task = helpers.call_action( - "task_status_update", {}, **_pending_task(resource["id"]) - ) - - # Update the resource, set a new URL - helpers.call_action( - "resource_update", - {}, - id=resource["id"], - package_id=dataset["id"], - url="http://example.com/another.file.csv", - format="CSV", - ) - # Not called - func.assert_not_called() - - # Call datapusher_hook with state complete, to mock the DataPusher - # finishing the job and telling CKAN - data_dict = { - "metadata": { - "resource_id": resource["id"], - "original_url": "http://example.com/file.csv", - "task_created": task["last_updated"], - }, - "status": "complete", - } - - helpers.call_action("datapusher_hook", {}, **data_dict) - - # datapusher_submit was called again - func.assert_called() - - def test_resubmits_if_upload_changes_in_the_meantime(self, monkeypatch): - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - - resource = helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - ) - - func.assert_called() - func.reset_mock() - # Create a task with a state pending to mimic an ongoing job - # on the DataPusher - task = helpers.call_action( - "task_status_update", {}, **_pending_task(resource["id"]) - ) - - # Update the resource, set a new last_modified (changes on file upload) - helpers.call_action( - "resource_update", - {}, - id=resource["id"], - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - last_modified=datetime.datetime.utcnow().isoformat(), - ) - # Not called - func.assert_not_called() - - # Call datapusher_hook with state complete, to mock the DataPusher - # finishing the job and telling CKAN - data_dict = { - "metadata": { - "resource_id": resource["id"], - "original_url": "http://example.com/file.csv", - "task_created": task["last_updated"], - }, - "status": "complete", - } - helpers.call_action("datapusher_hook", {}, **data_dict) - - # datapusher_submit was called again - func.assert_called() - - def test_does_not_resubmit_if_a_resource_field_changes_in_the_meantime( - self, monkeypatch - ): - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - - resource = helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - ) - - func.assert_called() - func.reset_mock() - - # Create a task with a state pending to mimic an ongoing job - # on the DataPusher - task = helpers.call_action( - "task_status_update", {}, **_pending_task(resource["id"]) - ) - - # Update the resource, set a new description - helpers.call_action( - "resource_update", - {}, - id=resource["id"], - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - description="Test", - ) - # Not called - func.assert_not_called() - - # Call datapusher_hook with state complete, to mock the DataPusher - # finishing the job and telling CKAN - data_dict = { - "metadata": { - "resource_id": resource["id"], - "original_url": "http://example.com/file.csv", - "task_created": task["last_updated"], - }, - "status": "complete", - } - helpers.call_action("datapusher_hook", {}, **data_dict) - - # Not called - func.assert_not_called() - - def test_does_not_resubmit_if_a_dataset_field_changes_in_the_meantime( - self, monkeypatch - ): - dataset = factories.Dataset() - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - - resource = helpers.call_action( - "resource_create", - {}, - package_id=dataset["id"], - url="http://example.com/file.csv", - format="CSV", - ) - - func.assert_called() - func.reset_mock() - # Create a task with a state pending to mimic an ongoing job - # on the DataPusher - task = helpers.call_action( - "task_status_update", {}, **_pending_task(resource["id"]) - ) - - # Update the parent dataset - helpers.call_action( - "package_update", - {}, - id=dataset["id"], - notes="Test notes", - resources=[resource], - ) - # Not called - func.assert_not_called() - # Call datapusher_hook with state complete, to mock the DataPusher - # finishing the job and telling CKAN - data_dict = { - "metadata": { - "resource_id": resource["id"], - "original_url": "http://example.com/file.csv", - "task_created": task["last_updated"], - }, - "status": "complete", - } - helpers.call_action("datapusher_hook", {}, **data_dict) - - # Not called - func.assert_not_called() - - def test_duplicated_tasks(self, app): - def submit(res, user): - return helpers.call_action( - "datapusher_submit", - context=dict(user=user["name"]), - resource_id=res["id"], - ) - - user = factories.User() - res = factories.Resource(user=user) - - with app.flask_app.test_request_context(): - with mock.patch("requests.post") as r_mock: - r_mock().json = mock.Mock( - side_effect=lambda: dict.fromkeys(["job_id", "job_key"]) - ) - r_mock.reset_mock() - submit(res, user) - submit(res, user) - - assert r_mock.call_count == 1 - - @pytest.mark.usefixtures("with_request_context") - def test_task_status_changes(self, create_with_upload): - """While updating task status, datapusher commits changes to database. - - Make sure that changes to task_status won't close session that is still - used on higher levels of API for resource updates. - - """ - user = factories.User() - dataset = factories.Dataset() - context = {"user": user["name"]} - resource = create_with_upload( - "id,name\n1,2\n3,4", "file.csv", package_id=dataset["id"], - context=context) - - create_with_upload( - "id,name\n1,2\n3,4", "file.csv", package_id=dataset["id"], - id=resource["id"], context=context, action="resource_update") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py deleted file mode 100644 index 2e7126c..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_default_views.py +++ /dev/null @@ -1,102 +0,0 @@ -# encoding: utf-8 - -import datetime - -import pytest - -from ckan.tests import helpers, factories -from ckanext.datapusher.tests import get_api_token - - -@pytest.mark.ckan_config("ckan.views.default_views", "recline_grid_view") -@pytest.mark.ckan_config( - "ckan.plugins", "datapusher datastore recline_grid_view" -) -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -def test_datapusher_creates_default_views_on_complete(): - - dataset = factories.Dataset() - - resource = factories.Resource(package_id=dataset["id"]) - - # Push data directly to the DataStore for the resource to be marked as - # `datastore_active=True`, so the grid view can be created - data = { - "resource_id": resource["id"], - "fields": [{"id": "a", "type": "text"}, {"id": "b", "type": "text"}], - "records": [{"a": "1", "b": "2"}], - "force": True, - } - helpers.call_action("datastore_create", **data) - - # Create a task for `datapusher_hook` to update - task_dict = { - "entity_id": resource["id"], - "entity_type": "resource", - "task_type": "datapusher", - "key": "datapusher", - "value": '{"job_id": "my_id", "job_key":"my_key"}', - "last_updated": str(datetime.datetime.now()), - "state": "pending", - } - helpers.call_action("task_status_update", context={}, **task_dict) - - # Call datapusher_hook with a status of complete to trigger the - # default views creation - params = { - "status": "complete", - "metadata": {"resource_id": resource["id"]}, - } - - helpers.call_action("datapusher_hook", context={}, **params) - - views = helpers.call_action("resource_view_list", id=resource["id"]) - - assert len(views) == 1 - assert views[0]["view_type"] == "recline_grid_view" - - -@pytest.mark.ckan_config("ckan.views.default_views", "recline_grid_view") -@pytest.mark.ckan_config( - "ckan.plugins", "datapusher datastore recline_grid_view" -) -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -def test_datapusher_does_not_create_default_views_on_pending(): - - dataset = factories.Dataset() - - resource = factories.Resource(package_id=dataset["id"]) - - # Push data directly to the DataStore for the resource to be marked as - # `datastore_active=True`, so the grid view can be created - data = { - "resource_id": resource["id"], - "fields": [{"id": "a", "type": "text"}, {"id": "b", "type": "text"}], - "records": [{"a": "1", "b": "2"}], - "force": True, - } - helpers.call_action("datastore_create", **data) - - # Create a task for `datapusher_hook` to update - task_dict = { - "entity_id": resource["id"], - "entity_type": "resource", - "task_type": "datapusher", - "key": "datapusher", - "value": '{"job_id": "my_id", "job_key":"my_key"}', - "last_updated": str(datetime.datetime.now()), - "state": "pending", - } - helpers.call_action("task_status_update", context={}, **task_dict) - - # Call datapusher_hook with a status of complete to trigger the - # default views creation - params = {"status": "pending", "metadata": {"resource_id": resource["id"]}} - - helpers.call_action("datapusher_hook", context={}, **params) - - views = helpers.call_action("resource_view_list", id=resource["id"]) - - assert len(views) == 0 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py deleted file mode 100644 index 6f126b2..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_interfaces.py +++ /dev/null @@ -1,121 +0,0 @@ -# encoding: utf-8 - -import datetime - -import json -import pytest -import responses - -import ckan.plugins as p -import ckanext.datapusher.interfaces as interfaces -from ckanext.datapusher.tests import get_api_token - -from ckan.tests import helpers, factories - - -class FakeDataPusherPlugin(p.SingletonPlugin): - p.implements(p.IConfigurable, inherit=True) - p.implements(interfaces.IDataPusher, inherit=True) - - def configure(self, _config): - self.after_upload_calls = 0 - - def can_upload(self, resource_id): - return False - - def after_upload(self, context, resource_dict, package_dict): - self.after_upload_calls += 1 - - -@pytest.mark.ckan_config( - "ckan.plugins", "datastore datapusher test_datapusher_plugin" -) -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestInterace(object): - - @responses.activate - @pytest.mark.parametrize("resource__url_type", ["datastore"]) - def test_send_datapusher_creates_task(self, test_request_context, resource): - sysadmin = factories.Sysadmin() - - responses.add( - responses.POST, - "http://datapusher.ckan.org/job", - content_type="application/json", - body=json.dumps({"job_id": "foo", "job_key": "bar"}), - ) - - context = {"ignore_auth": True, "user": sysadmin["name"]} - with test_request_context(): - result = p.toolkit.get_action("datapusher_submit")( - context, {"resource_id": resource["id"]} - ) - assert not result - - context.pop("task_status", None) - - with pytest.raises(p.toolkit.ObjectNotFound): - p.toolkit.get_action("task_status_show")( - context, - { - "entity_id": resource["id"], - "task_type": "datapusher", - "key": "datapusher", - }, - ) - - def test_after_upload_called(self): - dataset = factories.Dataset() - resource = factories.Resource(package_id=dataset["id"]) - - # Push data directly to the DataStore for the resource to be marked as - # `datastore_active=True`, so the grid view can be created - data = { - "resource_id": resource["id"], - "fields": [ - {"id": "a", "type": "text"}, - {"id": "b", "type": "text"}, - ], - "records": [{"a": "1", "b": "2"}], - "force": True, - } - helpers.call_action("datastore_create", **data) - - # Create a task for `datapusher_hook` to update - task_dict = { - "entity_id": resource["id"], - "entity_type": "resource", - "task_type": "datapusher", - "key": "datapusher", - "value": '{"job_id": "my_id", "job_key":"my_key"}', - "last_updated": str(datetime.datetime.now()), - "state": "pending", - } - helpers.call_action("task_status_update", context={}, **task_dict) - - # Call datapusher_hook with a status of complete to trigger the - # default views creation - params = { - "status": "complete", - "metadata": {"resource_id": resource["id"]}, - } - helpers.call_action("datapusher_hook", context={}, **params) - - total = sum( - plugin.after_upload_calls - for plugin in p.PluginImplementations(interfaces.IDataPusher) - ) - assert total == 1, total - - params = { - "status": "complete", - "metadata": {"resource_id": resource["id"]}, - } - helpers.call_action("datapusher_hook", context={}, **params) - - total = sum( - plugin.after_upload_calls - for plugin in p.PluginImplementations(interfaces.IDataPusher) - ) - assert total == 2, total diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py deleted file mode 100644 index bba6d89..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/tests/test_views.py +++ /dev/null @@ -1,38 +0,0 @@ -# encoding: utf-8 - -from unittest import mock -import pytest - -import ckan.tests.factories as factories -import ckan.model as model -from ckan.logic import _actions -from ckanext.datapusher.tests import get_api_token - - -@mock.patch("flask_login.utils._get_user") -@pytest.mark.ckan_config(u"ckan.plugins", u"datapusher datastore") -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures(u"non_clean_db", u"with_plugins", u"with_request_context") -def test_resource_data(current_user, app, monkeypatch): - user = factories.User() - user_obj = model.User.get(user["name"]) - # mock current_user - current_user.return_value = user_obj - - dataset = factories.Dataset(creator_user_id=user["id"]) - resource = factories.Resource( - package_id=dataset["id"], creator_user_id=user["id"] - ) - - url = u"/dataset/{id}/resource_data/{resource_id}".format( - id=str(dataset["name"]), resource_id=str(resource["id"]) - ) - - func = mock.Mock() - monkeypatch.setitem(_actions, 'datapusher_submit', func) - app.post(url=url, status=200) - func.assert_called() - func.reset_mock() - - app.get(url=url, status=200) - func.assert_not_called() diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/views.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/views.py deleted file mode 100644 index 650189b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datapusher/views.py +++ /dev/null @@ -1,75 +0,0 @@ -# encoding: utf-8 - -from flask import Blueprint -from flask.views import MethodView - -import ckan.plugins.toolkit as toolkit -import ckan.logic as logic -import ckan.lib.helpers as core_helpers -import ckan.lib.base as base - -from ckan.common import _ - -datapusher = Blueprint(u'datapusher', __name__) - - -def get_blueprints(): - return [datapusher] - - -class ResourceDataView(MethodView): - - def post(self, id: str, resource_id: str): - try: - toolkit.get_action(u'datapusher_submit')( - {}, { - u'resource_id': resource_id - } - ) - except logic.ValidationError: - pass - - return core_helpers.redirect_to( - u'datapusher.resource_data', id=id, resource_id=resource_id - ) - - def get(self, id: str, resource_id: str): - try: - pkg_dict = toolkit.get_action(u'package_show')({}, {u'id': id}) - resource = toolkit.get_action(u'resource_show' - )({}, { - u'id': resource_id - }) - - # backward compatibility with old templates - toolkit.g.pkg_dict = pkg_dict - toolkit.g.resource = resource - - except (logic.NotFound, logic.NotAuthorized): - base.abort(404, _(u'Resource not found')) - - try: - datapusher_status = toolkit.get_action(u'datapusher_status')( - {}, { - u'resource_id': resource_id - } - ) - except logic.NotFound: - datapusher_status = {} - except logic.NotAuthorized: - base.abort(403, _(u'Not authorized to see this page')) - - return base.render( - u'datapusher/resource_data.html', - extra_vars={ - u'status': datapusher_status, - u'pkg_dict': pkg_dict, - u'resource': resource, - } - ) - - -datapusher.add_url_rule( - u'/dataset//resource_data/', - view_func=ResourceDataView.as_view(str(u'resource_data')) -) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt deleted file mode 100644 index dd3840f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/allowed_functions.txt +++ /dev/null @@ -1,283 +0,0 @@ -abbrev -abs -abstime -abstimeeq -abstimege -abstimegt -abstimein -abstimele -abstimelt -abstimene -abstimeout -abstimerecv -abstimesend -acos -acosd -age -area -array_agg -array_append -array_cat -array_fill -array_length -array_lower -array_ndims -array_position -array_positions -array_prepend -array_remove -array_replace -array_to_json -array_to_string -array_to_tsvector -array_upper -avg -bit_and -bit_or -bool_and -bool_or -bound_box -box -broadcast -btrim -cardinality -cbrt -ceil -ceiling -center -chr -circle -clock_timestamp -concat -concat_ws -convert -convert_from -convert_to -corr -cos -cosd -cot -cotd -count -covar_pop -covar_samp -cume_dist -date_part -date_pl_interval -date_pli -date_recv -date_send -date_smaller -date_sortsupport -date_trunc -decode -degrees -dense_rank -diagonal -diameter -div -encode -enum_cmp -enum_eq -enum_first -enum_ge -enum_gt -enum_in -enum_larger -enum_last -enum_le -enum_lt -enum_ne -enum_range -enum_smaller -exp -factorial -family -first_value -floor -format -generate_series -get_bit -get_byte -get_current_ts_config -height -host -hostmask -inet_merge -inet_same_family -initcap -isclosed -isempty -isfinite -isopen -json_agg -json_array_element -json_array_element_text -json_array_elements -json_array_elements_text -json_array_length -json_build_array -json_build_object -json_each -json_each_text -json_extract_path -json_extract_path_text -json_object_agg -json_object_keys -json_populate_record -json_populate_recordset -jsonb_agg -jsonb_array_element -jsonb_array_element_text -jsonb_array_elements -jsonb_array_elements_text -jsonb_array_length -jsonb_build_array -jsonb_build_object -jsonb_extract_path -jsonb_extract_path_text -jsonb_insert -jsonb_pretty -jsonb_set -jsonb_strip_nulls -jsonb_typeof -justify_days -justify_hours -justify_interval -lag -last_value -lead -left -length -line -ln -log -lower -lower_inc -lower_inf -lpad -lseg -ltrim -macaddr8_set7bit -make_date -make_interval -make_time -make_timestamp -make_timestamptz -masklen -max -md5 -min -mod -mode -network -now -npoints -nth_value -ntile -num_nonnulls -num_nulls -numnode -parse_ident -path -pclose -percent_rank -percentile_cont -percentile_disc -phraseto_tsquery -pi -plainto_tsquery -point -polygon -popen -pow -power -querytree -quote_ident -quote_literal -quote_nullable -radians -radius -random -range_merge -rank -regexp_match -regexp_matches -regexp_replace -regexp_split_to_array -regexp_split_to_table -regr_avgx -regr_avgy -regr_count -regr_intercept -regr_r2 -regr_slope -regr_sxx -regr_sxy -regr_syy -repeat -replace -reverse -right -round -row_number -row_to_json -rpad -rtrim -scale -set_bit -set_byte -set_masklen -setweight -sign -sin -sind -slope -split_part -sqrt -statement_timestamp -stddev -stddev_pop -stddev_samp -string_agg -string_to_array -strip -strpos -substr -substring -sum -tan -tand -text -to_ascii -to_char -to_date -to_hex -to_json -to_jsonb -to_timestamp -to_tsquery -to_tsvector -transaction_timestamp -translate -trunc -ts_delete -ts_filter -ts_headline -ts_rank -ts_rewrite -tsvector_to_array -unnest -upper -upper_inc -upper_inf -var_pop -var_samp -variance -width -width_bucket -xmlagg -xmlcomment -xmlconcat -xpath -xpath_exists diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py deleted file mode 100644 index 7710ad8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/__init__.py +++ /dev/null @@ -1,238 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import annotations - -from ckan.types import Context -import re -import logging -from typing import Any, Container - -import ckan.plugins as plugins -from ckan.common import CKANConfig, config -from ckanext.datastore.interfaces import IDatastoreBackend - -log = logging.getLogger(__name__) - - -def get_all_resources_ids_in_datastore() -> list[str]: - """ - Helper for getting id of all resources in datastore. - - Uses `get_all_ids` of active datastore backend. - """ - DatastoreBackend.register_backends() - DatastoreBackend.set_active_backend(config) - backend = DatastoreBackend.get_active_backend() - backend.configure(config) - - return backend.get_all_ids() - - -def _parse_sort_clause( # type: ignore - clause: str, fields_types: Container[str]): - clause_match = re.match( - u'^(.+?)( +(asc|desc))?( nulls +(first|last) *)?$', clause, re.I - ) - - if not clause_match: - return False - - field = clause_match.group(1) - if field[0] == field[-1] == u'"': - field = field[1:-1] - sort = (clause_match.group(3) or u'asc').lower() - if clause_match.group(4): - sort += (clause_match.group(4)).lower() - - if field not in fields_types: - return False - - return field, sort - - -class DatastoreException(Exception): - pass - - -class InvalidDataError(Exception): - """Exception that's raised if you try to add invalid data to the datastore. - - For example if you have a column with type "numeric" and then you try to - add a non-numeric value like "foo" to it, this exception should be raised. - - """ - pass - - -class DatastoreBackend: - """Base class for all datastore backends. - - Very simple example of implementation based on SQLite can be found in - `ckanext.example_idatastorebackend`. In order to use it, set - datastore.write_url to - 'example-sqlite:////tmp/database-name-on-your-choice' - - :prop _backend: mapping(schema, class) of all registered backends - :type _backend: dictonary - :prop _active_backend: current active backend - :type _active_backend: DatastoreBackend - """ - - _backends = {} - _active_backend: "DatastoreBackend" - - @classmethod - def register_backends(cls): - """Register all backend implementations inside extensions. - """ - for plugin in plugins.PluginImplementations(IDatastoreBackend): - cls._backends.update(plugin.register_backends()) - - @classmethod - def set_active_backend(cls, config: CKANConfig): - """Choose most suitable backend depending on configuration - - :param config: configuration object - :rtype: ckan.common.CKANConfig - - """ - schema = config.get(u'ckan.datastore.write_url').split(u':')[0] - read_schema = config.get( - u'ckan.datastore.read_url').split(u':')[0] - assert read_schema == schema, u'Read and write engines are different' - cls._active_backend = cls._backends[schema]() - - @classmethod - def get_active_backend(cls): - """Return currently used backend - """ - return cls._active_backend - - def configure(self, config: CKANConfig): - """Configure backend, set inner variables, make some initial setup. - - :param config: configuration object - :returns: config - :rtype: CKANConfig - - """ - - return config - - def create(self, context: Context, data_dict: dict[str, Any]) -> Any: - """Create new resourct inside datastore. - - Called by `datastore_create`. - - :param data_dict: See `ckanext.datastore.logic.action.datastore_create` - :returns: The newly created data object - :rtype: dictonary - """ - raise NotImplementedError() - - def upsert(self, context: Context, data_dict: dict[str, Any]) -> Any: - """Update or create resource depending on data_dict param. - - Called by `datastore_upsert`. - - :param data_dict: See `ckanext.datastore.logic.action.datastore_upsert` - :returns: The modified data object - :rtype: dictonary - """ - raise NotImplementedError() - - def delete(self, context: Context, data_dict: dict[str, Any]) -> Any: - """Remove resource from datastore. - - Called by `datastore_delete`. - - :param data_dict: See `ckanext.datastore.logic.action.datastore_delete` - :returns: Original filters sent. - :rtype: dictonary - """ - raise NotImplementedError() - - def search(self, context: Context, data_dict: dict[str, Any]) -> Any: - """Base search. - - Called by `datastore_search`. - - :param data_dict: See `ckanext.datastore.logic.action.datastore_search` - :rtype: dictonary with following keys - - :param fields: fields/columns and their extra metadata - :type fields: list of dictionaries - :param offset: query offset value - :type offset: int - :param limit: query limit value - :type limit: int - :param filters: query filters - :type filters: list of dictionaries - :param total: number of total matching records - :type total: int - :param records: list of matching results - :type records: list of dictionaries - - """ - raise NotImplementedError() - - def search_sql(self, context: Context, data_dict: dict[str, Any]) -> Any: - """Advanced search. - - Called by `datastore_search_sql`. - :param sql: a single seach statement - :type sql: string - - :rtype: dictonary - :param fields: fields/columns and their extra metadata - :type fields: list of dictionaries - :param records: list of matching results - :type records: list of dictionaries - """ - raise NotImplementedError() - - def resource_exists(self, id: str) -> bool: - """Define whether resource exists in datastore. - """ - raise NotImplementedError() - - def resource_fields(self, id: str) -> Any: - """Return dictonary with resource description. - - Called by `datastore_info`. - :returns: A dictionary describing the columns and their types. - """ - raise NotImplementedError() - - def resource_info(self, id: str) -> Any: - """Return DataDictonary with resource's info - #3414 - """ - raise NotImplementedError() - - def resource_id_from_alias(self, alias: str) -> Any: - """Convert resource's alias to real id. - - :param alias: resource's alias or id - :type alias: string - :returns: real id of resource - :rtype: string - - """ - raise NotImplementedError() - - def get_all_ids(self) -> list[str]: - """Return id of all resource registered in datastore. - - :returns: all resources ids - :rtype: list of strings - """ - raise NotImplementedError() - - def create_function(self, *args: Any, **kwargs: Any) -> Any: - """Called by `datastore_function_create` action. - """ - raise NotImplementedError() - - def drop_function(self, *args: Any, **kwargs: Any) -> Any: - """Called by `datastore_function_delete` action. - """ - raise NotImplementedError() diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py deleted file mode 100644 index aaa951d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/backend/postgres.py +++ /dev/null @@ -1,2323 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import annotations -from typing_extensions import TypeAlias - -import sqlalchemy.exc -from sqlalchemy.engine.base import Engine -from ckan.types import Context, ErrorDict -import copy -import logging -import sys -from typing import ( - Any, Callable, Container, Dict, Iterable, Optional, Set, Union, - cast) -import sqlalchemy -import os -import pprint -import sqlalchemy.engine.url as sa_url -import datetime -import hashlib -import json -from collections import OrderedDict - -import six -from urllib.parse import ( - urlencode, urlunparse, parse_qsl, urlparse -) -from io import StringIO - -import ckan.plugins as p -import ckan.plugins.toolkit as toolkit -from ckan.lib.lazyjson import LazyJSONObject - -import ckanext.datastore.helpers as datastore_helpers -import ckanext.datastore.interfaces as interfaces - -from psycopg2.extras import register_default_json, register_composite -import distutils.version -from sqlalchemy.exc import (ProgrammingError, IntegrityError, - DBAPIError, DataError) - -import ckan.plugins as plugins -from ckan.common import CKANConfig, config - -from ckanext.datastore.backend import ( - DatastoreBackend, - DatastoreException, - _parse_sort_clause -) -from ckanext.datastore.backend import InvalidDataError - -log = logging.getLogger(__name__) - -_pg_types: dict[str, str] = {} -_type_names: Set[str] = set() -_engines: Dict[str, Engine] = {} - -_TIMEOUT = 60000 # milliseconds - -# See http://www.postgresql.org/docs/9.2/static/errcodes-appendix.html -_PG_ERR_CODE = { - 'unique_violation': '23505', - 'query_canceled': '57014', - 'undefined_object': '42704', - 'syntax_error': '42601', - 'permission_denied': '42501', - 'duplicate_table': '42P07', - 'duplicate_alias': '42712', -} - -_DATE_FORMATS = ['%Y-%m-%d', - '%Y-%m-%d %H:%M:%S', - '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%dT%H:%M:%SZ', - '%d/%m/%Y', - '%m/%d/%Y', - '%d-%m-%Y', - '%m-%d-%Y'] - -_INSERT = 'insert' -_UPSERT = 'upsert' -_UPDATE = 'update' - - -if not os.environ.get('DATASTORE_LOAD'): - ValidationError = toolkit.ValidationError # type: ignore -else: - log.warn("Running datastore without CKAN") - - class ValidationError(Exception): - def __init__(self, error_dict: ErrorDict): - pprint.pprint(error_dict) - -is_single_statement = datastore_helpers.is_single_statement - -_engines = {} - - -def literal_string(s: str): - """ - Return s as a postgres literal string - """ - return u"'" + s.replace(u"'", u"''").replace(u'\0', '') + u"'" - - -def identifier(s: str): - """ - Return s as a quoted postgres identifier - """ - return u'"' + s.replace(u'"', u'""').replace(u'\0', '') + u'"' - - -def get_read_engine(): - return _get_engine_from_url( - config['ckan.datastore.read_url'], - isolation_level='READ_UNCOMMITTED') - - -def get_write_engine(): - return _get_engine_from_url(config['ckan.datastore.write_url']) - - -def _get_engine_from_url(connection_url: str, **kwargs: Any) -> Engine: - '''Get either read or write engine.''' - engine = _engines.get(connection_url) - if not engine: - extras = {'url': connection_url} - config.setdefault('ckan.datastore.sqlalchemy.pool_pre_ping', True) - for key, value in kwargs.items(): - config.setdefault(key, value) - engine = sqlalchemy.engine_from_config(config, - 'ckan.datastore.sqlalchemy.', - **extras) - _engines[connection_url] = engine - - # don't automatically convert to python objects - # when using native json types in 9.2+ - # http://initd.org/psycopg/docs/extras.html#adapt-json - _loads: Callable[[Any], Any] = lambda x: x - register_default_json( - conn_or_curs=engine.raw_connection().connection, - globally=False, - loads=_loads) - - return engine - - -def _dispose_engines(): - '''Dispose all database engines.''' - global _engines - for _, engine in _engines.items(): - engine.dispose() - _engines = {} - - -def _pluck(field: str, arr: list[dict[str, Any]]): - return [x[field] for x in arr] - - -def _rename_json_field(data_dict: dict[str, Any]): - '''Rename json type to a corresponding type for the datastore since - pre 9.2 postgres versions do not support native json''' - return _rename_field(data_dict, 'json', 'nested') - - -def _unrename_json_field(data_dict: dict[str, Any]): - return _rename_field(data_dict, 'nested', 'json') - - -def _rename_field(data_dict: dict[str, Any], term: Any, replace: Any): - fields = data_dict.get('fields', []) - for i, field in enumerate(fields): - if 'type' in field and field['type'] == term: - data_dict['fields'][i]['type'] = replace - return data_dict - - -def _get_fields_types( - connection: Any, resource_id: str) -> 'OrderedDict[str, str]': - u''' - return a OrderedDict([(column_name, column_type)...]) for the passed - resource_id including '_id' but excluding other '_'-prefixed columns. - ''' - all_fields = _get_fields(connection, resource_id) - all_fields.insert(0, {'id': '_id', 'type': 'int'}) - field_types = OrderedDict([(f['id'], f['type']) for f in all_fields]) - return field_types - - -def _result_fields(fields_types: 'OrderedDict[str, str]', - field_info: dict[str, Any], fields: Optional[list[str]] - ) -> list[dict[str, Any]]: - u''' - return a list of field information based on the fields present, - passed and query passed. - - :param fields_types: OrderedDict returned from _get_fields_types(..) - with rank column types added by plugins' datastore_search - :param field_info: dict returned from _get_field_info(..) - :param fields: list of field names passed to datastore_search - or None for all - ''' - result_fields: list[dict[str, str]] = [] - - if fields is None: - fields = list(fields_types) - - for field_id in fields: - f = {u'id': field_id, u'type': fields_types[field_id]} - if field_id in field_info: - f[u'info'] = field_info[f['id']] - result_fields.append(f) - return result_fields - - -def _get_type(connection: Any, oid: str) -> str: - _cache_types(connection) - return _pg_types[oid] - - -def _guess_type(field: Any): - '''Simple guess type of field, only allowed are - integer, numeric and text''' - data_types = set([int, float]) - if isinstance(field, (dict, list)): - return 'nested' - if isinstance(field, int): - return 'int' - if isinstance(field, float): - return 'float' - for data_type in list(data_types): - try: - data_type(field) - except (TypeError, ValueError): - data_types.discard(data_type) - if not data_types: - break - if int in data_types: - return 'integer' - elif float in data_types: - return 'numeric' - - # try iso dates - for format in _DATE_FORMATS: - try: - datetime.datetime.strptime(field, format) - return 'timestamp' - except (ValueError, TypeError): - continue - return 'text' - - -def _get_unique_key(context: Context, data_dict: dict[str, Any]) -> list[str]: - sql_get_unique_key = ''' - SELECT - a.attname AS column_names - FROM - pg_class t, - pg_index idx, - pg_attribute a - WHERE - t.oid = idx.indrelid - AND a.attrelid = t.oid - AND a.attnum = ANY(idx.indkey) - AND t.relkind = 'r' - AND idx.indisunique = true - AND idx.indisprimary = false - AND t.relname = %s - ''' - key_parts = context['connection'].execute(sql_get_unique_key, - data_dict['resource_id']) - return [x[0] for x in key_parts] - - -def _get_field_info(connection: Any, resource_id: str) -> dict[str, Any]: - u'''return a dictionary mapping column names to their info data, - when present''' - qtext = sqlalchemy.text(u''' - select pa.attname as name, pd.description as info - from pg_class pc, pg_attribute pa, pg_description pd - where pa.attrelid = pc.oid and pd.objoid = pc.oid - and pd.objsubid = pa.attnum and pc.relname = :res_id - and pa.attnum > 0 - ''') - try: - return dict( - (n, json.loads(v)) for (n, v) in - connection.execute(qtext, res_id=resource_id).fetchall()) - except ValueError: # don't die on non-json comments - return {} - - -def _get_fields(connection: Any, resource_id: str): - u''' - return a list of {'id': column_name, 'type': column_type} dicts - for the passed resource_id, excluding '_'-prefixed columns. - ''' - fields: list[dict[str, Any]] = [] - all_fields = connection.execute( - u'SELECT * FROM "{0}" LIMIT 1'.format(resource_id) - ) - for field in all_fields.cursor.description: - if not field[0].startswith('_'): - fields.append({ - 'id': six.ensure_text(field[0]), - 'type': _get_type(connection, field[1]) - }) - return fields - - -def _cache_types(connection: Any) -> None: - if not _pg_types: - results = connection.execute( - 'SELECT oid, typname FROM pg_type;' - ) - for result in results: - _pg_types[result[0]] = result[1] - _type_names.add(result[1]) - if 'nested' not in _type_names: - native_json = _pg_version_is_at_least(connection, '9.2') - - log.info("Create nested type. Native JSON: {0!r}".format( - native_json)) - - backend = DatastorePostgresqlBackend.get_active_backend() - engine: Engine = backend._get_write_engine() # type: ignore - with cast(Any, engine.begin()) as write_connection: - write_connection.execute( - 'CREATE TYPE "nested" AS (json {0}, extra text)'.format( - 'json' if native_json else 'text')) - _pg_types.clear() - - # redo cache types with json now available. - return _cache_types(connection) - - register_composite('nested', connection.connection.connection, True) - - -def _pg_version_is_at_least(connection: Any, version: Any): - try: - v = distutils.version.LooseVersion(version) - pg_version = connection.execute('select version();').fetchone() - pg_version_number = pg_version[0].split()[1] - pv = distutils.version.LooseVersion(pg_version_number) - return v <= pv - except ValueError: - return False - - -def _is_array_type(field_type: str): - return field_type.startswith('_') - - -def _validate_record(record: Any, num: int, field_names: Iterable[str]): - # check record for sanity - if not isinstance(record, dict): - raise ValidationError({ - 'records': [u'row "{0}" is not a json object'.format(num)] - }) - # check for extra fields in data - extra_keys: set[str] = set(record.keys()) - set(field_names) - - if extra_keys: - raise ValidationError({ - 'records': [u'row "{0}" has extra keys "{1}"'.format( - num + 1, - ', '.join(list(extra_keys)) - )] - }) - - -def _where_clauses( - data_dict: dict[str, Any], fields_types: dict[str, Any] -) -> list[Any]: - filters = data_dict.get('filters', {}) - clauses: list[Any] = [] - - for field, value in filters.items(): - if field not in fields_types: - continue - field_array_type = _is_array_type(fields_types[field]) - # "%" needs to be escaped as "%%" in any query going to - # connection.execute, otherwise it will think the "%" is for - # substituting a bind parameter - field = field.replace('%', '%%') - if isinstance(value, list) and not field_array_type: - clause_str = (u'"{0}" in ({1})'.format(field, - ','.join(['%s'] * len(value)))) - clause = (clause_str,) + tuple(value) - else: - clause: tuple[Any, ...] = (u'"{0}" = %s'.format(field), value) - clauses.append(clause) - - # add full-text search where clause - q: Union[dict[str, str], str, Any] = data_dict.get('q') - full_text = data_dict.get('full_text') - if q and not full_text: - if isinstance(q, str): - ts_query_alias = _ts_query_alias() - clause_str = u'_full_text @@ {0}'.format(ts_query_alias) - clauses.append((clause_str,)) - elif isinstance(q, dict): - lang = _fts_lang(data_dict.get('language')) - for field, value in q.items(): - if field not in fields_types: - continue - query_field = _ts_query_alias(field) - - ftyp = fields_types[field] - if not datastore_helpers.should_fts_index_field_type(ftyp): - clause_str = u'_full_text @@ {0}'.format(query_field) - clauses.append((clause_str,)) - - clause_str = ( - u'to_tsvector({0}, cast({1} as text)) @@ {2}').format( - literal_string(lang), - identifier(field), - query_field) - clauses.append((clause_str,)) - elif (full_text and not q): - ts_query_alias = _ts_query_alias() - clause_str = u'_full_text @@ {0}'.format(ts_query_alias) - clauses.append((clause_str,)) - - elif full_text and isinstance(q, dict): - ts_query_alias = _ts_query_alias() - clause_str = u'_full_text @@ {0}'.format(ts_query_alias) - clauses.append((clause_str,)) - # update clauses with q dict - _update_where_clauses_on_q_dict(data_dict, fields_types, q, clauses) - - elif full_text and isinstance(q, str): - ts_query_alias = _ts_query_alias() - clause_str = u'_full_text @@ {0}'.format(ts_query_alias) - clauses.append((clause_str,)) - - return clauses - - -def _update_where_clauses_on_q_dict( - data_dict: dict[str, str], fields_types: dict[str, str], - q: dict[str, str], clauses: list[tuple[str]]) -> None: - lang = _fts_lang(data_dict.get('language')) - for field, _ in q.items(): - if field not in fields_types: - continue - query_field = _ts_query_alias(field) - - ftyp = fields_types[field] - if not datastore_helpers.should_fts_index_field_type(ftyp): - clause_str = u'_full_text @@ {0}'.format(query_field) - clauses.append((clause_str,)) - - clause_str = ( - u'to_tsvector({0}, cast({1} as text)) @@ {2}').format( - literal_string(lang), - identifier(field), - query_field) - clauses.append((clause_str,)) - - -def _textsearch_query( - lang: str, q: Optional[Union[str, dict[str, str], Any]], plain: bool, - full_text: Optional[str]) -> tuple[str, dict[str, str]]: - u''' - :param lang: language for to_tsvector - :param q: string to search _full_text or dict to search columns - :param plain: True to use plainto_tsquery, False for to_tsquery - :param full_text: string to search _full_text - - return (query, rank_columns) based on passed text/dict query - rank_columns is a {alias: statement} dict where alias is "rank" for - _full_text queries, and "rank " for column search - ''' - if not (q or full_text): - return '', {} - - statements: list[str] = [] - rank_columns: dict[str, str] = {} - if q and not full_text: - if isinstance(q, str): - query, rank = _build_query_and_rank_statements( - lang, q, plain) - statements.append(query) - rank_columns[u'rank'] = rank - elif isinstance(q, dict): - for field, value in q.items(): - query, rank = _build_query_and_rank_statements( - lang, value, plain, field) - statements.append(query) - rank_columns[u'rank ' + field] = rank - elif full_text and not q: - _update_rank_statements_and_columns( - statements, rank_columns, lang, full_text, plain - ) - elif full_text and isinstance(q, dict): - _update_rank_statements_and_columns( - statements, rank_columns, lang, full_text, plain) - for field, value in q.items(): - _update_rank_statements_and_columns( - statements, rank_columns, lang, value, plain, field - ) - elif full_text and isinstance(q, str): - _update_rank_statements_and_columns( - statements, rank_columns, lang, full_text, plain - ) - - statements_str = ', ' + ', '.join(statements) - return statements_str, rank_columns - - -def _update_rank_statements_and_columns( - statements: list[str], rank_columns: dict[str, str], lang: str, - query: str, plain: bool, field: Optional[str] = None): - query, rank = _build_query_and_rank_statements( - lang, query, plain, field) - statements.append(query) - if field: - rank_columns[u'rank ' + field] = rank - else: - rank_columns[u'rank'] = rank - - -def _build_query_and_rank_statements( - lang: str, query: str, plain: bool, field: Optional[str] = None): - query_alias = _ts_query_alias(field) - lang_literal = literal_string(lang) - query_literal = literal_string(query) - if plain: - statement = u"plainto_tsquery({lang_literal}, {literal}) {alias}" - else: - statement = u"to_tsquery({lang_literal}, {literal}) {alias}" - statement = statement.format( - lang_literal=lang_literal, - literal=query_literal, alias=query_alias) - if field is None: - rank_field = u'_full_text' - else: - rank_field = u'to_tsvector({0}, cast({1} as text))'.format( - lang_literal, identifier(field)) - rank_statement = u'ts_rank({0}, {1}, 32)'.format( - rank_field, query_alias) - return statement, rank_statement - - -def _fts_lang(lang: Optional[str] = None) -> str: - return lang or config.get('ckan.datastore.default_fts_lang') - - -def _sort(sort: Union[None, str, list[str]], fields_types: Container[str], - rank_columns: dict[str, Any]) -> list[str]: - u''' - :param sort: string or list sort parameter passed to datastore_search, - use None if not given - :param fields_types: OrderedDict returned from _get_fields_types(..) - :param rank_columns: rank_columns returned from _ts_query(..) - - returns sort expression as a string. When sort is None use rank_columns - to order by best text search match - ''' - if not sort: - rank_sorting: list[str] = [] - for column in rank_columns.values(): - rank_sorting.append(u'{0} DESC'.format(column)) - return rank_sorting - - clauses = datastore_helpers.get_list(sort, False) or [] - - clause_parsed: list[str] = [] - - for clause in clauses: - parsed = _parse_sort_clause(clause, fields_types) - if parsed: - field, sort = parsed - clause_parsed.append( - u'{0} {1}'.format(identifier(field), sort)) - return clause_parsed - - -def _ts_query_alias(field: Optional[str] = None): - query_alias = u'query' - if field: - query_alias += u' ' + field - return identifier(query_alias) - - -def _get_aliases(context: Context, data_dict: dict[str, Any]): - '''Get a list of aliases for a resource.''' - res_id = data_dict['resource_id'] - alias_sql = sqlalchemy.text( - u'SELECT name FROM "_table_metadata" WHERE alias_of = :id') - results = context['connection'].execute(alias_sql, id=res_id).fetchall() - return [x[0] for x in results] - - -def _get_resources(context: Context, alias: str): - '''Get a list of resources for an alias. There could be more than one alias - in a resource_dict.''' - alias_sql = sqlalchemy.text( - u'''SELECT alias_of FROM "_table_metadata" - WHERE name = :alias AND alias_of IS NOT NULL''') - results = context['connection'].execute(alias_sql, alias=alias).fetchall() - return [x[0] for x in results] - - -def create_alias(context: Context, data_dict: dict[str, Any]): - values: Optional[str] = data_dict.get('aliases') - aliases: Any = datastore_helpers.get_list(values) - alias = None - if aliases is not None: - # delete previous aliases - previous_aliases = _get_aliases(context, data_dict) - for alias in previous_aliases: - sql_alias_drop_string = u'DROP VIEW "{0}"'.format(alias) - context['connection'].execute(sql_alias_drop_string) - - try: - for alias in aliases: - sql_alias_string = u'''CREATE VIEW "{alias}" - AS SELECT * FROM "{main}"'''.format( - alias=alias, - main=data_dict['resource_id'] - ) - - res_ids = _get_resources(context, alias) - if res_ids: - raise ValidationError({ - 'alias': [(u'The alias "{0}" already exists.').format( - alias)] - }) - - context['connection'].execute(sql_alias_string) - except DBAPIError as e: - if e.orig.pgcode in [_PG_ERR_CODE['duplicate_table'], - _PG_ERR_CODE['duplicate_alias']]: - raise ValidationError({ - 'alias': [u'"{0}" already exists'.format(alias)] - }) - - -def _generate_index_name(resource_id: str, field: str): - value = (resource_id + field).encode('utf-8') - return hashlib.sha1(value).hexdigest() - - -def _get_fts_index_method() -> str: - return config.get('ckan.datastore.default_fts_index_method') - - -def _build_fts_indexes( - data_dict: dict[str, Any], # noqa - sql_index_str_method: str, fields: list[dict[str, Any]]): - fts_indexes: list[str] = [] - resource_id = data_dict['resource_id'] - fts_lang = data_dict.get( - 'language', config.get('ckan.datastore.default_fts_lang')) - - # create full-text search indexes - def to_tsvector(x: str): - return u"to_tsvector('{0}', {1})".format(fts_lang, x) - - def cast_as_text(x: str): - return u'cast("{0}" AS text)'.format(x) - - full_text_field = {'type': 'tsvector', 'id': '_full_text'} - for field in [full_text_field] + fields: - if not datastore_helpers.should_fts_index_field_type(field['type']): - continue - - field_str = field['id'] - if field['type'] not in ['text', 'tsvector']: - field_str = cast_as_text(field_str) - else: - field_str = u'"{0}"'.format(field_str) - if field['type'] != 'tsvector': - field_str = to_tsvector(field_str) - fts_indexes.append(sql_index_str_method.format( - res_id=resource_id, - unique='', - name=_generate_index_name(resource_id, field_str), - method=_get_fts_index_method(), fields=field_str)) - - return fts_indexes - - -def _drop_indexes(context: Context, data_dict: dict[str, Any], - unique: bool = False): - sql_drop_index = u'DROP INDEX "{0}" CASCADE' - sql_get_index_string = u""" - SELECT - i.relname AS index_name - FROM - pg_class t, - pg_class i, - pg_index idx - WHERE - t.oid = idx.indrelid - AND i.oid = idx.indexrelid - AND t.relkind = 'r' - AND idx.indisunique = {unique} - AND idx.indisprimary = false - AND t.relname = %s - """.format(unique='true' if unique else 'false') - indexes_to_drop = context['connection'].execute( - sql_get_index_string, data_dict['resource_id']).fetchall() - for index in indexes_to_drop: - context['connection'].execute( - sql_drop_index.format(index[0]).replace('%', '%%')) - - -def _get_index_names(connection: Any, resource_id: str): - sql = u""" - SELECT - i.relname AS index_name - FROM - pg_class t, - pg_class i, - pg_index idx - WHERE - t.oid = idx.indrelid - AND i.oid = idx.indexrelid - AND t.relkind = 'r' - AND t.relname = %s - """ - results = connection.execute(sql, resource_id).fetchall() - return [result[0] for result in results] - - -def _is_valid_pg_type(context: Context, type_name: str): - if type_name in _type_names: - return True - else: - connection = context['connection'] - try: - connection.execute('SELECT %s::regtype', type_name) - except ProgrammingError as e: - if e.orig.pgcode in [_PG_ERR_CODE['undefined_object'], - _PG_ERR_CODE['syntax_error']]: - return False - raise - else: - return True - - -def _execute_single_statement( - context: Context, sql_string: str, where_values: Any): - if not datastore_helpers.is_single_statement(sql_string): - raise ValidationError({ - 'query': ['Query is not a single statement.'] - }) - - results = context['connection'].execute(sql_string, [where_values]) - - return results - - -def _insert_links(data_dict: dict[str, Any], limit: int, offset: int): - '''Adds link to the next/prev part (same limit, offset=offset+limit) - and the resource page.''' - data_dict['_links'] = {} - - # get the url from the request - try: - urlstring = toolkit.request.environ['CKAN_CURRENT_URL'] - except (KeyError, TypeError, RuntimeError): - return # no links required for local actions - - # change the offset in the url - parsed = list(urlparse(urlstring)) - query = parsed[4] - - arguments = dict(parse_qsl(query)) - arguments_start = dict(arguments) - arguments_prev: dict[str, Any] = dict(arguments) - arguments_next: dict[str, Any] = dict(arguments) - if 'offset' in arguments_start: - arguments_start.pop('offset') - arguments_next['offset'] = int(offset) + int(limit) - arguments_prev['offset'] = int(offset) - int(limit) - - parsed_start = parsed[:] - parsed_prev = parsed[:] - parsed_next = parsed[:] - parsed_start[4] = urlencode(arguments_start) - parsed_next[4] = urlencode(arguments_next) - parsed_prev[4] = urlencode(arguments_prev) - - # add the links to the data dict - data_dict['_links']['start'] = urlunparse(parsed_start) - data_dict['_links']['next'] = urlunparse(parsed_next) - if int(offset) - int(limit) > 0: - data_dict['_links']['prev'] = urlunparse(parsed_prev) - - -def _where( - where_clauses_and_values: list[tuple[Any, ...]] -) -> tuple[str, list[Any]]: - '''Return a SQL WHERE clause from list with clauses and values - - :param where_clauses_and_values: list of tuples with format - (where_clause, param1, ...) - :type where_clauses_and_values: list of tuples - - :returns: SQL WHERE string with placeholders for the parameters, and list - of parameters - :rtype: string - ''' - where_clauses = [] - values: list[Any] = [] - - for clause_and_values in where_clauses_and_values: - where_clauses.append('(' + clause_and_values[0] + ')') - values += clause_and_values[1:] - - where_clause = u' AND '.join(where_clauses) - if where_clause: - where_clause = u'WHERE ' + where_clause - - return where_clause, values - - -def convert(data: Any, type_name: str) -> Any: - if data is None: - return None - if type_name == 'nested': - return json.loads(data[0]) - # array type - if type_name.startswith('_'): - sub_type = type_name[1:] - return [convert(item, sub_type) for item in data] - if type_name == 'tsvector': - return six.ensure_text(data) - if isinstance(data, datetime.datetime): - return data.isoformat() - if isinstance(data, (int, float)): - return data - return str(data) - - -def check_fields(context: Context, fields: Iterable[dict[str, Any]]): - '''Check if field types are valid.''' - for field in fields: - if field.get('type') and not _is_valid_pg_type(context, field['type']): - raise ValidationError({ - 'fields': [u'"{0}" is not a valid field type'.format( - field['type'])] - }) - elif not datastore_helpers.is_valid_field_name(field['id']): - raise ValidationError({ - 'fields': [u'"{0}" is not a valid field name'.format( - field['id'])] - }) - - -Indexes: TypeAlias = "Optional[list[Union[str, list[str]]]]" - - -def create_indexes(context: Context, data_dict: dict[str, Any]): - connection = context['connection'] - - indexes: Indexes = cast(Indexes, datastore_helpers.get_list( - data_dict.get('indexes', None))) - # primary key is not a real primary key - # it's just a unique key - primary_key: Any = datastore_helpers.get_list(data_dict.get('primary_key')) - - sql_index_tmpl = u'CREATE {unique} INDEX "{name}" ON "{res_id}"' - sql_index_string_method = sql_index_tmpl + u' USING {method}({fields})' - sql_index_string = sql_index_tmpl + u' ({fields})' - sql_index_strings: list[str] = [] - - fields = _get_fields(connection, data_dict['resource_id']) - field_ids = _pluck('id', fields) - json_fields = [x['id'] for x in fields if x['type'] == 'nested'] - - fts_indexes = _build_fts_indexes(data_dict, - sql_index_string_method, - fields) - sql_index_strings = sql_index_strings + fts_indexes - - if indexes is not None: - _drop_indexes(context, data_dict, False) - else: - indexes = [] - - if primary_key is not None: - unique_keys = _get_unique_key(context, data_dict) - if sorted(unique_keys) != sorted(primary_key): - _drop_indexes(context, data_dict, True) - indexes.append(primary_key) - - for index in indexes: - if not index: - continue - - index_fields = datastore_helpers.get_list(index) - assert index_fields is not None - for field in index_fields: - if field not in field_ids: - raise ValidationError({ - 'index': [ - u'The field "{0}" is not a valid column name.'.format( - index)] - }) - fields_string = u', '.join( - ['(("{0}").json::text)'.format(field) - if field in json_fields else - '"%s"' % field - for field in index_fields]) - sql_index_strings.append(sql_index_string.format( - res_id=data_dict['resource_id'], - unique='unique' if index == primary_key else '', - name=_generate_index_name(data_dict['resource_id'], fields_string), - fields=fields_string)) - - sql_index_strings = [x.replace('%', '%%') for x in sql_index_strings] - current_indexes = _get_index_names(context['connection'], - data_dict['resource_id']) - for sql_index_string in sql_index_strings: - has_index = [c for c in current_indexes - if sql_index_string.find(c) != -1] - if not has_index: - connection.execute(sql_index_string) - - -def create_table(context: Context, data_dict: dict[str, Any]): - '''Creates table, columns and column info (stored as comments). - - :param resource_id: The resource ID (i.e. postgres table name) - :type resource_id: string - :param fields: details of each field/column, each with properties: - id - field/column name - type - optional, otherwise it is guessed from the first record - info - some field/column properties, saved as a JSON string in postgres - as a column comment. e.g. "type_override", "label", "notes" - :type fields: list of dicts - :param records: records, of which the first is used when a field type needs - guessing. - :type records: list of dicts - ''' - - datastore_fields = [ - {'id': '_id', 'type': 'serial primary key'}, - {'id': '_full_text', 'type': 'tsvector'}, - ] - - # check first row of data for additional fields - extra_fields = [] - supplied_fields = data_dict.get('fields', []) - check_fields(context, supplied_fields) - field_ids = _pluck('id', supplied_fields) - records = data_dict.get('records') - - fields_errors = [] - - for field_id in field_ids: - # Postgres has a limit of 63 characters for a column name - if len(field_id) > 63: - message = 'Column heading "{0}" exceeds limit of 63 '\ - 'characters.'.format(field_id) - fields_errors.append(message) - - if fields_errors: - raise ValidationError({ - 'fields': fields_errors - }) - - # if type is field is not given try and guess or throw an error - for field in supplied_fields: - if 'type' not in field: - if not records or field['id'] not in records[0]: - raise ValidationError({ - 'fields': [u'"{0}" type not guessable'.format(field['id'])] - }) - field['type'] = _guess_type(records[0][field['id']]) - - # Check for duplicate fields - unique_fields = {f['id'] for f in supplied_fields} - if not len(unique_fields) == len(supplied_fields): - raise ValidationError({ - 'field': ['Duplicate column names are not supported'] - }) - - if records: - # check record for sanity - if not isinstance(records[0], dict): - raise ValidationError({ - 'records': ['The first row is not a json object'] - }) - supplied_field_ids = records[0].keys() - for field_id in supplied_field_ids: - if field_id not in field_ids: - extra_fields.append({ - 'id': field_id, - 'type': _guess_type(records[0][field_id]) - }) - - fields = datastore_fields + supplied_fields + extra_fields - sql_fields = u", ".join([u'{0} {1}'.format( - identifier(f['id']), f['type']) for f in fields]) - - sql_string = u'CREATE TABLE {0} ({1});'.format( - identifier(data_dict['resource_id']), - sql_fields - ) - - info_sql = [] - for f in supplied_fields: - info = f.get(u'info') - if isinstance(info, dict): - info_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format( - identifier(data_dict['resource_id']), - identifier(f['id']), - literal_string( - json.dumps(info, ensure_ascii=False)))) - - context['connection'].execute( - (sql_string + u';'.join(info_sql)).replace(u'%', u'%%')) - - -def alter_table(context: Context, data_dict: dict[str, Any]): - '''Adds new columns and updates column info (stored as comments). - - :param resource_id: The resource ID (i.e. postgres table name) - :type resource_id: string - :param fields: details of each field/column, each with properties: - id - field/column name - type - optional, otherwise it is guessed from the first record - info - some field/column properties, saved as a JSON string in postgres - as a column comment. e.g. "type_override", "label", "notes" - :type fields: list of dicts - :param records: records, of which the first is used when a field type needs - guessing. - :type records: list of dicts - ''' - supplied_fields = data_dict.get('fields', []) - current_fields = _get_fields( - context['connection'], data_dict['resource_id']) - if not supplied_fields: - supplied_fields = current_fields - check_fields(context, supplied_fields) - field_ids = _pluck('id', supplied_fields) - records = data_dict.get('records') - new_fields: list[dict[str, Any]] = [] - - for num, field in enumerate(supplied_fields): - # check to see if field definition is the same or and - # extension of current fields - if num < len(current_fields): - if field['id'] != current_fields[num]['id']: - raise ValidationError({ - 'fields': [(u'Supplied field "{0}" not ' - u'present or in wrong order').format( - field['id'])] - }) - # no need to check type as field already defined. - continue - - if 'type' not in field: - if not records or field['id'] not in records[0]: - raise ValidationError({ - 'fields': [u'"{0}" type not guessable'.format(field['id'])] - }) - field['type'] = _guess_type(records[0][field['id']]) - new_fields.append(field) - - if records: - # check record for sanity as they have not been - # checked during validation - if not isinstance(records, list): - raise ValidationError({ - 'records': ['Records has to be a list of dicts'] - }) - if not isinstance(records[0], dict): - raise ValidationError({ - 'records': ['The first row is not a json object'] - }) - supplied_field_ids = cast(Dict[str, Any], records[0]).keys() - for field_id in supplied_field_ids: - if field_id not in field_ids: - new_fields.append({ - 'id': field_id, - 'type': _guess_type(records[0][field_id]) - }) - - alter_sql = [] - for f in new_fields: - alter_sql.append(u'ALTER TABLE {0} ADD {1} {2};'.format( - identifier(data_dict['resource_id']), - identifier(f['id']), - f['type'])) - - for f in supplied_fields: - if u'info' in f: - info = f.get(u'info') - if isinstance(info, dict): - info_sql = literal_string( - json.dumps(info, ensure_ascii=False)) - else: - info_sql = 'NULL' - alter_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format( - identifier(data_dict['resource_id']), - identifier(f['id']), - info_sql)) - - if alter_sql: - context['connection'].execute( - u';'.join(alter_sql).replace(u'%', u'%%')) - - -def insert_data(context: Context, data_dict: dict[str, Any]): - """ - - :raises InvalidDataError: if there is an invalid value in the given data - - """ - data_dict['method'] = _INSERT - result = upsert_data(context, data_dict) - return result - - -def upsert_data(context: Context, data_dict: dict[str, Any]): - '''insert all data from records''' - if not data_dict.get('records'): - return - - method = data_dict.get('method', _UPSERT) - - fields = _get_fields(context['connection'], data_dict['resource_id']) - field_names = _pluck('id', fields) - records = data_dict['records'] - sql_columns = ", ".join( - identifier(name) for name in field_names) - - if method == _INSERT: - rows = [] - for num, record in enumerate(records): - _validate_record(record, num, field_names) - - row = [] - for field in fields: - value = record.get(field['id']) - if value is not None and field['type'].lower() == 'nested': - # a tuple with an empty second value - value = (json.dumps(value), '') - row.append(value) - rows.append(row) - - sql_string = u'''INSERT INTO {res_id} ({columns}) - VALUES ({values});'''.format( - res_id=identifier(data_dict['resource_id']), - columns=sql_columns.replace('%', '%%'), - values=', '.join(['%s' for _ in field_names]) - ) - - try: - context['connection'].execute(sql_string, rows) - except sqlalchemy.exc.DataError as err: - raise InvalidDataError( - toolkit._("The data was invalid: {}" - ).format(_programming_error_summary(err))) - except sqlalchemy.exc.DatabaseError as err: - raise ValidationError( - {u'records': [_programming_error_summary(err)]}) - - elif method in [_UPDATE, _UPSERT]: - unique_keys = _get_unique_key(context, data_dict) - - for num, record in enumerate(records): - if not unique_keys and '_id' not in record: - raise ValidationError({ - 'table': [u'unique key must be passed for update/upsert'] - }) - - elif '_id' not in record: - # all key columns have to be defined - missing_fields = [field for field in unique_keys - if field not in record] - if missing_fields: - raise ValidationError({ - 'key': [u'''fields "{fields}" are missing - but needed as key'''.format( - fields=', '.join(missing_fields))] - }) - - for field in fields: - value = record.get(field['id']) - if value is not None and field['type'].lower() == 'nested': - # a tuple with an empty second value - record[field['id']] = (json.dumps(value), '') - - non_existing_field_names = [ - field for field in record - if field not in field_names and field != '_id' - ] - if non_existing_field_names: - raise ValidationError({ - 'fields': [u'fields "{0}" do not exist'.format( - ', '.join(non_existing_field_names))] - }) - - if '_id' in record: - unique_values = [record['_id']] - pk_sql = '"_id"' - pk_values_sql = '%s' - else: - unique_values = [record[key] for key in unique_keys] - pk_sql = ','.join([identifier(part) for part in unique_keys]) - pk_values_sql = ','.join(['%s'] * len(unique_keys)) - - used_fields = [field for field in fields - if field['id'] in record] - - used_field_names = _pluck('id', used_fields) - - used_values = [record[field] for field in used_field_names] - - if method == _UPDATE: - sql_string = u''' - UPDATE {res_id} - SET ({columns}, "_full_text") = ({values}, NULL) - WHERE ({primary_key}) = ({primary_value}); - '''.format( - res_id=identifier(data_dict['resource_id']), - columns=u', '.join( - [identifier(field) - for field in used_field_names]).replace('%', '%%'), - values=u', '.join( - ['%s' for _ in used_field_names]), - primary_key=pk_sql.replace('%', '%%'), - primary_value=pk_values_sql, - ) - try: - results = context['connection'].execute( - sql_string, used_values + unique_values) - except sqlalchemy.exc.DatabaseError as err: - raise ValidationError({ - u'records': [_programming_error_summary(err)], - u'_records_row': num}) - - # validate that exactly one row has been updated - if results.rowcount != 1: - raise ValidationError({ - 'key': [u'key "{0}" not found'.format(unique_values)] - }) - - elif method == _UPSERT: - sql_string = u''' - UPDATE {res_id} - SET ({columns}, "_full_text") = ({values}, NULL) - WHERE ({primary_key}) = ({primary_value}); - INSERT INTO {res_id} ({columns}) - SELECT {values} - WHERE NOT EXISTS (SELECT 1 FROM {res_id} - WHERE ({primary_key}) = ({primary_value})); - '''.format( - res_id=identifier(data_dict['resource_id']), - columns=u', '.join( - [identifier(field) - for field in used_field_names]).replace('%', '%%'), - values=u', '.join(['%s::nested' - if field['type'] == 'nested' else '%s' - for field in used_fields]), - primary_key=pk_sql.replace('%', '%%'), - primary_value=pk_values_sql, - ) - try: - context['connection'].execute( - sql_string, - (used_values + unique_values) * 2) - except sqlalchemy.exc.DatabaseError as err: - raise ValidationError({ - u'records': [_programming_error_summary(err)], - u'_records_row': num}) - - -def validate(context: Context, data_dict: dict[str, Any]): - fields_types = _get_fields_types( - context['connection'], data_dict['resource_id']) - data_dict_copy = copy.deepcopy(data_dict) - - # TODO: Convert all attributes that can be a comma-separated string to - # lists - if 'fields' in data_dict_copy: - fields = datastore_helpers.get_list(data_dict_copy['fields']) - data_dict_copy['fields'] = fields - if 'sort' in data_dict_copy: - fields = datastore_helpers.get_list(data_dict_copy['sort'], False) - data_dict_copy['sort'] = fields - - for plugin in plugins.PluginImplementations(interfaces.IDatastore): - data_dict_copy = plugin.datastore_validate(context, - data_dict_copy, - fields_types) - - # Remove default elements in data_dict - data_dict_copy.pop('connection_url', None) - data_dict_copy.pop('resource_id', None) - - data_dict_copy.pop('id', None) - data_dict_copy.pop('include_total', None) - data_dict_copy.pop('total_estimation_threshold', None) - data_dict_copy.pop('records_format', None) - data_dict_copy.pop('calculate_record_count', None) - - for key, values in data_dict_copy.items(): - if not values: - continue - if isinstance(values, str): - value = values - elif isinstance(values, (list, tuple)): - value: Any = values[0] - elif isinstance(values, dict): - value: Any = list(values.keys())[0] - else: - value = values - - raise ValidationError({ - key: [u'invalid value "{0}"'.format(value)] - }) - - return True - - -def search_data(context: Context, data_dict: dict[str, Any]): - validate(context, data_dict) - fields_types = _get_fields_types( - context['connection'], data_dict['resource_id']) - - query_dict: dict[str, Any] = { - 'select': [], - 'sort': [], - 'where': [] - } - - for plugin in p.PluginImplementations(interfaces.IDatastore): - query_dict = plugin.datastore_search(context, data_dict, - fields_types, query_dict) - - where_clause, where_values = _where(query_dict['where']) - - # FIXME: Remove duplicates on select columns - select_columns = ', '.join(query_dict['select']).replace('%', '%%') - ts_query = cast(str, query_dict['ts_query']).replace('%', '%%') - resource_id = data_dict['resource_id'].replace('%', '%%') - sort = query_dict['sort'] - limit = query_dict['limit'] - offset = query_dict['offset'] - - if query_dict.get('distinct'): - distinct = 'DISTINCT' - else: - distinct = '' - - if not sort and not distinct: - sort = ['_id'] - - if sort: - sort_clause = 'ORDER BY %s' % (', '.join(sort)).replace('%', '%%') - else: - sort_clause = '' - - records_format = data_dict['records_format'] - if records_format == u'objects': - sql_fmt = u''' - SELECT array_to_json(array_agg(j))::text FROM ( - SELECT {distinct} {select} - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset} - ) AS j''' - elif records_format == u'lists': - select_columns = u" || ',' || ".join( - s for s in query_dict['select'] - ).replace('%', '%%') - sql_fmt = u''' - SELECT '[' || array_to_string(array_agg(j.v), ',') || ']' FROM ( - SELECT {distinct} '[' || {select} || ']' v - FROM ( - SELECT * FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset}) as z - ) AS j''' - elif records_format == u'csv': - sql_fmt = u''' - COPY ( - SELECT {distinct} {select} - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset} - ) TO STDOUT csv DELIMITER ',' ''' - elif records_format == u'tsv': - sql_fmt = u''' - COPY ( - SELECT {distinct} {select} - FROM "{resource}" {ts_query} - {where} {sort} LIMIT {limit} OFFSET {offset} - ) TO STDOUT csv DELIMITER '\t' ''' - else: - sql_fmt = u'' - sql_string = sql_fmt.format( - distinct=distinct, - select=select_columns, - resource=resource_id, - ts_query=ts_query, - where=where_clause, - sort=sort_clause, - limit=limit, - offset=offset) - if records_format == u'csv' or records_format == u'tsv': - buf = StringIO() - _execute_single_statement_copy_to( - context, sql_string, where_values, buf) - records = buf.getvalue() - else: - v = list(_execute_single_statement( - context, sql_string, where_values))[0][0] - if v is None or v == '[]': - records = [] - else: - records = LazyJSONObject(v) - data_dict['records'] = records - - data_dict['fields'] = _result_fields( - fields_types, - _get_field_info(context['connection'], data_dict['resource_id']), - datastore_helpers.get_list(data_dict.get('fields'))) - - _unrename_json_field(data_dict) - - _insert_links(data_dict, limit, offset) - - if data_dict.get('include_total', True): - total_estimation_threshold = \ - data_dict.get('total_estimation_threshold') - estimated_total = None - if total_estimation_threshold is not None and \ - not (where_clause or distinct): - # there are no filters, so we can try to use the estimated table - # row count from pg stats - # See: https://wiki.postgresql.org/wiki/Count_estimate - # (We also tried using the EXPLAIN to estimate filtered queries but - # it didn't estimate well in tests) - analyze_count_sql = sqlalchemy.text(''' - SELECT reltuples::BIGINT AS approximate_row_count - FROM pg_class - WHERE relname=:resource; - ''') - count_result = context['connection'].execute(analyze_count_sql, - resource=resource_id) - try: - estimated_total = count_result.fetchall()[0][0] - except ValueError: - # the table doesn't have the stats calculated yet. (This should - # be done by xloader/datapusher at the end of loading.) - # We could provoke their creation with an ANALYZE, but that - # takes 10x the time to run, compared to SELECT COUNT(*) so - # we'll just revert to the latter. At some point the autovacuum - # will run and create the stats so we can use an estimate in - # future. - pass - - if estimated_total is not None \ - and estimated_total >= total_estimation_threshold: - data_dict['total'] = estimated_total - data_dict['total_was_estimated'] = True - else: - # this is slow for large results - count_sql_string = u'''SELECT count(*) FROM ( - SELECT {distinct} {select} - FROM "{resource}" {ts_query} {where}) as t;'''.format( - distinct=distinct, - select=select_columns, - resource=resource_id, - ts_query=ts_query, - where=where_clause) - count_result = _execute_single_statement( - context, count_sql_string, where_values) - data_dict['total'] = count_result.fetchall()[0][0] - data_dict['total_was_estimated'] = False - - return data_dict - - -def _execute_single_statement_copy_to( - context: Context, sql_string: str, - where_values: Any, buf: Any): - if not datastore_helpers.is_single_statement(sql_string): - raise ValidationError({ - 'query': ['Query is not a single statement.'] - }) - - cursor = context['connection'].connection.cursor() - cursor.copy_expert(cursor.mogrify(sql_string, where_values), buf) - cursor.close() - - -def format_results(context: Context, results: Any, data_dict: dict[str, Any]): - result_fields: list[dict[str, Any]] = [] - for field in results.cursor.description: - result_fields.append({ - 'id': six.ensure_text(field[0]), - 'type': _get_type(context['connection'], field[1]) - }) - - records = [] - for row in results: - converted_row = {} - for field in result_fields: - converted_row[field['id']] = convert(row[field['id']], - field['type']) - records.append(converted_row) - data_dict['records'] = records - if data_dict.get('records_truncated', False): - data_dict['records'].pop() - data_dict['fields'] = result_fields - - return _unrename_json_field(data_dict) - - -def delete_data(context: Context, data_dict: dict[str, Any]): - validate(context, data_dict) - fields_types = _get_fields_types( - context['connection'], data_dict['resource_id']) - - query_dict: dict[str, Any] = { - 'where': [] - } - - for plugin in plugins.PluginImplementations(interfaces.IDatastore): - query_dict = plugin.datastore_delete(context, data_dict, - fields_types, query_dict) - - where_clause, where_values = _where(query_dict['where']) - sql_string = u'DELETE FROM "{0}" {1}'.format( - data_dict['resource_id'], - where_clause - ) - - _execute_single_statement(context, sql_string, where_values) - - -def _create_triggers(connection: Any, resource_id: str, - triggers: Iterable[dict[str, Any]]): - u''' - Delete existing triggers on table then create triggers - - Currently our schema requires "before insert or update" - triggers run on each row, so we're not reading "when" - or "for_each" parameters from triggers list. - ''' - existing = connection.execute( - u"""SELECT tgname FROM pg_trigger - WHERE tgrelid = %s::regclass AND tgname LIKE 't___'""", - resource_id) - sql_list = ( - [u'DROP TRIGGER {name} ON {table}'.format( - name=identifier(r[0]), - table=identifier(resource_id)) - for r in existing] + - [u'''CREATE TRIGGER {name} - BEFORE INSERT OR UPDATE ON {table} - FOR EACH ROW EXECUTE PROCEDURE {function}()'''.format( - # 1000 triggers per table should be plenty - name=identifier(u't%03d' % i), - table=identifier(resource_id), - function=identifier(t['function'])) - for i, t in enumerate(triggers)]) - try: - if sql_list: - connection.execute(u';\n'.join(sql_list)) - except ProgrammingError as pe: - raise ValidationError({u'triggers': [_programming_error_summary(pe)]}) - - -def _create_fulltext_trigger(connection: Any, resource_id: str): - connection.execute( - u'''CREATE TRIGGER zfulltext - BEFORE INSERT OR UPDATE ON {table} - FOR EACH ROW EXECUTE PROCEDURE populate_full_text_trigger()'''.format( - table=identifier(resource_id))) - - -def upsert(context: Context, data_dict: dict[str, Any]): - ''' - This method combines upsert insert and update on the datastore. The method - that will be used is defined in the mehtod variable. - - Any error results in total failure! For now pass back the actual error. - Should be transactional. - ''' - backend = DatastorePostgresqlBackend.get_active_backend() - engine = backend._get_write_engine() # type: ignore - context['connection'] = engine.connect() - timeout = context.get('query_timeout', _TIMEOUT) - - trans: Any = context['connection'].begin() - try: - # check if table already existes - context['connection'].execute( - u'SET LOCAL statement_timeout TO {0}'.format(timeout)) - upsert_data(context, data_dict) - if data_dict.get(u'dry_run', False): - trans.rollback() - else: - trans.commit() - return _unrename_json_field(data_dict) - except IntegrityError as e: - if e.orig.pgcode == _PG_ERR_CODE['unique_violation']: - raise ValidationError(cast(ErrorDict, { - 'constraints': ['Cannot insert records or create index because' - ' of uniqueness constraint'], - 'info': { - 'orig': str(e.orig), - 'pgcode': e.orig.pgcode - } - })) - raise - except DataError as e: - raise ValidationError(cast(ErrorDict, { - 'data': str(e), - 'info': { - 'orig': [str(e.orig)] - }})) - except DBAPIError as e: - if e.orig.pgcode == _PG_ERR_CODE['query_canceled']: - raise ValidationError({ - 'query': ['Query took too long'] - }) - raise - except Exception: - trans.rollback() - raise - finally: - context['connection'].close() - - -def search(context: Context, data_dict: dict[str, Any]): - backend = DatastorePostgresqlBackend.get_active_backend() - engine = backend._get_read_engine() # type: ignore - context['connection'] = engine.connect() - timeout = context.get('query_timeout', _TIMEOUT) - _cache_types(context['connection']) - - try: - context['connection'].execute( - u'SET LOCAL statement_timeout TO {0}'.format(timeout)) - return search_data(context, data_dict) - except DBAPIError as e: - if e.orig.pgcode == _PG_ERR_CODE['query_canceled']: - raise ValidationError({ - 'query': ['Search took too long'] - }) - raise ValidationError(cast(ErrorDict, { - 'query': ['Invalid query'], - 'info': { - 'statement': [e.statement], - 'params': [e.params], - 'orig': [str(e.orig)] - } - })) - finally: - context['connection'].close() - - -def search_sql(context: Context, data_dict: dict[str, Any]): - backend = DatastorePostgresqlBackend.get_active_backend() - engine = backend._get_read_engine() # type: ignore - - context['connection'] = engine.connect() - timeout = context.get('query_timeout', _TIMEOUT) - _cache_types(context['connection']) - - sql = data_dict['sql'].replace('%', '%%') - - # limit the number of results to ckan.datastore.search.rows_max + 1 - # (the +1 is so that we know if the results went over the limit or not) - rows_max = config.get('ckan.datastore.search.rows_max') - sql = 'SELECT * FROM ({0}) AS blah LIMIT {1} ;'.format(sql, rows_max + 1) - - try: - - context['connection'].execute( - u'SET LOCAL statement_timeout TO {0}'.format(timeout)) - - get_names = datastore_helpers.get_table_and_function_names_from_sql - table_names, function_names = get_names(context, sql) - log.debug('Tables involved in input SQL: {0!r}'.format(table_names)) - log.debug('Functions involved in input SQL: {0!r}'.format( - function_names)) - - if any(t.startswith('pg_') for t in table_names): - raise toolkit.NotAuthorized( - 'Not authorized to access system tables' - ) - context['check_access'](table_names) - - for f in function_names: - for name_variant in [f.lower(), '"{}"'.format(f)]: - if name_variant in \ - backend.allowed_sql_functions: # type: ignore - break - else: - raise toolkit.NotAuthorized( - 'Not authorized to call function {}'.format(f) - ) - - results: Any = context['connection'].execute(sql) - - if results.rowcount == rows_max + 1: - data_dict['records_truncated'] = True - - return format_results(context, results, data_dict) - - except ProgrammingError as e: - if e.orig.pgcode == _PG_ERR_CODE['permission_denied']: - raise toolkit.NotAuthorized('Not authorized to read resource.') - - def _remove_explain(msg: str): - return (msg.replace('EXPLAIN (VERBOSE, FORMAT JSON) ', '') - .replace('EXPLAIN ', '')) - - raise ValidationError(cast(ErrorDict, { - 'query': [_remove_explain(str(e))], - 'info': { - 'statement': [_remove_explain(e.statement)], - 'params': [e.params], - 'orig': [_remove_explain(str(e.orig))] - } - })) - except DBAPIError as e: - if e.orig.pgcode == _PG_ERR_CODE['query_canceled']: - raise ValidationError({ - 'query': ['Query took too long'] - }) - raise - finally: - context['connection'].close() - - -class DatastorePostgresqlBackend(DatastoreBackend): - - def _get_write_engine(self): - return _get_engine_from_url(self.write_url) - - def _get_read_engine(self): - return _get_engine_from_url(self.read_url) - - def _log_or_raise(self, message: str): - if self.config.get('debug'): - log.critical(message) - else: - raise DatastoreException(message) - - def _check_urls_and_permissions(self): - # Make sure that the right permissions are set - # so that no harmful queries can be made - - if self._same_ckan_and_datastore_db(): - self._log_or_raise( - 'CKAN and DataStore database cannot be the same.') - - if self._same_read_and_write_url(): - self._log_or_raise('The write and read-only database ' - 'connection urls are the same.') - - if not self._read_connection_has_correct_privileges(): - self._log_or_raise('The read-only user has write privileges.') - - def _is_postgresql_engine(self): - ''' Returns True if the read engine is a Postgresql Database. - - According to - http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql - all Postgres driver names start with `postgres`. - ''' - drivername = self._get_read_engine().engine.url.drivername - return drivername.startswith('postgres') - - def _is_read_only_database(self): - ''' Returns True if no connection has CREATE privileges on the public - schema. This is the case if replication is enabled.''' - for url in [self.ckan_url, self.write_url, self.read_url]: - connection = _get_engine_from_url(url).connect() - try: - sql = u"SELECT has_schema_privilege('public', 'CREATE')" - is_writable: bool = connection.execute(sql).one()[0] - finally: - connection.close() - if is_writable: - return False - return True - - def _same_ckan_and_datastore_db(self): - '''Returns True if the CKAN and DataStore db are the same''' - return self._get_db_from_url(self.ckan_url) == self._get_db_from_url( - self.read_url) - - def _get_db_from_url(self, url: str): - db_url = sa_url.make_url(url) - return db_url.host, db_url.port, db_url.database - - def _same_read_and_write_url(self) -> bool: - return self.write_url == self.read_url - - def _read_connection_has_correct_privileges(self): - ''' Returns True if the right permissions are set for the read - only user. A table is created by the write user to test the - read only user. - ''' - write_connection = self._get_write_engine().connect() - read_connection_user = sa_url.make_url(self.read_url).username - - drop_foo_sql = u'DROP TABLE IF EXISTS _foo' - - write_connection.execute(drop_foo_sql) - - try: - write_connection.execute(u'CREATE TEMP TABLE _foo ()') - for privilege in ['INSERT', 'UPDATE', 'DELETE']: - privilege_sql = u"SELECT has_table_privilege(%s, '_foo', %s)" - have_privilege: bool = write_connection.execute( - privilege_sql, - (read_connection_user, privilege) - ).one()[0] - if have_privilege: - return False - finally: - write_connection.execute(drop_foo_sql) - write_connection.close() - return True - - def configure(self, config: CKANConfig): - self.config = config - # check for ckan.datastore.write_url and ckan.datastore.read_url - if ('ckan.datastore.write_url' not in config): - error_msg = 'ckan.datastore.write_url not found in config' - raise DatastoreException(error_msg) - if ('ckan.datastore.read_url' not in config): - error_msg = 'ckan.datastore.read_url not found in config' - raise DatastoreException(error_msg) - - # Check whether users have disabled datastore_search_sql - self.enable_sql_search = self.config.get( - 'ckan.datastore.sqlsearch.enabled') - - if self.enable_sql_search: - allowed_sql_functions_file = self.config.get( - 'ckan.datastore.sqlsearch.allowed_functions_file' - ) - - def format_entry(line: str): - '''Prepare an entry from the 'allowed_functions' file - to be used in the whitelist. - - Leading and trailing whitespace is removed, and the - entry is lowercased unless enclosed in "double quotes". - ''' - entry = line.strip() - if not entry.startswith('"'): - entry = entry.lower() - return entry - - with open(allowed_sql_functions_file, 'r') as f: - self.allowed_sql_functions = set(format_entry(line) - for line in f) - - # Check whether we are running one of the paster commands which means - # that we should ignore the following tests. - args = sys.argv - if args[0].split('/')[-1] == 'paster' and 'datastore' in args[1:]: - log.warn('Omitting permission checks because you are ' - 'running paster commands.') - return - - self.ckan_url = self.config['sqlalchemy.url'] - self.write_url = self.config['ckan.datastore.write_url'] - self.read_url = self.config['ckan.datastore.read_url'] - - if not self._is_postgresql_engine(): - log.warn('We detected that you do not use a PostgreSQL ' - 'database. The DataStore will NOT work and DataStore ' - 'tests will be skipped.') - return - - if self._is_read_only_database(): - log.warn('We detected that CKAN is running on a read ' - 'only database. Permission checks and the creation ' - 'of _table_metadata are skipped.') - else: - self._check_urls_and_permissions() - - def datastore_delete( - self, context: Context, data_dict: dict[str, Any], # noqa - fields_types: dict[str, Any], query_dict: dict[str, Any]): - query_dict['where'] += _where_clauses(data_dict, fields_types) - return query_dict - - def datastore_search( - self, context: Context, data_dict: dict[str, Any], # noqa - fields_types: dict[str, Any], query_dict: dict[str, Any]): - - fields: str = data_dict.get('fields', '') - - ts_query, rank_columns = _textsearch_query( - _fts_lang(data_dict.get('language')), - data_dict.get('q'), - data_dict.get('plain', True), - data_dict.get('full_text')) - # mutate parameter to add rank columns for _result_fields - for rank_alias in rank_columns: - fields_types[rank_alias] = u'float' - fts_q = data_dict.get('full_text') - if fields and not fts_q: - field_ids = datastore_helpers.get_list(fields) - elif fields and fts_q: - field_ids = datastore_helpers.get_list(fields) - all_field_ids = list(fields_types.keys()) - if "rank" not in fields: - all_field_ids.remove("rank") - field_intersect = [x for x in field_ids - if x not in all_field_ids] - field_ids = all_field_ids + field_intersect - else: - field_ids = fields_types.keys() - - # add default limit here just in case - already defaulted in the schema - limit = data_dict.get('limit', 100) - offset = data_dict.get('offset', 0) - - sort = _sort( - data_dict.get('sort'), - fields_types, - rank_columns) - where = _where_clauses(data_dict, fields_types) - select_cols = [] - records_format = data_dict.get('records_format') - for field_id in field_ids: - fmt = '{0}' - if records_format == 'lists': - fmt = "coalesce(to_json({0}),'null')" - typ = fields_types.get(field_id, '') - if typ == 'nested': - fmt = "coalesce(({0}).json,'null')" - elif typ == 'timestamp': - fmt = "to_char({0}, 'YYYY-MM-DD\"T\"HH24:MI:SS')" - if records_format == 'lists': - fmt = f"coalesce(to_json({fmt}), 'null')" - elif typ.startswith('_') or typ.endswith('[]'): - fmt = "coalesce(array_to_json({0}),'null')" - - if field_id in rank_columns: - select_cols.append((fmt + ' as {1}').format( - rank_columns[field_id], identifier(field_id))) - continue - - if records_format == 'objects': - fmt += ' as {0}' - select_cols.append(fmt.format(identifier(field_id))) - - query_dict['distinct'] = data_dict.get('distinct', False) - query_dict['select'] += select_cols - query_dict['ts_query'] = ts_query - query_dict['sort'] += sort - query_dict['where'] += where - query_dict['limit'] = limit - query_dict['offset'] = offset - - return query_dict - - def delete(self, context: Context, data_dict: dict[str, Any]): - engine = self._get_write_engine() - context['connection'] = engine.connect() - _cache_types(context['connection']) - - trans = context['connection'].begin() - try: - # check if table exists - if 'filters' not in data_dict: - context['connection'].execute( - u'DROP TABLE "{0}" CASCADE'.format( - data_dict['resource_id']) - ) - else: - delete_data(context, data_dict) - - trans.commit() - return _unrename_json_field(data_dict) - except Exception: - trans.rollback() - raise - finally: - context['connection'].close() - - def create(self, context: Context, data_dict: dict[str, Any]): - ''' - The first row will be used to guess types not in the fields and the - guessed types will be added to the headers permanently. - Consecutive rows have to conform to the field definitions. - rows can be empty so that you can just set the fields. - fields are optional but needed if you want to do type hinting or - add extra information for certain columns or to explicitly - define ordering. - eg: [{"id": "dob", "type": "timestamp"}, - {"id": "name", "type": "text"}] - A header items values can not be changed after it has been defined - nor can the ordering of them be changed. They can be extended though. - Any error results in total failure! For now pass back the actual error. - Should be transactional. - :raises InvalidDataError: if there is an invalid value in the given - data - ''' - engine = get_write_engine() - context['connection'] = engine.connect() - timeout = context.get('query_timeout', _TIMEOUT) - _cache_types(context['connection']) - - _rename_json_field(data_dict) - - trans = context['connection'].begin() - try: - # check if table already exists - context['connection'].execute( - u'SET LOCAL statement_timeout TO {0}'.format(timeout)) - result = context['connection'].execute( - u'SELECT * FROM pg_tables WHERE tablename = %s', - data_dict['resource_id'] - ).fetchone() - if not result: - create_table(context, data_dict) - _create_fulltext_trigger( - context['connection'], - data_dict['resource_id']) - else: - alter_table(context, data_dict) - if 'triggers' in data_dict: - _create_triggers( - context['connection'], - data_dict['resource_id'], - data_dict['triggers']) - insert_data(context, data_dict) - create_indexes(context, data_dict) - create_alias(context, data_dict) - trans.commit() - return _unrename_json_field(data_dict) - except IntegrityError as e: - if e.orig.pgcode == _PG_ERR_CODE['unique_violation']: - raise ValidationError(cast(ErrorDict, { - 'constraints': ['Cannot insert records or create index' - 'because of uniqueness constraint'], - 'info': { - 'orig': str(e.orig), - 'pgcode': e.orig.pgcode - } - })) - raise - except DataError as e: - raise ValidationError(cast(ErrorDict, { - 'data': str(e), - 'info': { - 'orig': [str(e.orig)] - }})) - except DBAPIError as e: - if e.orig.pgcode == _PG_ERR_CODE['query_canceled']: - raise ValidationError({ - 'query': ['Query took too long'] - }) - raise - except Exception: - trans.rollback() - raise - finally: - context['connection'].close() - - def upsert(self, context: Context, data_dict: dict[str, Any]): - data_dict['connection_url'] = self.write_url - return upsert(context, data_dict) - - def search(self, context: Context, data_dict: dict[str, Any]): - data_dict['connection_url'] = self.write_url - return search(context, data_dict) - - def search_sql(self, context: Context, data_dict: dict[str, Any]): - sql = toolkit.get_or_bust(data_dict, 'sql') - data_dict['connection_url'] = self.read_url - - if not is_single_statement(sql): - raise toolkit.ValidationError({ - 'query': ['Query is not a single statement.'] - }) - return search_sql(context, data_dict) - - def resource_exists(self, id: str) -> bool: - resources_sql = sqlalchemy.text( - u'''SELECT 1 FROM "_table_metadata" - WHERE name = :id AND alias_of IS NULL''') - results = self._get_read_engine().execute(resources_sql, id=id) - res_exists = results.rowcount > 0 - return res_exists - - def resource_id_from_alias(self, alias: str) -> tuple[bool, Optional[str]]: - real_id: Optional[str] = None - resources_sql = sqlalchemy.text( - u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') - results = self._get_read_engine().execute(resources_sql, id=alias) - - res_exists = results.rowcount > 0 - if res_exists: - real_id = results.fetchone()[0] # type: ignore - return res_exists, real_id - - # def resource_info(self, id): - # pass - - def resource_fields(self, id: str) -> dict[str, Any]: - - info: dict[str, Any] = {'meta': {}, 'fields': []} - - try: - engine = self._get_read_engine() - - # resource id for deferencing aliases - info['meta']['id'] = id - - # count of rows in table - meta_sql = sqlalchemy.text( - u'SELECT count(_id) FROM "{0}"'.format(id)) - meta_results = engine.execute(meta_sql) - info['meta']['count'] = meta_results.fetchone()[0] # type: ignore - - # table_type - BASE TABLE, VIEW, FOREIGN TABLE, MATVIEW - tabletype_sql = sqlalchemy.text(u''' - SELECT table_type FROM INFORMATION_SCHEMA.TABLES - WHERE table_name = '{0}' - '''.format(id)) - tabletype_results = engine.execute(tabletype_sql) - info['meta']['table_type'] = \ - tabletype_results.fetchone()[0] # type: ignore - # MATERIALIZED VIEWS show as BASE TABLE, so - # we check pg_matviews - matview_sql = sqlalchemy.text(u''' - SELECT count(*) FROM pg_matviews - WHERE matviewname = '{0}' - '''.format(id)) - matview_results = engine.execute(matview_sql) - if matview_results.fetchone()[0]: # type: ignore - info['meta']['table_type'] = 'MATERIALIZED VIEW' - - # SIZE - size of table in bytes - size_sql = sqlalchemy.text( - u"SELECT pg_relation_size('{0}')".format(id)) - size_results = engine.execute(size_sql) - info['meta']['size'] = size_results.fetchone()[0] # type: ignore - - # DB_SIZE - size of database in bytes - dbsize_sql = sqlalchemy.text( - u"SELECT pg_database_size(current_database())") - dbsize_results = engine.execute(dbsize_sql) - info['meta']['db_size'] = \ - dbsize_results.fetchone()[0] # type: ignore - - # IDXSIZE - size of all indices for table in bytes - idxsize_sql = sqlalchemy.text( - u"SELECT pg_indexes_size('{0}')".format(id)) - idxsize_results = engine.execute(idxsize_sql) - info['meta']['idx_size'] = \ - idxsize_results.fetchone()[0] # type: ignore - - # all the aliases for this resource - alias_sql = sqlalchemy.text(u''' - SELECT name FROM "_table_metadata" WHERE alias_of = '{0}' - '''.format(id)) - alias_results = engine.execute(alias_sql) - aliases = [] - for alias in alias_results.fetchall(): - aliases.append(alias[0]) - info['meta']['aliases'] = aliases - - # get the data dictionary for the resource - data_dictionary = datastore_helpers.datastore_dictionary(id) - - schema_sql = sqlalchemy.text(u''' - SELECT - f.attname AS column_name, - pg_catalog.format_type(f.atttypid,f.atttypmod) AS native_type, - f.attnotnull AS notnull, - i.relname as index_name, - CASE - WHEN i.oid<>0 THEN True - ELSE False - END AS is_index, - CASE - WHEN p.contype = 'u' THEN True - WHEN p.contype = 'p' THEN True - ELSE False - END AS uniquekey - FROM pg_attribute f - JOIN pg_class c ON c.oid = f.attrelid - JOIN pg_type t ON t.oid = f.atttypid - LEFT JOIN pg_constraint p ON p.conrelid = c.oid - AND f.attnum = ANY (p.conkey) - LEFT JOIN pg_index AS ix ON f.attnum = ANY(ix.indkey) - AND c.oid = f.attrelid AND c.oid = ix.indrelid - LEFT JOIN pg_class AS i ON ix.indexrelid = i.oid - WHERE c.relkind = 'r'::char - AND c.relname = '{0}' - AND f.attnum > 0 - ORDER BY c.relname,f.attnum; - '''.format(id)) - schema_results = engine.execute(schema_sql) - schemainfo = {} - for row in schema_results.fetchall(): - row: Any # Row has incomplete type definition - colname: str = row.column_name - if colname.startswith('_'): # Skip internal rows - continue - colinfo: dict[str, Any] = {'native_type': row.native_type, - 'notnull': row.notnull, - 'index_name': row.index_name, - 'is_index': row.is_index, - 'uniquekey': row.uniquekey} - schemainfo[colname] = colinfo - - for field in data_dictionary: - field.update({'schema': schemainfo[field['id']]}) - info['fields'].append(field) - - except Exception: - pass - return info - - def get_all_ids(self): - resources_sql = sqlalchemy.text( - u'''SELECT name FROM "_table_metadata" - WHERE alias_of IS NULL''') - query = self._get_read_engine().execute(resources_sql) - return [q[0] for q in query.fetchall()] - - def create_function(self, *args: Any, **kwargs: Any): - return create_function(*args, **kwargs) - - def drop_function(self, *args: Any, **kwargs: Any): - return drop_function(*args, **kwargs) - - def before_fork(self): - # Called by DatastorePlugin.before_fork. Dispose SQLAlchemy engines - # to avoid sharing them between parent and child processes. - _dispose_engines() - - def calculate_record_count(self, resource_id: str): - ''' - Calculate an estimate of the record/row count and store it in - Postgresql's pg_stat_user_tables. This number will be used when - specifying `total_estimation_threshold` - ''' - connection = get_write_engine().connect() - sql = 'ANALYZE "{}"'.format(resource_id) - try: - connection.execute(sql) - except sqlalchemy.exc.DatabaseError as err: - raise DatastoreException(err) - - -def create_function(name: str, arguments: Iterable[dict[str, Any]], - rettype: Any, definition: str, or_replace: bool): - sql = u''' - CREATE {or_replace} FUNCTION - {name}({args}) RETURNS {rettype} AS {definition} - LANGUAGE plpgsql;'''.format( - or_replace=u'OR REPLACE' if or_replace else u'', - name=identifier(name), - args=u', '.join( - u'{argname} {argtype}'.format( - argname=identifier(a['argname']), - argtype=identifier(a['argtype'])) - for a in arguments), - rettype=identifier(rettype), - definition=literal_string(definition)) - - try: - _write_engine_execute(sql) - except ProgrammingError as pe: - already_exists = ( - u'function "{}" already exists with same argument types' - .format(name) - in pe.args[0]) - key = u'name' if already_exists else u'definition' - raise ValidationError({key: [_programming_error_summary(pe)]}) - - -def drop_function(name: str, if_exists: bool): - sql = u''' - DROP FUNCTION {if_exists} {name}(); - '''.format( - if_exists=u'IF EXISTS' if if_exists else u'', - name=identifier(name)) - - try: - _write_engine_execute(sql) - except ProgrammingError as pe: - raise ValidationError({u'name': [_programming_error_summary(pe)]}) - - -def _write_engine_execute(sql: str): - connection = get_write_engine().connect() - # No special meaning for '%' in sql parameter: - connection: Any = connection.execution_options(no_parameters=True) - trans = connection.begin() - try: - connection.execute(sql) - trans.commit() - except Exception: - trans.rollback() - raise - finally: - connection.close() - - -def _programming_error_summary(pe: Any): - u''' - return the text description of a sqlalchemy DatabaseError - without the actual SQL included, for raising as a - ValidationError to send back to API users - ''' - # first line only, after the '(ProgrammingError)' text - message = six.ensure_text(pe.args[0].split('\n')[0]) - return message.split(u') ', 1)[-1] diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py deleted file mode 100644 index ba10a69..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/blueprint.py +++ /dev/null @@ -1,290 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from typing import Any, Optional, cast, Union -from itertools import zip_longest - -from flask import Blueprint -from flask.wrappers import Response -from flask.views import MethodView - -import ckan.lib.navl.dictization_functions as dict_fns -from ckan.logic import ( - tuplize_dict, - parse_params, -) -from ckan.plugins.toolkit import ( - ObjectNotFound, NotAuthorized, get_action, get_validator, _, request, - abort, render, g, h -) -from ckan.types import Schema, ValidatorFactory -from ckanext.datastore.logic.schema import ( - list_of_strings_or_string, - json_validator, - unicode_or_json_validator, -) -from ckanext.datastore.writer import ( - csv_writer, - tsv_writer, - json_writer, - xml_writer, -) - -int_validator = get_validator(u'int_validator') -boolean_validator = get_validator(u'boolean_validator') -ignore_missing = get_validator(u'ignore_missing') -one_of = cast(ValidatorFactory, get_validator(u'one_of')) -default = cast(ValidatorFactory, get_validator(u'default')) -unicode_only = get_validator(u'unicode_only') - -DUMP_FORMATS = u'csv', u'tsv', u'json', u'xml' -PAGINATE_BY = 32000 - -datastore = Blueprint(u'datastore', __name__) - - -def dump_schema() -> Schema: - return { - u'offset': [default(0), int_validator], - u'limit': [ignore_missing, int_validator], - u'format': [default(u'csv'), one_of(DUMP_FORMATS)], - u'bom': [default(False), boolean_validator], - u'filters': [ignore_missing, json_validator], - u'q': [ignore_missing, unicode_or_json_validator], - u'distinct': [ignore_missing, boolean_validator], - u'plain': [ignore_missing, boolean_validator], - u'language': [ignore_missing, unicode_only], - u'fields': [ignore_missing, list_of_strings_or_string], - u'sort': [default(u'_id'), list_of_strings_or_string], - } - - -def dump(resource_id: str): - try: - get_action('datastore_search')({}, {'resource_id': resource_id, - 'limit': 0}) - except ObjectNotFound: - abort(404, _('DataStore resource not found')) - - data, errors = dict_fns.validate(request.args.to_dict(), dump_schema()) - if errors: - abort( - 400, '\n'.join( - '{0}: {1}'.format(k, ' '.join(e)) for k, e in errors.items() - ) - ) - - fmt = data['format'] - offset = data['offset'] - limit = data.get('limit') - options = {'bom': data['bom']} - sort = data['sort'] - search_params = { - k: v - for k, v in data.items() - if k in [ - 'filters', 'q', 'distinct', 'plain', 'language', - 'fields' - ] - } - - user_context = g.user - - content_type = None - content_disposition = None - - if fmt == 'csv': - content_disposition = 'attachment; filename="{name}.csv"'.format( - name=resource_id) - content_type = b'text/csv; charset=utf-8' - elif fmt == 'tsv': - content_disposition = 'attachment; filename="{name}.tsv"'.format( - name=resource_id) - content_type = b'text/tab-separated-values; charset=utf-8' - elif fmt == 'json': - content_disposition = 'attachment; filename="{name}.json"'.format( - name=resource_id) - content_type = b'application/json; charset=utf-8' - elif fmt == 'xml': - content_disposition = 'attachment; filename="{name}.xml"'.format( - name=resource_id) - content_type = b'text/xml; charset=utf-8' - else: - abort(404, _('Unsupported format')) - - headers = {} - if content_type: - headers['Content-Type'] = content_type - if content_disposition: - headers['Content-disposition'] = content_disposition - - try: - return Response(dump_to(resource_id, - fmt=fmt, - offset=offset, - limit=limit, - options=options, - sort=sort, - search_params=search_params, - user=user_context), - mimetype='application/octet-stream', - headers=headers) - except ObjectNotFound: - abort(404, _('DataStore resource not found')) - - -class DictionaryView(MethodView): - - def _prepare(self, id: str, resource_id: str) -> dict[str, Any]: - try: - # resource_edit_base template uses these - pkg_dict = get_action(u'package_show')({}, {u'id': id}) - resource = get_action(u'resource_show')({}, {u'id': resource_id}) - rec = get_action(u'datastore_search')( - {}, { - u'resource_id': resource_id, - u'limit': 0 - } - ) - return { - u'pkg_dict': pkg_dict, - u'resource': resource, - u'fields': [ - f for f in rec[u'fields'] if not f[u'id'].startswith(u'_') - ] - } - - except (ObjectNotFound, NotAuthorized): - abort(404, _(u'Resource not found')) - - def get(self, id: str, resource_id: str): - u'''Data dictionary view: show field labels and descriptions''' - - data_dict = self._prepare(id, resource_id) - - # global variables for backward compatibility - g.pkg_dict = data_dict[u'pkg_dict'] - g.resource = data_dict[u'resource'] - - return render(u'datastore/dictionary.html', data_dict) - - def post(self, id: str, resource_id: str): - u'''Data dictionary view: edit field labels and descriptions''' - data_dict = self._prepare(id, resource_id) - fields = data_dict[u'fields'] - data = dict_fns.unflatten(tuplize_dict(parse_params(request.form))) - info = data.get(u'info') - if not isinstance(info, list): - info = [] - info = info[:len(fields)] - - get_action(u'datastore_create')( - {}, { - u'resource_id': resource_id, - u'force': True, - u'fields': [{ - u'id': f[u'id'], - u'type': f[u'type'], - u'info': fi if isinstance(fi, dict) else {} - } for f, fi in zip_longest(fields, info)] - } - ) - - h.flash_success( - _( - u'Data Dictionary saved. Any type overrides will ' - u'take effect when the resource is next uploaded ' - u'to DataStore' - ) - ) - return h.redirect_to( - u'datastore.dictionary', id=id, resource_id=resource_id - ) - - -def dump_to( - resource_id: str, fmt: str, offset: int, - limit: Optional[int], options: dict[str, Any], sort: str, - search_params: dict[str, Any], user: str -): - if fmt == 'csv': - writer_factory = csv_writer - records_format = 'csv' - elif fmt == 'tsv': - writer_factory = tsv_writer - records_format = 'tsv' - elif fmt == 'json': - writer_factory = json_writer - records_format = 'lists' - elif fmt == 'xml': - writer_factory = xml_writer - records_format = 'objects' - else: - assert False, 'Unsupported format' - - bom = options.get('bom', False) - - def start_stream_writer(fields: list[dict[str, Any]]): - return writer_factory(fields, bom=bom) - - def stream_result_page(offs: int, lim: Union[None, int]): - return get_action('datastore_search')( - {'user': user}, - dict({ - 'resource_id': resource_id, - 'limit': PAGINATE_BY - if limit is None else min(PAGINATE_BY, lim), # type: ignore - 'offset': offs, - 'sort': sort, - 'records_format': records_format, - 'include_total': False, - }, **search_params) - ) - - def stream_dump(offset: int, limit: Union[None, int], - paginate_by: int, result: dict[str, Any]): - with start_stream_writer(result['fields']) as writer: - while True: - if limit is not None and limit <= 0: - break - - records = result['records'] - - yield writer.write_records(records) - - if records_format == 'objects' or records_format == 'lists': - if len(records) < paginate_by: - break - elif not records: - break - - offset += paginate_by - if limit is not None: - limit -= paginate_by - if limit <= 0: - break - - result = stream_result_page(offset, limit) - - yield writer.end_file() - - result = stream_result_page(offset, limit) - - if result['limit'] != limit: - # `limit` (from PAGINATE_BY) must have been more than - # ckan.datastore.search.rows_max, so datastore_search responded - # with a limit matching ckan.datastore.search.rows_max. - # So we need to paginate by that amount instead, otherwise - # we'll have gaps in the records. - paginate_by = result['limit'] - else: - paginate_by = PAGINATE_BY - - return stream_dump(offset, limit, paginate_by, result) - - -datastore.add_url_rule(u'/datastore/dump/', view_func=dump) -datastore.add_url_rule( - u'/dataset//dictionary/', - view_func=DictionaryView.as_view(str(u'dictionary')) -) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/cli.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/cli.py deleted file mode 100644 index 8091cba..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/cli.py +++ /dev/null @@ -1,179 +0,0 @@ -# encoding: utf-8 - -from typing import Any -import logging -import os - -import click - -from ckan.model import parse_db_config -from ckan.common import config -import ckan.logic as logic - -import ckanext.datastore as datastore_module -from ckanext.datastore.backend.postgres import identifier -from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to - -log = logging.getLogger(__name__) - - -@click.group(short_help=u"Perform commands to set up the datastore.") -def datastore(): - """Perform commands to set up the datastore. - """ - pass - - -@datastore.command( - u'set-permissions', - short_help=u'Generate SQL for permission configuration.' -) -def set_permissions(): - u'''Emit an SQL script that will set the permissions for the datastore - users as configured in your configuration file.''' - - write_url = _parse_db_config(u'ckan.datastore.write_url') - read_url = _parse_db_config(u'ckan.datastore.read_url') - db_url = _parse_db_config(u'sqlalchemy.url') - - # Basic validation that read and write URLs reference the same database. - # This obviously doesn't check they're the same database (the hosts/ports - # could be different), but it's better than nothing, I guess. - - if write_url[u'db_name'] != read_url[u'db_name']: - click.secho( - u'The datastore write_url and read_url must refer to the same ' - u'database!', - fg=u'red', - bold=True - ) - raise click.Abort() - - sql = permissions_sql( - maindb=db_url[u'db_name'], - datastoredb=write_url[u'db_name'], - mainuser=db_url[u'db_user'], - writeuser=write_url[u'db_user'], - readuser=read_url[u'db_user'] - ) - - click.echo(sql) - - -def permissions_sql(maindb: str, datastoredb: str, mainuser: str, - writeuser: str, readuser: str): - template_filename = os.path.join( - os.path.dirname(datastore_module.__file__), u'set_permissions.sql' - ) - with open(template_filename) as fp: - template = fp.read() - return template.format( - maindb=identifier(maindb), - datastoredb=identifier(datastoredb), - mainuser=identifier(mainuser), - writeuser=identifier(writeuser), - readuser=identifier(readuser) - ) - - -@datastore.command() -@click.argument(u'resource-id', nargs=1) -@click.argument( - u'output-file', - type=click.File(u'wb'), - default=click.get_binary_stream(u'stdout') -) -@click.option(u'--format', default=u'csv', type=click.Choice(DUMP_FORMATS)) -@click.option(u'--offset', type=click.IntRange(0, None), default=0) -@click.option(u'--limit', type=click.IntRange(0)) -@click.option(u'--bom', is_flag=True) -@click.pass_context -def dump(ctx: Any, resource_id: str, output_file: Any, format: str, - offset: int, limit: int, bom: bool): - u'''Dump a datastore resource. - ''' - flask_app = ctx.meta['flask_app'] - user = logic.get_action('get_site_user')( - {'ignore_auth': True}, {}) - with flask_app.test_request_context(): - for block in dump_to(resource_id, - fmt=format, - offset=offset, - limit=limit, - options={u'bom': bom}, - sort=u'_id', - search_params={}, - user=user['name']): - output_file.write(block) - - -def _parse_db_config(config_key: str = u'sqlalchemy.url'): - db_config = parse_db_config(config_key) - if not db_config: - click.secho( - u'Could not extract db details from url: %r' % config[config_key], - fg=u'red', - bold=True - ) - raise click.Abort() - return db_config - - -@datastore.command( - 'purge', - short_help='purge orphaned resources from the datastore.' -) -def purge(): - '''Purge orphaned resources from the datastore using the datastore_delete - action, which drops tables when called without filters.''' - - site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {}) - - result = logic.get_action('datastore_search')( - {'user': site_user['name']}, - {'resource_id': '_table_metadata'} - ) - - resource_id_list = [] - for record in result['records']: - try: - # ignore 'alias' records (views) as they are automatically - # deleted when the parent resource table is dropped - if record['alias_of']: - continue - - logic.get_action('resource_show')( - {'user': site_user['name']}, - {'id': record['name']} - ) - except logic.NotFound: - resource_id_list.append(record['name']) - click.echo("Resource '%s' orphaned - queued for drop" % - record[u'name']) - except KeyError: - continue - - orphaned_table_count = len(resource_id_list) - click.echo('%d orphaned tables found.' % orphaned_table_count) - - if not orphaned_table_count: - return - - click.confirm('Proceed with purge?', abort=True) - - # Drop the orphaned datastore tables. When datastore_delete is called - # without filters, it does a drop table cascade - drop_count = 0 - for resource_id in resource_id_list: - logic.get_action('datastore_delete')( - {'user': site_user['name']}, - {'resource_id': resource_id, 'force': True} - ) - click.echo("Table '%s' dropped)" % resource_id) - drop_count += 1 - - click.echo('Dropped %s tables' % drop_count) - - -def get_commands(): - return (set_permissions, dump, purge) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml deleted file mode 100644 index 5e30803..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/config_declaration.yaml +++ /dev/null @@ -1,101 +0,0 @@ -version: 1 -groups: -- annotation: Datastore settings - options: - - default: postgresql://ckan_default:pass@localhost/datastore_default - key: ckan.datastore.write_url - example: postgresql://ckanuser:pass@localhost/datastore - required: true - description: >- - The database connection to use for writing to the datastore (this can be - ignored if you're not using the :ref:`datastore`). Note that the database used - should not be the same as the normal CKAN database. The format is the same as - in :ref:`sqlalchemy.url`. - - - default: postgresql://datastore_default:pass@localhost/datastore_default - key: ckan.datastore.read_url - required: true - example: postgresql://readonlyuser:pass@localhost/datastore - description: >- - The database connection to use for reading from the datastore (this can be - ignored if you're not using the :ref:`datastore`). The database used must be - the same used in :ref:`ckan.datastore.write_url`, but the user should be one - with read permissions only. The format is the same as in :ref:`sqlalchemy.url`. - - - key: ckan.datastore.sqlsearch.allowed_functions_file - default_callable: ckanext.datastore.plugin:sql_functions_allowlist_file - example: /path/to/my_allowed_functions.txt - description: | - Allows to define the path to a text file listing the SQL functions that should be allowed to run - on queries sent to the :py:func:`~ckanext.datastore.logic.action.datastore_search_sql` function - (if enabled, see :ref:`ckan.datastore.sqlsearch.enabled`). Function names should be listed one on - each line, eg:: - - abbrev - abs - abstime - ... - - - key: ckan.datastore.sqlsearch.enabled - type: bool - example: 'true' - description: | - This option allows you to enable the :py:func:`~ckanext.datastore.logic.action.datastore_search_sql` action function, and corresponding API endpoint. - - This action function has protections from abuse including: - - - parsing of the query to prevent unsafe functions from being called, see :ref:`ckan.datastore.sqlsearch.allowed_functions_file` - - parsing of the query to prevent multiple statements - - prevention of data modification by using a read-only database role - - use of ``explain`` to resolve tables accessed in the query to check against user permissions - - use of a statement timeout to prevent queries from running indefinitely - - These protections offer some safety but are not designed to prevent all types of abuse. Depending on the sensitivity of private data in your datastore and the likelihood of abuse of your site you may choose to disable this action function or restrict its use with a :py:class:`~ckan.plugins.interfaces.IAuthFunctions` plugin. - - - default: 100 - key: ckan.datastore.search.rows_default - type: int - example: 1000 - description: | - Default number of rows returned by ``datastore_search``, unless the client - specifies a different ``limit`` (up to ``ckan.datastore.search.rows_max``). - - NB this setting does not affect ``datastore_search_sql``. - - - default: 32000 - key: ckan.datastore.search.rows_max - example: 1000000 - type: int - description: | - Maximum allowed value for the number of rows returned by the datastore. - - Specifically this limits: - - * ``datastore_search``'s ``limit`` parameter. - * ``datastore_search_sql`` queries have this limit inserted. - - - key: ckan.datastore.sqlalchemy.
    - - diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html deleted file mode 100644 index 939bf0b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/curl.html +++ /dev/null @@ -1,40 +0,0 @@ -{% call register_example('curl', 'button_label') %} -curl -{% endcall %} - - -{% call register_example('curl', 'request_limit') %} -
    curl {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }} \
-  -H"Authorization:$API_TOKEN" -d '
-{
-  "resource_id": "{{ resource_id }}",
-  "limit": 5,
-  "q": "jones"
-}'
    -{% endcall %} - - -{% call register_example('curl', 'request_filter') %} -
    curl {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }} \
--H"Authorization:$API_TOKEN" -d '
-{
-"resource_id": "{{ resource_id }}",
-  "filters": {
-    "subject": ["watershed", "survey"],
-    "stage": "active"
-  }
-}'
    -{% endcall %} - - -{% call register_example('curl', 'request_sql') %} -
    curl {{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }} \
-  -H"Authorization:$API_TOKEN" -d @- <<END
-{
-  "sql": "SELECT * FROM \"{{ resource_id }}\" WHERE title LIKE 'jones'"
-}
-END
    -{% endcall %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html deleted file mode 100644 index d0a065d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/javascript.html +++ /dev/null @@ -1,55 +0,0 @@ -{% call register_example('javascript', 'button_label') %} -JavaScript -{% endcall %} - -{% call register_example('javascript', 'request_limit') %} -
    const resp = await fetch(`{{
-  h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`, {
-    method: 'POST',
-    headers: {
-        'content-type': 'application/json',
-        authorization: API_TOKEN
-    },
-    body: JSON.stringify({
-        resource_id: '{{ resource_id }}',
-        limit: 5,
-        q: 'jones'
-    })
-})
-await resp.json()
    -{% endcall %} - - -{% call register_example('javascript', 'request_filter') %} -
    const resp = await fetch(`{{
-  h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`, {
-    method: 'POST',
-    headers: {
-        'content-type': 'application/json',
-        authorization: API_TOKEN
-    },
-    body: JSON.stringify({resource_id: '{{ resource_id }}', filters: {
-        subject: ['watershed', 'survey'],
-        stage: 'active'
-    }})})
-await resp.json()
    -{% endcall %} - - -{% call register_example('javascript', 'request_sql') %} -
    const resp = await fetch(`{{
-  h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}`, {
-    method: 'POST',
-    headers: {
-        'content-type': 'application/json',
-        authorization: API_TOKEN
-    },
-    body: JSON.stringify({
-        sql: `SELECT * FROM "{{ resource_id }}" WHERE title LIKE 'jones'`
-    })
-})
-await resp.json()
    -{% endcall %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html deleted file mode 100644 index c008d0a..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/powershell.html +++ /dev/null @@ -1,48 +0,0 @@ -{% call register_example('powershell', 'button_label') %} -PowerShell -{% endcall %} - - -{% call register_example('powershell', 'request_limit') %} -
    $json = @'
-{
-  "resource_id": "{{resource_id}}",
-  "limit": 5,
-  "q": "jones"
-}
-'@
-$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`
-  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
-$response.result.records
    -{% endcall %} - - -{% call register_example('powershell', 'request_filter') %} -
    $json = @'
-{
-  "resource_id": "{{resource_id}}",
-  "filters": {
-    "subject": ["watershed", "survey"],
-    "stage": "active"
-  }
-}
-'@
-$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`
-  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
-$response.result.records
    -{% endcall %} - - -{% call register_example('powershell', 'request_sql') %} -
    $json = @'
-{
-  "sql": "SELECT * from \"{{resource_id}}\" WHERE title LIKE 'jones'"
-}
-'@
-$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}`
-  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
-$response.result.records
    -{% endcall %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html deleted file mode 100644 index 9cd2045..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/python.html +++ /dev/null @@ -1,52 +0,0 @@ -{% call register_example('python', 'button_label') %} -Python -{% endcall %} - - -{% call register_example('python', 'request_limit') %} -
    -

    - {% trans url='https://github.com/ckan/ckanapi/blob/master/README.md#remoteckan' -%} - (using the ckanapi client library) - {%- endtrans %} -

    - 
    from ckanapi import RemoteCKAN
-
-rc = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
-result = rc.action.datastore_search(
-    resource_id="{{ resource_id }}",
-    limit=5,
-    q="jones",
-)
-print(result['records'])
    -{% endcall %} - - -{% call register_example('python', 'request_filter') %} -
    from ckanapi import RemoteCKAN
-
-rc = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
-result = rc.action.datastore_search(
-    resource_id="{{ resource_id }}",
-    filters={
-      "subject": ["watershed", "survey"],
-      "stage": "active",
-    },
-)
-print(result['records'])
    -{% endcall %} - - -{% call register_example('python', 'request_sql') %} -
    from ckanapi import RemoteCKAN
-
-rc = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
-result = rc.action.datastore_search_sql(
-    sql="""SELECT * from "{{resource_id}}" WHERE title LIKE 'jones'"""
-)
-print(result['records'])
    -{% endcall %} - diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html deleted file mode 100644 index 4ba2d60..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/api_examples/r.html +++ /dev/null @@ -1,51 +0,0 @@ -{% call register_example('r', 'button_label') %} -R -{% endcall %} - - -{% call register_example('r', 'request_limit') %} -
    library(httr2)
-
-req <- request("{{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}")
-result <- req %>% 
-    req_headers(Authorization = API_TOKEN) %>% 
-    req_body_json(list(
-        resource_id = '{{ resource_id }}',
-        limit = 5,
-        q = 'jones'))
-    req_perform %>% 
-    resp_body_json
    -{% endcall %} - - -{% call register_example('r', 'request_filter') %} -
    library(httr2)
-
-req <- request("{{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}")
-result <- req %>% 
-    req_headers(Authorization = API_TOKEN) %>% 
-    req_body_json(list(
-        resource_id='{{ resource_id }}', 
-        filters = list(
-            subject = list("watershed", "survey"), 
-            stage = "active")))
-    req_perform %>% 
-    resp_body_json
    -{% endcall %} - - -{% call register_example('r', 'request_sql') %} -
    library(httr2)
-
-req <- request("{{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}")
-result <- req %>% 
-    req_headers(Authorization = API_TOKEN) %>% 
-    req_body_json(list(
-        sql = "SELECT * FROM \"{{resource_id}}\" WHERE title LIKE 'jones'"))
-    req_perform %>% 
-    resp_body_json
-
    -{% endcall %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html deleted file mode 100644 index ac46294..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/dictionary.html +++ /dev/null @@ -1,22 +0,0 @@ -{% extends "package/resource_edit_base.html" %} - -{% import 'macros/form.html' as form %} - -{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block primary_content_inner %} - - {% set action = h.url_for('datastore.dictionary', id=pkg.name, resource_id=res.id) %} - -
    - {{ h.csrf_input() }} - {% block dictionary_form %} - {% for field in fields %} - {% snippet "datastore/snippets/dictionary_form.html", field=field, position=loop.index %} - {% endfor %} - {% endblock %} - -
    -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html deleted file mode 100644 index 41266c9..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/datastore/snippets/dictionary_form.html +++ /dev/null @@ -1,25 +0,0 @@ -{% import 'macros/form.html' as form %} - -

    {{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})

    - -{# - Data Dictionary fields may be added this snippet. New fields following - the 'info__' ~ position ~ '__namegoeshere' convention will be saved - as part of the "info" object on the column. -#} - -{{ form.select('info__' ~ position ~ '__type_override', - label=_('Type Override'), options=[ - {'name': '', 'value': ''}, - {'name': 'text', 'value': 'text'}, - {'name': 'numeric', 'value': 'numeric'}, - {'name': 'timestamp', 'value': 'timestamp'}, - ], selected=field.get('info', {}).get('type_override', '')) }} - -{{ form.input('info__' ~ position ~ '__label', - label=_('Label'), id='field-f' ~ position ~ 'label', - value=field.get('info', {}).get('label', ''), classes=['control-full']) }} - -{{ form.markdown('info__' ~ position ~ '__notes', - label=_('Description'), id='field-d' ~ position ~ 'notes', - value=field.get('info', {}).get('notes', '')) }} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html deleted file mode 100644 index 1152def..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_edit_base.html +++ /dev/null @@ -1,9 +0,0 @@ -{% ckan_extends %} - -{% block inner_primary_nav %} - {{ super() }} - {% if res.datastore_active %} - {{ h.build_nav_icon('datastore.dictionary', _('Data Dictionary'), - id=pkg.name, resource_id=res.id, icon='code') }} - {% endif %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html deleted file mode 100644 index 67edefb..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/resource_read.html +++ /dev/null @@ -1,49 +0,0 @@ -{% ckan_extends %} - -{% block resource_actions_inner %} - {{ super() }} - {% if res.datastore_active %} -
  • {% snippet 'package/snippets/data_api_button.html', resource=res %}
    • - {% endif %} -{% endblock %} - -{% block resource_additional_information_inner %} - {% if res.datastore_active %} - {% block resource_data_dictionary %} -
    -

    {{ _('Data Dictionary') }}

    - - - {% block resouce_data_dictionary_headers %} - - - - - - - {% endblock %} - - {% block resource_data_dictionary_data %} - {% set dict=h.datastore_dictionary(res.id) %} - {% for field in dict %} - {% snippet "package/snippets/dictionary_table.html", field=field %} - {% endfor %} - {% endblock %} -
    {{ _('Column') }}{{ _('Type') }}{{ _('Label') }}{{ _('Description') }}
    -
    - {% endblock %} - {% endif %} - {{ super() }} -{% endblock %} - -{% block action_manage_inner %} - {{ super() }} - {% if res.datastore_active %} -
  • {% link_for _('Data Dictionary'), named_route='datastore.dictionary', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='code' %} - {% endif %} -{% endblock %} - -{% block scripts %} - {{ super() }} - {% asset "ckanext_datastore/datastore" %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html deleted file mode 100644 index 7320529..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/data_api_button.html +++ /dev/null @@ -1,10 +0,0 @@ -{# Data API Help Button - -resource: the resource - -#} -{% if resource.datastore_active %} - {% set loading_text = _('Loading...') %} - {% set api_info_url = h.url_for('api.snippet', ver=1, snippet_path='api_info.html', resource_id=resource.id) %} - {{ _('Data API') }} -{% endif %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/dictionary_table.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/dictionary_table.html deleted file mode 100644 index adb07f3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/dictionary_table.html +++ /dev/null @@ -1,7 +0,0 @@ - - {{ field.id }} - {{ field.type }} - {{ h.get_translated(field.get('info', {}), 'label') }} - {{ h.render_markdown( - h.get_translated(field.get('info', {}), 'notes')) }} - diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html deleted file mode 100644 index 51daa78..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resource_item.html +++ /dev/null @@ -1,8 +0,0 @@ -{% ckan_extends %} - -{% block resource_item_explore_inner %} - {{ super() }} - {% if res.datastore_active %} -
  • {% link_for _('Data Dictionary'), named_route='datastore.dictionary', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='code' %} - {% endif %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html deleted file mode 100644 index 31fbae8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates-bs3/package/snippets/resources.html +++ /dev/null @@ -1,8 +0,0 @@ -{% ckan_extends %} - -{% block resources_list_edit_dropdown_inner %} - {{ super() }} - {% if resource.datastore_active %} -
  • {% link_for _('Data Dictionary'), named_route='datastore.dictionary', id=pkg.name, resource_id=resource.id, class_='dropdown-item', icon='code' %} - {% endif %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html deleted file mode 100644 index 77cfc29..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/ajax_snippets/api_info.html +++ /dev/null @@ -1,137 +0,0 @@ -{# -Displays information about accessing a resource via the API. - -resource_id - The resource id -embedded - If true will not include the "modal" classes on the snippet. - -Example - - {% snippet 'ajax_snippets/api_info.html', resource_id=resource_id, embedded=true %} - -#} - -{% set resource_id = h.sanitize_id(resource_id) %} -{% set sql_example_url = h.url_for('api.action', ver=3, logic_function='datastore_search_sql', qualified=True) + '?sql=SELECT * from "' + resource_id + '" WHERE title LIKE \'jones\'' %} -{# not urlencoding the sql because its clearer #} -
    -
    -
    -
    -

    - {{ _('CKAN Data API') }} -

    -
    -
    -

    {{ _('Access resource data via a web API with powerful query support') }}. - {% trans %} - Further information in the main - CKAN Data API and DataStore documentation.

    - {% endtrans %} -
    -
    -

    - -

    -
    -
    -

    {{ _('The Data API can be accessed via the following actions of the CKAN action API.') }}

    - - - - - - - - - - - - - - - - {% if h.datastore_search_sql_enabled() %} - - - - - {% endif %} - -
    {{ _('Create') }}{{ h.url_for('api.action', ver=3, logic_function='datastore_create', qualified=True) }}
    {{ _('Update / Insert') }}{{ h.url_for('api.action', ver=3, logic_function='datastore_upsert', qualified=True) }}
    {{ _('Query') }}{{ h.url_for('api.action', ver=3, logic_function='datastore_search', qualified=True) }}
    {{ _('Query (via SQL)') }}{{ h.url_for('api.action', ver=3, logic_function='datastore_search_sql', qualified=True) }}
    -
    -
    -
    - -
    -

    - -

    -
    -
    - {{ _('Query example (first 5 results)') }} -

    - {{ h.url_for('api.action', ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5, qualified=True) }} -

    - - {{ _('Query example (results containing \'jones\')') }} -

    - {{ h.url_for('api.action', ver=3, logic_function='datastore_search', resource_id=resource_id, q='jones', qualified=True) }} -

    - {% if h.datastore_search_sql_enabled() %} - {{ _('Query example (via SQL statement)') }} -

    - {{ sql_example_url }} -

    - {% endif %} -
    -
    -
    - -
    -

    - -

    -
    -
    -

    {{ _('A simple ajax (JSONP) request to the data API using jQuery.') }}

    - 
    -          var data = {
-            resource_id: '{{resource_id}}', // the resource id
-            limit: 5, // get 5 results
-            q: 'jones' // query for 'jones'
-          };
-          $.ajax({
-            url: '{{ h.url_for('api.action', ver=3, logic_function='datastore_search', qualified=True) }}',
-            data: data,
-            dataType: 'jsonp',
-            success: function(data) {
-              alert('Total results found: ' + data.result.total)
-            }
-          });
    -
    -
    -
    - -
    -
    - -
    -
    -
    - 
    -        import urllib.request
-        url = '{{ h.url_for('api.action', qualified=True, ver=3, logic_function='datastore_search', resource_id=resource_id, limit=5) + '&q=title:jones' }}'  {# not urlencoding the ":" because its clearer #}
-        fileobj = urllib.request.urlopen(url)
-        print(fileobj.read())
-
    -
    -
    -
    -
    - -
    -
    -
    - -
    diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/dictionary.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/dictionary.html deleted file mode 100644 index ac46294..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/dictionary.html +++ /dev/null @@ -1,22 +0,0 @@ -{% extends "package/resource_edit_base.html" %} - -{% import 'macros/form.html' as form %} - -{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block primary_content_inner %} - - {% set action = h.url_for('datastore.dictionary', id=pkg.name, resource_id=res.id) %} - -
    - {{ h.csrf_input() }} - {% block dictionary_form %} - {% for field in fields %} - {% snippet "datastore/snippets/dictionary_form.html", field=field, position=loop.index %} - {% endfor %} - {% endblock %} - -
    -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/snippets/dictionary_form.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/snippets/dictionary_form.html deleted file mode 100644 index 41266c9..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/datastore/snippets/dictionary_form.html +++ /dev/null @@ -1,25 +0,0 @@ -{% import 'macros/form.html' as form %} - -

    {{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})

    - -{# - Data Dictionary fields may be added this snippet. New fields following - the 'info__' ~ position ~ '__namegoeshere' convention will be saved - as part of the "info" object on the column. -#} - -{{ form.select('info__' ~ position ~ '__type_override', - label=_('Type Override'), options=[ - {'name': '', 'value': ''}, - {'name': 'text', 'value': 'text'}, - {'name': 'numeric', 'value': 'numeric'}, - {'name': 'timestamp', 'value': 'timestamp'}, - ], selected=field.get('info', {}).get('type_override', '')) }} - -{{ form.input('info__' ~ position ~ '__label', - label=_('Label'), id='field-f' ~ position ~ 'label', - value=field.get('info', {}).get('label', ''), classes=['control-full']) }} - -{{ form.markdown('info__' ~ position ~ '__notes', - label=_('Description'), id='field-d' ~ position ~ 'notes', - value=field.get('info', {}).get('notes', '')) }} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html deleted file mode 100644 index a5dddbd..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_edit_base.html +++ /dev/null @@ -1,8 +0,0 @@ -{% ckan_extends %} - -{% block inner_primary_nav %} - {{ super() }} - {% if res.datastore_active %} - {{ h.build_nav_icon('datastore.dictionary', _('Data Dictionary'), id=pkg.name, resource_id=res.id) }} - {% endif %} -{% endblock %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html deleted file mode 100644 index 2e1fdef..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/resource_read.html +++ /dev/null @@ -1,37 +0,0 @@ -{% ckan_extends %} - -{% block resource_actions_inner %} - {{ super() }} - {% if res.datastore_active %} -
  • {% snippet 'package/snippets/data_api_button.html', resource=res %}
    • - {% endif %} -{% endblock %} - -{% block resource_additional_information_inner %} - {% if res.datastore_active %} - {% block resource_data_dictionary %} -
    -

    {{ _('Data Dictionary') }}

    - - - {% block resouce_data_dictionary_headers %} - - - - - - - {% endblock %} - - {% block resource_data_dictionary_data %} - {% set dict=h.datastore_dictionary(res.id) %} - {% for field in dict %} - {% snippet "package/snippets/dictionary_table.html", field=field %} - {% endfor %} - {% endblock %} -
    {{ _('Column') }}{{ _('Type') }}{{ _('Label') }}{{ _('Description') }}
    -
    - {% endblock %} - {% endif %} - {{ super() }} -{% endblock %} \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html deleted file mode 100644 index 7320529..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/data_api_button.html +++ /dev/null @@ -1,10 +0,0 @@ -{# Data API Help Button - -resource: the resource - -#} -{% if resource.datastore_active %} - {% set loading_text = _('Loading...') %} - {% set api_info_url = h.url_for('api.snippet', ver=1, snippet_path='api_info.html', resource_id=resource.id) %} - {{ _('Data API') }} -{% endif %} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html deleted file mode 100644 index adb07f3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/templates/package/snippets/dictionary_table.html +++ /dev/null @@ -1,7 +0,0 @@ - - {{ field.id }} - {{ field.type }} - {{ h.get_translated(field.get('info', {}), 'label') }} - {{ h.render_markdown( - h.get_translated(field.get('info', {}), 'notes')) }} - diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt deleted file mode 100644 index 265e577..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/allowed_functions.txt +++ /dev/null @@ -1,2 +0,0 @@ -upper -"count" diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py deleted file mode 100644 index e2b13c6..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -import pytest -import ckanext.datastore.tests.helpers as test_helpers - - -@pytest.fixture -def clean_datastore(clean_db, clean_index): - engine = test_helpers.db.get_write_engine() - test_helpers.rebuild_all_dbs( - test_helpers.orm.scoped_session( - test_helpers.orm.sessionmaker(bind=engine) - ) - ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py deleted file mode 100644 index a8f1a05..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/helpers.py +++ /dev/null @@ -1,107 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import orm - -import ckan.model as model -from ckan.lib import search - -import ckan.plugins as p -from ckan.tests.helpers import FunctionalTestBase, reset_db -import ckanext.datastore.backend.postgres as db - - -def extract(d, keys): - return dict((k, d[k]) for k in keys if k in d) - - -def clear_db(Session): # noqa - drop_tables = u"""select 'drop table "' || tablename || '" cascade;' - from pg_tables where schemaname = 'public' """ - c = Session.connection() - results = c.execute(drop_tables) - for result in results: - c.execute(result[0]) - - drop_functions_sql = u""" - SELECT 'drop function if exists ' || quote_ident(proname) || '();' - FROM pg_proc - INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid) - WHERE ns.nspname = 'public' AND proname != 'populate_full_text_trigger' - """ - drop_functions = u"".join(r[0] for r in c.execute(drop_functions_sql)) - if drop_functions: - c.execute(drop_functions) - - Session.commit() - Session.remove() - - -def rebuild_all_dbs(Session): # noqa - """ If the tests are running on the same db, we have to make sure that - the ckan tables are recrated. - """ - db_read_url_parts = model.parse_db_config('ckan.datastore.write_url') - db_ckan_url_parts = model.parse_db_config('sqlalchemy.url') - same_db = db_read_url_parts["db_name"] == db_ckan_url_parts["db_name"] - - if same_db: - model.repo.tables_created_and_initialised = False - clear_db(Session) - model.repo.rebuild_db() - - -def set_url_type(resources, user): - context = {"user": user["name"]} - for resource in resources: - resource = p.toolkit.get_action("resource_show")( - context, {"id": resource.id} - ) - resource["url_type"] = "datastore" - p.toolkit.get_action("resource_update")(context, resource) - - -def execute_sql(sql, *args): - engine = db.get_write_engine() - session = orm.scoped_session(orm.sessionmaker(bind=engine)) - return session.connection().execute(sql, *args) - - -def when_was_last_analyze(resource_id): - results = execute_sql( - """SELECT last_analyze - FROM pg_stat_user_tables - WHERE relname=%s; - """, - resource_id, - ).fetchall() - return results[0][0] - - -class DatastoreFunctionalTestBase(FunctionalTestBase): - _load_plugins = (u"datastore",) - - @classmethod - def setup_class(cls): - engine = db.get_write_engine() - rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine))) - super(DatastoreFunctionalTestBase, cls).setup_class() - - -class DatastoreLegacyTestBase(object): - u""" - Tests that rely on data created in setup_class. No cleanup done between - each test method. Not recommended for new tests. - """ - - @classmethod - def setup_class(cls): - if not p.plugin_loaded(u"datastore"): - p.load(u"datastore") - reset_db() - search.clear_all() - engine = db.get_write_engine() - rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine))) - - @classmethod - def teardown_class(cls): - p.unload(u"datastore") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py deleted file mode 100644 index d82b813..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/sample_datastore_plugin.py +++ /dev/null @@ -1,56 +0,0 @@ -# encoding: utf-8 - -import ckan.plugins as p - -import ckanext.datastore.interfaces as interfaces - - -class SampleDataStorePlugin(p.SingletonPlugin): - p.implements(interfaces.IDatastore, inherit=True) - - def datastore_validate(self, context, data_dict, column_names): - valid_filters = ("age_between", "age_not_between", "insecure_filter") - filters = data_dict.get("filters", {}) - for key in list(filters.keys()): - if key in valid_filters: - del filters[key] - - return data_dict - - def datastore_search(self, context, data_dict, column_names, query_dict): - query_dict["where"] += self._where(data_dict) - return query_dict - - def datastore_delete(self, context, data_dict, column_names, query_dict): - query_dict["where"] += self._where(data_dict) - return query_dict - - def _where(self, data_dict): - filters = data_dict.get("filters", {}) - where_clauses = [] - - if "age_between" in filters: - age_between = filters["age_between"] - - clause = ( - '"age" >= %s AND "age" <= %s', - age_between[0], - age_between[1], - ) - where_clauses.append(clause) - if "age_not_between" in filters: - age_not_between = filters["age_not_between"] - - clause = ( - '"age" < %s OR "age" > %s', - age_not_between[0], - age_not_between[1], - ) - where_clauses.append(clause) - if "insecure_filter" in filters: - insecure_filter = filters["insecure_filter"] - - clause = (insecure_filter,) - where_clauses.append(clause) - - return where_clauses diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py deleted file mode 100644 index dcb931d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_auth.py +++ /dev/null @@ -1,278 +0,0 @@ -# encoding: utf-8 - -import pytest - -from ckan import model, logic -from ckan.tests import helpers, factories - - -@pytest.mark.ckan_config(u'ckan.plugins', u'datastore') -@pytest.mark.usefixtures(u'clean_db', u'with_plugins') -@pytest.mark.ckan_config(u'ckan.auth.allow_dataset_collaborators', True) -class TestCollaboratorsDataStore(): - - def _get_context(self, user): - - return { - u'model': model, - u'user': user if isinstance(user, str) else user.get(u'name') - } - - def test_datastore_search_private_editor(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_search', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor') - - assert helpers.call_auth( - u'datastore_search', - context=context, resource_id=resource[u'id']) - - def test_datastore_search_private_member(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_search', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'member') - - assert helpers.call_auth( - u'datastore_search', - context=context, resource_id=resource[u'id']) - - def test_datastore_info_private_editor(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_info', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor') - - assert helpers.call_auth( - u'datastore_info', - context=context, resource_id=resource[u'id']) - - def test_datastore_info_private_member(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_info', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'member') - - assert helpers.call_auth( - u'datastore_info', - context=context, resource_id=resource[u'id']) - - def test_datastore_search_sql_private_editor(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - context[u'table_names'] = [resource[u'id']] - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_search_sql', - context=context, sql=u'SELECT * FROM "{}"'.format( - resource[u'id'])) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor') - - assert helpers.call_auth( - u'datastore_search_sql', - context=context, sql=u'SELECT * FROM "{}"'.format(resource[u'id'])) - - def test_datastore_search_sql_private_member(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - context[u'table_names'] = [resource[u'id']] - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_search_sql', - context=context, sql=u'SELECT * FROM "{}"'.format( - resource[u'id'])) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'member') - - assert helpers.call_auth( - u'datastore_search_sql', - context=context, sql=u'SELECT * FROM "{}"'.format(resource[u'id'])) - - def test_datastore_create_private_editor(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_create', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor') - - assert helpers.call_auth( - u'datastore_create', - context=context, resource_id=resource[u'id']) - - def test_datastore_create_private_member(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_create', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'member') - - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_create', - context=context, resource_id=resource[u'id']) - - def test_datastore_upsert_private_editor(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_upsert', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor') - - assert helpers.call_auth( - u'datastore_upsert', - context=context, resource_id=resource[u'id']) - - def test_datastore_upsert_private_member(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_upsert', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'member') - - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_upsert', - context=context, resource_id=resource[u'id']) - - def test_datastore_delete_private_editor(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_delete', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'editor') - - assert helpers.call_auth( - u'datastore_delete', - context=context, resource_id=resource[u'id']) - - def test_datastore_delete_private_member(self): - - org = factories.Organization() - dataset = factories.Dataset(private=True, owner_org=org[u'id']) - resource = factories.Resource(package_id=dataset[u'id']) - user = factories.User() - - context = self._get_context(user) - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_delete', - context=context, resource_id=resource[u'id']) - - helpers.call_action( - u'package_collaborator_create', - id=dataset[u'id'], user_id=user[u'id'], capacity=u'member') - - with pytest.raises(logic.NotAuthorized): - helpers.call_auth( - u'datastore_delete', - context=context, resource_id=resource[u'id']) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py deleted file mode 100644 index e3ed8a1..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_action.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- - -import pytest - -import ckan.plugins as p -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers -from ckan.logic.action.get import package_list as core_package_list - - -package_list_message = u"The content of this message is largely irrelevant" - - -class ActionTestException(Exception): - pass - - -@p.toolkit.chained_action -def datastore_delete(up_func, context, data_dict): - res = helpers.call_action( - u"datastore_search", - resource_id=data_dict[u"resource_id"], - filters=data_dict[u"filters"], - limit=10, - ) - result = up_func(context, data_dict) - result["deleted_count"] = res.get(u"total", 0) - return result - - -@p.toolkit.chained_action -def package_list(next_func, context, data_dict): - # check it's received the core function as the first arg - assert next_func == core_package_list - raise ActionTestException(package_list_message) - - -class ExampleDataStoreDeletedWithCountPlugin(p.SingletonPlugin): - p.implements(p.IActions) - - def get_actions(self): - return { - u"datastore_delete": datastore_delete, - u"package_list": package_list, - } - - -@pytest.mark.usefixtures(u"with_request_context") -class TestChainedAction(object): - @pytest.mark.ckan_config( - u"ckan.plugins", - u"datastore example_datastore_deleted_with_count_plugin", - ) - @pytest.mark.usefixtures(u"with_plugins", u"clean_db") - def test_datastore_delete_filters(self): - records = [{u"age": 20}, {u"age": 30}, {u"age": 40}] - resource = self._create_datastore_resource(records) - filters = {u"age": 30} - - response = helpers.call_action( - u"datastore_delete", - resource_id=resource[u"id"], - force=True, - filters=filters, - ) - - result = helpers.call_action( - u"datastore_search", resource_id=resource[u"id"] - ) - - new_records_ages = [r[u"age"] for r in result[u"records"]] - new_records_ages.sort() - assert new_records_ages == [20, 40] - assert response["deleted_count"] == 1 - - def _create_datastore_resource(self, records): - dataset = factories.Dataset() - resource = factories.Resource(package=dataset) - - data = { - u"resource_id": resource[u"id"], - u"force": True, - u"records": records, - } - - helpers.call_action(u"datastore_create", **data) - - return resource - - @pytest.mark.ckan_config( - u"ckan.plugins", - u"datastore example_datastore_deleted_with_count_plugin", - ) - @pytest.mark.usefixtures(u"with_plugins", u"clean_db") - def test_chain_core_action(self): - with pytest.raises(ActionTestException) as raise_context: - helpers.call_action(u"package_list", {}) - assert raise_context.value.args == (package_list_message, ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py deleted file mode 100644 index 9c911a8..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_chained_auth_functions.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -import pytest - -import ckan.lib.create_test_data as ctd -import ckan.plugins as p -import ckan.tests.factories as factories -from ckan.logic import check_access, NotAuthorized -from ckan.logic.auth.get import user_list as core_user_list - - -auth_message = u"No search for you" -user_list_message = u"Nothing to see here" - - -class AuthTestException(Exception): - pass - - -@p.toolkit.chained_auth_function -def datastore_search_sql_auth(up_func, context, data_dict): - # simple checks to confirm that the up_func we've received is the original - # sql search auth function - assert up_func.auth_allow_anonymous_access - assert up_func(context, data_dict) == {u"success": True} - raise AuthTestException(auth_message) - - -@p.toolkit.chained_auth_function -def user_list(next_auth, context, data_dict): - # check it's received the core function as the first arg - assert next_auth == core_user_list - raise AuthTestException(user_list_message) - - -@p.toolkit.chained_auth_function -def user_create(next_auth, context, data_dict): - return next_auth(context, data_dict) - - -class ExampleDataStoreSearchSQLPlugin(p.SingletonPlugin): - p.implements(p.IAuthFunctions) - - def get_auth_functions(self): - return { - u"datastore_search_sql": datastore_search_sql_auth, - u"user_list": user_list, - } - - -@pytest.mark.ckan_config( - u"ckan.plugins", u"datastore example_data_store_search_sql_plugin" -) -@pytest.mark.usefixtures(u"with_request_context", u"with_plugins", u"clean_db") -class TestChainedAuth(object): - def test_datastore_search_sql_auth(self): - ctd.CreateTestData.create() - with pytest.raises(AuthTestException) as raise_context: - # checking access should call to our chained version defined above - # first, thus this should throw an exception - check_access( - u"datastore_search_sql", - {u"user": u"annafan", u"table_names": []}, - {}, - ) - # check that exception returned has the message from our auth function - assert raise_context.value.args == (auth_message, ) - - def test_chain_core_auth_functions(self): - user = factories.User() - context = {u"user": user[u"name"]} - with pytest.raises(AuthTestException) as raise_context: - check_access(u"user_list", context, {}) - assert raise_context.value.args == (user_list_message, ) - # check that the 'auth failed' msg doesn't fail because it's a partial - with pytest.raises(NotAuthorized): - check_access( - u"user_list", - {u"ignore_auth": False, u"user": u"not_a_real_user"}, - {}, - ) - - -class ExampleExternalProviderPlugin(p.SingletonPlugin): - p.implements(p.IAuthFunctions) - - def get_auth_functions(self): - return {u"user_create": user_create} - - -@pytest.mark.ckan_config( - u"ckan.plugins", u"datastore example_data_store_search_sql_plugin" -) -@pytest.mark.usefixtures(u"with_plugins", u"clean_db", u"with_request_context") -class TestChainedAuthBuiltInFallback(object): - - @pytest.mark.ckan_config("ckan.auth.create_user_via_web", True) - def test_user_create_chained_auth(self): - ctd.CreateTestData.create() - # check if chained auth fallbacks to built-in user_create - check_access(u"user_create", {u"user": u"annafan"}, {}) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py deleted file mode 100644 index 31ddf92..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_create.py +++ /dev/null @@ -1,1449 +0,0 @@ -# encoding: utf-8 - -import json -import pytest -import sqlalchemy.orm as orm - -import ckan.lib.create_test_data as ctd -import ckan.model as model -import ckan.plugins as p -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers -import ckanext.datastore.backend.postgres as db -from ckan.common import config -from ckan.plugins.toolkit import ValidationError -from ckanext.datastore.tests.helpers import ( - set_url_type, - execute_sql, - when_was_last_analyze, -) - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreCreateNewTests(object): - def _has_index_on_field(self, resource_id, field): - sql = u""" - SELECT - relname - FROM - pg_class - WHERE - pg_class.relname = %s - """ - index_name = db._generate_index_name(resource_id, field) - results = execute_sql(sql, index_name).fetchone() - return bool(results) - - def _get_index_names(self, resource_id): - sql = u""" - SELECT - i.relname AS index_name - FROM - pg_class t, - pg_class i, - pg_index idx - WHERE - t.oid = idx.indrelid - AND i.oid = idx.indexrelid - AND t.relkind = 'r' - AND t.relname = %s - """ - results = execute_sql(sql, resource_id).fetchall() - return [result[0] for result in results] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_works_with_empty_array_in_json_field(self): - package = factories.Dataset() - data = { - "resource": {"package_id": package["id"]}, - "fields": [ - {"id": "movie", "type": "text"}, - {"id": "directors", "type": "json"}, - ], - "records": [{"movie": "sideways", "directors": []}], - } - result = helpers.call_action("datastore_create", **data) - assert result["resource_id"] is not None - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_works_with_empty_object_in_json_field(self): - package = factories.Dataset() - data = { - "resource": {"package_id": package["id"]}, - "fields": [ - {"id": "movie", "type": "text"}, - {"id": "director", "type": "json"}, - ], - "records": [{"movie": "sideways", "director": {}}], - } - result = helpers.call_action("datastore_create", **data) - assert result["resource_id"] is not None - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_creates_index_on_primary_key(self): - package = factories.Dataset() - data = { - "resource": { - "boo%k": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - } - } - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - index_names = self._get_index_names(resource_id) - assert resource_id + "_pkey" in index_names - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_creates_url_with_site_name(self): - package = factories.Dataset() - data = {"resource": {"boo%k": "crime", "package_id": package["id"]}} - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - resource = helpers.call_action("resource_show", id=resource_id) - url = resource["url"] - assert url.startswith(config.get("ckan.site_url")) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_index_on_specific_fields(self): - package = factories.Dataset() - data = { - "resource": { - "boo%k": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - }, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "indexes": ["author"], - } - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - assert self._has_index_on_field(resource_id, '"author"') - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_adds_index_on_full_text_search_when_creating_other_indexes( - self - ): - package = factories.Dataset() - data = { - "resource": { - "boo%k": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - }, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "indexes": ["author"], - } - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - assert self._has_index_on_field(resource_id, '"_full_text"') - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_adds_index_on_full_text_search_when_not_creating_other_indexes( - self - ): - package = factories.Dataset() - data = { - "resource": { - "boo%k": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - }, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - assert self._has_index_on_field(resource_id, '"_full_text"') - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_add_full_text_search_indexes_on_every_text_field(self): - package = factories.Dataset() - data = { - "resource": { - "book": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - }, - "fields": [ - {"id": "boo%k", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "lang": "english", - } - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - assert self._has_index_on_field( - resource_id, "to_tsvector('english', \"boo%k\")" - ) - assert self._has_index_on_field( - resource_id, "to_tsvector('english', \"author\")" - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_doesnt_add_more_indexes_when_updating_data(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"book": "annakarenina", "author": "tolstoy"}], - } - result = helpers.call_action("datastore_create", **data) - previous_index_names = self._get_index_names(resource["id"]) - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"book": "warandpeace", "author": "tolstoy"}], - } - result = helpers.call_action("datastore_create", **data) - current_index_names = self._get_index_names(resource["id"]) - assert previous_index_names == current_index_names - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_duplicate_fields(self): - package = factories.Dataset() - data = { - "resource": { - "book": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - }, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "book", "type": "text"}, - ], - } - with pytest.raises(p.toolkit.ValidationError): - helpers.call_action("datastore_create", **data) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_sets_datastore_active_on_resource_on_create(self): - resource = factories.Resource() - - assert not (resource["datastore_active"]) - - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"book": "annakarenina", "author": "tolstoy"}], - } - - helpers.call_action("datastore_create", **data) - - resource = helpers.call_action("resource_show", id=resource["id"]) - - assert resource["datastore_active"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_sets_datastore_active_on_resource_on_delete(self): - resource = factories.Resource(datastore_active=True) - - assert resource["datastore_active"] - - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"book": "annakarenina", "author": "tolstoy"}], - } - - helpers.call_action("datastore_create", **data) - - helpers.call_action( - "datastore_delete", resource_id=resource["id"], force=True - ) - - resource = helpers.call_action("resource_show", id=resource["id"]) - - assert not (resource["datastore_active"]) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_exceeds_column_name_limit(self): - package = factories.Dataset() - data = { - "resource": {"package_id": package["id"]}, - "fields": [ - { - "id": "This is a really long name for a column. Column names " - "in Postgres have a limit of 63 characters", - "type": "text", - } - ], - } - with pytest.raises(p.toolkit.ValidationError): - helpers.call_action("datastore_create", **data) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_calculate_record_count_is_false(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "fields": [ - {"id": "name", "type": "text"}, - {"id": "age", "type": "text"}, - ], - "records": [ - {"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}, - ], - "force": True, - } - helpers.call_action("datastore_create", **data) - last_analyze = when_was_last_analyze(resource["id"]) - assert last_analyze is None - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.flaky(reruns=2) # because analyze is sometimes delayed - def test_calculate_record_count(self): - # how datapusher loads data (send_resource_to_datastore) - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "fields": [ - {"id": "name", "type": "text"}, - {"id": "age", "type": "text"}, - ], - "records": [ - {"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}, - ], - "calculate_record_count": True, - "force": True, - } - helpers.call_action("datastore_create", **data) - last_analyze = when_was_last_analyze(resource["id"]) - assert last_analyze is not None - - - -class TestDatastoreCreate(object): - sysadmin_user = None - normal_user = None - - @pytest.fixture(autouse=True) - def create_test_data(self, clean_datastore, test_request_context): - ctd.CreateTestData.create() - self.sysadmin_user = factories.Sysadmin() - self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"]) - self.sysadmin_token = self.sysadmin_token["token"] - self.normal_user = factories.User() - self.normal_user_token = factories.APIToken(user=self.normal_user["id"]) - self.normal_user_token = self.normal_user_token["token"] - engine = db.get_write_engine() - self.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) - with test_request_context(): - set_url_type( - model.Package.get("annakarenina").resources, self.sysadmin_user - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_requires_auth(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = {"resource_id": resource.id} - - res = app.post( - "/api/action/datastore_create", data=data - ) - assert res.status_code == 403 - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_empty_fails(self, app): - data = {} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - data=data, - extra_environ=auth, - ) - assert res.status_code == 409 - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_invalid_alias_name(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "aliases": u'foo"bar', - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - data = { - "resource_id": resource.id, - "aliases": u"fo%25bar", # alias with percent - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_duplicate_alias_name(self, app): - resource = factories.Resource(url_type = 'datastore') - data = {"resource_id": resource['id'], "aliases": u"myalias"} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - data=data, - extra_environ=auth, - ) - assert res.status_code == 200 - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - # try to create another table with the same alias - resource_2 = factories.Resource(url_type = 'datastore') - data = {"resource_id": resource_2['id'], "aliases": u"myalias"} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - data=data, - extra_environ=auth, - ) - assert res.status_code == 409 - - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - # try to create an alias that is a resource id - data = { - "resource_id": resource_2['id'], - "aliases": resource_2['id'], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - data=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_invalid_field_type(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "fields": [ - {"id": "book", "type": "int["}, # this is invalid - {"id": "author", "type": "INVALID"}, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_invalid_field_name(self, app): - resource = model.Package.get("annakarenina").resources[0] - auth = {"Authorization": self.sysadmin_token} - invalid_names = [ - "_author", - '"author', - "", - " author", - "author ", - "\tauthor", - "author\n", - ] - - for field_name in invalid_names: - data = { - "resource_id": resource.id, - "fields": [ - {"id": "book", "type": "text"}, - {"id": field_name, "type": "text"}, - ], - } - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_invalid_record_field(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [ - {"book": "annakarenina", "author": "tolstoy"}, - {"book": "warandpeace", "published": "1869"}, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_bad_records(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": ["bad"], # treat author as null - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is False - assert res_dict["error"]["__type"] == "Validation Error" - - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [ - {"book": "annakarenina", "author": "tolstoy"}, - [], - {"book": "warandpeace"}, - ], # treat author as null - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is False - assert res_dict["error"]["__type"] == "Validation Error" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_invalid_index(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "indexes": "book, dummy", - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_invalid_unique_index(self, app): - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "primary_key": "dummy", - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_alias_twice(self, app): - resource = model.Package.get("annakarenina").resources[1] - data = { - "resource_id": resource.id, - "aliases": "new_alias", - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True, res_dict - - resource = model.Package.get("annakarenina").resources[0] - data = { - "resource_id": resource.id, - "aliases": "new_alias", - "fields": [{"id": "more books", "type": "text"}], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False, res_dict - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_basic(self, app): - resource = model.Package.get("annakarenina").resources[0] - aliases = [u"great_list_of_books", u"another_list_of_b\xfcks"] - ### Firstly test to see whether resource has no datastore table yet - data = {"id": resource.id} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/resource_show", data=data, extra_environ=auth - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["datastore_active"] is False - data = { - "resource_id": resource.id, - "aliases": aliases, - "fields": [ - {"id": "boo%k", "type": "text"}, # column with percent - {"id": "author", "type": "json"}, - ], - "indexes": [["boo%k", "author"], "author"], - "records": [ - {"boo%k": "crime", "author": ["tolstoy", "dostoevsky"]}, - {"boo%k": "annakarenina", "author": ["tolstoy", "putin"]}, - {"boo%k": "warandpeace"}, - ], # treat author as null - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True - res = res_dict["result"] - assert res["resource_id"] == data["resource_id"] - assert res["fields"] == data["fields"], res["fields"] - - c = self.Session.connection() - results = c.execute('select * from "{0}"'.format(resource.id)) - - assert results.rowcount == 3 - for i, row in enumerate(results): - assert data["records"][i].get("boo%k") == row["boo%k"] - assert data["records"][i].get("author") == ( - json.loads(row["author"][0]) if row["author"] else None - ) - - results = c.execute( - """ - select * from "{0}" where _full_text @@ to_tsquery('warandpeace') - """.format( - resource.id - ) - ) - assert results.rowcount == 1, results.rowcount - - results = c.execute( - """ - select * from "{0}" where _full_text @@ to_tsquery('tolstoy') - """.format( - resource.id - ) - ) - assert results.rowcount == 2 - self.Session.remove() - - # check aliases for resource - c = self.Session.connection() - for alias in aliases: - - results = [ - row - for row in c.execute( - u'select * from "{0}"'.format(resource.id) - ) - ] - results_alias = [ - row for row in c.execute(u'select * from "{0}"'.format(alias)) - ] - - assert results == results_alias - - sql = ( - u"select * from _table_metadata " - "where alias_of=%s and name=%s" - ) - results = c.execute(sql, resource.id, alias) - assert results.rowcount == 1 - self.Session.remove() - - # check to test to see if resource now has a datastore table - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/resource_show", data={"id": resource.id}, extra_environ=auth - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["datastore_active"] - - ####### insert again simple - data2 = { - "resource_id": resource.id, - "records": [{"boo%k": "hagji murat", "author": ["tolstoy"]}], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data2, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True - - c = self.Session.connection() - results = c.execute('select * from "{0}"'.format(resource.id)) - self.Session.remove() - - assert results.rowcount == 4 - - all_data = data["records"] + data2["records"] - for i, row in enumerate(results): - assert all_data[i].get("boo%k") == row["boo%k"] - assert all_data[i].get("author") == ( - json.loads(row["author"][0]) if row["author"] else None - ) - - c = self.Session.connection() - results = c.execute( - """ - select * from "{0}" where _full_text @@ 'tolstoy' - """.format( - resource.id - ) - ) - self.Session.remove() - assert results.rowcount == 3 - - ####### insert again extra field - data3 = { - "resource_id": resource.id, - "records": [ - { - "boo%k": "crime and punsihment", - "author": ["dostoevsky"], - "rating": 2, - } - ], - "indexes": ["rating"], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data3, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True - - c = self.Session.connection() - results = c.execute('select * from "{0}"'.format(resource.id)) - - assert results.rowcount == 5 - - all_data = data["records"] + data2["records"] + data3["records"] - for i, row in enumerate(results): - assert all_data[i].get("boo%k") == row["boo%k"], ( - i, - all_data[i].get("boo%k"), - row["boo%k"], - ) - assert all_data[i].get("author") == ( - json.loads(row["author"][0]) if row["author"] else None - ) - - results = c.execute( - """select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') """.format( - resource.id - ) - ) - self.Session.remove() - assert results.rowcount == 2 - - ####### insert again which will fail because of unique book name - data4 = { - "resource_id": resource.id, - "records": [{"boo%k": "warandpeace"}], - "primary_key": "boo%k", - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data4, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is False - assert "constraints" in res_dict["error"], res_dict - - ####### insert again which should not fail because constraint is removed - data5 = { - "resource_id": resource.id, - "aliases": "another_alias", # replaces aliases - "records": [{"boo%k": "warandpeace"}], - "primary_key": "", - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data5, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True, res_dict - - # new aliases should replace old aliases - c = self.Session.connection() - for alias in aliases: - sql = ( - "select * from _table_metadata " - "where alias_of=%s and name=%s" - ) - results = c.execute(sql, resource.id, alias) - assert results.rowcount == 0 - - sql = "select * from _table_metadata " "where alias_of=%s and name=%s" - results = c.execute(sql, resource.id, "another_alias") - assert results.rowcount == 1 - self.Session.remove() - - ####### insert array type - data6 = { - "resource_id": resource.id, - "fields": [ - {"id": "boo%k", "type": "text"}, - {"id": "author", "type": "json"}, - {"id": "rating", "type": "int"}, - {"id": "characters", "type": "_text"}, - ], # this is an array of strings - "records": [ - { - "boo%k": "the hobbit", - "author": ["tolkien"], - "characters": ["Bilbo", "Gandalf"], - } - ], - "indexes": ["characters"], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data6, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True, res_dict - - ####### insert type that requires additional lookup - data7 = { - "resource_id": resource.id, - "fields": [ - {"id": "boo%k", "type": "text"}, - {"id": "author", "type": "json"}, - {"id": "rating", "type": "int"}, - {"id": "characters", "type": "_text"}, - {"id": "location", "type": "int[2]"}, - ], - "records": [ - { - "boo%k": "lord of the rings", - "author": ["tolkien"], - "location": [3, -42], - } - ], - "indexes": ["characters"], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data7, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True, res_dict - - ####### insert with parameter id rather than resource_id which is a shortcut - data8 = { - "id": resource.id, - # insert with percent - "records": [{"boo%k": "warandpeace", "author": "99% good"}], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data8, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True, res_dict - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_create_datastore_resource_on_dataset(self, app): - pkg = model.Package.get("annakarenina") - - data = { - "resource": {"package_id": pkg.id}, - "fields": [ - {"id": "boo%k", "type": "text"}, # column with percent - {"id": "author", "type": "json"}, - ], - "indexes": [["boo%k", "author"], "author"], - "records": [ - {"boo%k": "crime", "author": ["tolstoy", "dostoevsky"]}, - {"boo%k": "annakarenina", "author": ["tolstoy", "putin"]}, - {"boo%k": "warandpeace"}, - ], # treat author as null - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is True - res = res_dict["result"] - assert res["fields"] == data["fields"], res["fields"] - - # Get resource details - data = {"id": res["resource_id"]} - res = app.post("/api/action/resource_show", data=data) - res_dict = json.loads(res.data) - - assert res_dict["result"]["datastore_active"] is True - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_guess_types(self, app): - resource = model.Package.get("annakarenina").resources[1] - - data = {"resource_id": resource.id} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_delete", - data=data, - extra_environ=auth, - ) # ignore status - res_dict = json.loads(res.data) - - data = { - "resource_id": resource.id, - "fields": [ - {"id": "author", "type": "json"}, - {"id": "count"}, - {"id": "book"}, - {"id": "date"}, - ], - "records": [ - { - "book": "annakarenina", - "author": "tolstoy", - "count": 1, - "date": "2005-12-01", - "count2": 0.5, - }, - {"book": "crime", "author": ["tolstoy", "dostoevsky"]}, - {"book": "warandpeace"}, - ], # treat author as null - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - c = self.Session.connection() - results = c.execute("""select * from "{0}" """.format(resource.id)) - - types = [ - db._pg_types[field[1]] for field in results.cursor.description - ] - - assert types == [ - u"int4", - u"tsvector", - u"nested", - u"int4", - u"text", - u"timestamp", - u"float8", - ], types - - assert results.rowcount == 3 - for i, row in enumerate(results): - assert data["records"][i].get("book") == row["book"] - assert data["records"][i].get("author") == ( - json.loads(row["author"][0]) if row["author"] else None - ) - self.Session.remove() - - ### extend types - - data = { - "resource_id": resource.id, - "fields": [ - {"id": "author", "type": "text"}, - {"id": "count"}, - {"id": "book"}, - {"id": "date"}, - {"id": "count2"}, - {"id": "extra", "type": "text"}, - {"id": "date2"}, - ], - "records": [ - { - "book": "annakarenina", - "author": "tolstoy", - "count": 1, - "date": "2005-12-01", - "count2": 2, - "nested": [1, 2], - "date2": "2005-12-01", - } - ], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - ) - res_dict = json.loads(res.data) - - c = self.Session.connection() - results = c.execute("""select * from "{0}" """.format(resource.id)) - self.Session.remove() - - types = [ - db._pg_types[field[1]] for field in results.cursor.description - ] - - assert types == [ - u"int4", # id - u"tsvector", # fulltext - u"nested", # author - u"int4", # count - u"text", # book - u"timestamp", # date - u"float8", # count2 - u"text", # extra - u"timestamp", # date2 - u"nested", # count3 - ], types - - ### fields resupplied in wrong order - - data = { - "resource_id": resource.id, - "fields": [ - {"id": "author", "type": "text"}, - {"id": "count"}, - {"id": "date"}, # date and book in wrong order - {"id": "book"}, - {"id": "count2"}, - {"id": "extra", "type": "text"}, - {"id": "date2"}, - ], - "records": [ - { - "book": "annakarenina", - "author": "tolstoy", - "count": 1, - "date": "2005-12-01", - "count2": 2, - "count3": 432, - "date2": "2005-12-01", - } - ], - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins") - def test_datastore_create_with_invalid_data_value(self, app): - """datastore_create() should return an error for invalid data.""" - resource = factories.Resource(url_type="datastore") - data_dict = { - "resource_id": resource["id"], - "fields": [{"id": "value", "type": "numeric"}], - "records": [ - {"value": 0}, - {"value": 1}, - {"value": 2}, - {"value": 3}, - {"value": " "}, # Invalid numeric value. - {"value": 5}, - {"value": 6}, - {"value": 7}, - ], - "method": "insert", - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", - json=data_dict, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - - assert res_dict["success"] is False - assert res_dict["error"]["__type"] == "Validation Error" - assert res_dict["error"]["message"].startswith("The data was invalid") - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreFunctionCreate(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_nop_trigger(self): - helpers.call_action( - u"datastore_function_create", - name=u"test_nop", - rettype=u"trigger", - definition=u"BEGIN RETURN NEW; END;", - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_invalid_definition(self): - with pytest.raises(ValidationError) as error: - helpers.call_action( - u"datastore_function_create", - name=u"test_invalid_def", - rettype=u"trigger", - definition=u"HELLO WORLD", - ) - assert error.value.error_dict == { - u"definition": [u'syntax error at or near "HELLO"'] - } - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_redefined_trigger(self): - helpers.call_action( - u"datastore_function_create", - name=u"test_redefined", - rettype=u"trigger", - definition=u"BEGIN RETURN NEW; END;", - ) - with pytest.raises(ValidationError) as error: - helpers.call_action( - u"datastore_function_create", - name=u"test_redefined", - rettype=u"trigger", - definition=u"BEGIN RETURN NEW; END;", - ) - assert error.value.error_dict == { - u"name": [ - u'function "test_redefined" already exists ' - u"with same argument types" - ] - } - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_redefined_with_or_replace_trigger(self): - helpers.call_action( - u"datastore_function_create", - name=u"test_replaceme", - rettype=u"trigger", - definition=u"BEGIN RETURN NEW; END;", - ) - helpers.call_action( - u"datastore_function_create", - name=u"test_replaceme", - or_replace=True, - rettype=u"trigger", - definition=u"BEGIN RETURN NEW; END;", - ) - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreCreateTriggers(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_with_missing_trigger(self, app): - ds = factories.Dataset() - - with pytest.raises(ValidationError) as error: - with app.flask_app.test_request_context(): - helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}], - triggers=[{u"function": u"no_such_trigger_function"}], - ) - assert error.value.error_dict == { - u"triggers": [ - u"function no_such_trigger_function() does not exist" - ] - } - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_trigger_applies_to_records(self, app): - ds = factories.Dataset() - - helpers.call_action( - u"datastore_function_create", - name=u"spamify_trigger", - rettype=u"trigger", - definition=u""" - BEGIN - NEW.spam := 'spam spam ' || NEW.spam || ' spam'; - RETURN NEW; - END;""", - ) - - with app.flask_app.test_request_context(): - res = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}], - triggers=[{u"function": u"spamify_trigger"}], - ) - assert helpers.call_action( - u"datastore_search", - fields=[u"spam"], - resource_id=res["resource_id"], - )["records"] == [ - {u"spam": u"spam spam SPAM spam"}, - {u"spam": u"spam spam EGGS spam"}, - ] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_trigger_applies_to_records(self, app): - ds = factories.Dataset() - - helpers.call_action( - u"datastore_function_create", - name=u"more_spam_trigger", - rettype=u"trigger", - definition=u""" - BEGIN - NEW.spam := 'spam spam ' || NEW.spam || ' spam'; - RETURN NEW; - END;""", - ) - - with app.flask_app.test_request_context(): - res = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - triggers=[{u"function": u"more_spam_trigger"}], - ) - helpers.call_action( - u"datastore_upsert", - method=u"insert", - resource_id=res["resource_id"], - records=[{u"spam": u"BEANS"}, {u"spam": u"SPAM"}], - ) - assert helpers.call_action( - u"datastore_search", - fields=[u"spam"], - resource_id=res["resource_id"], - )["records"] == [ - {u"spam": u"spam spam BEANS spam"}, - {u"spam": u"spam spam SPAM spam"}, - ] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_trigger_exception(self, app): - ds = factories.Dataset() - - helpers.call_action( - u"datastore_function_create", - name=u"spamexception_trigger", - rettype=u"trigger", - definition=u""" - BEGIN - IF NEW.spam != 'spam' THEN - RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; - END IF; - RETURN NEW; - END;""", - ) - with pytest.raises(ValidationError) as error: - with app.flask_app.test_request_context(): - helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - records=[{u"spam": u"spam"}, {u"spam": u"EGGS"}], - triggers=[{u"function": u"spamexception_trigger"}], - ) - assert error.value.error_dict == {u"records": [u'"EGGS"? Yeeeeccch!']} - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_trigger_exception(self, app): - ds = factories.Dataset() - - helpers.call_action( - u"datastore_function_create", - name=u"spamonly_trigger", - rettype=u"trigger", - definition=u""" - BEGIN - IF NEW.spam != 'spam' THEN - RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; - END IF; - RETURN NEW; - END;""", - ) - with app.flask_app.test_request_context(): - res = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - triggers=[{u"function": u"spamonly_trigger"}], - ) - with pytest.raises(ValidationError) as error: - helpers.call_action( - u"datastore_upsert", - method=u"insert", - resource_id=res["resource_id"], - records=[{u"spam": u"spam"}, {u"spam": u"BEANS"}], - ) - assert error.value.error_dict == { - u"records": [u'"BEANS"? Yeeeeccch!'] - } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py deleted file mode 100644 index 01cc702..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_db.py +++ /dev/null @@ -1,225 +0,0 @@ -# encoding: utf-8 - -import unittest.mock as mock -import pytest -import sqlalchemy.exc - -import ckan.lib.jobs as jobs -import ckan.plugins as p -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers -import ckanext.datastore.backend as backend -import ckanext.datastore.backend.postgres as db - - -@pytest.mark.ckan_config("ckan.plugins", "datastore") -@pytest.mark.usefixtures("with_plugins") -class TestCreateIndexes(object): - def test_creates_fts_index_using_gist_by_default(self): - connection = mock.MagicMock() - context = {"connection": connection} - resource_id = "resource_id" - data_dict = {"resource_id": resource_id} - - db.create_indexes(context, data_dict) - - self._assert_created_index_on( - "_full_text", connection, resource_id, method="gist" - ) - - @pytest.mark.ckan_config("ckan.datastore.default_fts_index_method", "gin") - def test_default_fts_index_method_can_be_overwritten_by_config_var(self): - connection = mock.MagicMock() - context = {"connection": connection} - resource_id = "resource_id" - data_dict = {"resource_id": resource_id} - - db.create_indexes(context, data_dict) - - self._assert_created_index_on( - "_full_text", connection, resource_id, method="gin" - ) - - @mock.patch("ckanext.datastore.backend.postgres._get_fields") - def test_creates_fts_index_on_all_fields_except_dates_nested_and_arrays_with_english_as_default( - self, _get_fields - ): - _get_fields.return_value = [ - {"id": "text", "type": "text"}, - {"id": "number", "type": "number"}, - {"id": "nested", "type": "nested"}, - {"id": "date", "type": "date"}, - {"id": "text array", "type": "text[]"}, - {"id": "timestamp", "type": "timestamp"}, - ] - connection = mock.MagicMock() - context = {"connection": connection} - resource_id = "resource_id" - data_dict = {"resource_id": resource_id} - - db.create_indexes(context, data_dict) - - self._assert_created_index_on( - "text", connection, resource_id, "english" - ) - self._assert_created_index_on( - "number", connection, resource_id, "english", cast=True - ) - - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple") - @mock.patch("ckanext.datastore.backend.postgres._get_fields") - def test_creates_fts_index_on_textual_fields_can_overwrite_lang_with_config_var( - self, _get_fields - ): - _get_fields.return_value = [{"id": "foo", "type": "text"}] - connection = mock.MagicMock() - context = {"connection": connection} - resource_id = "resource_id" - data_dict = {"resource_id": resource_id} - - db.create_indexes(context, data_dict) - - self._assert_created_index_on("foo", connection, resource_id, "simple") - - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple") - @mock.patch("ckanext.datastore.backend.postgres._get_fields") - def test_creates_fts_index_on_textual_fields_can_overwrite_lang_using_lang_param( - self, _get_fields - ): - _get_fields.return_value = [{"id": "foo", "type": "text"}] - connection = mock.MagicMock() - context = {"connection": connection} - resource_id = "resource_id" - data_dict = {"resource_id": resource_id, "language": "french"} - - db.create_indexes(context, data_dict) - - self._assert_created_index_on("foo", connection, resource_id, "french") - - def _assert_created_index_on( - self, - field, - connection, - resource_id, - lang=None, - cast=False, - method="gist", - ): - field = u'"{0}"'.format(field) - if cast: - field = u"cast({0} AS text)".format(field) - if lang is not None: - sql_str = ( - u'ON "resource_id" ' - u"USING {method}(to_tsvector('{lang}', {field}))" - ) - sql_str = sql_str.format(method=method, lang=lang, field=field) - else: - sql_str = u"USING {method}({field})".format( - method=method, field=field - ) - - calls = connection.execute.call_args_list - was_called = [call for call in calls if call[0][0].find(sql_str) != -1] - - assert was_called, ( - "Expected 'connection.execute' to have been " - "called with a string containing '%s'" % sql_str - ) - - -@mock.patch("ckanext.datastore.backend.postgres._get_fields") -def test_upsert_with_insert_method_and_invalid_data(mock_get_fields_function): - """upsert_data() should raise InvalidDataError if given invalid data. - - If the type of a field is numeric and upsert_data() is given a whitespace - value like " ", it should raise DataError. - - In this case we're testing with "method": "insert" in the data_dict. - - """ - mock_connection = mock.Mock() - mock_connection.execute.side_effect = sqlalchemy.exc.DataError( - "statement", "params", "orig", connection_invalidated=False - ) - - context = {"connection": mock_connection} - data_dict = { - "fields": [{"id": "value", "type": "numeric"}], - "records": [ - {"value": 0}, - {"value": 1}, - {"value": 2}, - {"value": 3}, - {"value": " "}, # Invalid numeric value. - {"value": 5}, - {"value": 6}, - {"value": 7}, - ], - "method": "insert", - "resource_id": "fake-resource-id", - } - - mock_get_fields_function.return_value = data_dict["fields"] - - with pytest.raises(backend.InvalidDataError): - db.upsert_data(context, data_dict) - - -class TestGetAllResourcesIdsInDatastore(object): - @pytest.mark.ckan_config(u"ckan.plugins", u"datastore") - @pytest.mark.usefixtures(u"with_plugins", u"clean_db") - def test_get_all_resources_ids_in_datastore(self): - resource_in_datastore = factories.Resource() - resource_not_in_datastore = factories.Resource() - data = {"resource_id": resource_in_datastore["id"], "force": True} - helpers.call_action("datastore_create", **data) - - resource_ids = backend.get_all_resources_ids_in_datastore() - - assert resource_in_datastore["id"] in resource_ids - assert resource_not_in_datastore["id"] not in resource_ids - - -def datastore_job(res_id, value): - """ - A background job that uses the Datastore. - """ - app = helpers._get_test_app() - if not p.plugin_loaded(u"datastore"): - p.load("datastore") - data = { - "resource_id": res_id, - "method": "insert", - "records": [{"value": value}], - } - - with app.flask_app.test_request_context(): - helpers.call_action("datastore_upsert", **data) - - -class TestBackgroundJobs(helpers.RQTestBase): - """ - Test correct interaction with the background jobs system. - """ - @pytest.mark.ckan_config(u"ckan.plugins", u"datastore") - @pytest.mark.usefixtures(u"with_plugins", u"clean_db", u"with_request_context") - def test_worker_datastore_access(self, app): - """ - Test DataStore access from within a worker. - """ - pkg = factories.Dataset() - data = { - "resource": {"package_id": pkg["id"]}, - "fields": [{"id": "value", "type": "int"}], - } - - table = helpers.call_action("datastore_create", **data) - res_id = table["resource_id"] - for i in range(3): - self.enqueue(datastore_job, args=[res_id, i]) - jobs.Worker().work(burst=True) - # Aside from ensuring that the job succeeded, this also checks - # that accessing the Datastore still works in the main process. - result = helpers.call_action("datastore_search", resource_id=res_id) - assert [0, 1, 2] == [r["value"] for r in result["records"]] diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py deleted file mode 100644 index f12bc67..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_delete.py +++ /dev/null @@ -1,447 +0,0 @@ -# encoding: utf-8 - -import json -import pytest - -import sqlalchemy.orm as orm - -import ckan.lib.create_test_data as ctd -import ckan.model as model -from ckan.tests import helpers -from ckan.plugins.toolkit import ValidationError -import ckan.tests.factories as factories -from ckan.logic import NotFound -import ckanext.datastore.backend.postgres as db -from ckanext.datastore.tests.helpers import ( - set_url_type, - when_was_last_analyze, - execute_sql, -) - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreDelete(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_basic(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "aliases": u"b\xfck2", - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "rating with %", "type": "text"}, - ], - "records": [ - { - "book": "annakarenina", - "author": "tolstoy", - "rating with %": "90%", - }, - { - "book": "warandpeace", - "author": "tolstoy", - "rating with %": "42%", - }, - ], - } - helpers.call_action("datastore_create", **data) - data = {"resource_id": resource["id"], "force": True} - helpers.call_action("datastore_delete", **data) - - # regression test for #7832 - resobj = model.Resource.get(data["resource_id"]) - assert resobj.extras.get('datastore_active') is False - - results = execute_sql( - u"select 1 from pg_views where viewname = %s", u"b\xfck2" - ) - assert results.rowcount == 0 - - # check the table is gone - results = execute_sql( - u"""SELECT table_name - FROM information_schema.tables - WHERE table_name=%s;""", - resource["id"], - ) - assert results.rowcount == 0 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_calculate_record_count_is_false(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": "name", "type": "text"}, - {"id": "age", "type": "text"}, - ], - "records": [ - {"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}, - ], - } - helpers.call_action("datastore_create", **data) - data = { - "resource_id": resource["id"], - "filters": {"name": "Bowan"}, - "force": True, - } - helpers.call_action("datastore_delete", **data) - last_analyze = when_was_last_analyze(resource["id"]) - assert last_analyze is None - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.flaky(reruns=2) # because analyze is sometimes delayed - def test_calculate_record_count(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": "name", "type": "text"}, - {"id": "age", "type": "text"}, - ], - "records": [ - {"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}, - ], - } - helpers.call_action("datastore_create", **data) - data = { - "resource_id": resource["id"], - "filters": {"name": "Bowan"}, - "calculate_record_count": True, - "force": True, - } - helpers.call_action("datastore_delete", **data) - last_analyze = when_was_last_analyze(resource["id"]) - assert last_analyze is not None - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreDeleteLegacy(object): - sysadmin_user = None - normal_user = None - Session = None - - @pytest.fixture(autouse=True) - def initial_data(self, clean_datastore, app, test_request_context): - self.app = app - ctd.CreateTestData.create() - self.sysadmin_user = factories.Sysadmin() - self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"]) - self.sysadmin_token = self.sysadmin_token["token"] - self.normal_user = factories.User() - self.normal_user_token = factories.APIToken(user=self.normal_user["id"]) - self.normal_user_token = self.normal_user_token["token"] - resource = model.Package.get("annakarenina").resources[0] - self.data = { - "resource_id": resource.id, - "aliases": u"b\xfck2", - "fields": [ - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "rating with %", "type": "text"}, - ], - "records": [ - { - "book": "annakarenina", - "author": "tolstoy", - "rating with %": "90%", - }, - { - "book": "warandpeace", - "author": "tolstoy", - "rating with %": "42%", - }, - ], - } - - engine = db.get_write_engine() - - self.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) - with test_request_context(): - set_url_type( - model.Package.get("annakarenina").resources, self.sysadmin_user - ) - - def _create(self): - auth = {"Authorization": self.sysadmin_token} - res = self.app.post( - "/api/action/datastore_create", - json=self.data, - environ_overrides=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - return res_dict - - def _delete(self): - data = {"resource_id": self.data["resource_id"]} - auth = {"Authorization": self.sysadmin_token} - res = self.app.post( - "/api/action/datastore_delete", - data=data, - environ_overrides=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - assert res_dict["result"] == data - return res_dict - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("with_plugins", "with_request_context") - def test_datastore_deleted_during_resource_deletion(self): - package = factories.Dataset() - data = { - "resource": { - "boo%k": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - } - } - - result = helpers.call_action("datastore_create", **data) - resource_id = result["resource_id"] - helpers.call_action("resource_delete", id=resource_id) - - with pytest.raises(NotFound): - helpers.call_action("datastore_search", resource_id=resource_id) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_datastore_deleted_during_resource_only_for_deleted_resource(self): - package = factories.Dataset() - data = { - "boo%k": "crime", - "author": ["tolstoy", "dostoevsky"], - "package_id": package["id"], - } - - result_1 = helpers.call_action( - "datastore_create", resource=data.copy() - ) - resource_id_1 = result_1["resource_id"] - - result_2 = helpers.call_action( - "datastore_create", resource=data.copy() - ) - resource_id_2 = result_2["resource_id"] - - res_1 = model.Resource.get(resource_id_1) - res_2 = model.Resource.get(resource_id_2) - - # `synchronize_session=False` and session cache requires - # refreshing objects - model.Session.refresh(res_1) - model.Session.refresh(res_2) - assert res_1.extras["datastore_active"] - assert res_2.extras["datastore_active"] - - helpers.call_action("resource_delete", id=resource_id_1) - - with pytest.raises(NotFound): - helpers.call_action("datastore_search", resource_id=resource_id_1) - with pytest.raises(NotFound): - helpers.call_action("resource_show", id=resource_id_1) - model.Session.refresh(res_1) - model.Session.refresh(res_2) - assert not res_1.extras["datastore_active"] - assert res_2.extras["datastore_active"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_invalid_resource_id(self, app): - data = {"resource_id": "bad"} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_delete", - data=data, - environ_overrides=auth, - ) - assert res.status_code == 404 - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_filters(self, app): - self._create() - resource_id = self.data["resource_id"] - - # try and delete just the 'warandpeace' row - data = {"resource_id": resource_id, "filters": {"book": "warandpeace"}} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_delete", - json=data, - environ_overrides=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - c = self.Session.connection() - result = c.execute(u'select * from "{0}";'.format(resource_id)) - results = [r for r in result] - assert len(results) == 1 - assert results[0].book == "annakarenina" - self.Session.remove() - - # shouldn't delete anything - data = { - "resource_id": resource_id, - "filters": {"book": "annakarenina", "author": "bad"}, - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_delete", - json=data, - environ_overrides=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - c = self.Session.connection() - result = c.execute(u'select * from "{0}";'.format(resource_id)) - results = [r for r in result] - assert len(results) == 1 - assert results[0].book == "annakarenina" - self.Session.remove() - - # delete the 'annakarenina' row and also only use id - data = { - "id": resource_id, - "filters": {"book": "annakarenina", "author": "tolstoy"}, - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_delete", - json=data, - environ_overrides=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - c = self.Session.connection() - result = c.execute(u'select * from "{0}";'.format(resource_id)) - results = [r for r in result] - assert len(results) == 0 - self.Session.remove() - - self._delete() - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_is_unsuccessful_when_called_with_invalid_filters( - self, app - ): - self._create() - - data = { - "resource_id": self.data["resource_id"], - "filters": {"invalid-column-name": "value"}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_delete", - json=data, - environ_overrides=auth, - ) - assert res.status_code == 409 - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"].get("filters") is not None, res_dict["error"] - - self._delete() - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_is_unsuccessful_when_called_with_filters_not_as_dict( - self, app - ): - self._create() - - data = {"resource_id": self.data["resource_id"], "filters": []} - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_delete", - json=data, - environ_overrides=auth, - ) - assert res.status_code == 409 - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"].get("filters") is not None, res_dict["error"] - - self._delete() - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_with_blank_filters(self, app): - self._create() - - res = app.post( - "/api/action/datastore_delete", - json={"resource_id": self.data["resource_id"], "filters": {}}, - environ_overrides={"Authorization": self.normal_user_token}, - ) - assert res.status_code == 200 - results = json.loads(res.data) - assert results["success"] is True - - res = app.post( - "/api/action/datastore_search", - json={"resource_id": self.data["resource_id"]}, - environ_overrides={"Authorization": self.normal_user_token}, - ) - assert res.status_code == 200 - results = json.loads(res.data) - assert results["success"] is True - assert len(results["result"]["records"]) == 0 - - self._delete() - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreFunctionDelete(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_create_delete(self): - helpers.call_action( - u"datastore_function_create", - name=u"test_nop", - rettype=u"trigger", - definition=u"BEGIN RETURN NEW; END;", - ) - helpers.call_action(u"datastore_function_delete", name=u"test_nop") - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_nonexistant(self): - try: - helpers.call_action( - u"datastore_function_delete", name=u"test_not_there" - ) - except ValidationError as ve: - assert ve.error_dict == { - u"name": [u"function test_not_there() does not exist"] - } - else: - assert 0, u"no validation error" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_if_exists(self): - helpers.call_action( - u"datastore_function_delete", - name=u"test_not_there_either", - if_exists=True, - ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py deleted file mode 100644 index 7de3e1b..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dictionary.py +++ /dev/null @@ -1,35 +0,0 @@ -# encoding: utf-8 - -import pytest -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers - -from ckanext.datapusher.tests import get_api_token - - -@pytest.mark.ckan_config(u"ckan.plugins", u"datastore datapusher") -@pytest.mark.ckan_config("ckan.datapusher.api_token", get_api_token()) -@pytest.mark.usefixtures(u"clean_datastore", u"with_plugins", u"with_request_context") -def test_read(app): - user = factories.User() - user_token = factories.APIToken(user=user["id"]) - dataset = factories.Dataset(creator_user_id=user["id"]) - resource = factories.Resource( - package_id=dataset["id"], creator_user_id=user["id"] - ) - data = { - u"resource_id": resource["id"], - u"force": True, - u"records": [ - {u"from": u"Brazil", u"to": u"Brazil", u"num": 2}, - {u"from": u"Brazil", u"to": u"Italy", u"num": 22}, - ], - } - helpers.call_action(u"datastore_create", **data) - auth = {u"Authorization": user_token["token"]} - app.get( - url=u"/dataset/{id}/dictionary/{resource_id}".format( - id=str(dataset["name"]), resource_id=str(resource["id"]) - ), - extra_environ=auth, - ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py deleted file mode 100644 index 966e3f3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_disable.py +++ /dev/null @@ -1,28 +0,0 @@ -# encoding: utf-8 - -import pytest -import ckan.plugins as p - - -@pytest.mark.ckan_config("ckan.datastore.sqlsearch.enabled", None) -def test_disabled_by_default(): - with p.use_plugin("datastore"): - with pytest.raises( - KeyError, match=u"Action 'datastore_search_sql' not found" - ): - p.toolkit.get_action("datastore_search_sql") - - -@pytest.mark.ckan_config("ckan.datastore.sqlsearch.enabled", False) -def test_disable_sql_search(): - with p.use_plugin("datastore"): - with pytest.raises( - KeyError, match=u"Action 'datastore_search_sql' not found" - ): - p.toolkit.get_action("datastore_search_sql") - - -@pytest.mark.ckan_config("ckan.datastore.sqlsearch.enabled", True) -def test_enabled_sql_search(): - with p.use_plugin("datastore"): - p.toolkit.get_action("datastore_search_sql") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py deleted file mode 100644 index 6c0aa3e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_dump.py +++ /dev/null @@ -1,669 +0,0 @@ -# encoding: utf-8 - -import unittest.mock as mock -import json -import pytest -import six -import ckan.tests.helpers as helpers -import ckan.tests.factories as factories - - -class TestDatastoreDump(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_basic(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"book": "annakarenina"}, {u"book": "warandpeace"}], - } - helpers.call_action("datastore_create", **data) - - response = app.get("/datastore/dump/{0}".format(str(resource["id"]))) - assert ( - "_id,book\r\n" - "1,annakarenina\n" - "2,warandpeace\n" == six.ensure_text(response.data) - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_all_fields_types(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - response = app.get("/datastore/dump/{0}".format(str(resource["id"]))) - content = six.ensure_text(response.data) - expected = ( - u"_id,b\xfck,author,published" u",characters,random_letters,nested" - ) - assert content[: len(expected)] == expected - assert "warandpeace" in content - assert '"[""Princess Anna"",""Sergius""]"' in content - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_alias(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "aliases": "books", - "records": [{u"book": "annakarenina"}, {u"book": "warandpeace"}], - } - helpers.call_action("datastore_create", **data) - - # get with alias instead of id - response = app.get("/datastore/dump/books") - assert ( - "_id,book\r\n" - "1,annakarenina\n" - "2,warandpeace\n" == six.ensure_text(response.data) - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_does_not_exist_raises_404(self, app): - - app.get("/datastore/dump/does-not-exist", status=404) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_limit(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"book": "annakarenina"}, {u"book": "warandpeace"}], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=1".format(str(resource["id"])) - ) - content = six.ensure_text(response.data) - expected_content = u"_id,book\r\n" u"1,annakarenina\n" - assert content == expected_content - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_q_and_fields(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - u"/datastore/dump/{0}?q=warandpeace&fields=nested,author".format( - resource["id"] - ) - ) - content = six.ensure_text(response.data) - - expected_content = u"nested,author\r\n" u'"{""a"": ""b""}",tolstoy\n' - assert content == expected_content - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_csv_file_extension(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - u"/datastore/dump/{0}?limit=1&format=csv".format( - resource["id"] - ) - ) - - attachment_filename = res.headers['Content-disposition'] - - expected_attch_filename = 'attachment; filename="{0}.csv"'.format( - resource['id']) - - assert attachment_filename == expected_attch_filename - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_tsv(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - "/datastore/dump/{0}?limit=1&format=tsv".format( - str(resource["id"]) - ) - ) - content = six.ensure_text(res.data) - - expected_content = ( - u"_id\tb\xfck\tauthor\tpublished\tcharacters\trandom_letters\t" - u"nested\r\n1\tannakarenina\ttolstoy\t2005-03-01T00:00:00\t" - u'"[""Princess Anna"",""Sergius""]"\t' - u'"[""a"",""e"",""x""]"\t"[""b"", ' - u'{""moo"": ""moo""}]"\n' - ) - assert content == expected_content - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_tsv_file_extension(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - "/datastore/dump/{0}?limit=1&format=tsv".format( - str(resource["id"]) - ) - ) - - attachment_filename = res.headers['Content-disposition'] - - expected_attch_filename = 'attachment; filename="{0}.tsv"'.format( - resource['id']) - - assert attachment_filename == expected_attch_filename - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_json(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - "/datastore/dump/{0}?limit=1&format=json".format( - resource["id"] - ) - ) - - content = json.loads(six.ensure_text(res.data)) - expected_content = { - u'fields': [ - {u'id': u'_id', u'type': u'int'}, - {u'id': u'bük', u'type': u'text'}, - {u'id': u'author', u'type': u'text'}, - {u'id': u'published', u'type': u'timestamp'}, - {u'id': u'characters', u'type': u'_text'}, - {u'id': u'random_letters', u'type': u'_text'}, - {u'id': u'nested', u'type': u'json'} - ], - u'records': [ - [ - 1, u'annakarenina', u'tolstoy', u'2005-03-01T00:00:00', - [u'Princess Anna', u'Sergius'], - [u'a', u'e', u'x'], - [u'b', {u'moo': u'moo'}] - ] - ] - } - assert content == expected_content - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_json_file_extension(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - "/datastore/dump/{0}?limit=1&format=json".format( - resource["id"] - ) - ) - - attachment_filename = res.headers['Content-disposition'] - - expected_attch_filename = 'attachment; filename="{0}.json"'.format( - resource['id']) - - assert attachment_filename == expected_attch_filename - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_xml(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - "/datastore/dump/{0}?limit=1&format=xml".format( - str(resource["id"]) - ) - ) - content = six.ensure_text(res.data) - expected_content = ( - u'\n' - r'' - u"annakarenina" - u"tolstoy" - u"2005-03-01T00:00:00" - u"" - u'Princess Anna' - u'Sergius' - u"" - u"" - u'a' - u'e' - u'x' - u"" - u"" - u'b' - u'' - u'moo' - u"" - u"" - u"\n" - u"\n" - ) - assert content == expected_content - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_xml_file_extension(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "random_letters", "type": "text[]"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "random_letters": ["a", "e", "x"], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "random_letters": [], - }, - ], - } - helpers.call_action("datastore_create", **data) - - res = app.get( - "/datastore/dump/{0}?limit=1&format=xml".format( - str(resource["id"]) - ) - ) - - attachment_filename = res.headers['Content-disposition'] - - expected_attch_filename = 'attachment; filename="{0}.xml"'.format( - resource['id']) - - assert attachment_filename == expected_attch_filename - - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "3") - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dump_with_low_rows_max(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get("/datastore/dump/{0}".format(str(resource["id"]))) - assert get_csv_record_values(response.data) == list(range(12)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5) - def test_dump_pagination(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get("/datastore/dump/{0}".format(str(resource["id"]))) - assert get_csv_record_values(response.data) == list(range(12)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "7") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5) - def test_dump_pagination_with_limit(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=11".format(str(resource["id"])) - ) - assert get_csv_record_values(response.data) == list(range(11)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "7") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 6) - def test_dump_pagination_csv_with_limit_same_as_paginate(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=6".format(str(resource["id"])) - ) - assert get_csv_record_values(response.data) == list(range(6)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "6") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5) - def test_dump_pagination_with_rows_max(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=7".format(str(resource["id"])) - ) - assert get_csv_record_values(response.data) == list(range(7)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "6") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 6) - def test_dump_pagination_with_rows_max_same_as_paginate(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=7".format(str(resource["id"])) - ) - assert get_csv_record_values(response.data) == list(range(7)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "7") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5) - def test_dump_pagination_json_with_limit(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=6&format=json".format( - str(resource["id"]) - ) - ) - assert get_json_record_values(response.data) == list(range(6)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "6") - @mock.patch("ckanext.datastore.blueprint.PAGINATE_BY", 5) - def test_dump_pagination_json_with_rows_max(self, app): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{u"record": str(num)} for num in list(range(12))], - } - helpers.call_action("datastore_create", **data) - - response = app.get( - "/datastore/dump/{0}?limit=7&format=json".format( - str(resource["id"]) - ) - ) - assert get_json_record_values(response.data) == list(range(7)) - - -def get_csv_record_values(response_body): - return [ - int(record.split(",")[1]) for record in six.ensure_text( - response_body).split()[1:] - ] - - -def get_json_record_values(response_body): - return [record[1] for record in json.loads(response_body)["records"]] diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py deleted file mode 100644 index 201efec..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_helpers.py +++ /dev/null @@ -1,214 +0,0 @@ -# encoding: utf-8 -import re - -import pytest -import sqlalchemy.orm as orm -from sqlalchemy.exc import ProgrammingError - -import ckanext.datastore.backend.postgres as db -import ckanext.datastore.backend.postgres as postgres_backend -import ckanext.datastore.helpers as datastore_helpers - - -class TestTypeGetters(object): - def test_get_list(self): - get_list = datastore_helpers.get_list - assert get_list(None) is None - assert get_list([]) == [] - assert get_list("") == [] - assert get_list("foo") == ["foo"] - assert get_list("foo, bar") == ["foo", "bar"] - assert get_list('foo_"bar, baz') == ['foo_"bar', "baz"] - assert get_list('"foo", "bar"') == ["foo", "bar"] - assert get_list(u"foo, bar") == ["foo", "bar"] - assert get_list(["foo", "bar"]) == ["foo", "bar"] - assert get_list([u"foo", u"bar"]) == ["foo", "bar"] - assert get_list(["foo", ["bar", "baz"]]) == ["foo", ["bar", "baz"]] - - def test_is_single_statement(self): - singles = [ - "SELECT * FROM footable", - 'SELECT * FROM "bartable"', - 'SELECT * FROM "bartable";', - 'SELECT * FROM "bart;able";', - "select 'foo'||chr(59)||'bar'", - ] - - multiples = [ - "SELECT * FROM abc; SET LOCAL statement_timeout to" - "SET LOCAL statement_timeout to; SELECT * FROM abc", - 'SELECT * FROM "foo"; SELECT * FROM "abc"', - ] - - for single in singles: - assert postgres_backend.is_single_statement(single) is True - - for multiple in multiples: - assert postgres_backend.is_single_statement(multiple) is False - - def test_should_fts_index_field_type(self): - indexable_field_types = ["tsvector", "text", "number"] - - non_indexable_field_types = [ - "nested", - "timestamp", - "date", - "_text", - "text[]", - ] - - for indexable in indexable_field_types: - assert ( - datastore_helpers.should_fts_index_field_type(indexable) - is True - ) - - for non_indexable in non_indexable_field_types: - assert ( - datastore_helpers.should_fts_index_field_type(non_indexable) - is False - ) - - -@pytest.mark.ckan_config("ckan.plugins", "datastore") -@pytest.mark.usefixtures("clean_datastore", "with_plugins") -class TestGetTables(object): - def test_get_table_names(self): - engine = db.get_write_engine() - session = orm.scoped_session(orm.sessionmaker(bind=engine)) - create_tables = [ - u"CREATE TABLE test_a (id_a text)", - u"CREATE TABLE test_b (id_b text)", - u'CREATE TABLE "TEST_C" (id_c text)', - u'CREATE TABLE test_d ("α/α" integer)', - ] - for create_table_sql in create_tables: - session.execute(create_table_sql) - - test_cases = [ - (u"SELECT * FROM test_a", ["test_a"]), - (u"SELECT * FROM public.test_a", ["test_a"]), - (u'SELECT * FROM "TEST_C"', ["TEST_C"]), - (u'SELECT * FROM public."TEST_C"', ["TEST_C"]), - (u"SELECT * FROM pg_catalog.pg_database", ["pg_database"]), - (u"SELECT rolpassword FROM pg_roles", ["pg_authid"]), - ( - u"""SELECT p.rolpassword - FROM pg_roles p - JOIN test_b b - ON p.rolpassword = b.id_b""", - ["pg_authid", "test_b"], - ), - ( - u"""SELECT id_a, id_b, id_c - FROM ( - SELECT * - FROM ( - SELECT * - FROM "TEST_C") AS c, - test_b) AS b, - test_a AS a""", - ["test_a", "test_b", "TEST_C"], - ), - (u"INSERT INTO test_a VALUES ('a')", ["test_a"]), - (u'SELECT "α/α" FROM test_d', ["test_d"]), - (u'SELECT "α/α" FROM test_d WHERE "α/α" > 1000', ["test_d"]), - ] - - context = {"connection": session.connection()} - for case in test_cases: - assert sorted( - datastore_helpers.get_table_and_function_names_from_sql(context, case[0])[0] - ) == sorted(case[1]) - - -@pytest.mark.ckan_config("ckan.plugins", "datastore") -@pytest.mark.usefixtures("clean_datastore", "with_plugins") -class TestGetFunctions(object): - def test_get_function_names(self): - - engine = db.get_write_engine() - session = orm.scoped_session(orm.sessionmaker(bind=engine)) - create_tables = [ - u"CREATE TABLE test_a (id int, period date, subject_id text, result decimal)", - u"CREATE TABLE test_b (name text, subject_id text)", - ] - for create_table_sql in create_tables: - session.execute(create_table_sql) - - test_cases = [ - (u"SELECT max(id) from test_a", ["max"]), - (u"SELECT count(distinct(id)) FROM test_a", ["count", "distinct"]), - (u"SELECT trunc(avg(result),2) FROM test_a", ["trunc", "avg"]), - (u"SELECT trunc(avg(result),2), avg(result) FROM test_a", ["trunc", "avg"]), - (u"SELECT * from pg_settings", ["pg_show_all_settings"]), - (u"SELECT * from pg_settings UNION SELECT * from pg_settings", ["pg_show_all_settings"]), - (u"SELECT * from (SELECT * FROM pg_settings) AS tmp", ["pg_show_all_settings"]), - (u"SELECT query_to_xml('SELECT max(id) FROM test_a', true, true , '')", ["query_to_xml"]), - (u"select $$'$$, query_to_xml($X$SELECT table_name FROM information_schema.tables$X$,true,true,$X$$X$), $$'$$", ["query_to_xml"]) - ] - - context = {"connection": session.connection()} - for case in test_cases: - assert sorted( - datastore_helpers.get_table_and_function_names_from_sql(context, case[0])[1] - ) == sorted(case[1]) - - def test_get_function_names_custom_function(self): - - engine = db.get_write_engine() - session = orm.scoped_session(orm.sessionmaker(bind=engine)) - create_tables = [ - u"""CREATE FUNCTION add(integer, integer) RETURNS integer - AS 'select $1 + $2;' - LANGUAGE SQL - IMMUTABLE - RETURNS NULL ON NULL INPUT; - """ - ] - for create_table_sql in create_tables: - session.execute(create_table_sql) - - context = {"connection": session.connection()} - - sql = "SELECT add(1, 2);" - - assert datastore_helpers.get_table_and_function_names_from_sql(context, sql)[1] == ["add"] - - def test_get_function_names_crosstab(self): - """ - Crosstab functions need to be enabled in the database by executing the following using - a super user: - - CREATE extension tablefunc; - - """ - - engine = db.get_write_engine() - session = orm.scoped_session(orm.sessionmaker(bind=engine)) - create_tables = [ - u"CREATE TABLE test_a (id int, period date, subject_id text, result decimal)", - u"CREATE TABLE test_b (name text, subject_id text)", - ] - for create_table_sql in create_tables: - session.execute(create_table_sql) - - test_cases = [ - (u"""SELECT * - FROM crosstab( - 'SELECT extract(month from period)::text, test_b.name, trunc(avg(result),2) - FROM test_a, test_b - WHERE test_a.subject_id = test_b.subject_id') - AS final_result(month text, subject_1 numeric,subject_2 numeric);""", - ['crosstab', 'final_result', 'extract', 'trunc', 'avg']), - ] - - context = {"connection": session.connection()} - try: - for case in test_cases: - assert sorted( - datastore_helpers.get_table_and_function_names_from_sql(context, case[0])[1] - ) == sorted(case[1]) - except ProgrammingError as e: - if bool(re.search("function crosstab(.*) does not exist", str(e))): - pytest.skip("crosstab functions not enabled in DataStore database") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py deleted file mode 100644 index cb5ec51..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_info.py +++ /dev/null @@ -1,88 +0,0 @@ -# encoding: utf-8 - -import pytest -import six -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers -from ckan.lib import helpers as template_helpers - - -@pytest.mark.ckan_config("ckan.plugins", "datastore") -@pytest.mark.usefixtures("clean_datastore", "with_plugins") -def test_info_success(): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"from": "Brazil", "to": "Brazil", "num": 2}, - {"from": "Brazil", "to": "Italy", "num": 22}, - ], - } - helpers.call_action("datastore_create", **data) - - # aliases can only be created against an existing resource - data = { - "resource_id": resource["id"], - "force": True, - "aliases": "testalias1, testview2", - - } - helpers.call_action("datastore_create", **data) - - info = helpers.call_action("datastore_info", id=resource["id"]) - - assert len(info["meta"]) == 7, info["meta"] - assert info["meta"]["count"] == 2 - assert info["meta"]["table_type"] == "BASE TABLE" - assert len(info["meta"]["aliases"]) == 2 - assert info["meta"]["aliases"] == ["testview2", "testalias1"] - assert len(info["fields"]) == 3, info["fields"] - assert info["fields"][0]["id"] == "from" - assert info["fields"][0]["type"] == "text" - assert info["fields"][0]["schema"]["native_type"] == "text" - assert not info["fields"][0]["schema"]["is_index"] - assert info["fields"][2]["id"] == "num" - assert info["fields"][2]["schema"]["native_type"] == "integer" - - # check datastore_info with alias - info = helpers.call_action("datastore_info", id='testalias1') - - assert len(info["meta"]) == 7, info["meta"] - assert info["meta"]["count"] == 2 - assert info["meta"]["id"] == resource["id"] - - -@pytest.mark.ckan_config("ckan.plugins", "datastore") -@pytest.mark.usefixtures("clean_datastore", "with_plugins") -def test_api_info(app): - dataset = factories.Dataset() - resource = factories.Resource( - id="588dfa82-760c-45a2-b78a-e3bc314a4a9b", - package_id=dataset["id"], - datastore_active=True, - ) - - # the 'API info' is seen on the resource_read page, a snippet loaded by - # javascript via data_api_button.html - url = template_helpers.url_for( - "api.snippet", - ver=1, - snippet_path="api_info.html", - resource_id=resource["id"], - ) - - page = app.get(url, status=200) - - # check we built all the urls ok - expected_urls = ( - "http://test.ckan.net/api/3/action/datastore_create", - "http://test.ckan.net/api/3/action/datastore_upsert", - "http://test.ckan.net/api/3/action/datastore_search", - "http://test.ckan.net/api/3/action/datastore_search_sql", - "url: 'http://test.ckan.net/api/3/action/datastore_search'", - "http://test.ckan.net/api/3/action/datastore_search" - ) - content = six.ensure_text(page.data) - for url in expected_urls: - assert url in content diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py deleted file mode 100644 index 2c6fa2f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_interface.py +++ /dev/null @@ -1,169 +0,0 @@ -# encoding: utf-8 - -import pytest -import ckan.plugins as p -import ckan.tests.helpers as helpers -import ckan.tests.factories as factories - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_search_can_create_custom_filters(): - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - filters = {"age_between": [25, 35]} - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"], filters=filters - ) - - assert result["total"] == 1, result - assert result["records"][0]["age"] == 30, result - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_search_filters_sent_arent_modified(): - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - filters = {"age_between": [25, 35]} - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"], filters=filters.copy() - ) - - assert result["filters"] == filters - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_search_custom_filters_have_the_correct_operator_precedence(): - """ - We're testing that the WHERE clause becomes: - (age < 50 OR age > 60) AND age = 30 - And not: - age < 50 OR age > 60 AND age = 30 - """ - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - filters = {"age_not_between": [50, 60], "age": 30} - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"], filters=filters - ) - - assert result["total"] == 1, result - assert result["records"][0]["age"] == 30, result - assert result["filters"] == filters - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_search_insecure_filter(): - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - sql_inject = '1=1); DELETE FROM "%s"; COMMIT; SELECT * FROM "%s";--' % ( - resource["id"], - resource["id"], - ) - filters = {"insecure_filter": sql_inject} - - with pytest.raises(p.toolkit.ValidationError): - helpers.call_action( - "datastore_search", resource_id=resource["id"], filters=filters - ) - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"] - ) - - assert result["total"] == 3, result - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_delete_can_create_custom_filters(): - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - filters = {"age_between": [25, 35]} - - helpers.call_action( - "datastore_delete", - resource_id=resource["id"], - force=True, - filters=filters, - ) - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"] - ) - - new_records_ages = [r["age"] for r in result["records"]] - new_records_ages.sort() - assert new_records_ages == [20, 40] - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_delete_custom_filters_have_the_correct_operator_precedence(): - """ - We're testing that the WHERE clause becomes: - (age < 50 OR age > 60) AND age = 30 - And not: - age < 50 OR age > 60 AND age = 30 - """ - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - filters = {"age_not_between": [50, 60], "age": 30} - - helpers.call_action( - "datastore_delete", - resource_id=resource["id"], - force=True, - filters=filters, - ) - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"] - ) - - new_records_ages = [r["age"] for r in result["records"]] - new_records_ages.sort() - assert new_records_ages == [20, 40] - - -@pytest.mark.ckan_config("ckan.plugins", "datastore sample_datastore_plugin") -@pytest.mark.usefixtures("clean_datastore", "with_plugins", "with_request_context") -def test_datastore_delete_insecure_filter(): - records = [{"age": 20}, {"age": 30}, {"age": 40}] - resource = _create_datastore_resource(records) - sql_inject = '1=1); DELETE FROM "%s"; SELECT * FROM "%s";--' % ( - resource["id"], - resource["id"], - ) - filters = {"age": 20, "insecure_filter": sql_inject} - - with pytest.raises(p.toolkit.ValidationError): - helpers.call_action( - "datastore_delete", - resource_id=resource["id"], - force=True, - filters=filters, - ) - - result = helpers.call_action( - "datastore_search", resource_id=resource["id"] - ) - - assert result["total"] == 3, result - - -def _create_datastore_resource(records): - dataset = factories.Dataset() - resource = factories.Resource(package=dataset) - - data = {"resource_id": resource["id"], "force": True, "records": records} - - helpers.call_action("datastore_create", **data) - - return resource diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py deleted file mode 100644 index c08f473..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_plugin.py +++ /dev/null @@ -1,195 +0,0 @@ -# encoding: utf-8 - -import unittest.mock as mock -import pytest - -import ckan.plugins as p -import ckanext.datastore.interfaces as interfaces -import ckanext.datastore.plugin as plugin - - -DatastorePlugin = plugin.DatastorePlugin - - -class TestPluginLoadingOrder(object): - def setup(self): - if p.plugin_loaded("datastore"): - p.unload("datastore") - if p.plugin_loaded("sample_datastore_plugin"): - p.unload("sample_datastore_plugin") - - def teardown(self): - if p.plugin_loaded("sample_datastore_plugin"): - p.unload("sample_datastore_plugin") - if p.plugin_loaded("datastore"): - p.unload("datastore") - - def test_loading_datastore_first_works(self): - p.load("datastore") - p.load("sample_datastore_plugin") - p.unload("sample_datastore_plugin") - p.unload("datastore") - - def test_loading_datastore_last_doesnt_work(self): - # This test is complicated because we can't import - # ckanext.datastore.plugin before running it. If we did so, the - # DatastorePlugin class would be parsed which breaks the reason of our - # test. - p.load("sample_datastore_plugin") - thrown_exception = None - try: - p.load("datastore") - except Exception as e: - thrown_exception = e - idatastores = [ - x.__class__.__name__ - for x in p.PluginImplementations(interfaces.IDatastore) - ] - p.unload("sample_datastore_plugin") - - assert thrown_exception is not None, ( - 'Loading "datastore" after another IDatastore plugin was' - "loaded should raise DatastoreException" - ) - assert ( - thrown_exception.__class__.__name__ - == plugin.DatastoreException.__name__ - ) - assert plugin.DatastorePlugin.__name__ not in idatastores, ( - 'You shouldn\'t be able to load the "datastore" plugin after' - "another IDatastore plugin was loaded" - ) - - -@pytest.mark.ckan_config("ckan.plugins", "datastore") -@pytest.mark.usefixtures("clean_index", "with_plugins", "with_request_context") -class TestPluginDatastoreSearch(object): - def test_english_is_default_fts_language(self): - expected_ts_query = ", plainto_tsquery('english', 'foo') \"query\"" - data_dict = {"q": "foo"} - - result = self._datastore_search(data_dict=data_dict) - - assert result["ts_query"] == expected_ts_query - - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple") - def test_can_overwrite_default_fts_lang_using_config_variable(self): - expected_ts_query = ", plainto_tsquery('simple', 'foo') \"query\"" - data_dict = {"q": "foo"} - - result = self._datastore_search(data_dict=data_dict) - - assert result["ts_query"] == expected_ts_query - - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple") - def test_lang_parameter_overwrites_default_fts_lang(self): - expected_ts_query = ", plainto_tsquery('french', 'foo') \"query\"" - data_dict = {"q": "foo", "language": "french"} - - result = self._datastore_search(data_dict=data_dict) - - assert result["ts_query"] == expected_ts_query - - def test_fts_rank_column_uses_lang_when_casting_to_tsvector(self): - expected_select_content = ( - u"to_tsvector('french', cast(\"country\" as text))" - ) - data_dict = {"q": {"country": "Brazil"}, "language": "french"} - result = self._datastore_search(data_dict=data_dict, fields_types={}) - assert expected_select_content in result["select"][0] - - def test_adds_fts_on_full_text_field_when_q_is_a_string(self): - expected_where = [(u'_full_text @@ "query"',)] - data_dict = {"q": "foo"} - - result = self._datastore_search(data_dict=data_dict) - - assert result["where"] == expected_where - - def test_ignores_fts_searches_on_inexistent_fields(self): - data_dict = {"q": {"inexistent-field": "value"}} - - result = self._datastore_search(data_dict=data_dict, fields_types={}) - - assert result["where"] == [] - - def test_fts_where_clause_lang_uses_english_by_default(self): - expected_where = [ - ( - u"to_tsvector('english', cast(\"country\" as text))" - u' @@ "query country"', - ) - ] - data_dict = {"q": {"country": "Brazil"}} - fields_types = {"country": "text"} - - result = self._datastore_search( - data_dict=data_dict, fields_types=fields_types - ) - - assert result["where"] == expected_where - - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple") - def test_fts_where_clause_lang_can_be_overwritten_by_config(self): - expected_where = [ - ( - u"to_tsvector('simple', cast(\"country\" as text))" - u' @@ "query country"', - ) - ] - data_dict = {"q": {"country": "Brazil"}} - fields_types = {"country": "text"} - - result = self._datastore_search( - data_dict=data_dict, fields_types=fields_types - ) - - assert result["where"] == expected_where - - @pytest.mark.ckan_config("ckan.datastore.default_fts_lang", "simple") - def test_fts_where_clause_lang_can_be_overwritten_using_lang_param(self): - expected_where = [ - ( - u"to_tsvector('french', cast(\"country\" as text))" - u' @@ "query country"', - ) - ] - data_dict = {"q": {"country": "Brazil"}, "language": "french"} - fields_types = {"country": "text"} - - result = self._datastore_search( - data_dict=data_dict, fields_types=fields_types - ) - - assert result["where"] == expected_where - - @mock.patch("ckanext.datastore.helpers.should_fts_index_field_type") - def test_fts_adds_where_clause_on_full_text_when_querying_non_indexed_fields( - self, should_fts_index_field_type - ): - should_fts_index_field_type.return_value = False - expected_where = [ - ('_full_text @@ "query country"',), - ( - u"to_tsvector('english', cast(\"country\" as text))" - u' @@ "query country"', - ), - ] - data_dict = {"q": {"country": "Brazil"}, "language": "english"} - fields_types = {"country": "non-indexed field type"} - - result = self._datastore_search( - data_dict=data_dict, fields_types=fields_types - ) - - assert result["where"] == expected_where - - def _datastore_search( - self, context={}, data_dict={}, fields_types={}, query_dict={} - ): - _query_dict = {"select": [], "sort": [], "where": []} - _query_dict.update(query_dict) - - return DatastorePlugin().datastore_search( - context, data_dict, fields_types, _query_dict - ) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py deleted file mode 100644 index 7f41ce3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_search.py +++ /dev/null @@ -1,2181 +0,0 @@ -# encoding: utf-8 - -import json -import pytest -import sqlalchemy.orm as orm - -import ckan.lib.create_test_data as ctd -import ckan.logic as logic -import ckan.model as model -import ckan.plugins as p -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers -import ckanext.datastore.backend.postgres as db -from ckanext.datastore.tests.helpers import extract - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreSearch(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_fts_on_field_calculates_ranks_only_on_that_specific_field(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"from": "Brazil", "to": "Brazil"}, - {"from": "Brazil", "to": "Italy"}, - ], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "fields": "from, rank from", - "q": {"from": "Brazil"}, - } - result = helpers.call_action("datastore_search", **search_data) - ranks = [r["rank from"] for r in result["records"]] - assert len(result["records"]) == 2 - assert len(set(ranks)) == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_fts_works_on_non_textual_fields(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"from": "Brazil", "year": {"foo": 2014}}, - {"from": "Brazil", "year": {"foo": 1986}}, - ], - } - result = helpers.call_action("datastore_create", **data) - - search_data = { - "resource_id": resource["id"], - "fields": "year", - "plain": False, - "q": {"year": "20:*"}, - } - result = helpers.call_action("datastore_search", **search_data) - assert len(result["records"]) == 1 - assert result["records"][0]["year"] == {"foo": 2014} - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_all_params_work_with_fields_with_whitespaces(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "fields": "the year", - "sort": "the year", - "filters": {"the year": 2013}, - "q": {"the year": "2013"}, - } - result = helpers.call_action("datastore_search", **search_data) - result_years = [r["the year"] for r in result["records"]] - assert result_years == [2013] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_total(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - result = helpers.call_action("datastore_create", **data) - search_data = {"resource_id": resource["id"], "include_total": True} - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 2 - assert not (result.get("total_was_estimated")) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_without_total(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - result = helpers.call_action("datastore_create", **data) - search_data = {"resource_id": resource["id"], "include_total": False} - result = helpers.call_action("datastore_search", **search_data) - assert "total" not in result - assert "total_was_estimated" not in result - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(100)], - } - result = helpers.call_action("datastore_create", **data) - analyze_sql = """ - ANALYZE "{resource}"; - """.format( - resource=resource["id"] - ) - db.get_write_engine().execute(analyze_sql) - search_data = { - "resource_id": resource["id"], - "total_estimation_threshold": 50, - } - result = helpers.call_action("datastore_search", **search_data) - assert result.get("total_was_estimated") - assert 95 < result["total"] < 105, result["total"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total_with_filters(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(3)] * 10, - } - result = helpers.call_action("datastore_create", **data) - analyze_sql = """ - ANALYZE "{resource}"; - """.format( - resource=resource["id"] - ) - db.get_write_engine().execute(analyze_sql) - search_data = { - "resource_id": resource["id"], - "filters": {u"the year": 1901}, - "total_estimation_threshold": 5, - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 10 - # estimation is not compatible with filters - assert not (result.get("total_was_estimated")) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total_with_distinct(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(3)] * 10, - } - result = helpers.call_action("datastore_create", **data) - analyze_sql = """ - ANALYZE "{resource}"; - """.format( - resource=resource["id"] - ) - db.get_write_engine().execute(analyze_sql) - search_data = { - "resource_id": resource["id"], - "fields": ["the year"], - "distinct": True, - "total_estimation_threshold": 1, - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 3 - # estimation is not compatible with distinct - assert not (result.get("total_was_estimated")) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total_where_analyze_is_not_already_done(self): - # ANALYSE is done by latest datapusher/xloader, but need to cope in - # if tables created in other ways which may not have had an ANALYSE - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(100)], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "total_estimation_threshold": 50, - } - result = helpers.call_action("datastore_search", **search_data) - assert result.get("total_was_estimated") - assert 95 < result["total"] < 105, result["total"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total_with_zero_threshold(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(100)], - } - result = helpers.call_action("datastore_create", **data) - analyze_sql = """ - ANALYZE "{resource}"; - """.format( - resource=resource["id"] - ) - db.get_write_engine().execute(analyze_sql) - search_data = { - "resource_id": resource["id"], - "total_estimation_threshold": 0, - } - result = helpers.call_action("datastore_search", **search_data) - # threshold of 0 means always estimate - assert result.get("total_was_estimated") - assert 95 < result["total"] < 105, result["total"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total_off(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(100)], - } - result = helpers.call_action("datastore_create", **data) - analyze_sql = """ - ANALYZE "{resource}"; - """.format( - resource=resource["id"] - ) - db.get_write_engine().execute(analyze_sql) - search_data = { - "resource_id": resource["id"], - "total_estimation_threshold": None, - } - result = helpers.call_action("datastore_search", **search_data) - # threshold of None means don't estimate - assert not (result.get("total_was_estimated")) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_estimate_total_default_off(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 1900 + i} for i in range(100)], - } - result = helpers.call_action("datastore_create", **data) - analyze_sql = """ - ANALYZE "{resource}"; - """.format( - resource=resource["id"] - ) - db.get_write_engine().execute(analyze_sql) - search_data = { - "resource_id": resource["id"], - # don't specify total_estimation_threshold - } - result = helpers.call_action("datastore_search", **search_data) - # default threshold is None, meaning don't estimate - assert not (result.get("total_was_estimated")) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_limit(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - result = helpers.call_action("datastore_create", **data) - search_data = {"resource_id": resource["id"], "limit": 1} - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 2 - assert result["records"] == [{u"the year": 2014, u"_id": 1}] - assert result["limit"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_limit_invalid(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - helpers.call_action("datastore_create", **data) - search_data = {"resource_id": resource["id"], "limit": "bad"} - with pytest.raises(logic.ValidationError, match="Invalid integer"): - helpers.call_action("datastore_search", **search_data) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_limit_invalid_negative(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - helpers.call_action("datastore_create", **data) - search_data = {"resource_id": resource["id"], "limit": -1} - with pytest.raises( - logic.ValidationError, match="Must be a natural number" - ): - helpers.call_action("datastore_search", **search_data) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_default", "1") - def test_search_limit_config_default(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - # limit not specified - leaving to the configured default of 1 - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 2 - assert result["records"] == [{u"the year": 2014, u"_id": 1}] - assert result["limit"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_default", "1") - def test_search_limit_config(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"the year": 2015}, - {"the year": 2014}, - {"the year": 2013}, - ], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "limit": 2, # specified limit overrides the rows_default - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 3 - assert result["records"] == [ - {u"the year": 2015, u"_id": 1}, - {u"the year": 2014, u"_id": 2}, - ] - assert result["limit"] == 2 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "1") - def test_search_limit_config_max(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - # limit not specified - leaving to the configured default of 1 - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 2 - assert result["records"] == [{u"the year": 2014, u"_id": 1}] - assert result["limit"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_default", "1") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "2") - def test_search_limit_config_combination(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"the year": 2016}, - {"the year": 2015}, - {"the year": 2014}, - {"the year": 2013}, - ], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "limit": 3, # ignored because it is above rows_max - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 4 - # returns 2 records, - # ignoring the rows_default because we specified limit - # but limit is more than rows_max so rows_max=2 wins - assert result["records"] == [ - {u"the year": 2016, u"_id": 1}, - {u"the year": 2015, u"_id": 2}, - ] - assert result["limit"] == 2 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_filter_with_percent_in_column_name(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "bo%ok", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1%", "bo%ok": u"El Nino", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - search_data = { - "resource_id": resource["id"], - "filters": {u"bo%ok": "El Nino"}, - } - result = helpers.call_action("datastore_search", **search_data) - assert result["total"] == 1 - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreSearchLegacyTests(object): - sysadmin_user = None - normal_user = None - - @pytest.fixture(autouse=True) - def initial_data(self, clean_datastore, app): - ctd.CreateTestData.create() - self.sysadmin_user = factories.Sysadmin() - self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"]) - self.sysadmin_token = self.sysadmin_token["token"] - self.normal_user = factories.User() - self.normal_user_token = factories.APIToken(user=self.normal_user["id"]) - self.normal_user_token = self.normal_user_token["token"] - self.dataset = model.Package.get("annakarenina") - self.resource = self.dataset.resources[0] - self.data = { - "resource_id": self.resource.id, - "force": True, - "aliases": "books3", - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - {"id": u"characters", u"type": u"_text"}, - {"id": "rating with %"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - u"characters": [u"Princess Anna", u"Sergius"], - "rating with %": "60%", - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - "rating with %": "99%", - }, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", json=self.data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - # Make an organization, because private datasets must belong to one. - self.organization = helpers.call_action( - "organization_create", - {"user": self.sysadmin_user["name"]}, - name="test_org", - ) - - self.expected_records = [ - { - u"published": u"2005-03-01T00:00:00", - u"_id": 1, - u"nested": [u"b", {u"moo": u"moo"}], - u"b\xfck": u"annakarenina", - u"author": u"tolstoy", - u"characters": [u"Princess Anna", u"Sergius"], - u"rating with %": u"60%", - }, - { - u"published": None, - u"_id": 2, - u"nested": {u"a": u"b"}, - u"b\xfck": u"warandpeace", - u"author": u"tolstoy", - u"characters": None, - u"rating with %": u"99%", - }, - ] - - engine = db.get_write_engine() - self.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_basic(self, app): - data = {"resource_id": self.data["resource_id"]} - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == len(self.data["records"]) - assert result["records"] == self.expected_records, result["records"] - - # search with parameter id should yield the same results - data = {"id": self.data["resource_id"]} - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == len(self.data["records"]) - assert result["records"] == self.expected_records, result["records"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_private_dataset(self, app): - group = self.dataset.get_groups()[0] - context = { - "user": self.sysadmin_user["name"], - "ignore_auth": True, - "model": model, - } - package = p.toolkit.get_action("package_create")( - context, - { - "name": "privatedataset", - "private": True, - "owner_org": self.organization["id"], - "groups": [{"id": group.id}], - }, - ) - resource = p.toolkit.get_action("resource_create")( - context, - { - "name": "privateresource", - "url": "https://www.example.com/", - "package_id": package["id"], - }, - ) - helpers.call_action("datastore_create", resource_id=resource["id"], force=True) - data = {"resource_id": resource["id"]} - auth = {"Authorization": self.normal_user_token} - - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_alias(self, app): - data = {"resource_id": self.data["aliases"]} - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict_alias = json.loads(res.data) - result = res_dict_alias["result"] - assert result["total"] == len(self.data["records"]) - assert result["records"] == self.expected_records, result["records"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_invalid_field(self, app): - data = { - "resource_id": self.data["resource_id"], - "fields": [{"id": "bad"}], - } - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_fields(self, app): - data = {"resource_id": self.data["resource_id"], "fields": [u"b\xfck"]} - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == len(self.data["records"]) - assert result["records"] == [ - {u"b\xfck": "annakarenina"}, - {u"b\xfck": "warandpeace"}, - ], result["records"] - - data = { - "resource_id": self.data["resource_id"], - "fields": u"b\xfck, author", - } - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == len(self.data["records"]) - assert result["records"] == [ - {u"b\xfck": "annakarenina", "author": "tolstoy"}, - {u"b\xfck": "warandpeace", "author": "tolstoy"}, - ], result["records"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_distinct(self, app): - data = { - "resource_id": self.data["resource_id"], - "fields": [u"author"], - "distinct": True, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 1 - assert result["records"] == [{u"author": "tolstoy"}], result["records"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_filters(self, app): - data = { - "resource_id": self.data["resource_id"], - "filters": {u"b\xfck": "annakarenina"}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 1 - assert result["records"] == [self.expected_records[0]] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_filter_array_field(self, app): - data = { - "resource_id": self.data["resource_id"], - "filters": {u"characters": [u"Princess Anna", u"Sergius"]}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 1 - assert result["records"] == [self.expected_records[0]] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_multiple_filters_on_same_field(self, app): - data = { - "resource_id": self.data["resource_id"], - "filters": {u"b\xfck": [u"annakarenina", u"warandpeace"]}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 2 - assert result["records"] == self.expected_records - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_filter_normal_field_passing_multiple_values_in_array( - self, app - ): - data = { - "resource_id": self.data["resource_id"], - "filters": {u"b\xfck": [u"annakarenina", u"warandpeace"]}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 2 - assert result["records"] == self.expected_records, result["records"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_filters_get(self, app): - filters = {u"b\xfck": "annakarenina"} - res = app.get( - "/api/action/datastore_search?resource_id={0}&filters={1}".format( - self.data["resource_id"], json.dumps(filters) - ) - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 1 - assert result["records"] == [self.expected_records[0]] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_invalid_filter(self, app): - data = { - "resource_id": self.data["resource_id"], - # invalid because author is not a numeric field - "filters": {u"author": 42}, - } - - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_sort(self, app): - data = { - "resource_id": self.data["resource_id"], - "sort": u"b\xfck asc, author desc", - } - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 2 - - assert result["records"] == self.expected_records, result["records"] - - data = { - "resource_id": self.data["resource_id"], - "sort": [u"b\xfck desc", '"author" asc'], - } - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 2 - - assert result["records"] == self.expected_records[::-1] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_invalid(self, app): - data = { - "resource_id": self.data["resource_id"], - "sort": u"f\xfc\xfc asc", - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - error_msg = res_dict["error"]["sort"][0] - assert ( - u"f\xfc\xfc" in error_msg - ), 'Expected "{0}" to contain "{1}"'.format(error_msg, u"f\xfc\xfc") - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_offset(self, app): - data = { - "resource_id": self.data["resource_id"], - "limit": 1, - "offset": 1, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 2 - assert result["records"] == [self.expected_records[1]] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_invalid_offset(self, app): - data = {"resource_id": self.data["resource_id"], "offset": "bad"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - data = {"resource_id": self.data["resource_id"], "offset": -1} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_full_text(self, app): - data = {"resource_id": self.data["resource_id"], "q": "annakarenina"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 1 - - results = [ - extract( - result["records"][0], - [ - u"_id", - u"author", - u"b\xfck", - u"nested", - u"published", - u"characters", - u"rating with %", - ], - ) - ] - assert results == [self.expected_records[0]], results["records"] - - data = {"resource_id": self.data["resource_id"], "q": "tolstoy"} - - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 2 - results = [ - extract( - record, - [ - u"_id", - u"author", - u"b\xfck", - u"nested", - u"published", - u"characters", - u"rating with %", - ], - ) - for record in result["records"] - ] - assert results == self.expected_records, result["records"] - - expected_fields = [ - {u"type": u"int", u"id": u"_id"}, - {u"type": u"text", u"id": u"b\xfck"}, - {u"type": u"text", u"id": u"author"}, - {u"type": u"timestamp", u"id": u"published"}, - {u"type": u"json", u"id": u"nested"}, - ] - for field in expected_fields: - assert field in result["fields"] - - # test multiple word queries (connected with and) - data = { - "resource_id": self.data["resource_id"], - "plain": True, - "q": "tolstoy annakarenina", - } - - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["total"] == 1 - results = [ - extract( - result["records"][0], - [ - u"_id", - u"author", - u"b\xfck", - u"nested", - u"published", - u"characters", - u"rating with %", - ], - ) - ] - assert results == [self.expected_records[0]], results["records"] - - for field in expected_fields: - assert field in result["fields"], field - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_full_text_on_specific_column(self, app): - data = { - "resource_id": self.data["resource_id"], - "q": {u"b\xfck": "annakarenina"}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - assert len(res_dict["result"]["records"]) == 1 - assert ( - res_dict["result"]["records"][0]["_id"] - == self.expected_records[0]["_id"] - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_full_text_on_specific_column_even_if_q_is_a_json_string( - self, app - ): - data = { - "resource_id": self.data["resource_id"], - "q": u'{"b\xfck": "annakarenina"}', - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - assert len(res_dict["result"]["records"]) == 1 - assert ( - res_dict["result"]["records"][0]["_id"] - == self.expected_records[0]["_id"] - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_full_text_invalid_field_name(self, app): - data = { - "resource_id": self.data["resource_id"], - "q": {"invalid_field_name": "value"}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_full_text_invalid_field_value(self, app): - data = { - "resource_id": self.data["resource_id"], - "q": {"author": ["invalid", "value"]}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_table_metadata(self, app): - data = {"resource_id": "_table_metadata"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_is_unsuccessful_when_called_with_filters_not_as_dict( - self, app - ): - data = { - "resource_id": self.data["resource_id"], - "filters": "the-filter", - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"].get("filters") is not None, res_dict["error"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_is_unsuccessful_when_called_with_invalid_filters( - self, app - ): - data = { - "resource_id": self.data["resource_id"], - "filters": {"invalid-column-name": "value"}, - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"].get("filters") is not None, res_dict["error"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_is_unsuccessful_when_called_with_invalid_fields(self, app): - data = { - "resource_id": self.data["resource_id"], - "fields": ["invalid-column-name"], - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", - json=data, - extra_environ=auth, - status=409, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"].get("fields") is not None, res_dict["error"] - - -class TestDatastoreFullTextSearchLegacyTests(object): - @pytest.fixture(autouse=True) - def initial_data(self, clean_datastore, app): - ctd.CreateTestData.create() - self.sysadmin_user = factories.Sysadmin() - self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"]) - self.sysadmin_token = self.sysadmin_token["token"] - self.normal_user = factories.User() - self.normal_user_token = factories.APIToken(user=self.normal_user["id"]) - self.normal_user_token = self.normal_user_token["token"] - resource = model.Package.get("annakarenina").resources[0] - self.data = dict( - resource_id=resource.id, - force=True, - fields=[ - {"id": "id"}, - {"id": "date", "type": "date"}, - {"id": "x"}, - {"id": "y"}, - {"id": "z"}, - {"id": "country"}, - {"id": "title"}, - {"id": "lat"}, - {"id": "lon"}, - ], - records=[ - { - "id": 0, - "date": "2011-01-01", - "x": 1, - "y": 2, - "z": 3, - "country": "DE", - "title": "first 99", - "lat": 52.56, - "lon": 13.40, - }, - { - "id": 1, - "date": "2011-02-02", - "x": 2, - "y": 4, - "z": 24, - "country": "UK", - "title": "second", - "lat": 54.97, - "lon": -1.60, - }, - { - "id": 2, - "date": "2011-03-03", - "x": 3, - "y": 6, - "z": 9, - "country": "US", - "title": "third", - "lat": 40.00, - "lon": -75.5, - }, - { - "id": 3, - "date": "2011-04-04", - "x": 4, - "y": 8, - "z": 6, - "country": "UK", - "title": "fourth", - "lat": 57.27, - "lon": -6.20, - }, - { - "id": 4, - "date": "2011-05-04", - "x": 5, - "y": 10, - "z": 15, - "country": "UK", - "title": "fifth", - "lat": 51.58, - "lon": 0, - }, - { - "id": 5, - "date": "2011-06-02", - "x": 6, - "y": 12, - "z": 18, - "country": "DE", - "title": "sixth 53.56", - "lat": 51.04, - "lon": 7.9, - }, - ], - ) - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_create", json=self.data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_search_full_text(self, app): - data = {"resource_id": self.data["resource_id"], "q": "DE"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 2 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_advanced_search_full_text(self, app): - data = { - "resource_id": self.data["resource_id"], - "plain": "False", - "q": "DE | UK", - } - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 5 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_full_text_search_on_integers_within_text_strings(self, app): - data = {"resource_id": self.data["resource_id"], "q": "99"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_full_text_search_on_integers(self, app): - data = {"resource_id": self.data["resource_id"], "q": "4"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 3 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_full_text_search_on_decimal_within_text_strings(self, app): - data = {"resource_id": self.data["resource_id"], "q": "53.56"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_full_text_search_on_decimal(self, app): - data = {"resource_id": self.data["resource_id"], "q": "52.56"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_full_text_search_on_date(self, app): - data = {"resource_id": self.data["resource_id"], "q": "2011-01-01"} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["result"]["total"] == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_full_text_search_on_json_like_string_succeeds(self, app): - data = {"resource_id": self.data["resource_id"], "q": '"{}"'} - - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] - - -class TestDatastoreSQLLegacyTests(object): - sysadmin_user = None - normal_user = None - - @pytest.fixture(autouse=True) - def initial_data(self, clean_datastore, app): - ctd.CreateTestData.create() - self.sysadmin_user = factories.Sysadmin() - self.sysadmin_token = factories.APIToken(user=self.sysadmin_user["id"]) - self.sysadmin_token = self.sysadmin_token["token"] - self.normal_user = factories.User() - self.normal_user_token = factories.APIToken(user=self.normal_user["id"]) - self.normal_user_token = self.normal_user_token["token"] - self.dataset = model.Package.get("annakarenina") - resource = self.dataset.resources[0] - self.data = { - "resource_id": resource.id, - "force": True, - "aliases": "books4", - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "published"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - }, - ], - } - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_create", json=self.data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - - # Make an organization, because private datasets must belong to one. - self.organization = helpers.call_action( - "organization_create", - {"user": self.sysadmin_user["name"]}, - name="test_org", - ) - - self.expected_records = [ - { - u"_full_text": [ - u"'annakarenina'", - u"'b'", - u"'moo'", - u"'tolstoy'", - u"'2005'", - ], - u"_id": 1, - u"author": u"tolstoy", - u"b\xfck": u"annakarenina", - u"nested": [u"b", {u"moo": u"moo"}], - u"published": u"2005-03-01T00:00:00", - }, - { - u"_full_text": [u"'tolstoy'", u"'warandpeac'", u"'b'"], - u"_id": 2, - u"author": u"tolstoy", - u"b\xfck": u"warandpeace", - u"nested": {u"a": u"b"}, - u"published": None, - }, - ] - self.expected_join_results = [ - {u"first": 1, u"second": 1}, - {u"first": 1, u"second": 2}, - ] - - engine = db.get_write_engine() - self.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_select_where_like_with_percent(self, app): - query = 'SELECT * FROM public."{0}" WHERE "author" LIKE \'tol%\''.format( - self.data["resource_id"] - ) - data = {"sql": query} - auth = {"Authorization": self.sysadmin_token} - res = app.post( - "/api/action/datastore_search_sql", json=data, extra_environ=auth, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert len(result["records"]) == len(self.expected_records) - for (row_index, row) in enumerate(result["records"]): - expected_row = self.expected_records[row_index] - assert set(row.keys()) == set(expected_row.keys()) - for field in row: - if field == "_full_text": - for ft_value in expected_row["_full_text"]: - assert ft_value in row["_full_text"] - else: - assert row[field] == expected_row[field] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_self_join(self, app): - query = """ - select a._id as first, b._id as second - from "{0}" AS a, - "{0}" AS b - where a.author = b.author - limit 2 - """.format( - self.data["resource_id"] - ) - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search_sql", - json={"sql": query}, - extra_environ=auth, - ) - - res_dict = json.loads(res.data) - assert res_dict["success"] is True - result = res_dict["result"] - assert result["records"] == self.expected_join_results - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures( - "clean_datastore", "with_plugins", "with_request_context" - ) - def test_new_datastore_table_from_private_resource(self, app): - # make a private CKAN resource - group = self.dataset.get_groups()[0] - context = { - "user": self.sysadmin_user["name"], - "ignore_auth": True, - "model": model, - } - package = p.toolkit.get_action("package_create")( - context, - { - "name": "privatedataset", - "private": True, - "owner_org": self.organization["id"], - "groups": [{"id": group.id}], - }, - ) - resource = p.toolkit.get_action("resource_create")( - context, - { - "name": "privateresource", - "url": "https://www.example.com/", - "package_id": package["id"], - }, - ) - - auth = {"Authorization": self.sysadmin_token} - helpers.call_action( - "datastore_create", resource_id=resource["id"], force=True - ) - - # new resource should be private - query = 'SELECT * FROM "{0}"'.format(resource["id"]) - data = {"sql": query} - auth = {"Authorization": self.normal_user_token} - res = app.post( - "/api/action/datastore_search_sql", - json=data, - extra_environ=auth, - status=403, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"]["__type"] == "Authorization Error" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_not_authorized_to_access_system_tables(self, app): - test_cases = [ - "SELECT * FROM pg_roles", - "SELECT * FROM pg_catalog.pg_database", - "SELECT rolpassword FROM pg_roles", - """SELECT p.rolpassword - FROM pg_roles p - JOIN "{0}" r - ON p.rolpassword = r.author""".format( - self.data["resource_id"] - ), - ] - for query in test_cases: - data = {"sql": query.replace("\n", "")} - res = app.post( - "/api/action/datastore_search_sql", json=data, status=403, - ) - res_dict = json.loads(res.data) - assert res_dict["success"] is False - assert res_dict["error"]["__type"] == "Authorization Error" - - -class TestDatastoreSQLFunctional(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures( - "clean_datastore", "with_plugins", "with_request_context" - ) - def test_search_sql_enforces_private(self): - user1 = factories.User() - user2 = factories.User() - user3 = factories.User() - ctx1 = {u"user": user1["name"], u"ignore_auth": False} - ctx2 = {u"user": user2["name"], u"ignore_auth": False} - ctx3 = {u"user": user3["name"], u"ignore_auth": False} - - org1 = factories.Organization( - user=user1, - users=[{u"name": user3["name"], u"capacity": u"member"}], - ) - org2 = factories.Organization( - user=user2, - users=[{u"name": user3["name"], u"capacity": u"member"}], - ) - ds1 = factories.Dataset(owner_org=org1["id"], private=True) - ds2 = factories.Dataset(owner_org=org2["id"], private=True) - r1 = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds1["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - ) - r2 = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds2["id"]}, - fields=[{u"id": u"ham", u"type": u"text"}], - ) - - sql1 = 'SELECT spam FROM "{0}"'.format(r1["resource_id"]) - sql2 = 'SELECT ham FROM "{0}"'.format(r2["resource_id"]) - sql3 = 'SELECT spam, ham FROM "{0}", "{1}"'.format( - r1["resource_id"], r2["resource_id"] - ) - - with pytest.raises(p.toolkit.NotAuthorized): - helpers.call_action("datastore_search_sql", context=ctx2, sql=sql1) - with pytest.raises(p.toolkit.NotAuthorized): - helpers.call_action("datastore_search_sql", context=ctx1, sql=sql2) - with pytest.raises(p.toolkit.NotAuthorized): - helpers.call_action("datastore_search_sql", context=ctx1, sql=sql3) - with pytest.raises(p.toolkit.NotAuthorized): - helpers.call_action("datastore_search_sql", context=ctx2, sql=sql3) - helpers.call_action("datastore_search_sql", context=ctx1, sql=sql1) - helpers.call_action("datastore_search_sql", context=ctx2, sql=sql2) - helpers.call_action("datastore_search_sql", context=ctx3, sql=sql3) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_validates_sql_has_a_single_statement(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"the year": 2014}, {"the year": 2013}], - } - helpers.call_action("datastore_create", **data) - sql = 'SELECT * FROM public."{0}"; SELECT * FROM public."{0}";'.format( - resource["id"] - ) - with pytest.raises( - p.toolkit.ValidationError, match="Query is not a single statement" - ): - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_works_with_semicolons_inside_strings(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"author": "bob"}, {"author": "jane"}], - } - helpers.call_action("datastore_create", **data) - sql = 'SELECT * FROM public."{0}" WHERE "author" = \'foo; bar\''.format( - resource["id"] - ) - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_works_with_allowed_functions(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"author": "bob"}, {"author": "jane"}], - } - helpers.call_action("datastore_create", **data) - - sql = 'SELECT upper(author) from "{}"'.format( - resource["id"] - ) - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_not_authorized_with_disallowed_functions(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"author": "bob"}, {"author": "jane"}], - } - helpers.call_action("datastore_create", **data) - - sql = "SELECT query_to_xml('SELECT upper(author) from \"{}\"', true, true, '')".format( - resource["id"] - ) - with pytest.raises(p.toolkit.NotAuthorized): - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_allowed_functions_are_case_insensitive(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"author": "bob"}, {"author": "jane"}], - } - helpers.call_action("datastore_create", **data) - - sql = 'SELECT UpPeR(author) from "{}"'.format( - resource["id"] - ) - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_quoted_allowed_functions_are_case_sensitive(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [{"author": "bob"}, {"author": "jane"}], - } - helpers.call_action("datastore_create", **data) - - sql = 'SELECT count(*) from "{}"'.format( - resource["id"] - ) - helpers.call_action("datastore_search_sql", sql=sql) - - sql = 'SELECT CoUnT(*) from "{}"'.format( - resource["id"] - ) - with pytest.raises(p.toolkit.NotAuthorized): - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_invalid_statement(self): - sql = "SELECT ** FROM foobar" - with pytest.raises( - logic.ValidationError, match='syntax error at or near "FROM"' - ): - helpers.call_action("datastore_search_sql", sql=sql) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_select_basic(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - }, - ], - } - expected_records = [ - { - u"_full_text": [ - u"'annakarenina'", - u"'b'", - u"'moo'", - u"'tolstoy'", - u"'2005'", - ], - u"_id": 1, - u"author": u"tolstoy", - u"b\xfck": u"annakarenina", - u"nested": [u"b", {u"moo": u"moo"}], - u"published": u"2005-03-01T00:00:00", - }, - { - u"_full_text": [u"'tolstoy'", u"'warandpeac'", u"'b'"], - u"_id": 2, - u"author": u"tolstoy", - u"b\xfck": u"warandpeace", - u"nested": {u"a": u"b"}, - u"published": None, - }, - ] - helpers.call_action("datastore_create", **data) - sql = 'SELECT * FROM "{0}"'.format(resource["id"]) - result = helpers.call_action("datastore_search_sql", sql=sql) - assert len(result["records"]) == 2 - for (row_index, row) in enumerate(result["records"]): - expected_row = expected_records[row_index] - assert set(row.keys()) == set(expected_row.keys()) - for field in row: - if field == "_full_text": - for ft_value in expected_row["_full_text"]: - assert ft_value in row["_full_text"] - else: - assert row[field] == expected_row[field] - assert u"records_truncated" not in result - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_alias_search(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "aliases": "books4", - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - }, - ], - } - helpers.call_action("datastore_create", **data) - sql = 'SELECT * FROM "{0}"'.format(resource["id"]) - result = helpers.call_action("datastore_search_sql", sql=sql) - sql = 'SELECT * FROM "books4"' - result_with_alias = helpers.call_action( - "datastore_search_sql", sql=sql - ) - assert result["records"] == result_with_alias["records"] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "2") - def test_search_limit(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"the year": 2014}, - {"the year": 2013}, - {"the year": 2015}, - {"the year": 2016}, - ], - } - result = helpers.call_action("datastore_create", **data) - sql = 'SELECT * FROM "{0}"'.format(resource["id"]) - result = helpers.call_action("datastore_search_sql", sql=sql) - assert len(result["records"]) == 2 - assert [res[u"the year"] for res in result["records"]] == [2014, 2013] - assert result[u"records_truncated"] - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreSearchRecordsFormat(object): - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_sort_results_objects(self): - ds = factories.Dataset() - r = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[ - {u"id": u"num", u"type": u"numeric"}, - {u"id": u"dt", u"type": u"timestamp"}, - {u"id": u"txt", u"type": u"text"}, - ], - records=[ - {u"num": 10, u"dt": u"2020-01-01", u"txt": "aaab"}, - {u"num": 9, u"dt": u"2020-01-02", u"txt": "aaab"}, - {u"num": 9, u"dt": u"2020-01-01", u"txt": "aaac"}, - ], - ) - assert helpers.call_action( - "datastore_search", resource_id=r["resource_id"], sort=u"num, dt" - )["records"] == [ - { - u"_id": 3, - u"num": 9, - u"dt": u"2020-01-01T00:00:00", - u"txt": u"aaac", - }, - { - u"_id": 2, - u"num": 9, - u"dt": u"2020-01-02T00:00:00", - u"txt": u"aaab", - }, - { - u"_id": 1, - u"num": 10, - u"dt": u"2020-01-01T00:00:00", - u"txt": u"aaab", - }, - ] - assert helpers.call_action( - "datastore_search", resource_id=r["resource_id"], sort=u"dt, txt" - )["records"] == [ - { - u"_id": 1, - u"num": 10, - u"dt": u"2020-01-01T00:00:00", - u"txt": u"aaab", - }, - { - u"_id": 3, - u"num": 9, - u"dt": u"2020-01-01T00:00:00", - u"txt": u"aaac", - }, - { - u"_id": 2, - u"num": 9, - u"dt": u"2020-01-02T00:00:00", - u"txt": u"aaab", - }, - ] - assert helpers.call_action( - "datastore_search", resource_id=r["resource_id"], sort=u"txt, num" - )["records"] == [ - { - u"_id": 2, - u"num": 9, - u"dt": u"2020-01-02T00:00:00", - u"txt": u"aaab", - }, - { - u"_id": 1, - u"num": 10, - u"dt": u"2020-01-01T00:00:00", - u"txt": u"aaab", - }, - { - u"_id": 3, - u"num": 9, - u"dt": u"2020-01-01T00:00:00", - u"txt": u"aaac", - }, - ] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_sort_results_lists(self): - ds = factories.Dataset() - r = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[ - {u"id": u"num", u"type": u"numeric"}, - {u"id": u"dt", u"type": u"timestamp"}, - {u"id": u"txt", u"type": u"text"}, - ], - records=[ - {u"num": 10, u"dt": u"2020-01-01", u"txt": u"aaab"}, - {u"num": 9, u"dt": u"2020-01-02", u"txt": u"aaab"}, - {u"num": 9, u"dt": u"2020-01-01", u"txt": u"aaac"}, - ], - ) - assert helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"lists", - sort=u"num, dt", - )["records"] == [ - [3, 9, u"2020-01-01T00:00:00", u"aaac"], - [2, 9, u"2020-01-02T00:00:00", u"aaab"], - [1, 10, u"2020-01-01T00:00:00", u"aaab"], - ] - assert helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"lists", - sort=u"dt, txt", - )["records"] == [ - [1, 10, u"2020-01-01T00:00:00", u"aaab"], - [3, 9, u"2020-01-01T00:00:00", u"aaac"], - [2, 9, u"2020-01-02T00:00:00", u"aaab"], - ] - assert helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"lists", - sort=u"txt, num", - )["records"] == [ - [2, 9, u"2020-01-02T00:00:00", u"aaab"], - [1, 10, u"2020-01-01T00:00:00", u"aaab"], - [3, 9, u"2020-01-01T00:00:00", u"aaac"], - ] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_sort_results_csv(self): - ds = factories.Dataset() - r = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[ - {u"id": u"num", u"type": u"numeric"}, - {u"id": u"dt", u"type": u"timestamp"}, - {u"id": u"txt", u"type": u"text"}, - ], - records=[ - {u"num": 10, u"dt": u"2020-01-01", u"txt": u"aaab"}, - {u"num": 9, u"dt": u"2020-01-02", u"txt": u"aaab"}, - {u"num": 9, u"dt": u"2020-01-01", u"txt": u"aaac"}, - ], - ) - assert ( - helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"csv", - sort=u"num, dt", - )["records"] - == u"3,9,2020-01-01T00:00:00,aaac\n" - u"2,9,2020-01-02T00:00:00,aaab\n" - u"1,10,2020-01-01T00:00:00,aaab\n" - ) - assert ( - helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"csv", - sort=u"dt, txt", - )["records"] - == u"1,10,2020-01-01T00:00:00,aaab\n" - u"3,9,2020-01-01T00:00:00,aaac\n" - u"2,9,2020-01-02T00:00:00,aaab\n" - ) - assert ( - helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"csv", - sort=u"txt, num", - )["records"] - == u"2,9,2020-01-02T00:00:00,aaab\n" - u"1,10,2020-01-01T00:00:00,aaab\n" - u"3,9,2020-01-01T00:00:00,aaac\n" - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_fields_results_csv(self): - ds = factories.Dataset() - r = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[ - {u"id": u"num", u"type": u"numeric"}, - {u"id": u"dt", u"type": u"timestamp"}, - {u"id": u"txt", u"type": u"text"}, - ], - records=[ - {u"num": 9, u"dt": u"2020-01-02", u"txt": u"aaab"}, - {u"num": 9, u"dt": u"2020-01-01", u"txt": u"aaac"}, - ], - ) - r = helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"csv", - fields=u"dt, num, txt", - ) - assert r["fields"] == [ - {u"id": u"dt", u"type": u"timestamp"}, - {u"id": u"num", u"type": u"numeric"}, - {u"id": u"txt", u"type": u"text"}, - ] - assert ( - r["records"] == u"2020-01-02T00:00:00,9,aaab\n" - u"2020-01-01T00:00:00,9,aaac\n" - ) - r = helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"csv", - fields=u"dt", - q=u"aaac", - ) - assert r["fields"] == [{u"id": u"dt", u"type": u"timestamp"}] - assert r["records"] == u"2020-01-01T00:00:00\n" - r = helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"csv", - fields=u"txt, rank txt", - q={u"txt": u"aaac"}, - ) - assert r["fields"] == [ - {u"id": u"txt", u"type": u"text"}, - {u"id": u"rank txt", u"type": u"float"}, - ] - assert r["records"][:7] == u"aaac,0." - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_fts_on_field_calculates_ranks_specific_field_and_all_fields(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"from": "Brazil", "to": "Brazil"}, - {"from": "Brazil", "to": "Italy"}, - ], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "fields": "from, rank from", - "full_text": "Brazil", - "q": {"from": "Brazil"}, - } - result = helpers.call_action("datastore_search", **search_data) - ranks_from = [r["rank from"] for r in result["records"]] - assert len(result["records"]) == 2 - assert len(set(ranks_from)) == 1 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_fts_on_field_calculates_ranks_when_q_string_and_fulltext_is_given(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"from": "Brazil", "to": "Brazil"}, - {"from": "Brazil", "to": "Italy"}, - ], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "full_text": "Brazil", - "q": "Brazil", - } - result = helpers.call_action("datastore_search", **search_data) - ranks = [r["rank"] for r in result["records"]] - assert len(result["records"]) == 2 - assert len(set(ranks)) == 2 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_fts_on_field_calculates_ranks_when_full_text_is_given(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "records": [ - {"from": "Brazil", "to": "Brazil"}, - {"from": "Brazil", "to": "Italy"}, - ], - } - result = helpers.call_action("datastore_create", **data) - search_data = { - "resource_id": resource["id"], - "full_text": "Brazil", - } - result = helpers.call_action("datastore_search", **search_data) - ranks = [r["rank"] for r in result["records"]] - assert len(result["records"]) == 2 - assert len(set(ranks)) == 2 - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_results_with_nulls(self): - ds = factories.Dataset() - r = helpers.call_action( - "datastore_create", - resource={"package_id": ds["id"]}, - fields=[ - {"id": "num", "type": "numeric"}, - {"id": "dt", "type": "timestamp"}, - {"id": "txt", "type": "text"}, - {"id": "lst", "type": "_text"}, - ], - records=[ - {"num": 10, "dt": "2020-01-01", "txt": "aaab", "lst": ["one", "two"]}, - {"num": 9, "dt": "2020-01-02", "txt": "aaab"}, - {"num": 9, "txt": "aaac", "lst": ["one", "two"]}, - {}, # all nulls - ], - ) - assert helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"lists", - sort=u"num nulls last, dt nulls last", - )["records"] == [ - [2, 9, "2020-01-02T00:00:00", "aaab", None], - [3, 9, None, "aaac", ["one", "two"]], - [1, 10, "2020-01-01T00:00:00", "aaab", ["one", "two"]], - [4, None, None, None, None], - ] - assert helpers.call_action( - "datastore_search", - resource_id=r["resource_id"], - records_format=u"objects", - sort=u"num nulls last, dt nulls last", - )["records"] == [ - {"_id": 2, "num": 9, "dt": "2020-01-02T00:00:00", "txt": "aaab", "lst": None}, - {"_id": 3, "num": 9, "dt": None, "txt": "aaac", "lst": ["one", "two"]}, - {"_id": 1, "num": 10, "dt": "2020-01-01T00:00:00", "txt": "aaab", "lst": ["one", "two"]}, - {"_id": 4, "num": None, "dt": None, "txt": None, "lst": None}, - ] diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py deleted file mode 100644 index 9c25795..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_unit.py +++ /dev/null @@ -1,39 +0,0 @@ -# encoding: utf-8 - -import ckanext.datastore.backend.postgres as backend -import ckanext.datastore.backend.postgres as db -import ckanext.datastore.helpers as helpers -from ckan.common import config - - -postgres_backend = backend.DatastorePostgresqlBackend() -postgres_backend.configure(config) - - -def test_is_valid_field_name(): - assert helpers.is_valid_field_name("foo") - assert helpers.is_valid_field_name("foo bar") - assert helpers.is_valid_field_name("42") - assert not helpers.is_valid_field_name('foo"bar') - assert not helpers.is_valid_field_name('"') - assert helpers.is_valid_field_name("'") - assert not helpers.is_valid_field_name("") - assert helpers.is_valid_field_name("foo%bar") - - -def test_is_valid_table_name(): - assert helpers.is_valid_table_name("foo") - assert helpers.is_valid_table_name("foo bar") - assert helpers.is_valid_table_name("42") - assert not helpers.is_valid_table_name('foo"bar') - assert not helpers.is_valid_table_name('"') - assert helpers.is_valid_table_name("'") - assert not helpers.is_valid_table_name("") - assert not helpers.is_valid_table_name("foo%bar") - - -def test_pg_version_check(): - engine = db._get_engine_from_url(config["sqlalchemy.url"]) - connection = engine.connect() - assert db._pg_version_is_at_least(connection, "8.0") - assert not db._pg_version_is_at_least(connection, "20.0") diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py deleted file mode 100644 index 32a07e3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/tests/test_upsert.py +++ /dev/null @@ -1,953 +0,0 @@ -# encoding: utf-8 - -import pytest - -import ckan.tests.factories as factories -import ckan.tests.helpers as helpers -from ckan.plugins.toolkit import ValidationError, NotAuthorized -from ckanext.datastore.tests.helpers import when_was_last_analyze - - -def _search(resource_id): - return helpers.call_action(u"datastore_search", resource_id=resource_id) - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreUpsert(object): - # Test action 'datastore_upsert' with 'method': 'upsert' - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_requires_auth(self): - resource = factories.Resource(url_type=u"datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - ], - "records": [], - } - helpers.call_action("datastore_create", **data) - - data = {"resource_id": resource["id"]} - with pytest.raises(NotAuthorized) as context: - helpers.call_action( - "datastore_upsert", - context={"user": "", "ignore_auth": False}, - **data - ) - assert ( - u"Action datastore_upsert requires an authenticated user" - in str(context.value) - ) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_empty_fails(self): - resource = factories.Resource(url_type=u"datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - ], - "records": [], - } - helpers.call_action("datastore_create", **data) - - data = {} # empty - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u"'Missing value'" in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_basic_as_update(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [ - {"id": "1", "book": u"The boy", "author": u"F Torres"} - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["book"] == "The boy" - assert search_result["records"][0]["author"] == "F Torres" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_basic_as_insert(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [ - {"id": "2", "book": u"The boy", "author": u"F Torres"} - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 2 - assert search_result["records"][0]["book"] == u"El Niño" - assert search_result["records"][1]["book"] == u"The boy" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_only_one_field(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [ - {"id": "1", "book": u"The boy"} - ], # not changing the author - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["book"] == "The boy" - assert search_result["records"][0]["author"] == "Torres" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_field_types(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"b\xfck", - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "nested", "type": "json"}, - {"id": "characters", "type": "text[]"}, - {"id": "published"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - }, - { - "author": "adams", - "characters": ["Arthur", "Marvin"], - "nested": {"foo": "bar"}, - u"b\xfck": u"guide to the galaxy", - }, - ], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "upsert", - "records": [ - { - "author": "adams", - "characters": ["Bob", "Marvin"], - "nested": {"baz": 3}, - u"b\xfck": u"guide to the galaxy", - } - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 3 - assert ( - search_result["records"][0]["published"] == u"2005-03-01T00:00:00" - ) # i.e. stored in db as datetime - assert search_result["records"][2]["author"] == "adams" - assert search_result["records"][2]["characters"] == ["Bob", "Marvin"] - assert search_result["records"][2]["nested"] == {"baz": 3} - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_percent(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "bo%ok", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1%", "bo%ok": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [ - {"id": "1%", "bo%ok": u"The % boy", "author": u"F Torres"}, - {"id": "2%", "bo%ok": u"Gu%ide", "author": u"Adams"}, - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 2 - assert search_result["records"][0]["bo%ok"] == "The % boy" - assert search_result["records"][1]["bo%ok"] == "Gu%ide" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_missing_key(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": "guide", "author": "adams"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [{"book": u"El Niño", "author": "Torres"}], # no key - } - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u'fields "id" are missing' in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_non_existing_field(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "upsert", - "records": [{"id": "1", "dummy": "tolkien"}], # key not known - } - - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u'fields "dummy" do not exist' in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_works_with_empty_list_in_json_field(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "nested", "type": "json"}, - ], - "records": [{"id": "1", "nested": {"foo": "bar"}}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [{"id": "1", "nested": []}], # empty list - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["nested"] == [] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_delete_field_value(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [{"id": "1", "book": None}], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["book"] is None - assert search_result["records"][0]["author"] == "Torres" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_doesnt_crash_with_json_field(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "json"}, - {"id": "author", "type": "text"}, - ], - } - helpers.call_action("datastore_create", **data) - data = { - "resource_id": resource["id"], - "force": True, - "method": "insert", - "records": [ - { - "id": "1", - "book": {"code": "A", "title": u"ñ"}, - "author": "tolstoy", - } - ], - } - helpers.call_action("datastore_upsert", **data) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_upsert_doesnt_crash_with_json_field_with_string_value(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "json"}, - {"id": "author", "type": "text"}, - ], - } - helpers.call_action("datastore_create", **data) - data = { - "resource_id": resource["id"], - "force": True, - "method": "insert", - "records": [{"id": "1", "book": u"ñ", "author": "tolstoy"}], - } - helpers.call_action("datastore_upsert", **data) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dry_run(self): - ds = factories.Dataset() - table = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - primary_key=u"spam", - ) - helpers.call_action( - u"datastore_upsert", - resource_id=table["resource_id"], - records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}], - dry_run=True, - ) - result = helpers.call_action( - u"datastore_search", resource_id=table["resource_id"] - ) - assert result["records"] == [] - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dry_run_type_error(self): - ds = factories.Dataset() - table = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"numeric"}], - primary_key=u"spam", - ) - try: - helpers.call_action( - u"datastore_upsert", - resource_id=table["resource_id"], - records=[{u"spam": u"SPAM"}, {u"spam": u"EGGS"}], - dry_run=True, - ) - except ValidationError as ve: - assert ve.error_dict["records"] == [ - u'invalid input syntax for type numeric: "SPAM"' - ] - else: - assert 0, "error not raised" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_dry_run_trigger_error(self): - ds = factories.Dataset() - helpers.call_action( - u"datastore_function_create", - name=u"spamexception_trigger", - rettype=u"trigger", - definition=u""" - BEGIN - IF NEW.spam != 'spam' THEN - RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; - END IF; - RETURN NEW; - END;""", - ) - table = helpers.call_action( - u"datastore_create", - resource={u"package_id": ds["id"]}, - fields=[{u"id": u"spam", u"type": u"text"}], - primary_key=u"spam", - triggers=[{u"function": u"spamexception_trigger"}], - ) - try: - helpers.call_action( - u"datastore_upsert", - resource_id=table["resource_id"], - records=[{u"spam": u"EGGS"}], - dry_run=True, - ) - except ValidationError as ve: - assert ve.error_dict["records"] == [u'"EGGS"? Yeeeeccch!'] - else: - assert 0, "error not raised" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_calculate_record_count_is_false(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": "name", "type": "text"}, - {"id": "age", "type": "text"}, - ], - } - helpers.call_action("datastore_create", **data) - data = { - "resource_id": resource["id"], - "force": True, - "method": "insert", - "records": [ - {"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}, - ], - } - helpers.call_action("datastore_upsert", **data) - last_analyze = when_was_last_analyze(resource["id"]) - assert last_analyze is None - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - @pytest.mark.flaky(reruns=2) # because analyze is sometimes delayed - def test_calculate_record_count(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": "name", "type": "text"}, - {"id": "age", "type": "text"}, - ], - } - helpers.call_action("datastore_create", **data) - data = { - "resource_id": resource["id"], - "force": True, - "method": "insert", - "records": [ - {"name": "Sunita", "age": "51"}, - {"name": "Bowan", "age": "68"}, - ], - "calculate_record_count": True, - } - helpers.call_action("datastore_upsert", **data) - last_analyze = when_was_last_analyze(resource["id"]) - assert last_analyze is not None - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_no_pk_update(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": "book", "type": "text"}, - ], - "records": [{"book": u"El Niño"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [ - {"_id": "1", "book": u"The boy"} - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["book"] == "The boy" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_id_instead_of_pk_update(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "pk", - "fields": [ - {"id": "pk", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"pk": "1000", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "upsert", - "records": [ - {"_id": "1", "book": u"The boy", "author": u"F Torres"} - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["pk"] == "1000" - assert search_result["records"][0]["book"] == "The boy" - assert search_result["records"][0]["author"] == "F Torres" - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreInsert(object): - # Test action 'datastore_upsert' with 'method': 'insert' - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_basic_insert(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "insert", - "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["fields"] == [ - {u"id": "_id", u"type": "int"}, - {u"id": u"id", u"type": u"text"}, - {u"id": u"book", u"type": u"text"}, - {u"id": u"author", u"type": u"text"}, - ] - assert search_result["records"][0] == { - u"book": u"El Ni\xf1o", - u"_id": 1, - u"id": u"1", - u"author": u"Torres", - } - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_non_existing_field(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "insert", - "records": [{"id": "1", "dummy": "tolkien"}], # key not known - } - - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u'row "1" has extra keys "dummy"' in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_key_already_exists(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": "guide", "author": "adams"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "insert", - "records": [ - { - "id": "1", # already exists - "book": u"El Niño", - "author": "Torres", - } - ], - } - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u"duplicate key value violates unique constraint" in str( - context.value - ) - - -@pytest.mark.usefixtures("with_request_context") -class TestDatastoreUpdate(object): - # Test action 'datastore_upsert' with 'method': 'update' - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_basic(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "update", - "records": [{"id": "1", "book": u"The boy"}], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["book"] == "The boy" - assert search_result["records"][0]["author"] == "Torres" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_field_types(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"b\xfck", - "fields": [ - {"id": u"b\xfck", "type": "text"}, - {"id": "author", "type": "text"}, - {"id": "nested", "type": "json"}, - {"id": "characters", "type": "text[]"}, - {"id": "published"}, - ], - "records": [ - { - u"b\xfck": "annakarenina", - "author": "tolstoy", - "published": "2005-03-01", - "nested": ["b", {"moo": "moo"}], - }, - { - u"b\xfck": "warandpeace", - "author": "tolstoy", - "nested": {"a": "b"}, - }, - { - "author": "adams", - "characters": ["Arthur", "Marvin"], - "nested": {"foo": "bar"}, - u"b\xfck": u"guide to the galaxy", - }, - ], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "update", - "records": [ - { - "author": "adams", - "characters": ["Bob", "Marvin"], - "nested": {"baz": 3}, - u"b\xfck": u"guide to the galaxy", - } - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 3 - assert search_result["records"][2]["author"] == "adams" - assert search_result["records"][2]["characters"] == ["Bob", "Marvin"] - assert search_result["records"][2]["nested"] == {"baz": 3} - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_update_unspecified_key(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "update", - "records": [{"author": "tolkien"}], # no id - } - - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u'fields "id" are missing' in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_update_unknown_key(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "update", - "records": [{"id": "1", "author": "tolkien"}], # unknown - } - - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u"key \"[\\'1\\']\" not found" in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_update_non_existing_field(self): - resource = factories.Resource(url_type="datastore") - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": u"id", - "fields": [ - {"id": "id", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"id": "1", "book": "guide"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "method": "update", - "records": [{"id": "1", "dummy": "tolkien"}], # key not known - } - - with pytest.raises(ValidationError) as context: - helpers.call_action("datastore_upsert", **data) - assert u'fields "dummy" do not exist' in str(context.value) - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_no_pk_update(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "fields": [ - {"id": "book", "type": "text"}, - ], - "records": [{"book": u"El Niño"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "update", - "records": [ - {"_id": "1", "book": u"The boy"} - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["book"] == "The boy" - - @pytest.mark.ckan_config("ckan.plugins", "datastore") - @pytest.mark.usefixtures("clean_datastore", "with_plugins") - def test_id_instead_of_pk_update(self): - resource = factories.Resource() - data = { - "resource_id": resource["id"], - "force": True, - "primary_key": "pk", - "fields": [ - {"id": "pk", "type": "text"}, - {"id": "book", "type": "text"}, - {"id": "author", "type": "text"}, - ], - "records": [{"pk": "1000", "book": u"El Niño", "author": "Torres"}], - } - helpers.call_action("datastore_create", **data) - - data = { - "resource_id": resource["id"], - "force": True, - "method": "update", - "records": [ - {"_id": "1", "book": u"The boy", "author": u"F Torres"} - ], - } - helpers.call_action("datastore_upsert", **data) - - search_result = _search(resource["id"]) - assert search_result["total"] == 1 - assert search_result["records"][0]["pk"] == "1000" - assert search_result["records"][0]["book"] == "The boy" - assert search_result["records"][0]["author"] == "F Torres" diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/writer.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/writer.py deleted file mode 100644 index b46c42c..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datastore/writer.py +++ /dev/null @@ -1,179 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from io import StringIO, BytesIO - -from contextlib import contextmanager -from typing import Any, Optional -from simplejson import dumps - -from xml.etree.cElementTree import Element, SubElement, ElementTree - -import csv - -from codecs import BOM_UTF8 - - -BOM = "\N{bom}" - - -@contextmanager -def csv_writer(fields: list[dict[str, Any]], bom: bool = False): - '''Context manager for writing UTF-8 CSV data to file - - :param fields: list of datastore fields - :param bom: True to include a UTF-8 BOM at the start of the file - ''' - output = StringIO() - - if bom: - output.write(BOM) - - csv.writer(output).writerow( - f['id'] for f in fields) - yield TextWriter(output) - - -@contextmanager -def tsv_writer(fields: list[dict[str, Any]], bom: bool = False): - '''Context manager for writing UTF-8 TSV data to file - - :param fields: list of datastore fields - :param bom: True to include a UTF-8 BOM at the start of the file - ''' - output = StringIO() - - if bom: - output.write(BOM) - - csv.writer( - output, - dialect='excel-tab').writerow( - f['id'] for f in fields) - yield TextWriter(output) - - -class TextWriter(object): - u'text in, text out' - def __init__(self, output: StringIO): - self.output = output - - def write_records(self, records: list[Any]) -> bytes: - self.output.write(records) # type: ignore - self.output.seek(0) - output = self.output.read().encode('utf-8') - self.output.truncate(0) - self.output.seek(0) - return output - - def end_file(self) -> bytes: - return b'' - - -@contextmanager -def json_writer(fields: list[dict[str, Any]], bom: bool = False): - '''Context manager for writing UTF-8 JSON data to file - - :param fields: list of datastore fields - :param bom: True to include a UTF-8 BOM at the start of the file - ''' - output = StringIO() - - if bom: - output.write(BOM) - - output.write( - '{\n "fields": %s,\n "records": [' % dumps( - fields, ensure_ascii=False, separators=(u',', u':'))) - yield JSONWriter(output) - - -class JSONWriter(object): - def __init__(self, output: StringIO): - self.output = output - self.first = True - - def write_records(self, records: list[Any]) -> bytes: - for r in records: - if self.first: - self.first = False - self.output.write('\n ') - else: - self.output.write(',\n ') - - self.output.write(dumps( - r, ensure_ascii=False, separators=(',', ':'))) - - self.output.seek(0) - output = self.output.read().encode('utf-8') - self.output.truncate(0) - self.output.seek(0) - return output - - def end_file(self) -> bytes: - return b'\n]}\n' - - -@contextmanager -def xml_writer(fields: list[dict[str, Any]], bom: bool = False): - '''Context manager for writing UTF-8 XML data to file - - :param fields: list of datastore fields - :param bom: True to include a UTF-8 BOM at the start of the file - ''' - output = BytesIO() - - if bom: - output.write(BOM_UTF8) - - output.write( - b'\n') - yield XMLWriter(output, [f[u'id'] for f in fields]) - - -class XMLWriter(object): - _key_attr = 'key' - _value_tag = 'value' - - def __init__(self, output: BytesIO, columns: list[str]): - self.output = output - self.id_col = columns[0] == '_id' - if self.id_col: - columns = columns[1:] - self.columns = columns - - def _insert_node(self, root: Any, k: str, v: Any, - key_attr: Optional[Any] = None): - element = SubElement(root, k) - if v is None: - element.attrib['xsi:nil'] = 'true' - elif not isinstance(v, (list, dict)): - element.text = str(v) - else: - if isinstance(v, list): - it = enumerate(v) - else: - it = v.items() - for key, value in it: - self._insert_node(element, self._value_tag, value, key) - - if key_attr is not None: - element.attrib[self._key_attr] = str(key_attr) - - def write_records(self, records: list[Any]) -> bytes: - for r in records: - root = Element('row') - if self.id_col: - root.attrib['_id'] = str(r['_id']) - for c in self.columns: - self._insert_node(root, c, r[c]) - ElementTree(root).write(self.output, encoding='utf-8') - self.output.write(b'\n') - self.output.seek(0) - output = self.output.read() - self.output.truncate(0) - self.output.seek(0) - return output - - def end_file(self) -> bytes: - return b'\n' diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/__init__.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py deleted file mode 100644 index 45e2298..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/blueprint.py +++ /dev/null @@ -1,215 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from typing import Any -from urllib.parse import urlencode - -from flask import Blueprint - - -from ckan.common import json -from ckan.lib.helpers import decode_view_request_filters -from ckan.plugins.toolkit import get_action, request, h -import re - -datatablesview = Blueprint(u'datatablesview', __name__) - - -def merge_filters(view_filters: dict[str, Any], - user_filters: dict[str, Any] | None) -> dict[str, Any]: - u''' - view filters are built as part of the view, user filters - are selected by the user interacting with the view. Any filters - selected by user may only tighten filters set in the view, - others are ignored. - - >>> merge_filters({ - ... u'Department': [u'BTDT'], u'OnTime_Status': [u'ONTIME']}, - ... u'CASE_STATUS:Open|CASE_STATUS:Closed|Department:INFO') - {u'Department': [u'BTDT'], - u'OnTime_Status': [u'ONTIME'], - u'CASE_STATUS': [u'Open', u'Closed']} - ''' - filters = dict(view_filters) - if not user_filters: - return filters - combined_user_filters = {} - for k in user_filters: - if k not in view_filters or user_filters[k] in view_filters[k]: - combined_user_filters[k] = user_filters[k] - else: - combined_user_filters[k] = user_filters[k] + view_filters[k] - for k in combined_user_filters: - filters[k] = combined_user_filters[k] - return filters - - -def ajax(resource_view_id: str): - resource_view = get_action(u'resource_view_show' - )({}, { - u'id': resource_view_id - }) - - draw = int(request.form[u'draw']) - search_text = str(request.form[u'search[value]']) - offset = int(request.form[u'start']) - limit = int(request.form[u'length']) - view_filters = resource_view.get(u'filters', {}) - user_filters = decode_view_request_filters() - filters = merge_filters(view_filters, user_filters) - - datastore_search = get_action(u'datastore_search') - unfiltered_response = datastore_search( - {}, { - u"resource_id": resource_view[u'resource_id'], - u"limit": 0, - u"filters": view_filters, - } - ) - - cols = [f[u'id'] for f in unfiltered_response[u'fields']] - if u'show_fields' in resource_view: - cols = [c for c in cols if c in resource_view[u'show_fields']] - - sort_list = [] - i = 0 - while True: - if u'order[%d][column]' % i not in request.form: - break - sort_by_num = int(request.form[u'order[%d][column]' % i]) - sort_order = ( - u'desc' if request.form[u'order[%d][dir]' % - i] == u'desc' else u'asc' - ) - sort_list.append(cols[sort_by_num] + u' ' + sort_order) - i += 1 - - colsearch_dict = {} - i = 0 - while True: - if u'columns[%d][search][value]' % i not in request.form: - break - v = str(request.form[u'columns[%d][search][value]' % i]) - if v: - k = str(request.form[u'columns[%d][name]' % i]) - # replace non-alphanumeric characters with FTS wildcard (_) - v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v) - # append ':*' so we can do partial FTS searches - colsearch_dict[k] = v + u':*' - i += 1 - - if colsearch_dict: - search_text = json.dumps(colsearch_dict) - else: - search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_', - search_text) + u':*' if search_text else u'' - - try: - response = datastore_search( - {}, { - u"q": search_text, - u"resource_id": resource_view[u'resource_id'], - u'plain': False, - u'language': u'simple', - u"offset": offset, - u"limit": limit, - u"sort": u', '.join(sort_list), - u"filters": filters, - } - ) - except Exception: - query_error = u'Invalid search query... ' + search_text - dtdata = {u'error': query_error} - else: - data = [] - for row in response[u'records']: - record = {colname: str(row.get(colname, u'')) - for colname in cols} - # the DT_RowId is used in DT to set an element id for each record - record['DT_RowId'] = 'row' + str(row.get(u'_id', u'')) - data.append(record) - - dtdata = { - u'draw': draw, - u'recordsTotal': unfiltered_response.get(u'total', 0), - u'recordsFiltered': response.get(u'total', 0), - u'data': data - } - - return json.dumps(dtdata) - - -def filtered_download(resource_view_id: str): - params = json.loads(request.form[u'params']) - resource_view = get_action(u'resource_view_show' - )({}, { - u'id': resource_view_id - }) - - search_text = str(params[u'search'][u'value']) - view_filters = resource_view.get(u'filters', {}) - user_filters = decode_view_request_filters() - filters = merge_filters(view_filters, user_filters) - - datastore_search = get_action(u'datastore_search') - unfiltered_response = datastore_search( - {}, { - u"resource_id": resource_view[u'resource_id'], - u"limit": 0, - u"filters": view_filters, - } - ) - - cols = [f[u'id'] for f in unfiltered_response[u'fields']] - if u'show_fields' in resource_view: - cols = [c for c in cols if c in resource_view[u'show_fields']] - - sort_list = [] - for order in params[u'order']: - sort_by_num = int(order[u'column']) - sort_order = (u'desc' if order[u'dir'] == u'desc' else u'asc') - sort_list.append(cols[sort_by_num] + u' ' + sort_order) - - cols = [c for (c, v) in zip(cols, params[u'visible']) if v] - - colsearch_dict = {} - columns = params[u'columns'] - for column in columns: - if column[u'search'][u'value']: - v = column[u'search'][u'value'] - if v: - k = column[u'name'] - # replace non-alphanumeric characters with FTS wildcard (_) - v = re.sub(r'[^0-9a-zA-Z\-]+', '_', v) - # append ':*' so we can do partial FTS searches - colsearch_dict[k] = v + u':*' - - if colsearch_dict: - search_text = json.dumps(colsearch_dict) - else: - search_text = re.sub(r'[^0-9a-zA-Z\-]+', '_', - search_text) + u':*' if search_text else '' - - return h.redirect_to( - h.url_for( - u'datastore.dump', - resource_id=resource_view[u'resource_id']) + u'?' + urlencode( - { - u'q': search_text, - u'plain': False, - u'language': u'simple', - u'sort': u','.join(sort_list), - u'filters': json.dumps(filters), - u'format': request.form[u'format'], - u'fields': u','.join(cols), - })) - - -datatablesview.add_url_rule( - u'/datatables/ajax/', view_func=ajax, methods=[u'POST'] -) - -datatablesview.add_url_rule( - u'/datatables/filtered-download/', - view_func=filtered_download, methods=[u'POST'] -) diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml b/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml deleted file mode 100644 index fac8438..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/config_declaration.yaml +++ /dev/null @@ -1,95 +0,0 @@ -version: 1 -groups: -- annotation: datatables_view settings - options: - - key: ckan.datatables.page_length_choices - default: 20 50 100 500 1000 - description: https://datatables.net/examples/advanced_init/length_menu.html - type: list - example: 20 50 100 500 1000 5000 - description: | - Space-delimited list of the choices for the number of rows per page, with - the lowest value being the default initial value. - - .. note:: On larger screens, DataTables view will attempt to fill the - table with as many rows that can fit using the lowest closest choice. - - - key: ckan.datatables.state_saving - default: true - type: bool - example: 'false' - description: | - Enable or disable state saving. When enabled, DataTables view will store - state information such as pagination position, page length, row - selection/s, column visibility/ordering, filtering and sorting using the - browser's localStorage. When the end user reloads the page, the table's - state will be altered to match what they had previously set up. - - This also enables/disables the "Reset" and "Share current view" - buttons. "Reset" discards the saved state. "Share current view" base-64 - encodes the state and passes it as a url parameter, acting like a "saved - search" that can be used for embedding and sharing table searches. - - - key: ckan.datatables.state_duration - default: 7200 - type: int - example: 86400 - description: | - Duration (in seconds) for which the saved state information is considered - valid. After this period has elapsed, the table's state will be returned - to the default, and the state cleared from the browser's localStorage. - - .. note:: The value ``0`` is a special value as it indicates that the - state can be stored and retrieved indefinitely with no time limit. - - - key: ckan.datatables.data_dictionary_labels - default: true - type: bool - example: 'false' - description: > - Enable or disable data dictionary integration. When enabled, a column's - data dictionary label will be used in the table header. A tooltip for - each column with data dictionary information will also be integrated into - the header. - - - key: ckan.datatables.ellipsis_length - default: 100 - type: int - example: 100 - description: | - The maximum number of characters to show in a cell before it is - truncated. An ellipsis (...) will be added at the truncation point and - the full text of the cell will be available as a tooltip. This value can - be overridden at the resource level when configuring a DataTables - resource view. - - .. note:: The value ``0`` is a special value as it indicates that the - column's width will be determined by the column name, and cell content - will word-wrap. - - - key: ckan.datatables.date_format - default: llll - description: see Moment.js cheatsheet https://devhints.io/moment - example: YYYY-MM-DD dd ww - description: | - The `moment.js date format - `_ - to use to convert raw timestamps to a user-friendly date format using - CKAN's current locale language code. This value can be overridden at the - resource level when configuring a DataTables resource view. - - .. note:: The value ``NONE`` is a special value as it indicates that no - date formatting will be applied and the raw ISO-8601 timestamp will be - displayed. - - - key: ckan.datatables.default_view - default: table - example: list - description: | - Indicates the default view mode of the DataTable (valid values: ``table`` - or ``list``). Table view is the typical grid layout, with horizontal - scrolling. List view is a responsive table, automatically hiding columns - as required to fit the browser viewport. In addition, list view allows - the user to view, copy and print the details of a specific row. This - value can be overridden at the resource level when configuring a - DataTables resource view. diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/plugin.py b/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/plugin.py deleted file mode 100644 index 9d74106..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/plugin.py +++ /dev/null @@ -1,99 +0,0 @@ -# encoding: utf-8 -from __future__ import annotations - -from ckan.common import CKANConfig -from typing import Any, cast -from ckan.types import Context, ValidatorFactory -import ckan.plugins as p -import ckan.plugins.toolkit as toolkit -from ckanext.datatablesview import blueprint - -default = cast(ValidatorFactory, toolkit.get_validator(u'default')) -boolean_validator = toolkit.get_validator(u'boolean_validator') -natural_number_validator = toolkit.get_validator(u'natural_number_validator') -ignore_missing = toolkit.get_validator(u'ignore_missing') - - -@toolkit.blanket.config_declarations -class DataTablesView(p.SingletonPlugin): - u''' - DataTables table view plugin - ''' - p.implements(p.IConfigurer, inherit=True) - p.implements(p.IResourceView, inherit=True) - p.implements(p.IBlueprint) - - # IBlueprint - - def get_blueprint(self): - return blueprint.datatablesview - - # IConfigurer - - def update_config(self, config: CKANConfig): - u''' - Set up the resource library, public directory and - template directory for the view - ''' - - # https://datatables.net/reference/option/lengthMenu - self.page_length_choices = config.get( - u'ckan.datatables.page_length_choices') - - self.page_length_choices = [int(i) for i in self.page_length_choices] - self.state_saving = config.get(u'ckan.datatables.state_saving') - - # https://datatables.net/reference/option/stateDuration - self.state_duration = config.get( - u"ckan.datatables.state_duration") - self.data_dictionary_labels = config.get( - u"ckan.datatables.data_dictionary_labels") - self.ellipsis_length = config.get( - u"ckan.datatables.ellipsis_length") - self.date_format = config.get(u"ckan.datatables.date_format") - self.default_view = config.get(u"ckan.datatables.default_view") - - toolkit.add_template_directory(config, u'templates') - toolkit.add_public_directory(config, u'public') - toolkit.add_resource(u'public', u'ckanext-datatablesview') - - # IResourceView - - def can_view(self, data_dict: dict[str, Any]): - resource = data_dict['resource'] - return resource.get(u'datastore_active') - - def setup_template_variables(self, context: Context, - data_dict: dict[str, Any]) -> dict[str, Any]: - return {u'page_length_choices': self.page_length_choices, - u'state_saving': self.state_saving, - u'state_duration': self.state_duration, - u'data_dictionary_labels': self.data_dictionary_labels, - u'ellipsis_length': self.ellipsis_length, - u'date_format': self.date_format, - u'default_view': self.default_view} - - def view_template(self, context: Context, data_dict: dict[str, Any]): - return u'datatables/datatables_view.html' - - def form_template(self, context: Context, data_dict: dict[str, Any]): - return u'datatables/datatables_form.html' - - def info(self) -> dict[str, Any]: - return { - u'name': u'datatables_view', - u'title': u'Table', - u'filterable': True, - u'icon': u'table', - u'requires_datastore': True, - u'default_title': p.toolkit._(u'Table'), - u'preview_enabled': False, - u'schema': { - u'responsive': [default(False), boolean_validator], - u'ellipsis_length': [default(self.ellipsis_length), - natural_number_validator], - u'date_format': [default(self.date_format)], - u'show_fields': [ignore_missing], - u'filterable': [default(True), boolean_validator], - } - } diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatables_view.css b/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatables_view.css deleted file mode 100644 index 208ec26..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatables_view.css +++ /dev/null @@ -1,220 +0,0 @@ -body { - background: none; -} - -/* this is important for DataTables to do column sizing properly */ -table { - max-width: none !important; -} - -/* so user does not see table being dynamically created */ -body.dt-view { - visibility: hidden; -} - -/* we need to do this as the dt-print-view is cloned from dt-view */ -body.dt-print-view { - visibility: visible; -} - -table.dataTable { - margin-left: 0px; -} - -div.dataTables_scroll{ - width: 100% !important; -} - -/* processing message should hover over table and be visible */ -#dtprv_processing { - z-index: 100; -} - -/* for table header, don't wrap */ -table.dataTable th { - white-space: nowrap; - overflow: hidden; - text-overflow: clip; -} - -/* clip content if it will overrun column width */ -table.dataTable td { - white-space: nowrap; - overflow: hidden; - text-overflow: clip; -} - -/* but show the data on hover */ -table.dataTable td:hover { - overflow: visible; - text-overflow: visible; -} - -/* when ellipsis_length === 0, wrap to column name width */ -table.dataTable td.wrapcell { - white-space: normal !important; - word-break: normal !important; - text-overflow: clip !important; -} - - -/* make _id column darker ala Excel */ -div.DTFC_LeftBodyWrapper { - filter: brightness(0.85); -} - - -/* for dynamic resizing of datatable, needed for scrollresize */ -#resize_wrapper { - position: absolute; - top: 0.1em; - left: 0.1em; - right: 0.1em; - bottom: 0.1em; - max-width: 100vw; - max-height: 100vh; -} - -/* fine-tune positioning of various info elements for Bootstrap */ -.dataTables_length{ - display:inline; -} - -#dtprv_filter { - text-align: right; - display: inline-flex; - float: right; - margin-top: -1.4em; -} - -#filterinfoicon { - margin-top: 0.4em; -} - -div.resourceinfo { - display: inline; - font-weight: bold; -} - -div.sortinfo { - display: block; -} - -#dtprv_paginate { - margin-top: -2.5em; -} - -i.fa-info-circle { - color: darkblue; -} - -/* for webkit, blink browsers, use input type search cancel button */ -input[type="search"]::-webkit-search-cancel-button { - -webkit-appearance: none; - cursor:pointer; - height: 12px; - width: 12px; - background-image: url('vendor/FontAwesome/images/times-circle-solid.svg'); -} - -/* right align datatable buttons */ -div.dt-buttons { - position: relative; - float: right; -} - -/* tighten it up */ -button.dt-button { - margin-right: 0em; -} - -div.dt-button-collection.fixed.four-column { - margin-left: 0px; -} - -div.dt-button-background { - background: rgba(0, 0, 0, 0.7); - /* Fallback */ - background: -ms-radial-gradient(center, ellipse farthest-corner, rgba(0, 0, 0, 0.3) 0%, rgba(0, 0, 0, 0.7) 100%); - /* IE10 Consumer Preview */ - background: -moz-radial-gradient(center, ellipse farthest-corner, rgba(0, 0, 0, 0.3) 0%, rgba(0, 0, 0, 0.7) 100%); - /* Firefox */ - background: -o-radial-gradient(center, ellipse farthest-corner, rgba(0, 0, 0, 0.3) 0%, rgba(0, 0, 0, 0.7) 100%); - /* Opera */ - background: -webkit-gradient(radial, center center, 0, center center, 497, color-stop(0, rgba(0, 0, 0, 0.3)), color-stop(1, rgba(0, 0, 0, 0.7))); - /* Webkit (Safari/Chrome 10) */ - background: -webkit-radial-gradient(center, ellipse farthest-corner, rgba(0, 0, 0, 0.3) 0%, rgba(0, 0, 0, 0.7) 100%); - /* Webkit (Chrome 11+) */ - background: radial-gradient(ellipse farthest-corner at center, rgba(0, 0, 0, 0.3) 0%, rgba(0, 0, 0, 0.7) 100%); - /* W3C Markup, IE10 Release Preview */ -} - -div.dt-button-collection.fixed { - position: fixed; - top: 50%; - left: 50%; - transform: translate(-65%, -20%); -} - -.table-striped > tbody > tr.selected > td, -.table-striped > tbody > tr.selected > th { -background-color: #0275d8; -} - -/* for zooming thumbnails on hover */ -.zoomthumb { --webkit-transition: all 0.3s ease-in-out; --moz-transition: all 0.3s ease-in-out; -transition: all 0.3s ease-in-out; -cursor: -webkit-zoom-in; -cursor: -moz-zoom-in; -cursor: zoom-in; -} - -.zoomthumb:hover, -.zoomthumb:active, -.zoomthumb:focus { -/**adjust scale to desired size, -add browser prefixes**/ --ms-transform: scale(2.5); --moz-transform: scale(2.5); --webkit-transform: scale(2.5); --o-transform: scale(2.5); -transform: scale(2.5); -position:relative; -z-index:100; -} - -.zoomthumb img { - display: block; - width: 100%; - height: auto; -} - -/* Animation CSS */ -#target { - box-sizing: border-box; - width: 40px; - height: 40px; - border-radius: 4px; - position: absolute; - top: calc(50% - 20px); - left: calc(50% - 20px); - background: #7d0; - box-shadow: inset 1px 1px 0 0 rgba(255, 255, 255, .2), inset -1px -1px 0 0 rgba(0, 0, 0, .05); -} - -[class^="animated-"], -[class*=" animated-"] { - animation-fill-mode: both; -} - -@keyframes shake { - 0%, 100% {transform: translateX(0);} - 20%, 60% {transform: translateX(-6px);} - 40%, 80% {transform: translateX(6px);} -} - -.animated-shake { - animation: shake .4s; -} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatablesview.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatablesview.js deleted file mode 100644 index 9c66d39..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/datatablesview/public/datatablesview.js +++ /dev/null @@ -1,944 +0,0 @@ -/* global $ jQuery gdataDict gresviewId */ - -// global vars used for state saving/deeplinking -let gsavedPage -let gsavedPagelen -let gsavedSelected -// global var for current view mode (table/list) -let gcurrentView = 'table' -// global var for sort info, global so we can show it in copy/print -let gsortInfo = '' -// global vars for filter info labels -let gtableSearchText = '' -let gcolFilterText = '' - -let datatable -const gisFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1 -let gsearchMode = '' -let gstartTime = 0 -let gelapsedTime - -// HELPER FUNCTIONS -// helper for filtered downloads -const run_query = function (params, format) { - const form = $('#filtered-datatables-download') - const p = $('') - p.attr('value', JSON.stringify(params)) - form.append(p) - const f = $('') - f.attr('value', format) - form.append(f) - form.submit() - p.remove() - f.remove() -} - -// helper for setting expiring localstorage, ttl in secs -function setWithExpiry (key, value, ttl) { - const now = new Date() - - // `item` is an object which contains the original value - // as well as the time when it's supposed to expire - const item = { - value: value, - expiry: ttl > 0 ? now.getTime() + (ttl * 1000) : 0 - } - window.localStorage.setItem(key, JSON.stringify(item)) -} - -// helper for getting expiring localstorage -function getWithExpiry (key) { - const itemStr = window.localStorage.getItem(key) - // if the item doesn't exist, return null - if (!itemStr) { - return null - } - let item - try { - item = JSON.parse(itemStr) - } catch { - return null - } - const now = new Date() - // compare the expiry time of the item with the current time - if (item.expiry && now.getTime() > item.expiry) { - // If the item is expired, delete the item from storage - // and return null - window.localStorage.removeItem(key) - return null - } - return item.value -} - -// helper for modal print -function printModal (title) { - const contents = document.querySelector('.dtr-details').innerHTML - const prtWindow = window.open('', '_blank') - prtWindow.document.write('

    ' + title + '

    ') - prtWindow.document.write(contents) - prtWindow.document.write('
    ') - prtWindow.print() - prtWindow.close() -} - -// helper for modal clipboard copy -function copyModal (title) { - const el = document.querySelector('.dtr-details') - const body = document.body - let range - let sel - if (document.createRange && window.getSelection) { - range = document.createRange() - sel = window.getSelection() - sel.removeAllRanges() - try { - range.selectNodeContents(el) - sel.addRange(range) - } catch (e) { - range.selectNode(el) - sel.addRange(range) - } - } else if (body.createTextRange) { - range = body.createTextRange() - range.moveToElementText(el) - range.select() - } - document.execCommand('copy') - window.getSelection().removeAllRanges() -} - -// force column auto width adjustment to kick in -// used by "Autofit columns" button -function fitColText () { - const dt = $('#dtprv').DataTable({ retrieve: true }) - if (gcurrentView === 'list') { - dt.responsive.recalc() - } - dt.columns.adjust().draw(false) -} - -// ensure element id is valid -function validateId (id) { - id = id.toLowerCase() - // Make alphanumeric (removes all other characters) - id = id.replace(/[^a-z0-9_\s-]/g, '') - // Convert whitespaces and underscore to # - id = id.replace(/[\s_]/g, '#') - // Convert multiple # to hyphen - id = id.replace(/[#]+/g, '-') - return id -} - -// compile sort & active filters for display in print, clipboard copy & search tooltip -function filterInfo (dt, noHtml = false, justFilterInfo = false, wrapped = false) { - let filtermsg = justFilterInfo ? '' : document.getElementById('dtprv_info').innerText - - const selinfo = document.getElementsByClassName('select-info')[0] - - if (selinfo !== undefined) { - filtermsg = filtermsg.replace(selinfo.innerText, ', ' + selinfo.innerText) - } - - // add active filter info to messageTop - if (gsearchMode === 'table') { - filtermsg = filtermsg + '
    ' + gtableSearchText + ': ' + dt.search() - } else if (gsearchMode === 'column') { - let colsearchflag = false - let colsearchmsg = '' - dt.columns().every(function () { - const colsearch = this.search() - const colname = this.name() - - if (colsearch) { - colsearchflag = true - colsearchmsg = colsearchmsg + ' ' + colname + ':
    ' + colsearch + ', ' - } - }) - if (colsearchflag) { - filtermsg = filtermsg + '
    ' + gcolFilterText + ':
    ' + colsearchmsg.slice(0, -2) - } - } - filtermsg = justFilterInfo ? filtermsg : filtermsg + '
    ' + gsortInfo - filtermsg = noHtml ? filtermsg.replace(/(<([^>]+)>)/ig, '') : filtermsg - filtermsg = wrapped ? filtermsg.replace(/,/g, '\n') : filtermsg - return filtermsg.trim() -}; - -// Copy deeplink to clipboard -function copyLink (dt, deeplink, shareText, sharemsgText) { - const hiddenDiv = $('
    ') - .css({ - height: 1, - width: 1, - overflow: 'hidden', - position: 'fixed', - top: 0, - left: 0 - }) - - const textarea = $(' \ -
    \ -
    \ - \ - \ -
    \ -
    \ -
    \ - ', - - onEditClick: function(e) { - var editing = this.$el.find('.data-table-cell-editor-editor'); - if (editing.length > 0) { - editing.parents('.data-table-cell-value').html(editing.text()).siblings('.data-table-cell-edit').removeClass("hidden"); - } - $(e.target).addClass("hidden"); - var cell = $(e.target).siblings('.data-table-cell-value'); - cell.data("previousContents", cell.text()); - var templated = Mustache.render(this.cellEditorTemplate, {value: cell.text()}); - cell.html(templated); - }, - - onEditorOK: function(e) { - var self = this; - var cell = $(e.target); - var rowId = cell.parents('tr').attr('data-id'); - var field = cell.parents('td').attr('data-field'); - var newValue = cell.parents('.data-table-cell-editor').find('.data-table-cell-editor-editor').val(); - var newData = {}; - newData[field] = newValue; - this.model.set(newData); - this.trigger('recline:flash', {message: "Updating row...", loader: true}); - this.model.save().then(function(response) { - this.trigger('recline:flash', {message: "Row updated successfully", category: 'success'}); - }) - .fail(function() { - this.trigger('recline:flash', { - message: 'Error saving row', - category: 'error', - persist: true - }); - }); - }, - - onEditorCancel: function(e) { - var cell = $(e.target).parents('.data-table-cell-value'); - cell.html(cell.data('previousContents')).siblings('.data-table-cell-edit').removeClass("hidden"); - } -}); - -})(jQuery, recline.View); -/*jshint multistr:true */ - -this.recline = this.recline || {}; -this.recline.View = this.recline.View || {}; - -(function($, my) { - "use strict"; -// ## Map view for a Dataset using Leaflet mapping library. -// -// This view allows to plot gereferenced records on a map. The location -// information can be provided in 2 ways: -// -// 1. Via a single field. This field must be either a geo_point or -// [GeoJSON](http://geojson.org) object -// 2. Via two fields with latitude and longitude coordinates. -// -// Which fields in the data these correspond to can be configured via the state -// (and are guessed if no info is provided). -// -// Initialization arguments are as standard for Dataset Views. State object may -// have the following (optional) configuration options: -// -// 
    -//   {
-//     // geomField if specified will be used in preference to lat/lon
-//     geomField: {id of field containing geometry in the dataset}
-//     lonField: {id of field containing longitude in the dataset}
-//     latField: {id of field containing latitude in the dataset}
-//     autoZoom: true,
-//     // use cluster support
-//     // cluster: true = always on
-//     // cluster: false = always off
-//     cluster: false
-//   }
-//
    -// -// Useful attributes to know about (if e.g. customizing) -// -// * map: the Leaflet map (L.Map) -// * features: Leaflet GeoJSON layer containing all the features (L.GeoJSON) -my.Map = Backbone.View.extend({ - template: ' \ -
    \ -
    \ -
    \ -', - - // These are the default (case-insensitive) names of field that are used if found. - // If not found, the user will need to define the fields via the editor. - latitudeFieldNames: ['lat','latitude'], - longitudeFieldNames: ['lon','longitude'], - geometryFieldNames: ['geojson', 'geom','the_geom','geometry','spatial','location', 'geo', 'lonlat'], - - initialize: function(options) { - var self = this; - this.visible = this.$el.is(':visible'); - this.mapReady = false; - // this will be the Leaflet L.Map object (setup below) - this.map = null; - - var stateData = _.extend({ - geomField: null, - lonField: null, - latField: null, - autoZoom: true, - cluster: false - }, - options.state - ); - this.state = new recline.Model.ObjectState(stateData); - - this._clusterOptions = { - zoomToBoundsOnClick: true, - //disableClusteringAtZoom: 10, - maxClusterRadius: 80, - singleMarkerMode: false, - skipDuplicateAddTesting: true, - animateAddingMarkers: false - }; - - // Listen to changes in the fields - this.listenTo(this.model.fields, 'change', function() { - self._setupGeometryField(); - self.render(); - }); - - // Listen to changes in the records - this.listenTo(this.model.records, 'add', function(doc){self.redraw('add',doc);}); - this.listenTo(this.model.records, 'change', function(doc){ - self.redraw('remove',doc); - self.redraw('add',doc); - }); - this.listenTo(this.model.records, 'remove', function(doc){self.redraw('remove',doc);}); - this.listenTo(this.model.records, 'reset', function(){self.redraw('reset');}); - - this.menu = new my.MapMenu({ - model: this.model, - state: this.state.toJSON() - }); - this.listenTo(this.menu.state, 'change', function() { - self.state.set(self.menu.state.toJSON()); - self.redraw(); - }); - this.listenTo(this.state, 'change', function() { - self.redraw(); - }); - this.elSidebar = this.menu.$el; - }, - - // ## Customization Functions - // - // The following methods are designed for overriding in order to customize - // behaviour - - // ### infobox - // - // Function to create infoboxes used in popups. The default behaviour is very simple and just lists all attributes. - // - // Users should override this function to customize behaviour i.e. - // - // view = new View({...}); - // view.infobox = function(record) { - // ... - // } - infobox: function(record) { - var html = ''; - for (var key in record.attributes){ - if (!(this.state.get('geomField') && key == this.state.get('geomField'))){ - html += '
    ' + key + ': '+ record.attributes[key] + '
    '; - } - } - return html; - }, - - // Options to use for the [Leaflet GeoJSON layer](http://leaflet.cloudmade.com/reference.html#geojson) - // See also - // - // e.g. - // - // pointToLayer: function(feature, latLng) - // onEachFeature: function(feature, layer) - // - // See defaults for examples - geoJsonLayerOptions: { - // pointToLayer function to use when creating points - // - // Default behaviour shown here is to create a marker using the - // popupContent set on the feature properties (created via infobox function - // during feature generation) - // - // NB: inside pointToLayer `this` will be set to point to this map view - // instance (which allows e.g. this.markers to work in this default case) - pointToLayer: function (feature, latlng) { - var marker = new L.Marker(latlng); - marker.bindPopup(feature.properties.popupContent); - // this is for cluster case - this.markers.addLayer(marker); - return marker; - }, - // onEachFeature default which adds popup in - onEachFeature: function(feature, layer) { - if (feature.properties && feature.properties.popupContent) { - layer.bindPopup(feature.properties.popupContent); - } - } - }, - - // END: Customization section - // ---- - - // ### Public: Adds the necessary elements to the page. - // - // Also sets up the editor fields and the map if necessary. - render: function() { - var self = this; - var htmls = Mustache.render(this.template, this.model.toTemplateJSON()); - this.$el.html(htmls); - this.$map = this.$el.find('.panel.map'); - this.redraw(); - return this; - }, - - // ### Public: Redraws the features on the map according to the action provided - // - // Actions can be: - // - // * reset: Clear all features - // * add: Add one or n features (records) - // * remove: Remove one or n features (records) - // * refresh: Clear existing features and add all current records - redraw: function(action, doc){ - var self = this; - action = action || 'refresh'; - // try to set things up if not already - if (!self._geomReady()){ - self._setupGeometryField(); - } - if (!self.mapReady){ - self._setupMap(); - } - - if (this._geomReady() && this.mapReady){ - // removing ad re-adding the layer enables faster bulk loading - this.map.removeLayer(this.features); - this.map.removeLayer(this.markers); - - var countBefore = 0; - this.features.eachLayer(function(){countBefore++;}); - - if (action == 'refresh' || action == 'reset') { - this.features.clearLayers(); - // recreate cluster group because of issues with clearLayer - this.map.removeLayer(this.markers); - this.markers = new L.MarkerClusterGroup(this._clusterOptions); - this._add(this.model.records.models); - } else if (action == 'add' && doc){ - this._add(doc); - } else if (action == 'remove' && doc){ - this._remove(doc); - } - - // this must come before zooming! - // if not: errors when using e.g. circle markers like - // "Cannot call method 'project' of undefined" - if (this.state.get('cluster')) { - this.map.addLayer(this.markers); - } else { - this.map.addLayer(this.features); - } - - if (this.state.get('autoZoom')){ - if (this.visible){ - this._zoomToFeatures(); - } else { - this._zoomPending = true; - } - } - } - }, - - show: function() { - // If the div was hidden, Leaflet needs to recalculate some sizes - // to display properly - if (this.map){ - this.map.invalidateSize(); - if (this._zoomPending && this.state.get('autoZoom')) { - this._zoomToFeatures(); - this._zoomPending = false; - } - } - this.visible = true; - }, - - hide: function() { - this.visible = false; - }, - - _geomReady: function() { - return Boolean(this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); - }, - - // Private: Add one or n features to the map - // - // For each record passed, a GeoJSON geometry will be extracted and added - // to the features layer. If an exception is thrown, the process will be - // stopped and an error notification shown. - // - // Each feature will have a popup associated with all the record fields. - // - _add: function(docs){ - var self = this; - - if (!(docs instanceof Array)) docs = [docs]; - - var count = 0; - var wrongSoFar = 0; - _.every(docs, function(doc){ - count += 1; - var feature = self._getGeometryFromRecord(doc); - if (typeof feature === 'undefined' || feature === null){ - // Empty field - return true; - } else if (feature instanceof Object){ - feature.properties = { - popupContent: self.infobox(doc), - // Add a reference to the model id, which will allow us to - // link this Leaflet layer to a Recline doc - cid: doc.cid - }; - - try { - self.features.addData(feature); - } catch (except) { - wrongSoFar += 1; - var msg = 'Wrong geometry value'; - if (except.message) msg += ' (' + except.message + ')'; - if (wrongSoFar <= 10) { - self.trigger('recline:flash', {message: msg, category:'error'}); - } - } - } else { - wrongSoFar += 1; - if (wrongSoFar <= 10) { - self.trigger('recline:flash', {message: 'Wrong geometry value', category:'error'}); - } - } - return true; - }); - }, - - // Private: Remove one or n features from the map - // - _remove: function(docs){ - - var self = this; - - if (!(docs instanceof Array)) docs = [docs]; - - _.each(docs,function(doc){ - for (var key in self.features._layers){ - if (self.features._layers[key].feature.geometry.properties.cid == doc.cid){ - self.features.removeLayer(self.features._layers[key]); - } - } - }); - - }, - - // Private: convert DMS coordinates to decimal - // - // north and east are positive, south and west are negative - // - _parseCoordinateString: function(coord){ - if (typeof(coord) != 'string') { - return(parseFloat(coord)); - } - var dms = coord.split(/[^-?\.\d\w]+/); - var deg = 0; var m = 0; - var toDeg = [1, 60, 3600]; // conversion factors for Deg, min, sec - var i; - for (i = 0; i < dms.length; ++i) { - if (isNaN(parseFloat(dms[i]))) { - continue; - } - deg += parseFloat(dms[i]) / toDeg[m]; - m += 1; - } - if (coord.match(/[SW]/)) { - deg = -1*deg; - } - return(deg); - }, - - // Private: Return a GeoJSON geomtry extracted from the record fields - // - _getGeometryFromRecord: function(doc){ - if (this.state.get('geomField')){ - var value = doc.get(this.state.get('geomField')); - if (typeof(value) === 'string'){ - // We *may* have a GeoJSON string representation - try { - value = $.parseJSON(value); - } catch(e) {} - } - if (typeof(value) === 'string') { - value = value.replace('(', '').replace(')', ''); - var parts = value.split(','); - var lat = this._parseCoordinateString(parts[0]); - var lon = this._parseCoordinateString(parts[1]); - - if (!isNaN(lon) && !isNaN(parseFloat(lat))) { - return { - "type": "Point", - "coordinates": [lon, lat] - }; - } else { - return null; - } - } else if (value && _.isArray(value)) { - // [ lon, lat ] - return { - "type": "Point", - "coordinates": [value[0], value[1]] - }; - } else if (value && value.lat) { - // of form { lat: ..., lon: ...} - return { - "type": "Point", - "coordinates": [value.lon || value.lng, value.lat] - }; - } - // We o/w assume that contents of the field are a valid GeoJSON object - return value; - } else if (this.state.get('lonField') && this.state.get('latField')){ - // We'll create a GeoJSON like point object from the two lat/lon fields - var lon = doc.get(this.state.get('lonField')); - var lat = doc.get(this.state.get('latField')); - lon = this._parseCoordinateString(lon); - lat = this._parseCoordinateString(lat); - - if (!isNaN(parseFloat(lon)) && !isNaN(parseFloat(lat))) { - return { - type: 'Point', - coordinates: [lon,lat] - }; - } - } - return null; - }, - - // Private: Check if there is a field with GeoJSON geometries or alternatively, - // two fields with lat/lon values. - // - // If not found, the user can define them via the UI form. - _setupGeometryField: function(){ - // should not overwrite if we have already set this (e.g. explicitly via state) - if (!this._geomReady()) { - this.state.set({ - geomField: this._checkField(this.geometryFieldNames), - latField: this._checkField(this.latitudeFieldNames), - lonField: this._checkField(this.longitudeFieldNames) - }); - this.menu.state.set(this.state.toJSON()); - } - }, - - // Private: Check if a field in the current model exists in the provided - // list of names. - // - // - _checkField: function(fieldNames){ - var field; - var modelFieldNames = this.model.fields.pluck('id'); - for (var i = 0; i < fieldNames.length; i++){ - for (var j = 0; j < modelFieldNames.length; j++){ - if (modelFieldNames[j].toLowerCase() == fieldNames[i].toLowerCase()) - return modelFieldNames[j]; - } - } - return null; - }, - - // Private: Zoom to map to current features extent if any, or to the full - // extent if none. - // - _zoomToFeatures: function(){ - var bounds = this.features.getBounds(); - if (bounds && bounds.getNorthEast() && bounds.getSouthWest()){ - this.map.fitBounds(bounds); - } else { - this.map.setView([0, 0], 2); - } - }, - - // Private: Sets up the Leaflet map control and the features layer. - // - // The map uses a base layer from [Stamen](http://maps.stamen.com) based - // on [OpenStreetMap data](http://openstreetmap.org) by default, but it can - // be configured passing the `mapTilesURL` and `mapTilesAttribution` options - // (`mapTilesSubdomains` is also supported), eg: - // - // view = new recline.View.Map({ - // model: dataset, - // mapTilesURL: '//{s}.tiles.mapbox.com/v4/mapbox.mapbox-streets-v7/{z}/{x}/{y}.png?access_token=pk.XXXX', - // mapTilesAttribution: '© MapBox etc..', - // mapTilesSubdomains: 'ab' - // }) - // - // - _setupMap: function(){ - var self = this; - this.map = new L.Map(this.$map.get(0)); - var mapUrl = this.options.mapTilesURL || 'https://stamen-tiles-{s}.a.ssl.fastly.net/terrain/{z}/{x}/{y}.png'; - var attribution = this.options.mapTilesAttribution ||'Map tiles by Stamen Design (CC BY 3.0). Data by OpenStreetMap (CC BY SA)'; - var subdomains = this.options.mapTilesSubdomains || 'abc'; - - var bg = new L.TileLayer(mapUrl, {maxZoom: 19, attribution: attribution, subdomains: subdomains}); - this.map.addLayer(bg); - - this.markers = new L.MarkerClusterGroup(this._clusterOptions); - - // rebind this (as needed in e.g. default case above) - this.geoJsonLayerOptions.pointToLayer = _.bind( - this.geoJsonLayerOptions.pointToLayer, - this); - this.features = new L.GeoJSON(null, this.geoJsonLayerOptions); - - this.map.setView([0, 0], 2); - - this.mapReady = true; - }, - - // Private: Helper function to select an option from a select list - // - _selectOption: function(id,value){ - var options = $('.' + id + ' > select > option'); - if (options){ - options.each(function(opt){ - if (this.value == value) { - $(this).attr('selected','selected'); - return false; - } - }); - } - } -}); - -my.MapMenu = Backbone.View.extend({ - className: 'editor', - - template: ' \ -
    \ -
    \ -
    \ - \ - \ -
    \ -
    \ - \ -
    \ - \ -
    \ - \ -
    \ - \ -
    \ -
    \ - \ -
    \ -
    \ - \ -
    \ -
    \ - \ - \ -
    \ - \ -
    \ - ', - - // Define here events for UI elements - events: { - 'click .editor-update-map': 'onEditorSubmit', - 'change .editor-field-type': 'onFieldTypeChange', - 'click #editor-auto-zoom': 'onAutoZoomChange', - 'click #editor-cluster': 'onClusteringChange' - }, - - initialize: function(options) { - var self = this; - _.bindAll(this, 'render'); - this.listenTo(this.model.fields, 'change', this.render); - this.state = new recline.Model.ObjectState(options.state); - this.listenTo(this.state, 'change', this.render); - this.render(); - }, - - // ### Public: Adds the necessary elements to the page. - // - // Also sets up the editor fields and the map if necessary. - render: function() { - var self = this; - var htmls = Mustache.render(this.template, this.model.toTemplateJSON()); - this.$el.html(htmls); - - if (this._geomReady() && this.model.fields.length){ - if (this.state.get('geomField')){ - this._selectOption('editor-geom-field',this.state.get('geomField')); - this.$el.find('#editor-field-type-geom').attr('checked','checked').change(); - } else{ - this._selectOption('editor-lon-field',this.state.get('lonField')); - this._selectOption('editor-lat-field',this.state.get('latField')); - this.$el.find('#editor-field-type-latlon').attr('checked','checked').change(); - } - } - if (this.state.get('autoZoom')) { - this.$el.find('#editor-auto-zoom').attr('checked', 'checked'); - } else { - this.$el.find('#editor-auto-zoom').removeAttr('checked'); - } - if (this.state.get('cluster')) { - this.$el.find('#editor-cluster').attr('checked', 'checked'); - } else { - this.$el.find('#editor-cluster').removeAttr('checked'); - } - return this; - }, - - _geomReady: function() { - return Boolean(this.state.get('geomField') || (this.state.get('latField') && this.state.get('lonField'))); - }, - - // ## UI Event handlers - // - - // Public: Update map with user options - // - // Right now the only configurable option is what field(s) contains the - // location information. - // - onEditorSubmit: function(e){ - e.preventDefault(); - if (this.$el.find('#editor-field-type-geom').attr('checked')){ - this.state.set({ - geomField: this.$el.find('.editor-geom-field > select > option:selected').val(), - lonField: null, - latField: null - }); - } else { - this.state.set({ - geomField: null, - lonField: this.$el.find('.editor-lon-field > select > option:selected').val(), - latField: this.$el.find('.editor-lat-field > select > option:selected').val() - }); - } - return false; - }, - - // Public: Shows the relevant select lists depending on the location field - // type selected. - // - onFieldTypeChange: function(e){ - if (e.target.value == 'geom'){ - this.$el.find('.editor-field-type-geom').show(); - this.$el.find('.editor-field-type-latlon').hide(); - } else { - this.$el.find('.editor-field-type-geom').hide(); - this.$el.find('.editor-field-type-latlon').show(); - } - }, - - onAutoZoomChange: function(e){ - this.state.set({autoZoom: !this.state.get('autoZoom')}); - }, - - onClusteringChange: function(e){ - this.state.set({cluster: !this.state.get('cluster')}); - }, - - // Private: Helper function to select an option from a select list - // - _selectOption: function(id,value){ - var options = this.$el.find('.' + id + ' > select > option'); - if (options){ - options.each(function(opt){ - if (this.value == value) { - $(this).attr('selected','selected'); - return false; - } - }); - } - } -}); - -})(jQuery, recline.View); -/*jshint multistr:true */ - -// Standard JS module setup -this.recline = this.recline || {}; -this.recline.View = this.recline.View || {}; - -(function($, my) { - "use strict"; -// ## MultiView -// -// Manage multiple views together along with query editor etc. Usage: -// -// 
    -// var myExplorer = new recline.View.MultiView({
-//   model: {{recline.Model.Dataset instance}}
-//   el: {{an existing dom element}}
-//   views: {{dataset views}}
-//   state: {{state configuration -- see below}}
-// });
-//
    -// -// ### Parameters -// -// **model**: (required) recline.model.Dataset instance. -// -// **el**: (required) DOM element to bind to. NB: the element already -// being in the DOM is important for rendering of some subviews (e.g. -// Graph). -// -// **views**: (optional) the dataset views (Grid, Graph etc) for -// MultiView to show. This is an array of view hashes. If not provided -// initialize with (recline.View.)Grid, Graph, and Map views (with obvious id -// and labels!). -// -// 
    -// var views = [
-//   {
-//     id: 'grid', // used for routing
-//     label: 'Grid', // used for view switcher
-//     view: new recline.View.Grid({
-//       model: dataset
-//     })
-//   },
-//   {
-//     id: 'graph',
-//     label: 'Graph',
-//     view: new recline.View.Graph({
-//       model: dataset
-//     })
-//   }
-// ];
-//
    -// -// **sidebarViews**: (optional) the sidebar views (Filters, Fields) for -// MultiView to show. This is an array of view hashes. If not provided -// initialize with (recline.View.)FilterEditor and Fields views (with obvious -// id and labels!). -// -// 
    -// var sidebarViews = [
-//   {
-//     id: 'filterEditor', // used for routing
-//     label: 'Filters', // used for view switcher
-//     view: new recline.View.FilterEditor({
-//       model: dataset
-//     })
-//   },
-//   {
-//     id: 'fieldsView',
-//     label: 'Fields',
-//     view: new recline.View.Fields({
-//       model: dataset
-//     })
-//   }
-// ];
-//
    -// -// **state**: standard state config for this view. This state is slightly -// special as it includes config of many of the subviews. -// -// 
    -// var state = {
-//     query: {dataset query state - see dataset.queryState object}
-//     'view-{id1}': {view-state for this view}
-//     'view-{id2}': {view-state for }
-//     ...
-//     // Explorer
-//     currentView: id of current view (defaults to first view if not specified)
-//     readOnly: (default: false) run in read-only mode
-// }
-//
    -// -// Note that at present we do *not* serialize information about the actual set -// of views in use -- e.g. those specified by the views argument -- but instead -// expect either that the default views are fine or that the client to have -// initialized the MultiView with the relevant views themselves. -my.MultiView = Backbone.View.extend({ - template: ' \ -
    \ -
    \ - \ -
    \ -
    \ -
    \ - {{#views}} \ - \ - {{/views}} \ -
    \ -
    \ -
    \ - {{#recordCountWasEstimated}} \ - about \ - {{/recordCountWasEstimated}} \ - {{recordCount}} records \ -
    \ -
    \ -
    \ - {{#sidebarViews}} \ - \ - {{/sidebarViews}} \ -
    \ -
    \ -
    \ -
    \ -
    \ -
    \ -
    \ - ', - events: { - 'click .menu-right button': '_onMenuClick', - 'click .navigation button': '_onSwitchView' - }, - - initialize: function(options) { - var self = this; - this._setupState(options.state); - - // Hash of 'page' views (i.e. those for whole page) keyed by page name - if (options.views) { - this.pageViews = options.views; - } else { - this.pageViews = [{ - id: 'grid', - label: 'Grid', - view: new my.SlickGrid({ - model: this.model, - state: this.state.get('view-grid') - }) - }, { - id: 'graph', - label: 'Graph', - view: new my.Graph({ - model: this.model, - state: this.state.get('view-graph') - }) - }, { - id: 'map', - label: 'Map', - view: new my.Map({ - model: this.model, - state: this.state.get('view-map') - }) - }, { - id: 'timeline', - label: 'Timeline', - view: new my.Timeline({ - model: this.model, - state: this.state.get('view-timeline') - }) - }]; - } - // Hashes of sidebar elements - if(options.sidebarViews) { - this.sidebarViews = options.sidebarViews; - } else { - this.sidebarViews = [{ - id: 'filterEditor', - label: 'Filters', - view: new my.FilterEditor({ - model: this.model - }) - }, { - id: 'fieldsView', - label: 'Fields', - view: new my.Fields({ - model: this.model - }) - }]; - } - // these must be called after pageViews are created - this.render(); - this._bindStateChanges(); - this._bindFlashNotifications(); - // now do updates based on state (need to come after render) - if (this.state.get('readOnly')) { - this.setReadOnly(); - } - if (this.state.get('currentView')) { - this.updateNav(this.state.get('currentView')); - } else { - this.updateNav(this.pageViews[0].id); - } - this._showHideSidebar(); - - this.listenTo(this.model, 'query:start', function() { - self.notify({loader: true, persist: true}); - }); - this.listenTo(this.model, 'query:done', function() { - self.clearNotifications(); - self.$el.find('.doc-count').text(self.model.recordCount || 'Unknown'); - self.$el.find('.doc-count-approx').text(self.model.recordCountWasEstimated && 'about' || ''); - }); - this.listenTo(this.model, 'query:fail', function(error) { - self.clearNotifications(); - var msg = ''; - if (typeof(error) == 'string') { - msg = error; - } else if (typeof(error) == 'object') { - if (error.title) { - msg = error.title + ': '; - } - if (error.message) { - msg += error.message; - } - } else { - msg = 'There was an error querying the backend'; - } - self.notify({message: msg, category: 'error', persist: true}); - }); - - // retrieve basic data like fields etc - // note this.model and dataset returned are the same - // TODO: set query state ...? - this.model.queryState.set(self.state.get('query'), {silent: true}); - }, - - setReadOnly: function() { - this.$el.addClass('recline-read-only'); - }, - - render: function() { - var tmplData = this.model.toTemplateJSON(); - tmplData.views = this.pageViews; - tmplData.sidebarViews = this.sidebarViews; - var template = Mustache.render(this.template, tmplData); - this.$el.html(template); - - // now create and append other views - var $dataViewContainer = this.$el.find('.data-view-container'); - var $dataSidebar = this.$el.find('.data-view-sidebar'); - - // the main views - _.each(this.pageViews, function(view, pageName) { - view.view.render(); - if (view.view.redraw) { - view.view.redraw(); - } - $dataViewContainer.append(view.view.el); - if (view.view.elSidebar) { - $dataSidebar.append(view.view.elSidebar); - } - }); - - _.each(this.sidebarViews, function(view) { - this['$'+view.id] = view.view.$el; - $dataSidebar.append(view.view.el); - }, this); - - this.pager = new recline.View.Pager({ - model: this.model - }); - this.$el.find('.recline-results-info').after(this.pager.el); - - this.queryEditor = new recline.View.QueryEditor({ - model: this.model.queryState - }); - this.$el.find('.query-editor-here').append(this.queryEditor.el); - - }, - - remove: function () { - _.each(this.pageViews, function (view) { - view.view.remove(); - }); - _.each(this.sidebarViews, function (view) { - view.view.remove(); - }); - this.pager.remove(); - this.queryEditor.remove(); - Backbone.View.prototype.remove.apply(this, arguments); - }, - - // hide the sidebar if empty - _showHideSidebar: function() { - var $dataSidebar = this.$el.find('.data-view-sidebar'); - var visibleChildren = $dataSidebar.children().filter(function() { - return $(this).css("display") != "none"; - }).length; - - if (visibleChildren > 0) { - $dataSidebar.show(); - } else { - $dataSidebar.hide(); - } - }, - - updateNav: function(pageName) { - this.$el.find('.navigation button').removeClass('active'); - var $el = this.$el.find('.navigation button[data-view="' + pageName + '"]'); - $el.addClass('active'); - - // add/remove sidebars and hide inactive views - _.each(this.pageViews, function(view, idx) { - if (view.id === pageName) { - view.view.$el.show(); - if (view.view.elSidebar) { - view.view.elSidebar.show(); - } - } else { - view.view.$el.hide(); - if (view.view.elSidebar) { - view.view.elSidebar.hide(); - } - if (view.view.hide) { - view.view.hide(); - } - } - }); - - this._showHideSidebar(); - - // call view.view.show after sidebar visibility has been determined so - // that views can correctly calculate their maximum width - _.each(this.pageViews, function(view, idx) { - if (view.id === pageName) { - if (view.view.show) { - view.view.show(); - } - } - }); - }, - - _onMenuClick: function(e) { - e.preventDefault(); - var action = $(e.target).attr('data-action'); - this['$'+action].toggle(); - this._showHideSidebar(); - }, - - _onSwitchView: function(e) { - e.preventDefault(); - var viewName = $(e.target).attr('data-view'); - this.updateNav(viewName); - this.state.set({currentView: viewName}); - }, - - // create a state object for this view and do the job of - // - // a) initializing it from both data passed in and other sources (e.g. hash url) - // - // b) ensure the state object is updated in responese to changes in subviews, query etc. - _setupState: function(initialState) { - var self = this; - // get data from the query string / hash url plus some defaults - var qs = my.parseHashQueryString(); - var query = qs.reclineQuery; - query = query ? JSON.parse(query) : self.model.queryState.toJSON(); - // backwards compatability (now named view-graph but was named graph) - var graphState = qs['view-graph'] || qs.graph; - graphState = graphState ? JSON.parse(graphState) : {}; - - // now get default data + hash url plus initial state and initial our state object with it - var stateData = _.extend({ - query: query, - 'view-graph': graphState, - backend: this.model.backend.__type__, - url: this.model.get('url'), - dataset: this.model.toJSON(), - currentView: null, - readOnly: false - }, - initialState); - this.state = new recline.Model.ObjectState(stateData); - }, - - _bindStateChanges: function() { - var self = this; - // finally ensure we update our state object when state of sub-object changes so that state is always up to date - this.listenTo(this.model.queryState, 'change', function() { - self.state.set({query: self.model.queryState.toJSON()}); - }); - _.each(this.pageViews, function(pageView) { - if (pageView.view.state && pageView.view.state.bind) { - var update = {}; - update['view-' + pageView.id] = pageView.view.state.toJSON(); - self.state.set(update); - self.listenTo(pageView.view.state, 'change', function() { - var update = {}; - update['view-' + pageView.id] = pageView.view.state.toJSON(); - // had problems where change not being triggered for e.g. grid view so let's do it explicitly - self.state.set(update, {silent: true}); - self.state.trigger('change'); - }); - } - }); - }, - - _bindFlashNotifications: function() { - var self = this; - _.each(this.pageViews, function(pageView) { - self.listenTo(pageView.view, 'recline:flash', function(flash) { - self.notify(flash); - }); - }); - }, - - // ### notify - // - // Create a notification (a div.alert in div.alert-messsages) using provided - // flash object. Flash attributes (all are optional): - // - // * message: message to show. - // * category: warning (default), success, error - // * persist: if true alert is persistent, o/w hidden after 3s (default = false) - // * loader: if true show loading spinner - notify: function(flash) { - var tmplData = _.extend({ - message: 'Loading', - category: 'warning', - loader: false - }, - flash - ); - var _template; - if (tmplData.loader) { - _template = ' \ -
    \ - {{message}} \ -   \ -
    '; - } else { - _template = ' \ -
    × \ - {{message}} \ -
    '; - } - var _templated = $(Mustache.render(_template, tmplData)); - _templated = $(_templated).appendTo($('.recline-data-explorer .alert-messages')); - if (!flash.persist) { - setTimeout(function() { - $(_templated).fadeOut(1000, function() { - $(this).remove(); - }); - }, 1000); - } - }, - - // ### clearNotifications - // - // Clear all existing notifications - clearNotifications: function() { - var $notifications = $('.recline-data-explorer .alert-messages .alert'); - $notifications.fadeOut(1500, function() { - $(this).remove(); - }); - } -}); - -// ### MultiView.restore -// -// Restore a MultiView instance from a serialized state including the associated dataset -// -// This inverts the state serialization process in Multiview -my.MultiView.restore = function(state) { - // hack-y - restoring a memory dataset does not mean much ... (but useful for testing!) - var datasetInfo; - if (state.backend === 'memory') { - datasetInfo = { - backend: 'memory', - records: [{stub: 'this is a stub dataset because we do not restore memory datasets'}] - }; - } else { - datasetInfo = _.extend({ - url: state.url, - backend: state.backend - }, - state.dataset - ); - } - var dataset = new recline.Model.Dataset(datasetInfo); - var explorer = new my.MultiView({ - model: dataset, - state: state - }); - return explorer; -}; - -// ## Miscellaneous Utilities -var urlPathRegex = /^([^?]+)(\?.*)?/; - -// Parse the Hash section of a URL into path and query string -my.parseHashUrl = function(hashUrl) { - var parsed = urlPathRegex.exec(hashUrl); - if (parsed === null) { - return {}; - } else { - return { - path: parsed[1], - query: parsed[2] || '' - }; - } -}; - -// Parse a URL query string (?xyz=abc...) into a dictionary. -my.parseQueryString = function(q) { - if (!q) { - return {}; - } - var urlParams = {}, - e, d = function (s) { - return unescape(s.replace(/\+/g, " ")); - }, - r = /([^&=]+)=?([^&]*)/g; - - if (q && q.length && q[0] === '?') { - q = q.slice(1); - } - while (e = r.exec(q)) { - // TODO: have values be array as query string allow repetition of keys - urlParams[d(e[1])] = d(e[2]); - } - return urlParams; -}; - -// Parse the query string out of the URL hash -my.parseHashQueryString = function() { - var q = my.parseHashUrl(window.location.hash).query; - return my.parseQueryString(q); -}; - -// Compse a Query String -my.composeQueryString = function(queryParams) { - var queryString = '?'; - var items = []; - $.each(queryParams, function(key, value) { - if (typeof(value) === 'object') { - value = JSON.stringify(value); - } - items.push(key + '=' + encodeURIComponent(value)); - }); - queryString += items.join('&'); - return queryString; -}; - -my.getNewHashForQueryString = function(queryParams) { - var queryPart = my.composeQueryString(queryParams); - if (window.location.hash) { - // slice(1) to remove # at start - return window.location.hash.split('?')[0].slice(1) + queryPart; - } else { - return queryPart; - } -}; - -my.setHashQueryString = function(queryParams) { - window.location.hash = my.getNewHashForQueryString(queryParams); -}; - -})(jQuery, recline.View); - -/*jshint multistr:true */ - -this.recline = this.recline || {}; -this.recline.View = this.recline.View || {}; - -(function($, my) { - "use strict"; - -// ## SlickGrid Dataset View -// -// Provides a tabular view on a Dataset, based on SlickGrid. -// -// https://github.com/mleibman/SlickGrid -// -// Initialize it with a `recline.Model.Dataset`. -// -// Additional options to drive SlickGrid grid can be given through state. -// The following keys allow for customization: -// * gridOptions: to add options at grid level -// * columnsEditor: to add editor for editable columns -// -// For example: -// var grid = new recline.View.SlickGrid({ -// model: dataset, -// el: $el, -// state: { -// gridOptions: { -// editable: true, -// enableAddRow: true -// // Enable support for row delete -// enabledDelRow: true, -// // Enable support for row Reorder -// enableReOrderRow:true, -// ... -// }, -// columnsEditor: [ -// {column: 'date', editor: Slick.Editors.Date }, -// {column: 'title', editor: Slick.Editors.Text} -// ] -// } -// }); -//// NB: you need an explicit height on the element for slickgrid to work -my.SlickGrid = Backbone.View.extend({ - initialize: function(modelEtc) { - var self = this; - this.$el.addClass('recline-slickgrid'); - - // Template for row delete menu , change it if you don't love - this.templates = { - "deleterow" : '' - }; - - _.bindAll(this, 'render', 'onRecordChanged'); - this.listenTo(this.model.records, 'add remove reset', this.render); - this.listenTo(this.model.records, 'change', this.onRecordChanged); - var state = _.extend({ - hiddenColumns: [], - columnsOrder: [], - columnsSort: {}, - columnsWidth: [], - columnsEditor: [], - options: {}, - fitColumns: false - }, modelEtc.state - - ); - this.state = new recline.Model.ObjectState(state); - this._slickHandler = new Slick.EventHandler(); - - //add menu for new row , check if enableAddRow is set to true or not set - if(this.state.get("gridOptions") - && this.state.get("gridOptions").enabledAddRow != undefined - && this.state.get("gridOptions").enabledAddRow == true ){ - this.editor = new my.GridControl() - this.elSidebar = this.editor.$el - this.listenTo(this.editor.state, 'change', function(){ - this.model.records.add(new recline.Model.Record()) - }); - } - }, - - onRecordChanged: function(record) { - // Ignore if the grid is not yet drawn - if (!this.grid) { - return; - } - // Let's find the row corresponding to the index - var row_index = this.grid.getData().getModelRow( record ); - this.grid.invalidateRow(row_index); - this.grid.getData().updateItem(record, row_index); - this.grid.render(); - }, - - render: function() { - var self = this; - var options = _.extend({ - enableCellNavigation: true, - enableColumnReorder: true, - explicitInitialization: true, - syncColumnCellResize: true, - forceFitColumns: this.state.get('fitColumns') - }, self.state.get('gridOptions')); - - // We need all columns, even the hidden ones, to show on the column picker - var columns = []; - - // custom formatter as default one escapes html - // plus this way we distinguish between rendering/formatting and computed value (so e.g. sort still works ...) - // row = row index, cell = cell index, value = value, columnDef = column definition, dataContext = full row values - var formatter = function(row, cell, value, columnDef, dataContext) { - if(columnDef.id == "del"){ - return self.templates.deleterow - } - var field = self.model.fields.get(columnDef.id); - if (field.renderer) { - return field.renderer(value, field, dataContext); - } else { - return value - } - }; - - // we need to be sure that user is entering a valid input , for exemple if - // field is date type and field.format ='YY-MM-DD', we should be sure that - // user enter a correct value - var validator = function(field) { - return function(value){ - if (field.type == "date" && isNaN(Date.parse(value))){ - return { - valid: false, - msg: "A date is required, check field field-date-format" - }; - } else { - return {valid: true, msg :null } - } - } - }; - - // Add column for row reorder support - if (this.state.get("gridOptions") && this.state.get("gridOptions").enableReOrderRow == true) { - columns.push({ - id: "#", - name: "", - width: 22, - behavior: "selectAndMove", - selectable: false, - resizable: false, - cssClass: "recline-cell-reorder" - }) - } - // Add column for row delete support - if (this.state.get("gridOptions") && this.state.get("gridOptions").enabledDelRow == true) { - columns.push({ - id: 'del', - name: '', - field: 'del', - sortable: true, - width: 38, - formatter: formatter, - validator:validator - }) - } - - function sanitizeFieldName(name) { - return $('
    ').text(name).html(); - } - - _.each(this.model.fields.toJSON(),function(field){ - var column = { - id: field.id, - name: sanitizeFieldName(field.label), - field: field.id, - sortable: true, - minWidth: 80, - formatter: formatter, - validator:validator(field) - }; - var widthInfo = _.find(self.state.get('columnsWidth'),function(c){return c.column === field.id;}); - if (widthInfo){ - column.width = widthInfo.width; - } - var editInfo = _.find(self.state.get('columnsEditor'),function(c){return c.column === field.id;}); - if (editInfo){ - column.editor = editInfo.editor; - } else { - // guess editor type - var typeToEditorMap = { - 'string': Slick.Editors.LongText, - 'integer': Slick.Editors.IntegerEditor, - 'number': Slick.Editors.Text, - // TODO: need a way to ensure we format date in the right way - // Plus what if dates are in distant past or future ... (?) - // 'date': Slick.Editors.DateEditor, - 'date': Slick.Editors.Text, - 'boolean': Slick.Editors.YesNoSelectEditor - // TODO: (?) percent ... - }; - if (field.type in typeToEditorMap) { - column.editor = typeToEditorMap[field.type] - } else { - column.editor = Slick.Editors.LongText; - } - } - columns.push(column); - }); - // Restrict the visible columns - var visibleColumns = _.filter(columns, function(column) { - return _.indexOf(self.state.get('hiddenColumns'), column.id) === -1; - }); - // Order them if there is ordering info on the state - if (this.state.get('columnsOrder') && this.state.get('columnsOrder').length > 0) { - visibleColumns = visibleColumns.sort(function(a,b){ - return _.indexOf(self.state.get('columnsOrder'),a.id) > _.indexOf(self.state.get('columnsOrder'),b.id) ? 1 : -1; - }); - columns = columns.sort(function(a,b){ - return _.indexOf(self.state.get('columnsOrder'),a.id) > _.indexOf(self.state.get('columnsOrder'),b.id) ? 1 : -1; - }); - } - - // Move hidden columns to the end, so they appear at the bottom of the - // column picker - var tempHiddenColumns = []; - for (var i = columns.length -1; i >= 0; i--){ - if (_.indexOf(_.pluck(visibleColumns,'id'),columns[i].id) === -1){ - tempHiddenColumns.push(columns.splice(i,1)[0]); - } - } - columns = columns.concat(tempHiddenColumns); - - // Transform a model object into a row - function toRow(m) { - var row = {}; - self.model.fields.each(function(field) { - var render = ""; - //when adding row from slickgrid the field value is undefined - if(!_.isUndefined(m.getFieldValueUnrendered(field))){ - render =m.getFieldValueUnrendered(field) - } - row[field.id] = render - }); - return row; - } - - function RowSet() { - var models = []; - var rows = []; - - this.push = function(model, row) { - models.push(model); - rows.push(row); - }; - - this.getLength = function() {return rows.length; }; - this.getItem = function(index) {return rows[index];}; - this.getItemMetadata = function(index) {return {};}; - this.getModel = function(index) {return models[index];}; - this.getModelRow = function(m) {return _.indexOf(models, m);}; - this.updateItem = function(m,i) { - rows[i] = toRow(m); - models[i] = m; - }; - } - - var data = new RowSet(); - - this.model.records.each(function(doc){ - data.push(doc, toRow(doc)); - }); - - this.grid = new Slick.Grid(this.el, data, visibleColumns, options); - // Column sorting - var sortInfo = this.model.queryState.get('sort'); - if (sortInfo){ - var column = sortInfo[0].field; - var sortAsc = sortInfo[0].order !== 'desc'; - this.grid.setSortColumn(column, sortAsc); - } - - if (this.state.get("gridOptions") && this.state.get("gridOptions").enableReOrderRow) { - this._setupRowReordering(); - } - - this._slickHandler.subscribe(this.grid.onSort, function(e, args){ - var order = (args.sortAsc) ? 'asc':'desc'; - var sort = [{ - field: args.sortCol.field, - order: order - }]; - self.model.query({sort: sort}); - }); - - this._slickHandler.subscribe(this.grid.onColumnsReordered, function(e, args){ - self.state.set({columnsOrder: _.pluck(self.grid.getColumns(),'id')}); - }); - - this.grid.onColumnsResized.subscribe(function(e, args){ - var columns = args.grid.getColumns(); - var defaultColumnWidth = args.grid.getOptions().defaultColumnWidth; - var columnsWidth = []; - _.each(columns,function(column){ - if (column.width != defaultColumnWidth){ - columnsWidth.push({column:column.id,width:column.width}); - } - }); - self.state.set({columnsWidth:columnsWidth}); - }); - - this._slickHandler.subscribe(this.grid.onCellChange, function (e, args) { - // We need to change the model associated value - var grid = args.grid; - var model = data.getModel(args.row); - var field = grid.getColumns()[args.cell].id; - var v = {}; - v[field] = args.item[field]; - model.set(v); - }); - this._slickHandler.subscribe(this.grid.onClick,function(e, args){ - //try catch , because this fail in qunit , but no - //error on browser. - try{e.preventDefault()}catch(e){} - - // The cell of grid that handle row delete is The first cell (0) if - // The grid ReOrder is not present ie enableReOrderRow == false - // else it is The the second cell (1) , because The 0 is now cell - // that handle row Reoder. - var cell =0 - if(self.state.get("gridOptions") - && self.state.get("gridOptions").enableReOrderRow != undefined - && self.state.get("gridOptions").enableReOrderRow == true ){ - cell =1 - } - if (args.cell == cell && self.state.get("gridOptions").enabledDelRow == true){ - // We need to delete the associated model - var model = data.getModel(args.row); - model.destroy() - } - }) ; - var columnpicker = new Slick.Controls.ColumnPicker(columns, this.grid, - _.extend(options,{state:this.state})); - if (self.visible){ - self.grid.init(); - self.rendered = true; - } else { - // Defer rendering until the view is visible - self.rendered = false; - } - return this; - }, - - // Row reordering support based on - // https://github.com/mleibman/SlickGrid/blob/gh-pages/examples/example9-row-reordering.html - _setupRowReordering: function() { - var self = this; - self.grid.setSelectionModel(new Slick.RowSelectionModel()); - - var moveRowsPlugin = new Slick.RowMoveManager({ - cancelEditOnDrag: true - }); - - moveRowsPlugin.onBeforeMoveRows.subscribe(function (e, data) { - for (var i = 0; i < data.rows.length; i++) { - // no point in moving before or after itself - if (data.rows[i] == data.insertBefore || data.rows[i] == data.insertBefore - 1) { - e.stopPropagation(); - return false; - } - } - return true; - }); - - moveRowsPlugin.onMoveRows.subscribe(function (e, args) { - var extractedRows = [], left, right; - var rows = args.rows; - var insertBefore = args.insertBefore; - - var data = self.model.records.toJSON() - left = data.slice(0, insertBefore); - right= data.slice(insertBefore, data.length); - - rows.sort(function(a,b) { return a-b; }); - - for (var i = 0; i < rows.length; i++) { - extractedRows.push(data[rows[i]]); - } - - rows.reverse(); - - for (var i = 0; i < rows.length; i++) { - var row = rows[i]; - if (row < insertBefore) { - left.splice(row, 1); - } else { - right.splice(row - insertBefore, 1); - } - } - - data = left.concat(extractedRows.concat(right)); - var selectedRows = []; - for (var i = 0; i < rows.length; i++) - selectedRows.push(left.length + i); - - self.model.records.reset(data) - - }); - //register The plugin to handle row Reorder - if(this.state.get("gridOptions") && this.state.get("gridOptions").enableReOrderRow) { - self.grid.registerPlugin(moveRowsPlugin); - } - }, - - remove: function () { - this._slickHandler.unsubscribeAll(); - Backbone.View.prototype.remove.apply(this, arguments); - }, - - show: function() { - // If the div is hidden, SlickGrid will calculate wrongly some - // sizes so we must render it explicitly when the view is visible - if (!this.rendered){ - if (!this.grid){ - this.render(); - } - this.grid.init(); - this.rendered = true; - } - this.visible = true; - }, - - hide: function() { - this.visible = false; - } -}); - -// Add new grid Control to display a new row add menu bouton -// It display a simple side-bar menu ,for user to add new -// row to grid -my.GridControl= Backbone.View.extend({ - className: "recline-row-add", - // Template for row edit menu , change it if you don't love - template: '

    ', - - initialize: function(options){ - var self = this; - _.bindAll(this, 'render'); - this.state = new recline.Model.ObjectState(); - this.render(); - }, - - render: function() { - var self = this; - this.$el.html(this.template) - }, - - events : { - "click .recline-row-add" : "addNewRow" - }, - - addNewRow : function(e){ - e.preventDefault() - this.state.trigger("change") - } -}); - -})(jQuery, recline.View); - -/* -* Context menu for the column picker, adapted from -* http://mleibman.github.com/SlickGrid/examples/example-grouping -* -*/ -(function ($) { - function SlickColumnPicker(columns, grid, options) { - var $menu; - var columnCheckboxes; - - var defaults = { - fadeSpeed:250 - }; - - function init() { - grid.onHeaderContextMenu.subscribe(handleHeaderContextMenu); - options = $.extend({}, defaults, options); - - $menu = $('
    ").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}), -d=document.activeElement;c.wrap(b);if(c[0]===d||f.contains(c[0],d))f(d).focus();b=c.parent();if(c.css("position")=="static"){b.css({position:"relative"});c.css({position:"relative"})}else{f.extend(a,{position:c.css("position"),zIndex:c.css("z-index")});f.each(["top","left","bottom","right"],function(e,g){a[g]=c.css(g);if(isNaN(parseInt(a[g],10)))a[g]="auto"});c.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})}return b.css(a).show()},removeWrapper:function(c){var a,b=document.activeElement; -if(c.parent().is(".ui-effects-wrapper")){a=c.parent().replaceWith(c);if(c[0]===b||f.contains(c[0],b))f(b).focus();return a}return c},setTransition:function(c,a,b,d){d=d||{};f.each(a,function(e,g){unit=c.cssUnit(g);if(unit[0]>0)d[g]=unit[0]*b+unit[1]});return d}});f.fn.extend({effect:function(c){var a=k.apply(this,arguments),b={options:a[1],duration:a[2],callback:a[3]};a=b.options.mode;var d=f.effects[c];if(f.fx.off||!d)return a?this[a](b.duration,b.callback):this.each(function(){b.callback&&b.callback.call(this)}); -return d.call(this,b)},_show:f.fn.show,show:function(c){if(l(c))return this._show.apply(this,arguments);else{var a=k.apply(this,arguments);a[1].mode="show";return this.effect.apply(this,a)}},_hide:f.fn.hide,hide:function(c){if(l(c))return this._hide.apply(this,arguments);else{var a=k.apply(this,arguments);a[1].mode="hide";return this.effect.apply(this,a)}},__toggle:f.fn.toggle,toggle:function(c){if(l(c)||typeof c==="boolean"||f.isFunction(c))return this.__toggle.apply(this,arguments);else{var a=k.apply(this, -arguments);a[1].mode="toggle";return this.effect.apply(this,a)}},cssUnit:function(c){var a=this.css(c),b=[];f.each(["em","px","%","pt"],function(d,e){if(a.indexOf(e)>0)b=[parseFloat(a),e]});return b}});f.easing.jswing=f.easing.swing;f.extend(f.easing,{def:"easeOutQuad",swing:function(c,a,b,d,e){return f.easing[f.easing.def](c,a,b,d,e)},easeInQuad:function(c,a,b,d,e){return d*(a/=e)*a+b},easeOutQuad:function(c,a,b,d,e){return-d*(a/=e)*(a-2)+b},easeInOutQuad:function(c,a,b,d,e){if((a/=e/2)<1)return d/ -2*a*a+b;return-d/2*(--a*(a-2)-1)+b},easeInCubic:function(c,a,b,d,e){return d*(a/=e)*a*a+b},easeOutCubic:function(c,a,b,d,e){return d*((a=a/e-1)*a*a+1)+b},easeInOutCubic:function(c,a,b,d,e){if((a/=e/2)<1)return d/2*a*a*a+b;return d/2*((a-=2)*a*a+2)+b},easeInQuart:function(c,a,b,d,e){return d*(a/=e)*a*a*a+b},easeOutQuart:function(c,a,b,d,e){return-d*((a=a/e-1)*a*a*a-1)+b},easeInOutQuart:function(c,a,b,d,e){if((a/=e/2)<1)return d/2*a*a*a*a+b;return-d/2*((a-=2)*a*a*a-2)+b},easeInQuint:function(c,a,b, -d,e){return d*(a/=e)*a*a*a*a+b},easeOutQuint:function(c,a,b,d,e){return d*((a=a/e-1)*a*a*a*a+1)+b},easeInOutQuint:function(c,a,b,d,e){if((a/=e/2)<1)return d/2*a*a*a*a*a+b;return d/2*((a-=2)*a*a*a*a+2)+b},easeInSine:function(c,a,b,d,e){return-d*Math.cos(a/e*(Math.PI/2))+d+b},easeOutSine:function(c,a,b,d,e){return d*Math.sin(a/e*(Math.PI/2))+b},easeInOutSine:function(c,a,b,d,e){return-d/2*(Math.cos(Math.PI*a/e)-1)+b},easeInExpo:function(c,a,b,d,e){return a==0?b:d*Math.pow(2,10*(a/e-1))+b},easeOutExpo:function(c, -a,b,d,e){return a==e?b+d:d*(-Math.pow(2,-10*a/e)+1)+b},easeInOutExpo:function(c,a,b,d,e){if(a==0)return b;if(a==e)return b+d;if((a/=e/2)<1)return d/2*Math.pow(2,10*(a-1))+b;return d/2*(-Math.pow(2,-10*--a)+2)+b},easeInCirc:function(c,a,b,d,e){return-d*(Math.sqrt(1-(a/=e)*a)-1)+b},easeOutCirc:function(c,a,b,d,e){return d*Math.sqrt(1-(a=a/e-1)*a)+b},easeInOutCirc:function(c,a,b,d,e){if((a/=e/2)<1)return-d/2*(Math.sqrt(1-a*a)-1)+b;return d/2*(Math.sqrt(1-(a-=2)*a)+1)+b},easeInElastic:function(c,a,b, -d,e){c=1.70158;var g=0,h=d;if(a==0)return b;if((a/=e)==1)return b+d;g||(g=e*0.3);if(h").css({position:"absolute",visibility:"visible",left:-f*(h/d),top:-e*(i/c)}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:h/d,height:i/c,left:g.left+f*(h/d)+(a.options.mode=="show"?(f-Math.floor(d/2))*(h/d):0),top:g.top+e*(i/c)+(a.options.mode=="show"?(e-Math.floor(c/2))*(i/c):0),opacity:a.options.mode=="show"?0:1}).animate({left:g.left+f*(h/d)+(a.options.mode=="show"?0:(f-Math.floor(d/2))*(h/d)),top:g.top+ -e*(i/c)+(a.options.mode=="show"?0:(e-Math.floor(c/2))*(i/c)),opacity:a.options.mode=="show"?1:0},a.duration||500);setTimeout(function(){a.options.mode=="show"?b.css({visibility:"visible"}):b.css({visibility:"visible"}).hide();a.callback&&a.callback.apply(b[0]);b.dequeue();j("div.ui-effects-explode").remove()},a.duration||500)})}})(jQuery); -;/* - * jQuery UI Effects Fade 1.8.16 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI/Effects/Fade - * - * Depends: - * jquery.effects.core.js - */ -(function(b){b.effects.fade=function(a){return this.queue(function(){var c=b(this),d=b.effects.setMode(c,a.options.mode||"hide");c.animate({opacity:d},{queue:false,duration:a.duration,easing:a.options.easing,complete:function(){a.callback&&a.callback.apply(this,arguments);c.dequeue()}})})}})(jQuery); -;/* - * jQuery UI Effects Fold 1.8.16 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI/Effects/Fold - * - * Depends: - * jquery.effects.core.js - */ -(function(c){c.effects.fold=function(a){return this.queue(function(){var b=c(this),j=["position","top","bottom","left","right"],d=c.effects.setMode(b,a.options.mode||"hide"),g=a.options.size||15,h=!!a.options.horizFirst,k=a.duration?a.duration/2:c.fx.speeds._default/2;c.effects.save(b,j);b.show();var e=c.effects.createWrapper(b).css({overflow:"hidden"}),f=d=="show"!=h,l=f?["width","height"]:["height","width"];f=f?[e.width(),e.height()]:[e.height(),e.width()];var i=/([0-9]+)%/.exec(g);if(i)g=parseInt(i[1], -10)/100*f[d=="hide"?0:1];if(d=="show")e.css(h?{height:0,width:g}:{height:g,width:0});h={};i={};h[l[0]]=d=="show"?f[0]:g;i[l[1]]=d=="show"?f[1]:0;e.animate(h,k,a.options.easing).animate(i,k,a.options.easing,function(){d=="hide"&&b.hide();c.effects.restore(b,j);c.effects.removeWrapper(b);a.callback&&a.callback.apply(b[0],arguments);b.dequeue()})})}})(jQuery); -;/* - * jQuery UI Effects Highlight 1.8.16 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI/Effects/Highlight - * - * Depends: - * jquery.effects.core.js - */ -(function(b){b.effects.highlight=function(c){return this.queue(function(){var a=b(this),e=["backgroundImage","backgroundColor","opacity"],d=b.effects.setMode(a,c.options.mode||"show"),f={backgroundColor:a.css("backgroundColor")};if(d=="hide")f.opacity=0;b.effects.save(a,e);a.show().css({backgroundImage:"none",backgroundColor:c.options.color||"#ffff99"}).animate(f,{queue:false,duration:c.duration,easing:c.options.easing,complete:function(){d=="hide"&&a.hide();b.effects.restore(a,e);d=="show"&&!b.support.opacity&& -this.style.removeAttribute("filter");c.callback&&c.callback.apply(this,arguments);a.dequeue()}})})}})(jQuery); -;/* - * jQuery UI Effects Pulsate 1.8.16 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI/Effects/Pulsate - * - * Depends: - * jquery.effects.core.js - */ -(function(d){d.effects.pulsate=function(a){return this.queue(function(){var b=d(this),c=d.effects.setMode(b,a.options.mode||"show");times=(a.options.times||5)*2-1;duration=a.duration?a.duration/2:d.fx.speeds._default/2;isVisible=b.is(":visible");animateTo=0;if(!isVisible){b.css("opacity",0).show();animateTo=1}if(c=="hide"&&isVisible||c=="show"&&!isVisible)times--;for(c=0;c').appendTo(document.body).addClass(a.options.className).css({top:d.top,left:d.left,height:b.innerHeight(),width:b.innerWidth(),position:"absolute"}).animate(c,a.duration,a.options.easing,function(){f.remove();a.callback&&a.callback.apply(b[0],arguments); -b.dequeue()})})}})(jQuery); -; \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drag-2.2.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drag-2.2.js deleted file mode 100644 index f2c1d57..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drag-2.2.js +++ /dev/null @@ -1,402 +0,0 @@ -/*! - * jquery.event.drag - v 2.2 - * Copyright (c) 2010 Three Dub Media - http://threedubmedia.com - * Open Source MIT License - http://threedubmedia.com/code/license - */ -// Created: 2008-06-04 -// Updated: 2012-05-21 -// REQUIRES: jquery 1.7.x - -;(function( $ ){ - -// add the jquery instance method -$.fn.drag = function( str, arg, opts ){ - // figure out the event type - var type = typeof str == "string" ? str : "", - // figure out the event handler... - fn = $.isFunction( str ) ? str : $.isFunction( arg ) ? arg : null; - // fix the event type - if ( type.indexOf("drag") !== 0 ) - type = "drag"+ type; - // were options passed - opts = ( str == fn ? arg : opts ) || {}; - // trigger or bind event handler - return fn ? this.bind( type, opts, fn ) : this.trigger( type ); -}; - -// local refs (increase compression) -var $event = $.event, -$special = $event.special, -// configure the drag special event -drag = $special.drag = { - - // these are the default settings - defaults: { - which: 1, // mouse button pressed to start drag sequence - distance: 0, // distance dragged before dragstart - not: ':input', // selector to suppress dragging on target elements - handle: null, // selector to match handle target elements - relative: false, // true to use "position", false to use "offset" - drop: true, // false to suppress drop events, true or selector to allow - click: false // false to suppress click events after dragend (no proxy) - }, - - // the key name for stored drag data - datakey: "dragdata", - - // prevent bubbling for better performance - noBubble: true, - - // count bound related events - add: function( obj ){ - // read the interaction data - var data = $.data( this, drag.datakey ), - // read any passed options - opts = obj.data || {}; - // count another realted event - data.related += 1; - // extend data options bound with this event - // don't iterate "opts" in case it is a node - $.each( drag.defaults, function( key, def ){ - if ( opts[ key ] !== undefined ) - data[ key ] = opts[ key ]; - }); - }, - - // forget unbound related events - remove: function(){ - $.data( this, drag.datakey ).related -= 1; - }, - - // configure interaction, capture settings - setup: function(){ - // check for related events - if ( $.data( this, drag.datakey ) ) - return; - // initialize the drag data with copied defaults - var data = $.extend({ related:0 }, drag.defaults ); - // store the interaction data - $.data( this, drag.datakey, data ); - // bind the mousedown event, which starts drag interactions - $event.add( this, "touchstart mousedown", drag.init, data ); - // prevent image dragging in IE... - if ( this.attachEvent ) - this.attachEvent("ondragstart", drag.dontstart ); - }, - - // destroy configured interaction - teardown: function(){ - var data = $.data( this, drag.datakey ) || {}; - // check for related events - if ( data.related ) - return; - // remove the stored data - $.removeData( this, drag.datakey ); - // remove the mousedown event - $event.remove( this, "touchstart mousedown", drag.init ); - // enable text selection - drag.textselect( true ); - // un-prevent image dragging in IE... - if ( this.detachEvent ) - this.detachEvent("ondragstart", drag.dontstart ); - }, - - // initialize the interaction - init: function( event ){ - // sorry, only one touch at a time - if ( drag.touched ) - return; - // the drag/drop interaction data - var dd = event.data, results; - // check the which directive - if ( event.which != 0 && dd.which > 0 && event.which != dd.which ) - return; - // check for suppressed selector - if ( $( event.target ).is( dd.not ) ) - return; - // check for handle selector - if ( dd.handle && !$( event.target ).closest( dd.handle, event.currentTarget ).length ) - return; - - drag.touched = event.type == 'touchstart' ? this : null; - dd.propagates = 1; - dd.mousedown = this; - dd.interactions = [ drag.interaction( this, dd ) ]; - dd.target = event.target; - dd.pageX = event.pageX; - dd.pageY = event.pageY; - dd.dragging = null; - // handle draginit event... - results = drag.hijack( event, "draginit", dd ); - // early cancel - if ( !dd.propagates ) - return; - // flatten the result set - results = drag.flatten( results ); - // insert new interaction elements - if ( results && results.length ){ - dd.interactions = []; - $.each( results, function(){ - dd.interactions.push( drag.interaction( this, dd ) ); - }); - } - // remember how many interactions are propagating - dd.propagates = dd.interactions.length; - // locate and init the drop targets - if ( dd.drop !== false && $special.drop ) - $special.drop.handler( event, dd ); - // disable text selection - drag.textselect( false ); - // bind additional events... - if ( drag.touched ) - $event.add( drag.touched, "touchmove touchend", drag.handler, dd ); - else - $event.add( document, "mousemove mouseup", drag.handler, dd ); - // helps prevent text selection or scrolling - if ( !drag.touched || dd.live ) - return false; - }, - - // returns an interaction object - interaction: function( elem, dd ){ - var offset = $( elem )[ dd.relative ? "position" : "offset" ]() || { top:0, left:0 }; - return { - drag: elem, - callback: new drag.callback(), - droppable: [], - offset: offset - }; - }, - - // handle drag-releatd DOM events - handler: function( event ){ - // read the data before hijacking anything - var dd = event.data; - // handle various events - switch ( event.type ){ - // mousemove, check distance, start dragging - case !dd.dragging && 'touchmove': - event.preventDefault(); - case !dd.dragging && 'mousemove': - // drag tolerance, x≤ + y≤ = distance≤ - if ( Math.pow( event.pageX-dd.pageX, 2 ) + Math.pow( event.pageY-dd.pageY, 2 ) < Math.pow( dd.distance, 2 ) ) - break; // distance tolerance not reached - event.target = dd.target; // force target from "mousedown" event (fix distance issue) - drag.hijack( event, "dragstart", dd ); // trigger "dragstart" - if ( dd.propagates ) // "dragstart" not rejected - dd.dragging = true; // activate interaction - // mousemove, dragging - case 'touchmove': - event.preventDefault(); - case 'mousemove': - if ( dd.dragging ){ - // trigger "drag" - drag.hijack( event, "drag", dd ); - if ( dd.propagates ){ - // manage drop events - if ( dd.drop !== false && $special.drop ) - $special.drop.handler( event, dd ); // "dropstart", "dropend" - break; // "drag" not rejected, stop - } - event.type = "mouseup"; // helps "drop" handler behave - } - // mouseup, stop dragging - case 'touchend': - case 'mouseup': - default: - if ( drag.touched ) - $event.remove( drag.touched, "touchmove touchend", drag.handler ); // remove touch events - else - $event.remove( document, "mousemove mouseup", drag.handler ); // remove page events - if ( dd.dragging ){ - if ( dd.drop !== false && $special.drop ) - $special.drop.handler( event, dd ); // "drop" - drag.hijack( event, "dragend", dd ); // trigger "dragend" - } - drag.textselect( true ); // enable text selection - // if suppressing click events... - if ( dd.click === false && dd.dragging ) - $.data( dd.mousedown, "suppress.click", new Date().getTime() + 5 ); - dd.dragging = drag.touched = false; // deactivate element - break; - } - }, - - // re-use event object for custom events - hijack: function( event, type, dd, x, elem ){ - // not configured - if ( !dd ) - return; - // remember the original event and type - var orig = { event:event.originalEvent, type:event.type }, - // is the event drag related or drog related? - mode = type.indexOf("drop") ? "drag" : "drop", - // iteration vars - result, i = x || 0, ia, $elems, callback, - len = !isNaN( x ) ? x : dd.interactions.length; - // modify the event type - event.type = type; - // remove the original event - event.originalEvent = null; - // initialize the results - dd.results = []; - // handle each interacted element - do if ( ia = dd.interactions[ i ] ){ - // validate the interaction - if ( type !== "dragend" && ia.cancelled ) - continue; - // set the dragdrop properties on the event object - callback = drag.properties( event, dd, ia ); - // prepare for more results - ia.results = []; - // handle each element - $( elem || ia[ mode ] || dd.droppable ).each(function( p, subject ){ - // identify drag or drop targets individually - callback.target = subject; - // force propagtion of the custom event - event.isPropagationStopped = function(){ return false; }; - // handle the event - result = subject ? $event.dispatch.call( subject, event, callback ) : null; - // stop the drag interaction for this element - if ( result === false ){ - if ( mode == "drag" ){ - ia.cancelled = true; - dd.propagates -= 1; - } - if ( type == "drop" ){ - ia[ mode ][p] = null; - } - } - // assign any dropinit elements - else if ( type == "dropinit" ) - ia.droppable.push( drag.element( result ) || subject ); - // accept a returned proxy element - if ( type == "dragstart" ) - ia.proxy = $( drag.element( result ) || ia.drag )[0]; - // remember this result - ia.results.push( result ); - // forget the event result, for recycling - delete event.result; - // break on cancelled handler - if ( type !== "dropinit" ) - return result; - }); - // flatten the results - dd.results[ i ] = drag.flatten( ia.results ); - // accept a set of valid drop targets - if ( type == "dropinit" ) - ia.droppable = drag.flatten( ia.droppable ); - // locate drop targets - if ( type == "dragstart" && !ia.cancelled ) - callback.update(); - } - while ( ++i < len ) - // restore the original event & type - event.type = orig.type; - event.originalEvent = orig.event; - // return all handler results - return drag.flatten( dd.results ); - }, - - // extend the callback object with drag/drop properties... - properties: function( event, dd, ia ){ - var obj = ia.callback; - // elements - obj.drag = ia.drag; - obj.proxy = ia.proxy || ia.drag; - // starting mouse position - obj.startX = dd.pageX; - obj.startY = dd.pageY; - // current distance dragged - obj.deltaX = event.pageX - dd.pageX; - obj.deltaY = event.pageY - dd.pageY; - // original element position - obj.originalX = ia.offset.left; - obj.originalY = ia.offset.top; - // adjusted element position - obj.offsetX = obj.originalX + obj.deltaX; - obj.offsetY = obj.originalY + obj.deltaY; - // assign the drop targets information - obj.drop = drag.flatten( ( ia.drop || [] ).slice() ); - obj.available = drag.flatten( ( ia.droppable || [] ).slice() ); - return obj; - }, - - // determine is the argument is an element or jquery instance - element: function( arg ){ - if ( arg && ( arg.jquery || arg.nodeType == 1 ) ) - return arg; - }, - - // flatten nested jquery objects and arrays into a single dimension array - flatten: function( arr ){ - return $.map( arr, function( member ){ - return member && member.jquery ? $.makeArray( member ) : - member && member.length ? drag.flatten( member ) : member; - }); - }, - - // toggles text selection attributes ON (true) or OFF (false) - textselect: function( bool ){ - $( document )[ bool ? "unbind" : "bind" ]("selectstart", drag.dontstart ) - .css("MozUserSelect", bool ? "" : "none" ); - // .attr("unselectable", bool ? "off" : "on" ) - document.unselectable = bool ? "off" : "on"; - }, - - // suppress "selectstart" and "ondragstart" events - dontstart: function(){ - return false; - }, - - // a callback instance contructor - callback: function(){} - -}; - -// callback methods -drag.callback.prototype = { - update: function(){ - if ( $special.drop && this.available.length ) - $.each( this.available, function( i ){ - $special.drop.locate( this, i ); - }); - } -}; - -// patch $.event.$dispatch to allow suppressing clicks -var $dispatch = $event.dispatch; -$event.dispatch = function( event ){ - if ( $.data( this, "suppress."+ event.type ) - new Date().getTime() > 0 ){ - $.removeData( this, "suppress."+ event.type ); - return; - } - return $dispatch.apply( this, arguments ); -}; - -// event fix hooks for touch events... -var touchHooks = -$event.fixHooks.touchstart = -$event.fixHooks.touchmove = -$event.fixHooks.touchend = -$event.fixHooks.touchcancel = { - props: "clientX clientY pageX pageY screenX screenY".split( " " ), - filter: function( event, orig ) { - if ( orig ){ - var touched = ( orig.touches && orig.touches[0] ) - || ( orig.changedTouches && orig.changedTouches[0] ) - || null; - // iOS webkit: touchstart, touchmove, touchend - if ( touched ) - $.each( touchHooks.props, function( i, prop ){ - event[ prop ] = touched[ prop ]; - }); - } - return event; - } -}; - -// share the same special event configuration with related events... -$special.draginit = $special.dragstart = $special.dragend = drag; - -})( jQuery ); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drop-2.2.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drop-2.2.js deleted file mode 100644 index 7599ef9..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drop-2.2.js +++ /dev/null @@ -1,302 +0,0 @@ -/*! - * jquery.event.drop - v 2.2 - * Copyright (c) 2010 Three Dub Media - http://threedubmedia.com - * Open Source MIT License - http://threedubmedia.com/code/license - */ -// Created: 2008-06-04 -// Updated: 2012-05-21 -// REQUIRES: jquery 1.7.x, event.drag 2.2 - -;(function($){ // secure $ jQuery alias - -// Events: drop, dropstart, dropend - -// add the jquery instance method -$.fn.drop = function( str, arg, opts ){ - // figure out the event type - var type = typeof str == "string" ? str : "", - // figure out the event handler... - fn = $.isFunction( str ) ? str : $.isFunction( arg ) ? arg : null; - // fix the event type - if ( type.indexOf("drop") !== 0 ) - type = "drop"+ type; - // were options passed - opts = ( str == fn ? arg : opts ) || {}; - // trigger or bind event handler - return fn ? this.bind( type, opts, fn ) : this.trigger( type ); -}; - -// DROP MANAGEMENT UTILITY -// returns filtered drop target elements, caches their positions -$.drop = function( opts ){ - opts = opts || {}; - // safely set new options... - drop.multi = opts.multi === true ? Infinity : - opts.multi === false ? 1 : !isNaN( opts.multi ) ? opts.multi : drop.multi; - drop.delay = opts.delay || drop.delay; - drop.tolerance = $.isFunction( opts.tolerance ) ? opts.tolerance : - opts.tolerance === null ? null : drop.tolerance; - drop.mode = opts.mode || drop.mode || 'intersect'; -}; - -// local refs (increase compression) -var $event = $.event, -$special = $event.special, -// configure the drop special event -drop = $.event.special.drop = { - - // these are the default settings - multi: 1, // allow multiple drop winners per dragged element - delay: 20, // async timeout delay - mode: 'overlap', // drop tolerance mode - - // internal cache - targets: [], - - // the key name for stored drop data - datakey: "dropdata", - - // prevent bubbling for better performance - noBubble: true, - - // count bound related events - add: function( obj ){ - // read the interaction data - var data = $.data( this, drop.datakey ); - // count another realted event - data.related += 1; - }, - - // forget unbound related events - remove: function(){ - $.data( this, drop.datakey ).related -= 1; - }, - - // configure the interactions - setup: function(){ - // check for related events - if ( $.data( this, drop.datakey ) ) - return; - // initialize the drop element data - var data = { - related: 0, - active: [], - anyactive: 0, - winner: 0, - location: {} - }; - // store the drop data on the element - $.data( this, drop.datakey, data ); - // store the drop target in internal cache - drop.targets.push( this ); - }, - - // destroy the configure interaction - teardown: function(){ - var data = $.data( this, drop.datakey ) || {}; - // check for related events - if ( data.related ) - return; - // remove the stored data - $.removeData( this, drop.datakey ); - // reference the targeted element - var element = this; - // remove from the internal cache - drop.targets = $.grep( drop.targets, function( target ){ - return ( target !== element ); - }); - }, - - // shared event handler - handler: function( event, dd ){ - // local vars - var results, $targets; - // make sure the right data is available - if ( !dd ) - return; - // handle various events - switch ( event.type ){ - // draginit, from $.event.special.drag - case 'mousedown': // DROPINIT >> - case 'touchstart': // DROPINIT >> - // collect and assign the drop targets - $targets = $( drop.targets ); - if ( typeof dd.drop == "string" ) - $targets = $targets.filter( dd.drop ); - // reset drop data winner properties - $targets.each(function(){ - var data = $.data( this, drop.datakey ); - data.active = []; - data.anyactive = 0; - data.winner = 0; - }); - // set available target elements - dd.droppable = $targets; - // activate drop targets for the initial element being dragged - $special.drag.hijack( event, "dropinit", dd ); - break; - // drag, from $.event.special.drag - case 'mousemove': // TOLERATE >> - case 'touchmove': // TOLERATE >> - drop.event = event; // store the mousemove event - if ( !drop.timer ) - // monitor drop targets - drop.tolerate( dd ); - break; - // dragend, from $.event.special.drag - case 'mouseup': // DROP >> DROPEND >> - case 'touchend': // DROP >> DROPEND >> - drop.timer = clearTimeout( drop.timer ); // delete timer - if ( dd.propagates ){ - $special.drag.hijack( event, "drop", dd ); - $special.drag.hijack( event, "dropend", dd ); - } - break; - - } - }, - - // returns the location positions of an element - locate: function( elem, index ){ - var data = $.data( elem, drop.datakey ), - $elem = $( elem ), - posi = $elem.offset() || {}, - height = $elem.outerHeight(), - width = $elem.outerWidth(), - location = { - elem: elem, - width: width, - height: height, - top: posi.top, - left: posi.left, - right: posi.left + width, - bottom: posi.top + height - }; - // drag elements might not have dropdata - if ( data ){ - data.location = location; - data.index = index; - data.elem = elem; - } - return location; - }, - - // test the location positions of an element against another OR an X,Y coord - contains: function( target, test ){ // target { location } contains test [x,y] or { location } - return ( ( test[0] || test.left ) >= target.left && ( test[0] || test.right ) <= target.right - && ( test[1] || test.top ) >= target.top && ( test[1] || test.bottom ) <= target.bottom ); - }, - - // stored tolerance modes - modes: { // fn scope: "$.event.special.drop" object - // target with mouse wins, else target with most overlap wins - 'intersect': function( event, proxy, target ){ - return this.contains( target, [ event.pageX, event.pageY ] ) ? // check cursor - 1e9 : this.modes.overlap.apply( this, arguments ); // check overlap - }, - // target with most overlap wins - 'overlap': function( event, proxy, target ){ - // calculate the area of overlap... - return Math.max( 0, Math.min( target.bottom, proxy.bottom ) - Math.max( target.top, proxy.top ) ) - * Math.max( 0, Math.min( target.right, proxy.right ) - Math.max( target.left, proxy.left ) ); - }, - // proxy is completely contained within target bounds - 'fit': function( event, proxy, target ){ - return this.contains( target, proxy ) ? 1 : 0; - }, - // center of the proxy is contained within target bounds - 'middle': function( event, proxy, target ){ - return this.contains( target, [ proxy.left + proxy.width * .5, proxy.top + proxy.height * .5 ] ) ? 1 : 0; - } - }, - - // sort drop target cache by by winner (dsc), then index (asc) - sort: function( a, b ){ - return ( b.winner - a.winner ) || ( a.index - b.index ); - }, - - // async, recursive tolerance execution - tolerate: function( dd ){ - // declare local refs - var i, drp, drg, data, arr, len, elem, - // interaction iteration variables - x = 0, ia, end = dd.interactions.length, - // determine the mouse coords - xy = [ drop.event.pageX, drop.event.pageY ], - // custom or stored tolerance fn - tolerance = drop.tolerance || drop.modes[ drop.mode ]; - // go through each passed interaction... - do if ( ia = dd.interactions[x] ){ - // check valid interaction - if ( !ia ) - return; - // initialize or clear the drop data - ia.drop = []; - // holds the drop elements - arr = []; - len = ia.droppable.length; - // determine the proxy location, if needed - if ( tolerance ) - drg = drop.locate( ia.proxy ); - // reset the loop - i = 0; - // loop each stored drop target - do if ( elem = ia.droppable[i] ){ - data = $.data( elem, drop.datakey ); - drp = data.location; - if ( !drp ) continue; - // find a winner: tolerance function is defined, call it - data.winner = tolerance ? tolerance.call( drop, drop.event, drg, drp ) - // mouse position is always the fallback - : drop.contains( drp, xy ) ? 1 : 0; - arr.push( data ); - } while ( ++i < len ); // loop - // sort the drop targets - arr.sort( drop.sort ); - // reset the loop - i = 0; - // loop through all of the targets again - do if ( data = arr[ i ] ){ - // winners... - if ( data.winner && ia.drop.length < drop.multi ){ - // new winner... dropstart - if ( !data.active[x] && !data.anyactive ){ - // check to make sure that this is not prevented - if ( $special.drag.hijack( drop.event, "dropstart", dd, x, data.elem )[0] !== false ){ - data.active[x] = 1; - data.anyactive += 1; - } - // if false, it is not a winner - else - data.winner = 0; - } - // if it is still a winner - if ( data.winner ) - ia.drop.push( data.elem ); - } - // losers... - else if ( data.active[x] && data.anyactive == 1 ){ - // former winner... dropend - $special.drag.hijack( drop.event, "dropend", dd, x, data.elem ); - data.active[x] = 0; - data.anyactive -= 1; - } - } while ( ++i < len ); // loop - } while ( ++x < end ) // loop - // check if the mouse is still moving or is idle - if ( drop.last && xy[0] == drop.last.pageX && xy[1] == drop.last.pageY ) - delete drop.timer; // idle, don't recurse - else // recurse - drop.timer = setTimeout(function(){ - drop.tolerate( dd ); - }, drop.delay ); - // remember event, to compare idleness - drop.last = drop.event; - } - -}; - -// share the same special event configuration with related events... -$special.dropinit = $special.dropstart = $special.dropend = drop; - -})(jQuery); // confine scope \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.autotooltips.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.autotooltips.js deleted file mode 100644 index 955684f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.autotooltips.js +++ /dev/null @@ -1,83 +0,0 @@ -(function ($) { - // Register namespace - $.extend(true, window, { - "Slick": { - "AutoTooltips": AutoTooltips - } - }); - - /** - * AutoTooltips plugin to show/hide tooltips when columns are too narrow to fit content. - * @constructor - * @param {boolean} [options.enableForCells=true] - Enable tooltip for grid cells - * @param {boolean} [options.enableForHeaderCells=false] - Enable tooltip for header cells - * @param {number} [options.maxToolTipLength=null] - The maximum length for a tooltip - */ - function AutoTooltips(options) { - var _grid; - var _self = this; - var _defaults = { - enableForCells: true, - enableForHeaderCells: false, - maxToolTipLength: null - }; - - /** - * Initialize plugin. - */ - function init(grid) { - options = $.extend(true, {}, _defaults, options); - _grid = grid; - if (options.enableForCells) _grid.onMouseEnter.subscribe(handleMouseEnter); - if (options.enableForHeaderCells) _grid.onHeaderMouseEnter.subscribe(handleHeaderMouseEnter); - } - - /** - * Destroy plugin. - */ - function destroy() { - if (options.enableForCells) _grid.onMouseEnter.unsubscribe(handleMouseEnter); - if (options.enableForHeaderCells) _grid.onHeaderMouseEnter.unsubscribe(handleHeaderMouseEnter); - } - - /** - * Handle mouse entering grid cell to add/remove tooltip. - * @param {jQuery.Event} e - The event - */ - function handleMouseEnter(e) { - var cell = _grid.getCellFromEvent(e); - if (cell) { - var $node = $(_grid.getCellNode(cell.row, cell.cell)); - var text; - if ($node.innerWidth() < $node[0].scrollWidth) { - text = $.trim($node.text()); - if (options.maxToolTipLength && text.length > options.maxToolTipLength) { - text = text.substr(0, options.maxToolTipLength - 3) + "..."; - } - } else { - text = ""; - } - $node.attr("title", text); - } - } - - /** - * Handle mouse entering header cell to add/remove tooltip. - * @param {jQuery.Event} e - The event - * @param {object} args.column - The column definition - */ - function handleHeaderMouseEnter(e, args) { - var column = args.column, - $node = $(e.target).closest(".slick-header-column"); - if (!column.toolTip) { - $node.attr("title", ($node.innerWidth() < $node[0].scrollWidth) ? column.name : ""); - } - } - - // Public API - $.extend(this, { - "init": init, - "destroy": destroy - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellcopymanager.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellcopymanager.js deleted file mode 100644 index c74018d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellcopymanager.js +++ /dev/null @@ -1,86 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "CellCopyManager": CellCopyManager - } - }); - - - function CellCopyManager() { - var _grid; - var _self = this; - var _copiedRanges; - - function init(grid) { - _grid = grid; - _grid.onKeyDown.subscribe(handleKeyDown); - } - - function destroy() { - _grid.onKeyDown.unsubscribe(handleKeyDown); - } - - function handleKeyDown(e, args) { - var ranges; - if (!_grid.getEditorLock().isActive()) { - if (e.which == $.ui.keyCode.ESCAPE) { - if (_copiedRanges) { - e.preventDefault(); - clearCopySelection(); - _self.onCopyCancelled.notify({ranges: _copiedRanges}); - _copiedRanges = null; - } - } - - if (e.which == 67 && (e.ctrlKey || e.metaKey)) { - ranges = _grid.getSelectionModel().getSelectedRanges(); - if (ranges.length != 0) { - e.preventDefault(); - _copiedRanges = ranges; - markCopySelection(ranges); - _self.onCopyCells.notify({ranges: ranges}); - } - } - - if (e.which == 86 && (e.ctrlKey || e.metaKey)) { - if (_copiedRanges) { - e.preventDefault(); - clearCopySelection(); - ranges = _grid.getSelectionModel().getSelectedRanges(); - _self.onPasteCells.notify({from: _copiedRanges, to: ranges}); - _copiedRanges = null; - } - } - } - } - - function markCopySelection(ranges) { - var columns = _grid.getColumns(); - var hash = {}; - for (var i = 0; i < ranges.length; i++) { - for (var j = ranges[i].fromRow; j <= ranges[i].toRow; j++) { - hash[j] = {}; - for (var k = ranges[i].fromCell; k <= ranges[i].toCell; k++) { - hash[j][columns[k].id] = "copied"; - } - } - } - _grid.setCellCssStyles("copy-manager", hash); - } - - function clearCopySelection() { - _grid.removeCellCssStyles("copy-manager"); - } - - $.extend(this, { - "init": init, - "destroy": destroy, - "clearCopySelection": clearCopySelection, - - "onCopyCells": new Slick.Event(), - "onCopyCancelled": new Slick.Event(), - "onPasteCells": new Slick.Event() - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangedecorator.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangedecorator.js deleted file mode 100644 index 0cbe71d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangedecorator.js +++ /dev/null @@ -1,66 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "CellRangeDecorator": CellRangeDecorator - } - }); - - /*** - * Displays an overlay on top of a given cell range. - * - * TODO: - * Currently, it blocks mouse events to DOM nodes behind it. - * Use FF and WebKit-specific "pointer-events" CSS style, or some kind of event forwarding. - * Could also construct the borders separately using 4 individual DIVs. - * - * @param {Grid} grid - * @param {Object} options - */ - function CellRangeDecorator(grid, options) { - var _elem; - var _defaults = { - selectionCssClass: 'slick-range-decorator', - selectionCss: { - "zIndex": "9999", - "border": "2px dashed red" - } - }; - - options = $.extend(true, {}, _defaults, options); - - - function show(range) { - if (!_elem) { - _elem = $("
    ", {css: options.selectionCss}) - .addClass(options.selectionCssClass) - .css("position", "absolute") - .appendTo(grid.getCanvasNode()); - } - - var from = grid.getCellNodeBox(range.fromRow, range.fromCell); - var to = grid.getCellNodeBox(range.toRow, range.toCell); - - _elem.css({ - top: from.top - 1, - left: from.left - 1, - height: to.bottom - from.top - 2, - width: to.right - from.left - 2 - }); - - return _elem; - } - - function hide() { - if (_elem) { - _elem.remove(); - _elem = null; - } - } - - $.extend(this, { - "show": show, - "hide": hide - }); - } -})(jQuery); diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangeselector.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangeselector.js deleted file mode 100644 index 520b17f..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangeselector.js +++ /dev/null @@ -1,113 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "CellRangeSelector": CellRangeSelector - } - }); - - - function CellRangeSelector(options) { - var _grid; - var _canvas; - var _dragging; - var _decorator; - var _self = this; - var _handler = new Slick.EventHandler(); - var _defaults = { - selectionCss: { - "border": "2px dashed blue" - } - }; - - - function init(grid) { - options = $.extend(true, {}, _defaults, options); - _decorator = new Slick.CellRangeDecorator(grid, options); - _grid = grid; - _canvas = _grid.getCanvasNode(); - _handler - .subscribe(_grid.onDragInit, handleDragInit) - .subscribe(_grid.onDragStart, handleDragStart) - .subscribe(_grid.onDrag, handleDrag) - .subscribe(_grid.onDragEnd, handleDragEnd); - } - - function destroy() { - _handler.unsubscribeAll(); - } - - function handleDragInit(e, dd) { - // prevent the grid from cancelling drag'n'drop by default - e.stopImmediatePropagation(); - } - - function handleDragStart(e, dd) { - var cell = _grid.getCellFromEvent(e); - if (_self.onBeforeCellRangeSelected.notify(cell) !== false) { - if (_grid.canCellBeSelected(cell.row, cell.cell)) { - _dragging = true; - e.stopImmediatePropagation(); - } - } - if (!_dragging) { - return; - } - - _grid.focus(); - - var start = _grid.getCellFromPoint( - dd.startX - $(_canvas).offset().left, - dd.startY - $(_canvas).offset().top); - - dd.range = {start: start, end: {}}; - - return _decorator.show(new Slick.Range(start.row, start.cell)); - } - - function handleDrag(e, dd) { - if (!_dragging) { - return; - } - e.stopImmediatePropagation(); - - var end = _grid.getCellFromPoint( - e.pageX - $(_canvas).offset().left, - e.pageY - $(_canvas).offset().top); - - if (!_grid.canCellBeSelected(end.row, end.cell)) { - return; - } - - dd.range.end = end; - _decorator.show(new Slick.Range(dd.range.start.row, dd.range.start.cell, end.row, end.cell)); - } - - function handleDragEnd(e, dd) { - if (!_dragging) { - return; - } - - _dragging = false; - e.stopImmediatePropagation(); - - _decorator.hide(); - _self.onCellRangeSelected.notify({ - range: new Slick.Range( - dd.range.start.row, - dd.range.start.cell, - dd.range.end.row, - dd.range.end.cell - ) - }); - } - - $.extend(this, { - "init": init, - "destroy": destroy, - - "onBeforeCellRangeSelected": new Slick.Event(), - "onCellRangeSelected": new Slick.Event() - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellselectionmodel.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellselectionmodel.js deleted file mode 100644 index 74bc3eb..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellselectionmodel.js +++ /dev/null @@ -1,154 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "CellSelectionModel": CellSelectionModel - } - }); - - - function CellSelectionModel(options) { - var _grid; - var _canvas; - var _ranges = []; - var _self = this; - var _selector = new Slick.CellRangeSelector({ - "selectionCss": { - "border": "2px solid black" - } - }); - var _options; - var _defaults = { - selectActiveCell: true - }; - - - function init(grid) { - _options = $.extend(true, {}, _defaults, options); - _grid = grid; - _canvas = _grid.getCanvasNode(); - _grid.onActiveCellChanged.subscribe(handleActiveCellChange); - _grid.onKeyDown.subscribe(handleKeyDown); - grid.registerPlugin(_selector); - _selector.onCellRangeSelected.subscribe(handleCellRangeSelected); - _selector.onBeforeCellRangeSelected.subscribe(handleBeforeCellRangeSelected); - } - - function destroy() { - _grid.onActiveCellChanged.unsubscribe(handleActiveCellChange); - _grid.onKeyDown.unsubscribe(handleKeyDown); - _selector.onCellRangeSelected.unsubscribe(handleCellRangeSelected); - _selector.onBeforeCellRangeSelected.unsubscribe(handleBeforeCellRangeSelected); - _grid.unregisterPlugin(_selector); - } - - function removeInvalidRanges(ranges) { - var result = []; - - for (var i = 0; i < ranges.length; i++) { - var r = ranges[i]; - if (_grid.canCellBeSelected(r.fromRow, r.fromCell) && _grid.canCellBeSelected(r.toRow, r.toCell)) { - result.push(r); - } - } - - return result; - } - - function setSelectedRanges(ranges) { - _ranges = removeInvalidRanges(ranges); - _self.onSelectedRangesChanged.notify(_ranges); - } - - function getSelectedRanges() { - return _ranges; - } - - function handleBeforeCellRangeSelected(e, args) { - if (_grid.getEditorLock().isActive()) { - e.stopPropagation(); - return false; - } - } - - function handleCellRangeSelected(e, args) { - setSelectedRanges([args.range]); - } - - function handleActiveCellChange(e, args) { - if (_options.selectActiveCell && args.row != null && args.cell != null) { - setSelectedRanges([new Slick.Range(args.row, args.cell)]); - } - } - - function handleKeyDown(e) { - /*** - * Кey codes - * 37 left - * 38 up - * 39 right - * 40 down - */ - var ranges, last; - var active = _grid.getActiveCell(); - - if ( active && e.shiftKey && !e.ctrlKey && !e.altKey && - (e.which == 37 || e.which == 39 || e.which == 38 || e.which == 40) ) { - - ranges = getSelectedRanges(); - if (!ranges.length) - ranges.push(new Slick.Range(active.row, active.cell)); - - // keyboard can work with last range only - last = ranges.pop(); - - // can't handle selection out of active cell - if (!last.contains(active.row, active.cell)) - last = new Slick.Range(active.row, active.cell); - - var dRow = last.toRow - last.fromRow, - dCell = last.toCell - last.fromCell, - // walking direction - dirRow = active.row == last.fromRow ? 1 : -1, - dirCell = active.cell == last.fromCell ? 1 : -1; - - if (e.which == 37) { - dCell -= dirCell; - } else if (e.which == 39) { - dCell += dirCell ; - } else if (e.which == 38) { - dRow -= dirRow; - } else if (e.which == 40) { - dRow += dirRow; - } - - // define new selection range - var new_last = new Slick.Range(active.row, active.cell, active.row + dirRow*dRow, active.cell + dirCell*dCell); - if (removeInvalidRanges([new_last]).length) { - ranges.push(new_last); - var viewRow = dirRow > 0 ? new_last.toRow : new_last.fromRow; - var viewCell = dirCell > 0 ? new_last.toCell : new_last.fromCell; - _grid.scrollRowIntoView(viewRow); - _grid.scrollCellIntoView(viewRow, viewCell); - } - else - ranges.push(last); - - setSelectedRanges(ranges); - - e.preventDefault(); - e.stopPropagation(); - } - } - - $.extend(this, { - "getSelectedRanges": getSelectedRanges, - "setSelectedRanges": setSelectedRanges, - - "init": init, - "destroy": destroy, - - "onSelectedRangesChanged": new Slick.Event() - }); - } -})(jQuery); diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.checkboxselectcolumn.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.checkboxselectcolumn.js deleted file mode 100644 index 83d8d50..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.checkboxselectcolumn.js +++ /dev/null @@ -1,153 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "CheckboxSelectColumn": CheckboxSelectColumn - } - }); - - - function CheckboxSelectColumn(options) { - var _grid; - var _self = this; - var _handler = new Slick.EventHandler(); - var _selectedRowsLookup = {}; - var _defaults = { - columnId: "_checkbox_selector", - cssClass: null, - toolTip: "Select/Deselect All", - width: 30 - }; - - var _options = $.extend(true, {}, _defaults, options); - - function init(grid) { - _grid = grid; - _handler - .subscribe(_grid.onSelectedRowsChanged, handleSelectedRowsChanged) - .subscribe(_grid.onClick, handleClick) - .subscribe(_grid.onHeaderClick, handleHeaderClick) - .subscribe(_grid.onKeyDown, handleKeyDown); - } - - function destroy() { - _handler.unsubscribeAll(); - } - - function handleSelectedRowsChanged(e, args) { - var selectedRows = _grid.getSelectedRows(); - var lookup = {}, row, i; - for (i = 0; i < selectedRows.length; i++) { - row = selectedRows[i]; - lookup[row] = true; - if (lookup[row] !== _selectedRowsLookup[row]) { - _grid.invalidateRow(row); - delete _selectedRowsLookup[row]; - } - } - for (i in _selectedRowsLookup) { - _grid.invalidateRow(i); - } - _selectedRowsLookup = lookup; - _grid.render(); - - if (selectedRows.length && selectedRows.length == _grid.getDataLength()) { - _grid.updateColumnHeader(_options.columnId, "", _options.toolTip); - } else { - _grid.updateColumnHeader(_options.columnId, "", _options.toolTip); - } - } - - function handleKeyDown(e, args) { - if (e.which == 32) { - if (_grid.getColumns()[args.cell].id === _options.columnId) { - // if editing, try to commit - if (!_grid.getEditorLock().isActive() || _grid.getEditorLock().commitCurrentEdit()) { - toggleRowSelection(args.row); - } - e.preventDefault(); - e.stopImmediatePropagation(); - } - } - } - - function handleClick(e, args) { - // clicking on a row select checkbox - if (_grid.getColumns()[args.cell].id === _options.columnId && $(e.target).is(":checkbox")) { - // if editing, try to commit - if (_grid.getEditorLock().isActive() && !_grid.getEditorLock().commitCurrentEdit()) { - e.preventDefault(); - e.stopImmediatePropagation(); - return; - } - - toggleRowSelection(args.row); - e.stopPropagation(); - e.stopImmediatePropagation(); - } - } - - function toggleRowSelection(row) { - if (_selectedRowsLookup[row]) { - _grid.setSelectedRows($.grep(_grid.getSelectedRows(), function (n) { - return n != row - })); - } else { - _grid.setSelectedRows(_grid.getSelectedRows().concat(row)); - } - } - - function handleHeaderClick(e, args) { - if (args.column.id == _options.columnId && $(e.target).is(":checkbox")) { - // if editing, try to commit - if (_grid.getEditorLock().isActive() && !_grid.getEditorLock().commitCurrentEdit()) { - e.preventDefault(); - e.stopImmediatePropagation(); - return; - } - - if ($(e.target).is(":checked")) { - var rows = []; - for (var i = 0; i < _grid.getDataLength(); i++) { - rows.push(i); - } - _grid.setSelectedRows(rows); - } else { - _grid.setSelectedRows([]); - } - e.stopPropagation(); - e.stopImmediatePropagation(); - } - } - - function getColumnDefinition() { - return { - id: _options.columnId, - name: "", - toolTip: _options.toolTip, - field: "sel", - width: _options.width, - resizable: false, - sortable: false, - cssClass: _options.cssClass, - formatter: checkboxSelectionFormatter - }; - } - - function checkboxSelectionFormatter(row, cell, value, columnDef, dataContext) { - if (dataContext) { - return _selectedRowsLookup[row] - ? "" - : ""; - } - return null; - } - - $.extend(this, { - "init": init, - "destroy": destroy, - - "getColumnDefinition": getColumnDefinition - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.css b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.css deleted file mode 100644 index 0ba79ea..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.css +++ /dev/null @@ -1,39 +0,0 @@ -.slick-column-name, -.slick-sort-indicator { - /** - * This makes all "float:right" elements after it that spill over to the next line - * display way below the lower boundary of the column thus hiding them. - */ - display: inline-block; - float: left; - margin-bottom: 100px; -} - -.slick-header-button { - display: inline-block; - float: right; - vertical-align: top; - margin: 1px; - /** - * This makes all "float:right" elements after it that spill over to the next line - * display way below the lower boundary of the column thus hiding them. - */ - margin-bottom: 100px; - height: 15px; - width: 15px; - background-repeat: no-repeat; - background-position: center center; - cursor: pointer; -} - -.slick-header-button-hidden { - width: 0; - - -webkit-transition: 0.2s width; - -ms-transition: 0.2s width; - transition: 0.2s width; -} - -.slick-header-column:hover > .slick-header-button { - width: 15px; -} \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.js deleted file mode 100644 index 8e61273..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.js +++ /dev/null @@ -1,177 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "Plugins": { - "HeaderButtons": HeaderButtons - } - } - }); - - - /*** - * A plugin to add custom buttons to column headers. - * - * USAGE: - * - * Add the plugin .js & .css files and register it with the grid. - * - * To specify a custom button in a column header, extend the column definition like so: - * - * var columns = [ - * { - * id: 'myColumn', - * name: 'My column', - * - * // This is the relevant part - * header: { - * buttons: [ - * { - * // button options - * }, - * { - * // button options - * } - * ] - * } - * } - * ]; - * - * Available button options: - * cssClass: CSS class to add to the button. - * image: Relative button image path. - * tooltip: Button tooltip. - * showOnHover: Only show the button on hover. - * handler: Button click handler. - * command: A command identifier to be passed to the onCommand event handlers. - * - * The plugin exposes the following events: - * onCommand: Fired on button click for buttons with 'command' specified. - * Event args: - * grid: Reference to the grid. - * column: Column definition. - * command: Button command identified. - * button: Button options. Note that you can change the button options in your - * event handler, and the column header will be automatically updated to - * reflect them. This is useful if you want to implement something like a - * toggle button. - * - * - * @param options {Object} Options: - * buttonCssClass: a CSS class to use for buttons (default 'slick-header-button') - * @class Slick.Plugins.HeaderButtons - * @constructor - */ - function HeaderButtons(options) { - var _grid; - var _self = this; - var _handler = new Slick.EventHandler(); - var _defaults = { - buttonCssClass: "slick-header-button" - }; - - - function init(grid) { - options = $.extend(true, {}, _defaults, options); - _grid = grid; - _handler - .subscribe(_grid.onHeaderCellRendered, handleHeaderCellRendered) - .subscribe(_grid.onBeforeHeaderCellDestroy, handleBeforeHeaderCellDestroy); - - // Force the grid to re-render the header now that the events are hooked up. - _grid.setColumns(_grid.getColumns()); - } - - - function destroy() { - _handler.unsubscribeAll(); - } - - - function handleHeaderCellRendered(e, args) { - var column = args.column; - - if (column.header && column.header.buttons) { - // Append buttons in reverse order since they are floated to the right. - var i = column.header.buttons.length; - while (i--) { - var button = column.header.buttons[i]; - var btn = $("
    ") - .addClass(options.buttonCssClass) - .data("column", column) - .data("button", button); - - if (button.showOnHover) { - btn.addClass("slick-header-button-hidden"); - } - - if (button.image) { - btn.css("backgroundImage", "url(" + button.image + ")"); - } - - if (button.cssClass) { - btn.addClass(button.cssClass); - } - - if (button.tooltip) { - btn.attr("title", button.tooltip); - } - - if (button.command) { - btn.data("command", button.command); - } - - if (button.handler) { - btn.bind("click", button.handler); - } - - btn - .bind("click", handleButtonClick) - .appendTo(args.node); - } - } - } - - - function handleBeforeHeaderCellDestroy(e, args) { - var column = args.column; - - if (column.header && column.header.buttons) { - // Removing buttons via jQuery will also clean up any event handlers and data. - // NOTE: If you attach event handlers directly or using a different framework, - // you must also clean them up here to avoid memory leaks. - $(args.node).find("." + options.buttonCssClass).remove(); - } - } - - - function handleButtonClick(e) { - var command = $(this).data("command"); - var columnDef = $(this).data("column"); - var button = $(this).data("button"); - - if (command != null) { - _self.onCommand.notify({ - "grid": _grid, - "column": columnDef, - "command": command, - "button": button - }, e, _self); - - // Update the header in case the user updated the button definition in the handler. - _grid.updateColumnHeader(columnDef.id); - } - - // Stop propagation so that it doesn't register as a header click event. - e.preventDefault(); - e.stopPropagation(); - } - - $.extend(this, { - "init": init, - "destroy": destroy, - - "onCommand": new Slick.Event() - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.css b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.css deleted file mode 100644 index 8b0b6a9..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.css +++ /dev/null @@ -1,59 +0,0 @@ -/* Menu button */ -.slick-header-menubutton { - position: absolute; - right: 0; - top: 0; - bottom: 0; - width: 14px; - background-repeat: no-repeat; - background-position: left center; - background-image: url(../images/down.gif); - cursor: pointer; - - display: none; - border-left: thin ridge silver; -} - -.slick-header-column:hover > .slick-header-menubutton, -.slick-header-column-active .slick-header-menubutton { - display: inline-block; -} - -/* Menu */ -.slick-header-menu { - position: absolute; - display: inline-block; - margin: 0; - padding: 2px; - cursor: default; -} - - -/* Menu items */ -.slick-header-menuitem { - list-style: none; - margin: 0; - padding: 0; - cursor: pointer; -} - -.slick-header-menuicon { - display: inline-block; - width: 16px; - height: 16px; - vertical-align: middle; - margin-right: 4px; - background-repeat: no-repeat; - background-position: center center; -} - -.slick-header-menucontent { - display: inline-block; - vertical-align: middle; -} - - -/* Disabled */ -.slick-header-menuitem-disabled { - color: silver; -} diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.js deleted file mode 100644 index ec8244d..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.js +++ /dev/null @@ -1,275 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "Plugins": { - "HeaderMenu": HeaderMenu - } - } - }); - - - /*** - * A plugin to add drop-down menus to column headers. - * - * USAGE: - * - * Add the plugin .js & .css files and register it with the grid. - * - * To specify a menu in a column header, extend the column definition like so: - * - * var columns = [ - * { - * id: 'myColumn', - * name: 'My column', - * - * // This is the relevant part - * header: { - * menu: { - * items: [ - * { - * // menu item options - * }, - * { - * // menu item options - * } - * ] - * } - * } - * } - * ]; - * - * - * Available menu options: - * tooltip: Menu button tooltip. - * - * - * Available menu item options: - * title: Menu item text. - * disabled: Whether the item is disabled. - * tooltip: Item tooltip. - * command: A command identifier to be passed to the onCommand event handlers. - * iconCssClass: A CSS class to be added to the menu item icon. - * iconImage: A url to the icon image. - * - * - * The plugin exposes the following events: - * onBeforeMenuShow: Fired before the menu is shown. You can customize the menu or dismiss it by returning false. - * Event args: - * grid: Reference to the grid. - * column: Column definition. - * menu: Menu options. Note that you can change the menu items here. - * - * onCommand: Fired on menu item click for buttons with 'command' specified. - * Event args: - * grid: Reference to the grid. - * column: Column definition. - * command: Button command identified. - * button: Button options. Note that you can change the button options in your - * event handler, and the column header will be automatically updated to - * reflect them. This is useful if you want to implement something like a - * toggle button. - * - * - * @param options {Object} Options: - * buttonCssClass: an extra CSS class to add to the menu button - * buttonImage: a url to the menu button image (default '../images/down.gif') - * @class Slick.Plugins.HeaderButtons - * @constructor - */ - function HeaderMenu(options) { - var _grid; - var _self = this; - var _handler = new Slick.EventHandler(); - var _defaults = { - buttonCssClass: null, - buttonImage: null - }; - var $menu; - var $activeHeaderColumn; - - - function init(grid) { - options = $.extend(true, {}, _defaults, options); - _grid = grid; - _handler - .subscribe(_grid.onHeaderCellRendered, handleHeaderCellRendered) - .subscribe(_grid.onBeforeHeaderCellDestroy, handleBeforeHeaderCellDestroy); - - // Force the grid to re-render the header now that the events are hooked up. - _grid.setColumns(_grid.getColumns()); - - // Hide the menu on outside click. - $(document.body).bind("mousedown", handleBodyMouseDown); - } - - - function destroy() { - _handler.unsubscribeAll(); - $(document.body).unbind("mousedown", handleBodyMouseDown); - } - - - function handleBodyMouseDown(e) { - if ($menu && $menu[0] != e.target && !$.contains($menu[0], e.target)) { - hideMenu(); - } - } - - - function hideMenu() { - if ($menu) { - $menu.remove(); - $menu = null; - $activeHeaderColumn - .removeClass("slick-header-column-active"); - } - } - - function handleHeaderCellRendered(e, args) { - var column = args.column; - var menu = column.header && column.header.menu; - - if (menu) { - var $el = $("
    ") - .addClass("slick-header-menubutton") - .data("column", column) - .data("menu", menu); - - if (options.buttonCssClass) { - $el.addClass(options.buttonCssClass); - } - - if (options.buttonImage) { - $el.css("background-image", "url(" + options.buttonImage + ")"); - } - - if (menu.tooltip) { - $el.attr("title", menu.tooltip); - } - - $el - .bind("click", showMenu) - .appendTo(args.node); - } - } - - - function handleBeforeHeaderCellDestroy(e, args) { - var column = args.column; - - if (column.header && column.header.menu) { - $(args.node).find(".slick-header-menubutton").remove(); - } - } - - - function showMenu(e) { - var $menuButton = $(this); - var menu = $menuButton.data("menu"); - var columnDef = $menuButton.data("column"); - - // Let the user modify the menu or cancel altogether, - // or provide alternative menu implementation. - if (_self.onBeforeMenuShow.notify({ - "grid": _grid, - "column": columnDef, - "menu": menu - }, e, _self) == false) { - return; - } - - - if (!$menu) { - $menu = $("
    ") - .appendTo(_grid.getContainerNode()); - } - $menu.empty(); - - - // Construct the menu items. - for (var i = 0; i < menu.items.length; i++) { - var item = menu.items[i]; - - var $li = $("
    ") - .data("command", item.command || '') - .data("column", columnDef) - .data("item", item) - .bind("click", handleMenuItemClick) - .appendTo($menu); - - if (item.disabled) { - $li.addClass("slick-header-menuitem-disabled"); - } - - if (item.tooltip) { - $li.attr("title", item.tooltip); - } - - var $icon = $("
    ") - .appendTo($li); - - if (item.iconCssClass) { - $icon.addClass(item.iconCssClass); - } - - if (item.iconImage) { - $icon.css("background-image", "url(" + item.iconImage + ")"); - } - - $("") - .text(item.title) - .appendTo($li); - } - - - // Position the menu. - $menu - .offset({ top: $(this).offset().top + $(this).height(), left: $(this).offset().left }); - - - // Mark the header as active to keep the highlighting. - $activeHeaderColumn = $menuButton.closest(".slick-header-column"); - $activeHeaderColumn - .addClass("slick-header-column-active"); - - // Stop propagation so that it doesn't register as a header click event. - e.preventDefault(); - e.stopPropagation(); - } - - - function handleMenuItemClick(e) { - var command = $(this).data("command"); - var columnDef = $(this).data("column"); - var item = $(this).data("item"); - - if (item.disabled) { - return; - } - - hideMenu(); - - if (command != null && command != '') { - _self.onCommand.notify({ - "grid": _grid, - "column": columnDef, - "command": command, - "item": item - }, e, _self); - } - - // Stop propagation so that it doesn't register as a header click event. - e.preventDefault(); - e.stopPropagation(); - } - - $.extend(this, { - "init": init, - "destroy": destroy, - - "onBeforeMenuShow": new Slick.Event(), - "onCommand": new Slick.Event() - }); - } -})(jQuery); diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowmovemanager.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowmovemanager.js deleted file mode 100644 index 5f87a1e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowmovemanager.js +++ /dev/null @@ -1,138 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "RowMoveManager": RowMoveManager - } - }); - - function RowMoveManager(options) { - var _grid; - var _canvas; - var _dragging; - var _self = this; - var _handler = new Slick.EventHandler(); - var _defaults = { - cancelEditOnDrag: false - }; - - function init(grid) { - options = $.extend(true, {}, _defaults, options); - _grid = grid; - _canvas = _grid.getCanvasNode(); - _handler - .subscribe(_grid.onDragInit, handleDragInit) - .subscribe(_grid.onDragStart, handleDragStart) - .subscribe(_grid.onDrag, handleDrag) - .subscribe(_grid.onDragEnd, handleDragEnd); - } - - function destroy() { - _handler.unsubscribeAll(); - } - - function handleDragInit(e, dd) { - // prevent the grid from cancelling drag'n'drop by default - e.stopImmediatePropagation(); - } - - function handleDragStart(e, dd) { - var cell = _grid.getCellFromEvent(e); - - if (options.cancelEditOnDrag && _grid.getEditorLock().isActive()) { - _grid.getEditorLock().cancelCurrentEdit(); - } - - if (_grid.getEditorLock().isActive() || !/move|selectAndMove/.test(_grid.getColumns()[cell.cell].behavior)) { - return false; - } - - _dragging = true; - e.stopImmediatePropagation(); - - var selectedRows = _grid.getSelectedRows(); - - if (selectedRows.length == 0 || $.inArray(cell.row, selectedRows) == -1) { - selectedRows = [cell.row]; - _grid.setSelectedRows(selectedRows); - } - - var rowHeight = _grid.getOptions().rowHeight; - - dd.selectedRows = selectedRows; - - dd.selectionProxy = $("
    ") - .css("position", "absolute") - .css("zIndex", "99999") - .css("width", $(_canvas).innerWidth()) - .css("height", rowHeight * selectedRows.length) - .appendTo(_canvas); - - dd.guide = $("
    ") - .css("position", "absolute") - .css("zIndex", "99998") - .css("width", $(_canvas).innerWidth()) - .css("top", -1000) - .appendTo(_canvas); - - dd.insertBefore = -1; - } - - function handleDrag(e, dd) { - if (!_dragging) { - return; - } - - e.stopImmediatePropagation(); - - var top = e.pageY - $(_canvas).offset().top; - dd.selectionProxy.css("top", top - 5); - - var insertBefore = Math.max(0, Math.min(Math.round(top / _grid.getOptions().rowHeight), _grid.getDataLength())); - if (insertBefore !== dd.insertBefore) { - var eventData = { - "rows": dd.selectedRows, - "insertBefore": insertBefore - }; - - if (_self.onBeforeMoveRows.notify(eventData) === false) { - dd.guide.css("top", -1000); - dd.canMove = false; - } else { - dd.guide.css("top", insertBefore * _grid.getOptions().rowHeight); - dd.canMove = true; - } - - dd.insertBefore = insertBefore; - } - } - - function handleDragEnd(e, dd) { - if (!_dragging) { - return; - } - _dragging = false; - e.stopImmediatePropagation(); - - dd.guide.remove(); - dd.selectionProxy.remove(); - - if (dd.canMove) { - var eventData = { - "rows": dd.selectedRows, - "insertBefore": dd.insertBefore - }; - // TODO: _grid.remapCellCssClasses ? - _self.onMoveRows.notify(eventData); - } - } - - $.extend(this, { - "onBeforeMoveRows": new Slick.Event(), - "onMoveRows": new Slick.Event(), - - "init": init, - "destroy": destroy - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowselectionmodel.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowselectionmodel.js deleted file mode 100644 index 0de8dd3..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.rowselectionmodel.js +++ /dev/null @@ -1,187 +0,0 @@ -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "RowSelectionModel": RowSelectionModel - } - }); - - function RowSelectionModel(options) { - var _grid; - var _ranges = []; - var _self = this; - var _handler = new Slick.EventHandler(); - var _inHandler; - var _options; - var _defaults = { - selectActiveRow: true - }; - - function init(grid) { - _options = $.extend(true, {}, _defaults, options); - _grid = grid; - _handler.subscribe(_grid.onActiveCellChanged, - wrapHandler(handleActiveCellChange)); - _handler.subscribe(_grid.onKeyDown, - wrapHandler(handleKeyDown)); - _handler.subscribe(_grid.onClick, - wrapHandler(handleClick)); - } - - function destroy() { - _handler.unsubscribeAll(); - } - - function wrapHandler(handler) { - return function () { - if (!_inHandler) { - _inHandler = true; - handler.apply(this, arguments); - _inHandler = false; - } - }; - } - - function rangesToRows(ranges) { - var rows = []; - for (var i = 0; i < ranges.length; i++) { - for (var j = ranges[i].fromRow; j <= ranges[i].toRow; j++) { - rows.push(j); - } - } - return rows; - } - - function rowsToRanges(rows) { - var ranges = []; - var lastCell = _grid.getColumns().length - 1; - for (var i = 0; i < rows.length; i++) { - ranges.push(new Slick.Range(rows[i], 0, rows[i], lastCell)); - } - return ranges; - } - - function getRowsRange(from, to) { - var i, rows = []; - for (i = from; i <= to; i++) { - rows.push(i); - } - for (i = to; i < from; i++) { - rows.push(i); - } - return rows; - } - - function getSelectedRows() { - return rangesToRows(_ranges); - } - - function setSelectedRows(rows) { - setSelectedRanges(rowsToRanges(rows)); - } - - function setSelectedRanges(ranges) { - _ranges = ranges; - _self.onSelectedRangesChanged.notify(_ranges); - } - - function getSelectedRanges() { - return _ranges; - } - - function handleActiveCellChange(e, data) { - if (_options.selectActiveRow && data.row != null) { - setSelectedRanges([new Slick.Range(data.row, 0, data.row, _grid.getColumns().length - 1)]); - } - } - - function handleKeyDown(e) { - var activeRow = _grid.getActiveCell(); - if (activeRow && e.shiftKey && !e.ctrlKey && !e.altKey && !e.metaKey && (e.which == 38 || e.which == 40)) { - var selectedRows = getSelectedRows(); - selectedRows.sort(function (x, y) { - return x - y - }); - - if (!selectedRows.length) { - selectedRows = [activeRow.row]; - } - - var top = selectedRows[0]; - var bottom = selectedRows[selectedRows.length - 1]; - var active; - - if (e.which == 40) { - active = activeRow.row < bottom || top == bottom ? ++bottom : ++top; - } else { - active = activeRow.row < bottom ? --bottom : --top; - } - - if (active >= 0 && active < _grid.getDataLength()) { - _grid.scrollRowIntoView(active); - _ranges = rowsToRanges(getRowsRange(top, bottom)); - setSelectedRanges(_ranges); - } - - e.preventDefault(); - e.stopPropagation(); - } - } - - function handleClick(e) { - var cell = _grid.getCellFromEvent(e); - if (!cell || !_grid.canCellBeActive(cell.row, cell.cell)) { - return false; - } - - if (!_grid.getOptions().multiSelect || ( - !e.ctrlKey && !e.shiftKey && !e.metaKey)) { - return false; - } - - var selection = rangesToRows(_ranges); - var idx = $.inArray(cell.row, selection); - - if (idx === -1 && (e.ctrlKey || e.metaKey)) { - selection.push(cell.row); - _grid.setActiveCell(cell.row, cell.cell); - } else if (idx !== -1 && (e.ctrlKey || e.metaKey)) { - selection = $.grep(selection, function (o, i) { - return (o !== cell.row); - }); - _grid.setActiveCell(cell.row, cell.cell); - } else if (selection.length && e.shiftKey) { - var last = selection.pop(); - var from = Math.min(cell.row, last); - var to = Math.max(cell.row, last); - selection = []; - for (var i = from; i <= to; i++) { - if (i !== last) { - selection.push(i); - } - } - selection.push(last); - _grid.setActiveCell(cell.row, cell.cell); - } - - _ranges = rowsToRanges(selection); - setSelectedRanges(_ranges); - e.stopImmediatePropagation(); - - return true; - } - - $.extend(this, { - "getSelectedRows": getSelectedRows, - "setSelectedRows": setSelectedRows, - - "getSelectedRanges": getSelectedRanges, - "setSelectedRanges": setSelectedRanges, - - "init": init, - "destroy": destroy, - - "onSelectedRangesChanged": new Slick.Event() - }); - } -})(jQuery); \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick-default-theme.css b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick-default-theme.css deleted file mode 100644 index efc7415..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick-default-theme.css +++ /dev/null @@ -1,118 +0,0 @@ -/* -IMPORTANT: -In order to preserve the uniform grid appearance, all cell styles need to have padding, margin and border sizes. -No built-in (selected, editable, highlight, flashing, invalid, loading, :focus) or user-specified CSS -classes should alter those! -*/ - -.slick-header-columns { - background: url('images/header-columns-bg.gif') repeat-x center bottom; - border-bottom: 1px solid silver; -} - -.slick-header-column { - background: url('images/header-columns-bg.gif') repeat-x center bottom; - border-right: 1px solid silver; -} - -.slick-header-column:hover, .slick-header-column-active { - background: white url('images/header-columns-over-bg.gif') repeat-x center bottom; -} - -.slick-headerrow { - background: #fafafa; -} - -.slick-headerrow-column { - background: #fafafa; - border-bottom: 0; - height: 100%; -} - -.slick-row.ui-state-active { - background: #F5F7D7; -} - -.slick-row { - position: absolute; - background: white; - border: 0px; - line-height: 20px; -} - -.slick-row.selected { - z-index: 10; - background: #DFE8F6; -} - -.slick-cell { - padding-left: 4px; - padding-right: 4px; -} - -.slick-group { - border-bottom: 2px solid silver; -} - -.slick-group-toggle { - width: 9px; - height: 9px; - margin-right: 5px; -} - -.slick-group-toggle.expanded { - background: url(images/collapse.gif) no-repeat center center; -} - -.slick-group-toggle.collapsed { - background: url(images/expand.gif) no-repeat center center; -} - -.slick-group-totals { - color: gray; - background: white; -} - -.slick-cell.selected { - background-color: beige; -} - -.slick-cell.active { - border-color: gray; - border-style: solid; -} - -.slick-sortable-placeholder { - background: silver !important; -} - -.slick-row.odd { - background: #fafafa; -} - -.slick-row.ui-state-active { - background: #F5F7D7; -} - -.slick-row.loading { - opacity: 0.5; - filter: alpha(opacity = 50); -} - -.slick-cell.invalid { - border-color: red; - -moz-animation-duration: 0.2s; - -webkit-animation-duration: 0.2s; - -moz-animation-name: slickgrid-invalid-hilite; - -webkit-animation-name: slickgrid-invalid-hilite; -} - -@-moz-keyframes slickgrid-invalid-hilite { - from { box-shadow: 0 0 6px red; } - to { box-shadow: none; } -} - -@-webkit-keyframes slickgrid-invalid-hilite { - from { box-shadow: 0 0 6px red; } - to { box-shadow: none; } -} \ No newline at end of file diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.core.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.core.js deleted file mode 100644 index 2f097b1..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.core.js +++ /dev/null @@ -1,467 +0,0 @@ -/*** - * Contains core SlickGrid classes. - * @module Core - * @namespace Slick - */ - -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "Event": Event, - "EventData": EventData, - "EventHandler": EventHandler, - "Range": Range, - "NonDataRow": NonDataItem, - "Group": Group, - "GroupTotals": GroupTotals, - "EditorLock": EditorLock, - - /*** - * A global singleton editor lock. - * @class GlobalEditorLock - * @static - * @constructor - */ - "GlobalEditorLock": new EditorLock() - } - }); - - /*** - * An event object for passing data to event handlers and letting them control propagation. - *

    This is pretty much identical to how W3C and jQuery implement events.

    - * @class EventData - * @constructor - */ - function EventData() { - var isPropagationStopped = false; - var isImmediatePropagationStopped = false; - - /*** - * Stops event from propagating up the DOM tree. - * @method stopPropagation - */ - this.stopPropagation = function () { - isPropagationStopped = true; - }; - - /*** - * Returns whether stopPropagation was called on this event object. - * @method isPropagationStopped - * @return {Boolean} - */ - this.isPropagationStopped = function () { - return isPropagationStopped; - }; - - /*** - * Prevents the rest of the handlers from being executed. - * @method stopImmediatePropagation - */ - this.stopImmediatePropagation = function () { - isImmediatePropagationStopped = true; - }; - - /*** - * Returns whether stopImmediatePropagation was called on this event object.\ - * @method isImmediatePropagationStopped - * @return {Boolean} - */ - this.isImmediatePropagationStopped = function () { - return isImmediatePropagationStopped; - } - } - - /*** - * A simple publisher-subscriber implementation. - * @class Event - * @constructor - */ - function Event() { - var handlers = []; - - /*** - * Adds an event handler to be called when the event is fired. - *

    Event handler will receive two arguments - an EventData and the data - * object the event was fired with.

    - * @method subscribe - * @param fn {Function} Event handler. - */ - this.subscribe = function (fn) { - handlers.push(fn); - }; - - /*** - * Removes an event handler added with subscribe(fn). - * @method unsubscribe - * @param fn {Function} Event handler to be removed. - */ - this.unsubscribe = function (fn) { - for (var i = handlers.length - 1; i >= 0; i--) { - if (handlers[i] === fn) { - handlers.splice(i, 1); - } - } - }; - - /*** - * Fires an event notifying all subscribers. - * @method notify - * @param args {Object} Additional data object to be passed to all handlers. - * @param e {EventData} - * Optional. - * An EventData object to be passed to all handlers. - * For DOM events, an existing W3C/jQuery event object can be passed in. - * @param scope {Object} - * Optional. - * The scope ("this") within which the handler will be executed. - * If not specified, the scope will be set to the Event instance. - */ - this.notify = function (args, e, scope) { - e = e || new EventData(); - scope = scope || this; - - var returnValue; - for (var i = 0; i < handlers.length && !(e.isPropagationStopped() || e.isImmediatePropagationStopped()); i++) { - returnValue = handlers[i].call(scope, e, args); - } - - return returnValue; - }; - } - - function EventHandler() { - var handlers = []; - - this.subscribe = function (event, handler) { - handlers.push({ - event: event, - handler: handler - }); - event.subscribe(handler); - - return this; // allow chaining - }; - - this.unsubscribe = function (event, handler) { - var i = handlers.length; - while (i--) { - if (handlers[i].event === event && - handlers[i].handler === handler) { - handlers.splice(i, 1); - event.unsubscribe(handler); - return; - } - } - - return this; // allow chaining - }; - - this.unsubscribeAll = function () { - var i = handlers.length; - while (i--) { - handlers[i].event.unsubscribe(handlers[i].handler); - } - handlers = []; - - return this; // allow chaining - } - } - - /*** - * A structure containing a range of cells. - * @class Range - * @constructor - * @param fromRow {Integer} Starting row. - * @param fromCell {Integer} Starting cell. - * @param toRow {Integer} Optional. Ending row. Defaults to fromRow. - * @param toCell {Integer} Optional. Ending cell. Defaults to fromCell. - */ - function Range(fromRow, fromCell, toRow, toCell) { - if (toRow === undefined && toCell === undefined) { - toRow = fromRow; - toCell = fromCell; - } - - /*** - * @property fromRow - * @type {Integer} - */ - this.fromRow = Math.min(fromRow, toRow); - - /*** - * @property fromCell - * @type {Integer} - */ - this.fromCell = Math.min(fromCell, toCell); - - /*** - * @property toRow - * @type {Integer} - */ - this.toRow = Math.max(fromRow, toRow); - - /*** - * @property toCell - * @type {Integer} - */ - this.toCell = Math.max(fromCell, toCell); - - /*** - * Returns whether a range represents a single row. - * @method isSingleRow - * @return {Boolean} - */ - this.isSingleRow = function () { - return this.fromRow == this.toRow; - }; - - /*** - * Returns whether a range represents a single cell. - * @method isSingleCell - * @return {Boolean} - */ - this.isSingleCell = function () { - return this.fromRow == this.toRow && this.fromCell == this.toCell; - }; - - /*** - * Returns whether a range contains a given cell. - * @method contains - * @param row {Integer} - * @param cell {Integer} - * @return {Boolean} - */ - this.contains = function (row, cell) { - return row >= this.fromRow && row <= this.toRow && - cell >= this.fromCell && cell <= this.toCell; - }; - - /*** - * Returns a readable representation of a range. - * @method toString - * @return {String} - */ - this.toString = function () { - if (this.isSingleCell()) { - return "(" + this.fromRow + ":" + this.fromCell + ")"; - } - else { - return "(" + this.fromRow + ":" + this.fromCell + " - " + this.toRow + ":" + this.toCell + ")"; - } - } - } - - - /*** - * A base class that all special / non-data rows (like Group and GroupTotals) derive from. - * @class NonDataItem - * @constructor - */ - function NonDataItem() { - this.__nonDataRow = true; - } - - - /*** - * Information about a group of rows. - * @class Group - * @extends Slick.NonDataItem - * @constructor - */ - function Group() { - this.__group = true; - - /** - * Grouping level, starting with 0. - * @property level - * @type {Number} - */ - this.level = 0; - - /*** - * Number of rows in the group. - * @property count - * @type {Integer} - */ - this.count = 0; - - /*** - * Grouping value. - * @property value - * @type {Object} - */ - this.value = null; - - /*** - * Formatted display value of the group. - * @property title - * @type {String} - */ - this.title = null; - - /*** - * Whether a group is collapsed. - * @property collapsed - * @type {Boolean} - */ - this.collapsed = false; - - /*** - * GroupTotals, if any. - * @property totals - * @type {GroupTotals} - */ - this.totals = null; - - /** - * Rows that are part of the group. - * @property rows - * @type {Array} - */ - this.rows = []; - - /** - * Sub-groups that are part of the group. - * @property groups - * @type {Array} - */ - this.groups = null; - - /** - * A unique key used to identify the group. This key can be used in calls to DataView - * collapseGroup() or expandGroup(). - * @property groupingKey - * @type {Object} - */ - this.groupingKey = null; - } - - Group.prototype = new NonDataItem(); - - /*** - * Compares two Group instances. - * @method equals - * @return {Boolean} - * @param group {Group} Group instance to compare to. - */ - Group.prototype.equals = function (group) { - return this.value === group.value && - this.count === group.count && - this.collapsed === group.collapsed && - this.title === group.title; - }; - - /*** - * Information about group totals. - * An instance of GroupTotals will be created for each totals row and passed to the aggregators - * so that they can store arbitrary data in it. That data can later be accessed by group totals - * formatters during the display. - * @class GroupTotals - * @extends Slick.NonDataItem - * @constructor - */ - function GroupTotals() { - this.__groupTotals = true; - - /*** - * Parent Group. - * @param group - * @type {Group} - */ - this.group = null; - - /*** - * Whether the totals have been fully initialized / calculated. - * Will be set to false for lazy-calculated group totals. - * @param initialized - * @type {Boolean} - */ - this.initialized = false; - } - - GroupTotals.prototype = new NonDataItem(); - - /*** - * A locking helper to track the active edit controller and ensure that only a single controller - * can be active at a time. This prevents a whole class of state and validation synchronization - * issues. An edit controller (such as SlickGrid) can query if an active edit is in progress - * and attempt a commit or cancel before proceeding. - * @class EditorLock - * @constructor - */ - function EditorLock() { - var activeEditController = null; - - /*** - * Returns true if a specified edit controller is active (has the edit lock). - * If the parameter is not specified, returns true if any edit controller is active. - * @method isActive - * @param editController {EditController} - * @return {Boolean} - */ - this.isActive = function (editController) { - return (editController ? activeEditController === editController : activeEditController !== null); - }; - - /*** - * Sets the specified edit controller as the active edit controller (acquire edit lock). - * If another edit controller is already active, and exception will be thrown. - * @method activate - * @param editController {EditController} edit controller acquiring the lock - */ - this.activate = function (editController) { - if (editController === activeEditController) { // already activated? - return; - } - if (activeEditController !== null) { - throw "SlickGrid.EditorLock.activate: an editController is still active, can't activate another editController"; - } - if (!editController.commitCurrentEdit) { - throw "SlickGrid.EditorLock.activate: editController must implement .commitCurrentEdit()"; - } - if (!editController.cancelCurrentEdit) { - throw "SlickGrid.EditorLock.activate: editController must implement .cancelCurrentEdit()"; - } - activeEditController = editController; - }; - - /*** - * Unsets the specified edit controller as the active edit controller (release edit lock). - * If the specified edit controller is not the active one, an exception will be thrown. - * @method deactivate - * @param editController {EditController} edit controller releasing the lock - */ - this.deactivate = function (editController) { - if (activeEditController !== editController) { - throw "SlickGrid.EditorLock.deactivate: specified editController is not the currently active one"; - } - activeEditController = null; - }; - - /*** - * Attempts to commit the current edit by calling "commitCurrentEdit" method on the active edit - * controller and returns whether the commit attempt was successful (commit may fail due to validation - * errors, etc.). Edit controller's "commitCurrentEdit" must return true if the commit has succeeded - * and false otherwise. If no edit controller is active, returns true. - * @method commitCurrentEdit - * @return {Boolean} - */ - this.commitCurrentEdit = function () { - return (activeEditController ? activeEditController.commitCurrentEdit() : true); - }; - - /*** - * Attempts to cancel the current edit by calling "cancelCurrentEdit" method on the active edit - * controller and returns whether the edit was successfully cancelled. If no edit controller is - * active, returns true. - * @method cancelCurrentEdit - * @return {Boolean} - */ - this.cancelCurrentEdit = function cancelCurrentEdit() { - return (activeEditController ? activeEditController.cancelCurrentEdit() : true); - }; - } -})(jQuery); - - diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.dataview.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.dataview.js deleted file mode 100644 index f1c1b5e..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.dataview.js +++ /dev/null @@ -1,1126 +0,0 @@ -(function ($) { - $.extend(true, window, { - Slick: { - Data: { - DataView: DataView, - Aggregators: { - Avg: AvgAggregator, - Min: MinAggregator, - Max: MaxAggregator, - Sum: SumAggregator - } - } - } - }); - - - /*** - * A sample Model implementation. - * Provides a filtered view of the underlying data. - * - * Relies on the data item having an "id" property uniquely identifying it. - */ - function DataView(options) { - var self = this; - - var defaults = { - groupItemMetadataProvider: null, - inlineFilters: false - }; - - - // private - var idProperty = "id"; // property holding a unique row id - var items = []; // data by index - var rows = []; // data by row - var idxById = {}; // indexes by id - var rowsById = null; // rows by id; lazy-calculated - var filter = null; // filter function - var updated = null; // updated item ids - var suspend = false; // suspends the recalculation - var sortAsc = true; - var fastSortField; - var sortComparer; - var refreshHints = {}; - var prevRefreshHints = {}; - var filterArgs; - var filteredItems = []; - var compiledFilter; - var compiledFilterWithCaching; - var filterCache = []; - - // grouping - var groupingInfoDefaults = { - getter: null, - formatter: null, - comparer: function(a, b) { return a.value - b.value; }, - predefinedValues: [], - aggregators: [], - aggregateEmpty: false, - aggregateCollapsed: false, - aggregateChildGroups: false, - collapsed: false, - displayTotalsRow: true, - lazyTotalsCalculation: false - }; - var groupingInfos = []; - var groups = []; - var toggledGroupsByLevel = []; - var groupingDelimiter = ':|:'; - - var pagesize = 0; - var pagenum = 0; - var totalRows = 0; - - // events - var onRowCountChanged = new Slick.Event(); - var onRowsChanged = new Slick.Event(); - var onPagingInfoChanged = new Slick.Event(); - - options = $.extend(true, {}, defaults, options); - - - function beginUpdate() { - suspend = true; - } - - function endUpdate() { - suspend = false; - refresh(); - } - - function setRefreshHints(hints) { - refreshHints = hints; - } - - function setFilterArgs(args) { - filterArgs = args; - } - - function updateIdxById(startingIndex) { - startingIndex = startingIndex || 0; - var id; - for (var i = startingIndex, l = items.length; i < l; i++) { - id = items[i][idProperty]; - if (id === undefined) { - throw "Each data element must implement a unique 'id' property"; - } - idxById[id] = i; - } - } - - function ensureIdUniqueness() { - var id; - for (var i = 0, l = items.length; i < l; i++) { - id = items[i][idProperty]; - if (id === undefined || idxById[id] !== i) { - throw "Each data element must implement a unique 'id' property"; - } - } - } - - function getItems() { - return items; - } - - function setItems(data, objectIdProperty) { - if (objectIdProperty !== undefined) { - idProperty = objectIdProperty; - } - items = filteredItems = data; - idxById = {}; - updateIdxById(); - ensureIdUniqueness(); - refresh(); - } - - function setPagingOptions(args) { - if (args.pageSize != undefined) { - pagesize = args.pageSize; - pagenum = pagesize ? Math.min(pagenum, Math.max(0, Math.ceil(totalRows / pagesize) - 1)) : 0; - } - - if (args.pageNum != undefined) { - pagenum = Math.min(args.pageNum, Math.max(0, Math.ceil(totalRows / pagesize) - 1)); - } - - onPagingInfoChanged.notify(getPagingInfo(), null, self); - - refresh(); - } - - function getPagingInfo() { - var totalPages = pagesize ? Math.max(1, Math.ceil(totalRows / pagesize)) : 1; - return {pageSize: pagesize, pageNum: pagenum, totalRows: totalRows, totalPages: totalPages}; - } - - function sort(comparer, ascending) { - sortAsc = ascending; - sortComparer = comparer; - fastSortField = null; - if (ascending === false) { - items.reverse(); - } - items.sort(comparer); - if (ascending === false) { - items.reverse(); - } - idxById = {}; - updateIdxById(); - refresh(); - } - - /*** - * Provides a workaround for the extremely slow sorting in IE. - * Does a [lexicographic] sort on a give column by temporarily overriding Object.prototype.toString - * to return the value of that field and then doing a native Array.sort(). - */ - function fastSort(field, ascending) { - sortAsc = ascending; - fastSortField = field; - sortComparer = null; - var oldToString = Object.prototype.toString; - Object.prototype.toString = (typeof field == "function") ? field : function () { - return this[field] - }; - // an extra reversal for descending sort keeps the sort stable - // (assuming a stable native sort implementation, which isn't true in some cases) - if (ascending === false) { - items.reverse(); - } - items.sort(); - Object.prototype.toString = oldToString; - if (ascending === false) { - items.reverse(); - } - idxById = {}; - updateIdxById(); - refresh(); - } - - function reSort() { - if (sortComparer) { - sort(sortComparer, sortAsc); - } else if (fastSortField) { - fastSort(fastSortField, sortAsc); - } - } - - function setFilter(filterFn) { - filter = filterFn; - if (options.inlineFilters) { - compiledFilter = compileFilter(); - compiledFilterWithCaching = compileFilterWithCaching(); - } - refresh(); - } - - function getGrouping() { - return groupingInfos; - } - - function setGrouping(groupingInfo) { - if (!options.groupItemMetadataProvider) { - options.groupItemMetadataProvider = new Slick.Data.GroupItemMetadataProvider(); - } - - groups = []; - toggledGroupsByLevel = []; - groupingInfo = groupingInfo || []; - groupingInfos = (groupingInfo instanceof Array) ? groupingInfo : [groupingInfo]; - - for (var i = 0; i < groupingInfos.length; i++) { - var gi = groupingInfos[i] = $.extend(true, {}, groupingInfoDefaults, groupingInfos[i]); - gi.getterIsAFn = typeof gi.getter === "function"; - - // pre-compile accumulator loops - gi.compiledAccumulators = []; - var idx = gi.aggregators.length; - while (idx--) { - gi.compiledAccumulators[idx] = compileAccumulatorLoop(gi.aggregators[idx]); - } - - toggledGroupsByLevel[i] = {}; - } - - refresh(); - } - - /** - * @deprecated Please use {@link setGrouping}. - */ - function groupBy(valueGetter, valueFormatter, sortComparer) { - if (valueGetter == null) { - setGrouping([]); - return; - } - - setGrouping({ - getter: valueGetter, - formatter: valueFormatter, - comparer: sortComparer - }); - } - - /** - * @deprecated Please use {@link setGrouping}. - */ - function setAggregators(groupAggregators, includeCollapsed) { - if (!groupingInfos.length) { - throw new Error("At least one grouping must be specified before calling setAggregators()."); - } - - groupingInfos[0].aggregators = groupAggregators; - groupingInfos[0].aggregateCollapsed = includeCollapsed; - - setGrouping(groupingInfos); - } - - function getItemByIdx(i) { - return items[i]; - } - - function getIdxById(id) { - return idxById[id]; - } - - function ensureRowsByIdCache() { - if (!rowsById) { - rowsById = {}; - for (var i = 0, l = rows.length; i < l; i++) { - rowsById[rows[i][idProperty]] = i; - } - } - } - - function getRowById(id) { - ensureRowsByIdCache(); - return rowsById[id]; - } - - function getItemById(id) { - return items[idxById[id]]; - } - - function mapIdsToRows(idArray) { - var rows = []; - ensureRowsByIdCache(); - for (var i = 0, l = idArray.length; i < l; i++) { - var row = rowsById[idArray[i]]; - if (row != null) { - rows[rows.length] = row; - } - } - return rows; - } - - function mapRowsToIds(rowArray) { - var ids = []; - for (var i = 0, l = rowArray.length; i < l; i++) { - if (rowArray[i] < rows.length) { - ids[ids.length] = rows[rowArray[i]][idProperty]; - } - } - return ids; - } - - function updateItem(id, item) { - if (idxById[id] === undefined || id !== item[idProperty]) { - throw "Invalid or non-matching id"; - } - items[idxById[id]] = item; - if (!updated) { - updated = {}; - } - updated[id] = true; - refresh(); - } - - function insertItem(insertBefore, item) { - items.splice(insertBefore, 0, item); - updateIdxById(insertBefore); - refresh(); - } - - function addItem(item) { - items.push(item); - updateIdxById(items.length - 1); - refresh(); - } - - function deleteItem(id) { - var idx = idxById[id]; - if (idx === undefined) { - throw "Invalid id"; - } - delete idxById[id]; - items.splice(idx, 1); - updateIdxById(idx); - refresh(); - } - - function getLength() { - return rows.length; - } - - function getItem(i) { - var item = rows[i]; - - // if this is a group row, make sure totals are calculated and update the title - if (item && item.__group && item.totals && !item.totals.initialized) { - var gi = groupingInfos[item.level]; - if (!gi.displayTotalsRow) { - calculateTotals(item.totals); - item.title = gi.formatter ? gi.formatter(item) : item.value; - } - } - // if this is a totals row, make sure it's calculated - else if (item && item.__groupTotals && !item.initialized) { - calculateTotals(item); - } - - return item; - } - - function getItemMetadata(i) { - var item = rows[i]; - if (item === undefined) { - return null; - } - - // overrides for grouping rows - if (item.__group) { - return options.groupItemMetadataProvider.getGroupRowMetadata(item); - } - - // overrides for totals rows - if (item.__groupTotals) { - return options.groupItemMetadataProvider.getTotalsRowMetadata(item); - } - - return null; - } - - function expandCollapseAllGroups(level, collapse) { - if (level == null) { - for (var i = 0; i < groupingInfos.length; i++) { - toggledGroupsByLevel[i] = {}; - groupingInfos[i].collapsed = collapse; - } - } else { - toggledGroupsByLevel[level] = {}; - groupingInfos[level].collapsed = collapse; - } - refresh(); - } - - /** - * @param level {Number} Optional level to collapse. If not specified, applies to all levels. - */ - function collapseAllGroups(level) { - expandCollapseAllGroups(level, true); - } - - /** - * @param level {Number} Optional level to expand. If not specified, applies to all levels. - */ - function expandAllGroups(level) { - expandCollapseAllGroups(level, false); - } - - function expandCollapseGroup(level, groupingKey, collapse) { - toggledGroupsByLevel[level][groupingKey] = groupingInfos[level].collapsed ^ collapse; - refresh(); - } - - /** - * @param varArgs Either a Slick.Group's "groupingKey" property, or a - * variable argument list of grouping values denoting a unique path to the row. For - * example, calling collapseGroup('high', '10%') will collapse the '10%' subgroup of - * the 'high' group. - */ - function collapseGroup(varArgs) { - var args = Array.prototype.slice.call(arguments); - var arg0 = args[0]; - if (args.length == 1 && arg0.indexOf(groupingDelimiter) != -1) { - expandCollapseGroup(arg0.split(groupingDelimiter).length - 1, arg0, true); - } else { - expandCollapseGroup(args.length - 1, args.join(groupingDelimiter), true); - } - } - - /** - * @param varArgs Either a Slick.Group's "groupingKey" property, or a - * variable argument list of grouping values denoting a unique path to the row. For - * example, calling expandGroup('high', '10%') will expand the '10%' subgroup of - * the 'high' group. - */ - function expandGroup(varArgs) { - var args = Array.prototype.slice.call(arguments); - var arg0 = args[0]; - if (args.length == 1 && arg0.indexOf(groupingDelimiter) != -1) { - expandCollapseGroup(arg0.split(groupingDelimiter).length - 1, arg0, false); - } else { - expandCollapseGroup(args.length - 1, args.join(groupingDelimiter), false); - } - } - - function getGroups() { - return groups; - } - - function extractGroups(rows, parentGroup) { - var group; - var val; - var groups = []; - var groupsByVal = {}; - var r; - var level = parentGroup ? parentGroup.level + 1 : 0; - var gi = groupingInfos[level]; - - for (var i = 0, l = gi.predefinedValues.length; i < l; i++) { - val = gi.predefinedValues[i]; - group = groupsByVal[val]; - if (!group) { - group = new Slick.Group(); - group.value = val; - group.level = level; - group.groupingKey = (parentGroup ? parentGroup.groupingKey + groupingDelimiter : '') + val; - groups[groups.length] = group; - groupsByVal[val] = group; - } - } - - for (var i = 0, l = rows.length; i < l; i++) { - r = rows[i]; - val = gi.getterIsAFn ? gi.getter(r) : r[gi.getter]; - group = groupsByVal[val]; - if (!group) { - group = new Slick.Group(); - group.value = val; - group.level = level; - group.groupingKey = (parentGroup ? parentGroup.groupingKey + groupingDelimiter : '') + val; - groups[groups.length] = group; - groupsByVal[val] = group; - } - - group.rows[group.count++] = r; - } - - if (level < groupingInfos.length - 1) { - for (var i = 0; i < groups.length; i++) { - group = groups[i]; - group.groups = extractGroups(group.rows, group); - } - } - - groups.sort(groupingInfos[level].comparer); - - return groups; - } - - function calculateTotals(totals) { - var group = totals.group; - var gi = groupingInfos[group.level]; - var isLeafLevel = (group.level == groupingInfos.length); - var agg, idx = gi.aggregators.length; - - if (!isLeafLevel && gi.aggregateChildGroups) { - // make sure all the subgroups are calculated - var i = group.groups.length; - while (i--) { - if (!group.groups[i].initialized) { - calculateTotals(group.groups[i]); - } - } - } - - while (idx--) { - agg = gi.aggregators[idx]; - agg.init(); - if (!isLeafLevel && gi.aggregateChildGroups) { - gi.compiledAccumulators[idx].call(agg, group.groups); - } else { - gi.compiledAccumulators[idx].call(agg, group.rows); - } - agg.storeResult(totals); - } - totals.initialized = true; - } - - function addGroupTotals(group) { - var gi = groupingInfos[group.level]; - var totals = new Slick.GroupTotals(); - totals.group = group; - group.totals = totals; - if (!gi.lazyTotalsCalculation) { - calculateTotals(totals); - } - } - - function addTotals(groups, level) { - level = level || 0; - var gi = groupingInfos[level]; - var groupCollapsed = gi.collapsed; - var toggledGroups = toggledGroupsByLevel[level]; - var idx = groups.length, g; - while (idx--) { - g = groups[idx]; - - if (g.collapsed && !gi.aggregateCollapsed) { - continue; - } - - // Do a depth-first aggregation so that parent group aggregators can access subgroup totals. - if (g.groups) { - addTotals(g.groups, level + 1); - } - - if (gi.aggregators.length && ( - gi.aggregateEmpty || g.rows.length || (g.groups && g.groups.length))) { - addGroupTotals(g); - } - - g.collapsed = groupCollapsed ^ toggledGroups[g.groupingKey]; - g.title = gi.formatter ? gi.formatter(g) : g.value; - } - } - - function flattenGroupedRows(groups, level) { - level = level || 0; - var gi = groupingInfos[level]; - var groupedRows = [], rows, gl = 0, g; - for (var i = 0, l = groups.length; i < l; i++) { - g = groups[i]; - groupedRows[gl++] = g; - - if (!g.collapsed) { - rows = g.groups ? flattenGroupedRows(g.groups, level + 1) : g.rows; - for (var j = 0, jj = rows.length; j < jj; j++) { - groupedRows[gl++] = rows[j]; - } - } - - if (g.totals && gi.displayTotalsRow && (!g.collapsed || gi.aggregateCollapsed)) { - groupedRows[gl++] = g.totals; - } - } - return groupedRows; - } - - function getFunctionInfo(fn) { - var fnRegex = /^function[^(]*\(([^)]*)\)\s*{([\s\S]*)}$/; - var matches = fn.toString().match(fnRegex); - return { - params: matches[1].split(","), - body: matches[2] - }; - } - - function compileAccumulatorLoop(aggregator) { - var accumulatorInfo = getFunctionInfo(aggregator.accumulate); - var fn = new Function( - "_items", - "for (var " + accumulatorInfo.params[0] + ", _i=0, _il=_items.length; _i<_il; _i++) {" + - accumulatorInfo.params[0] + " = _items[_i]; " + - accumulatorInfo.body + - "}" - ); - fn.displayName = fn.name = "compiledAccumulatorLoop"; - return fn; - } - - function compileFilter() { - var filterInfo = getFunctionInfo(filter); - - var filterBody = filterInfo.body - .replace(/return false\s*([;}]|$)/gi, "{ continue _coreloop; }$1") - .replace(/return true\s*([;}]|$)/gi, "{ _retval[_idx++] = $item$; continue _coreloop; }$1") - .replace(/return ([^;}]+?)\s*([;}]|$)/gi, - "{ if ($1) { _retval[_idx++] = $item$; }; continue _coreloop; }$2"); - - // This preserves the function template code after JS compression, - // so that replace() commands still work as expected. - var tpl = [ - //"function(_items, _args) { ", - "var _retval = [], _idx = 0; ", - "var $item$, $args$ = _args; ", - "_coreloop: ", - "for (var _i = 0, _il = _items.length; _i < _il; _i++) { ", - "$item$ = _items[_i]; ", - "$filter$; ", - "} ", - "return _retval; " - //"}" - ].join(""); - tpl = tpl.replace(/\$filter\$/gi, filterBody); - tpl = tpl.replace(/\$item\$/gi, filterInfo.params[0]); - tpl = tpl.replace(/\$args\$/gi, filterInfo.params[1]); - - var fn = new Function("_items,_args", tpl); - fn.displayName = fn.name = "compiledFilter"; - return fn; - } - - function compileFilterWithCaching() { - var filterInfo = getFunctionInfo(filter); - - var filterBody = filterInfo.body - .replace(/return false\s*([;}]|$)/gi, "{ continue _coreloop; }$1") - .replace(/return true\s*([;}]|$)/gi, "{ _cache[_i] = true;_retval[_idx++] = $item$; continue _coreloop; }$1") - .replace(/return ([^;}]+?)\s*([;}]|$)/gi, - "{ if ((_cache[_i] = $1)) { _retval[_idx++] = $item$; }; continue _coreloop; }$2"); - - // This preserves the function template code after JS compression, - // so that replace() commands still work as expected. - var tpl = [ - //"function(_items, _args, _cache) { ", - "var _retval = [], _idx = 0; ", - "var $item$, $args$ = _args; ", - "_coreloop: ", - "for (var _i = 0, _il = _items.length; _i < _il; _i++) { ", - "$item$ = _items[_i]; ", - "if (_cache[_i]) { ", - "_retval[_idx++] = $item$; ", - "continue _coreloop; ", - "} ", - "$filter$; ", - "} ", - "return _retval; " - //"}" - ].join(""); - tpl = tpl.replace(/\$filter\$/gi, filterBody); - tpl = tpl.replace(/\$item\$/gi, filterInfo.params[0]); - tpl = tpl.replace(/\$args\$/gi, filterInfo.params[1]); - - var fn = new Function("_items,_args,_cache", tpl); - fn.displayName = fn.name = "compiledFilterWithCaching"; - return fn; - } - - function uncompiledFilter(items, args) { - var retval = [], idx = 0; - - for (var i = 0, ii = items.length; i < ii; i++) { - if (filter(items[i], args)) { - retval[idx++] = items[i]; - } - } - - return retval; - } - - function uncompiledFilterWithCaching(items, args, cache) { - var retval = [], idx = 0, item; - - for (var i = 0, ii = items.length; i < ii; i++) { - item = items[i]; - if (cache[i]) { - retval[idx++] = item; - } else if (filter(item, args)) { - retval[idx++] = item; - cache[i] = true; - } - } - - return retval; - } - - function getFilteredAndPagedItems(items) { - if (filter) { - var batchFilter = options.inlineFilters ? compiledFilter : uncompiledFilter; - var batchFilterWithCaching = options.inlineFilters ? compiledFilterWithCaching : uncompiledFilterWithCaching; - - if (refreshHints.isFilterNarrowing) { - filteredItems = batchFilter(filteredItems, filterArgs); - } else if (refreshHints.isFilterExpanding) { - filteredItems = batchFilterWithCaching(items, filterArgs, filterCache); - } else if (!refreshHints.isFilterUnchanged) { - filteredItems = batchFilter(items, filterArgs); - } - } else { - // special case: if not filtering and not paging, the resulting - // rows collection needs to be a copy so that changes due to sort - // can be caught - filteredItems = pagesize ? items : items.concat(); - } - - // get the current page - var paged; - if (pagesize) { - if (filteredItems.length < pagenum * pagesize) { - pagenum = Math.floor(filteredItems.length / pagesize); - } - paged = filteredItems.slice(pagesize * pagenum, pagesize * pagenum + pagesize); - } else { - paged = filteredItems; - } - - return {totalRows: filteredItems.length, rows: paged}; - } - - function getRowDiffs(rows, newRows) { - var item, r, eitherIsNonData, diff = []; - var from = 0, to = newRows.length; - - if (refreshHints && refreshHints.ignoreDiffsBefore) { - from = Math.max(0, - Math.min(newRows.length, refreshHints.ignoreDiffsBefore)); - } - - if (refreshHints && refreshHints.ignoreDiffsAfter) { - to = Math.min(newRows.length, - Math.max(0, refreshHints.ignoreDiffsAfter)); - } - - for (var i = from, rl = rows.length; i < to; i++) { - if (i >= rl) { - diff[diff.length] = i; - } else { - item = newRows[i]; - r = rows[i]; - - if ((groupingInfos.length && (eitherIsNonData = (item.__nonDataRow) || (r.__nonDataRow)) && - item.__group !== r.__group || - item.__group && !item.equals(r)) - || (eitherIsNonData && - // no good way to compare totals since they are arbitrary DTOs - // deep object comparison is pretty expensive - // always considering them 'dirty' seems easier for the time being - (item.__groupTotals || r.__groupTotals)) - || item[idProperty] != r[idProperty] - || (updated && updated[item[idProperty]]) - ) { - diff[diff.length] = i; - } - } - } - return diff; - } - - function recalc(_items) { - rowsById = null; - - if (refreshHints.isFilterNarrowing != prevRefreshHints.isFilterNarrowing || - refreshHints.isFilterExpanding != prevRefreshHints.isFilterExpanding) { - filterCache = []; - } - - var filteredItems = getFilteredAndPagedItems(_items); - totalRows = filteredItems.totalRows; - var newRows = filteredItems.rows; - - groups = []; - if (groupingInfos.length) { - groups = extractGroups(newRows); - if (groups.length) { - addTotals(groups); - newRows = flattenGroupedRows(groups); - } - } - - var diff = getRowDiffs(rows, newRows); - - rows = newRows; - - return diff; - } - - function refresh() { - if (suspend) { - return; - } - - var countBefore = rows.length; - var totalRowsBefore = totalRows; - - var diff = recalc(items, filter); // pass as direct refs to avoid closure perf hit - - // if the current page is no longer valid, go to last page and recalc - // we suffer a performance penalty here, but the main loop (recalc) remains highly optimized - if (pagesize && totalRows < pagenum * pagesize) { - pagenum = Math.max(0, Math.ceil(totalRows / pagesize) - 1); - diff = recalc(items, filter); - } - - updated = null; - prevRefreshHints = refreshHints; - refreshHints = {}; - - if (totalRowsBefore != totalRows) { - onPagingInfoChanged.notify(getPagingInfo(), null, self); - } - if (countBefore != rows.length) { - onRowCountChanged.notify({previous: countBefore, current: rows.length}, null, self); - } - if (diff.length > 0) { - onRowsChanged.notify({rows: diff}, null, self); - } - } - - /*** - * Wires the grid and the DataView together to keep row selection tied to item ids. - * This is useful since, without it, the grid only knows about rows, so if the items - * move around, the same rows stay selected instead of the selection moving along - * with the items. - * - * NOTE: This doesn't work with cell selection model. - * - * @param grid {Slick.Grid} The grid to sync selection with. - * @param preserveHidden {Boolean} Whether to keep selected items that go out of the - * view due to them getting filtered out. - * @param preserveHiddenOnSelectionChange {Boolean} Whether to keep selected items - * that are currently out of the view (see preserveHidden) as selected when selection - * changes. - * @return {Slick.Event} An event that notifies when an internal list of selected row ids - * changes. This is useful since, in combination with the above two options, it allows - * access to the full list selected row ids, and not just the ones visible to the grid. - * @method syncGridSelection - */ - function syncGridSelection(grid, preserveHidden, preserveHiddenOnSelectionChange) { - var self = this; - var inHandler; - var selectedRowIds = self.mapRowsToIds(grid.getSelectedRows()); - var onSelectedRowIdsChanged = new Slick.Event(); - - function setSelectedRowIds(rowIds) { - if (selectedRowIds.join(",") == rowIds.join(",")) { - return; - } - - selectedRowIds = rowIds; - - onSelectedRowIdsChanged.notify({ - "grid": grid, - "ids": selectedRowIds - }, new Slick.EventData(), self); - } - - function update() { - if (selectedRowIds.length > 0) { - inHandler = true; - var selectedRows = self.mapIdsToRows(selectedRowIds); - if (!preserveHidden) { - setSelectedRowIds(self.mapRowsToIds(selectedRows)); - } - grid.setSelectedRows(selectedRows); - inHandler = false; - } - } - - grid.onSelectedRowsChanged.subscribe(function(e, args) { - if (inHandler) { return; } - var newSelectedRowIds = self.mapRowsToIds(grid.getSelectedRows()); - if (!preserveHiddenOnSelectionChange || !grid.getOptions().multiSelect) { - setSelectedRowIds(newSelectedRowIds); - } else { - // keep the ones that are hidden - var existing = $.grep(selectedRowIds, function(id) { return self.getRowById(id) === undefined; }); - // add the newly selected ones - setSelectedRowIds(existing.concat(newSelectedRowIds)); - } - }); - - this.onRowsChanged.subscribe(update); - - this.onRowCountChanged.subscribe(update); - - return onSelectedRowIdsChanged; - } - - function syncGridCellCssStyles(grid, key) { - var hashById; - var inHandler; - - // since this method can be called after the cell styles have been set, - // get the existing ones right away - storeCellCssStyles(grid.getCellCssStyles(key)); - - function storeCellCssStyles(hash) { - hashById = {}; - for (var row in hash) { - var id = rows[row][idProperty]; - hashById[id] = hash[row]; - } - } - - function update() { - if (hashById) { - inHandler = true; - ensureRowsByIdCache(); - var newHash = {}; - for (var id in hashById) { - var row = rowsById[id]; - if (row != undefined) { - newHash[row] = hashById[id]; - } - } - grid.setCellCssStyles(key, newHash); - inHandler = false; - } - } - - grid.onCellCssStylesChanged.subscribe(function(e, args) { - if (inHandler) { return; } - if (key != args.key) { return; } - if (args.hash) { - storeCellCssStyles(args.hash); - } - }); - - this.onRowsChanged.subscribe(update); - - this.onRowCountChanged.subscribe(update); - } - - $.extend(this, { - // methods - "beginUpdate": beginUpdate, - "endUpdate": endUpdate, - "setPagingOptions": setPagingOptions, - "getPagingInfo": getPagingInfo, - "getItems": getItems, - "setItems": setItems, - "setFilter": setFilter, - "sort": sort, - "fastSort": fastSort, - "reSort": reSort, - "setGrouping": setGrouping, - "getGrouping": getGrouping, - "groupBy": groupBy, - "setAggregators": setAggregators, - "collapseAllGroups": collapseAllGroups, - "expandAllGroups": expandAllGroups, - "collapseGroup": collapseGroup, - "expandGroup": expandGroup, - "getGroups": getGroups, - "getIdxById": getIdxById, - "getRowById": getRowById, - "getItemById": getItemById, - "getItemByIdx": getItemByIdx, - "mapRowsToIds": mapRowsToIds, - "mapIdsToRows": mapIdsToRows, - "setRefreshHints": setRefreshHints, - "setFilterArgs": setFilterArgs, - "refresh": refresh, - "updateItem": updateItem, - "insertItem": insertItem, - "addItem": addItem, - "deleteItem": deleteItem, - "syncGridSelection": syncGridSelection, - "syncGridCellCssStyles": syncGridCellCssStyles, - - // data provider methods - "getLength": getLength, - "getItem": getItem, - "getItemMetadata": getItemMetadata, - - // events - "onRowCountChanged": onRowCountChanged, - "onRowsChanged": onRowsChanged, - "onPagingInfoChanged": onPagingInfoChanged - }); - } - - function AvgAggregator(field) { - this.field_ = field; - - this.init = function () { - this.count_ = 0; - this.nonNullCount_ = 0; - this.sum_ = 0; - }; - - this.accumulate = function (item) { - var val = item[this.field_]; - this.count_++; - if (val != null && val !== "" && val !== NaN) { - this.nonNullCount_++; - this.sum_ += parseFloat(val); - } - }; - - this.storeResult = function (groupTotals) { - if (!groupTotals.avg) { - groupTotals.avg = {}; - } - if (this.nonNullCount_ != 0) { - groupTotals.avg[this.field_] = this.sum_ / this.nonNullCount_; - } - }; - } - - function MinAggregator(field) { - this.field_ = field; - - this.init = function () { - this.min_ = null; - }; - - this.accumulate = function (item) { - var val = item[this.field_]; - if (val != null && val !== "" && val !== NaN) { - if (this.min_ == null || val < this.min_) { - this.min_ = val; - } - } - }; - - this.storeResult = function (groupTotals) { - if (!groupTotals.min) { - groupTotals.min = {}; - } - groupTotals.min[this.field_] = this.min_; - } - } - - function MaxAggregator(field) { - this.field_ = field; - - this.init = function () { - this.max_ = null; - }; - - this.accumulate = function (item) { - var val = item[this.field_]; - if (val != null && val !== "" && val !== NaN) { - if (this.max_ == null || val > this.max_) { - this.max_ = val; - } - } - }; - - this.storeResult = function (groupTotals) { - if (!groupTotals.max) { - groupTotals.max = {}; - } - groupTotals.max[this.field_] = this.max_; - } - } - - function SumAggregator(field) { - this.field_ = field; - - this.init = function () { - this.sum_ = null; - }; - - this.accumulate = function (item) { - var val = item[this.field_]; - if (val != null && val !== "" && val !== NaN) { - this.sum_ += parseFloat(val); - } - }; - - this.storeResult = function (groupTotals) { - if (!groupTotals.sum) { - groupTotals.sum = {}; - } - groupTotals.sum[this.field_] = this.sum_; - } - } - - // TODO: add more built-in aggregators - // TODO: merge common aggregators in one to prevent needles iterating - -})(jQuery); diff --git a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.editors.js b/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.editors.js deleted file mode 100644 index 04b20d2..0000000 --- a/compose/2.10/src/ckanext-d4science_theme/ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.editors.js +++ /dev/null @@ -1,512 +0,0 @@ -/*** - * Contains basic SlickGrid editors. - * @module Editors - * @namespace Slick - */ - -(function ($) { - // register namespace - $.extend(true, window, { - "Slick": { - "Editors": { - "Text": TextEditor, - "Integer": IntegerEditor, - "Date": DateEditor, - "YesNoSelect": YesNoSelectEditor, - "Checkbox": CheckboxEditor, - "PercentComplete": PercentCompleteEditor, - "LongText": LongTextEditor - } - } - }); - - function TextEditor(args) { - var $input; - var defaultValue; - var scope = this; - - this.init = function () { - $input = $("") - .appendTo(args.container) - .bind("keydown.nav", function (e) { - if (e.keyCode === $.ui.keyCode.LEFT || e.keyCode === $.ui.keyCode.RIGHT) { - e.stopImmediatePropagation(); - } - }) - .focus() - .select(); - }; - - this.destroy = function () { - $input.remove(); - }; - - this.focus = function () { - $input.focus(); - }; - - this.getValue = function () { - return $input.val(); - }; - - this.setValue = function (val) { - $input.val(val); - }; - - this.loadValue = function (item) { - defaultValue = item[args.column.field] || ""; - $input.val(defaultValue); - $input[0].defaultValue = defaultValue; - $input.select(); - }; - - this.serializeValue = function () { - return $input.val(); - }; - - this.applyValue = function (item, state) { - item[args.column.field] = state; - }; - - this.isValueChanged = function () { - return (!($input.val() == "" && defaultValue == null)) && ($input.val() != defaultValue); - }; - - this.validate = function () { - if (args.column.validator) { - var validationResults = args.column.validator($input.val()); - if (!validationResults.valid) { - return validationResults; - } - } - - return { - valid: true, - msg: null - }; - }; - - this.init(); - } - - function IntegerEditor(args) { - var $input; - var defaultValue; - var scope = this; - - this.init = function () { - $input = $(""); - - $input.bind("keydown.nav", function (e) { - if (e.keyCode === $.ui.keyCode.LEFT || e.keyCode === $.ui.keyCode.RIGHT) { - e.stopImmediatePropagation(); - } - }); - - $input.appendTo(args.container); - $input.focus().select(); - }; - - this.destroy = function () { - $input.remove(); - }; - - this.focus = function () { - $input.focus(); - }; - - this.loadValue = function (item) { - defaultValue = item[args.column.field]; - $input.val(defaultValue); - $input[0].defaultValue = defaultValue; - $input.select(); - }; - - this.serializeValue = function () { - return parseInt($input.val(), 10) || 0; - }; - - this.applyValue = function (item, state) { - item[args.column.field] = state; - }; - - this.isValueChanged = function () { - return (!($input.val() == "" && defaultValue == null)) && ($input.val() != defaultValue); - }; - - this.validate = function () { - if (isNaN($input.val())) { - return { - valid: false, - msg: "Please enter a valid integer" - }; - } - - return { - valid: true, - msg: null - }; - }; - - this.init(); - } - - function DateEditor(args) { - var $input; - var defaultValue; - var scope = this; - var calendarOpen = false; - - this.init = function () { - $input = $(""); - $input.appendTo(args.container); - $input.focus().select(); - $input.datepicker({ - showOn: "button", - buttonImageOnly: true, - buttonImage: "../images/calendar.gif", - beforeShow: function () { - calendarOpen = true - }, - onClose: function () { - calendarOpen = false - } - }); - $input.width($input.width() - 18); - }; - - this.destroy = function () { - $.datepicker.dpDiv.stop(true, true); - $input.datepicker("hide"); - $input.datepicker("destroy"); - $input.remove(); - }; - - this.show = function () { - if (calendarOpen) { - $.datepicker.dpDiv.stop(true, true).show(); - } - }; - - this.hide = function () { - if (calendarOpen) { - $.datepicker.dpDiv.stop(true, true).hide(); - } - }; - - this.position = function (position) { - if (!calendarOpen) { - return; - } - $.datepicker.dpDiv - .css("top", position.top + 30) - .css("left", position.left); - }; - - this.focus = function () { - $input.focus(); - }; - - this.loadValue = function (item) { - defaultValue = item[args.column.field]; - $input.val(defaultValue); - $input[0].defaultValue = defaultValue; - $input.select(); - }; - - this.serializeValue = function () { - return $input.val(); - }; - - this.applyValue = function (item, state) { - item[args.column.field] = state; - }; - - this.isValueChanged = function () { - return (!($input.val() == "" && defaultValue == null)) && ($input.val() != defaultValue); - }; - - this.validate = function () { - return { - valid: true, - msg: null - }; - }; - - this.init(); - } - - function YesNoSelectEditor(args) { - var $select; - var defaultValue; - var scope = this; - - this.init = function () { - $select = $(""); - $select.appendTo(args.container); - $select.focus(); - }; - - this.destroy = function () { - $select.remove(); - }; - - this.focus = function () { - $select.focus(); - }; - - this.loadValue = function (item) { - $select.val((defaultValue = item[args.column.field]) ? "yes" : "no"); - $select.select(); - }; - - this.serializeValue = function () { - return ($select.val() == "yes"); - }; - - this.applyValue = function (item, state) { - item[args.column.field] = state; - }; - - this.isValueChanged = function () { - return ($select.val() != defaultValue); - }; - - this.validate = function () { - return { - valid: true, - msg: null - }; - }; - - this.init(); - } - - function CheckboxEditor(args) { - var $select; - var defaultValue; - var scope = this; - - this.init = function () { - $select = $(""); - $select.appendTo(args.container); - $select.focus(); - }; - - this.destroy = function () { - $select.remove(); - }; - - this.focus = function () { - $select.focus(); - }; - - this.loadValue = function (item) { - defaultValue = !!item[args.column.field]; - if (defaultValue) { - $select.prop('checked', true); - } else { - $select.prop('checked', false); - } - }; - - this.serializeValue = function () { - return $select.prop('checked'); - }; - - this.applyValue = function (item, state) { - item[args.column.field] = state; - }; - - this.isValueChanged = function () { - return (this.serializeValue() !== defaultValue); - }; - - this.validate = function () { - return { - valid: true, - msg: null - }; - }; - - this.init(); - } - - function PercentCompleteEditor(args) { - var $input, $picker; - var defaultValue; - var scope = this; - - this.init = function () { - $input = $(""); - $input.width($(args.container).innerWidth() - 25); - $input.appendTo(args.container); - - $picker = $("

    ").appendTo(args.container); - $picker.append("
    "); - - $picker.find(".editor-percentcomplete-buttons").append("

    "); - - $input.focus().select(); - - $picker.find(".editor-percentcomplete-slider").slider({ - orientation: "vertical", - range: "min", - value: defaultValue, - slide: function (event, ui) { - $input.val(ui.value) - } - }); - - $picker.find(".editor-percentcomplete-buttons button").bind("click", function (e) { - $input.val($(this).attr("val")); - $picker.find(".editor-percentcomplete-slider").slider("value", $(this).attr("val")); - }) - }; - - this.destroy = function () { - $input.remove(); - $picker.remove(); - }; - - this.focus = function () { - $input.focus(); - }; - - this.loadValue = function (item) { - $input.val(defaultValue = item[args.column.field]); - $input.select(); - }; - - this.serializeValue = function () { - return parseInt($input.val(), 10) || 0; - }; - - this.applyValue = function (item, state) { - item[args.column.field] = state; - }; - - this.isValueChanged = function () { - return (!($input.val() == "" && defaultValue == null)) && ((parseInt($input.val(), 10) || 0) != defaultValue); - }; - - this.validate = function () { - if (isNaN(parseInt($input.val(), 10))) { - return { - valid: false, - msg: "Please enter a valid positive number" - }; - } - - return { - valid: true, - msg: null - }; - }; - - this.init(); - } - - /* - * An example of a "detached" editor. - * The UI is added onto document BODY and .position(), .show() and .hide() are implemented. - * KeyDown events are also handled to provide handling for Tab, Shift-Tab, Esc and Ctrl-Enter. - */ - function LongTextEditor(args) { - var $input, $wrapper; - var defaultValue; - var scope = this; - - this.init = function () { - var $container = $("body"); - - $wrapper = $("
    ") - .appendTo($container); - - $input = $("