From 6d55c38b334a8657647028e0a66913995894f2b8 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 10:08:35 +0200 Subject: [PATCH 01/20] Install requirements on tests --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e45c7c7..c68c308 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,6 +49,7 @@ jobs: - uses: actions/checkout@v2 - name: Install requirements run: | + pip install -r requirements.txt pip install -r dev-requirements.txt pip install -e . # Replace default path to CKAN core config file with the one on the container From f678a708427559a5d2e054b24cc97c2c7beb84fa Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 10:12:52 +0200 Subject: [PATCH 02/20] Use toolkit to access action --- ckanext/googleanalytics/controller.py | 3 +-- ckanext/googleanalytics/views.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ckanext/googleanalytics/controller.py b/ckanext/googleanalytics/controller.py index 3104c49..0777fd7 100644 --- a/ckanext/googleanalytics/controller.py +++ b/ckanext/googleanalytics/controller.py @@ -4,7 +4,6 @@ import logging from ckan.lib.base import BaseController, c, render, request from . import dbutil -import ckan.logic as logic import hashlib from . import plugin @@ -56,7 +55,7 @@ class GAApiController(ApiController): def action(self, logic_function, ver=None): try: - function = logic.get_action(logic_function) + function = tk.get_action(logic_function) side_effect_free = getattr(function, "side_effect_free", False) request_data = self._get_request_data( try_url_params=side_effect_free diff --git a/ckanext/googleanalytics/views.py b/ckanext/googleanalytics/views.py index 392e613..904d76e 100644 --- a/ckanext/googleanalytics/views.py +++ b/ckanext/googleanalytics/views.py @@ -7,7 +7,6 @@ import six from flask import Blueprint from werkzeug.utils import import_string -import ckan.logic as logic import ckan.plugins.toolkit as tk import ckan.views.api as api import ckan.views.resource as resource @@ -22,7 +21,7 @@ ga = Blueprint("google_analytics", "google_analytics") def action(logic_function, ver=api.API_MAX_VERSION): try: - function = logic.get_action(logic_function) + function = tk.get_action(logic_function) side_effect_free = getattr(function, "side_effect_free", False) request_data = api._get_request_data(try_url_params=side_effect_free) if isinstance(request_data, dict): From bd0678b2d5e377d6d42c259edfc9f976b013575a Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 11:14:23 +0200 Subject: [PATCH 03/20] Add pytest-cov --- .github/workflows/test.yml | 2 +- dev-requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c68c308..8d027b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,4 +65,4 @@ jobs: paster --plugin=ckan db init -c test.ini paster --plugin=ckanext-googleanalytics initdb -c test.ini - name: Run tests - run: pytest --ckan-ini=test.ini --disable-warnings ckanext/googleanalytics/tests + run: pytest --ckan-ini=test.ini --cov=ckanext.googleanalytics --disable-warnings ckanext/googleanalytics/tests diff --git a/dev-requirements.txt b/dev-requirements.txt index dfb701d..6aac721 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,2 +1,3 @@ pytest-ckan pytest-factoryboy +pytest-cov \ No newline at end of file From e223d3c71e6b202153e422916bb1e12aaab895f4 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 11:22:21 +0200 Subject: [PATCH 04/20] Use CKAN Core dev-requirements --- .github/workflows/test.yml | 1 - dev-requirements.txt | 3 --- 2 files changed, 4 deletions(-) delete mode 100644 dev-requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d027b5..037cae3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,6 @@ jobs: - name: Install requirements run: | pip install -r requirements.txt - pip install -r dev-requirements.txt pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 6aac721..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest-ckan -pytest-factoryboy -pytest-cov \ No newline at end of file From f63b879b1898aba4c8167b44923cb26432e01aa1 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 11:27:05 +0200 Subject: [PATCH 05/20] Restore dev-requirements.txt --- .github/workflows/test.yml | 1 + ckanext/dev-requirements.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 ckanext/dev-requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 037cae3..8d027b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,7 @@ jobs: - name: Install requirements run: | pip install -r requirements.txt + pip install -r dev-requirements.txt pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini diff --git a/ckanext/dev-requirements.txt b/ckanext/dev-requirements.txt new file mode 100644 index 0000000..172902c --- /dev/null +++ b/ckanext/dev-requirements.txt @@ -0,0 +1 @@ +pytest-ckan \ No newline at end of file From 483d4d20ce92c00670b28bc36249d6e3f0767e3c Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 11:30:20 +0200 Subject: [PATCH 06/20] Fix file folder --- ckanext/dev-requirements.txt => dev-requirements.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ckanext/dev-requirements.txt => dev-requirements.txt (100%) diff --git a/ckanext/dev-requirements.txt b/dev-requirements.txt similarity index 100% rename from ckanext/dev-requirements.txt rename to dev-requirements.txt From 97448b4887787840377fb8bc33c25fb10a9bc365 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 11:42:27 +0200 Subject: [PATCH 07/20] Manage pytest requirements for CKAN versions --- .github/workflows/test.yml | 9 ++++++++- dev-requirements-2.9.txt | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 dev-requirements-2.9.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d027b5..9c0b363 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,18 +50,25 @@ jobs: - name: Install requirements run: | pip install -r requirements.txt - pip install -r dev-requirements.txt pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Setup extension (CKAN 2.10) + if: ${{ matrix.ckan-version == '2.10' }} + run: | + pip install -r dev-requirements.txt + ckan -c test.ini db init + ckan -c test.ini db upgrade -p googleanalytics - name: Setup extension (CKAN >= 2.9) if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} run: | + pip install -r dev-requirements-2.9.txt ckan -c test.ini db init ckan -c test.ini db upgrade -p googleanalytics - name: Setup extension (CKAN < 2.9) if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} run: | + pip install -r dev-requirements-2.9.txt paster --plugin=ckan db init -c test.ini paster --plugin=ckanext-googleanalytics initdb -c test.ini - name: Run tests diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt new file mode 100644 index 0000000..72345ad --- /dev/null +++ b/dev-requirements-2.9.txt @@ -0,0 +1,2 @@ +pytest-ckan +pytestfactoryboy \ No newline at end of file From 589890ef74883fe6bd72d51a0ca7cb127c8d1067 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 11:59:02 +0200 Subject: [PATCH 08/20] Fix typos --- .github/workflows/test.yml | 4 ++-- dev-requirements-2.9.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c0b363..79668cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,8 +59,8 @@ jobs: pip install -r dev-requirements.txt ckan -c test.ini db init ckan -c test.ini db upgrade -p googleanalytics - - name: Setup extension (CKAN >= 2.9) - if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} + - name: Setup extension (CKAN == 2.9) + if: ${{ matrix.ckan-version == '2.9' }} run: | pip install -r dev-requirements-2.9.txt ckan -c test.ini db init diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt index 72345ad..2fc9171 100644 --- a/dev-requirements-2.9.txt +++ b/dev-requirements-2.9.txt @@ -1,2 +1,2 @@ pytest-ckan -pytestfactoryboy \ No newline at end of file +pytest-factoryboy \ No newline at end of file From 4e57fbaef286f875f60d279679a2789ad8840e0e Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:06:34 +0200 Subject: [PATCH 09/20] Pin pytest in CKAN 2.0 --- dev-requirements-2.9.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt index 2fc9171..54a3328 100644 --- a/dev-requirements-2.9.txt +++ b/dev-requirements-2.9.txt @@ -1,2 +1,3 @@ +pytest==6.0.2 pytest-ckan pytest-factoryboy \ No newline at end of file From 1d41a85079647dfb183c1b82cdafc179eefa7b10 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:12:59 +0200 Subject: [PATCH 10/20] Fix pinned version --- .github/workflows/test.yml | 2 +- dev-requirements-2.9.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 79668cd..9e7122b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: ckan -c test.ini db init ckan -c test.ini db upgrade -p googleanalytics - name: Setup extension (CKAN == 2.9) - if: ${{ matrix.ckan-version == '2.9' }} + if: ${{ matrix.ckan-version == '2.9' || matrix.ckan-version == '2.9-py2' }} run: | pip install -r dev-requirements-2.9.txt ckan -c test.ini db init diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt index 54a3328..afab1de 100644 --- a/dev-requirements-2.9.txt +++ b/dev-requirements-2.9.txt @@ -1,3 +1,3 @@ -pytest==6.0.2 +pytest<=6.0.2 pytest-ckan pytest-factoryboy \ No newline at end of file From 259d9768e20b45ee350f7718eb8d7ce0a7d81b61 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:17:20 +0200 Subject: [PATCH 11/20] Fix dev requirements 2.0 --- dev-requirements-2.9.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt index afab1de..53c02d8 100644 --- a/dev-requirements-2.9.txt +++ b/dev-requirements-2.9.txt @@ -1,3 +1,4 @@ pytest<=6.0.2 pytest-ckan -pytest-factoryboy \ No newline at end of file +pytest-factoryboy +pytest-cov \ No newline at end of file From df5a7d198f4d57b8cf731a02d32ad91f68b64ad0 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:19:43 +0200 Subject: [PATCH 12/20] Remove broken method --- ckanext/googleanalytics/dbutil.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/ckanext/googleanalytics/dbutil.py b/ckanext/googleanalytics/dbutil.py index 9602d75..53f019e 100644 --- a/ckanext/googleanalytics/dbutil.py +++ b/ckanext/googleanalytics/dbutil.py @@ -78,36 +78,6 @@ def get_resource_visits_for_url(url): return count and count[0] or "" -""" get_top_packages is broken, and needs to be rewritten to work with -CKAN 2.*. This is because ckan.authz has been removed in CKAN 2.* - -See commit ffa86c010d5d25fa1881c6b915e48f3b44657612 -""" - - -def get_top_packages(limit=20): - items = [] - # caveat emptor: the query below will not filter out private - # or deleted datasets (TODO) - q = model.Session.query(model.Package) - connection = model.Session.connection() - package_stats = get_table("package_stats") - s = select( - [ - package_stats.c.package_id, - package_stats.c.visits_recently, - package_stats.c.visits_ever, - ] - ).order_by(package_stats.c.visits_recently.desc()) - res = connection.execute(s).fetchmany(limit) - for package_id, recent, ever in res: - item = q.filter(text("package.id = '%s'" % package_id)) - if not item.count(): - continue - items.append((item.first(), recent, ever)) - return items - - def get_top_resources(limit=20): items = [] connection = model.Session.connection() From 1813413b33170f87cc9fb721c7b5a8cdecfab722 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:26:50 +0200 Subject: [PATCH 13/20] Fix fixture for older versions --- ckanext/googleanalytics/tests/conftest.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ckanext/googleanalytics/tests/conftest.py b/ckanext/googleanalytics/tests/conftest.py index 7fe6cf5..b5f5422 100644 --- a/ckanext/googleanalytics/tests/conftest.py +++ b/ckanext/googleanalytics/tests/conftest.py @@ -3,16 +3,25 @@ import factory from factory.alchemy import SQLAlchemyModelFactory from pytest_factoryboy import register +from ckan.plugins import toolkit import ckan.model as model from ckanext.googleanalytics.model import PackageStats, ResourceStats -@pytest.fixture() -def clean_db(reset_db, migrate_db_for): - reset_db() - migrate_db_for("googleanalytics") +if toolkit.requires_ckan_version("2.9"): + @pytest.fixture() + def clean_db(reset_db, migrate_db_for): + reset_db() + migrate_db_for("googleanalytics") +else: + from dbutil import init_tables + @pytest.fixture() + def clean_db(reset_db): + reset_db() + init_tables() + @register From 2bc1d0e0dda26f2ac44f59d02b687c7368314e0d Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:30:05 +0200 Subject: [PATCH 14/20] Fix pep8 --- ckanext/googleanalytics/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckanext/googleanalytics/tests/conftest.py b/ckanext/googleanalytics/tests/conftest.py index b5f5422..176f5f6 100644 --- a/ckanext/googleanalytics/tests/conftest.py +++ b/ckanext/googleanalytics/tests/conftest.py @@ -23,7 +23,6 @@ else: init_tables() - @register class PackageStatsFactory(SQLAlchemyModelFactory): class Meta: From c4dd52fc5633ca6180b92644e6a92be7c2a46fed Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:37:09 +0200 Subject: [PATCH 15/20] Fix pep8 --- ckanext/googleanalytics/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/googleanalytics/tests/conftest.py b/ckanext/googleanalytics/tests/conftest.py index 176f5f6..cc13c50 100644 --- a/ckanext/googleanalytics/tests/conftest.py +++ b/ckanext/googleanalytics/tests/conftest.py @@ -17,6 +17,7 @@ if toolkit.requires_ckan_version("2.9"): migrate_db_for("googleanalytics") else: from dbutil import init_tables + @pytest.fixture() def clean_db(reset_db): reset_db() From dc5c5c5f2f982e7eb8f07600ead0d3d30fcfac96 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:38:28 +0200 Subject: [PATCH 16/20] Fix more pep8 --- ckanext/googleanalytics/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/googleanalytics/tests/conftest.py b/ckanext/googleanalytics/tests/conftest.py index cc13c50..784f74b 100644 --- a/ckanext/googleanalytics/tests/conftest.py +++ b/ckanext/googleanalytics/tests/conftest.py @@ -17,7 +17,7 @@ if toolkit.requires_ckan_version("2.9"): migrate_db_for("googleanalytics") else: from dbutil import init_tables - + @pytest.fixture() def clean_db(reset_db): reset_db() From 688270801777e48898b9ac68cab73c215b03bf69 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:41:36 +0200 Subject: [PATCH 17/20] Fix import --- ckanext/googleanalytics/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/googleanalytics/tests/conftest.py b/ckanext/googleanalytics/tests/conftest.py index 784f74b..1f63978 100644 --- a/ckanext/googleanalytics/tests/conftest.py +++ b/ckanext/googleanalytics/tests/conftest.py @@ -16,7 +16,7 @@ if toolkit.requires_ckan_version("2.9"): reset_db() migrate_db_for("googleanalytics") else: - from dbutil import init_tables + from ckanext.googleanalytics.dbutil import init_tables @pytest.fixture() def clean_db(reset_db): From 8a135677d4e365f3e636342fb23a8f799e256186 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 12:52:37 +0200 Subject: [PATCH 18/20] Clean obsolete code --- ckanext/googleanalytics/controller.py | 8 -------- ckanext/googleanalytics/tests/conftest.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/ckanext/googleanalytics/controller.py b/ckanext/googleanalytics/controller.py index 0777fd7..33b6873 100644 --- a/ckanext/googleanalytics/controller.py +++ b/ckanext/googleanalytics/controller.py @@ -11,17 +11,9 @@ from paste.util.multidict import MultiDict from ckan.controllers.api import ApiController -from ckan.exceptions import CkanVersionException import ckan.plugins.toolkit as tk from ckanext.googleanalytics import utils -try: - tk.requires_ckan_version("2.9") -except CkanVersionException: - pass -else: - from builtins import str - log = logging.getLogger("ckanext.googleanalytics") diff --git a/ckanext/googleanalytics/tests/conftest.py b/ckanext/googleanalytics/tests/conftest.py index 1f63978..9669f92 100644 --- a/ckanext/googleanalytics/tests/conftest.py +++ b/ckanext/googleanalytics/tests/conftest.py @@ -10,7 +10,7 @@ import ckan.model as model from ckanext.googleanalytics.model import PackageStats, ResourceStats -if toolkit.requires_ckan_version("2.9"): +if toolkit.check_ckan_version("2.9"): @pytest.fixture() def clean_db(reset_db, migrate_db_for): reset_db() From 4455f9ab8972fb69c74fd2d2cccafe81768f5411 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 30 May 2022 13:03:01 +0200 Subject: [PATCH 19/20] Update Readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d5ad362..ba52192 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ See `Googles' documentation Date: Wed, 8 Jun 2022 11:46:59 +0200 Subject: [PATCH 20/20] Remove 2.7 from test matrix --- .github/workflows/test.yml | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e7122b..6747672 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7] + ckan-version: ["2.10", 2.9, 2.9-py2, 2.8] fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -66,7 +66,7 @@ jobs: ckan -c test.ini db init ckan -c test.ini db upgrade -p googleanalytics - name: Setup extension (CKAN < 2.9) - if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} + if: ${{ matrix.ckan-version == '2.8' }} run: | pip install -r dev-requirements-2.9.txt paster --plugin=ckan db init -c test.ini diff --git a/README.md b/README.md index ba52192..117ae25 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Status:** Production -**CKAN Version:** >= 2.7 +**CKAN Version:** >= 2.8 A CKAN extension that both sends tracking data to Google Analytics and retrieves statistics from Google Analytics and inserts them into CKAN pages.