From dc75b4c472edb2d1074fd650936de8cfc2c424f1 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 6 May 2022 19:28:52 +0300 Subject: [PATCH] Configure tests --- .github/workflows/test.yml | 68 ++++++++++++++++++++ ckanext/googleanalytics/cli.py | 19 +++--- ckanext/googleanalytics/commands.py | 17 +++-- ckanext/googleanalytics/dbutil.py | 7 +- ckanext/googleanalytics/ga_auth.py | 9 +-- ckanext/googleanalytics/gasnippet.py | 9 ++- ckanext/googleanalytics/plugin/__init__.py | 2 +- ckanext/googleanalytics/tests/test_plugin.py | 7 ++ dev-requirements.txt | 1 + setup.cfg | 1 + test.ini | 46 +++++++++++++ 11 files changed, 155 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 ckanext/googleanalytics/tests/test_plugin.py create mode 100644 dev-requirements.txt create mode 100644 test.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..8d027b5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,68 @@ +name: Tests +on: [push, pull_request] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + - name: Run flake8 + run: flake8 . --count --max-line-length=127 --statistics --exclude ckan + + test: + needs: lint + strategy: + matrix: + ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7] + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + container: + image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + services: + solr: + image: ckan/ckan-solr:${{ matrix.ckan-version }} + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + + steps: + - uses: actions/checkout@v2 + - name: Install requirements + run: | + pip install -r requirements.txt + pip install -r dev-requirements.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Setup extension (CKAN >= 2.9) + if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} + run: | + ckan -c test.ini db init + ckan -c test.ini db upgrade -p googleanalytics + - name: Setup extension (CKAN < 2.9) + if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} + run: | + paster --plugin=ckan db init -c test.ini + paster --plugin=ckanext-googleanalytics initdb -c test.ini + - name: Run tests + run: pytest --ckan-ini=test.ini --cov=ckanext.googleanalytics --disable-warnings ckanext/googleanalytics/tests diff --git a/ckanext/googleanalytics/cli.py b/ckanext/googleanalytics/cli.py index a46420a..ddcd36a 100644 --- a/ckanext/googleanalytics/cli.py +++ b/ckanext/googleanalytics/cli.py @@ -10,8 +10,6 @@ import ckan.model as model from . import dbutil, utils -import ckan.plugins.toolkit as tk - log = logging.getLogger(__name__) PACKAGE_URL = "/dataset/" # XXX get from routes... @@ -105,7 +103,8 @@ def internal_save(packages_data, summary_date): SET package_id = COALESCE( (SELECT id FROM package p WHERE t.url = %s || p.name) ,'~~not~found~~') - WHERE t.package_id = '~~not~found~~' AND tracking_type = 'page';""" + WHERE t.package_id = '~~not~found~~' + AND tracking_type = 'page';""" engine.execute(sql, "%sedit/" % PACKAGE_URL) # update summary totals for resources @@ -120,7 +119,8 @@ def internal_save(packages_data, summary_date): SELECT sum(count) FROM tracking_summary t2 WHERE t1.url = t2.url - AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s + AND t2.tracking_date <= t1.tracking_date + AND t2.tracking_date >= t1.tracking_date - %s ) + t1.count WHERE t1.running_total = 0 AND tracking_type = 'resource';""" engine.execute(sql, utils.config_recent_view_days()) @@ -137,7 +137,8 @@ def internal_save(packages_data, summary_date): SELECT sum(count) FROM tracking_summary t2 WHERE t1.package_id = t2.package_id - AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s + AND t2.tracking_date <= t1.tracking_date + AND t2.tracking_date >= t1.tracking_date - %s ) + t1.count WHERE t1.running_total = 0 AND tracking_type = 'page' AND t1.package_id IS NOT NULL @@ -243,7 +244,7 @@ def save_ga_data(packages_data): ever = visits.get("ever", 0) matches = RESOURCE_URL_REGEX.match(identifier) if matches: - resource_url = identifier[len(utils.config_prefix()) :] + resource_url = identifier[len(utils.config_prefix()):] resource = ( model.Session.query(model.Resource) .autoflush(True) @@ -256,7 +257,7 @@ def save_ga_data(packages_data): dbutil.update_resource_visits(resource.id, recently, ever) log.info("Updated %s with %s visits" % (resource.id, visits)) else: - package_name = identifier[len(PACKAGE_URL) :] + package_name = identifier[len(PACKAGE_URL):] if "/" in package_name: log.warning("%s not a valid package name" % package_name) continue @@ -337,8 +338,8 @@ def get_ga_data(service, profile_id, query_filter): package = "/" + "/".join(package.split("/")[2:]) count = result[1] - # Make sure we add the different representations of the same - # dataset /mysite.com & /www.mysite.com ... + # Make sure we add the different representations of the + # same dataset /mysite.com & /www.mysite.com ... val = 0 if package in packages and date_name in packages[package]: val += packages[package][date_name] diff --git a/ckanext/googleanalytics/commands.py b/ckanext/googleanalytics/commands.py index 69d9cbc..d0ed148 100644 --- a/ckanext/googleanalytics/commands.py +++ b/ckanext/googleanalytics/commands.py @@ -103,7 +103,8 @@ class LoadAnalytics(CkanCommand): SET package_id = COALESCE( (SELECT id FROM package p WHERE t.url = %s || p.name) ,'~~not~found~~') - WHERE t.package_id = '~~not~found~~' AND tracking_type = 'page';""" + WHERE t.package_id = '~~not~found~~' + AND tracking_type = 'page';""" engine.execute(sql, "%sedit/" % PACKAGE_URL) # update summary totals for resources @@ -118,7 +119,8 @@ class LoadAnalytics(CkanCommand): SELECT sum(count) FROM tracking_summary t2 WHERE t1.url = t2.url - AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s + AND t2.tracking_date <= t1.tracking_date + AND t2.tracking_date >= t1.tracking_date - %s ) + t1.count WHERE t1.running_total = 0 AND tracking_type = 'resource';""" engine.execute(sql, self.recent_view_days) @@ -135,7 +137,8 @@ class LoadAnalytics(CkanCommand): SELECT sum(count) FROM tracking_summary t2 WHERE t1.package_id = t2.package_id - AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s + AND t2.tracking_date <= t1.tracking_date + AND t2.tracking_date >= t1.tracking_date - %s ) + t1.count WHERE t1.running_total = 0 AND tracking_type = 'page' AND t1.package_id IS NOT NULL @@ -265,7 +268,7 @@ class LoadAnalytics(CkanCommand): ever = visits.get("ever", 0) matches = RESOURCE_URL_REGEX.match(identifier) if matches: - resource_url = identifier[len(self.resource_url_tag) :] + resource_url = identifier[len(self.resource_url_tag):] resource = ( model.Session.query(model.Resource) .autoflush(True) @@ -278,7 +281,7 @@ class LoadAnalytics(CkanCommand): dbutil.update_resource_visits(resource.id, recently, ever) log.info("Updated %s with %s visits" % (resource.id, visits)) else: - package_name = identifier[len(PACKAGE_URL) :] + package_name = identifier[len(PACKAGE_URL):] if "/" in package_name: log.warning("%s not a valid package name" % package_name) continue @@ -360,8 +363,8 @@ class LoadAnalytics(CkanCommand): package = "/" + "/".join(package.split("/")[2:]) count = result[1] - # Make sure we add the different representations of the same - # dataset /mysite.com & /www.mysite.com ... + # Make sure we add the different representations of the + # same dataset /mysite.com & /www.mysite.com ... val = 0 if ( package in packages diff --git a/ckanext/googleanalytics/dbutil.py b/ckanext/googleanalytics/dbutil.py index 1314e8c..9602d75 100644 --- a/ckanext/googleanalytics/dbutil.py +++ b/ckanext/googleanalytics/dbutil.py @@ -4,22 +4,19 @@ from sqlalchemy import func import ckan.model as model -# from ckan.model.authz import PSEUDO_USER__VISITOR -from ckan.lib.base import * - cached_tables = {} def init_tables(): metadata = MetaData() - package_stats = Table( + Table( "package_stats", metadata, Column("package_id", String(60), primary_key=True), Column("visits_recently", Integer), Column("visits_ever", Integer), ) - resource_stats = Table( + Table( "resource_stats", metadata, Column("resource_id", String(60), primary_key=True), diff --git a/ckanext/googleanalytics/ga_auth.py b/ckanext/googleanalytics/ga_auth.py index d960006..3c76cab 100644 --- a/ckanext/googleanalytics/ga_auth.py +++ b/ckanext/googleanalytics/ga_auth.py @@ -50,12 +50,9 @@ def get_profile_id(service): accountId = acc.get("id") # TODO: check, whether next line is doing something useful. - webproperties = ( - service.management() - .webproperties() - .list(accountId=accountId) - .execute() - ) + service.management().webproperties().list( + accountId=accountId + ).execute() profiles = ( service.management() diff --git a/ckanext/googleanalytics/gasnippet.py b/ckanext/googleanalytics/gasnippet.py index 8a12f38..1329aa1 100644 --- a/ckanext/googleanalytics/gasnippet.py +++ b/ckanext/googleanalytics/gasnippet.py @@ -5,9 +5,12 @@ header_code = """ _gaq.push(['_setDomainName', '%s']); _gaq.push(['_trackPageview']); (function() { - var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; - ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; - var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); + var ga = document.createElement('script'); + ga.type = 'text/javascript'; ga.async = true; + ga.src = ('https:' == document.location.protocol ? + 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; + var s = document.getElementsByTagName('script')[0]; + s.parentNode.insertBefore(ga, s); })(); """ diff --git a/ckanext/googleanalytics/plugin/__init__.py b/ckanext/googleanalytics/plugin/__init__.py index 38abc87..7e30ef8 100644 --- a/ckanext/googleanalytics/plugin/__init__.py +++ b/ckanext/googleanalytics/plugin/__init__.py @@ -43,7 +43,7 @@ class AnalyticsPostThread(threading.Thread): data = urlencode(data_dict) log.debug("Sending API event to Google Analytics: " + data) # send analytics - res = requests.post( + requests.post( "http://www.google-analytics.com/collect", data, timeout=10, diff --git a/ckanext/googleanalytics/tests/test_plugin.py b/ckanext/googleanalytics/tests/test_plugin.py new file mode 100644 index 0000000..15682bf --- /dev/null +++ b/ckanext/googleanalytics/tests/test_plugin.py @@ -0,0 +1,7 @@ +import pytest + + +@pytest.mark.usefixtures("clean_db") +def test_script(app): + resp = app.get("/") + assert "GoogleAnalyticsObject" in resp diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..eac82b4 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1 @@ +pytest-ckan diff --git a/setup.cfg b/setup.cfg index a972f71..65432cf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ requirements = [options.entry_points] ckan.plugins = googleanalytics = ckanext.googleanalytics.plugin:GoogleAnalyticsPlugin + paste.paster_command = loadanalytics = ckanext.googleanalytics.commands:LoadAnalytics initdb = ckanext.googleanalytics.commands:InitDB diff --git a/test.ini b/test.ini new file mode 100644 index 0000000..f112f25 --- /dev/null +++ b/test.ini @@ -0,0 +1,46 @@ +[DEFAULT] +debug = false +smtp_server = localhost +error_email_from = ckan@localhost + +[app:main] +use = config:../ckan/test-core.ini + +# Insert any custom config settings to be used when running your extension's +# tests here. These will override the one defined in CKAN core's test-core.ini +ckan.plugins = googleanalytics + +googleanalytics.id = UA-000000000-1 + +# Logging configuration +[loggers] +keys = root, ckan, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_ckan] +qualname = ckan +handlers = +level = INFO + +[logger_sqlalchemy] +handlers = +qualname = sqlalchemy.engine +level = WARN + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s