CKAN-Extensions
/
ckanext-googleanalytics
mirror of https://github.com/ckan/ckanext-googleanalytics.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Configure tests

This commit is contained in:
Sergey Motornyuk 2022-05-06 19:28:52 +03:00
parent 7f5c2c03d9
commit dc75b4c472
11 changed files with 155 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
.github/workflows/test.yml vendored Normal file
View File

@ -0,0 +1,68 @@
name: Tests
on: [push, pull_request]
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.6'
- name: Install requirements
run: pip install flake8 pycodestyle
- name: Check syntax
run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
- name: Run flake8
run: flake8 . --count --max-line-length=127 --statistics --exclude ckan
test:
needs: lint
strategy:
matrix:
ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
fail-fast: false
name: CKAN ${{ matrix.ckan-version }}
runs-on: ubuntu-latest
container:
image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
services:
solr:
image: ckan/ckan-solr:${{ matrix.ckan-version }}
postgres:
image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
redis:
image: redis:3
env:
CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
CKAN_SOLR_URL: http://solr:8983/solr/ckan
CKAN_REDIS_URL: redis://redis:6379/1
steps:
- uses: actions/checkout@v2
- name: Install requirements
run: |
pip install -r requirements.txt
pip install -r dev-requirements.txt
pip install -e .
# Replace default path to CKAN core config file with the one on the container
sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
- name: Setup extension (CKAN >= 2.9)
if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
run: |
ckan -c test.ini db init
ckan -c test.ini db upgrade -p googleanalytics
- name: Setup extension (CKAN < 2.9)
if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
run: |
paster --plugin=ckan db init -c test.ini
paster --plugin=ckanext-googleanalytics initdb -c test.ini
- name: Run tests
run: pytest --ckan-ini=test.ini --cov=ckanext.googleanalytics --disable-warnings ckanext/googleanalytics/tests

19
ckanext/googleanalytics/cli.py
View File

@ -10,8 +10,6 @@ import ckan.model as model
from . import dbutil, utils
import ckan.plugins.toolkit as tk
log = logging.getLogger(__name__)
PACKAGE_URL = "/dataset/" # XXX get from routes...
@ -105,7 +103,8 @@ def internal_save(packages_data, summary_date):
SET package_id = COALESCE(
(SELECT id FROM package p WHERE t.url = %s || p.name)
,'~~not~found~~')
WHERE t.package_id = '~~not~found~~' AND tracking_type = 'page';"""
WHERE t.package_id = '~~not~found~~'
AND tracking_type = 'page';"""
engine.execute(sql, "%sedit/" % PACKAGE_URL)
# update summary totals for resources
@ -120,7 +119,8 @@ def internal_save(packages_data, summary_date):
SELECT sum(count)
FROM tracking_summary t2
WHERE t1.url = t2.url
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
AND t2.tracking_date <= t1.tracking_date
AND t2.tracking_date >= t1.tracking_date - %s
) + t1.count
WHERE t1.running_total = 0 AND tracking_type = 'resource';"""
engine.execute(sql, utils.config_recent_view_days())
@ -137,7 +137,8 @@ def internal_save(packages_data, summary_date):
SELECT sum(count)
FROM tracking_summary t2
WHERE t1.package_id = t2.package_id
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
AND t2.tracking_date <= t1.tracking_date
AND t2.tracking_date >= t1.tracking_date - %s
) + t1.count
WHERE t1.running_total = 0 AND tracking_type = 'page'
AND t1.package_id IS NOT NULL
@ -243,7 +244,7 @@ def save_ga_data(packages_data):
ever = visits.get("ever", 0)
matches = RESOURCE_URL_REGEX.match(identifier)
if matches:
resource_url = identifier[len(utils.config_prefix()) :]
resource_url = identifier[len(utils.config_prefix()):]
resource = (
model.Session.query(model.Resource)
.autoflush(True)
@ -256,7 +257,7 @@ def save_ga_data(packages_data):
dbutil.update_resource_visits(resource.id, recently, ever)
log.info("Updated %s with %s visits" % (resource.id, visits))
else:
package_name = identifier[len(PACKAGE_URL) :]
package_name = identifier[len(PACKAGE_URL):]
if "/" in package_name:
log.warning("%s not a valid package name" % package_name)
continue
@ -337,8 +338,8 @@ def get_ga_data(service, profile_id, query_filter):
package = "/" + "/".join(package.split("/")[2:])
count = result[1]
# Make sure we add the different representations of the same
# dataset /mysite.com & /www.mysite.com ...
# Make sure we add the different representations of the
# same dataset /mysite.com & /www.mysite.com ...
val = 0
if package in packages and date_name in packages[package]:
val += packages[package][date_name]

17
ckanext/googleanalytics/commands.py
View File

@ -103,7 +103,8 @@ class LoadAnalytics(CkanCommand):
SET package_id = COALESCE(
(SELECT id FROM package p WHERE t.url = %s || p.name)
,'~~not~found~~')
WHERE t.package_id = '~~not~found~~' AND tracking_type = 'page';"""
WHERE t.package_id = '~~not~found~~'
AND tracking_type = 'page';"""
engine.execute(sql, "%sedit/" % PACKAGE_URL)
# update summary totals for resources
@ -118,7 +119,8 @@ class LoadAnalytics(CkanCommand):
SELECT sum(count)
FROM tracking_summary t2
WHERE t1.url = t2.url
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
AND t2.tracking_date <= t1.tracking_date
AND t2.tracking_date >= t1.tracking_date - %s
) + t1.count
WHERE t1.running_total = 0 AND tracking_type = 'resource';"""
engine.execute(sql, self.recent_view_days)
@ -135,7 +137,8 @@ class LoadAnalytics(CkanCommand):
SELECT sum(count)
FROM tracking_summary t2
WHERE t1.package_id = t2.package_id
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
AND t2.tracking_date <= t1.tracking_date
AND t2.tracking_date >= t1.tracking_date - %s
) + t1.count
WHERE t1.running_total = 0 AND tracking_type = 'page'
AND t1.package_id IS NOT NULL
@ -265,7 +268,7 @@ class LoadAnalytics(CkanCommand):
ever = visits.get("ever", 0)
matches = RESOURCE_URL_REGEX.match(identifier)
if matches:
resource_url = identifier[len(self.resource_url_tag) :]
resource_url = identifier[len(self.resource_url_tag):]
resource = (
model.Session.query(model.Resource)
.autoflush(True)
@ -278,7 +281,7 @@ class LoadAnalytics(CkanCommand):
dbutil.update_resource_visits(resource.id, recently, ever)
log.info("Updated %s with %s visits" % (resource.id, visits))
else:
package_name = identifier[len(PACKAGE_URL) :]
package_name = identifier[len(PACKAGE_URL):]
if "/" in package_name:
log.warning("%s not a valid package name" % package_name)
continue
@ -360,8 +363,8 @@ class LoadAnalytics(CkanCommand):
package = "/" + "/".join(package.split("/")[2:])
count = result[1]
# Make sure we add the different representations of the same
# dataset /mysite.com & /www.mysite.com ...
# Make sure we add the different representations of the
# same dataset /mysite.com & /www.mysite.com ...
val = 0
if (
package in packages

7
ckanext/googleanalytics/dbutil.py
View File

@ -4,22 +4,19 @@ from sqlalchemy import func
import ckan.model as model
# from ckan.model.authz import PSEUDO_USER__VISITOR
from ckan.lib.base import *
cached_tables = {}
def init_tables():
metadata = MetaData()
package_stats = Table(
Table(
"package_stats",
metadata,
Column("package_id", String(60), primary_key=True),
Column("visits_recently", Integer),
Column("visits_ever", Integer),
)
resource_stats = Table(
Table(
"resource_stats",
metadata,
Column("resource_id", String(60), primary_key=True),

9
ckanext/googleanalytics/ga_auth.py
View File

@ -50,12 +50,9 @@ def get_profile_id(service):
accountId = acc.get("id")
# TODO: check, whether next line is doing something useful.
webproperties = (
service.management()
.webproperties()
.list(accountId=accountId)
.execute()
)
service.management().webproperties().list(
accountId=accountId
).execute()
profiles = (
service.management()

9
ckanext/googleanalytics/gasnippet.py
View File

@ -5,9 +5,12 @@ header_code = """
_gaq.push(['_setDomainName', '%s']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
var ga = document.createElement('script');
ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ?
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);
})();
</script>
"""

2
ckanext/googleanalytics/plugin/__init__.py
View File

@ -43,7 +43,7 @@ class AnalyticsPostThread(threading.Thread):
data = urlencode(data_dict)
log.debug("Sending API event to Google Analytics: " + data)
# send analytics
res = requests.post(
requests.post(
"http://www.google-analytics.com/collect",
data,
timeout=10,

7
ckanext/googleanalytics/tests/test_plugin.py Normal file
View File

@ -0,0 +1,7 @@
import pytest
@pytest.mark.usefixtures("clean_db")
def test_script(app):
resp = app.get("/")
assert "GoogleAnalyticsObject" in resp

1
dev-requirements.txt Normal file
View File

@ -0,0 +1 @@
pytest-ckan

1
setup.cfg
View File

@ -38,6 +38,7 @@ requirements =
[options.entry_points]
ckan.plugins =
googleanalytics = ckanext.googleanalytics.plugin:GoogleAnalyticsPlugin
paste.paster_command =
loadanalytics = ckanext.googleanalytics.commands:LoadAnalytics
initdb = ckanext.googleanalytics.commands:InitDB

46
test.ini Normal file
View File

@ -0,0 +1,46 @@
[DEFAULT]
debug = false
smtp_server = localhost
error_email_from = ckan@localhost
[app:main]
use = config:../ckan/test-core.ini
# Insert any custom config settings to be used when running your extension's
# tests here. These will override the one defined in CKAN core's test-core.ini
ckan.plugins = googleanalytics
googleanalytics.id = UA-000000000-1
# Logging configuration
[loggers]
keys = root, ckan, sqlalchemy
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_ckan]
qualname = ckan
handlers =
level = INFO
[logger_sqlalchemy]
handlers =
qualname = sqlalchemy.engine
level = WARN
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s