Configure tests
This commit is contained in:
parent
7f5c2c03d9
commit
dc75b4c472
|
@ -0,0 +1,68 @@
|
||||||
|
name: Tests
|
||||||
|
on: [push, pull_request]
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: '3.6'
|
||||||
|
- name: Install requirements
|
||||||
|
run: pip install flake8 pycodestyle
|
||||||
|
- name: Check syntax
|
||||||
|
run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
|
||||||
|
- name: Run flake8
|
||||||
|
run: flake8 . --count --max-line-length=127 --statistics --exclude ckan
|
||||||
|
|
||||||
|
test:
|
||||||
|
needs: lint
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
|
||||||
|
fail-fast: false
|
||||||
|
|
||||||
|
name: CKAN ${{ matrix.ckan-version }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
|
||||||
|
services:
|
||||||
|
solr:
|
||||||
|
image: ckan/ckan-solr:${{ matrix.ckan-version }}
|
||||||
|
postgres:
|
||||||
|
image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: postgres
|
||||||
|
POSTGRES_PASSWORD: postgres
|
||||||
|
POSTGRES_DB: postgres
|
||||||
|
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||||||
|
redis:
|
||||||
|
image: redis:3
|
||||||
|
env:
|
||||||
|
CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
|
||||||
|
CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
|
||||||
|
CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
|
||||||
|
CKAN_SOLR_URL: http://solr:8983/solr/ckan
|
||||||
|
CKAN_REDIS_URL: redis://redis:6379/1
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: Install requirements
|
||||||
|
run: |
|
||||||
|
pip install -r requirements.txt
|
||||||
|
pip install -r dev-requirements.txt
|
||||||
|
pip install -e .
|
||||||
|
# Replace default path to CKAN core config file with the one on the container
|
||||||
|
sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
|
||||||
|
- name: Setup extension (CKAN >= 2.9)
|
||||||
|
if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
|
||||||
|
run: |
|
||||||
|
ckan -c test.ini db init
|
||||||
|
ckan -c test.ini db upgrade -p googleanalytics
|
||||||
|
- name: Setup extension (CKAN < 2.9)
|
||||||
|
if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
|
||||||
|
run: |
|
||||||
|
paster --plugin=ckan db init -c test.ini
|
||||||
|
paster --plugin=ckanext-googleanalytics initdb -c test.ini
|
||||||
|
- name: Run tests
|
||||||
|
run: pytest --ckan-ini=test.ini --cov=ckanext.googleanalytics --disable-warnings ckanext/googleanalytics/tests
|
|
@ -10,8 +10,6 @@ import ckan.model as model
|
||||||
|
|
||||||
from . import dbutil, utils
|
from . import dbutil, utils
|
||||||
|
|
||||||
import ckan.plugins.toolkit as tk
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
PACKAGE_URL = "/dataset/" # XXX get from routes...
|
PACKAGE_URL = "/dataset/" # XXX get from routes...
|
||||||
|
|
||||||
|
@ -105,7 +103,8 @@ def internal_save(packages_data, summary_date):
|
||||||
SET package_id = COALESCE(
|
SET package_id = COALESCE(
|
||||||
(SELECT id FROM package p WHERE t.url = %s || p.name)
|
(SELECT id FROM package p WHERE t.url = %s || p.name)
|
||||||
,'~~not~found~~')
|
,'~~not~found~~')
|
||||||
WHERE t.package_id = '~~not~found~~' AND tracking_type = 'page';"""
|
WHERE t.package_id = '~~not~found~~'
|
||||||
|
AND tracking_type = 'page';"""
|
||||||
engine.execute(sql, "%sedit/" % PACKAGE_URL)
|
engine.execute(sql, "%sedit/" % PACKAGE_URL)
|
||||||
|
|
||||||
# update summary totals for resources
|
# update summary totals for resources
|
||||||
|
@ -120,7 +119,8 @@ def internal_save(packages_data, summary_date):
|
||||||
SELECT sum(count)
|
SELECT sum(count)
|
||||||
FROM tracking_summary t2
|
FROM tracking_summary t2
|
||||||
WHERE t1.url = t2.url
|
WHERE t1.url = t2.url
|
||||||
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
|
AND t2.tracking_date <= t1.tracking_date
|
||||||
|
AND t2.tracking_date >= t1.tracking_date - %s
|
||||||
) + t1.count
|
) + t1.count
|
||||||
WHERE t1.running_total = 0 AND tracking_type = 'resource';"""
|
WHERE t1.running_total = 0 AND tracking_type = 'resource';"""
|
||||||
engine.execute(sql, utils.config_recent_view_days())
|
engine.execute(sql, utils.config_recent_view_days())
|
||||||
|
@ -137,7 +137,8 @@ def internal_save(packages_data, summary_date):
|
||||||
SELECT sum(count)
|
SELECT sum(count)
|
||||||
FROM tracking_summary t2
|
FROM tracking_summary t2
|
||||||
WHERE t1.package_id = t2.package_id
|
WHERE t1.package_id = t2.package_id
|
||||||
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
|
AND t2.tracking_date <= t1.tracking_date
|
||||||
|
AND t2.tracking_date >= t1.tracking_date - %s
|
||||||
) + t1.count
|
) + t1.count
|
||||||
WHERE t1.running_total = 0 AND tracking_type = 'page'
|
WHERE t1.running_total = 0 AND tracking_type = 'page'
|
||||||
AND t1.package_id IS NOT NULL
|
AND t1.package_id IS NOT NULL
|
||||||
|
@ -337,8 +338,8 @@ def get_ga_data(service, profile_id, query_filter):
|
||||||
package = "/" + "/".join(package.split("/")[2:])
|
package = "/" + "/".join(package.split("/")[2:])
|
||||||
|
|
||||||
count = result[1]
|
count = result[1]
|
||||||
# Make sure we add the different representations of the same
|
# Make sure we add the different representations of the
|
||||||
# dataset /mysite.com & /www.mysite.com ...
|
# same dataset /mysite.com & /www.mysite.com ...
|
||||||
val = 0
|
val = 0
|
||||||
if package in packages and date_name in packages[package]:
|
if package in packages and date_name in packages[package]:
|
||||||
val += packages[package][date_name]
|
val += packages[package][date_name]
|
||||||
|
|
|
@ -103,7 +103,8 @@ class LoadAnalytics(CkanCommand):
|
||||||
SET package_id = COALESCE(
|
SET package_id = COALESCE(
|
||||||
(SELECT id FROM package p WHERE t.url = %s || p.name)
|
(SELECT id FROM package p WHERE t.url = %s || p.name)
|
||||||
,'~~not~found~~')
|
,'~~not~found~~')
|
||||||
WHERE t.package_id = '~~not~found~~' AND tracking_type = 'page';"""
|
WHERE t.package_id = '~~not~found~~'
|
||||||
|
AND tracking_type = 'page';"""
|
||||||
engine.execute(sql, "%sedit/" % PACKAGE_URL)
|
engine.execute(sql, "%sedit/" % PACKAGE_URL)
|
||||||
|
|
||||||
# update summary totals for resources
|
# update summary totals for resources
|
||||||
|
@ -118,7 +119,8 @@ class LoadAnalytics(CkanCommand):
|
||||||
SELECT sum(count)
|
SELECT sum(count)
|
||||||
FROM tracking_summary t2
|
FROM tracking_summary t2
|
||||||
WHERE t1.url = t2.url
|
WHERE t1.url = t2.url
|
||||||
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
|
AND t2.tracking_date <= t1.tracking_date
|
||||||
|
AND t2.tracking_date >= t1.tracking_date - %s
|
||||||
) + t1.count
|
) + t1.count
|
||||||
WHERE t1.running_total = 0 AND tracking_type = 'resource';"""
|
WHERE t1.running_total = 0 AND tracking_type = 'resource';"""
|
||||||
engine.execute(sql, self.recent_view_days)
|
engine.execute(sql, self.recent_view_days)
|
||||||
|
@ -135,7 +137,8 @@ class LoadAnalytics(CkanCommand):
|
||||||
SELECT sum(count)
|
SELECT sum(count)
|
||||||
FROM tracking_summary t2
|
FROM tracking_summary t2
|
||||||
WHERE t1.package_id = t2.package_id
|
WHERE t1.package_id = t2.package_id
|
||||||
AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - %s
|
AND t2.tracking_date <= t1.tracking_date
|
||||||
|
AND t2.tracking_date >= t1.tracking_date - %s
|
||||||
) + t1.count
|
) + t1.count
|
||||||
WHERE t1.running_total = 0 AND tracking_type = 'page'
|
WHERE t1.running_total = 0 AND tracking_type = 'page'
|
||||||
AND t1.package_id IS NOT NULL
|
AND t1.package_id IS NOT NULL
|
||||||
|
@ -360,8 +363,8 @@ class LoadAnalytics(CkanCommand):
|
||||||
package = "/" + "/".join(package.split("/")[2:])
|
package = "/" + "/".join(package.split("/")[2:])
|
||||||
|
|
||||||
count = result[1]
|
count = result[1]
|
||||||
# Make sure we add the different representations of the same
|
# Make sure we add the different representations of the
|
||||||
# dataset /mysite.com & /www.mysite.com ...
|
# same dataset /mysite.com & /www.mysite.com ...
|
||||||
val = 0
|
val = 0
|
||||||
if (
|
if (
|
||||||
package in packages
|
package in packages
|
||||||
|
|
|
@ -4,22 +4,19 @@ from sqlalchemy import func
|
||||||
|
|
||||||
import ckan.model as model
|
import ckan.model as model
|
||||||
|
|
||||||
# from ckan.model.authz import PSEUDO_USER__VISITOR
|
|
||||||
from ckan.lib.base import *
|
|
||||||
|
|
||||||
cached_tables = {}
|
cached_tables = {}
|
||||||
|
|
||||||
|
|
||||||
def init_tables():
|
def init_tables():
|
||||||
metadata = MetaData()
|
metadata = MetaData()
|
||||||
package_stats = Table(
|
Table(
|
||||||
"package_stats",
|
"package_stats",
|
||||||
metadata,
|
metadata,
|
||||||
Column("package_id", String(60), primary_key=True),
|
Column("package_id", String(60), primary_key=True),
|
||||||
Column("visits_recently", Integer),
|
Column("visits_recently", Integer),
|
||||||
Column("visits_ever", Integer),
|
Column("visits_ever", Integer),
|
||||||
)
|
)
|
||||||
resource_stats = Table(
|
Table(
|
||||||
"resource_stats",
|
"resource_stats",
|
||||||
metadata,
|
metadata,
|
||||||
Column("resource_id", String(60), primary_key=True),
|
Column("resource_id", String(60), primary_key=True),
|
||||||
|
|
|
@ -50,12 +50,9 @@ def get_profile_id(service):
|
||||||
accountId = acc.get("id")
|
accountId = acc.get("id")
|
||||||
|
|
||||||
# TODO: check, whether next line is doing something useful.
|
# TODO: check, whether next line is doing something useful.
|
||||||
webproperties = (
|
service.management().webproperties().list(
|
||||||
service.management()
|
accountId=accountId
|
||||||
.webproperties()
|
).execute()
|
||||||
.list(accountId=accountId)
|
|
||||||
.execute()
|
|
||||||
)
|
|
||||||
|
|
||||||
profiles = (
|
profiles = (
|
||||||
service.management()
|
service.management()
|
||||||
|
|
|
@ -5,9 +5,12 @@ header_code = """
|
||||||
_gaq.push(['_setDomainName', '%s']);
|
_gaq.push(['_setDomainName', '%s']);
|
||||||
_gaq.push(['_trackPageview']);
|
_gaq.push(['_trackPageview']);
|
||||||
(function() {
|
(function() {
|
||||||
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
|
var ga = document.createElement('script');
|
||||||
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
|
ga.type = 'text/javascript'; ga.async = true;
|
||||||
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
|
ga.src = ('https:' == document.location.protocol ?
|
||||||
|
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
|
||||||
|
var s = document.getElementsByTagName('script')[0];
|
||||||
|
s.parentNode.insertBefore(ga, s);
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -43,7 +43,7 @@ class AnalyticsPostThread(threading.Thread):
|
||||||
data = urlencode(data_dict)
|
data = urlencode(data_dict)
|
||||||
log.debug("Sending API event to Google Analytics: " + data)
|
log.debug("Sending API event to Google Analytics: " + data)
|
||||||
# send analytics
|
# send analytics
|
||||||
res = requests.post(
|
requests.post(
|
||||||
"http://www.google-analytics.com/collect",
|
"http://www.google-analytics.com/collect",
|
||||||
data,
|
data,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("clean_db")
|
||||||
|
def test_script(app):
|
||||||
|
resp = app.get("/")
|
||||||
|
assert "GoogleAnalyticsObject" in resp
|
|
@ -0,0 +1 @@
|
||||||
|
pytest-ckan
|
|
@ -38,6 +38,7 @@ requirements =
|
||||||
[options.entry_points]
|
[options.entry_points]
|
||||||
ckan.plugins =
|
ckan.plugins =
|
||||||
googleanalytics = ckanext.googleanalytics.plugin:GoogleAnalyticsPlugin
|
googleanalytics = ckanext.googleanalytics.plugin:GoogleAnalyticsPlugin
|
||||||
|
|
||||||
paste.paster_command =
|
paste.paster_command =
|
||||||
loadanalytics = ckanext.googleanalytics.commands:LoadAnalytics
|
loadanalytics = ckanext.googleanalytics.commands:LoadAnalytics
|
||||||
initdb = ckanext.googleanalytics.commands:InitDB
|
initdb = ckanext.googleanalytics.commands:InitDB
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
[DEFAULT]
|
||||||
|
debug = false
|
||||||
|
smtp_server = localhost
|
||||||
|
error_email_from = ckan@localhost
|
||||||
|
|
||||||
|
[app:main]
|
||||||
|
use = config:../ckan/test-core.ini
|
||||||
|
|
||||||
|
# Insert any custom config settings to be used when running your extension's
|
||||||
|
# tests here. These will override the one defined in CKAN core's test-core.ini
|
||||||
|
ckan.plugins = googleanalytics
|
||||||
|
|
||||||
|
googleanalytics.id = UA-000000000-1
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root, ckan, sqlalchemy
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
|
||||||
|
[logger_ckan]
|
||||||
|
qualname = ckan
|
||||||
|
handlers =
|
||||||
|
level = INFO
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
level = WARN
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
|
Loading…
Reference in New Issue