From 8906260eb15715a8b45a23c829fa962f21225c09 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 29 Nov 2011 17:39:49 +0000 Subject: [PATCH] update plugin filter method and tests for latest ckan (with dataset view changes) --- ckanext/googleanalytics/commands.py | 18 +++--- ckanext/googleanalytics/plugin.py | 94 +++++++++++++++-------------- tests/mockgoogleanalytics.py | 10 +-- tests/packagefixture.xml | 10 +-- tests/test_general.py | 20 +++--- 5 files changed, 77 insertions(+), 75 deletions(-) diff --git a/ckanext/googleanalytics/commands.py b/ckanext/googleanalytics/commands.py index e200c3c..2ee0551 100644 --- a/ckanext/googleanalytics/commands.py +++ b/ckanext/googleanalytics/commands.py @@ -8,7 +8,7 @@ import ckan.model as model import dbutil log = logging.getLogger('ckanext.googleanalytics') -PACKAGE_URL = '/package/' # XXX get from routes... +PACKAGE_URL = '/dataset/' # XXX get from routes... DEFAULT_RESOURCE_URL_TAG = '/downloads/' @@ -39,10 +39,13 @@ class LoadAnalytics(CkanCommand): max_args = 0 min_args = 0 TEST_HOST = None - CONFIG = pylonsconfig + CONFIG = None def command(self): - self._load_config() + if not self.CONFIG: + self._load_config() + self.CONFIG = pylonsconfig + self.resource_url_tag = self.CONFIG.get( 'googleanalytics.resource_prefix', DEFAULT_RESOURCE_URL_TAG) @@ -54,9 +57,7 @@ class LoadAnalytics(CkanCommand): self.parse_and_save() def parse_and_save(self): - """Grab raw data from Google Analytics and save to the - database - """ + """Grab raw data from Google Analytics and save to the database""" packages_data = self.get_ga_data() self.save_ga_data(packages_data) log.info("Saved %s records from google" % len(packages_data)) @@ -104,9 +105,7 @@ class LoadAnalytics(CkanCommand): http_client=self.TEST_HOST) else: my_client = client.AnalyticsClient(source=SOURCE_APP_NAME) - my_client.ClientLogin(username, - password, - SOURCE_APP_NAME) + my_client.ClientLogin(username, password, SOURCE_APP_NAME) account_query = client.AccountFeedQuery({'max-results': '300'}) feed = my_client.GetAccountFeed(account_query) table_id = None @@ -166,3 +165,4 @@ class LoadAnalytics(CkanCommand): 'ga:uniquePageviews').value or 0 packages.setdefault(package, {})[date_name] = count return packages + diff --git a/ckanext/googleanalytics/plugin.py b/ckanext/googleanalytics/plugin.py index 8ed6d2c..91112e8 100644 --- a/ckanext/googleanalytics/plugin.py +++ b/ckanext/googleanalytics/plugin.py @@ -6,7 +6,7 @@ from genshi.filters import Transformer from genshi import HTML from genshi.core import START, TEXT, END from genshi.filters.transform import INSIDE, EXIT -from pylons import config +from pylons import config, request from ckan.plugins import implements, SingletonPlugin from ckan.plugins import IGenshiStreamFilter, IConfigurable, IRoutes from ckan.plugins import IConfigurer @@ -29,7 +29,6 @@ class GoogleAnalyticsPlugin(SingletonPlugin): def configure(self, config): self.config = config - log.info("Loading Google Analytics plugin...") if (not 'googleanalytics.id' in config): msg = "Missing googleanalytics.id in config" raise GoogleAnalyticsException(msg) @@ -45,50 +44,52 @@ class GoogleAnalyticsPlugin(SingletonPlugin): show_downloads = asbool(config.get('googleanalytics.show_downloads', True)) - # add download tracking link - def js_attr(name, event): - attrs = event[1][1] - href = attrs.get('href').encode('utf-8') - link = '%s%s' % (resource_url, - urllib.quote(href)) - js = "javascript: _gaq.push(['_trackPageview', '%s']);" % link - return js + routes = request.environ.get('pylons.routes_dict') + if (routes.get('controller') == 'package' and + routes.get('action') == 'read'): - # add some stats - def download_adder(stream): - download_html = ''' - (downloaded %s times)''' - count = None - for mark, (kind, data, pos) in stream: - if mark and kind == START: - href = data[1].get('href') - if href: - count = dbutil.get_resource_visits_for_url(href) - if count and mark is EXIT: - # emit count - yield INSIDE, (TEXT, - HTML(download_html % count), - pos) - yield mark, (kind, data, pos) + # add download tracking link + def js_attr(name, event): + attrs = event[1][1] + href = attrs.get('href').encode('utf-8') + link = '%s%s' % (resource_url, urllib.quote(href)) + js = "javascript: _gaq.push(['_trackPageview', '%s']);" % link + return js - # and some styling - download_style = '''''' + # add some stats + def download_adder(stream): + download_html = ''' + (downloaded %s times)''' + count = None + for mark, (kind, data, pos) in stream: + if mark and kind == START: + href = data[1].get('href') + if href: + count = dbutil.get_resource_visits_for_url(href) + if count and mark is EXIT: + # emit count + yield INSIDE, (TEXT, HTML(download_html % count), pos) + yield mark, (kind, data, pos) + + # and some styling + download_style = ''' + + ''' + + # perform the stream transform + stream = stream | Transformer('//div[@class="resource-url"]//a')\ + .attr('onclick', js_attr) + + if show_downloads: + stream = stream | Transformer('//div[@class="resource-url"]//a')\ + .apply(download_adder) + stream = stream | Transformer('//link[@rel="stylesheet"]')\ + .append(HTML(download_style)) - # perform the stream transform - stream = stream | Transformer( - '//div[@id="package"]//td/a')\ - .attr('onclick', js_attr) - if show_downloads: - stream = stream | Transformer( - '//div[@id="package"]//td/a')\ - .apply(download_adder) - stream = stream | Transformer( - '//link[@rel="stylesheet"]')\ - .append(HTML(download_style)) return stream def after_map(self, map): @@ -101,6 +102,7 @@ class GoogleAnalyticsPlugin(SingletonPlugin): here = os.path.dirname(__file__) rootdir = os.path.dirname(os.path.dirname(here)) template_dir = os.path.join(rootdir, 'ckanext', - 'googleanalytics', 'templates') - config['extra_template_paths'] = ','.join([template_dir, - config.get('extra_template_paths', '')]) + 'googleanalytics', 'templates') + config['extra_template_paths'] = ','.join( + [template_dir, config.get('extra_template_paths', '')] + ) diff --git a/tests/mockgoogleanalytics.py b/tests/mockgoogleanalytics.py index 2373d89..8cfc831 100644 --- a/tests/mockgoogleanalytics.py +++ b/tests/mockgoogleanalytics.py @@ -1,8 +1,6 @@ import os import BaseHTTPServer import threading -import gdata.data -import atom.core here_dir = os.path.dirname(os.path.abspath(__file__)) @@ -15,12 +13,10 @@ class MockHandler(BaseHTTPServer.BaseHTTPRequestHandler): fixture = os.path.join(here_dir, "accountsfixture.xml") content = open(fixture, "r").read() elif "analytics/feeds/data" in self.path: - if "package" in self.path: - fixture = os.path.join(here_dir, - "packagefixture.xml") + if "dataset" in self.path: + fixture = os.path.join(here_dir, "packagefixture.xml") elif "download" in self.path: - fixture = os.path.join(here_dir, - "downloadfixture.xml") + fixture = os.path.join(here_dir, "downloadfixture.xml") self.send_response(200) self.end_headers() content = open(fixture, "r").read() diff --git a/tests/packagefixture.xml b/tests/packagefixture.xml index c99e6a8..26a5417 100644 --- a/tests/packagefixture.xml +++ b/tests/packagefixture.xml @@ -1,9 +1,9 @@ -http://www.google.com/analytics/feeds/data?ids=ga:42156377&dimensions=ga:pagePath&metrics=ga:newVisits,ga:uniquePageviews,ga:visitors,ga:visits&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-05datagm.staging.ckan.net/ga:421563772011-04-05T03:08:57.106-07:00false +http://www.google.com/analytics/feeds/data?ids=ga:42156377&dimensions=ga:pagePath&metrics=ga:newVisits,ga:uniquePageviews,ga:visitors,ga:visits&filters=ga:pagePath%3D~%5E/dataset/&start-date=2011-03-22&end-date=2011-04-05datagm.staging.ckan.net/ga:421563772011-04-05T03:08:57.106-07:00false -http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/package/annakarenina&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/package/annakarenina +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/dataset/annakarenina&filters=ga:pagePath%3D~%5E/dataset/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/dataset/annakarenina -http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/package/annakarenina/invalid&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/package/annakarenina/invalid +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/dataset/annakarenina/invalid&filters=ga:pagePath%3D~%5E/dataset/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/dataset/annakarenina/invalid -http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/package/annakarenina-invalid&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/package/annakarenina-invalid +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/dataset/annakarenina-invalid&filters=ga:pagePath%3D~%5E/dataset/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/dataset/annakarenina-invalid -Google Analytics1Google Analytics Data for Profile 42156377Google Analytics2011-04-051522011-03-2210000 +Google Analytics1Google Analytics Data for Profile 42156377Google Analytics2011-04-051522011-03-2210000 diff --git a/tests/test_general.py b/tests/test_general.py index 6652f15..e803bef 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -1,3 +1,4 @@ +import os import httplib from unittest import TestCase @@ -18,13 +19,11 @@ class MockClient(httplib.HTTPConnection): filters = http_request.uri.query.get('filters') path = http_request.uri.path if filters: - if "package" in filters: - path += "/package" + if "dataset" in filters: + path += "/dataset" else: path += "/download" - httplib.HTTPConnection.request(self, - http_request.method, - path) + httplib.HTTPConnection.request(self, http_request.method, path) resp = self.getresponse() return resp @@ -42,6 +41,8 @@ class TestConfig(TestCase): class TestLoadCommand(TestCase): @classmethod def setup_class(cls): + InitDB("initdb").run([]) # set up database tables + config = appconfig('config:test.ini', relative_to=conf_dir) config.local_conf['ckan.plugins'] = 'googleanalytics' config.local_conf['googleanalytics.username'] = 'borf' @@ -69,7 +70,6 @@ class TestLoadCommand(TestCase): assert code in response.body def test_top_packages(self): - InitDB("initdb").run([]) # set up database tables command = LoadAnalytics("loadanalytics") command.TEST_HOST = MockClient('localhost', 6969) command.CONFIG = self.config @@ -80,12 +80,16 @@ class TestLoadCommand(TestCase): self.assertEquals(resources[0][1], 4) def test_download_count_inserted(self): - InitDB("initdb").run([]) # set up database tables command = LoadAnalytics("loadanalytics") command.TEST_HOST = MockClient('localhost', 6969) command.CONFIG = self.config command.run([]) + # command.run(['--config=%s' % os.path.join(conf_dir, 'test.ini')]) + # config = appconfig('config:test.ini', relative_to=conf_dir) + # for k in config: + # print k response = self.app.get(url_for(controller='package', action='read', id='annakarenina')) - assert "(downloaded 4 times)" in response.body, response.body + assert "(downloaded 4 times)" in response.body +