update plugin filter method and tests for latest

ckan (with dataset view changes)
This commit is contained in:
John Glover 2011-11-29 17:39:49 +00:00
parent 251f896bc6
commit 8906260eb1
5 changed files with 77 additions and 75 deletions

View File

@ -8,7 +8,7 @@ import ckan.model as model
import dbutil
log = logging.getLogger('ckanext.googleanalytics')
PACKAGE_URL = '/package/' # XXX get from routes...
PACKAGE_URL = '/dataset/' # XXX get from routes...
DEFAULT_RESOURCE_URL_TAG = '/downloads/'
@ -39,10 +39,13 @@ class LoadAnalytics(CkanCommand):
max_args = 0
min_args = 0
TEST_HOST = None
CONFIG = pylonsconfig
CONFIG = None
def command(self):
self._load_config()
if not self.CONFIG:
self._load_config()
self.CONFIG = pylonsconfig
self.resource_url_tag = self.CONFIG.get(
'googleanalytics.resource_prefix',
DEFAULT_RESOURCE_URL_TAG)
@ -54,9 +57,7 @@ class LoadAnalytics(CkanCommand):
self.parse_and_save()
def parse_and_save(self):
"""Grab raw data from Google Analytics and save to the
database
"""
"""Grab raw data from Google Analytics and save to the database"""
packages_data = self.get_ga_data()
self.save_ga_data(packages_data)
log.info("Saved %s records from google" % len(packages_data))
@ -104,9 +105,7 @@ class LoadAnalytics(CkanCommand):
http_client=self.TEST_HOST)
else:
my_client = client.AnalyticsClient(source=SOURCE_APP_NAME)
my_client.ClientLogin(username,
password,
SOURCE_APP_NAME)
my_client.ClientLogin(username, password, SOURCE_APP_NAME)
account_query = client.AccountFeedQuery({'max-results': '300'})
feed = my_client.GetAccountFeed(account_query)
table_id = None
@ -166,3 +165,4 @@ class LoadAnalytics(CkanCommand):
'ga:uniquePageviews').value or 0
packages.setdefault(package, {})[date_name] = count
return packages

View File

@ -6,7 +6,7 @@ from genshi.filters import Transformer
from genshi import HTML
from genshi.core import START, TEXT, END
from genshi.filters.transform import INSIDE, EXIT
from pylons import config
from pylons import config, request
from ckan.plugins import implements, SingletonPlugin
from ckan.plugins import IGenshiStreamFilter, IConfigurable, IRoutes
from ckan.plugins import IConfigurer
@ -29,7 +29,6 @@ class GoogleAnalyticsPlugin(SingletonPlugin):
def configure(self, config):
self.config = config
log.info("Loading Google Analytics plugin...")
if (not 'googleanalytics.id' in config):
msg = "Missing googleanalytics.id in config"
raise GoogleAnalyticsException(msg)
@ -45,50 +44,52 @@ class GoogleAnalyticsPlugin(SingletonPlugin):
show_downloads = asbool(config.get('googleanalytics.show_downloads',
True))
# add download tracking link
def js_attr(name, event):
attrs = event[1][1]
href = attrs.get('href').encode('utf-8')
link = '%s%s' % (resource_url,
urllib.quote(href))
js = "javascript: _gaq.push(['_trackPageview', '%s']);" % link
return js
routes = request.environ.get('pylons.routes_dict')
if (routes.get('controller') == 'package' and
routes.get('action') == 'read'):
# add some stats
def download_adder(stream):
download_html = ''' <span class="downloads-count">
(downloaded %s times)</span>'''
count = None
for mark, (kind, data, pos) in stream:
if mark and kind == START:
href = data[1].get('href')
if href:
count = dbutil.get_resource_visits_for_url(href)
if count and mark is EXIT:
# emit count
yield INSIDE, (TEXT,
HTML(download_html % count),
pos)
yield mark, (kind, data, pos)
# add download tracking link
def js_attr(name, event):
attrs = event[1][1]
href = attrs.get('href').encode('utf-8')
link = '%s%s' % (resource_url, urllib.quote(href))
js = "javascript: _gaq.push(['_trackPageview', '%s']);" % link
return js
# and some styling
download_style = '''<style type="text/css">
span.downloads-count {
font-size: 75%;
}
</style>'''
# add some stats
def download_adder(stream):
download_html = ''' <span class="downloads-count">
(downloaded %s times)</span>'''
count = None
for mark, (kind, data, pos) in stream:
if mark and kind == START:
href = data[1].get('href')
if href:
count = dbutil.get_resource_visits_for_url(href)
if count and mark is EXIT:
# emit count
yield INSIDE, (TEXT, HTML(download_html % count), pos)
yield mark, (kind, data, pos)
# and some styling
download_style = '''
<style type="text/css">
span.downloads-count {
font-size: 0.9em;
}
</style>
'''
# perform the stream transform
stream = stream | Transformer('//div[@class="resource-url"]//a')\
.attr('onclick', js_attr)
if show_downloads:
stream = stream | Transformer('//div[@class="resource-url"]//a')\
.apply(download_adder)
stream = stream | Transformer('//link[@rel="stylesheet"]')\
.append(HTML(download_style))
# perform the stream transform
stream = stream | Transformer(
'//div[@id="package"]//td/a')\
.attr('onclick', js_attr)
if show_downloads:
stream = stream | Transformer(
'//div[@id="package"]//td/a')\
.apply(download_adder)
stream = stream | Transformer(
'//link[@rel="stylesheet"]')\
.append(HTML(download_style))
return stream
def after_map(self, map):
@ -101,6 +102,7 @@ class GoogleAnalyticsPlugin(SingletonPlugin):
here = os.path.dirname(__file__)
rootdir = os.path.dirname(os.path.dirname(here))
template_dir = os.path.join(rootdir, 'ckanext',
'googleanalytics', 'templates')
config['extra_template_paths'] = ','.join([template_dir,
config.get('extra_template_paths', '')])
'googleanalytics', 'templates')
config['extra_template_paths'] = ','.join(
[template_dir, config.get('extra_template_paths', '')]
)

View File

@ -1,8 +1,6 @@
import os
import BaseHTTPServer
import threading
import gdata.data
import atom.core
here_dir = os.path.dirname(os.path.abspath(__file__))
@ -15,12 +13,10 @@ class MockHandler(BaseHTTPServer.BaseHTTPRequestHandler):
fixture = os.path.join(here_dir, "accountsfixture.xml")
content = open(fixture, "r").read()
elif "analytics/feeds/data" in self.path:
if "package" in self.path:
fixture = os.path.join(here_dir,
"packagefixture.xml")
if "dataset" in self.path:
fixture = os.path.join(here_dir, "packagefixture.xml")
elif "download" in self.path:
fixture = os.path.join(here_dir,
"downloadfixture.xml")
fixture = os.path.join(here_dir, "downloadfixture.xml")
self.send_response(200)
self.end_headers()
content = open(fixture, "r").read()

View File

@ -1,9 +1,9 @@
<ns0:feed ns1:etag="W/&quot;AkYHRn87fip7I2A9WhZSGUQ.&quot;" ns1:kind="analytics#data" xmlns:ns0="http://www.w3.org/2005/Atom" xmlns:ns1="http://schemas.google.com/g/2005"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;dimensions=ga:pagePath&amp;metrics=ga:newVisits,ga:uniquePageviews,ga:visitors,ga:visits&amp;filters=ga:pagePath%3D~%5E/package/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns2:dataSource xmlns:ns2="http://schemas.google.com/analytics/2009"><ns2:tableName>datagm.staging.ckan.net/</ns2:tableName><ns2:tableId>ga:42156377</ns2:tableId><ns2:property name="ga:profileId" value="42156377" /><ns2:property name="ga:webPropertyId" value="UA-21313878-1" /><ns2:property name="ga:accountName" value="http://datagm.staging.ckan.net/" /></ns2:dataSource><ns2:aggregates xmlns:ns2="http://schemas.google.com/analytics/2009"><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="122" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="526" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="83" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="606" /></ns2:aggregates><ns0:updated>2011-04-05T03:08:57.106-07:00</ns0:updated><ns2:containsSampledData xmlns:ns2="http://schemas.google.com/analytics/2009">false</ns2:containsSampledData>
<ns0:feed ns1:etag="W/&quot;AkYHRn87fip7I2A9WhZSGUQ.&quot;" ns1:kind="analytics#data" xmlns:ns0="http://www.w3.org/2005/Atom" xmlns:ns1="http://schemas.google.com/g/2005"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;dimensions=ga:pagePath&amp;metrics=ga:newVisits,ga:uniquePageviews,ga:visitors,ga:visits&amp;filters=ga:pagePath%3D~%5E/dataset/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns2:dataSource xmlns:ns2="http://schemas.google.com/analytics/2009"><ns2:tableName>datagm.staging.ckan.net/</ns2:tableName><ns2:tableId>ga:42156377</ns2:tableId><ns2:property name="ga:profileId" value="42156377" /><ns2:property name="ga:webPropertyId" value="UA-21313878-1" /><ns2:property name="ga:accountName" value="http://datagm.staging.ckan.net/" /></ns2:dataSource><ns2:aggregates xmlns:ns2="http://schemas.google.com/analytics/2009"><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="122" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="526" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="83" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="606" /></ns2:aggregates><ns0:updated>2011-04-05T03:08:57.106-07:00</ns0:updated><ns2:containsSampledData xmlns:ns2="http://schemas.google.com/analytics/2009">false</ns2:containsSampledData>
<ns0:entry ns1:etag="W/&quot;C0EEQX47eSp7I2A9WhZSGUs.&quot;" ns1:kind="analytics#datarow"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;ga:pagePath=/package/annakarenina&amp;filters=ga:pagePath%3D~%5E/package/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns0:updated>2011-04-04T17:00:00.001-07:00</ns0:updated><ns2:dimension name="ga:pagePath" value="/package/annakarenina" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:title>ga:pagePath=/package/annakarenina</ns0:title><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:link href="http://www.google.com/analytics" rel="alternate" type="text/html" /></ns0:entry>
<ns0:entry ns1:etag="W/&quot;C0EEQX47eSp7I2A9WhZSGUs.&quot;" ns1:kind="analytics#datarow"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;ga:pagePath=/dataset/annakarenina&amp;filters=ga:pagePath%3D~%5E/dataset/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns0:updated>2011-04-04T17:00:00.001-07:00</ns0:updated><ns2:dimension name="ga:pagePath" value="/dataset/annakarenina" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:title>ga:pagePath=/dataset/annakarenina</ns0:title><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:link href="http://www.google.com/analytics" rel="alternate" type="text/html" /></ns0:entry>
<ns0:entry ns1:etag="W/&quot;C0EEQX47eSp7I2A9WhZSGUs.&quot;" ns1:kind="analytics#datarow"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;ga:pagePath=/package/annakarenina/invalid&amp;filters=ga:pagePath%3D~%5E/package/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns0:updated>2011-04-04T17:00:00.001-07:00</ns0:updated><ns2:dimension name="ga:pagePath" value="/package/annakarenina/invalid" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:title>ga:pagePath=/package/annakarenina/invalid</ns0:title><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:link href="http://www.google.com/analytics" rel="alternate" type="text/html" /></ns0:entry>
<ns0:entry ns1:etag="W/&quot;C0EEQX47eSp7I2A9WhZSGUs.&quot;" ns1:kind="analytics#datarow"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;ga:pagePath=/dataset/annakarenina/invalid&amp;filters=ga:pagePath%3D~%5E/dataset/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns0:updated>2011-04-04T17:00:00.001-07:00</ns0:updated><ns2:dimension name="ga:pagePath" value="/dataset/annakarenina/invalid" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:title>ga:pagePath=/dataset/annakarenina/invalid</ns0:title><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:link href="http://www.google.com/analytics" rel="alternate" type="text/html" /></ns0:entry>
<ns0:entry ns1:etag="W/&quot;C0EEQX47eSp7I2A9WhZSGUs.&quot;" ns1:kind="analytics#datarow"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;ga:pagePath=/package/annakarenina-invalid&amp;filters=ga:pagePath%3D~%5E/package/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns0:updated>2011-04-04T17:00:00.001-07:00</ns0:updated><ns2:dimension name="ga:pagePath" value="/package/annakarenina-invalid" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:title>ga:pagePath=/package/annakarenina-invalid</ns0:title><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:link href="http://www.google.com/analytics" rel="alternate" type="text/html" /></ns0:entry>
<ns0:entry ns1:etag="W/&quot;C0EEQX47eSp7I2A9WhZSGUs.&quot;" ns1:kind="analytics#datarow"><ns0:id>http://www.google.com/analytics/feeds/data?ids=ga:42156377&amp;ga:pagePath=/dataset/annakarenina-invalid&amp;filters=ga:pagePath%3D~%5E/dataset/&amp;start-date=2011-03-22&amp;end-date=2011-04-05</ns0:id><ns0:updated>2011-04-04T17:00:00.001-07:00</ns0:updated><ns2:dimension name="ga:pagePath" value="/dataset/annakarenina-invalid" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:title>ga:pagePath=/dataset/annakarenina-invalid</ns0:title><ns2:metric confidenceInterval="0.0" name="ga:visits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:visitors" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:newVisits" type="integer" value="0" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns2:metric confidenceInterval="0.0" name="ga:uniquePageviews" type="integer" value="2" xmlns:ns2="http://schemas.google.com/analytics/2009" /><ns0:link href="http://www.google.com/analytics" rel="alternate" type="text/html" /></ns0:entry>
<ns0:generator version="1.0">Google Analytics</ns0:generator><ns2:startIndex xmlns:ns2="http://a9.com/-/spec/opensearchrss/1.0/">1</ns2:startIndex><ns0:title>Google Analytics Data for Profile 42156377</ns0:title><ns0:author><ns0:name>Google Analytics</ns0:name></ns0:author><ns0:link href="https://www.google.com/analytics/feeds/data?max-results=10000&amp;sort=-ga%3AnewVisits&amp;end-date=2011-04-05&amp;start-date=2011-03-22&amp;metrics=ga%3Avisits%2Cga%3Avisitors%2Cga%3AnewVisits%2Cga%3AuniquePageviews&amp;ids=ga%3A42156377&amp;dimensions=ga%3ApagePath&amp;filters=ga%3ApagePath%3D%7E%5E%2Fpackage%2F" rel="self" type="application/atom+xml" /><ns2:endDate xmlns:ns2="http://schemas.google.com/analytics/2009">2011-04-05</ns2:endDate><ns2:totalResults xmlns:ns2="http://a9.com/-/spec/opensearchrss/1.0/">152</ns2:totalResults><ns2:startDate xmlns:ns2="http://schemas.google.com/analytics/2009">2011-03-22</ns2:startDate><ns2:itemsPerPage xmlns:ns2="http://a9.com/-/spec/opensearchrss/1.0/">10000</ns2:itemsPerPage></ns0:feed>
<ns0:generator version="1.0">Google Analytics</ns0:generator><ns2:startIndex xmlns:ns2="http://a9.com/-/spec/opensearchrss/1.0/">1</ns2:startIndex><ns0:title>Google Analytics Data for Profile 42156377</ns0:title><ns0:author><ns0:name>Google Analytics</ns0:name></ns0:author><ns0:link href="https://www.google.com/analytics/feeds/data?max-results=10000&amp;sort=-ga%3AnewVisits&amp;end-date=2011-04-05&amp;start-date=2011-03-22&amp;metrics=ga%3Avisits%2Cga%3Avisitors%2Cga%3AnewVisits%2Cga%3AuniquePageviews&amp;ids=ga%3A42156377&amp;dimensions=ga%3ApagePath&amp;filters=ga%3ApagePath%3D%7E%5E%2Fdataset%2F" rel="self" type="application/atom+xml" /><ns2:endDate xmlns:ns2="http://schemas.google.com/analytics/2009">2011-04-05</ns2:endDate><ns2:totalResults xmlns:ns2="http://a9.com/-/spec/opensearchrss/1.0/">152</ns2:totalResults><ns2:startDate xmlns:ns2="http://schemas.google.com/analytics/2009">2011-03-22</ns2:startDate><ns2:itemsPerPage xmlns:ns2="http://a9.com/-/spec/opensearchrss/1.0/">10000</ns2:itemsPerPage></ns0:feed>

View File

@ -1,3 +1,4 @@
import os
import httplib
from unittest import TestCase
@ -18,13 +19,11 @@ class MockClient(httplib.HTTPConnection):
filters = http_request.uri.query.get('filters')
path = http_request.uri.path
if filters:
if "package" in filters:
path += "/package"
if "dataset" in filters:
path += "/dataset"
else:
path += "/download"
httplib.HTTPConnection.request(self,
http_request.method,
path)
httplib.HTTPConnection.request(self, http_request.method, path)
resp = self.getresponse()
return resp
@ -42,6 +41,8 @@ class TestConfig(TestCase):
class TestLoadCommand(TestCase):
@classmethod
def setup_class(cls):
InitDB("initdb").run([]) # set up database tables
config = appconfig('config:test.ini', relative_to=conf_dir)
config.local_conf['ckan.plugins'] = 'googleanalytics'
config.local_conf['googleanalytics.username'] = 'borf'
@ -69,7 +70,6 @@ class TestLoadCommand(TestCase):
assert code in response.body
def test_top_packages(self):
InitDB("initdb").run([]) # set up database tables
command = LoadAnalytics("loadanalytics")
command.TEST_HOST = MockClient('localhost', 6969)
command.CONFIG = self.config
@ -80,12 +80,16 @@ class TestLoadCommand(TestCase):
self.assertEquals(resources[0][1], 4)
def test_download_count_inserted(self):
InitDB("initdb").run([]) # set up database tables
command = LoadAnalytics("loadanalytics")
command.TEST_HOST = MockClient('localhost', 6969)
command.CONFIG = self.config
command.run([])
# command.run(['--config=%s' % os.path.join(conf_dir, 'test.ini')])
# config = appconfig('config:test.ini', relative_to=conf_dir)
# for k in config:
# print k
response = self.app.get(url_for(controller='package',
action='read',
id='annakarenina'))
assert "(downloaded 4 times)" in response.body, response.body
assert "(downloaded 4 times)" in response.body