From fefac7548d993e3105ff364e5e2ea492a903d7f0 Mon Sep 17 00:00:00 2001 From: Seb Bacon Date: Tue, 5 Apr 2011 15:08:24 +0100 Subject: [PATCH] add tests, and associated fixage/harness support to main software --- README.txt | 9 +++ ckanext/googleanalytics/commands.py | 24 +++++--- ckanext/googleanalytics/dbutil.py | 3 +- tests/__init__.py | 0 tests/accountsfixture.xml | 1 + tests/downloadfixture.xml | 8 +++ tests/mockgoogleanalytics.py | 66 ++++++++++++++++++++++ tests/packagefixture.xml | 9 +++ tests/test_general.py | 87 +++++++++++++++++++++++++++++ 9 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/accountsfixture.xml create mode 100644 tests/downloadfixture.xml create mode 100644 tests/mockgoogleanalytics.py create mode 100644 tests/packagefixture.xml create mode 100644 tests/test_general.py diff --git a/README.txt b/README.txt index 6f1393c..9966f43 100644 --- a/README.txt +++ b/README.txt @@ -47,6 +47,15 @@ Installation 6. Consider putting the import command as a daily cron job, or remember to run it by hand! +Testing +======= + +There are some very high-level functional tests that you can run using:: + + (pyenv)~/pyenv/src/ckan$ nosetests --ckan ../ckanext-googleanalytics/tests/ + +(note -- that's run from the CKAN software root, not the extension root) + TODO ==== diff --git a/ckanext/googleanalytics/commands.py b/ckanext/googleanalytics/commands.py index b60e69f..f43985a 100644 --- a/ckanext/googleanalytics/commands.py +++ b/ckanext/googleanalytics/commands.py @@ -1,6 +1,6 @@ import logging import datetime -from pylons import config +from pylons import config as pylonsconfig from ckan.lib.cli import CkanCommand from gdata.analytics import client import ckan.model as model @@ -21,11 +21,14 @@ class LoadAnalytics(CkanCommand): usage = __doc__ max_args = 0 min_args = 0 - + TEST_HOST = None + CONFIG = pylonsconfig + def command(self): self._load_config() - self.resource_url_tag = config.get('googleanalytics.resource_prefix', - DEFAULT_RESOURCE_URL_TAG) + self.resource_url_tag = self.CONFIG.get( + 'googleanalytics.resource_prefix', + DEFAULT_RESOURCE_URL_TAG) self.setup_ga_connection() # funny dance we need to do to make sure we've got a # configured session @@ -67,12 +70,16 @@ class LoadAnalytics(CkanCommand): def setup_ga_connection(self): SOURCE_APP_NAME = "CKAN Google Analytics Plugin" - username = config.get('googleanalytics.username') - password = config.get('googleanalytics.password') - profile_name = config.get('googleanalytics.profile_name') + username = self.CONFIG.get('googleanalytics.username') + password = self.CONFIG.get('googleanalytics.password') + profile_name = self.CONFIG.get('googleanalytics.profile_name') if not username or not password or not profile_name: raise Exception("No googleanalytics profile info in config") - my_client = client.AnalyticsClient(source=SOURCE_APP_NAME) + if self.TEST_HOST: + my_client = client.AnalyticsClient(source=SOURCE_APP_NAME, + http_client=self.TEST_HOST) + else: + my_client = client.AnalyticsClient(source=SOURCE_APP_NAME) my_client.ClientLogin(username, password, SOURCE_APP_NAME) @@ -129,4 +136,3 @@ class LoadAnalytics(CkanCommand): 'ga:uniquePageviews').value or 0 packages.setdefault(package, {})[date_name] = count return packages - diff --git a/ckanext/googleanalytics/dbutil.py b/ckanext/googleanalytics/dbutil.py index 9c93a9b..4f82a90 100644 --- a/ckanext/googleanalytics/dbutil.py +++ b/ckanext/googleanalytics/dbutil.py @@ -61,7 +61,8 @@ def update_package_visits(package_id, recently, ever): WHERE package_id = '%s'""" % package_id).fetchone() if count[0]: connection.execute( - """UPDATE package_stats SET visits = %s + """UPDATE package_stats SET visits_recently = %s, + visits_ever = %s WHERE package_id = '%s'""" % (recently, ever, package_id) ) else: diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/accountsfixture.xml b/tests/accountsfixture.xml new file mode 100644 index 0000000..a6631c3 --- /dev/null +++ b/tests/accountsfixture.xml @@ -0,0 +1 @@ +2011-03-13T01:59:55.435-08:00http://www.google.com/analytics/feeds/accounts/seb.bacon@okfn.orgGoogle AnalyticsGoogle Analytics ga:visitorType==New Visitorga:visitorType==Returning Visitorga:medium==cpa,ga:medium==cpc,ga:medium==cpm,ga:medium==cpp,ga:medium==cpv,ga:medium==ppcga:medium==organicga:medium==cpa,ga:medium==cpc,ga:medium==cpm,ga:medium==cpp,ga:medium==cpv,ga:medium==organic,ga:medium==ppcga:medium==(none)ga:medium==referralga:goalCompletionsAll>0ga:transactions>0ga:isMobile==Yesga:bounces==011300http://www.google.com/analytics/feeds/accounts/ga:42156377ga:421563772011-03-13T01:59:55.435-08:00borfProfile list for seb.bacon@okfn.org diff --git a/tests/downloadfixture.xml b/tests/downloadfixture.xml new file mode 100644 index 0000000..6db89d5 --- /dev/null +++ b/tests/downloadfixture.xml @@ -0,0 +1,8 @@ +http://www.google.com/analytics/feeds/data?ids=ga:42156377&dimensions=ga:pagePath&metrics=ga:newVisits,ga:uniquePageviews,ga:visitors,ga:visits&filters=ga:pagePath%3D~%5E/downloads/&start-date=2011-03-22&end-date=2011-04-05datagm.staging.ckan.net/ga:421563772011-04-05T03:08:58.420-07:00false + +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/downloads/http%3A%2F%2Fwww.annakarenina.com%2Findex.json&filters=ga:pagePath%3D~%5E/downloads/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/downloads/http%3A%2F%2Fwww.annakarenina.com%2Findex.json + + +missingthing2011-04-04T17:00:00.001-07:00ga:pagePath=/downloads/missingthing + +Google Analytics1Google Analytics Data for Profile 42156377Google Analytics2011-04-05432011-03-2210000 diff --git a/tests/mockgoogleanalytics.py b/tests/mockgoogleanalytics.py new file mode 100644 index 0000000..2373d89 --- /dev/null +++ b/tests/mockgoogleanalytics.py @@ -0,0 +1,66 @@ +import os +import BaseHTTPServer +import threading +import gdata.data +import atom.core + +here_dir = os.path.dirname(os.path.abspath(__file__)) + + +class MockHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_GET(self): + if "feeds/accounts/default" in self.path: + self.send_response(200) + self.end_headers() + fixture = os.path.join(here_dir, "accountsfixture.xml") + content = open(fixture, "r").read() + elif "analytics/feeds/data" in self.path: + if "package" in self.path: + fixture = os.path.join(here_dir, + "packagefixture.xml") + elif "download" in self.path: + fixture = os.path.join(here_dir, + "downloadfixture.xml") + self.send_response(200) + self.end_headers() + content = open(fixture, "r").read() + else: + self.send_response(200) + self.end_headers() + content = "empty" + self.wfile.write(content) + + def do_POST(self): + if "ClientLogin" in self.path: + self.send_response(200) + self.end_headers() + content = "Auth=blah" + else: + self.send_response(200) + self.end_headers() + content = "empty" + self.wfile.write(content) + + def do_QUIT(self): + self.send_response(200) + self.end_headers() + self.server.stop = True + + +class ReusableServer(BaseHTTPServer.HTTPServer): + allow_reuse_address = 1 + + def serve_til_quit(self): + self.stop = False + while not self.stop: + self.handle_request() + + +def runmockserver(): + server_address = ('localhost', 6969) + httpd = ReusableServer(server_address, + MockHandler) + httpd_thread = threading.Thread(target=httpd.serve_til_quit) + httpd_thread.setDaemon(True) + httpd_thread.start() + return httpd_thread diff --git a/tests/packagefixture.xml b/tests/packagefixture.xml new file mode 100644 index 0000000..c99e6a8 --- /dev/null +++ b/tests/packagefixture.xml @@ -0,0 +1,9 @@ +http://www.google.com/analytics/feeds/data?ids=ga:42156377&dimensions=ga:pagePath&metrics=ga:newVisits,ga:uniquePageviews,ga:visitors,ga:visits&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-05datagm.staging.ckan.net/ga:421563772011-04-05T03:08:57.106-07:00false + +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/package/annakarenina&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/package/annakarenina + +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/package/annakarenina/invalid&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/package/annakarenina/invalid + +http://www.google.com/analytics/feeds/data?ids=ga:42156377&ga:pagePath=/package/annakarenina-invalid&filters=ga:pagePath%3D~%5E/package/&start-date=2011-03-22&end-date=2011-04-052011-04-04T17:00:00.001-07:00ga:pagePath=/package/annakarenina-invalid + +Google Analytics1Google Analytics Data for Profile 42156377Google Analytics2011-04-051522011-03-2210000 diff --git a/tests/test_general.py b/tests/test_general.py new file mode 100644 index 0000000..621dfad --- /dev/null +++ b/tests/test_general.py @@ -0,0 +1,87 @@ +import httplib + + +from ckan.config.middleware import make_app +from paste.deploy import appconfig +import paste.fixture +from ckan.tests import conf_dir, url_for, CreateTestData + +from mockgoogleanalytics import runmockserver +from ckanext.googleanalytics.commands import LoadAnalytics +from ckanext.googleanalytics import dbutil + + +class MockClient(httplib.HTTPConnection): + def request(self, http_request): + filters = http_request.uri.query.get('filters') + path = http_request.uri.path + if filters: + if "package" in filters: + path += "/package" + else: + path += "/download" + httplib.HTTPConnection.request(self, + http_request.method, + path) + resp = self.getresponse() + return resp + + +class TestConfig: + def test_config(self): + config = appconfig('config:test.ini', relative_to=conf_dir) + config.local_conf['ckan.plugins'] = 'googleanalytics' + command = LoadAnalytics("loadanalytics") + command.CONFIG = config.local_conf + command.run([]) + + @classmethod + def teardown_class(cls): + CreateTestData.delete() + + +class TestLoadCommand: + @classmethod + def setup_class(cls): + config = appconfig('config:test.ini', relative_to=conf_dir) + config.local_conf['ckan.plugins'] = 'googleanalytics' + config.local_conf['googleanalytics.username'] \ + = 'borf' + config.local_conf['googleanalytics.password'] \ + = 'borf' + config.local_conf['googleanalytics.profile_name'] \ + = 'borf' + cls.config = config.local_conf + wsgiapp = make_app(config.global_conf, **config.local_conf) + env = {'HTTP_ACCEPT': ('text/html;q=0.9,text/plain;' + 'q=0.8,image/png,*/*;q=0.5')} + cls.app = paste.fixture.TestApp(wsgiapp, extra_environ=env) + CreateTestData.create() + runmockserver() + + @classmethod + def teardown_class(cls): + CreateTestData.delete() + conn = httplib.HTTPConnection("localhost:%d" % 6969) + conn.request("QUIT", "/") + conn.getresponse() + + def test_top_packages(self): + command = LoadAnalytics("loadanalytics") + command.TEST_HOST = MockClient('localhost', 6969) + command.CONFIG = self.config + command.run([]) + packages = dbutil.get_top_packages() + resources = dbutil.get_top_resources() + assert packages[0][1] == 2 + assert resources[0][1] == 4 + + def test_download_count_inserted(self): + command = LoadAnalytics("loadanalytics") + command.TEST_HOST = MockClient('localhost', 6969) + command.CONFIG = self.config + command.run([]) + response = self.app.get(url_for(controller='package', + action='read', + id='annakarenina')) + assert "(4 downloads)" in response.body, response.body