From 56fbb853a45217657ae5e3887840aad2915e4f1b Mon Sep 17 00:00:00 2001 From: Alex Sadleir Date: Sun, 20 Oct 2013 02:04:28 +1100 Subject: [PATCH 1/6] Adds Google Analytics Event Tracking to some API calls so that usage of the CKAN API can be reported on via Google Analytics. --- README.rst | 3 + ckanext/googleanalytics/controller.py | 81 +++++++++++++++++++++++++++ ckanext/googleanalytics/plugin.py | 53 ++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/README.rst b/README.rst index 5fb1a1e..ce6ad12 100644 --- a/README.rst +++ b/README.rst @@ -18,6 +18,9 @@ Features resource downloads will be displayed as Events in the Google Analytics reporting interface. +* Adds Google Analytics Event Tracking to some API calls so that usage of the + API can be reported on via Google Analytics. + * Adds Google Analytics Event Tracking to group links on the home page, user profile links, editing and saving user profiles, etc. diff --git a/ckanext/googleanalytics/controller.py b/ckanext/googleanalytics/controller.py index e226db8..1efceda 100644 --- a/ckanext/googleanalytics/controller.py +++ b/ckanext/googleanalytics/controller.py @@ -2,6 +2,19 @@ import logging from ckan.lib.base import BaseController, c, render import dbutil +import urllib +from pprint import pprint +import logging +import ckan.logic as logic +import hashlib +import threading +from pylons import config + +from webob.multidict import UnicodeMultiDict +from paste.util.multidict import MultiDict + +from ckan.controllers.api import ApiController + log = logging.getLogger('ckanext.googleanalytics') class GAController(BaseController): @@ -10,3 +23,71 @@ class GAController(BaseController): c.top_packages = dbutil.get_top_packages(limit=10) c.top_resources = dbutil.get_top_resources(limit=10) return render('summary.html') + +class GAApiController(ApiController): + # intercept API calls to record via google analytics + def _post_analytics(self,user,request_obj_type,request_function,request_id): + if (config.get('googleanalytics.id') != None): + data = urllib.urlencode({ + "v":1, + "tid":config.get('googleanalytics.id'), + "cid":hashlib.md5(user).hexdigest(), #customer id should be obfuscated + "t":"event", + "dh":c.environ['HTTP_HOST'], + "dp":c.environ['PATH_INFO'], + "dr":c.environ.get('HTTP_REFERER',''), + "ec":"CKAN API Request", + "ea":request_obj_type+request_function, + "el":request_id, + }) + log.debug("Sending API event to Google Analytics: "+data) + # send analytics asynchronously + threading.Thread(target=urllib.urlopen,args=("http://www.google-analytics.com/collect", data)).start() + + + def action(self, logic_function, ver=None): + try: + function = logic.get_action(logic_function) + except Exception,e: + log.debug(e) + pass + try: + side_effect_free = getattr(function, 'side_effect_free', False) + request_data = self._get_request_data(try_url_params=side_effect_free) + if isinstance(request_data, dict): + id = request_data.get('id','') + if 'q' in request_data.keys(): + id = request_data['q'] + if 'query' in request_data.keys(): + id = request_data['query'] + self._post_analytics(c.user,logic_function,'', id) + except Exception,e: + print log.debug(e) + pass + + return ApiController.action(self,logic_function, ver) + + def list(self, ver=None, register=None, subregister=None, id=None): + self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"list",id) + return ApiController.list(self,ver, register, subregister, id) + def show(self, ver=None, register=None, subregister=None, id=None, id2=None): + self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"show",id) + return ApiController.show(self,ver, register, subregister, id,id2) + def update(self, ver=None, register=None, subregister=None, id=None, id2=None): + self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"update",id) + return ApiController.update(self,ver, register, subregister, id,id2) + def delete(self, ver=None, register=None, subregister=None, id=None, id2=None): + self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"delete",id) + return ApiController.delete(self,ver, register, subregister, id,id2) + def search(self, ver=None, register=None): + id = None + try: + params = MultiDict(self._get_search_params(request.params)) + if 'q' in params.keys(): + id = params['q'] + if 'query' in params.keys(): + id = params['query'] + except ValueError, e: + print str(e) + pass + self._post_analytics(c.user,register,"search",id) diff --git a/ckanext/googleanalytics/plugin.py b/ckanext/googleanalytics/plugin.py index febcee8..bfebc54 100644 --- a/ckanext/googleanalytics/plugin.py +++ b/ckanext/googleanalytics/plugin.py @@ -8,6 +8,7 @@ import pylons import ckan.lib.helpers as h import ckan.plugins as p import gasnippet +from routes.mapper import SubMapper, Mapper as _Mapper log = logging.getLogger('ckanext.googleanalytics') @@ -67,6 +68,58 @@ class GoogleAnalyticsPlugin(p.SingletonPlugin): else: p.toolkit.add_template_directory(config, 'templates') + def before_map(self, map): + '''Add new routes that this extension's controllers handle. + + See IRoutes. + + ''' + # Helpers to reduce code clutter + GET = dict(method=['GET']) + PUT = dict(method=['PUT']) + POST = dict(method=['POST']) + DELETE = dict(method=['DELETE']) + GET_POST = dict(method=['GET', 'POST']) + # intercept API calls that we want to capture analytics on + register_list = [ + 'package', + 'dataset', + 'resource', + 'tag', + 'group', + 'related', + 'revision', + 'licenses', + 'rating', + 'user', + 'activity' + ] + register_list_str = '|'.join(register_list) + # /api ver 3 or none + with SubMapper(map, controller='ckanext.googleanalytics.controller:GAApiController', path_prefix='/api{ver:/3|}', + ver='/3') as m: + m.connect('/action/{logic_function}', action='action', + conditions=GET_POST) + + # /api ver 1, 2, 3 or none + with SubMapper(map, controller='ckanext.googleanalytics.controller:GAApiController', path_prefix='/api{ver:/1|/2|/3|}', + ver='/1') as m: + m.connect('/search/{register}', action='search') + + # /api/rest ver 1, 2 or none + with SubMapper(map, controller='ckanext.googleanalytics.controller:GAApiController', path_prefix='/api{ver:/1|/2|}', + ver='/1', requirements=dict(register=register_list_str) + ) as m: + + m.connect('/rest/{register}', action='list', conditions=GET) + m.connect('/rest/{register}', action='create', conditions=POST) + m.connect('/rest/{register}/{id}', action='show', conditions=GET) + m.connect('/rest/{register}/{id}', action='update', conditions=PUT) + m.connect('/rest/{register}/{id}', action='update', conditions=POST) + m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE) + + return map + def after_map(self, map): '''Add new routes that this extension's controllers handle. From 482d8e2adaa0a40e6abc3e92868b9cf777bd4ce1 Mon Sep 17 00:00:00 2001 From: "CKAN data.gov.au" Date: Thu, 22 May 2014 06:59:28 +0000 Subject: [PATCH 2/6] Use universal analytics --- .../googleanalytics_event_tracking.js | 2 +- .../snippets/googleanalytics_header.html | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js b/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js index 10a50b3..34cdd5f 100644 --- a/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js +++ b/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js @@ -8,7 +8,7 @@ this.ckan.module('google-analytics', function(jQuery, _) { jQuery('a.resource-url-analytics').on('click', function() { var resource_url = encodeURIComponent(jQuery(this).prop('href')); if (resource_url) { - _gaq.push(['_trackEvent', 'Resource', 'Download', resource_url]); + ga('send', 'event', 'Resource', 'Download', resource_url]); } }); } diff --git a/ckanext/googleanalytics/templates/googleanalytics/snippets/googleanalytics_header.html b/ckanext/googleanalytics/templates/googleanalytics/snippets/googleanalytics_header.html index 863a1e6..1486393 100644 --- a/ckanext/googleanalytics/templates/googleanalytics/snippets/googleanalytics_header.html +++ b/ckanext/googleanalytics/templates/googleanalytics/snippets/googleanalytics_header.html @@ -1,11 +1,10 @@ From f83664c98e3641bf9c3db89aead92b032feed8b0 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 24 Jul 2014 22:58:40 +0000 Subject: [PATCH 3/6] fix typo --- .../fanstatic_library/googleanalytics_event_tracking.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js b/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js index 34cdd5f..60bcb5c 100644 --- a/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js +++ b/ckanext/googleanalytics/fanstatic_library/googleanalytics_event_tracking.js @@ -8,7 +8,7 @@ this.ckan.module('google-analytics', function(jQuery, _) { jQuery('a.resource-url-analytics').on('click', function() { var resource_url = encodeURIComponent(jQuery(this).prop('href')); if (resource_url) { - ga('send', 'event', 'Resource', 'Download', resource_url]); + ga('send', 'event', 'Resource', 'Download', resource_url); } }); } From d9f811cea7ce48091e7661941ea9aad06e0f02f6 Mon Sep 17 00:00:00 2001 From: Alex Sadleir Date: Wed, 5 Nov 2014 12:43:24 +1100 Subject: [PATCH 4/6] PEP8 fixes for Google Analytics Event Tracking for CKAN API --- ckanext/googleanalytics/controller.py | 128 ++++++++++++++++---------- 1 file changed, 78 insertions(+), 50 deletions(-) diff --git a/ckanext/googleanalytics/controller.py b/ckanext/googleanalytics/controller.py index 1efceda..6ffd5e4 100644 --- a/ckanext/googleanalytics/controller.py +++ b/ckanext/googleanalytics/controller.py @@ -1,9 +1,9 @@ import logging -from ckan.lib.base import BaseController, c, render +from ckan.lib.base import BaseController, c, render, request import dbutil import urllib -from pprint import pprint + import logging import ckan.logic as logic import hashlib @@ -17,6 +17,7 @@ from ckan.controllers.api import ApiController log = logging.getLogger('ckanext.googleanalytics') + class GAController(BaseController): def view(self): # get package objects corresponding to popular GA content @@ -24,61 +25,88 @@ class GAController(BaseController): c.top_resources = dbutil.get_top_resources(limit=10) return render('summary.html') + class GAApiController(ApiController): # intercept API calls to record via google analytics - def _post_analytics(self,user,request_obj_type,request_function,request_id): - if (config.get('googleanalytics.id') != None): - data = urllib.urlencode({ - "v":1, - "tid":config.get('googleanalytics.id'), - "cid":hashlib.md5(user).hexdigest(), #customer id should be obfuscated - "t":"event", - "dh":c.environ['HTTP_HOST'], - "dp":c.environ['PATH_INFO'], - "dr":c.environ.get('HTTP_REFERER',''), - "ec":"CKAN API Request", - "ea":request_obj_type+request_function, - "el":request_id, - }) + def _post_analytics( + self, user, request_obj_type, request_function, request_id): + if config.get('googleanalytics.id'): + data_dict = { + "v": 1, + "tid": config.get('googleanalytics.id'), + "cid": hashlib.md5(user).hexdigest(), + # customer id should be obfuscated + "t": "event", + "dh": c.environ['HTTP_HOST'], + "dp": c.environ['PATH_INFO'], + "dr": c.environ.get('HTTP_REFERER', ''), + "ec": "CKAN API Request", + "ea": request_obj_type+request_function, + "el": request_id, + } + data = urllib.urlencode(data_dict) log.debug("Sending API event to Google Analytics: "+data) # send analytics asynchronously - threading.Thread(target=urllib.urlopen,args=("http://www.google-analytics.com/collect", data)).start() - + threading.Thread(target=urllib.urlopen, + args=( + "http://www.google-analytics.com/collect", + data)).start() def action(self, logic_function, ver=None): try: - function = logic.get_action(logic_function) - except Exception,e: - log.debug(e) - pass - try: - side_effect_free = getattr(function, 'side_effect_free', False) - request_data = self._get_request_data(try_url_params=side_effect_free) - if isinstance(request_data, dict): - id = request_data.get('id','') - if 'q' in request_data.keys(): - id = request_data['q'] - if 'query' in request_data.keys(): - id = request_data['query'] - self._post_analytics(c.user,logic_function,'', id) - except Exception,e: - print log.debug(e) - pass + function = logic.get_action(logic_function) + side_effect_free = getattr(function, 'side_effect_free', False) + request_data = self._get_request_data( + try_url_params=side_effect_free) + if isinstance(request_data, dict): + id = request_data.get('id', '') + if 'q' in request_data: + id = request_data['q'] + if 'query' in request_data: + id = request_data['query'] + self._post_analytics(c.user, logic_function, '', id) + except Exception, e: + log.debug(e) + pass - return ApiController.action(self,logic_function, ver) + return ApiController.action(self, logic_function, ver) + + def list(self, ver=None, register=None, + subregister=None, id=None): + self._post_analytics(c.user, + register + + ("_"+str(subregister) if subregister else ""), + "list", + id) + return ApiController.list(self, ver, register, subregister, id) + + def show(self, ver=None, register=None, + subregister=None, id=None, id2=None): + self._post_analytics(c.user, + register + + ("_"+str(subregister) if subregister else ""), + "show", + id) + return ApiController.show(self, ver, register, subregister, id, id2) + + def update(self, ver=None, register=None, + subregister=None, id=None, id2=None): + self._post_analytics(c.user, + register + + ("_"+str(subregister) if subregister else ""), + "update", + id) + return ApiController.update(self, ver, register, subregister, id, id2) + + def delete(self, ver=None, register=None, + subregister=None, id=None, id2=None): + self._post_analytics(c.user, + register + + ("_"+str(subregister) if subregister else ""), + "delete", + id) + return ApiController.delete(self, ver, register, subregister, id, id2) - def list(self, ver=None, register=None, subregister=None, id=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"list",id) - return ApiController.list(self,ver, register, subregister, id) - def show(self, ver=None, register=None, subregister=None, id=None, id2=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"show",id) - return ApiController.show(self,ver, register, subregister, id,id2) - def update(self, ver=None, register=None, subregister=None, id=None, id2=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"update",id) - return ApiController.update(self,ver, register, subregister, id,id2) - def delete(self, ver=None, register=None, subregister=None, id=None, id2=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"delete",id) - return ApiController.delete(self,ver, register, subregister, id,id2) def search(self, ver=None, register=None): id = None try: @@ -88,6 +116,6 @@ class GAApiController(ApiController): if 'query' in params.keys(): id = params['query'] except ValueError, e: - print str(e) + log.debug(str(e)) pass - self._post_analytics(c.user,register,"search",id) + self._post_analytics(c.user, register, "search", id) From 5809f205f5da74b0925be1c7212a5455b5d32521 Mon Sep 17 00:00:00 2001 From: Alex Sadleir Date: Wed, 5 Nov 2014 12:51:51 +1100 Subject: [PATCH 5/6] Use urllib2 timeout for Google Analytics Event Tracking requests --- ckanext/googleanalytics/controller.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckanext/googleanalytics/controller.py b/ckanext/googleanalytics/controller.py index 6ffd5e4..58e4919 100644 --- a/ckanext/googleanalytics/controller.py +++ b/ckanext/googleanalytics/controller.py @@ -3,6 +3,7 @@ from ckan.lib.base import BaseController, c, render, request import dbutil import urllib +import urllib2 import logging import ckan.logic as logic @@ -47,10 +48,12 @@ class GAApiController(ApiController): data = urllib.urlencode(data_dict) log.debug("Sending API event to Google Analytics: "+data) # send analytics asynchronously - threading.Thread(target=urllib.urlopen, + threading.Thread(target=urllib2.urlopen, args=( "http://www.google-analytics.com/collect", - data)).start() + data, + # timeout in seconds https://docs.python.org/2/library/urllib2.html#urllib2.urlopen + 10)).start() def action(self, logic_function, ver=None): try: From 803b3f7f81c6682a64e28e8c63cf1c2f30b4293e Mon Sep 17 00:00:00 2001 From: Alex Sadleir Date: Wed, 5 Nov 2014 13:39:53 +1100 Subject: [PATCH 6/6] Use pool of threads and queue for Google Analytics Event Tracking requests --- ckanext/googleanalytics/controller.py | 45 +++++++++++++++++++++------ 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/ckanext/googleanalytics/controller.py b/ckanext/googleanalytics/controller.py index 58e4919..d9eb969 100644 --- a/ckanext/googleanalytics/controller.py +++ b/ckanext/googleanalytics/controller.py @@ -8,6 +8,7 @@ import urllib2 import logging import ckan.logic as logic import hashlib +import Queue import threading from pylons import config @@ -27,8 +28,42 @@ class GAController(BaseController): return render('summary.html') +class AnalyticsPostThread(threading.Thread): + """Threaded Url POST""" + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + while True: + # grabs host from queue + data_dict = self.queue.get() + + data = urllib.urlencode(data_dict) + log.debug("Sending API event to Google Analytics: " + data) + # send analytics + urllib2.urlopen( + "http://www.google-analytics.com/collect", + data, + # timeout in seconds + # https://docs.python.org/2/library/urllib2.html#urllib2.urlopen + 10) + + # signals to queue job is done + self.queue.task_done() + + class GAApiController(ApiController): # intercept API calls to record via google analytics + analytics_queue = Queue.Queue() + + def __init__(self): + # spawn a pool of 5 threads, and pass them queue instance + for i in range(5): + t = AnalyticsPostThread(self.analytics_queue) + t.setDaemon(True) + t.start() + def _post_analytics( self, user, request_obj_type, request_function, request_id): if config.get('googleanalytics.id'): @@ -45,15 +80,7 @@ class GAApiController(ApiController): "ea": request_obj_type+request_function, "el": request_id, } - data = urllib.urlencode(data_dict) - log.debug("Sending API event to Google Analytics: "+data) - # send analytics asynchronously - threading.Thread(target=urllib2.urlopen, - args=( - "http://www.google-analytics.com/collect", - data, - # timeout in seconds https://docs.python.org/2/library/urllib2.html#urllib2.urlopen - 10)).start() + self.analytics_queue.put(data_dict) def action(self, logic_function, ver=None): try: