Futurize codebase
This commit is contained in:
parent
a5be073c04
commit
ec9465654e
|
@ -1,3 +1,5 @@
|
||||||
|
from __future__ import print_function
|
||||||
|
from __future__ import absolute_import
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import logging
|
import logging
|
||||||
|
@ -8,7 +10,7 @@ from pylons import config as pylonsconfig
|
||||||
from ckan.lib.cli import CkanCommand
|
from ckan.lib.cli import CkanCommand
|
||||||
import ckan.model as model
|
import ckan.model as model
|
||||||
|
|
||||||
import dbutil
|
from . import dbutil
|
||||||
|
|
||||||
log = logging.getLogger("ckanext.googleanalytics")
|
log = logging.getLogger("ckanext.googleanalytics")
|
||||||
PACKAGE_URL = "/dataset/" # XXX get from routes...
|
PACKAGE_URL = "/dataset/" # XXX get from routes...
|
||||||
|
@ -78,7 +80,7 @@ class LoadAnalytics(CkanCommand):
|
||||||
)
|
)
|
||||||
engine.execute(sql)
|
engine.execute(sql)
|
||||||
|
|
||||||
for url, count in packages_data.iteritems():
|
for url, count in packages_data.items():
|
||||||
# If it matches the resource then we should mark it as a resource.
|
# If it matches the resource then we should mark it as a resource.
|
||||||
# For resources we don't currently find the package ID.
|
# For resources we don't currently find the package ID.
|
||||||
if RESOURCE_URL_REGEX.match(url):
|
if RESOURCE_URL_REGEX.match(url):
|
||||||
|
@ -174,7 +176,7 @@ class LoadAnalytics(CkanCommand):
|
||||||
time.sleep(0.25)
|
time.sleep(0.25)
|
||||||
start_date = stop_date
|
start_date = stop_date
|
||||||
log.info("%s received %s" % (len(packages_data), start_date))
|
log.info("%s received %s" % (len(packages_data), start_date))
|
||||||
print "%s received %s" % (len(packages_data), start_date)
|
print("%s received %s" % (len(packages_data), start_date))
|
||||||
|
|
||||||
def get_ga_data_new(self, start_date=None, end_date=None):
|
def get_ga_data_new(self, start_date=None, end_date=None):
|
||||||
"""Get raw data from Google Analtyics for packages and
|
"""Get raw data from Google Analtyics for packages and
|
||||||
|
@ -234,7 +236,7 @@ class LoadAnalytics(CkanCommand):
|
||||||
|
|
||||||
def parse_and_save(self):
|
def parse_and_save(self):
|
||||||
"""Grab raw data from Google Analytics and save to the database"""
|
"""Grab raw data from Google Analytics and save to the database"""
|
||||||
from ga_auth import init_service, get_profile_id
|
from .ga_auth import init_service, get_profile_id
|
||||||
|
|
||||||
tokenfile = self.args[0]
|
tokenfile = self.args[0]
|
||||||
if not os.path.exists(tokenfile):
|
if not os.path.exists(tokenfile):
|
||||||
|
@ -262,7 +264,7 @@ class LoadAnalytics(CkanCommand):
|
||||||
def save_ga_data(self, packages_data):
|
def save_ga_data(self, packages_data):
|
||||||
"""Save tuples of packages_data to the database
|
"""Save tuples of packages_data to the database
|
||||||
"""
|
"""
|
||||||
for identifier, visits in packages_data.items():
|
for identifier, visits in list(packages_data.items()):
|
||||||
recently = visits.get("recent", 0)
|
recently = visits.get("recent", 0)
|
||||||
ever = visits.get("ever", 0)
|
ever = visits.get("ever", 0)
|
||||||
matches = RESOURCE_URL_REGEX.match(identifier)
|
matches = RESOURCE_URL_REGEX.match(identifier)
|
||||||
|
@ -314,7 +316,7 @@ class LoadAnalytics(CkanCommand):
|
||||||
if not sort:
|
if not sort:
|
||||||
sort = "-ga:uniquePageviews"
|
sort = "-ga:uniquePageviews"
|
||||||
|
|
||||||
print "%s -> %s" % (from_date, to_date)
|
print("%s -> %s" % (from_date, to_date))
|
||||||
|
|
||||||
results = (
|
results = (
|
||||||
self.service.data()
|
self.service.data()
|
||||||
|
@ -349,7 +351,7 @@ class LoadAnalytics(CkanCommand):
|
||||||
packages = {}
|
packages = {}
|
||||||
queries = ["ga:pagePath=~%s" % PACKAGE_URL]
|
queries = ["ga:pagePath=~%s" % PACKAGE_URL]
|
||||||
dates = {"recent": recent_date, "ever": floor_date}
|
dates = {"recent": recent_date, "ever": floor_date}
|
||||||
for date_name, date in dates.iteritems():
|
for date_name, date in dates.items():
|
||||||
for query in queries:
|
for query in queries:
|
||||||
results = self.ga_query(
|
results = self.ga_query(
|
||||||
query_filter=query,
|
query_filter=query,
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from builtins import str
|
||||||
import logging
|
import logging
|
||||||
from ckan.lib.base import BaseController, c, render, request
|
from ckan.lib.base import BaseController, c, render, request
|
||||||
import dbutil
|
from . import dbutil
|
||||||
|
|
||||||
import ckan.logic as logic
|
import ckan.logic as logic
|
||||||
import hashlib
|
import hashlib
|
||||||
import plugin
|
from . import plugin
|
||||||
from pylons import config
|
from pylons import config
|
||||||
|
|
||||||
from paste.util.multidict import MultiDict
|
from paste.util.multidict import MultiDict
|
||||||
|
@ -56,7 +58,7 @@ class GAApiController(ApiController):
|
||||||
if "query" in request_data:
|
if "query" in request_data:
|
||||||
id = request_data["query"]
|
id = request_data["query"]
|
||||||
self._post_analytics(c.user, logic_function, "", id)
|
self._post_analytics(c.user, logic_function, "", id)
|
||||||
except Exception, e:
|
except Exception as e:
|
||||||
log.debug(e)
|
log.debug(e)
|
||||||
pass
|
pass
|
||||||
return ApiController.action(self, logic_function, ver)
|
return ApiController.action(self, logic_function, ver)
|
||||||
|
@ -107,11 +109,11 @@ class GAApiController(ApiController):
|
||||||
id = None
|
id = None
|
||||||
try:
|
try:
|
||||||
params = MultiDict(self._get_search_params(request.params))
|
params = MultiDict(self._get_search_params(request.params))
|
||||||
if "q" in params.keys():
|
if "q" in list(params.keys()):
|
||||||
id = params["q"]
|
id = params["q"]
|
||||||
if "query" in params.keys():
|
if "query" in list(params.keys()):
|
||||||
id = params["query"]
|
id = params["query"]
|
||||||
except ValueError, e:
|
except ValueError as e:
|
||||||
log.debug(str(e))
|
log.debug(str(e))
|
||||||
pass
|
pass
|
||||||
self._post_analytics(c.user, register, "search", id)
|
self._post_analytics(c.user, register, "search", id)
|
||||||
|
|
|
@ -1,13 +1,17 @@
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
|
from future import standard_library
|
||||||
|
standard_library.install_aliases()
|
||||||
|
from builtins import str
|
||||||
|
from builtins import range
|
||||||
import ast
|
import ast
|
||||||
import logging
|
import logging
|
||||||
import urllib
|
import urllib.request, urllib.parse, urllib.error
|
||||||
import ckanext.googleanalytics.commands as commands
|
import ckanext.googleanalytics.commands as commands
|
||||||
import paste.deploy.converters as converters
|
import paste.deploy.converters as converters
|
||||||
import ckan.lib.helpers as h
|
import ckan.lib.helpers as h
|
||||||
import ckan.plugins as p
|
import ckan.plugins as p
|
||||||
import ckan.plugins.toolkit as tk
|
import ckan.plugins.toolkit as tk
|
||||||
import urllib2
|
import urllib.request, urllib.error, urllib.parse
|
||||||
from ckan.exceptions import CkanVersionException
|
from ckan.exceptions import CkanVersionException
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
|
@ -37,10 +41,10 @@ class AnalyticsPostThread(threading.Thread):
|
||||||
# grabs host from queue
|
# grabs host from queue
|
||||||
data_dict = self.queue.get()
|
data_dict = self.queue.get()
|
||||||
|
|
||||||
data = urllib.urlencode(data_dict)
|
data = urllib.parse.urlencode(data_dict)
|
||||||
log.debug("Sending API event to Google Analytics: " + data)
|
log.debug("Sending API event to Google Analytics: " + data)
|
||||||
# send analytics
|
# send analytics
|
||||||
urllib2.urlopen(
|
urllib.request.urlopen(
|
||||||
"http://www.google-analytics.com/collect",
|
"http://www.google-analytics.com/collect",
|
||||||
data,
|
data,
|
||||||
# timeout in seconds
|
# timeout in seconds
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
import Queue
|
from future import standard_library
|
||||||
|
standard_library.install_aliases()
|
||||||
|
import queue
|
||||||
|
|
||||||
import ckan.plugins as plugins
|
import ckan.plugins as plugins
|
||||||
|
|
||||||
|
@ -9,7 +11,7 @@ from ckanext.googleanalytics.views import ga
|
||||||
class GAMixinPlugin(plugins.SingletonPlugin):
|
class GAMixinPlugin(plugins.SingletonPlugin):
|
||||||
plugins.implements(plugins.IBlueprint)
|
plugins.implements(plugins.IBlueprint)
|
||||||
|
|
||||||
analytics_queue = Queue.Queue()
|
analytics_queue = queue.Queue()
|
||||||
|
|
||||||
def get_blueprint(self):
|
def get_blueprint(self):
|
||||||
return [ga]
|
return [ga]
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
import Queue
|
from future import standard_library
|
||||||
|
standard_library.install_aliases()
|
||||||
|
import queue
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import importlib
|
import importlib
|
||||||
|
@ -15,7 +17,7 @@ from routes.mapper import SubMapper
|
||||||
class GAMixinPlugin(plugins.SingletonPlugin):
|
class GAMixinPlugin(plugins.SingletonPlugin):
|
||||||
plugins.implements(plugins.IRoutes)
|
plugins.implements(plugins.IRoutes)
|
||||||
|
|
||||||
analytics_queue = Queue.Queue()
|
analytics_queue = queue.Queue()
|
||||||
|
|
||||||
def before_map(self, map):
|
def before_map(self, map):
|
||||||
"""Add new routes that this extension's controllers handle.
|
"""Add new routes that this extension's controllers handle.
|
||||||
|
|
|
@ -25,7 +25,7 @@ def action(logic_function, ver=api.API_MAX_VERSION):
|
||||||
if "query" in request_data:
|
if "query" in request_data:
|
||||||
id = request_data["query"]
|
id = request_data["query"]
|
||||||
_post_analytics(g.user, "CKAN API Request", logic_function, "", id)
|
_post_analytics(g.user, "CKAN API Request", logic_function, "", id)
|
||||||
except Exception, e:
|
except Exception as e:
|
||||||
log.debug(e)
|
log.debug(e)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue