add cachetools, handle invalid user

This commit is contained in:
Jay Lee
2017-10-09 13:44:29 -04:00
parent b1a2eb4de5
commit 63ca5f8054
10 changed files with 746 additions and 4 deletions

112
src/cachetools/__init__.py Normal file
View File

@@ -0,0 +1,112 @@
"""Extensible memoizing collections and decorators."""
from __future__ import absolute_import
import functools
from . import keys
from .cache import Cache
from .lfu import LFUCache
from .lru import LRUCache
from .rr import RRCache
from .ttl import TTLCache
__all__ = (
'Cache', 'LFUCache', 'LRUCache', 'RRCache', 'TTLCache',
'cached', 'cachedmethod'
)
__version__ = '2.0.1'
if hasattr(functools.update_wrapper(lambda f: f(), lambda: 42), '__wrapped__'):
_update_wrapper = functools.update_wrapper
else:
def _update_wrapper(wrapper, wrapped):
functools.update_wrapper(wrapper, wrapped)
wrapper.__wrapped__ = wrapped
return wrapper
def cached(cache, key=keys.hashkey, lock=None):
"""Decorator to wrap a function with a memoizing callable that saves
results in a cache.
"""
def decorator(func):
if cache is None:
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
elif lock is None:
def wrapper(*args, **kwargs):
k = key(*args, **kwargs)
try:
return cache[k]
except KeyError:
pass # key not found
v = func(*args, **kwargs)
try:
cache[k] = v
except ValueError:
pass # value too large
return v
else:
def wrapper(*args, **kwargs):
k = key(*args, **kwargs)
try:
with lock:
return cache[k]
except KeyError:
pass # key not found
v = func(*args, **kwargs)
try:
with lock:
cache[k] = v
except ValueError:
pass # value too large
return v
return _update_wrapper(wrapper, func)
return decorator
def cachedmethod(cache, key=keys.hashkey, lock=None):
"""Decorator to wrap a class or instance method with a memoizing
callable that saves results in a cache.
"""
def decorator(method):
if lock is None:
def wrapper(self, *args, **kwargs):
c = cache(self)
if c is None:
return method(self, *args, **kwargs)
k = key(self, *args, **kwargs)
try:
return c[k]
except KeyError:
pass # key not found
v = method(self, *args, **kwargs)
try:
c[k] = v
except ValueError:
pass # value too large
return v
else:
def wrapper(self, *args, **kwargs):
c = cache(self)
if c is None:
return method(self, *args, **kwargs)
k = key(self, *args, **kwargs)
try:
with lock(self):
return c[k]
except KeyError:
pass # key not found
v = method(self, *args, **kwargs)
try:
with lock(self):
c[k] = v
except ValueError:
pass # value too large
return v
return _update_wrapper(wrapper, method)
return decorator

49
src/cachetools/abc.py Normal file
View File

@@ -0,0 +1,49 @@
from __future__ import absolute_import
import collections
from abc import abstractmethod
class DefaultMapping(collections.MutableMapping):
__slots__ = ()
@abstractmethod
def __contains__(self, key): # pragma: nocover
return False
@abstractmethod
def __getitem__(self, key): # pragma: nocover
if hasattr(self.__class__, '__missing__'):
return self.__class__.__missing__(self, key)
else:
raise KeyError(key)
def get(self, key, default=None):
if key in self:
return self[key]
else:
return default
__marker = object()
def pop(self, key, default=__marker):
if key in self:
value = self[key]
del self[key]
elif default is self.__marker:
raise KeyError(key)
else:
value = default
return value
def setdefault(self, key, default=None):
if key in self:
value = self[key]
else:
self[key] = value = default
return value
DefaultMapping.register(dict)

104
src/cachetools/cache.py Normal file
View File

@@ -0,0 +1,104 @@
from __future__ import absolute_import
from .abc import DefaultMapping
class _DefaultSize(object):
def __getitem__(self, _):
return 1
def __setitem__(self, _, value):
assert value == 1
def pop(self, _):
return 1
class Cache(DefaultMapping):
"""Mutable mapping to serve as a simple cache or cache base class."""
__size = _DefaultSize()
def __init__(self, maxsize, missing=None, getsizeof=None):
if missing:
self.__missing = missing
if getsizeof:
self.__getsizeof = getsizeof
self.__size = dict()
self.__data = dict()
self.__currsize = 0
self.__maxsize = maxsize
def __repr__(self):
return '%s(%r, maxsize=%r, currsize=%r)' % (
self.__class__.__name__,
list(self.__data.items()),
self.__maxsize,
self.__currsize,
)
def __getitem__(self, key):
try:
return self.__data[key]
except KeyError:
return self.__missing__(key)
def __setitem__(self, key, value):
maxsize = self.__maxsize
size = self.getsizeof(value)
if size > maxsize:
raise ValueError('value too large')
if key not in self.__data or self.__size[key] < size:
while self.__currsize + size > maxsize:
self.popitem()
if key in self.__data:
diffsize = size - self.__size[key]
else:
diffsize = size
self.__data[key] = value
self.__size[key] = size
self.__currsize += diffsize
def __delitem__(self, key):
size = self.__size.pop(key)
del self.__data[key]
self.__currsize -= size
def __contains__(self, key):
return key in self.__data
def __missing__(self, key):
value = self.__missing(key)
try:
self.__setitem__(key, value)
except ValueError:
pass # value too large
return value
def __iter__(self):
return iter(self.__data)
def __len__(self):
return len(self.__data)
@staticmethod
def __getsizeof(value):
return 1
@staticmethod
def __missing(key):
raise KeyError(key)
@property
def maxsize(self):
"""The maximum size of the cache."""
return self.__maxsize
@property
def currsize(self):
"""The current size of the cache."""
return self.__currsize
def getsizeof(self, value):
"""Return the size of a cache element's value."""
return self.__getsizeof(value)

106
src/cachetools/func.py Normal file
View File

@@ -0,0 +1,106 @@
"""`functools.lru_cache` compatible memoizing function decorators."""
from __future__ import absolute_import
import collections
import functools
import random
import time
try:
from threading import RLock
except ImportError:
from dummy_threading import RLock
from . import keys
from .lfu import LFUCache
from .lru import LRUCache
from .rr import RRCache
from .ttl import TTLCache
__all__ = ('lfu_cache', 'lru_cache', 'rr_cache', 'ttl_cache')
_CacheInfo = collections.namedtuple('CacheInfo', [
'hits', 'misses', 'maxsize', 'currsize'
])
def _cache(cache, typed=False):
def decorator(func):
key = keys.typedkey if typed else keys.hashkey
lock = RLock()
stats = [0, 0]
def cache_info():
with lock:
hits, misses = stats
maxsize = cache.maxsize
currsize = cache.currsize
return _CacheInfo(hits, misses, maxsize, currsize)
def cache_clear():
with lock:
try:
cache.clear()
finally:
stats[:] = [0, 0]
def wrapper(*args, **kwargs):
k = key(*args, **kwargs)
with lock:
try:
v = cache[k]
stats[0] += 1
return v
except KeyError:
stats[1] += 1
v = func(*args, **kwargs)
try:
with lock:
cache[k] = v
except ValueError:
pass # value too large
return v
functools.update_wrapper(wrapper, func)
if not hasattr(wrapper, '__wrapped__'):
wrapper.__wrapped__ = func # Python 2.7
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return wrapper
return decorator
def lfu_cache(maxsize=128, typed=False):
"""Decorator to wrap a function with a memoizing callable that saves
up to `maxsize` results based on a Least Frequently Used (LFU)
algorithm.
"""
return _cache(LFUCache(maxsize), typed)
def lru_cache(maxsize=128, typed=False):
"""Decorator to wrap a function with a memoizing callable that saves
up to `maxsize` results based on a Least Recently Used (LRU)
algorithm.
"""
return _cache(LRUCache(maxsize), typed)
def rr_cache(maxsize=128, choice=random.choice, typed=False):
"""Decorator to wrap a function with a memoizing callable that saves
up to `maxsize` results based on a Random Replacement (RR)
algorithm.
"""
return _cache(RRCache(maxsize, choice), typed)
def ttl_cache(maxsize=128, ttl=600, timer=time.time, typed=False):
"""Decorator to wrap a function with a memoizing callable that saves
up to `maxsize` results based on a Least Recently Used (LRU)
algorithm with a per-item time-to-live (TTL) value.
"""
return _cache(TTLCache(maxsize, ttl, timer), typed)

43
src/cachetools/keys.py Normal file
View File

@@ -0,0 +1,43 @@
"""Key functions for memoizing decorators."""
from __future__ import absolute_import
__all__ = ('hashkey', 'typedkey')
class _HashedTuple(tuple):
__hashvalue = None
def __hash__(self, hash=tuple.__hash__):
hashvalue = self.__hashvalue
if hashvalue is None:
self.__hashvalue = hashvalue = hash(self)
return hashvalue
def __add__(self, other, add=tuple.__add__):
return _HashedTuple(add(self, other))
def __radd__(self, other, add=tuple.__add__):
return _HashedTuple(add(other, self))
_kwmark = (object(),)
def hashkey(*args, **kwargs):
"""Return a cache key for the specified hashable arguments."""
if kwargs:
return _HashedTuple(args + sum(sorted(kwargs.items()), _kwmark))
else:
return _HashedTuple(args)
def typedkey(*args, **kwargs):
"""Return a typed cache key for the specified hashable arguments."""
key = hashkey(*args, **kwargs)
key += tuple(type(v) for v in args)
key += tuple(type(v) for _, v in sorted(kwargs.items()))
return key

35
src/cachetools/lfu.py Normal file
View File

@@ -0,0 +1,35 @@
from __future__ import absolute_import
import collections
from .cache import Cache
class LFUCache(Cache):
"""Least Frequently Used (LFU) cache implementation."""
def __init__(self, maxsize, missing=None, getsizeof=None):
Cache.__init__(self, maxsize, missing, getsizeof)
self.__counter = collections.Counter()
def __getitem__(self, key, cache_getitem=Cache.__getitem__):
value = cache_getitem(self, key)
self.__counter[key] -= 1
return value
def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
cache_setitem(self, key, value)
self.__counter[key] -= 1
def __delitem__(self, key, cache_delitem=Cache.__delitem__):
cache_delitem(self, key)
del self.__counter[key]
def popitem(self):
"""Remove and return the `(key, value)` pair least frequently used."""
try:
(key, _), = self.__counter.most_common(1)
except ValueError:
raise KeyError('%s is empty' % self.__class__.__name__)
else:
return (key, self.pop(key))

48
src/cachetools/lru.py Normal file
View File

@@ -0,0 +1,48 @@
from __future__ import absolute_import
import collections
from .cache import Cache
class LRUCache(Cache):
"""Least Recently Used (LRU) cache implementation."""
def __init__(self, maxsize, missing=None, getsizeof=None):
Cache.__init__(self, maxsize, missing, getsizeof)
self.__order = collections.OrderedDict()
def __getitem__(self, key, cache_getitem=Cache.__getitem__):
value = cache_getitem(self, key)
self.__update(key)
return value
def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
cache_setitem(self, key, value)
self.__update(key)
def __delitem__(self, key, cache_delitem=Cache.__delitem__):
cache_delitem(self, key)
del self.__order[key]
def popitem(self):
"""Remove and return the `(key, value)` pair least recently used."""
try:
key = next(iter(self.__order))
except StopIteration:
raise KeyError('%s is empty' % self.__class__.__name__)
else:
return (key, self.pop(key))
if hasattr(collections.OrderedDict, 'move_to_end'):
def __update(self, key):
try:
self.__order.move_to_end(key)
except KeyError:
self.__order[key] = None
else:
def __update(self, key):
try:
self.__order[key] = self.__order.pop(key)
except KeyError:
self.__order[key] = None

28
src/cachetools/rr.py Normal file
View File

@@ -0,0 +1,28 @@
from __future__ import absolute_import
import random
from .cache import Cache
class RRCache(Cache):
"""Random Replacement (RR) cache implementation."""
def __init__(self, maxsize, choice=random.choice, missing=None,
getsizeof=None):
Cache.__init__(self, maxsize, missing, getsizeof)
self.__choice = choice
@property
def choice(self):
"""The `choice` function used by the cache."""
return self.__choice
def popitem(self):
"""Remove and return a random `(key, value)` pair."""
try:
key = self.__choice(list(self))
except IndexError:
raise KeyError('%s is empty' % self.__class__.__name__)
else:
return (key, self.pop(key))

217
src/cachetools/ttl.py Normal file
View File

@@ -0,0 +1,217 @@
from __future__ import absolute_import
import collections
import time
from .cache import Cache
class _Link(object):
__slots__ = ('key', 'expire', 'next', 'prev')
def __init__(self, key=None, expire=None):
self.key = key
self.expire = expire
def __reduce__(self):
return _Link, (self.key, self.expire)
def unlink(self):
next = self.next
prev = self.prev
prev.next = next
next.prev = prev
class _Timer(object):
def __init__(self, timer):
self.__timer = timer
self.__nesting = 0
def __call__(self):
if self.__nesting == 0:
return self.__timer()
else:
return self.__time
def __enter__(self):
if self.__nesting == 0:
self.__time = time = self.__timer()
else:
time = self.__time
self.__nesting += 1
return time
def __exit__(self, *exc):
self.__nesting -= 1
def __reduce__(self):
return _Timer, (self.__timer,)
def __getattr__(self, name):
return getattr(self.__timer, name)
class TTLCache(Cache):
"""LRU Cache implementation with per-item time-to-live (TTL) value."""
def __init__(self, maxsize, ttl, timer=time.time, missing=None,
getsizeof=None):
Cache.__init__(self, maxsize, missing, getsizeof)
self.__root = root = _Link()
root.prev = root.next = root
self.__links = collections.OrderedDict()
self.__timer = _Timer(timer)
self.__ttl = ttl
def __contains__(self, key):
try:
link = self.__links[key] # no reordering
except KeyError:
return False
else:
return not (link.expire < self.__timer())
def __getitem__(self, key, cache_getitem=Cache.__getitem__):
try:
link = self.__getlink(key)
except KeyError:
expired = False
else:
expired = link.expire < self.__timer()
if expired:
return self.__missing__(key)
else:
return cache_getitem(self, key)
def __setitem__(self, key, value, cache_setitem=Cache.__setitem__):
with self.__timer as time:
self.expire(time)
cache_setitem(self, key, value)
try:
link = self.__getlink(key)
except KeyError:
self.__links[key] = link = _Link(key)
else:
link.unlink()
link.expire = time + self.__ttl
link.next = root = self.__root
link.prev = prev = root.prev
prev.next = root.prev = link
def __delitem__(self, key, cache_delitem=Cache.__delitem__):
cache_delitem(self, key)
link = self.__links.pop(key)
link.unlink()
if link.expire < self.__timer():
raise KeyError(key)
def __iter__(self):
root = self.__root
curr = root.next
while curr is not root:
# "freeze" time for iterator access
with self.__timer as time:
if not (curr.expire < time):
yield curr.key
curr = curr.next
def __len__(self):
root = self.__root
curr = root.next
time = self.__timer()
count = len(self.__links)
while curr is not root and curr.expire < time:
count -= 1
curr = curr.next
return count
def __setstate__(self, state):
self.__dict__.update(state)
root = self.__root
root.prev = root.next = root
for link in sorted(self.__links.values(), key=lambda obj: obj.expire):
link.next = root
link.prev = prev = root.prev
prev.next = root.prev = link
self.expire(self.__timer())
def __repr__(self, cache_repr=Cache.__repr__):
with self.__timer as time:
self.expire(time)
return cache_repr(self)
@property
def currsize(self):
with self.__timer as time:
self.expire(time)
return super(TTLCache, self).currsize
@property
def timer(self):
"""The timer function used by the cache."""
return self.__timer
@property
def ttl(self):
"""The time-to-live value of the cache's items."""
return self.__ttl
def expire(self, time=None):
"""Remove expired items from the cache."""
if time is None:
time = self.__timer()
root = self.__root
curr = root.next
links = self.__links
cache_delitem = Cache.__delitem__
while curr is not root and curr.expire < time:
cache_delitem(self, curr.key)
del links[curr.key]
next = curr.next
curr.unlink()
curr = next
def clear(self):
with self.__timer as time:
self.expire(time)
Cache.clear(self)
def get(self, *args, **kwargs):
with self.__timer:
return Cache.get(self, *args, **kwargs)
def pop(self, *args, **kwargs):
with self.__timer:
return Cache.pop(self, *args, **kwargs)
def setdefault(self, *args, **kwargs):
with self.__timer:
return Cache.setdefault(self, *args, **kwargs)
def popitem(self):
"""Remove and return the `(key, value)` pair least recently used that
has not already expired.
"""
with self.__timer as time:
self.expire(time)
try:
key = next(iter(self.__links))
except StopIteration:
raise KeyError('%s is empty' % self.__class__.__name__)
else:
return (key, self.pop(key))
if hasattr(collections.OrderedDict, 'move_to_end'):
def __getlink(self, key):
value = self.__links[key]
self.__links.move_to_end(key)
return value
else:
def __getlink(self, key):
value = self.__links.pop(key)
self.__links[key] = value
return value

View File

@@ -995,14 +995,14 @@ def buildGAPIServiceObject(api, act_as, use_scopes=None):
GM_Globals[GM_CURRENT_API_SCOPES] = use_scopes or API_SCOPE_MAPPING[api]
credentials = getSvcAcctCredentials(GM_Globals[GM_CURRENT_API_SCOPES], act_as)
request = google_auth_httplib2.Request(http)
credentials.refresh(request)
try:
credentials.refresh(request)
service._http = google_auth_httplib2.AuthorizedHttp(credentials, http=http)
except httplib2.ServerNotFoundError as e:
systemErrorExit(4, e)
except oauth2client.client.AccessTokenRefreshError as e:
stderrErrorMsg(u'User {0}: {1}'.format(GM_Globals[GM_CURRENT_API_USER], str(e)))
return handleOAuthTokenError(str(e), True)
except google.auth.exceptions.RefreshError as e:
stderrErrorMsg(u'User {0}: {1}'.format(GM_Globals[GM_CURRENT_API_USER], str(e[0])))
return handleOAuthTokenError(str(e[0]), True)
return service
def buildActivityGAPIObject(user):