diff --git a/.travis.yml b/.travis.yml index ac086155..4a84e89a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,25 +3,29 @@ if: tag IS blank matrix: include: - os: linux + name: "Linux 64-bit" + dist: xenial language: python - dist: precise - python: - - "2.7.15" + python: + - 3.7.3 env: - GAMOS=linux - PLATFORM=x86_64 - os: osx + name: "MacOS 64-bit" language: generic osx_image: xcode10.1 env: - GAMOS=macos - PLATFORM=x86_64 - os: windows + name: "Windows 64-bit" language: shell env: - GAMOS=windows PLATFORM=x86_64 - os: windows + name: "Windows 32-bit" language: shell env: - GAMOS=windows @@ -48,5 +52,6 @@ deploy: file: gam-$GAMVERSION-* skip_cleanup: true draft: true + all_branches: true on: repo: jay0lee/GAM diff --git a/src/httplib2/cacerts.txt b/src/cacerts.txt similarity index 99% rename from src/httplib2/cacerts.txt rename to src/cacerts.txt index a2a9833d..8020c1b4 100644 --- a/src/httplib2/cacerts.txt +++ b/src/cacerts.txt @@ -2194,3 +2194,4 @@ Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl MrY= -----END CERTIFICATE----- + diff --git a/src/cachetools/__init__.py b/src/cachetools/__init__.py deleted file mode 100644 index e9a61f78..00000000 --- a/src/cachetools/__init__.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Extensible memoizing collections and decorators.""" - -from __future__ import absolute_import - -import functools - -from . import keys -from .cache import Cache -from .lfu import LFUCache -from .lru import LRUCache -from .rr import RRCache -from .ttl import TTLCache - -__all__ = ( - 'Cache', 'LFUCache', 'LRUCache', 'RRCache', 'TTLCache', - 'cached', 'cachedmethod' -) - -__version__ = '3.1.0' - -if hasattr(functools.update_wrapper(lambda f: f(), lambda: 42), '__wrapped__'): - _update_wrapper = functools.update_wrapper -else: - def _update_wrapper(wrapper, wrapped): - functools.update_wrapper(wrapper, wrapped) - wrapper.__wrapped__ = wrapped - return wrapper - - -def cached(cache, key=keys.hashkey, lock=None): - """Decorator to wrap a function with a memoizing callable that saves - results in a cache. - - """ - def decorator(func): - if cache is None: - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - elif lock is None: - def wrapper(*args, **kwargs): - k = key(*args, **kwargs) - try: - return cache[k] - except KeyError: - pass # key not found - v = func(*args, **kwargs) - try: - cache[k] = v - except ValueError: - pass # value too large - return v - else: - def wrapper(*args, **kwargs): - k = key(*args, **kwargs) - try: - with lock: - return cache[k] - except KeyError: - pass # key not found - v = func(*args, **kwargs) - try: - with lock: - cache[k] = v - except ValueError: - pass # value too large - return v - return _update_wrapper(wrapper, func) - return decorator - - -def cachedmethod(cache, key=keys.hashkey, lock=None): - """Decorator to wrap a class or instance method with a memoizing - callable that saves results in a cache. - - """ - def decorator(method): - if lock is None: - def wrapper(self, *args, **kwargs): - c = cache(self) - if c is None: - return method(self, *args, **kwargs) - k = key(*args, **kwargs) - try: - return c[k] - except KeyError: - pass # key not found - v = method(self, *args, **kwargs) - try: - c[k] = v - except ValueError: - pass # value too large - return v - else: - def wrapper(self, *args, **kwargs): - c = cache(self) - if c is None: - return method(self, *args, **kwargs) - k = key(*args, **kwargs) - try: - with lock(self): - return c[k] - except KeyError: - pass # key not found - v = method(self, *args, **kwargs) - try: - with lock(self): - c[k] = v - except ValueError: - pass # value too large - return v - return _update_wrapper(wrapper, method) - return decorator diff --git a/src/cachetools/abc.py b/src/cachetools/abc.py deleted file mode 100644 index 3bc43cc4..00000000 --- a/src/cachetools/abc.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import absolute_import - -from abc import abstractmethod - -try: - from collections.abc import MutableMapping -except ImportError: - from collections import MutableMapping - - -class DefaultMapping(MutableMapping): - - __slots__ = () - - @abstractmethod - def __contains__(self, key): # pragma: nocover - return False - - @abstractmethod - def __getitem__(self, key): # pragma: nocover - if hasattr(self.__class__, '__missing__'): - return self.__class__.__missing__(self, key) - else: - raise KeyError(key) - - def get(self, key, default=None): - if key in self: - return self[key] - else: - return default - - __marker = object() - - def pop(self, key, default=__marker): - if key in self: - value = self[key] - del self[key] - elif default is self.__marker: - raise KeyError(key) - else: - value = default - return value - - def setdefault(self, key, default=None): - if key in self: - value = self[key] - else: - self[key] = value = default - return value - - -DefaultMapping.register(dict) diff --git a/src/cachetools/cache.py b/src/cachetools/cache.py deleted file mode 100644 index 5cb80715..00000000 --- a/src/cachetools/cache.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import absolute_import - -from .abc import DefaultMapping - - -class _DefaultSize(object): - def __getitem__(self, _): - return 1 - - def __setitem__(self, _, value): - assert value == 1 - - def pop(self, _): - return 1 - - -class Cache(DefaultMapping): - """Mutable mapping to serve as a simple cache or cache base class.""" - - __size = _DefaultSize() - - def __init__(self, maxsize, getsizeof=None): - if getsizeof: - self.getsizeof = getsizeof - if self.getsizeof is not Cache.getsizeof: - self.__size = dict() - self.__data = dict() - self.__currsize = 0 - self.__maxsize = maxsize - - def __repr__(self): - return '%s(%r, maxsize=%r, currsize=%r)' % ( - self.__class__.__name__, - list(self.__data.items()), - self.__maxsize, - self.__currsize, - ) - - def __getitem__(self, key): - try: - return self.__data[key] - except KeyError: - return self.__missing__(key) - - def __setitem__(self, key, value): - maxsize = self.__maxsize - size = self.getsizeof(value) - if size > maxsize: - raise ValueError('value too large') - if key not in self.__data or self.__size[key] < size: - while self.__currsize + size > maxsize: - self.popitem() - if key in self.__data: - diffsize = size - self.__size[key] - else: - diffsize = size - self.__data[key] = value - self.__size[key] = size - self.__currsize += diffsize - - def __delitem__(self, key): - size = self.__size.pop(key) - del self.__data[key] - self.__currsize -= size - - def __contains__(self, key): - return key in self.__data - - def __missing__(self, key): - raise KeyError(key) - - def __iter__(self): - return iter(self.__data) - - def __len__(self): - return len(self.__data) - - @property - def maxsize(self): - """The maximum size of the cache.""" - return self.__maxsize - - @property - def currsize(self): - """The current size of the cache.""" - return self.__currsize - - @staticmethod - def getsizeof(value): - """Return the size of a cache element's value.""" - return 1 diff --git a/src/cachetools/func.py b/src/cachetools/func.py deleted file mode 100644 index 8ced5dda..00000000 --- a/src/cachetools/func.py +++ /dev/null @@ -1,140 +0,0 @@ -"""`functools.lru_cache` compatible memoizing function decorators.""" - -from __future__ import absolute_import - -import collections -import functools -import random - -try: - from time import monotonic as default_timer -except ImportError: - from time import time as default_timer - -try: - from threading import RLock -except ImportError: # pragma: no cover - from dummy_threading import RLock - -from . import keys -from .lfu import LFUCache -from .lru import LRUCache -from .rr import RRCache -from .ttl import TTLCache - -__all__ = ('lfu_cache', 'lru_cache', 'rr_cache', 'ttl_cache') - - -_CacheInfo = collections.namedtuple('CacheInfo', [ - 'hits', 'misses', 'maxsize', 'currsize' -]) - - -class _UnboundCache(dict): - - maxsize = None - - @property - def currsize(self): - return len(self) - - -class _UnboundTTLCache(TTLCache): - def __init__(self, ttl, timer): - TTLCache.__init__(self, float('inf'), ttl, timer) - - @property - def maxsize(self): - return None - - -def _cache(cache, typed=False): - def decorator(func): - key = keys.typedkey if typed else keys.hashkey - lock = RLock() - stats = [0, 0] - - def cache_info(): - with lock: - hits, misses = stats - maxsize = cache.maxsize - currsize = cache.currsize - return _CacheInfo(hits, misses, maxsize, currsize) - - def cache_clear(): - with lock: - try: - cache.clear() - finally: - stats[:] = [0, 0] - - def wrapper(*args, **kwargs): - k = key(*args, **kwargs) - with lock: - try: - v = cache[k] - stats[0] += 1 - return v - except KeyError: - stats[1] += 1 - v = func(*args, **kwargs) - try: - with lock: - cache[k] = v - except ValueError: - pass # value too large - return v - functools.update_wrapper(wrapper, func) - if not hasattr(wrapper, '__wrapped__'): - wrapper.__wrapped__ = func # Python 2.7 - wrapper.cache_info = cache_info - wrapper.cache_clear = cache_clear - return wrapper - return decorator - - -def lfu_cache(maxsize=128, typed=False): - """Decorator to wrap a function with a memoizing callable that saves - up to `maxsize` results based on a Least Frequently Used (LFU) - algorithm. - - """ - if maxsize is None: - return _cache(_UnboundCache(), typed) - else: - return _cache(LFUCache(maxsize), typed) - - -def lru_cache(maxsize=128, typed=False): - """Decorator to wrap a function with a memoizing callable that saves - up to `maxsize` results based on a Least Recently Used (LRU) - algorithm. - - """ - if maxsize is None: - return _cache(_UnboundCache(), typed) - else: - return _cache(LRUCache(maxsize), typed) - - -def rr_cache(maxsize=128, choice=random.choice, typed=False): - """Decorator to wrap a function with a memoizing callable that saves - up to `maxsize` results based on a Random Replacement (RR) - algorithm. - - """ - if maxsize is None: - return _cache(_UnboundCache(), typed) - else: - return _cache(RRCache(maxsize, choice), typed) - - -def ttl_cache(maxsize=128, ttl=600, timer=default_timer, typed=False): - """Decorator to wrap a function with a memoizing callable that saves - up to `maxsize` results based on a Least Recently Used (LRU) - algorithm with a per-item time-to-live (TTL) value. - """ - if maxsize is None: - return _cache(_UnboundTTLCache(ttl, timer), typed) - else: - return _cache(TTLCache(maxsize, ttl, timer), typed) diff --git a/src/cachetools/keys.py b/src/cachetools/keys.py deleted file mode 100644 index adb9dad4..00000000 --- a/src/cachetools/keys.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Key functions for memoizing decorators.""" - -from __future__ import absolute_import - -__all__ = ('hashkey', 'typedkey') - - -class _HashedTuple(tuple): - - __hashvalue = None - - def __hash__(self, hash=tuple.__hash__): - hashvalue = self.__hashvalue - if hashvalue is None: - self.__hashvalue = hashvalue = hash(self) - return hashvalue - - def __add__(self, other, add=tuple.__add__): - return _HashedTuple(add(self, other)) - - def __radd__(self, other, add=tuple.__add__): - return _HashedTuple(add(other, self)) - - -_kwmark = (object(),) - - -def hashkey(*args, **kwargs): - """Return a cache key for the specified hashable arguments.""" - - if kwargs: - return _HashedTuple(args + sum(sorted(kwargs.items()), _kwmark)) - else: - return _HashedTuple(args) - - -def typedkey(*args, **kwargs): - """Return a typed cache key for the specified hashable arguments.""" - - key = hashkey(*args, **kwargs) - key += tuple(type(v) for v in args) - key += tuple(type(v) for _, v in sorted(kwargs.items())) - return key diff --git a/src/cachetools/lfu.py b/src/cachetools/lfu.py deleted file mode 100644 index 4857c4e9..00000000 --- a/src/cachetools/lfu.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import absolute_import - -import collections - -from .cache import Cache - - -class LFUCache(Cache): - """Least Frequently Used (LFU) cache implementation.""" - - def __init__(self, maxsize, getsizeof=None): - Cache.__init__(self, maxsize, getsizeof) - self.__counter = collections.Counter() - - def __getitem__(self, key, cache_getitem=Cache.__getitem__): - value = cache_getitem(self, key) - self.__counter[key] -= 1 - return value - - def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): - cache_setitem(self, key, value) - self.__counter[key] -= 1 - - def __delitem__(self, key, cache_delitem=Cache.__delitem__): - cache_delitem(self, key) - del self.__counter[key] - - def popitem(self): - """Remove and return the `(key, value)` pair least frequently used.""" - try: - (key, _), = self.__counter.most_common(1) - except ValueError: - raise KeyError('%s is empty' % self.__class__.__name__) - else: - return (key, self.pop(key)) diff --git a/src/cachetools/lru.py b/src/cachetools/lru.py deleted file mode 100644 index 44ec4f1c..00000000 --- a/src/cachetools/lru.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import absolute_import - -import collections - -from .cache import Cache - - -class LRUCache(Cache): - """Least Recently Used (LRU) cache implementation.""" - - def __init__(self, maxsize, getsizeof=None): - Cache.__init__(self, maxsize, getsizeof) - self.__order = collections.OrderedDict() - - def __getitem__(self, key, cache_getitem=Cache.__getitem__): - value = cache_getitem(self, key) - self.__update(key) - return value - - def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): - cache_setitem(self, key, value) - self.__update(key) - - def __delitem__(self, key, cache_delitem=Cache.__delitem__): - cache_delitem(self, key) - del self.__order[key] - - def popitem(self): - """Remove and return the `(key, value)` pair least recently used.""" - try: - key = next(iter(self.__order)) - except StopIteration: - raise KeyError('%s is empty' % self.__class__.__name__) - else: - return (key, self.pop(key)) - - if hasattr(collections.OrderedDict, 'move_to_end'): - def __update(self, key): - try: - self.__order.move_to_end(key) - except KeyError: - self.__order[key] = None - else: - def __update(self, key): - try: - self.__order[key] = self.__order.pop(key) - except KeyError: - self.__order[key] = None diff --git a/src/cachetools/rr.py b/src/cachetools/rr.py deleted file mode 100644 index 09ff7708..00000000 --- a/src/cachetools/rr.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import absolute_import - -import random - -from .cache import Cache - - -# random.choice cannot be pickled in Python 2.7 -def _choice(seq): - return random.choice(seq) - - -class RRCache(Cache): - """Random Replacement (RR) cache implementation.""" - - def __init__(self, maxsize, choice=random.choice, getsizeof=None): - Cache.__init__(self, maxsize, getsizeof) - # TODO: use None as default, assing to self.choice directly? - if choice is random.choice: - self.__choice = _choice - else: - self.__choice = choice - - @property - def choice(self): - """The `choice` function used by the cache.""" - return self.__choice - - def popitem(self): - """Remove and return a random `(key, value)` pair.""" - try: - key = self.__choice(list(self)) - except IndexError: - raise KeyError('%s is empty' % self.__class__.__name__) - else: - return (key, self.pop(key)) diff --git a/src/cachetools/ttl.py b/src/cachetools/ttl.py deleted file mode 100644 index 1edde3ab..00000000 --- a/src/cachetools/ttl.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import absolute_import - -import collections - -try: - from time import monotonic as default_timer -except ImportError: - from time import time as default_timer - -from .cache import Cache - - -class _Link(object): - - __slots__ = ('key', 'expire', 'next', 'prev') - - def __init__(self, key=None, expire=None): - self.key = key - self.expire = expire - - def __reduce__(self): - return _Link, (self.key, self.expire) - - def unlink(self): - next = self.next - prev = self.prev - prev.next = next - next.prev = prev - - -class _Timer(object): - - def __init__(self, timer): - self.__timer = timer - self.__nesting = 0 - - def __call__(self): - if self.__nesting == 0: - return self.__timer() - else: - return self.__time - - def __enter__(self): - if self.__nesting == 0: - self.__time = time = self.__timer() - else: - time = self.__time - self.__nesting += 1 - return time - - def __exit__(self, *exc): - self.__nesting -= 1 - - def __reduce__(self): - return _Timer, (self.__timer,) - - def __getattr__(self, name): - return getattr(self.__timer, name) - - -class TTLCache(Cache): - """LRU Cache implementation with per-item time-to-live (TTL) value.""" - - def __init__(self, maxsize, ttl, timer=default_timer, getsizeof=None): - Cache.__init__(self, maxsize, getsizeof) - self.__root = root = _Link() - root.prev = root.next = root - self.__links = collections.OrderedDict() - self.__timer = _Timer(timer) - self.__ttl = ttl - - def __contains__(self, key): - try: - link = self.__links[key] # no reordering - except KeyError: - return False - else: - return not (link.expire < self.__timer()) - - def __getitem__(self, key, cache_getitem=Cache.__getitem__): - try: - link = self.__getlink(key) - except KeyError: - expired = False - else: - expired = link.expire < self.__timer() - if expired: - return self.__missing__(key) - else: - return cache_getitem(self, key) - - def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): - with self.__timer as time: - self.expire(time) - cache_setitem(self, key, value) - try: - link = self.__getlink(key) - except KeyError: - self.__links[key] = link = _Link(key) - else: - link.unlink() - link.expire = time + self.__ttl - link.next = root = self.__root - link.prev = prev = root.prev - prev.next = root.prev = link - - def __delitem__(self, key, cache_delitem=Cache.__delitem__): - cache_delitem(self, key) - link = self.__links.pop(key) - link.unlink() - if link.expire < self.__timer(): - raise KeyError(key) - - def __iter__(self): - root = self.__root - curr = root.next - while curr is not root: - # "freeze" time for iterator access - with self.__timer as time: - if not (curr.expire < time): - yield curr.key - curr = curr.next - - def __len__(self): - root = self.__root - curr = root.next - time = self.__timer() - count = len(self.__links) - while curr is not root and curr.expire < time: - count -= 1 - curr = curr.next - return count - - def __setstate__(self, state): - self.__dict__.update(state) - root = self.__root - root.prev = root.next = root - for link in sorted(self.__links.values(), key=lambda obj: obj.expire): - link.next = root - link.prev = prev = root.prev - prev.next = root.prev = link - self.expire(self.__timer()) - - def __repr__(self, cache_repr=Cache.__repr__): - with self.__timer as time: - self.expire(time) - return cache_repr(self) - - @property - def currsize(self): - with self.__timer as time: - self.expire(time) - return super(TTLCache, self).currsize - - @property - def timer(self): - """The timer function used by the cache.""" - return self.__timer - - @property - def ttl(self): - """The time-to-live value of the cache's items.""" - return self.__ttl - - def expire(self, time=None): - """Remove expired items from the cache.""" - if time is None: - time = self.__timer() - root = self.__root - curr = root.next - links = self.__links - cache_delitem = Cache.__delitem__ - while curr is not root and curr.expire < time: - cache_delitem(self, curr.key) - del links[curr.key] - next = curr.next - curr.unlink() - curr = next - - def clear(self): - with self.__timer as time: - self.expire(time) - Cache.clear(self) - - def get(self, *args, **kwargs): - with self.__timer: - return Cache.get(self, *args, **kwargs) - - def pop(self, *args, **kwargs): - with self.__timer: - return Cache.pop(self, *args, **kwargs) - - def setdefault(self, *args, **kwargs): - with self.__timer: - return Cache.setdefault(self, *args, **kwargs) - - def popitem(self): - """Remove and return the `(key, value)` pair least recently used that - has not already expired. - - """ - with self.__timer as time: - self.expire(time) - try: - key = next(iter(self.__links)) - except StopIteration: - raise KeyError('%s is empty' % self.__class__.__name__) - else: - return (key, self.pop(key)) - - if hasattr(collections.OrderedDict, 'move_to_end'): - def __getlink(self, key): - value = self.__links[key] - self.__links.move_to_end(key) - return value - else: - def __getlink(self, key): - value = self.__links.pop(key) - self.__links[key] = value - return value diff --git a/src/dateutil/__init__.py b/src/dateutil/__init__.py deleted file mode 100644 index 0defb82e..00000000 --- a/src/dateutil/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -try: - from ._version import version as __version__ -except ImportError: - __version__ = 'unknown' - -__all__ = ['easter', 'parser', 'relativedelta', 'rrule', 'tz', - 'utils', 'zoneinfo'] diff --git a/src/dateutil/_common.py b/src/dateutil/_common.py deleted file mode 100644 index 4eb2659b..00000000 --- a/src/dateutil/_common.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Common code used in multiple modules. -""" - - -class weekday(object): - __slots__ = ["weekday", "n"] - - def __init__(self, weekday, n=None): - self.weekday = weekday - self.n = n - - def __call__(self, n): - if n == self.n: - return self - else: - return self.__class__(self.weekday, n) - - def __eq__(self, other): - try: - if self.weekday != other.weekday or self.n != other.n: - return False - except AttributeError: - return False - return True - - def __hash__(self): - return hash(( - self.weekday, - self.n, - )) - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - s = ("MO", "TU", "WE", "TH", "FR", "SA", "SU")[self.weekday] - if not self.n: - return s - else: - return "%s(%+d)" % (s, self.n) - -# vim:ts=4:sw=4:et diff --git a/src/dateutil/_version.py b/src/dateutil/_version.py deleted file mode 100644 index 670d7ab7..00000000 --- a/src/dateutil/_version.py +++ /dev/null @@ -1,4 +0,0 @@ -# coding: utf-8 -# file generated by setuptools_scm -# don't change, don't track in version control -version = '2.8.0' diff --git a/src/dateutil/easter.py b/src/dateutil/easter.py deleted file mode 100644 index 53b7c789..00000000 --- a/src/dateutil/easter.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module offers a generic easter computing method for any given year, using -Western, Orthodox or Julian algorithms. -""" - -import datetime - -__all__ = ["easter", "EASTER_JULIAN", "EASTER_ORTHODOX", "EASTER_WESTERN"] - -EASTER_JULIAN = 1 -EASTER_ORTHODOX = 2 -EASTER_WESTERN = 3 - - -def easter(year, method=EASTER_WESTERN): - """ - This method was ported from the work done by GM Arts, - on top of the algorithm by Claus Tondering, which was - based in part on the algorithm of Ouding (1940), as - quoted in "Explanatory Supplement to the Astronomical - Almanac", P. Kenneth Seidelmann, editor. - - This algorithm implements three different easter - calculation methods: - - 1 - Original calculation in Julian calendar, valid in - dates after 326 AD - 2 - Original method, with date converted to Gregorian - calendar, valid in years 1583 to 4099 - 3 - Revised method, in Gregorian calendar, valid in - years 1583 to 4099 as well - - These methods are represented by the constants: - - * ``EASTER_JULIAN = 1`` - * ``EASTER_ORTHODOX = 2`` - * ``EASTER_WESTERN = 3`` - - The default method is method 3. - - More about the algorithm may be found at: - - `GM Arts: Easter Algorithms `_ - - and - - `The Calendar FAQ: Easter `_ - - """ - - if not (1 <= method <= 3): - raise ValueError("invalid method") - - # g - Golden year - 1 - # c - Century - # h - (23 - Epact) mod 30 - # i - Number of days from March 21 to Paschal Full Moon - # j - Weekday for PFM (0=Sunday, etc) - # p - Number of days from March 21 to Sunday on or before PFM - # (-6 to 28 methods 1 & 3, to 56 for method 2) - # e - Extra days to add for method 2 (converting Julian - # date to Gregorian date) - - y = year - g = y % 19 - e = 0 - if method < 3: - # Old method - i = (19*g + 15) % 30 - j = (y + y//4 + i) % 7 - if method == 2: - # Extra dates to convert Julian to Gregorian date - e = 10 - if y > 1600: - e = e + y//100 - 16 - (y//100 - 16)//4 - else: - # New method - c = y//100 - h = (c - c//4 - (8*c + 13)//25 + 19*g + 15) % 30 - i = h - (h//28)*(1 - (h//28)*(29//(h + 1))*((21 - g)//11)) - j = (y + y//4 + i + 2 - c + c//4) % 7 - - # p can be from -6 to 56 corresponding to dates 22 March to 23 May - # (later dates apply to method 2, although 23 May never actually occurs) - p = i - j + e - d = 1 + (p + 27 + (p + 6)//40) % 31 - m = 3 + (p + 26)//30 - return datetime.date(int(y), int(m), int(d)) diff --git a/src/dateutil/parser/__init__.py b/src/dateutil/parser/__init__.py deleted file mode 100644 index 216762c0..00000000 --- a/src/dateutil/parser/__init__.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -from ._parser import parse, parser, parserinfo -from ._parser import DEFAULTPARSER, DEFAULTTZPARSER -from ._parser import UnknownTimezoneWarning - -from ._parser import __doc__ - -from .isoparser import isoparser, isoparse - -__all__ = ['parse', 'parser', 'parserinfo', - 'isoparse', 'isoparser', - 'UnknownTimezoneWarning'] - - -### -# Deprecate portions of the private interface so that downstream code that -# is improperly relying on it is given *some* notice. - - -def __deprecated_private_func(f): - from functools import wraps - import warnings - - msg = ('{name} is a private function and may break without warning, ' - 'it will be moved and or renamed in future versions.') - msg = msg.format(name=f.__name__) - - @wraps(f) - def deprecated_func(*args, **kwargs): - warnings.warn(msg, DeprecationWarning) - return f(*args, **kwargs) - - return deprecated_func - -def __deprecate_private_class(c): - import warnings - - msg = ('{name} is a private class and may break without warning, ' - 'it will be moved and or renamed in future versions.') - msg = msg.format(name=c.__name__) - - class private_class(c): - __doc__ = c.__doc__ - - def __init__(self, *args, **kwargs): - warnings.warn(msg, DeprecationWarning) - super(private_class, self).__init__(*args, **kwargs) - - private_class.__name__ = c.__name__ - - return private_class - - -from ._parser import _timelex, _resultbase -from ._parser import _tzparser, _parsetz - -_timelex = __deprecate_private_class(_timelex) -_tzparser = __deprecate_private_class(_tzparser) -_resultbase = __deprecate_private_class(_resultbase) -_parsetz = __deprecated_private_func(_parsetz) diff --git a/src/dateutil/parser/_parser.py b/src/dateutil/parser/_parser.py deleted file mode 100644 index 0da0f3e6..00000000 --- a/src/dateutil/parser/_parser.py +++ /dev/null @@ -1,1580 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module offers a generic date/time string parser which is able to parse -most known formats to represent a date and/or time. - -This module attempts to be forgiving with regards to unlikely input formats, -returning a datetime object even for dates which are ambiguous. If an element -of a date/time stamp is omitted, the following rules are applied: - -- If AM or PM is left unspecified, a 24-hour clock is assumed, however, an hour - on a 12-hour clock (``0 <= hour <= 12``) *must* be specified if AM or PM is - specified. -- If a time zone is omitted, a timezone-naive datetime is returned. - -If any other elements are missing, they are taken from the -:class:`datetime.datetime` object passed to the parameter ``default``. If this -results in a day number exceeding the valid number of days per month, the -value falls back to the end of the month. - -Additional resources about date/time string formats can be found below: - -- `A summary of the international standard date and time notation - `_ -- `W3C Date and Time Formats `_ -- `Time Formats (Planetary Rings Node) `_ -- `CPAN ParseDate module - `_ -- `Java SimpleDateFormat Class - `_ -""" -from __future__ import unicode_literals - -import datetime -import re -import string -import time -import warnings - -from calendar import monthrange -from io import StringIO - -import six -from six import integer_types, text_type - -from decimal import Decimal - -from warnings import warn - -from .. import relativedelta -from .. import tz - -__all__ = ["parse", "parserinfo"] - - -# TODO: pandas.core.tools.datetimes imports this explicitly. Might be worth -# making public and/or figuring out if there is something we can -# take off their plate. -class _timelex(object): - # Fractional seconds are sometimes split by a comma - _split_decimal = re.compile("([.,])") - - def __init__(self, instream): - if six.PY2: - # In Python 2, we can't duck type properly because unicode has - # a 'decode' function, and we'd be double-decoding - if isinstance(instream, (bytes, bytearray)): - instream = instream.decode() - else: - if getattr(instream, 'decode', None) is not None: - instream = instream.decode() - - if isinstance(instream, text_type): - instream = StringIO(instream) - elif getattr(instream, 'read', None) is None: - raise TypeError('Parser must be a string or character stream, not ' - '{itype}'.format(itype=instream.__class__.__name__)) - - self.instream = instream - self.charstack = [] - self.tokenstack = [] - self.eof = False - - def get_token(self): - """ - This function breaks the time string into lexical units (tokens), which - can be parsed by the parser. Lexical units are demarcated by changes in - the character set, so any continuous string of letters is considered - one unit, any continuous string of numbers is considered one unit. - - The main complication arises from the fact that dots ('.') can be used - both as separators (e.g. "Sep.20.2009") or decimal points (e.g. - "4:30:21.447"). As such, it is necessary to read the full context of - any dot-separated strings before breaking it into tokens; as such, this - function maintains a "token stack", for when the ambiguous context - demands that multiple tokens be parsed at once. - """ - if self.tokenstack: - return self.tokenstack.pop(0) - - seenletters = False - token = None - state = None - - while not self.eof: - # We only realize that we've reached the end of a token when we - # find a character that's not part of the current token - since - # that character may be part of the next token, it's stored in the - # charstack. - if self.charstack: - nextchar = self.charstack.pop(0) - else: - nextchar = self.instream.read(1) - while nextchar == '\x00': - nextchar = self.instream.read(1) - - if not nextchar: - self.eof = True - break - elif not state: - # First character of the token - determines if we're starting - # to parse a word, a number or something else. - token = nextchar - if self.isword(nextchar): - state = 'a' - elif self.isnum(nextchar): - state = '0' - elif self.isspace(nextchar): - token = ' ' - break # emit token - else: - break # emit token - elif state == 'a': - # If we've already started reading a word, we keep reading - # letters until we find something that's not part of a word. - seenletters = True - if self.isword(nextchar): - token += nextchar - elif nextchar == '.': - token += nextchar - state = 'a.' - else: - self.charstack.append(nextchar) - break # emit token - elif state == '0': - # If we've already started reading a number, we keep reading - # numbers until we find something that doesn't fit. - if self.isnum(nextchar): - token += nextchar - elif nextchar == '.' or (nextchar == ',' and len(token) >= 2): - token += nextchar - state = '0.' - else: - self.charstack.append(nextchar) - break # emit token - elif state == 'a.': - # If we've seen some letters and a dot separator, continue - # parsing, and the tokens will be broken up later. - seenletters = True - if nextchar == '.' or self.isword(nextchar): - token += nextchar - elif self.isnum(nextchar) and token[-1] == '.': - token += nextchar - state = '0.' - else: - self.charstack.append(nextchar) - break # emit token - elif state == '0.': - # If we've seen at least one dot separator, keep going, we'll - # break up the tokens later. - if nextchar == '.' or self.isnum(nextchar): - token += nextchar - elif self.isword(nextchar) and token[-1] == '.': - token += nextchar - state = 'a.' - else: - self.charstack.append(nextchar) - break # emit token - - if (state in ('a.', '0.') and (seenletters or token.count('.') > 1 or - token[-1] in '.,')): - l = self._split_decimal.split(token) - token = l[0] - for tok in l[1:]: - if tok: - self.tokenstack.append(tok) - - if state == '0.' and token.count('.') == 0: - token = token.replace(',', '.') - - return token - - def __iter__(self): - return self - - def __next__(self): - token = self.get_token() - if token is None: - raise StopIteration - - return token - - def next(self): - return self.__next__() # Python 2.x support - - @classmethod - def split(cls, s): - return list(cls(s)) - - @classmethod - def isword(cls, nextchar): - """ Whether or not the next character is part of a word """ - return nextchar.isalpha() - - @classmethod - def isnum(cls, nextchar): - """ Whether the next character is part of a number """ - return nextchar.isdigit() - - @classmethod - def isspace(cls, nextchar): - """ Whether the next character is whitespace """ - return nextchar.isspace() - - -class _resultbase(object): - - def __init__(self): - for attr in self.__slots__: - setattr(self, attr, None) - - def _repr(self, classname): - l = [] - for attr in self.__slots__: - value = getattr(self, attr) - if value is not None: - l.append("%s=%s" % (attr, repr(value))) - return "%s(%s)" % (classname, ", ".join(l)) - - def __len__(self): - return (sum(getattr(self, attr) is not None - for attr in self.__slots__)) - - def __repr__(self): - return self._repr(self.__class__.__name__) - - -class parserinfo(object): - """ - Class which handles what inputs are accepted. Subclass this to customize - the language and acceptable values for each parameter. - - :param dayfirst: - Whether to interpret the first value in an ambiguous 3-integer date - (e.g. 01/05/09) as the day (``True``) or month (``False``). If - ``yearfirst`` is set to ``True``, this distinguishes between YDM - and YMD. Default is ``False``. - - :param yearfirst: - Whether to interpret the first value in an ambiguous 3-integer date - (e.g. 01/05/09) as the year. If ``True``, the first number is taken - to be the year, otherwise the last number is taken to be the year. - Default is ``False``. - """ - - # m from a.m/p.m, t from ISO T separator - JUMP = [" ", ".", ",", ";", "-", "/", "'", - "at", "on", "and", "ad", "m", "t", "of", - "st", "nd", "rd", "th"] - - WEEKDAYS = [("Mon", "Monday"), - ("Tue", "Tuesday"), # TODO: "Tues" - ("Wed", "Wednesday"), - ("Thu", "Thursday"), # TODO: "Thurs" - ("Fri", "Friday"), - ("Sat", "Saturday"), - ("Sun", "Sunday")] - MONTHS = [("Jan", "January"), - ("Feb", "February"), # TODO: "Febr" - ("Mar", "March"), - ("Apr", "April"), - ("May", "May"), - ("Jun", "June"), - ("Jul", "July"), - ("Aug", "August"), - ("Sep", "Sept", "September"), - ("Oct", "October"), - ("Nov", "November"), - ("Dec", "December")] - HMS = [("h", "hour", "hours"), - ("m", "minute", "minutes"), - ("s", "second", "seconds")] - AMPM = [("am", "a"), - ("pm", "p")] - UTCZONE = ["UTC", "GMT", "Z", "z"] - PERTAIN = ["of"] - TZOFFSET = {} - # TODO: ERA = ["AD", "BC", "CE", "BCE", "Stardate", - # "Anno Domini", "Year of Our Lord"] - - def __init__(self, dayfirst=False, yearfirst=False): - self._jump = self._convert(self.JUMP) - self._weekdays = self._convert(self.WEEKDAYS) - self._months = self._convert(self.MONTHS) - self._hms = self._convert(self.HMS) - self._ampm = self._convert(self.AMPM) - self._utczone = self._convert(self.UTCZONE) - self._pertain = self._convert(self.PERTAIN) - - self.dayfirst = dayfirst - self.yearfirst = yearfirst - - self._year = time.localtime().tm_year - self._century = self._year // 100 * 100 - - def _convert(self, lst): - dct = {} - for i, v in enumerate(lst): - if isinstance(v, tuple): - for v in v: - dct[v.lower()] = i - else: - dct[v.lower()] = i - return dct - - def jump(self, name): - return name.lower() in self._jump - - def weekday(self, name): - try: - return self._weekdays[name.lower()] - except KeyError: - pass - return None - - def month(self, name): - try: - return self._months[name.lower()] + 1 - except KeyError: - pass - return None - - def hms(self, name): - try: - return self._hms[name.lower()] - except KeyError: - return None - - def ampm(self, name): - try: - return self._ampm[name.lower()] - except KeyError: - return None - - def pertain(self, name): - return name.lower() in self._pertain - - def utczone(self, name): - return name.lower() in self._utczone - - def tzoffset(self, name): - if name in self._utczone: - return 0 - - return self.TZOFFSET.get(name) - - def convertyear(self, year, century_specified=False): - """ - Converts two-digit years to year within [-50, 49] - range of self._year (current local time) - """ - - # Function contract is that the year is always positive - assert year >= 0 - - if year < 100 and not century_specified: - # assume current century to start - year += self._century - - if year >= self._year + 50: # if too far in future - year -= 100 - elif year < self._year - 50: # if too far in past - year += 100 - - return year - - def validate(self, res): - # move to info - if res.year is not None: - res.year = self.convertyear(res.year, res.century_specified) - - if ((res.tzoffset == 0 and not res.tzname) or - (res.tzname == 'Z' or res.tzname == 'z')): - res.tzname = "UTC" - res.tzoffset = 0 - elif res.tzoffset != 0 and res.tzname and self.utczone(res.tzname): - res.tzoffset = 0 - return True - - -class _ymd(list): - def __init__(self, *args, **kwargs): - super(self.__class__, self).__init__(*args, **kwargs) - self.century_specified = False - self.dstridx = None - self.mstridx = None - self.ystridx = None - - @property - def has_year(self): - return self.ystridx is not None - - @property - def has_month(self): - return self.mstridx is not None - - @property - def has_day(self): - return self.dstridx is not None - - def could_be_day(self, value): - if self.has_day: - return False - elif not self.has_month: - return 1 <= value <= 31 - elif not self.has_year: - # Be permissive, assume leapyear - month = self[self.mstridx] - return 1 <= value <= monthrange(2000, month)[1] - else: - month = self[self.mstridx] - year = self[self.ystridx] - return 1 <= value <= monthrange(year, month)[1] - - def append(self, val, label=None): - if hasattr(val, '__len__'): - if val.isdigit() and len(val) > 2: - self.century_specified = True - if label not in [None, 'Y']: # pragma: no cover - raise ValueError(label) - label = 'Y' - elif val > 100: - self.century_specified = True - if label not in [None, 'Y']: # pragma: no cover - raise ValueError(label) - label = 'Y' - - super(self.__class__, self).append(int(val)) - - if label == 'M': - if self.has_month: - raise ValueError('Month is already set') - self.mstridx = len(self) - 1 - elif label == 'D': - if self.has_day: - raise ValueError('Day is already set') - self.dstridx = len(self) - 1 - elif label == 'Y': - if self.has_year: - raise ValueError('Year is already set') - self.ystridx = len(self) - 1 - - def _resolve_from_stridxs(self, strids): - """ - Try to resolve the identities of year/month/day elements using - ystridx, mstridx, and dstridx, if enough of these are specified. - """ - if len(self) == 3 and len(strids) == 2: - # we can back out the remaining stridx value - missing = [x for x in range(3) if x not in strids.values()] - key = [x for x in ['y', 'm', 'd'] if x not in strids] - assert len(missing) == len(key) == 1 - key = key[0] - val = missing[0] - strids[key] = val - - assert len(self) == len(strids) # otherwise this should not be called - out = {key: self[strids[key]] for key in strids} - return (out.get('y'), out.get('m'), out.get('d')) - - def resolve_ymd(self, yearfirst, dayfirst): - len_ymd = len(self) - year, month, day = (None, None, None) - - strids = (('y', self.ystridx), - ('m', self.mstridx), - ('d', self.dstridx)) - - strids = {key: val for key, val in strids if val is not None} - if (len(self) == len(strids) > 0 or - (len(self) == 3 and len(strids) == 2)): - return self._resolve_from_stridxs(strids) - - mstridx = self.mstridx - - if len_ymd > 3: - raise ValueError("More than three YMD values") - elif len_ymd == 1 or (mstridx is not None and len_ymd == 2): - # One member, or two members with a month string - if mstridx is not None: - month = self[mstridx] - # since mstridx is 0 or 1, self[mstridx-1] always - # looks up the other element - other = self[mstridx - 1] - else: - other = self[0] - - if len_ymd > 1 or mstridx is None: - if other > 31: - year = other - else: - day = other - - elif len_ymd == 2: - # Two members with numbers - if self[0] > 31: - # 99-01 - year, month = self - elif self[1] > 31: - # 01-99 - month, year = self - elif dayfirst and self[1] <= 12: - # 13-01 - day, month = self - else: - # 01-13 - month, day = self - - elif len_ymd == 3: - # Three members - if mstridx == 0: - if self[1] > 31: - # Apr-2003-25 - month, year, day = self - else: - month, day, year = self - elif mstridx == 1: - if self[0] > 31 or (yearfirst and self[2] <= 31): - # 99-Jan-01 - year, month, day = self - else: - # 01-Jan-01 - # Give precendence to day-first, since - # two-digit years is usually hand-written. - day, month, year = self - - elif mstridx == 2: - # WTF!? - if self[1] > 31: - # 01-99-Jan - day, year, month = self - else: - # 99-01-Jan - year, day, month = self - - else: - if (self[0] > 31 or - self.ystridx == 0 or - (yearfirst and self[1] <= 12 and self[2] <= 31)): - # 99-01-01 - if dayfirst and self[2] <= 12: - year, day, month = self - else: - year, month, day = self - elif self[0] > 12 or (dayfirst and self[1] <= 12): - # 13-01-01 - day, month, year = self - else: - # 01-13-01 - month, day, year = self - - return year, month, day - - -class parser(object): - def __init__(self, info=None): - self.info = info or parserinfo() - - def parse(self, timestr, default=None, - ignoretz=False, tzinfos=None, **kwargs): - """ - Parse the date/time string into a :class:`datetime.datetime` object. - - :param timestr: - Any date/time string using the supported formats. - - :param default: - The default datetime object, if this is a datetime object and not - ``None``, elements specified in ``timestr`` replace elements in the - default object. - - :param ignoretz: - If set ``True``, time zones in parsed strings are ignored and a - naive :class:`datetime.datetime` object is returned. - - :param tzinfos: - Additional time zone names / aliases which may be present in the - string. This argument maps time zone names (and optionally offsets - from those time zones) to time zones. This parameter can be a - dictionary with timezone aliases mapping time zone names to time - zones or a function taking two parameters (``tzname`` and - ``tzoffset``) and returning a time zone. - - The timezones to which the names are mapped can be an integer - offset from UTC in seconds or a :class:`tzinfo` object. - - .. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> from dateutil.parser import parse - >>> from dateutil.tz import gettz - >>> tzinfos = {"BRST": -7200, "CST": gettz("America/Chicago")} - >>> parse("2012-01-19 17:21:00 BRST", tzinfos=tzinfos) - datetime.datetime(2012, 1, 19, 17, 21, tzinfo=tzoffset(u'BRST', -7200)) - >>> parse("2012-01-19 17:21:00 CST", tzinfos=tzinfos) - datetime.datetime(2012, 1, 19, 17, 21, - tzinfo=tzfile('/usr/share/zoneinfo/America/Chicago')) - - This parameter is ignored if ``ignoretz`` is set. - - :param \\*\\*kwargs: - Keyword arguments as passed to ``_parse()``. - - :return: - Returns a :class:`datetime.datetime` object or, if the - ``fuzzy_with_tokens`` option is ``True``, returns a tuple, the - first element being a :class:`datetime.datetime` object, the second - a tuple containing the fuzzy tokens. - - :raises ValueError: - Raised for invalid or unknown string format, if the provided - :class:`tzinfo` is not in a valid format, or if an invalid date - would be created. - - :raises TypeError: - Raised for non-string or character stream input. - - :raises OverflowError: - Raised if the parsed date exceeds the largest valid C integer on - your system. - """ - - if default is None: - default = datetime.datetime.now().replace(hour=0, minute=0, - second=0, microsecond=0) - - res, skipped_tokens = self._parse(timestr, **kwargs) - - if res is None: - raise ValueError("Unknown string format:", timestr) - - if len(res) == 0: - raise ValueError("String does not contain a date:", timestr) - - ret = self._build_naive(res, default) - - if not ignoretz: - ret = self._build_tzaware(ret, res, tzinfos) - - if kwargs.get('fuzzy_with_tokens', False): - return ret, skipped_tokens - else: - return ret - - class _result(_resultbase): - __slots__ = ["year", "month", "day", "weekday", - "hour", "minute", "second", "microsecond", - "tzname", "tzoffset", "ampm","any_unused_tokens"] - - def _parse(self, timestr, dayfirst=None, yearfirst=None, fuzzy=False, - fuzzy_with_tokens=False): - """ - Private method which performs the heavy lifting of parsing, called from - ``parse()``, which passes on its ``kwargs`` to this function. - - :param timestr: - The string to parse. - - :param dayfirst: - Whether to interpret the first value in an ambiguous 3-integer date - (e.g. 01/05/09) as the day (``True``) or month (``False``). If - ``yearfirst`` is set to ``True``, this distinguishes between YDM - and YMD. If set to ``None``, this value is retrieved from the - current :class:`parserinfo` object (which itself defaults to - ``False``). - - :param yearfirst: - Whether to interpret the first value in an ambiguous 3-integer date - (e.g. 01/05/09) as the year. If ``True``, the first number is taken - to be the year, otherwise the last number is taken to be the year. - If this is set to ``None``, the value is retrieved from the current - :class:`parserinfo` object (which itself defaults to ``False``). - - :param fuzzy: - Whether to allow fuzzy parsing, allowing for string like "Today is - January 1, 2047 at 8:21:00AM". - - :param fuzzy_with_tokens: - If ``True``, ``fuzzy`` is automatically set to True, and the parser - will return a tuple where the first element is the parsed - :class:`datetime.datetime` datetimestamp and the second element is - a tuple containing the portions of the string which were ignored: - - .. doctest:: - - >>> from dateutil.parser import parse - >>> parse("Today is January 1, 2047 at 8:21:00AM", fuzzy_with_tokens=True) - (datetime.datetime(2047, 1, 1, 8, 21), (u'Today is ', u' ', u'at ')) - - """ - if fuzzy_with_tokens: - fuzzy = True - - info = self.info - - if dayfirst is None: - dayfirst = info.dayfirst - - if yearfirst is None: - yearfirst = info.yearfirst - - res = self._result() - l = _timelex.split(timestr) # Splits the timestr into tokens - - skipped_idxs = [] - - # year/month/day list - ymd = _ymd() - - len_l = len(l) - i = 0 - try: - while i < len_l: - - # Check if it's a number - value_repr = l[i] - try: - value = float(value_repr) - except ValueError: - value = None - - if value is not None: - # Numeric token - i = self._parse_numeric_token(l, i, info, ymd, res, fuzzy) - - # Check weekday - elif info.weekday(l[i]) is not None: - value = info.weekday(l[i]) - res.weekday = value - - # Check month name - elif info.month(l[i]) is not None: - value = info.month(l[i]) - ymd.append(value, 'M') - - if i + 1 < len_l: - if l[i + 1] in ('-', '/'): - # Jan-01[-99] - sep = l[i + 1] - ymd.append(l[i + 2]) - - if i + 3 < len_l and l[i + 3] == sep: - # Jan-01-99 - ymd.append(l[i + 4]) - i += 2 - - i += 2 - - elif (i + 4 < len_l and l[i + 1] == l[i + 3] == ' ' and - info.pertain(l[i + 2])): - # Jan of 01 - # In this case, 01 is clearly year - if l[i + 4].isdigit(): - # Convert it here to become unambiguous - value = int(l[i + 4]) - year = str(info.convertyear(value)) - ymd.append(year, 'Y') - else: - # Wrong guess - pass - # TODO: not hit in tests - i += 4 - - # Check am/pm - elif info.ampm(l[i]) is not None: - value = info.ampm(l[i]) - val_is_ampm = self._ampm_valid(res.hour, res.ampm, fuzzy) - - if val_is_ampm: - res.hour = self._adjust_ampm(res.hour, value) - res.ampm = value - - elif fuzzy: - skipped_idxs.append(i) - - # Check for a timezone name - elif self._could_be_tzname(res.hour, res.tzname, res.tzoffset, l[i]): - res.tzname = l[i] - res.tzoffset = info.tzoffset(res.tzname) - - # Check for something like GMT+3, or BRST+3. Notice - # that it doesn't mean "I am 3 hours after GMT", but - # "my time +3 is GMT". If found, we reverse the - # logic so that timezone parsing code will get it - # right. - if i + 1 < len_l and l[i + 1] in ('+', '-'): - l[i + 1] = ('+', '-')[l[i + 1] == '+'] - res.tzoffset = None - if info.utczone(res.tzname): - # With something like GMT+3, the timezone - # is *not* GMT. - res.tzname = None - - # Check for a numbered timezone - elif res.hour is not None and l[i] in ('+', '-'): - signal = (-1, 1)[l[i] == '+'] - len_li = len(l[i + 1]) - - # TODO: check that l[i + 1] is integer? - if len_li == 4: - # -0300 - hour_offset = int(l[i + 1][:2]) - min_offset = int(l[i + 1][2:]) - elif i + 2 < len_l and l[i + 2] == ':': - # -03:00 - hour_offset = int(l[i + 1]) - min_offset = int(l[i + 3]) # TODO: Check that l[i+3] is minute-like? - i += 2 - elif len_li <= 2: - # -[0]3 - hour_offset = int(l[i + 1][:2]) - min_offset = 0 - else: - raise ValueError(timestr) - - res.tzoffset = signal * (hour_offset * 3600 + min_offset * 60) - - # Look for a timezone name between parenthesis - if (i + 5 < len_l and - info.jump(l[i + 2]) and l[i + 3] == '(' and - l[i + 5] == ')' and - 3 <= len(l[i + 4]) and - self._could_be_tzname(res.hour, res.tzname, - None, l[i + 4])): - # -0300 (BRST) - res.tzname = l[i + 4] - i += 4 - - i += 1 - - # Check jumps - elif not (info.jump(l[i]) or fuzzy): - raise ValueError(timestr) - - else: - skipped_idxs.append(i) - i += 1 - - # Process year/month/day - year, month, day = ymd.resolve_ymd(yearfirst, dayfirst) - - res.century_specified = ymd.century_specified - res.year = year - res.month = month - res.day = day - - except (IndexError, ValueError): - return None, None - - if not info.validate(res): - return None, None - - if fuzzy_with_tokens: - skipped_tokens = self._recombine_skipped(l, skipped_idxs) - return res, tuple(skipped_tokens) - else: - return res, None - - def _parse_numeric_token(self, tokens, idx, info, ymd, res, fuzzy): - # Token is a number - value_repr = tokens[idx] - try: - value = self._to_decimal(value_repr) - except Exception as e: - six.raise_from(ValueError('Unknown numeric token'), e) - - len_li = len(value_repr) - - len_l = len(tokens) - - if (len(ymd) == 3 and len_li in (2, 4) and - res.hour is None and - (idx + 1 >= len_l or - (tokens[idx + 1] != ':' and - info.hms(tokens[idx + 1]) is None))): - # 19990101T23[59] - s = tokens[idx] - res.hour = int(s[:2]) - - if len_li == 4: - res.minute = int(s[2:]) - - elif len_li == 6 or (len_li > 6 and tokens[idx].find('.') == 6): - # YYMMDD or HHMMSS[.ss] - s = tokens[idx] - - if not ymd and '.' not in tokens[idx]: - ymd.append(s[:2]) - ymd.append(s[2:4]) - ymd.append(s[4:]) - else: - # 19990101T235959[.59] - - # TODO: Check if res attributes already set. - res.hour = int(s[:2]) - res.minute = int(s[2:4]) - res.second, res.microsecond = self._parsems(s[4:]) - - elif len_li in (8, 12, 14): - # YYYYMMDD - s = tokens[idx] - ymd.append(s[:4], 'Y') - ymd.append(s[4:6]) - ymd.append(s[6:8]) - - if len_li > 8: - res.hour = int(s[8:10]) - res.minute = int(s[10:12]) - - if len_li > 12: - res.second = int(s[12:]) - - elif self._find_hms_idx(idx, tokens, info, allow_jump=True) is not None: - # HH[ ]h or MM[ ]m or SS[.ss][ ]s - hms_idx = self._find_hms_idx(idx, tokens, info, allow_jump=True) - (idx, hms) = self._parse_hms(idx, tokens, info, hms_idx) - if hms is not None: - # TODO: checking that hour/minute/second are not - # already set? - self._assign_hms(res, value_repr, hms) - - elif idx + 2 < len_l and tokens[idx + 1] == ':': - # HH:MM[:SS[.ss]] - res.hour = int(value) - value = self._to_decimal(tokens[idx + 2]) # TODO: try/except for this? - (res.minute, res.second) = self._parse_min_sec(value) - - if idx + 4 < len_l and tokens[idx + 3] == ':': - res.second, res.microsecond = self._parsems(tokens[idx + 4]) - - idx += 2 - - idx += 2 - - elif idx + 1 < len_l and tokens[idx + 1] in ('-', '/', '.'): - sep = tokens[idx + 1] - ymd.append(value_repr) - - if idx + 2 < len_l and not info.jump(tokens[idx + 2]): - if tokens[idx + 2].isdigit(): - # 01-01[-01] - ymd.append(tokens[idx + 2]) - else: - # 01-Jan[-01] - value = info.month(tokens[idx + 2]) - - if value is not None: - ymd.append(value, 'M') - else: - raise ValueError() - - if idx + 3 < len_l and tokens[idx + 3] == sep: - # We have three members - value = info.month(tokens[idx + 4]) - - if value is not None: - ymd.append(value, 'M') - else: - ymd.append(tokens[idx + 4]) - idx += 2 - - idx += 1 - idx += 1 - - elif idx + 1 >= len_l or info.jump(tokens[idx + 1]): - if idx + 2 < len_l and info.ampm(tokens[idx + 2]) is not None: - # 12 am - hour = int(value) - res.hour = self._adjust_ampm(hour, info.ampm(tokens[idx + 2])) - idx += 1 - else: - # Year, month or day - ymd.append(value) - idx += 1 - - elif info.ampm(tokens[idx + 1]) is not None and (0 <= value < 24): - # 12am - hour = int(value) - res.hour = self._adjust_ampm(hour, info.ampm(tokens[idx + 1])) - idx += 1 - - elif ymd.could_be_day(value): - ymd.append(value) - - elif not fuzzy: - raise ValueError() - - return idx - - def _find_hms_idx(self, idx, tokens, info, allow_jump): - len_l = len(tokens) - - if idx+1 < len_l and info.hms(tokens[idx+1]) is not None: - # There is an "h", "m", or "s" label following this token. We take - # assign the upcoming label to the current token. - # e.g. the "12" in 12h" - hms_idx = idx + 1 - - elif (allow_jump and idx+2 < len_l and tokens[idx+1] == ' ' and - info.hms(tokens[idx+2]) is not None): - # There is a space and then an "h", "m", or "s" label. - # e.g. the "12" in "12 h" - hms_idx = idx + 2 - - elif idx > 0 and info.hms(tokens[idx-1]) is not None: - # There is a "h", "m", or "s" preceeding this token. Since neither - # of the previous cases was hit, there is no label following this - # token, so we use the previous label. - # e.g. the "04" in "12h04" - hms_idx = idx-1 - - elif (1 < idx == len_l-1 and tokens[idx-1] == ' ' and - info.hms(tokens[idx-2]) is not None): - # If we are looking at the final token, we allow for a - # backward-looking check to skip over a space. - # TODO: Are we sure this is the right condition here? - hms_idx = idx - 2 - - else: - hms_idx = None - - return hms_idx - - def _assign_hms(self, res, value_repr, hms): - # See GH issue #427, fixing float rounding - value = self._to_decimal(value_repr) - - if hms == 0: - # Hour - res.hour = int(value) - if value % 1: - res.minute = int(60*(value % 1)) - - elif hms == 1: - (res.minute, res.second) = self._parse_min_sec(value) - - elif hms == 2: - (res.second, res.microsecond) = self._parsems(value_repr) - - def _could_be_tzname(self, hour, tzname, tzoffset, token): - return (hour is not None and - tzname is None and - tzoffset is None and - len(token) <= 5 and - (all(x in string.ascii_uppercase for x in token) - or token in self.info.UTCZONE)) - - def _ampm_valid(self, hour, ampm, fuzzy): - """ - For fuzzy parsing, 'a' or 'am' (both valid English words) - may erroneously trigger the AM/PM flag. Deal with that - here. - """ - val_is_ampm = True - - # If there's already an AM/PM flag, this one isn't one. - if fuzzy and ampm is not None: - val_is_ampm = False - - # If AM/PM is found and hour is not, raise a ValueError - if hour is None: - if fuzzy: - val_is_ampm = False - else: - raise ValueError('No hour specified with AM or PM flag.') - elif not 0 <= hour <= 12: - # If AM/PM is found, it's a 12 hour clock, so raise - # an error for invalid range - if fuzzy: - val_is_ampm = False - else: - raise ValueError('Invalid hour specified for 12-hour clock.') - - return val_is_ampm - - def _adjust_ampm(self, hour, ampm): - if hour < 12 and ampm == 1: - hour += 12 - elif hour == 12 and ampm == 0: - hour = 0 - return hour - - def _parse_min_sec(self, value): - # TODO: Every usage of this function sets res.second to the return - # value. Are there any cases where second will be returned as None and - # we *dont* want to set res.second = None? - minute = int(value) - second = None - - sec_remainder = value % 1 - if sec_remainder: - second = int(60 * sec_remainder) - return (minute, second) - - def _parsems(self, value): - """Parse a I[.F] seconds value into (seconds, microseconds).""" - if "." not in value: - return int(value), 0 - else: - i, f = value.split(".") - return int(i), int(f.ljust(6, "0")[:6]) - - def _parse_hms(self, idx, tokens, info, hms_idx): - # TODO: Is this going to admit a lot of false-positives for when we - # just happen to have digits and "h", "m" or "s" characters in non-date - # text? I guess hex hashes won't have that problem, but there's plenty - # of random junk out there. - if hms_idx is None: - hms = None - new_idx = idx - elif hms_idx > idx: - hms = info.hms(tokens[hms_idx]) - new_idx = hms_idx - else: - # Looking backwards, increment one. - hms = info.hms(tokens[hms_idx]) + 1 - new_idx = idx - - return (new_idx, hms) - - def _recombine_skipped(self, tokens, skipped_idxs): - """ - >>> tokens = ["foo", " ", "bar", " ", "19June2000", "baz"] - >>> skipped_idxs = [0, 1, 2, 5] - >>> _recombine_skipped(tokens, skipped_idxs) - ["foo bar", "baz"] - """ - skipped_tokens = [] - for i, idx in enumerate(sorted(skipped_idxs)): - if i > 0 and idx - 1 == skipped_idxs[i - 1]: - skipped_tokens[-1] = skipped_tokens[-1] + tokens[idx] - else: - skipped_tokens.append(tokens[idx]) - - return skipped_tokens - - def _build_tzinfo(self, tzinfos, tzname, tzoffset): - if callable(tzinfos): - tzdata = tzinfos(tzname, tzoffset) - else: - tzdata = tzinfos.get(tzname) - # handle case where tzinfo is paased an options that returns None - # eg tzinfos = {'BRST' : None} - if isinstance(tzdata, datetime.tzinfo) or tzdata is None: - tzinfo = tzdata - elif isinstance(tzdata, text_type): - tzinfo = tz.tzstr(tzdata) - elif isinstance(tzdata, integer_types): - tzinfo = tz.tzoffset(tzname, tzdata) - return tzinfo - - def _build_tzaware(self, naive, res, tzinfos): - if (callable(tzinfos) or (tzinfos and res.tzname in tzinfos)): - tzinfo = self._build_tzinfo(tzinfos, res.tzname, res.tzoffset) - aware = naive.replace(tzinfo=tzinfo) - aware = self._assign_tzname(aware, res.tzname) - - elif res.tzname and res.tzname in time.tzname: - aware = naive.replace(tzinfo=tz.tzlocal()) - - # Handle ambiguous local datetime - aware = self._assign_tzname(aware, res.tzname) - - # This is mostly relevant for winter GMT zones parsed in the UK - if (aware.tzname() != res.tzname and - res.tzname in self.info.UTCZONE): - aware = aware.replace(tzinfo=tz.tzutc()) - - elif res.tzoffset == 0: - aware = naive.replace(tzinfo=tz.tzutc()) - - elif res.tzoffset: - aware = naive.replace(tzinfo=tz.tzoffset(res.tzname, res.tzoffset)) - - elif not res.tzname and not res.tzoffset: - # i.e. no timezone information was found. - aware = naive - - elif res.tzname: - # tz-like string was parsed but we don't know what to do - # with it - warnings.warn("tzname {tzname} identified but not understood. " - "Pass `tzinfos` argument in order to correctly " - "return a timezone-aware datetime. In a future " - "version, this will raise an " - "exception.".format(tzname=res.tzname), - category=UnknownTimezoneWarning) - aware = naive - - return aware - - def _build_naive(self, res, default): - repl = {} - for attr in ("year", "month", "day", "hour", - "minute", "second", "microsecond"): - value = getattr(res, attr) - if value is not None: - repl[attr] = value - - if 'day' not in repl: - # If the default day exceeds the last day of the month, fall back - # to the end of the month. - cyear = default.year if res.year is None else res.year - cmonth = default.month if res.month is None else res.month - cday = default.day if res.day is None else res.day - - if cday > monthrange(cyear, cmonth)[1]: - repl['day'] = monthrange(cyear, cmonth)[1] - - naive = default.replace(**repl) - - if res.weekday is not None and not res.day: - naive = naive + relativedelta.relativedelta(weekday=res.weekday) - - return naive - - def _assign_tzname(self, dt, tzname): - if dt.tzname() != tzname: - new_dt = tz.enfold(dt, fold=1) - if new_dt.tzname() == tzname: - return new_dt - - return dt - - def _to_decimal(self, val): - try: - decimal_value = Decimal(val) - # See GH 662, edge case, infinite value should not be converted via `_to_decimal` - if not decimal_value.is_finite(): - raise ValueError("Converted decimal value is infinite or NaN") - except Exception as e: - msg = "Could not convert %s to decimal" % val - six.raise_from(ValueError(msg), e) - else: - return decimal_value - - -DEFAULTPARSER = parser() - - -def parse(timestr, parserinfo=None, **kwargs): - """ - - Parse a string in one of the supported formats, using the - ``parserinfo`` parameters. - - :param timestr: - A string containing a date/time stamp. - - :param parserinfo: - A :class:`parserinfo` object containing parameters for the parser. - If ``None``, the default arguments to the :class:`parserinfo` - constructor are used. - - The ``**kwargs`` parameter takes the following keyword arguments: - - :param default: - The default datetime object, if this is a datetime object and not - ``None``, elements specified in ``timestr`` replace elements in the - default object. - - :param ignoretz: - If set ``True``, time zones in parsed strings are ignored and a naive - :class:`datetime` object is returned. - - :param tzinfos: - Additional time zone names / aliases which may be present in the - string. This argument maps time zone names (and optionally offsets - from those time zones) to time zones. This parameter can be a - dictionary with timezone aliases mapping time zone names to time - zones or a function taking two parameters (``tzname`` and - ``tzoffset``) and returning a time zone. - - The timezones to which the names are mapped can be an integer - offset from UTC in seconds or a :class:`tzinfo` object. - - .. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> from dateutil.parser import parse - >>> from dateutil.tz import gettz - >>> tzinfos = {"BRST": -7200, "CST": gettz("America/Chicago")} - >>> parse("2012-01-19 17:21:00 BRST", tzinfos=tzinfos) - datetime.datetime(2012, 1, 19, 17, 21, tzinfo=tzoffset(u'BRST', -7200)) - >>> parse("2012-01-19 17:21:00 CST", tzinfos=tzinfos) - datetime.datetime(2012, 1, 19, 17, 21, - tzinfo=tzfile('/usr/share/zoneinfo/America/Chicago')) - - This parameter is ignored if ``ignoretz`` is set. - - :param dayfirst: - Whether to interpret the first value in an ambiguous 3-integer date - (e.g. 01/05/09) as the day (``True``) or month (``False``). If - ``yearfirst`` is set to ``True``, this distinguishes between YDM and - YMD. If set to ``None``, this value is retrieved from the current - :class:`parserinfo` object (which itself defaults to ``False``). - - :param yearfirst: - Whether to interpret the first value in an ambiguous 3-integer date - (e.g. 01/05/09) as the year. If ``True``, the first number is taken to - be the year, otherwise the last number is taken to be the year. If - this is set to ``None``, the value is retrieved from the current - :class:`parserinfo` object (which itself defaults to ``False``). - - :param fuzzy: - Whether to allow fuzzy parsing, allowing for string like "Today is - January 1, 2047 at 8:21:00AM". - - :param fuzzy_with_tokens: - If ``True``, ``fuzzy`` is automatically set to True, and the parser - will return a tuple where the first element is the parsed - :class:`datetime.datetime` datetimestamp and the second element is - a tuple containing the portions of the string which were ignored: - - .. doctest:: - - >>> from dateutil.parser import parse - >>> parse("Today is January 1, 2047 at 8:21:00AM", fuzzy_with_tokens=True) - (datetime.datetime(2047, 1, 1, 8, 21), (u'Today is ', u' ', u'at ')) - - :return: - Returns a :class:`datetime.datetime` object or, if the - ``fuzzy_with_tokens`` option is ``True``, returns a tuple, the - first element being a :class:`datetime.datetime` object, the second - a tuple containing the fuzzy tokens. - - :raises ValueError: - Raised for invalid or unknown string format, if the provided - :class:`tzinfo` is not in a valid format, or if an invalid date - would be created. - - :raises OverflowError: - Raised if the parsed date exceeds the largest valid C integer on - your system. - """ - if parserinfo: - return parser(parserinfo).parse(timestr, **kwargs) - else: - return DEFAULTPARSER.parse(timestr, **kwargs) - - -class _tzparser(object): - - class _result(_resultbase): - - __slots__ = ["stdabbr", "stdoffset", "dstabbr", "dstoffset", - "start", "end"] - - class _attr(_resultbase): - __slots__ = ["month", "week", "weekday", - "yday", "jyday", "day", "time"] - - def __repr__(self): - return self._repr("") - - def __init__(self): - _resultbase.__init__(self) - self.start = self._attr() - self.end = self._attr() - - def parse(self, tzstr): - res = self._result() - l = [x for x in re.split(r'([,:.]|[a-zA-Z]+|[0-9]+)',tzstr) if x] - used_idxs = list() - try: - - len_l = len(l) - - i = 0 - while i < len_l: - # BRST+3[BRDT[+2]] - j = i - while j < len_l and not [x for x in l[j] - if x in "0123456789:,-+"]: - j += 1 - if j != i: - if not res.stdabbr: - offattr = "stdoffset" - res.stdabbr = "".join(l[i:j]) - else: - offattr = "dstoffset" - res.dstabbr = "".join(l[i:j]) - - for ii in range(j): - used_idxs.append(ii) - i = j - if (i < len_l and (l[i] in ('+', '-') or l[i][0] in - "0123456789")): - if l[i] in ('+', '-'): - # Yes, that's right. See the TZ variable - # documentation. - signal = (1, -1)[l[i] == '+'] - used_idxs.append(i) - i += 1 - else: - signal = -1 - len_li = len(l[i]) - if len_li == 4: - # -0300 - setattr(res, offattr, (int(l[i][:2]) * 3600 + - int(l[i][2:]) * 60) * signal) - elif i + 1 < len_l and l[i + 1] == ':': - # -03:00 - setattr(res, offattr, - (int(l[i]) * 3600 + - int(l[i + 2]) * 60) * signal) - used_idxs.append(i) - i += 2 - elif len_li <= 2: - # -[0]3 - setattr(res, offattr, - int(l[i][:2]) * 3600 * signal) - else: - return None - used_idxs.append(i) - i += 1 - if res.dstabbr: - break - else: - break - - - if i < len_l: - for j in range(i, len_l): - if l[j] == ';': - l[j] = ',' - - assert l[i] == ',' - - i += 1 - - if i >= len_l: - pass - elif (8 <= l.count(',') <= 9 and - not [y for x in l[i:] if x != ',' - for y in x if y not in "0123456789+-"]): - # GMT0BST,3,0,30,3600,10,0,26,7200[,3600] - for x in (res.start, res.end): - x.month = int(l[i]) - used_idxs.append(i) - i += 2 - if l[i] == '-': - value = int(l[i + 1]) * -1 - used_idxs.append(i) - i += 1 - else: - value = int(l[i]) - used_idxs.append(i) - i += 2 - if value: - x.week = value - x.weekday = (int(l[i]) - 1) % 7 - else: - x.day = int(l[i]) - used_idxs.append(i) - i += 2 - x.time = int(l[i]) - used_idxs.append(i) - i += 2 - if i < len_l: - if l[i] in ('-', '+'): - signal = (-1, 1)[l[i] == "+"] - used_idxs.append(i) - i += 1 - else: - signal = 1 - used_idxs.append(i) - res.dstoffset = (res.stdoffset + int(l[i]) * signal) - - # This was a made-up format that is not in normal use - warn(('Parsed time zone "%s"' % tzstr) + - 'is in a non-standard dateutil-specific format, which ' + - 'is now deprecated; support for parsing this format ' + - 'will be removed in future versions. It is recommended ' + - 'that you switch to a standard format like the GNU ' + - 'TZ variable format.', tz.DeprecatedTzFormatWarning) - elif (l.count(',') == 2 and l[i:].count('/') <= 2 and - not [y for x in l[i:] if x not in (',', '/', 'J', 'M', - '.', '-', ':') - for y in x if y not in "0123456789"]): - for x in (res.start, res.end): - if l[i] == 'J': - # non-leap year day (1 based) - used_idxs.append(i) - i += 1 - x.jyday = int(l[i]) - elif l[i] == 'M': - # month[-.]week[-.]weekday - used_idxs.append(i) - i += 1 - x.month = int(l[i]) - used_idxs.append(i) - i += 1 - assert l[i] in ('-', '.') - used_idxs.append(i) - i += 1 - x.week = int(l[i]) - if x.week == 5: - x.week = -1 - used_idxs.append(i) - i += 1 - assert l[i] in ('-', '.') - used_idxs.append(i) - i += 1 - x.weekday = (int(l[i]) - 1) % 7 - else: - # year day (zero based) - x.yday = int(l[i]) + 1 - - used_idxs.append(i) - i += 1 - - if i < len_l and l[i] == '/': - used_idxs.append(i) - i += 1 - # start time - len_li = len(l[i]) - if len_li == 4: - # -0300 - x.time = (int(l[i][:2]) * 3600 + - int(l[i][2:]) * 60) - elif i + 1 < len_l and l[i + 1] == ':': - # -03:00 - x.time = int(l[i]) * 3600 + int(l[i + 2]) * 60 - used_idxs.append(i) - i += 2 - if i + 1 < len_l and l[i + 1] == ':': - used_idxs.append(i) - i += 2 - x.time += int(l[i]) - elif len_li <= 2: - # -[0]3 - x.time = (int(l[i][:2]) * 3600) - else: - return None - used_idxs.append(i) - i += 1 - - assert i == len_l or l[i] == ',' - - i += 1 - - assert i >= len_l - - except (IndexError, ValueError, AssertionError): - return None - - unused_idxs = set(range(len_l)).difference(used_idxs) - res.any_unused_tokens = not {l[n] for n in unused_idxs}.issubset({",",":"}) - return res - - -DEFAULTTZPARSER = _tzparser() - - -def _parsetz(tzstr): - return DEFAULTTZPARSER.parse(tzstr) - -class UnknownTimezoneWarning(RuntimeWarning): - """Raised when the parser finds a timezone it cannot parse into a tzinfo""" -# vim:ts=4:sw=4:et diff --git a/src/dateutil/parser/isoparser.py b/src/dateutil/parser/isoparser.py deleted file mode 100644 index e3cf6d8c..00000000 --- a/src/dateutil/parser/isoparser.py +++ /dev/null @@ -1,411 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module offers a parser for ISO-8601 strings - -It is intended to support all valid date, time and datetime formats per the -ISO-8601 specification. - -..versionadded:: 2.7.0 -""" -from datetime import datetime, timedelta, time, date -import calendar -from dateutil import tz - -from functools import wraps - -import re -import six - -__all__ = ["isoparse", "isoparser"] - - -def _takes_ascii(f): - @wraps(f) - def func(self, str_in, *args, **kwargs): - # If it's a stream, read the whole thing - str_in = getattr(str_in, 'read', lambda: str_in)() - - # If it's unicode, turn it into bytes, since ISO-8601 only covers ASCII - if isinstance(str_in, six.text_type): - # ASCII is the same in UTF-8 - try: - str_in = str_in.encode('ascii') - except UnicodeEncodeError as e: - msg = 'ISO-8601 strings should contain only ASCII characters' - six.raise_from(ValueError(msg), e) - - return f(self, str_in, *args, **kwargs) - - return func - - -class isoparser(object): - def __init__(self, sep=None): - """ - :param sep: - A single character that separates date and time portions. If - ``None``, the parser will accept any single character. - For strict ISO-8601 adherence, pass ``'T'``. - """ - if sep is not None: - if (len(sep) != 1 or ord(sep) >= 128 or sep in '0123456789'): - raise ValueError('Separator must be a single, non-numeric ' + - 'ASCII character') - - sep = sep.encode('ascii') - - self._sep = sep - - @_takes_ascii - def isoparse(self, dt_str): - """ - Parse an ISO-8601 datetime string into a :class:`datetime.datetime`. - - An ISO-8601 datetime string consists of a date portion, followed - optionally by a time portion - the date and time portions are separated - by a single character separator, which is ``T`` in the official - standard. Incomplete date formats (such as ``YYYY-MM``) may *not* be - combined with a time portion. - - Supported date formats are: - - Common: - - - ``YYYY`` - - ``YYYY-MM`` or ``YYYYMM`` - - ``YYYY-MM-DD`` or ``YYYYMMDD`` - - Uncommon: - - - ``YYYY-Www`` or ``YYYYWww`` - ISO week (day defaults to 0) - - ``YYYY-Www-D`` or ``YYYYWwwD`` - ISO week and day - - The ISO week and day numbering follows the same logic as - :func:`datetime.date.isocalendar`. - - Supported time formats are: - - - ``hh`` - - ``hh:mm`` or ``hhmm`` - - ``hh:mm:ss`` or ``hhmmss`` - - ``hh:mm:ss.ssssss`` (Up to 6 sub-second digits) - - Midnight is a special case for `hh`, as the standard supports both - 00:00 and 24:00 as a representation. The decimal separator can be - either a dot or a comma. - - - .. caution:: - - Support for fractional components other than seconds is part of the - ISO-8601 standard, but is not currently implemented in this parser. - - Supported time zone offset formats are: - - - `Z` (UTC) - - `±HH:MM` - - `±HHMM` - - `±HH` - - Offsets will be represented as :class:`dateutil.tz.tzoffset` objects, - with the exception of UTC, which will be represented as - :class:`dateutil.tz.tzutc`. Time zone offsets equivalent to UTC (such - as `+00:00`) will also be represented as :class:`dateutil.tz.tzutc`. - - :param dt_str: - A string or stream containing only an ISO-8601 datetime string - - :return: - Returns a :class:`datetime.datetime` representing the string. - Unspecified components default to their lowest value. - - .. warning:: - - As of version 2.7.0, the strictness of the parser should not be - considered a stable part of the contract. Any valid ISO-8601 string - that parses correctly with the default settings will continue to - parse correctly in future versions, but invalid strings that - currently fail (e.g. ``2017-01-01T00:00+00:00:00``) are not - guaranteed to continue failing in future versions if they encode - a valid date. - - .. versionadded:: 2.7.0 - """ - components, pos = self._parse_isodate(dt_str) - - if len(dt_str) > pos: - if self._sep is None or dt_str[pos:pos + 1] == self._sep: - components += self._parse_isotime(dt_str[pos + 1:]) - else: - raise ValueError('String contains unknown ISO components') - - if len(components) > 3 and components[3] == 24: - components[3] = 0 - return datetime(*components) + timedelta(days=1) - - return datetime(*components) - - @_takes_ascii - def parse_isodate(self, datestr): - """ - Parse the date portion of an ISO string. - - :param datestr: - The string portion of an ISO string, without a separator - - :return: - Returns a :class:`datetime.date` object - """ - components, pos = self._parse_isodate(datestr) - if pos < len(datestr): - raise ValueError('String contains unknown ISO ' + - 'components: {}'.format(datestr)) - return date(*components) - - @_takes_ascii - def parse_isotime(self, timestr): - """ - Parse the time portion of an ISO string. - - :param timestr: - The time portion of an ISO string, without a separator - - :return: - Returns a :class:`datetime.time` object - """ - components = self._parse_isotime(timestr) - if components[0] == 24: - components[0] = 0 - return time(*components) - - @_takes_ascii - def parse_tzstr(self, tzstr, zero_as_utc=True): - """ - Parse a valid ISO time zone string. - - See :func:`isoparser.isoparse` for details on supported formats. - - :param tzstr: - A string representing an ISO time zone offset - - :param zero_as_utc: - Whether to return :class:`dateutil.tz.tzutc` for zero-offset zones - - :return: - Returns :class:`dateutil.tz.tzoffset` for offsets and - :class:`dateutil.tz.tzutc` for ``Z`` and (if ``zero_as_utc`` is - specified) offsets equivalent to UTC. - """ - return self._parse_tzstr(tzstr, zero_as_utc=zero_as_utc) - - # Constants - _DATE_SEP = b'-' - _TIME_SEP = b':' - _FRACTION_REGEX = re.compile(b'[\\.,]([0-9]+)') - - def _parse_isodate(self, dt_str): - try: - return self._parse_isodate_common(dt_str) - except ValueError: - return self._parse_isodate_uncommon(dt_str) - - def _parse_isodate_common(self, dt_str): - len_str = len(dt_str) - components = [1, 1, 1] - - if len_str < 4: - raise ValueError('ISO string too short') - - # Year - components[0] = int(dt_str[0:4]) - pos = 4 - if pos >= len_str: - return components, pos - - has_sep = dt_str[pos:pos + 1] == self._DATE_SEP - if has_sep: - pos += 1 - - # Month - if len_str - pos < 2: - raise ValueError('Invalid common month') - - components[1] = int(dt_str[pos:pos + 2]) - pos += 2 - - if pos >= len_str: - if has_sep: - return components, pos - else: - raise ValueError('Invalid ISO format') - - if has_sep: - if dt_str[pos:pos + 1] != self._DATE_SEP: - raise ValueError('Invalid separator in ISO string') - pos += 1 - - # Day - if len_str - pos < 2: - raise ValueError('Invalid common day') - components[2] = int(dt_str[pos:pos + 2]) - return components, pos + 2 - - def _parse_isodate_uncommon(self, dt_str): - if len(dt_str) < 4: - raise ValueError('ISO string too short') - - # All ISO formats start with the year - year = int(dt_str[0:4]) - - has_sep = dt_str[4:5] == self._DATE_SEP - - pos = 4 + has_sep # Skip '-' if it's there - if dt_str[pos:pos + 1] == b'W': - # YYYY-?Www-?D? - pos += 1 - weekno = int(dt_str[pos:pos + 2]) - pos += 2 - - dayno = 1 - if len(dt_str) > pos: - if (dt_str[pos:pos + 1] == self._DATE_SEP) != has_sep: - raise ValueError('Inconsistent use of dash separator') - - pos += has_sep - - dayno = int(dt_str[pos:pos + 1]) - pos += 1 - - base_date = self._calculate_weekdate(year, weekno, dayno) - else: - # YYYYDDD or YYYY-DDD - if len(dt_str) - pos < 3: - raise ValueError('Invalid ordinal day') - - ordinal_day = int(dt_str[pos:pos + 3]) - pos += 3 - - if ordinal_day < 1 or ordinal_day > (365 + calendar.isleap(year)): - raise ValueError('Invalid ordinal day' + - ' {} for year {}'.format(ordinal_day, year)) - - base_date = date(year, 1, 1) + timedelta(days=ordinal_day - 1) - - components = [base_date.year, base_date.month, base_date.day] - return components, pos - - def _calculate_weekdate(self, year, week, day): - """ - Calculate the day of corresponding to the ISO year-week-day calendar. - - This function is effectively the inverse of - :func:`datetime.date.isocalendar`. - - :param year: - The year in the ISO calendar - - :param week: - The week in the ISO calendar - range is [1, 53] - - :param day: - The day in the ISO calendar - range is [1 (MON), 7 (SUN)] - - :return: - Returns a :class:`datetime.date` - """ - if not 0 < week < 54: - raise ValueError('Invalid week: {}'.format(week)) - - if not 0 < day < 8: # Range is 1-7 - raise ValueError('Invalid weekday: {}'.format(day)) - - # Get week 1 for the specific year: - jan_4 = date(year, 1, 4) # Week 1 always has January 4th in it - week_1 = jan_4 - timedelta(days=jan_4.isocalendar()[2] - 1) - - # Now add the specific number of weeks and days to get what we want - week_offset = (week - 1) * 7 + (day - 1) - return week_1 + timedelta(days=week_offset) - - def _parse_isotime(self, timestr): - len_str = len(timestr) - components = [0, 0, 0, 0, None] - pos = 0 - comp = -1 - - if len(timestr) < 2: - raise ValueError('ISO time too short') - - has_sep = len_str >= 3 and timestr[2:3] == self._TIME_SEP - - while pos < len_str and comp < 5: - comp += 1 - - if timestr[pos:pos + 1] in b'-+Zz': - # Detect time zone boundary - components[-1] = self._parse_tzstr(timestr[pos:]) - pos = len_str - break - - if comp < 3: - # Hour, minute, second - components[comp] = int(timestr[pos:pos + 2]) - pos += 2 - if (has_sep and pos < len_str and - timestr[pos:pos + 1] == self._TIME_SEP): - pos += 1 - - if comp == 3: - # Fraction of a second - frac = self._FRACTION_REGEX.match(timestr[pos:]) - if not frac: - continue - - us_str = frac.group(1)[:6] # Truncate to microseconds - components[comp] = int(us_str) * 10**(6 - len(us_str)) - pos += len(frac.group()) - - if pos < len_str: - raise ValueError('Unused components in ISO string') - - if components[0] == 24: - # Standard supports 00:00 and 24:00 as representations of midnight - if any(component != 0 for component in components[1:4]): - raise ValueError('Hour may only be 24 at 24:00:00.000') - - return components - - def _parse_tzstr(self, tzstr, zero_as_utc=True): - if tzstr == b'Z' or tzstr == b'z': - return tz.tzutc() - - if len(tzstr) not in {3, 5, 6}: - raise ValueError('Time zone offset must be 1, 3, 5 or 6 characters') - - if tzstr[0:1] == b'-': - mult = -1 - elif tzstr[0:1] == b'+': - mult = 1 - else: - raise ValueError('Time zone offset requires sign') - - hours = int(tzstr[1:3]) - if len(tzstr) == 3: - minutes = 0 - else: - minutes = int(tzstr[(4 if tzstr[3:4] == self._TIME_SEP else 3):]) - - if zero_as_utc and hours == 0 and minutes == 0: - return tz.tzutc() - else: - if minutes > 59: - raise ValueError('Invalid minutes in time zone offset') - - if hours > 23: - raise ValueError('Invalid hours in time zone offset') - - return tz.tzoffset(None, mult * (hours * 60 + minutes) * 60) - - -DEFAULT_ISOPARSER = isoparser() -isoparse = DEFAULT_ISOPARSER.isoparse diff --git a/src/dateutil/relativedelta.py b/src/dateutil/relativedelta.py deleted file mode 100644 index c65c66e6..00000000 --- a/src/dateutil/relativedelta.py +++ /dev/null @@ -1,599 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -import calendar - -import operator -from math import copysign - -from six import integer_types -from warnings import warn - -from ._common import weekday - -MO, TU, WE, TH, FR, SA, SU = weekdays = tuple(weekday(x) for x in range(7)) - -__all__ = ["relativedelta", "MO", "TU", "WE", "TH", "FR", "SA", "SU"] - - -class relativedelta(object): - """ - The relativedelta type is designed to be applied to an existing datetime and - can replace specific components of that datetime, or represents an interval - of time. - - It is based on the specification of the excellent work done by M.-A. Lemburg - in his - `mx.DateTime `_ extension. - However, notice that this type does *NOT* implement the same algorithm as - his work. Do *NOT* expect it to behave like mx.DateTime's counterpart. - - There are two different ways to build a relativedelta instance. The - first one is passing it two date/datetime classes:: - - relativedelta(datetime1, datetime2) - - The second one is passing it any number of the following keyword arguments:: - - relativedelta(arg1=x,arg2=y,arg3=z...) - - year, month, day, hour, minute, second, microsecond: - Absolute information (argument is singular); adding or subtracting a - relativedelta with absolute information does not perform an arithmetic - operation, but rather REPLACES the corresponding value in the - original datetime with the value(s) in relativedelta. - - years, months, weeks, days, hours, minutes, seconds, microseconds: - Relative information, may be negative (argument is plural); adding - or subtracting a relativedelta with relative information performs - the corresponding aritmetic operation on the original datetime value - with the information in the relativedelta. - - weekday: - One of the weekday instances (MO, TU, etc) available in the - relativedelta module. These instances may receive a parameter N, - specifying the Nth weekday, which could be positive or negative - (like MO(+1) or MO(-2)). Not specifying it is the same as specifying - +1. You can also use an integer, where 0=MO. This argument is always - relative e.g. if the calculated date is already Monday, using MO(1) - or MO(-1) won't change the day. To effectively make it absolute, use - it in combination with the day argument (e.g. day=1, MO(1) for first - Monday of the month). - - leapdays: - Will add given days to the date found, if year is a leap - year, and the date found is post 28 of february. - - yearday, nlyearday: - Set the yearday or the non-leap year day (jump leap days). - These are converted to day/month/leapdays information. - - There are relative and absolute forms of the keyword - arguments. The plural is relative, and the singular is - absolute. For each argument in the order below, the absolute form - is applied first (by setting each attribute to that value) and - then the relative form (by adding the value to the attribute). - - The order of attributes considered when this relativedelta is - added to a datetime is: - - 1. Year - 2. Month - 3. Day - 4. Hours - 5. Minutes - 6. Seconds - 7. Microseconds - - Finally, weekday is applied, using the rule described above. - - For example - - >>> from datetime import datetime - >>> from dateutil.relativedelta import relativedelta, MO - >>> dt = datetime(2018, 4, 9, 13, 37, 0) - >>> delta = relativedelta(hours=25, day=1, weekday=MO(1)) - >>> dt + delta - datetime.datetime(2018, 4, 2, 14, 37) - - First, the day is set to 1 (the first of the month), then 25 hours - are added, to get to the 2nd day and 14th hour, finally the - weekday is applied, but since the 2nd is already a Monday there is - no effect. - - """ - - def __init__(self, dt1=None, dt2=None, - years=0, months=0, days=0, leapdays=0, weeks=0, - hours=0, minutes=0, seconds=0, microseconds=0, - year=None, month=None, day=None, weekday=None, - yearday=None, nlyearday=None, - hour=None, minute=None, second=None, microsecond=None): - - if dt1 and dt2: - # datetime is a subclass of date. So both must be date - if not (isinstance(dt1, datetime.date) and - isinstance(dt2, datetime.date)): - raise TypeError("relativedelta only diffs datetime/date") - - # We allow two dates, or two datetimes, so we coerce them to be - # of the same type - if (isinstance(dt1, datetime.datetime) != - isinstance(dt2, datetime.datetime)): - if not isinstance(dt1, datetime.datetime): - dt1 = datetime.datetime.fromordinal(dt1.toordinal()) - elif not isinstance(dt2, datetime.datetime): - dt2 = datetime.datetime.fromordinal(dt2.toordinal()) - - self.years = 0 - self.months = 0 - self.days = 0 - self.leapdays = 0 - self.hours = 0 - self.minutes = 0 - self.seconds = 0 - self.microseconds = 0 - self.year = None - self.month = None - self.day = None - self.weekday = None - self.hour = None - self.minute = None - self.second = None - self.microsecond = None - self._has_time = 0 - - # Get year / month delta between the two - months = (dt1.year - dt2.year) * 12 + (dt1.month - dt2.month) - self._set_months(months) - - # Remove the year/month delta so the timedelta is just well-defined - # time units (seconds, days and microseconds) - dtm = self.__radd__(dt2) - - # If we've overshot our target, make an adjustment - if dt1 < dt2: - compare = operator.gt - increment = 1 - else: - compare = operator.lt - increment = -1 - - while compare(dt1, dtm): - months += increment - self._set_months(months) - dtm = self.__radd__(dt2) - - # Get the timedelta between the "months-adjusted" date and dt1 - delta = dt1 - dtm - self.seconds = delta.seconds + delta.days * 86400 - self.microseconds = delta.microseconds - else: - # Check for non-integer values in integer-only quantities - if any(x is not None and x != int(x) for x in (years, months)): - raise ValueError("Non-integer years and months are " - "ambiguous and not currently supported.") - - # Relative information - self.years = int(years) - self.months = int(months) - self.days = days + weeks * 7 - self.leapdays = leapdays - self.hours = hours - self.minutes = minutes - self.seconds = seconds - self.microseconds = microseconds - - # Absolute information - self.year = year - self.month = month - self.day = day - self.hour = hour - self.minute = minute - self.second = second - self.microsecond = microsecond - - if any(x is not None and int(x) != x - for x in (year, month, day, hour, - minute, second, microsecond)): - # For now we'll deprecate floats - later it'll be an error. - warn("Non-integer value passed as absolute information. " + - "This is not a well-defined condition and will raise " + - "errors in future versions.", DeprecationWarning) - - if isinstance(weekday, integer_types): - self.weekday = weekdays[weekday] - else: - self.weekday = weekday - - yday = 0 - if nlyearday: - yday = nlyearday - elif yearday: - yday = yearday - if yearday > 59: - self.leapdays = -1 - if yday: - ydayidx = [31, 59, 90, 120, 151, 181, 212, - 243, 273, 304, 334, 366] - for idx, ydays in enumerate(ydayidx): - if yday <= ydays: - self.month = idx+1 - if idx == 0: - self.day = yday - else: - self.day = yday-ydayidx[idx-1] - break - else: - raise ValueError("invalid year day (%d)" % yday) - - self._fix() - - def _fix(self): - if abs(self.microseconds) > 999999: - s = _sign(self.microseconds) - div, mod = divmod(self.microseconds * s, 1000000) - self.microseconds = mod * s - self.seconds += div * s - if abs(self.seconds) > 59: - s = _sign(self.seconds) - div, mod = divmod(self.seconds * s, 60) - self.seconds = mod * s - self.minutes += div * s - if abs(self.minutes) > 59: - s = _sign(self.minutes) - div, mod = divmod(self.minutes * s, 60) - self.minutes = mod * s - self.hours += div * s - if abs(self.hours) > 23: - s = _sign(self.hours) - div, mod = divmod(self.hours * s, 24) - self.hours = mod * s - self.days += div * s - if abs(self.months) > 11: - s = _sign(self.months) - div, mod = divmod(self.months * s, 12) - self.months = mod * s - self.years += div * s - if (self.hours or self.minutes or self.seconds or self.microseconds - or self.hour is not None or self.minute is not None or - self.second is not None or self.microsecond is not None): - self._has_time = 1 - else: - self._has_time = 0 - - @property - def weeks(self): - return int(self.days / 7.0) - - @weeks.setter - def weeks(self, value): - self.days = self.days - (self.weeks * 7) + value * 7 - - def _set_months(self, months): - self.months = months - if abs(self.months) > 11: - s = _sign(self.months) - div, mod = divmod(self.months * s, 12) - self.months = mod * s - self.years = div * s - else: - self.years = 0 - - def normalized(self): - """ - Return a version of this object represented entirely using integer - values for the relative attributes. - - >>> relativedelta(days=1.5, hours=2).normalized() - relativedelta(days=+1, hours=+14) - - :return: - Returns a :class:`dateutil.relativedelta.relativedelta` object. - """ - # Cascade remainders down (rounding each to roughly nearest microsecond) - days = int(self.days) - - hours_f = round(self.hours + 24 * (self.days - days), 11) - hours = int(hours_f) - - minutes_f = round(self.minutes + 60 * (hours_f - hours), 10) - minutes = int(minutes_f) - - seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8) - seconds = int(seconds_f) - - microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds)) - - # Constructor carries overflow back up with call to _fix() - return self.__class__(years=self.years, months=self.months, - days=days, hours=hours, minutes=minutes, - seconds=seconds, microseconds=microseconds, - leapdays=self.leapdays, year=self.year, - month=self.month, day=self.day, - weekday=self.weekday, hour=self.hour, - minute=self.minute, second=self.second, - microsecond=self.microsecond) - - def __add__(self, other): - if isinstance(other, relativedelta): - return self.__class__(years=other.years + self.years, - months=other.months + self.months, - days=other.days + self.days, - hours=other.hours + self.hours, - minutes=other.minutes + self.minutes, - seconds=other.seconds + self.seconds, - microseconds=(other.microseconds + - self.microseconds), - leapdays=other.leapdays or self.leapdays, - year=(other.year if other.year is not None - else self.year), - month=(other.month if other.month is not None - else self.month), - day=(other.day if other.day is not None - else self.day), - weekday=(other.weekday if other.weekday is not None - else self.weekday), - hour=(other.hour if other.hour is not None - else self.hour), - minute=(other.minute if other.minute is not None - else self.minute), - second=(other.second if other.second is not None - else self.second), - microsecond=(other.microsecond if other.microsecond - is not None else - self.microsecond)) - if isinstance(other, datetime.timedelta): - return self.__class__(years=self.years, - months=self.months, - days=self.days + other.days, - hours=self.hours, - minutes=self.minutes, - seconds=self.seconds + other.seconds, - microseconds=self.microseconds + other.microseconds, - leapdays=self.leapdays, - year=self.year, - month=self.month, - day=self.day, - weekday=self.weekday, - hour=self.hour, - minute=self.minute, - second=self.second, - microsecond=self.microsecond) - if not isinstance(other, datetime.date): - return NotImplemented - elif self._has_time and not isinstance(other, datetime.datetime): - other = datetime.datetime.fromordinal(other.toordinal()) - year = (self.year or other.year)+self.years - month = self.month or other.month - if self.months: - assert 1 <= abs(self.months) <= 12 - month += self.months - if month > 12: - year += 1 - month -= 12 - elif month < 1: - year -= 1 - month += 12 - day = min(calendar.monthrange(year, month)[1], - self.day or other.day) - repl = {"year": year, "month": month, "day": day} - for attr in ["hour", "minute", "second", "microsecond"]: - value = getattr(self, attr) - if value is not None: - repl[attr] = value - days = self.days - if self.leapdays and month > 2 and calendar.isleap(year): - days += self.leapdays - ret = (other.replace(**repl) - + datetime.timedelta(days=days, - hours=self.hours, - minutes=self.minutes, - seconds=self.seconds, - microseconds=self.microseconds)) - if self.weekday: - weekday, nth = self.weekday.weekday, self.weekday.n or 1 - jumpdays = (abs(nth) - 1) * 7 - if nth > 0: - jumpdays += (7 - ret.weekday() + weekday) % 7 - else: - jumpdays += (ret.weekday() - weekday) % 7 - jumpdays *= -1 - ret += datetime.timedelta(days=jumpdays) - return ret - - def __radd__(self, other): - return self.__add__(other) - - def __rsub__(self, other): - return self.__neg__().__radd__(other) - - def __sub__(self, other): - if not isinstance(other, relativedelta): - return NotImplemented # In case the other object defines __rsub__ - return self.__class__(years=self.years - other.years, - months=self.months - other.months, - days=self.days - other.days, - hours=self.hours - other.hours, - minutes=self.minutes - other.minutes, - seconds=self.seconds - other.seconds, - microseconds=self.microseconds - other.microseconds, - leapdays=self.leapdays or other.leapdays, - year=(self.year if self.year is not None - else other.year), - month=(self.month if self.month is not None else - other.month), - day=(self.day if self.day is not None else - other.day), - weekday=(self.weekday if self.weekday is not None else - other.weekday), - hour=(self.hour if self.hour is not None else - other.hour), - minute=(self.minute if self.minute is not None else - other.minute), - second=(self.second if self.second is not None else - other.second), - microsecond=(self.microsecond if self.microsecond - is not None else - other.microsecond)) - - def __abs__(self): - return self.__class__(years=abs(self.years), - months=abs(self.months), - days=abs(self.days), - hours=abs(self.hours), - minutes=abs(self.minutes), - seconds=abs(self.seconds), - microseconds=abs(self.microseconds), - leapdays=self.leapdays, - year=self.year, - month=self.month, - day=self.day, - weekday=self.weekday, - hour=self.hour, - minute=self.minute, - second=self.second, - microsecond=self.microsecond) - - def __neg__(self): - return self.__class__(years=-self.years, - months=-self.months, - days=-self.days, - hours=-self.hours, - minutes=-self.minutes, - seconds=-self.seconds, - microseconds=-self.microseconds, - leapdays=self.leapdays, - year=self.year, - month=self.month, - day=self.day, - weekday=self.weekday, - hour=self.hour, - minute=self.minute, - second=self.second, - microsecond=self.microsecond) - - def __bool__(self): - return not (not self.years and - not self.months and - not self.days and - not self.hours and - not self.minutes and - not self.seconds and - not self.microseconds and - not self.leapdays and - self.year is None and - self.month is None and - self.day is None and - self.weekday is None and - self.hour is None and - self.minute is None and - self.second is None and - self.microsecond is None) - # Compatibility with Python 2.x - __nonzero__ = __bool__ - - def __mul__(self, other): - try: - f = float(other) - except TypeError: - return NotImplemented - - return self.__class__(years=int(self.years * f), - months=int(self.months * f), - days=int(self.days * f), - hours=int(self.hours * f), - minutes=int(self.minutes * f), - seconds=int(self.seconds * f), - microseconds=int(self.microseconds * f), - leapdays=self.leapdays, - year=self.year, - month=self.month, - day=self.day, - weekday=self.weekday, - hour=self.hour, - minute=self.minute, - second=self.second, - microsecond=self.microsecond) - - __rmul__ = __mul__ - - def __eq__(self, other): - if not isinstance(other, relativedelta): - return NotImplemented - if self.weekday or other.weekday: - if not self.weekday or not other.weekday: - return False - if self.weekday.weekday != other.weekday.weekday: - return False - n1, n2 = self.weekday.n, other.weekday.n - if n1 != n2 and not ((not n1 or n1 == 1) and (not n2 or n2 == 1)): - return False - return (self.years == other.years and - self.months == other.months and - self.days == other.days and - self.hours == other.hours and - self.minutes == other.minutes and - self.seconds == other.seconds and - self.microseconds == other.microseconds and - self.leapdays == other.leapdays and - self.year == other.year and - self.month == other.month and - self.day == other.day and - self.hour == other.hour and - self.minute == other.minute and - self.second == other.second and - self.microsecond == other.microsecond) - - def __hash__(self): - return hash(( - self.weekday, - self.years, - self.months, - self.days, - self.hours, - self.minutes, - self.seconds, - self.microseconds, - self.leapdays, - self.year, - self.month, - self.day, - self.hour, - self.minute, - self.second, - self.microsecond, - )) - - def __ne__(self, other): - return not self.__eq__(other) - - def __div__(self, other): - try: - reciprocal = 1 / float(other) - except TypeError: - return NotImplemented - - return self.__mul__(reciprocal) - - __truediv__ = __div__ - - def __repr__(self): - l = [] - for attr in ["years", "months", "days", "leapdays", - "hours", "minutes", "seconds", "microseconds"]: - value = getattr(self, attr) - if value: - l.append("{attr}={value:+g}".format(attr=attr, value=value)) - for attr in ["year", "month", "day", "weekday", - "hour", "minute", "second", "microsecond"]: - value = getattr(self, attr) - if value is not None: - l.append("{attr}={value}".format(attr=attr, value=repr(value))) - return "{classname}({attrs})".format(classname=self.__class__.__name__, - attrs=", ".join(l)) - - -def _sign(x): - return int(copysign(1, x)) - -# vim:ts=4:sw=4:et diff --git a/src/dateutil/rrule.py b/src/dateutil/rrule.py deleted file mode 100644 index 20a0c4ac..00000000 --- a/src/dateutil/rrule.py +++ /dev/null @@ -1,1736 +0,0 @@ -# -*- coding: utf-8 -*- -""" -The rrule module offers a small, complete, and very fast, implementation of -the recurrence rules documented in the -`iCalendar RFC `_, -including support for caching of results. -""" -import itertools -import datetime -import calendar -import re -import sys - -try: - from math import gcd -except ImportError: - from fractions import gcd - -from six import advance_iterator, integer_types -from six.moves import _thread, range -import heapq - -from ._common import weekday as weekdaybase -from .tz import tzutc, tzlocal - -# For warning about deprecation of until and count -from warnings import warn - -__all__ = ["rrule", "rruleset", "rrulestr", - "YEARLY", "MONTHLY", "WEEKLY", "DAILY", - "HOURLY", "MINUTELY", "SECONDLY", - "MO", "TU", "WE", "TH", "FR", "SA", "SU"] - -# Every mask is 7 days longer to handle cross-year weekly periods. -M366MASK = tuple([1]*31+[2]*29+[3]*31+[4]*30+[5]*31+[6]*30 + - [7]*31+[8]*31+[9]*30+[10]*31+[11]*30+[12]*31+[1]*7) -M365MASK = list(M366MASK) -M29, M30, M31 = list(range(1, 30)), list(range(1, 31)), list(range(1, 32)) -MDAY366MASK = tuple(M31+M29+M31+M30+M31+M30+M31+M31+M30+M31+M30+M31+M31[:7]) -MDAY365MASK = list(MDAY366MASK) -M29, M30, M31 = list(range(-29, 0)), list(range(-30, 0)), list(range(-31, 0)) -NMDAY366MASK = tuple(M31+M29+M31+M30+M31+M30+M31+M31+M30+M31+M30+M31+M31[:7]) -NMDAY365MASK = list(NMDAY366MASK) -M366RANGE = (0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366) -M365RANGE = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365) -WDAYMASK = [0, 1, 2, 3, 4, 5, 6]*55 -del M29, M30, M31, M365MASK[59], MDAY365MASK[59], NMDAY365MASK[31] -MDAY365MASK = tuple(MDAY365MASK) -M365MASK = tuple(M365MASK) - -FREQNAMES = ['YEARLY', 'MONTHLY', 'WEEKLY', 'DAILY', 'HOURLY', 'MINUTELY', 'SECONDLY'] - -(YEARLY, - MONTHLY, - WEEKLY, - DAILY, - HOURLY, - MINUTELY, - SECONDLY) = list(range(7)) - -# Imported on demand. -easter = None -parser = None - - -class weekday(weekdaybase): - """ - This version of weekday does not allow n = 0. - """ - def __init__(self, wkday, n=None): - if n == 0: - raise ValueError("Can't create weekday with n==0") - - super(weekday, self).__init__(wkday, n) - - -MO, TU, WE, TH, FR, SA, SU = weekdays = tuple(weekday(x) for x in range(7)) - - -def _invalidates_cache(f): - """ - Decorator for rruleset methods which may invalidate the - cached length. - """ - def inner_func(self, *args, **kwargs): - rv = f(self, *args, **kwargs) - self._invalidate_cache() - return rv - - return inner_func - - -class rrulebase(object): - def __init__(self, cache=False): - if cache: - self._cache = [] - self._cache_lock = _thread.allocate_lock() - self._invalidate_cache() - else: - self._cache = None - self._cache_complete = False - self._len = None - - def __iter__(self): - if self._cache_complete: - return iter(self._cache) - elif self._cache is None: - return self._iter() - else: - return self._iter_cached() - - def _invalidate_cache(self): - if self._cache is not None: - self._cache = [] - self._cache_complete = False - self._cache_gen = self._iter() - - if self._cache_lock.locked(): - self._cache_lock.release() - - self._len = None - - def _iter_cached(self): - i = 0 - gen = self._cache_gen - cache = self._cache - acquire = self._cache_lock.acquire - release = self._cache_lock.release - while gen: - if i == len(cache): - acquire() - if self._cache_complete: - break - try: - for j in range(10): - cache.append(advance_iterator(gen)) - except StopIteration: - self._cache_gen = gen = None - self._cache_complete = True - break - release() - yield cache[i] - i += 1 - while i < self._len: - yield cache[i] - i += 1 - - def __getitem__(self, item): - if self._cache_complete: - return self._cache[item] - elif isinstance(item, slice): - if item.step and item.step < 0: - return list(iter(self))[item] - else: - return list(itertools.islice(self, - item.start or 0, - item.stop or sys.maxsize, - item.step or 1)) - elif item >= 0: - gen = iter(self) - try: - for i in range(item+1): - res = advance_iterator(gen) - except StopIteration: - raise IndexError - return res - else: - return list(iter(self))[item] - - def __contains__(self, item): - if self._cache_complete: - return item in self._cache - else: - for i in self: - if i == item: - return True - elif i > item: - return False - return False - - # __len__() introduces a large performance penality. - def count(self): - """ Returns the number of recurrences in this set. It will have go - trough the whole recurrence, if this hasn't been done before. """ - if self._len is None: - for x in self: - pass - return self._len - - def before(self, dt, inc=False): - """ Returns the last recurrence before the given datetime instance. The - inc keyword defines what happens if dt is an occurrence. With - inc=True, if dt itself is an occurrence, it will be returned. """ - if self._cache_complete: - gen = self._cache - else: - gen = self - last = None - if inc: - for i in gen: - if i > dt: - break - last = i - else: - for i in gen: - if i >= dt: - break - last = i - return last - - def after(self, dt, inc=False): - """ Returns the first recurrence after the given datetime instance. The - inc keyword defines what happens if dt is an occurrence. With - inc=True, if dt itself is an occurrence, it will be returned. """ - if self._cache_complete: - gen = self._cache - else: - gen = self - if inc: - for i in gen: - if i >= dt: - return i - else: - for i in gen: - if i > dt: - return i - return None - - def xafter(self, dt, count=None, inc=False): - """ - Generator which yields up to `count` recurrences after the given - datetime instance, equivalent to `after`. - - :param dt: - The datetime at which to start generating recurrences. - - :param count: - The maximum number of recurrences to generate. If `None` (default), - dates are generated until the recurrence rule is exhausted. - - :param inc: - If `dt` is an instance of the rule and `inc` is `True`, it is - included in the output. - - :yields: Yields a sequence of `datetime` objects. - """ - - if self._cache_complete: - gen = self._cache - else: - gen = self - - # Select the comparison function - if inc: - comp = lambda dc, dtc: dc >= dtc - else: - comp = lambda dc, dtc: dc > dtc - - # Generate dates - n = 0 - for d in gen: - if comp(d, dt): - if count is not None: - n += 1 - if n > count: - break - - yield d - - def between(self, after, before, inc=False, count=1): - """ Returns all the occurrences of the rrule between after and before. - The inc keyword defines what happens if after and/or before are - themselves occurrences. With inc=True, they will be included in the - list, if they are found in the recurrence set. """ - if self._cache_complete: - gen = self._cache - else: - gen = self - started = False - l = [] - if inc: - for i in gen: - if i > before: - break - elif not started: - if i >= after: - started = True - l.append(i) - else: - l.append(i) - else: - for i in gen: - if i >= before: - break - elif not started: - if i > after: - started = True - l.append(i) - else: - l.append(i) - return l - - -class rrule(rrulebase): - """ - That's the base of the rrule operation. It accepts all the keywords - defined in the RFC as its constructor parameters (except byday, - which was renamed to byweekday) and more. The constructor prototype is:: - - rrule(freq) - - Where freq must be one of YEARLY, MONTHLY, WEEKLY, DAILY, HOURLY, MINUTELY, - or SECONDLY. - - .. note:: - Per RFC section 3.3.10, recurrence instances falling on invalid dates - and times are ignored rather than coerced: - - Recurrence rules may generate recurrence instances with an invalid - date (e.g., February 30) or nonexistent local time (e.g., 1:30 AM - on a day where the local time is moved forward by an hour at 1:00 - AM). Such recurrence instances MUST be ignored and MUST NOT be - counted as part of the recurrence set. - - This can lead to possibly surprising behavior when, for example, the - start date occurs at the end of the month: - - >>> from dateutil.rrule import rrule, MONTHLY - >>> from datetime import datetime - >>> start_date = datetime(2014, 12, 31) - >>> list(rrule(freq=MONTHLY, count=4, dtstart=start_date)) - ... # doctest: +NORMALIZE_WHITESPACE - [datetime.datetime(2014, 12, 31, 0, 0), - datetime.datetime(2015, 1, 31, 0, 0), - datetime.datetime(2015, 3, 31, 0, 0), - datetime.datetime(2015, 5, 31, 0, 0)] - - Additionally, it supports the following keyword arguments: - - :param dtstart: - The recurrence start. Besides being the base for the recurrence, - missing parameters in the final recurrence instances will also be - extracted from this date. If not given, datetime.now() will be used - instead. - :param interval: - The interval between each freq iteration. For example, when using - YEARLY, an interval of 2 means once every two years, but with HOURLY, - it means once every two hours. The default interval is 1. - :param wkst: - The week start day. Must be one of the MO, TU, WE constants, or an - integer, specifying the first day of the week. This will affect - recurrences based on weekly periods. The default week start is got - from calendar.firstweekday(), and may be modified by - calendar.setfirstweekday(). - :param count: - If given, this determines how many occurrences will be generated. - - .. note:: - As of version 2.5.0, the use of the keyword ``until`` in conjunction - with ``count`` is deprecated, to make sure ``dateutil`` is fully - compliant with `RFC-5545 Sec. 3.3.10 `_. Therefore, ``until`` and ``count`` - **must not** occur in the same call to ``rrule``. - :param until: - If given, this must be a datetime instance specifying the upper-bound - limit of the recurrence. The last recurrence in the rule is the greatest - datetime that is less than or equal to the value specified in the - ``until`` parameter. - - .. note:: - As of version 2.5.0, the use of the keyword ``until`` in conjunction - with ``count`` is deprecated, to make sure ``dateutil`` is fully - compliant with `RFC-5545 Sec. 3.3.10 `_. Therefore, ``until`` and ``count`` - **must not** occur in the same call to ``rrule``. - :param bysetpos: - If given, it must be either an integer, or a sequence of integers, - positive or negative. Each given integer will specify an occurrence - number, corresponding to the nth occurrence of the rule inside the - frequency period. For example, a bysetpos of -1 if combined with a - MONTHLY frequency, and a byweekday of (MO, TU, WE, TH, FR), will - result in the last work day of every month. - :param bymonth: - If given, it must be either an integer, or a sequence of integers, - meaning the months to apply the recurrence to. - :param bymonthday: - If given, it must be either an integer, or a sequence of integers, - meaning the month days to apply the recurrence to. - :param byyearday: - If given, it must be either an integer, or a sequence of integers, - meaning the year days to apply the recurrence to. - :param byeaster: - If given, it must be either an integer, or a sequence of integers, - positive or negative. Each integer will define an offset from the - Easter Sunday. Passing the offset 0 to byeaster will yield the Easter - Sunday itself. This is an extension to the RFC specification. - :param byweekno: - If given, it must be either an integer, or a sequence of integers, - meaning the week numbers to apply the recurrence to. Week numbers - have the meaning described in ISO8601, that is, the first week of - the year is that containing at least four days of the new year. - :param byweekday: - If given, it must be either an integer (0 == MO), a sequence of - integers, one of the weekday constants (MO, TU, etc), or a sequence - of these constants. When given, these variables will define the - weekdays where the recurrence will be applied. It's also possible to - use an argument n for the weekday instances, which will mean the nth - occurrence of this weekday in the period. For example, with MONTHLY, - or with YEARLY and BYMONTH, using FR(+1) in byweekday will specify the - first friday of the month where the recurrence happens. Notice that in - the RFC documentation, this is specified as BYDAY, but was renamed to - avoid the ambiguity of that keyword. - :param byhour: - If given, it must be either an integer, or a sequence of integers, - meaning the hours to apply the recurrence to. - :param byminute: - If given, it must be either an integer, or a sequence of integers, - meaning the minutes to apply the recurrence to. - :param bysecond: - If given, it must be either an integer, or a sequence of integers, - meaning the seconds to apply the recurrence to. - :param cache: - If given, it must be a boolean value specifying to enable or disable - caching of results. If you will use the same rrule instance multiple - times, enabling caching will improve the performance considerably. - """ - def __init__(self, freq, dtstart=None, - interval=1, wkst=None, count=None, until=None, bysetpos=None, - bymonth=None, bymonthday=None, byyearday=None, byeaster=None, - byweekno=None, byweekday=None, - byhour=None, byminute=None, bysecond=None, - cache=False): - super(rrule, self).__init__(cache) - global easter - if not dtstart: - if until and until.tzinfo: - dtstart = datetime.datetime.now(tz=until.tzinfo).replace(microsecond=0) - else: - dtstart = datetime.datetime.now().replace(microsecond=0) - elif not isinstance(dtstart, datetime.datetime): - dtstart = datetime.datetime.fromordinal(dtstart.toordinal()) - else: - dtstart = dtstart.replace(microsecond=0) - self._dtstart = dtstart - self._tzinfo = dtstart.tzinfo - self._freq = freq - self._interval = interval - self._count = count - - # Cache the original byxxx rules, if they are provided, as the _byxxx - # attributes do not necessarily map to the inputs, and this can be - # a problem in generating the strings. Only store things if they've - # been supplied (the string retrieval will just use .get()) - self._original_rule = {} - - if until and not isinstance(until, datetime.datetime): - until = datetime.datetime.fromordinal(until.toordinal()) - self._until = until - - if self._dtstart and self._until: - if (self._dtstart.tzinfo is not None) != (self._until.tzinfo is not None): - # According to RFC5545 Section 3.3.10: - # https://tools.ietf.org/html/rfc5545#section-3.3.10 - # - # > If the "DTSTART" property is specified as a date with UTC - # > time or a date with local time and time zone reference, - # > then the UNTIL rule part MUST be specified as a date with - # > UTC time. - raise ValueError( - 'RRULE UNTIL values must be specified in UTC when DTSTART ' - 'is timezone-aware' - ) - - if count is not None and until: - warn("Using both 'count' and 'until' is inconsistent with RFC 5545" - " and has been deprecated in dateutil. Future versions will " - "raise an error.", DeprecationWarning) - - if wkst is None: - self._wkst = calendar.firstweekday() - elif isinstance(wkst, integer_types): - self._wkst = wkst - else: - self._wkst = wkst.weekday - - if bysetpos is None: - self._bysetpos = None - elif isinstance(bysetpos, integer_types): - if bysetpos == 0 or not (-366 <= bysetpos <= 366): - raise ValueError("bysetpos must be between 1 and 366, " - "or between -366 and -1") - self._bysetpos = (bysetpos,) - else: - self._bysetpos = tuple(bysetpos) - for pos in self._bysetpos: - if pos == 0 or not (-366 <= pos <= 366): - raise ValueError("bysetpos must be between 1 and 366, " - "or between -366 and -1") - - if self._bysetpos: - self._original_rule['bysetpos'] = self._bysetpos - - if (byweekno is None and byyearday is None and bymonthday is None and - byweekday is None and byeaster is None): - if freq == YEARLY: - if bymonth is None: - bymonth = dtstart.month - self._original_rule['bymonth'] = None - bymonthday = dtstart.day - self._original_rule['bymonthday'] = None - elif freq == MONTHLY: - bymonthday = dtstart.day - self._original_rule['bymonthday'] = None - elif freq == WEEKLY: - byweekday = dtstart.weekday() - self._original_rule['byweekday'] = None - - # bymonth - if bymonth is None: - self._bymonth = None - else: - if isinstance(bymonth, integer_types): - bymonth = (bymonth,) - - self._bymonth = tuple(sorted(set(bymonth))) - - if 'bymonth' not in self._original_rule: - self._original_rule['bymonth'] = self._bymonth - - # byyearday - if byyearday is None: - self._byyearday = None - else: - if isinstance(byyearday, integer_types): - byyearday = (byyearday,) - - self._byyearday = tuple(sorted(set(byyearday))) - self._original_rule['byyearday'] = self._byyearday - - # byeaster - if byeaster is not None: - if not easter: - from dateutil import easter - if isinstance(byeaster, integer_types): - self._byeaster = (byeaster,) - else: - self._byeaster = tuple(sorted(byeaster)) - - self._original_rule['byeaster'] = self._byeaster - else: - self._byeaster = None - - # bymonthday - if bymonthday is None: - self._bymonthday = () - self._bynmonthday = () - else: - if isinstance(bymonthday, integer_types): - bymonthday = (bymonthday,) - - bymonthday = set(bymonthday) # Ensure it's unique - - self._bymonthday = tuple(sorted(x for x in bymonthday if x > 0)) - self._bynmonthday = tuple(sorted(x for x in bymonthday if x < 0)) - - # Storing positive numbers first, then negative numbers - if 'bymonthday' not in self._original_rule: - self._original_rule['bymonthday'] = tuple( - itertools.chain(self._bymonthday, self._bynmonthday)) - - # byweekno - if byweekno is None: - self._byweekno = None - else: - if isinstance(byweekno, integer_types): - byweekno = (byweekno,) - - self._byweekno = tuple(sorted(set(byweekno))) - - self._original_rule['byweekno'] = self._byweekno - - # byweekday / bynweekday - if byweekday is None: - self._byweekday = None - self._bynweekday = None - else: - # If it's one of the valid non-sequence types, convert to a - # single-element sequence before the iterator that builds the - # byweekday set. - if isinstance(byweekday, integer_types) or hasattr(byweekday, "n"): - byweekday = (byweekday,) - - self._byweekday = set() - self._bynweekday = set() - for wday in byweekday: - if isinstance(wday, integer_types): - self._byweekday.add(wday) - elif not wday.n or freq > MONTHLY: - self._byweekday.add(wday.weekday) - else: - self._bynweekday.add((wday.weekday, wday.n)) - - if not self._byweekday: - self._byweekday = None - elif not self._bynweekday: - self._bynweekday = None - - if self._byweekday is not None: - self._byweekday = tuple(sorted(self._byweekday)) - orig_byweekday = [weekday(x) for x in self._byweekday] - else: - orig_byweekday = () - - if self._bynweekday is not None: - self._bynweekday = tuple(sorted(self._bynweekday)) - orig_bynweekday = [weekday(*x) for x in self._bynweekday] - else: - orig_bynweekday = () - - if 'byweekday' not in self._original_rule: - self._original_rule['byweekday'] = tuple(itertools.chain( - orig_byweekday, orig_bynweekday)) - - # byhour - if byhour is None: - if freq < HOURLY: - self._byhour = {dtstart.hour} - else: - self._byhour = None - else: - if isinstance(byhour, integer_types): - byhour = (byhour,) - - if freq == HOURLY: - self._byhour = self.__construct_byset(start=dtstart.hour, - byxxx=byhour, - base=24) - else: - self._byhour = set(byhour) - - self._byhour = tuple(sorted(self._byhour)) - self._original_rule['byhour'] = self._byhour - - # byminute - if byminute is None: - if freq < MINUTELY: - self._byminute = {dtstart.minute} - else: - self._byminute = None - else: - if isinstance(byminute, integer_types): - byminute = (byminute,) - - if freq == MINUTELY: - self._byminute = self.__construct_byset(start=dtstart.minute, - byxxx=byminute, - base=60) - else: - self._byminute = set(byminute) - - self._byminute = tuple(sorted(self._byminute)) - self._original_rule['byminute'] = self._byminute - - # bysecond - if bysecond is None: - if freq < SECONDLY: - self._bysecond = ((dtstart.second,)) - else: - self._bysecond = None - else: - if isinstance(bysecond, integer_types): - bysecond = (bysecond,) - - self._bysecond = set(bysecond) - - if freq == SECONDLY: - self._bysecond = self.__construct_byset(start=dtstart.second, - byxxx=bysecond, - base=60) - else: - self._bysecond = set(bysecond) - - self._bysecond = tuple(sorted(self._bysecond)) - self._original_rule['bysecond'] = self._bysecond - - if self._freq >= HOURLY: - self._timeset = None - else: - self._timeset = [] - for hour in self._byhour: - for minute in self._byminute: - for second in self._bysecond: - self._timeset.append( - datetime.time(hour, minute, second, - tzinfo=self._tzinfo)) - self._timeset.sort() - self._timeset = tuple(self._timeset) - - def __str__(self): - """ - Output a string that would generate this RRULE if passed to rrulestr. - This is mostly compatible with RFC5545, except for the - dateutil-specific extension BYEASTER. - """ - - output = [] - h, m, s = [None] * 3 - if self._dtstart: - output.append(self._dtstart.strftime('DTSTART:%Y%m%dT%H%M%S')) - h, m, s = self._dtstart.timetuple()[3:6] - - parts = ['FREQ=' + FREQNAMES[self._freq]] - if self._interval != 1: - parts.append('INTERVAL=' + str(self._interval)) - - if self._wkst: - parts.append('WKST=' + repr(weekday(self._wkst))[0:2]) - - if self._count is not None: - parts.append('COUNT=' + str(self._count)) - - if self._until: - parts.append(self._until.strftime('UNTIL=%Y%m%dT%H%M%S')) - - if self._original_rule.get('byweekday') is not None: - # The str() method on weekday objects doesn't generate - # RFC5545-compliant strings, so we should modify that. - original_rule = dict(self._original_rule) - wday_strings = [] - for wday in original_rule['byweekday']: - if wday.n: - wday_strings.append('{n:+d}{wday}'.format( - n=wday.n, - wday=repr(wday)[0:2])) - else: - wday_strings.append(repr(wday)) - - original_rule['byweekday'] = wday_strings - else: - original_rule = self._original_rule - - partfmt = '{name}={vals}' - for name, key in [('BYSETPOS', 'bysetpos'), - ('BYMONTH', 'bymonth'), - ('BYMONTHDAY', 'bymonthday'), - ('BYYEARDAY', 'byyearday'), - ('BYWEEKNO', 'byweekno'), - ('BYDAY', 'byweekday'), - ('BYHOUR', 'byhour'), - ('BYMINUTE', 'byminute'), - ('BYSECOND', 'bysecond'), - ('BYEASTER', 'byeaster')]: - value = original_rule.get(key) - if value: - parts.append(partfmt.format(name=name, vals=(','.join(str(v) - for v in value)))) - - output.append('RRULE:' + ';'.join(parts)) - return '\n'.join(output) - - def replace(self, **kwargs): - """Return new rrule with same attributes except for those attributes given new - values by whichever keyword arguments are specified.""" - new_kwargs = {"interval": self._interval, - "count": self._count, - "dtstart": self._dtstart, - "freq": self._freq, - "until": self._until, - "wkst": self._wkst, - "cache": False if self._cache is None else True } - new_kwargs.update(self._original_rule) - new_kwargs.update(kwargs) - return rrule(**new_kwargs) - - def _iter(self): - year, month, day, hour, minute, second, weekday, yearday, _ = \ - self._dtstart.timetuple() - - # Some local variables to speed things up a bit - freq = self._freq - interval = self._interval - wkst = self._wkst - until = self._until - bymonth = self._bymonth - byweekno = self._byweekno - byyearday = self._byyearday - byweekday = self._byweekday - byeaster = self._byeaster - bymonthday = self._bymonthday - bynmonthday = self._bynmonthday - bysetpos = self._bysetpos - byhour = self._byhour - byminute = self._byminute - bysecond = self._bysecond - - ii = _iterinfo(self) - ii.rebuild(year, month) - - getdayset = {YEARLY: ii.ydayset, - MONTHLY: ii.mdayset, - WEEKLY: ii.wdayset, - DAILY: ii.ddayset, - HOURLY: ii.ddayset, - MINUTELY: ii.ddayset, - SECONDLY: ii.ddayset}[freq] - - if freq < HOURLY: - timeset = self._timeset - else: - gettimeset = {HOURLY: ii.htimeset, - MINUTELY: ii.mtimeset, - SECONDLY: ii.stimeset}[freq] - if ((freq >= HOURLY and - self._byhour and hour not in self._byhour) or - (freq >= MINUTELY and - self._byminute and minute not in self._byminute) or - (freq >= SECONDLY and - self._bysecond and second not in self._bysecond)): - timeset = () - else: - timeset = gettimeset(hour, minute, second) - - total = 0 - count = self._count - while True: - # Get dayset with the right frequency - dayset, start, end = getdayset(year, month, day) - - # Do the "hard" work ;-) - filtered = False - for i in dayset[start:end]: - if ((bymonth and ii.mmask[i] not in bymonth) or - (byweekno and not ii.wnomask[i]) or - (byweekday and ii.wdaymask[i] not in byweekday) or - (ii.nwdaymask and not ii.nwdaymask[i]) or - (byeaster and not ii.eastermask[i]) or - ((bymonthday or bynmonthday) and - ii.mdaymask[i] not in bymonthday and - ii.nmdaymask[i] not in bynmonthday) or - (byyearday and - ((i < ii.yearlen and i+1 not in byyearday and - -ii.yearlen+i not in byyearday) or - (i >= ii.yearlen and i+1-ii.yearlen not in byyearday and - -ii.nextyearlen+i-ii.yearlen not in byyearday)))): - dayset[i] = None - filtered = True - - # Output results - if bysetpos and timeset: - poslist = [] - for pos in bysetpos: - if pos < 0: - daypos, timepos = divmod(pos, len(timeset)) - else: - daypos, timepos = divmod(pos-1, len(timeset)) - try: - i = [x for x in dayset[start:end] - if x is not None][daypos] - time = timeset[timepos] - except IndexError: - pass - else: - date = datetime.date.fromordinal(ii.yearordinal+i) - res = datetime.datetime.combine(date, time) - if res not in poslist: - poslist.append(res) - poslist.sort() - for res in poslist: - if until and res > until: - self._len = total - return - elif res >= self._dtstart: - if count is not None: - count -= 1 - if count < 0: - self._len = total - return - total += 1 - yield res - else: - for i in dayset[start:end]: - if i is not None: - date = datetime.date.fromordinal(ii.yearordinal + i) - for time in timeset: - res = datetime.datetime.combine(date, time) - if until and res > until: - self._len = total - return - elif res >= self._dtstart: - if count is not None: - count -= 1 - if count < 0: - self._len = total - return - - total += 1 - yield res - - # Handle frequency and interval - fixday = False - if freq == YEARLY: - year += interval - if year > datetime.MAXYEAR: - self._len = total - return - ii.rebuild(year, month) - elif freq == MONTHLY: - month += interval - if month > 12: - div, mod = divmod(month, 12) - month = mod - year += div - if month == 0: - month = 12 - year -= 1 - if year > datetime.MAXYEAR: - self._len = total - return - ii.rebuild(year, month) - elif freq == WEEKLY: - if wkst > weekday: - day += -(weekday+1+(6-wkst))+self._interval*7 - else: - day += -(weekday-wkst)+self._interval*7 - weekday = wkst - fixday = True - elif freq == DAILY: - day += interval - fixday = True - elif freq == HOURLY: - if filtered: - # Jump to one iteration before next day - hour += ((23-hour)//interval)*interval - - if byhour: - ndays, hour = self.__mod_distance(value=hour, - byxxx=self._byhour, - base=24) - else: - ndays, hour = divmod(hour+interval, 24) - - if ndays: - day += ndays - fixday = True - - timeset = gettimeset(hour, minute, second) - elif freq == MINUTELY: - if filtered: - # Jump to one iteration before next day - minute += ((1439-(hour*60+minute))//interval)*interval - - valid = False - rep_rate = (24*60) - for j in range(rep_rate // gcd(interval, rep_rate)): - if byminute: - nhours, minute = \ - self.__mod_distance(value=minute, - byxxx=self._byminute, - base=60) - else: - nhours, minute = divmod(minute+interval, 60) - - div, hour = divmod(hour+nhours, 24) - if div: - day += div - fixday = True - filtered = False - - if not byhour or hour in byhour: - valid = True - break - - if not valid: - raise ValueError('Invalid combination of interval and ' + - 'byhour resulting in empty rule.') - - timeset = gettimeset(hour, minute, second) - elif freq == SECONDLY: - if filtered: - # Jump to one iteration before next day - second += (((86399 - (hour * 3600 + minute * 60 + second)) - // interval) * interval) - - rep_rate = (24 * 3600) - valid = False - for j in range(0, rep_rate // gcd(interval, rep_rate)): - if bysecond: - nminutes, second = \ - self.__mod_distance(value=second, - byxxx=self._bysecond, - base=60) - else: - nminutes, second = divmod(second+interval, 60) - - div, minute = divmod(minute+nminutes, 60) - if div: - hour += div - div, hour = divmod(hour, 24) - if div: - day += div - fixday = True - - if ((not byhour or hour in byhour) and - (not byminute or minute in byminute) and - (not bysecond or second in bysecond)): - valid = True - break - - if not valid: - raise ValueError('Invalid combination of interval, ' + - 'byhour and byminute resulting in empty' + - ' rule.') - - timeset = gettimeset(hour, minute, second) - - if fixday and day > 28: - daysinmonth = calendar.monthrange(year, month)[1] - if day > daysinmonth: - while day > daysinmonth: - day -= daysinmonth - month += 1 - if month == 13: - month = 1 - year += 1 - if year > datetime.MAXYEAR: - self._len = total - return - daysinmonth = calendar.monthrange(year, month)[1] - ii.rebuild(year, month) - - def __construct_byset(self, start, byxxx, base): - """ - If a `BYXXX` sequence is passed to the constructor at the same level as - `FREQ` (e.g. `FREQ=HOURLY,BYHOUR={2,4,7},INTERVAL=3`), there are some - specifications which cannot be reached given some starting conditions. - - This occurs whenever the interval is not coprime with the base of a - given unit and the difference between the starting position and the - ending position is not coprime with the greatest common denominator - between the interval and the base. For example, with a FREQ of hourly - starting at 17:00 and an interval of 4, the only valid values for - BYHOUR would be {21, 1, 5, 9, 13, 17}, because 4 and 24 are not - coprime. - - :param start: - Specifies the starting position. - :param byxxx: - An iterable containing the list of allowed values. - :param base: - The largest allowable value for the specified frequency (e.g. - 24 hours, 60 minutes). - - This does not preserve the type of the iterable, returning a set, since - the values should be unique and the order is irrelevant, this will - speed up later lookups. - - In the event of an empty set, raises a :exception:`ValueError`, as this - results in an empty rrule. - """ - - cset = set() - - # Support a single byxxx value. - if isinstance(byxxx, integer_types): - byxxx = (byxxx, ) - - for num in byxxx: - i_gcd = gcd(self._interval, base) - # Use divmod rather than % because we need to wrap negative nums. - if i_gcd == 1 or divmod(num - start, i_gcd)[1] == 0: - cset.add(num) - - if len(cset) == 0: - raise ValueError("Invalid rrule byxxx generates an empty set.") - - return cset - - def __mod_distance(self, value, byxxx, base): - """ - Calculates the next value in a sequence where the `FREQ` parameter is - specified along with a `BYXXX` parameter at the same "level" - (e.g. `HOURLY` specified with `BYHOUR`). - - :param value: - The old value of the component. - :param byxxx: - The `BYXXX` set, which should have been generated by - `rrule._construct_byset`, or something else which checks that a - valid rule is present. - :param base: - The largest allowable value for the specified frequency (e.g. - 24 hours, 60 minutes). - - If a valid value is not found after `base` iterations (the maximum - number before the sequence would start to repeat), this raises a - :exception:`ValueError`, as no valid values were found. - - This returns a tuple of `divmod(n*interval, base)`, where `n` is the - smallest number of `interval` repetitions until the next specified - value in `byxxx` is found. - """ - accumulator = 0 - for ii in range(1, base + 1): - # Using divmod() over % to account for negative intervals - div, value = divmod(value + self._interval, base) - accumulator += div - if value in byxxx: - return (accumulator, value) - - -class _iterinfo(object): - __slots__ = ["rrule", "lastyear", "lastmonth", - "yearlen", "nextyearlen", "yearordinal", "yearweekday", - "mmask", "mrange", "mdaymask", "nmdaymask", - "wdaymask", "wnomask", "nwdaymask", "eastermask"] - - def __init__(self, rrule): - for attr in self.__slots__: - setattr(self, attr, None) - self.rrule = rrule - - def rebuild(self, year, month): - # Every mask is 7 days longer to handle cross-year weekly periods. - rr = self.rrule - if year != self.lastyear: - self.yearlen = 365 + calendar.isleap(year) - self.nextyearlen = 365 + calendar.isleap(year + 1) - firstyday = datetime.date(year, 1, 1) - self.yearordinal = firstyday.toordinal() - self.yearweekday = firstyday.weekday() - - wday = datetime.date(year, 1, 1).weekday() - if self.yearlen == 365: - self.mmask = M365MASK - self.mdaymask = MDAY365MASK - self.nmdaymask = NMDAY365MASK - self.wdaymask = WDAYMASK[wday:] - self.mrange = M365RANGE - else: - self.mmask = M366MASK - self.mdaymask = MDAY366MASK - self.nmdaymask = NMDAY366MASK - self.wdaymask = WDAYMASK[wday:] - self.mrange = M366RANGE - - if not rr._byweekno: - self.wnomask = None - else: - self.wnomask = [0]*(self.yearlen+7) - # no1wkst = firstwkst = self.wdaymask.index(rr._wkst) - no1wkst = firstwkst = (7-self.yearweekday+rr._wkst) % 7 - if no1wkst >= 4: - no1wkst = 0 - # Number of days in the year, plus the days we got - # from last year. - wyearlen = self.yearlen+(self.yearweekday-rr._wkst) % 7 - else: - # Number of days in the year, minus the days we - # left in last year. - wyearlen = self.yearlen-no1wkst - div, mod = divmod(wyearlen, 7) - numweeks = div+mod//4 - for n in rr._byweekno: - if n < 0: - n += numweeks+1 - if not (0 < n <= numweeks): - continue - if n > 1: - i = no1wkst+(n-1)*7 - if no1wkst != firstwkst: - i -= 7-firstwkst - else: - i = no1wkst - for j in range(7): - self.wnomask[i] = 1 - i += 1 - if self.wdaymask[i] == rr._wkst: - break - if 1 in rr._byweekno: - # Check week number 1 of next year as well - # TODO: Check -numweeks for next year. - i = no1wkst+numweeks*7 - if no1wkst != firstwkst: - i -= 7-firstwkst - if i < self.yearlen: - # If week starts in next year, we - # don't care about it. - for j in range(7): - self.wnomask[i] = 1 - i += 1 - if self.wdaymask[i] == rr._wkst: - break - if no1wkst: - # Check last week number of last year as - # well. If no1wkst is 0, either the year - # started on week start, or week number 1 - # got days from last year, so there are no - # days from last year's last week number in - # this year. - if -1 not in rr._byweekno: - lyearweekday = datetime.date(year-1, 1, 1).weekday() - lno1wkst = (7-lyearweekday+rr._wkst) % 7 - lyearlen = 365+calendar.isleap(year-1) - if lno1wkst >= 4: - lno1wkst = 0 - lnumweeks = 52+(lyearlen + - (lyearweekday-rr._wkst) % 7) % 7//4 - else: - lnumweeks = 52+(self.yearlen-no1wkst) % 7//4 - else: - lnumweeks = -1 - if lnumweeks in rr._byweekno: - for i in range(no1wkst): - self.wnomask[i] = 1 - - if (rr._bynweekday and (month != self.lastmonth or - year != self.lastyear)): - ranges = [] - if rr._freq == YEARLY: - if rr._bymonth: - for month in rr._bymonth: - ranges.append(self.mrange[month-1:month+1]) - else: - ranges = [(0, self.yearlen)] - elif rr._freq == MONTHLY: - ranges = [self.mrange[month-1:month+1]] - if ranges: - # Weekly frequency won't get here, so we may not - # care about cross-year weekly periods. - self.nwdaymask = [0]*self.yearlen - for first, last in ranges: - last -= 1 - for wday, n in rr._bynweekday: - if n < 0: - i = last+(n+1)*7 - i -= (self.wdaymask[i]-wday) % 7 - else: - i = first+(n-1)*7 - i += (7-self.wdaymask[i]+wday) % 7 - if first <= i <= last: - self.nwdaymask[i] = 1 - - if rr._byeaster: - self.eastermask = [0]*(self.yearlen+7) - eyday = easter.easter(year).toordinal()-self.yearordinal - for offset in rr._byeaster: - self.eastermask[eyday+offset] = 1 - - self.lastyear = year - self.lastmonth = month - - def ydayset(self, year, month, day): - return list(range(self.yearlen)), 0, self.yearlen - - def mdayset(self, year, month, day): - dset = [None]*self.yearlen - start, end = self.mrange[month-1:month+1] - for i in range(start, end): - dset[i] = i - return dset, start, end - - def wdayset(self, year, month, day): - # We need to handle cross-year weeks here. - dset = [None]*(self.yearlen+7) - i = datetime.date(year, month, day).toordinal()-self.yearordinal - start = i - for j in range(7): - dset[i] = i - i += 1 - # if (not (0 <= i < self.yearlen) or - # self.wdaymask[i] == self.rrule._wkst): - # This will cross the year boundary, if necessary. - if self.wdaymask[i] == self.rrule._wkst: - break - return dset, start, i - - def ddayset(self, year, month, day): - dset = [None] * self.yearlen - i = datetime.date(year, month, day).toordinal() - self.yearordinal - dset[i] = i - return dset, i, i + 1 - - def htimeset(self, hour, minute, second): - tset = [] - rr = self.rrule - for minute in rr._byminute: - for second in rr._bysecond: - tset.append(datetime.time(hour, minute, second, - tzinfo=rr._tzinfo)) - tset.sort() - return tset - - def mtimeset(self, hour, minute, second): - tset = [] - rr = self.rrule - for second in rr._bysecond: - tset.append(datetime.time(hour, minute, second, tzinfo=rr._tzinfo)) - tset.sort() - return tset - - def stimeset(self, hour, minute, second): - return (datetime.time(hour, minute, second, - tzinfo=self.rrule._tzinfo),) - - -class rruleset(rrulebase): - """ The rruleset type allows more complex recurrence setups, mixing - multiple rules, dates, exclusion rules, and exclusion dates. The type - constructor takes the following keyword arguments: - - :param cache: If True, caching of results will be enabled, improving - performance of multiple queries considerably. """ - - class _genitem(object): - def __init__(self, genlist, gen): - try: - self.dt = advance_iterator(gen) - genlist.append(self) - except StopIteration: - pass - self.genlist = genlist - self.gen = gen - - def __next__(self): - try: - self.dt = advance_iterator(self.gen) - except StopIteration: - if self.genlist[0] is self: - heapq.heappop(self.genlist) - else: - self.genlist.remove(self) - heapq.heapify(self.genlist) - - next = __next__ - - def __lt__(self, other): - return self.dt < other.dt - - def __gt__(self, other): - return self.dt > other.dt - - def __eq__(self, other): - return self.dt == other.dt - - def __ne__(self, other): - return self.dt != other.dt - - def __init__(self, cache=False): - super(rruleset, self).__init__(cache) - self._rrule = [] - self._rdate = [] - self._exrule = [] - self._exdate = [] - - @_invalidates_cache - def rrule(self, rrule): - """ Include the given :py:class:`rrule` instance in the recurrence set - generation. """ - self._rrule.append(rrule) - - @_invalidates_cache - def rdate(self, rdate): - """ Include the given :py:class:`datetime` instance in the recurrence - set generation. """ - self._rdate.append(rdate) - - @_invalidates_cache - def exrule(self, exrule): - """ Include the given rrule instance in the recurrence set exclusion - list. Dates which are part of the given recurrence rules will not - be generated, even if some inclusive rrule or rdate matches them. - """ - self._exrule.append(exrule) - - @_invalidates_cache - def exdate(self, exdate): - """ Include the given datetime instance in the recurrence set - exclusion list. Dates included that way will not be generated, - even if some inclusive rrule or rdate matches them. """ - self._exdate.append(exdate) - - def _iter(self): - rlist = [] - self._rdate.sort() - self._genitem(rlist, iter(self._rdate)) - for gen in [iter(x) for x in self._rrule]: - self._genitem(rlist, gen) - exlist = [] - self._exdate.sort() - self._genitem(exlist, iter(self._exdate)) - for gen in [iter(x) for x in self._exrule]: - self._genitem(exlist, gen) - lastdt = None - total = 0 - heapq.heapify(rlist) - heapq.heapify(exlist) - while rlist: - ritem = rlist[0] - if not lastdt or lastdt != ritem.dt: - while exlist and exlist[0] < ritem: - exitem = exlist[0] - advance_iterator(exitem) - if exlist and exlist[0] is exitem: - heapq.heapreplace(exlist, exitem) - if not exlist or ritem != exlist[0]: - total += 1 - yield ritem.dt - lastdt = ritem.dt - advance_iterator(ritem) - if rlist and rlist[0] is ritem: - heapq.heapreplace(rlist, ritem) - self._len = total - - - - -class _rrulestr(object): - """ Parses a string representation of a recurrence rule or set of - recurrence rules. - - :param s: - Required, a string defining one or more recurrence rules. - - :param dtstart: - If given, used as the default recurrence start if not specified in the - rule string. - - :param cache: - If set ``True`` caching of results will be enabled, improving - performance of multiple queries considerably. - - :param unfold: - If set ``True`` indicates that a rule string is split over more - than one line and should be joined before processing. - - :param forceset: - If set ``True`` forces a :class:`dateutil.rrule.rruleset` to - be returned. - - :param compatible: - If set ``True`` forces ``unfold`` and ``forceset`` to be ``True``. - - :param ignoretz: - If set ``True``, time zones in parsed strings are ignored and a naive - :class:`datetime.datetime` object is returned. - - :param tzids: - If given, a callable or mapping used to retrieve a - :class:`datetime.tzinfo` from a string representation. - Defaults to :func:`dateutil.tz.gettz`. - - :param tzinfos: - Additional time zone names / aliases which may be present in a string - representation. See :func:`dateutil.parser.parse` for more - information. - - :return: - Returns a :class:`dateutil.rrule.rruleset` or - :class:`dateutil.rrule.rrule` - """ - - _freq_map = {"YEARLY": YEARLY, - "MONTHLY": MONTHLY, - "WEEKLY": WEEKLY, - "DAILY": DAILY, - "HOURLY": HOURLY, - "MINUTELY": MINUTELY, - "SECONDLY": SECONDLY} - - _weekday_map = {"MO": 0, "TU": 1, "WE": 2, "TH": 3, - "FR": 4, "SA": 5, "SU": 6} - - def _handle_int(self, rrkwargs, name, value, **kwargs): - rrkwargs[name.lower()] = int(value) - - def _handle_int_list(self, rrkwargs, name, value, **kwargs): - rrkwargs[name.lower()] = [int(x) for x in value.split(',')] - - _handle_INTERVAL = _handle_int - _handle_COUNT = _handle_int - _handle_BYSETPOS = _handle_int_list - _handle_BYMONTH = _handle_int_list - _handle_BYMONTHDAY = _handle_int_list - _handle_BYYEARDAY = _handle_int_list - _handle_BYEASTER = _handle_int_list - _handle_BYWEEKNO = _handle_int_list - _handle_BYHOUR = _handle_int_list - _handle_BYMINUTE = _handle_int_list - _handle_BYSECOND = _handle_int_list - - def _handle_FREQ(self, rrkwargs, name, value, **kwargs): - rrkwargs["freq"] = self._freq_map[value] - - def _handle_UNTIL(self, rrkwargs, name, value, **kwargs): - global parser - if not parser: - from dateutil import parser - try: - rrkwargs["until"] = parser.parse(value, - ignoretz=kwargs.get("ignoretz"), - tzinfos=kwargs.get("tzinfos")) - except ValueError: - raise ValueError("invalid until date") - - def _handle_WKST(self, rrkwargs, name, value, **kwargs): - rrkwargs["wkst"] = self._weekday_map[value] - - def _handle_BYWEEKDAY(self, rrkwargs, name, value, **kwargs): - """ - Two ways to specify this: +1MO or MO(+1) - """ - l = [] - for wday in value.split(','): - if '(' in wday: - # If it's of the form TH(+1), etc. - splt = wday.split('(') - w = splt[0] - n = int(splt[1][:-1]) - elif len(wday): - # If it's of the form +1MO - for i in range(len(wday)): - if wday[i] not in '+-0123456789': - break - n = wday[:i] or None - w = wday[i:] - if n: - n = int(n) - else: - raise ValueError("Invalid (empty) BYDAY specification.") - - l.append(weekdays[self._weekday_map[w]](n)) - rrkwargs["byweekday"] = l - - _handle_BYDAY = _handle_BYWEEKDAY - - def _parse_rfc_rrule(self, line, - dtstart=None, - cache=False, - ignoretz=False, - tzinfos=None): - if line.find(':') != -1: - name, value = line.split(':') - if name != "RRULE": - raise ValueError("unknown parameter name") - else: - value = line - rrkwargs = {} - for pair in value.split(';'): - name, value = pair.split('=') - name = name.upper() - value = value.upper() - try: - getattr(self, "_handle_"+name)(rrkwargs, name, value, - ignoretz=ignoretz, - tzinfos=tzinfos) - except AttributeError: - raise ValueError("unknown parameter '%s'" % name) - except (KeyError, ValueError): - raise ValueError("invalid '%s': %s" % (name, value)) - return rrule(dtstart=dtstart, cache=cache, **rrkwargs) - - def _parse_date_value(self, date_value, parms, rule_tzids, - ignoretz, tzids, tzinfos): - global parser - if not parser: - from dateutil import parser - - datevals = [] - value_found = False - TZID = None - - for parm in parms: - if parm.startswith("TZID="): - try: - tzkey = rule_tzids[parm.split('TZID=')[-1]] - except KeyError: - continue - if tzids is None: - from . import tz - tzlookup = tz.gettz - elif callable(tzids): - tzlookup = tzids - else: - tzlookup = getattr(tzids, 'get', None) - if tzlookup is None: - msg = ('tzids must be a callable, mapping, or None, ' - 'not %s' % tzids) - raise ValueError(msg) - - TZID = tzlookup(tzkey) - continue - - # RFC 5445 3.8.2.4: The VALUE parameter is optional, but may be found - # only once. - if parm not in {"VALUE=DATE-TIME", "VALUE=DATE"}: - raise ValueError("unsupported parm: " + parm) - else: - if value_found: - msg = ("Duplicate value parameter found in: " + parm) - raise ValueError(msg) - value_found = True - - for datestr in date_value.split(','): - date = parser.parse(datestr, ignoretz=ignoretz, tzinfos=tzinfos) - if TZID is not None: - if date.tzinfo is None: - date = date.replace(tzinfo=TZID) - else: - raise ValueError('DTSTART/EXDATE specifies multiple timezone') - datevals.append(date) - - return datevals - - def _parse_rfc(self, s, - dtstart=None, - cache=False, - unfold=False, - forceset=False, - compatible=False, - ignoretz=False, - tzids=None, - tzinfos=None): - global parser - if compatible: - forceset = True - unfold = True - - TZID_NAMES = dict(map( - lambda x: (x.upper(), x), - re.findall('TZID=(?P[^:]+):', s) - )) - s = s.upper() - if not s.strip(): - raise ValueError("empty string") - if unfold: - lines = s.splitlines() - i = 0 - while i < len(lines): - line = lines[i].rstrip() - if not line: - del lines[i] - elif i > 0 and line[0] == " ": - lines[i-1] += line[1:] - del lines[i] - else: - i += 1 - else: - lines = s.split() - if (not forceset and len(lines) == 1 and (s.find(':') == -1 or - s.startswith('RRULE:'))): - return self._parse_rfc_rrule(lines[0], cache=cache, - dtstart=dtstart, ignoretz=ignoretz, - tzinfos=tzinfos) - else: - rrulevals = [] - rdatevals = [] - exrulevals = [] - exdatevals = [] - for line in lines: - if not line: - continue - if line.find(':') == -1: - name = "RRULE" - value = line - else: - name, value = line.split(':', 1) - parms = name.split(';') - if not parms: - raise ValueError("empty property name") - name = parms[0] - parms = parms[1:] - if name == "RRULE": - for parm in parms: - raise ValueError("unsupported RRULE parm: "+parm) - rrulevals.append(value) - elif name == "RDATE": - for parm in parms: - if parm != "VALUE=DATE-TIME": - raise ValueError("unsupported RDATE parm: "+parm) - rdatevals.append(value) - elif name == "EXRULE": - for parm in parms: - raise ValueError("unsupported EXRULE parm: "+parm) - exrulevals.append(value) - elif name == "EXDATE": - exdatevals.extend( - self._parse_date_value(value, parms, - TZID_NAMES, ignoretz, - tzids, tzinfos) - ) - elif name == "DTSTART": - dtvals = self._parse_date_value(value, parms, TZID_NAMES, - ignoretz, tzids, tzinfos) - if len(dtvals) != 1: - raise ValueError("Multiple DTSTART values specified:" + - value) - dtstart = dtvals[0] - else: - raise ValueError("unsupported property: "+name) - if (forceset or len(rrulevals) > 1 or rdatevals - or exrulevals or exdatevals): - if not parser and (rdatevals or exdatevals): - from dateutil import parser - rset = rruleset(cache=cache) - for value in rrulevals: - rset.rrule(self._parse_rfc_rrule(value, dtstart=dtstart, - ignoretz=ignoretz, - tzinfos=tzinfos)) - for value in rdatevals: - for datestr in value.split(','): - rset.rdate(parser.parse(datestr, - ignoretz=ignoretz, - tzinfos=tzinfos)) - for value in exrulevals: - rset.exrule(self._parse_rfc_rrule(value, dtstart=dtstart, - ignoretz=ignoretz, - tzinfos=tzinfos)) - for value in exdatevals: - rset.exdate(value) - if compatible and dtstart: - rset.rdate(dtstart) - return rset - else: - return self._parse_rfc_rrule(rrulevals[0], - dtstart=dtstart, - cache=cache, - ignoretz=ignoretz, - tzinfos=tzinfos) - - def __call__(self, s, **kwargs): - return self._parse_rfc(s, **kwargs) - - -rrulestr = _rrulestr() - -# vim:ts=4:sw=4:et diff --git a/src/dateutil/tz/__init__.py b/src/dateutil/tz/__init__.py deleted file mode 100644 index 5a2d9cd6..00000000 --- a/src/dateutil/tz/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -from .tz import * -from .tz import __doc__ - -#: Convenience constant providing a :class:`tzutc()` instance -#: -#: .. versionadded:: 2.7.0 -UTC = tzutc() - -__all__ = ["tzutc", "tzoffset", "tzlocal", "tzfile", "tzrange", - "tzstr", "tzical", "tzwin", "tzwinlocal", "gettz", - "enfold", "datetime_ambiguous", "datetime_exists", - "resolve_imaginary", "UTC", "DeprecatedTzFormatWarning"] - - -class DeprecatedTzFormatWarning(Warning): - """Warning raised when time zones are parsed from deprecated formats.""" diff --git a/src/dateutil/tz/_common.py b/src/dateutil/tz/_common.py deleted file mode 100644 index 594e0823..00000000 --- a/src/dateutil/tz/_common.py +++ /dev/null @@ -1,419 +0,0 @@ -from six import PY2 - -from functools import wraps - -from datetime import datetime, timedelta, tzinfo - - -ZERO = timedelta(0) - -__all__ = ['tzname_in_python2', 'enfold'] - - -def tzname_in_python2(namefunc): - """Change unicode output into bytestrings in Python 2 - - tzname() API changed in Python 3. It used to return bytes, but was changed - to unicode strings - """ - if PY2: - @wraps(namefunc) - def adjust_encoding(*args, **kwargs): - name = namefunc(*args, **kwargs) - if name is not None: - name = name.encode() - - return name - - return adjust_encoding - else: - return namefunc - - -# The following is adapted from Alexander Belopolsky's tz library -# https://github.com/abalkin/tz -if hasattr(datetime, 'fold'): - # This is the pre-python 3.6 fold situation - def enfold(dt, fold=1): - """ - Provides a unified interface for assigning the ``fold`` attribute to - datetimes both before and after the implementation of PEP-495. - - :param fold: - The value for the ``fold`` attribute in the returned datetime. This - should be either 0 or 1. - - :return: - Returns an object for which ``getattr(dt, 'fold', 0)`` returns - ``fold`` for all versions of Python. In versions prior to - Python 3.6, this is a ``_DatetimeWithFold`` object, which is a - subclass of :py:class:`datetime.datetime` with the ``fold`` - attribute added, if ``fold`` is 1. - - .. versionadded:: 2.6.0 - """ - return dt.replace(fold=fold) - -else: - class _DatetimeWithFold(datetime): - """ - This is a class designed to provide a PEP 495-compliant interface for - Python versions before 3.6. It is used only for dates in a fold, so - the ``fold`` attribute is fixed at ``1``. - - .. versionadded:: 2.6.0 - """ - __slots__ = () - - def replace(self, *args, **kwargs): - """ - Return a datetime with the same attributes, except for those - attributes given new values by whichever keyword arguments are - specified. Note that tzinfo=None can be specified to create a naive - datetime from an aware datetime with no conversion of date and time - data. - - This is reimplemented in ``_DatetimeWithFold`` because pypy3 will - return a ``datetime.datetime`` even if ``fold`` is unchanged. - """ - argnames = ( - 'year', 'month', 'day', 'hour', 'minute', 'second', - 'microsecond', 'tzinfo' - ) - - for arg, argname in zip(args, argnames): - if argname in kwargs: - raise TypeError('Duplicate argument: {}'.format(argname)) - - kwargs[argname] = arg - - for argname in argnames: - if argname not in kwargs: - kwargs[argname] = getattr(self, argname) - - dt_class = self.__class__ if kwargs.get('fold', 1) else datetime - - return dt_class(**kwargs) - - @property - def fold(self): - return 1 - - def enfold(dt, fold=1): - """ - Provides a unified interface for assigning the ``fold`` attribute to - datetimes both before and after the implementation of PEP-495. - - :param fold: - The value for the ``fold`` attribute in the returned datetime. This - should be either 0 or 1. - - :return: - Returns an object for which ``getattr(dt, 'fold', 0)`` returns - ``fold`` for all versions of Python. In versions prior to - Python 3.6, this is a ``_DatetimeWithFold`` object, which is a - subclass of :py:class:`datetime.datetime` with the ``fold`` - attribute added, if ``fold`` is 1. - - .. versionadded:: 2.6.0 - """ - if getattr(dt, 'fold', 0) == fold: - return dt - - args = dt.timetuple()[:6] - args += (dt.microsecond, dt.tzinfo) - - if fold: - return _DatetimeWithFold(*args) - else: - return datetime(*args) - - -def _validate_fromutc_inputs(f): - """ - The CPython version of ``fromutc`` checks that the input is a ``datetime`` - object and that ``self`` is attached as its ``tzinfo``. - """ - @wraps(f) - def fromutc(self, dt): - if not isinstance(dt, datetime): - raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") - - return f(self, dt) - - return fromutc - - -class _tzinfo(tzinfo): - """ - Base class for all ``dateutil`` ``tzinfo`` objects. - """ - - def is_ambiguous(self, dt): - """ - Whether or not the "wall time" of a given datetime is ambiguous in this - zone. - - :param dt: - A :py:class:`datetime.datetime`, naive or time zone aware. - - - :return: - Returns ``True`` if ambiguous, ``False`` otherwise. - - .. versionadded:: 2.6.0 - """ - - dt = dt.replace(tzinfo=self) - - wall_0 = enfold(dt, fold=0) - wall_1 = enfold(dt, fold=1) - - same_offset = wall_0.utcoffset() == wall_1.utcoffset() - same_dt = wall_0.replace(tzinfo=None) == wall_1.replace(tzinfo=None) - - return same_dt and not same_offset - - def _fold_status(self, dt_utc, dt_wall): - """ - Determine the fold status of a "wall" datetime, given a representation - of the same datetime as a (naive) UTC datetime. This is calculated based - on the assumption that ``dt.utcoffset() - dt.dst()`` is constant for all - datetimes, and that this offset is the actual number of hours separating - ``dt_utc`` and ``dt_wall``. - - :param dt_utc: - Representation of the datetime as UTC - - :param dt_wall: - Representation of the datetime as "wall time". This parameter must - either have a `fold` attribute or have a fold-naive - :class:`datetime.tzinfo` attached, otherwise the calculation may - fail. - """ - if self.is_ambiguous(dt_wall): - delta_wall = dt_wall - dt_utc - _fold = int(delta_wall == (dt_utc.utcoffset() - dt_utc.dst())) - else: - _fold = 0 - - return _fold - - def _fold(self, dt): - return getattr(dt, 'fold', 0) - - def _fromutc(self, dt): - """ - Given a timezone-aware datetime in a given timezone, calculates a - timezone-aware datetime in a new timezone. - - Since this is the one time that we *know* we have an unambiguous - datetime object, we take this opportunity to determine whether the - datetime is ambiguous and in a "fold" state (e.g. if it's the first - occurence, chronologically, of the ambiguous datetime). - - :param dt: - A timezone-aware :class:`datetime.datetime` object. - """ - - # Re-implement the algorithm from Python's datetime.py - dtoff = dt.utcoffset() - if dtoff is None: - raise ValueError("fromutc() requires a non-None utcoffset() " - "result") - - # The original datetime.py code assumes that `dst()` defaults to - # zero during ambiguous times. PEP 495 inverts this presumption, so - # for pre-PEP 495 versions of python, we need to tweak the algorithm. - dtdst = dt.dst() - if dtdst is None: - raise ValueError("fromutc() requires a non-None dst() result") - delta = dtoff - dtdst - - dt += delta - # Set fold=1 so we can default to being in the fold for - # ambiguous dates. - dtdst = enfold(dt, fold=1).dst() - if dtdst is None: - raise ValueError("fromutc(): dt.dst gave inconsistent " - "results; cannot convert") - return dt + dtdst - - @_validate_fromutc_inputs - def fromutc(self, dt): - """ - Given a timezone-aware datetime in a given timezone, calculates a - timezone-aware datetime in a new timezone. - - Since this is the one time that we *know* we have an unambiguous - datetime object, we take this opportunity to determine whether the - datetime is ambiguous and in a "fold" state (e.g. if it's the first - occurance, chronologically, of the ambiguous datetime). - - :param dt: - A timezone-aware :class:`datetime.datetime` object. - """ - dt_wall = self._fromutc(dt) - - # Calculate the fold status given the two datetimes. - _fold = self._fold_status(dt, dt_wall) - - # Set the default fold value for ambiguous dates - return enfold(dt_wall, fold=_fold) - - -class tzrangebase(_tzinfo): - """ - This is an abstract base class for time zones represented by an annual - transition into and out of DST. Child classes should implement the following - methods: - - * ``__init__(self, *args, **kwargs)`` - * ``transitions(self, year)`` - this is expected to return a tuple of - datetimes representing the DST on and off transitions in standard - time. - - A fully initialized ``tzrangebase`` subclass should also provide the - following attributes: - * ``hasdst``: Boolean whether or not the zone uses DST. - * ``_dst_offset`` / ``_std_offset``: :class:`datetime.timedelta` objects - representing the respective UTC offsets. - * ``_dst_abbr`` / ``_std_abbr``: Strings representing the timezone short - abbreviations in DST and STD, respectively. - * ``_hasdst``: Whether or not the zone has DST. - - .. versionadded:: 2.6.0 - """ - def __init__(self): - raise NotImplementedError('tzrangebase is an abstract base class') - - def utcoffset(self, dt): - isdst = self._isdst(dt) - - if isdst is None: - return None - elif isdst: - return self._dst_offset - else: - return self._std_offset - - def dst(self, dt): - isdst = self._isdst(dt) - - if isdst is None: - return None - elif isdst: - return self._dst_base_offset - else: - return ZERO - - @tzname_in_python2 - def tzname(self, dt): - if self._isdst(dt): - return self._dst_abbr - else: - return self._std_abbr - - def fromutc(self, dt): - """ Given a datetime in UTC, return local time """ - if not isinstance(dt, datetime): - raise TypeError("fromutc() requires a datetime argument") - - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") - - # Get transitions - if there are none, fixed offset - transitions = self.transitions(dt.year) - if transitions is None: - return dt + self.utcoffset(dt) - - # Get the transition times in UTC - dston, dstoff = transitions - - dston -= self._std_offset - dstoff -= self._std_offset - - utc_transitions = (dston, dstoff) - dt_utc = dt.replace(tzinfo=None) - - isdst = self._naive_isdst(dt_utc, utc_transitions) - - if isdst: - dt_wall = dt + self._dst_offset - else: - dt_wall = dt + self._std_offset - - _fold = int(not isdst and self.is_ambiguous(dt_wall)) - - return enfold(dt_wall, fold=_fold) - - def is_ambiguous(self, dt): - """ - Whether or not the "wall time" of a given datetime is ambiguous in this - zone. - - :param dt: - A :py:class:`datetime.datetime`, naive or time zone aware. - - - :return: - Returns ``True`` if ambiguous, ``False`` otherwise. - - .. versionadded:: 2.6.0 - """ - if not self.hasdst: - return False - - start, end = self.transitions(dt.year) - - dt = dt.replace(tzinfo=None) - return (end <= dt < end + self._dst_base_offset) - - def _isdst(self, dt): - if not self.hasdst: - return False - elif dt is None: - return None - - transitions = self.transitions(dt.year) - - if transitions is None: - return False - - dt = dt.replace(tzinfo=None) - - isdst = self._naive_isdst(dt, transitions) - - # Handle ambiguous dates - if not isdst and self.is_ambiguous(dt): - return not self._fold(dt) - else: - return isdst - - def _naive_isdst(self, dt, transitions): - dston, dstoff = transitions - - dt = dt.replace(tzinfo=None) - - if dston < dstoff: - isdst = dston <= dt < dstoff - else: - isdst = not dstoff <= dt < dston - - return isdst - - @property - def _dst_base_offset(self): - return self._dst_offset - self._std_offset - - __hash__ = None - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - return "%s(...)" % self.__class__.__name__ - - __reduce__ = object.__reduce__ diff --git a/src/dateutil/tz/_factories.py b/src/dateutil/tz/_factories.py deleted file mode 100644 index d2560eb7..00000000 --- a/src/dateutil/tz/_factories.py +++ /dev/null @@ -1,73 +0,0 @@ -from datetime import timedelta -import weakref -from collections import OrderedDict - - -class _TzSingleton(type): - def __init__(cls, *args, **kwargs): - cls.__instance = None - super(_TzSingleton, cls).__init__(*args, **kwargs) - - def __call__(cls): - if cls.__instance is None: - cls.__instance = super(_TzSingleton, cls).__call__() - return cls.__instance - - -class _TzFactory(type): - def instance(cls, *args, **kwargs): - """Alternate constructor that returns a fresh instance""" - return type.__call__(cls, *args, **kwargs) - - -class _TzOffsetFactory(_TzFactory): - def __init__(cls, *args, **kwargs): - cls.__instances = weakref.WeakValueDictionary() - cls.__strong_cache = OrderedDict() - cls.__strong_cache_size = 8 - - def __call__(cls, name, offset): - if isinstance(offset, timedelta): - key = (name, offset.total_seconds()) - else: - key = (name, offset) - - instance = cls.__instances.get(key, None) - if instance is None: - instance = cls.__instances.setdefault(key, - cls.instance(name, offset)) - - cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance) - - # Remove an item if the strong cache is overpopulated - # TODO: Maybe this should be under a lock? - if len(cls.__strong_cache) > cls.__strong_cache_size: - cls.__strong_cache.popitem(last=False) - - return instance - - -class _TzStrFactory(_TzFactory): - def __init__(cls, *args, **kwargs): - cls.__instances = weakref.WeakValueDictionary() - cls.__strong_cache = OrderedDict() - cls.__strong_cache_size = 8 - - def __call__(cls, s, posix_offset=False): - key = (s, posix_offset) - instance = cls.__instances.get(key, None) - - if instance is None: - instance = cls.__instances.setdefault(key, - cls.instance(s, posix_offset)) - - cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance) - - - # Remove an item if the strong cache is overpopulated - # TODO: Maybe this should be under a lock? - if len(cls.__strong_cache) > cls.__strong_cache_size: - cls.__strong_cache.popitem(last=False) - - return instance - diff --git a/src/dateutil/tz/tz.py b/src/dateutil/tz/tz.py deleted file mode 100644 index d05414e7..00000000 --- a/src/dateutil/tz/tz.py +++ /dev/null @@ -1,1836 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module offers timezone implementations subclassing the abstract -:py:class:`datetime.tzinfo` type. There are classes to handle tzfile format -files (usually are in :file:`/etc/localtime`, :file:`/usr/share/zoneinfo`, -etc), TZ environment string (in all known formats), given ranges (with help -from relative deltas), local machine timezone, fixed offset timezone, and UTC -timezone. -""" -import datetime -import struct -import time -import sys -import os -import bisect -import weakref -from collections import OrderedDict - -import six -from six import string_types -from six.moves import _thread -from ._common import tzname_in_python2, _tzinfo -from ._common import tzrangebase, enfold -from ._common import _validate_fromutc_inputs - -from ._factories import _TzSingleton, _TzOffsetFactory -from ._factories import _TzStrFactory -try: - from .win import tzwin, tzwinlocal -except ImportError: - tzwin = tzwinlocal = None - -# For warning about rounding tzinfo -from warnings import warn - -ZERO = datetime.timedelta(0) -EPOCH = datetime.datetime.utcfromtimestamp(0) -EPOCHORDINAL = EPOCH.toordinal() - - -@six.add_metaclass(_TzSingleton) -class tzutc(datetime.tzinfo): - """ - This is a tzinfo object that represents the UTC time zone. - - **Examples:** - - .. doctest:: - - >>> from datetime import * - >>> from dateutil.tz import * - - >>> datetime.now() - datetime.datetime(2003, 9, 27, 9, 40, 1, 521290) - - >>> datetime.now(tzutc()) - datetime.datetime(2003, 9, 27, 12, 40, 12, 156379, tzinfo=tzutc()) - - >>> datetime.now(tzutc()).tzname() - 'UTC' - - .. versionchanged:: 2.7.0 - ``tzutc()`` is now a singleton, so the result of ``tzutc()`` will - always return the same object. - - .. doctest:: - - >>> from dateutil.tz import tzutc, UTC - >>> tzutc() is tzutc() - True - >>> tzutc() is UTC - True - """ - def utcoffset(self, dt): - return ZERO - - def dst(self, dt): - return ZERO - - @tzname_in_python2 - def tzname(self, dt): - return "UTC" - - def is_ambiguous(self, dt): - """ - Whether or not the "wall time" of a given datetime is ambiguous in this - zone. - - :param dt: - A :py:class:`datetime.datetime`, naive or time zone aware. - - - :return: - Returns ``True`` if ambiguous, ``False`` otherwise. - - .. versionadded:: 2.6.0 - """ - return False - - @_validate_fromutc_inputs - def fromutc(self, dt): - """ - Fast track version of fromutc() returns the original ``dt`` object for - any valid :py:class:`datetime.datetime` object. - """ - return dt - - def __eq__(self, other): - if not isinstance(other, (tzutc, tzoffset)): - return NotImplemented - - return (isinstance(other, tzutc) or - (isinstance(other, tzoffset) and other._offset == ZERO)) - - __hash__ = None - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - return "%s()" % self.__class__.__name__ - - __reduce__ = object.__reduce__ - - -@six.add_metaclass(_TzOffsetFactory) -class tzoffset(datetime.tzinfo): - """ - A simple class for representing a fixed offset from UTC. - - :param name: - The timezone name, to be returned when ``tzname()`` is called. - :param offset: - The time zone offset in seconds, or (since version 2.6.0, represented - as a :py:class:`datetime.timedelta` object). - """ - def __init__(self, name, offset): - self._name = name - - try: - # Allow a timedelta - offset = offset.total_seconds() - except (TypeError, AttributeError): - pass - - self._offset = datetime.timedelta(seconds=_get_supported_offset(offset)) - - def utcoffset(self, dt): - return self._offset - - def dst(self, dt): - return ZERO - - @tzname_in_python2 - def tzname(self, dt): - return self._name - - @_validate_fromutc_inputs - def fromutc(self, dt): - return dt + self._offset - - def is_ambiguous(self, dt): - """ - Whether or not the "wall time" of a given datetime is ambiguous in this - zone. - - :param dt: - A :py:class:`datetime.datetime`, naive or time zone aware. - :return: - Returns ``True`` if ambiguous, ``False`` otherwise. - - .. versionadded:: 2.6.0 - """ - return False - - def __eq__(self, other): - if not isinstance(other, tzoffset): - return NotImplemented - - return self._offset == other._offset - - __hash__ = None - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - return "%s(%s, %s)" % (self.__class__.__name__, - repr(self._name), - int(self._offset.total_seconds())) - - __reduce__ = object.__reduce__ - - -class tzlocal(_tzinfo): - """ - A :class:`tzinfo` subclass built around the ``time`` timezone functions. - """ - def __init__(self): - super(tzlocal, self).__init__() - - self._std_offset = datetime.timedelta(seconds=-time.timezone) - if time.daylight: - self._dst_offset = datetime.timedelta(seconds=-time.altzone) - else: - self._dst_offset = self._std_offset - - self._dst_saved = self._dst_offset - self._std_offset - self._hasdst = bool(self._dst_saved) - self._tznames = tuple(time.tzname) - - def utcoffset(self, dt): - if dt is None and self._hasdst: - return None - - if self._isdst(dt): - return self._dst_offset - else: - return self._std_offset - - def dst(self, dt): - if dt is None and self._hasdst: - return None - - if self._isdst(dt): - return self._dst_offset - self._std_offset - else: - return ZERO - - @tzname_in_python2 - def tzname(self, dt): - return self._tznames[self._isdst(dt)] - - def is_ambiguous(self, dt): - """ - Whether or not the "wall time" of a given datetime is ambiguous in this - zone. - - :param dt: - A :py:class:`datetime.datetime`, naive or time zone aware. - - - :return: - Returns ``True`` if ambiguous, ``False`` otherwise. - - .. versionadded:: 2.6.0 - """ - naive_dst = self._naive_is_dst(dt) - return (not naive_dst and - (naive_dst != self._naive_is_dst(dt - self._dst_saved))) - - def _naive_is_dst(self, dt): - timestamp = _datetime_to_timestamp(dt) - return time.localtime(timestamp + time.timezone).tm_isdst - - def _isdst(self, dt, fold_naive=True): - # We can't use mktime here. It is unstable when deciding if - # the hour near to a change is DST or not. - # - # timestamp = time.mktime((dt.year, dt.month, dt.day, dt.hour, - # dt.minute, dt.second, dt.weekday(), 0, -1)) - # return time.localtime(timestamp).tm_isdst - # - # The code above yields the following result: - # - # >>> import tz, datetime - # >>> t = tz.tzlocal() - # >>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() - # 'BRDT' - # >>> datetime.datetime(2003,2,16,0,tzinfo=t).tzname() - # 'BRST' - # >>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() - # 'BRST' - # >>> datetime.datetime(2003,2,15,22,tzinfo=t).tzname() - # 'BRDT' - # >>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() - # 'BRDT' - # - # Here is a more stable implementation: - # - if not self._hasdst: - return False - - # Check for ambiguous times: - dstval = self._naive_is_dst(dt) - fold = getattr(dt, 'fold', None) - - if self.is_ambiguous(dt): - if fold is not None: - return not self._fold(dt) - else: - return True - - return dstval - - def __eq__(self, other): - if isinstance(other, tzlocal): - return (self._std_offset == other._std_offset and - self._dst_offset == other._dst_offset) - elif isinstance(other, tzutc): - return (not self._hasdst and - self._tznames[0] in {'UTC', 'GMT'} and - self._std_offset == ZERO) - elif isinstance(other, tzoffset): - return (not self._hasdst and - self._tznames[0] == other._name and - self._std_offset == other._offset) - else: - return NotImplemented - - __hash__ = None - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - return "%s()" % self.__class__.__name__ - - __reduce__ = object.__reduce__ - - -class _ttinfo(object): - __slots__ = ["offset", "delta", "isdst", "abbr", - "isstd", "isgmt", "dstoffset"] - - def __init__(self): - for attr in self.__slots__: - setattr(self, attr, None) - - def __repr__(self): - l = [] - for attr in self.__slots__: - value = getattr(self, attr) - if value is not None: - l.append("%s=%s" % (attr, repr(value))) - return "%s(%s)" % (self.__class__.__name__, ", ".join(l)) - - def __eq__(self, other): - if not isinstance(other, _ttinfo): - return NotImplemented - - return (self.offset == other.offset and - self.delta == other.delta and - self.isdst == other.isdst and - self.abbr == other.abbr and - self.isstd == other.isstd and - self.isgmt == other.isgmt and - self.dstoffset == other.dstoffset) - - __hash__ = None - - def __ne__(self, other): - return not (self == other) - - def __getstate__(self): - state = {} - for name in self.__slots__: - state[name] = getattr(self, name, None) - return state - - def __setstate__(self, state): - for name in self.__slots__: - if name in state: - setattr(self, name, state[name]) - - -class _tzfile(object): - """ - Lightweight class for holding the relevant transition and time zone - information read from binary tzfiles. - """ - attrs = ['trans_list', 'trans_list_utc', 'trans_idx', 'ttinfo_list', - 'ttinfo_std', 'ttinfo_dst', 'ttinfo_before', 'ttinfo_first'] - - def __init__(self, **kwargs): - for attr in self.attrs: - setattr(self, attr, kwargs.get(attr, None)) - - -class tzfile(_tzinfo): - """ - This is a ``tzinfo`` subclass thant allows one to use the ``tzfile(5)`` - format timezone files to extract current and historical zone information. - - :param fileobj: - This can be an opened file stream or a file name that the time zone - information can be read from. - - :param filename: - This is an optional parameter specifying the source of the time zone - information in the event that ``fileobj`` is a file object. If omitted - and ``fileobj`` is a file stream, this parameter will be set either to - ``fileobj``'s ``name`` attribute or to ``repr(fileobj)``. - - See `Sources for Time Zone and Daylight Saving Time Data - `_ for more information. - Time zone files can be compiled from the `IANA Time Zone database files - `_ with the `zic time zone compiler - `_ - - .. note:: - - Only construct a ``tzfile`` directly if you have a specific timezone - file on disk that you want to read into a Python ``tzinfo`` object. - If you want to get a ``tzfile`` representing a specific IANA zone, - (e.g. ``'America/New_York'``), you should call - :func:`dateutil.tz.gettz` with the zone identifier. - - - **Examples:** - - Using the US Eastern time zone as an example, we can see that a ``tzfile`` - provides time zone information for the standard Daylight Saving offsets: - - .. testsetup:: tzfile - - from dateutil.tz import gettz - from datetime import datetime - - .. doctest:: tzfile - - >>> NYC = gettz('America/New_York') - >>> NYC - tzfile('/usr/share/zoneinfo/America/New_York') - - >>> print(datetime(2016, 1, 3, tzinfo=NYC)) # EST - 2016-01-03 00:00:00-05:00 - - >>> print(datetime(2016, 7, 7, tzinfo=NYC)) # EDT - 2016-07-07 00:00:00-04:00 - - - The ``tzfile`` structure contains a fully history of the time zone, - so historical dates will also have the right offsets. For example, before - the adoption of the UTC standards, New York used local solar mean time: - - .. doctest:: tzfile - - >>> print(datetime(1901, 4, 12, tzinfo=NYC)) # LMT - 1901-04-12 00:00:00-04:56 - - And during World War II, New York was on "Eastern War Time", which was a - state of permanent daylight saving time: - - .. doctest:: tzfile - - >>> print(datetime(1944, 2, 7, tzinfo=NYC)) # EWT - 1944-02-07 00:00:00-04:00 - - """ - - def __init__(self, fileobj, filename=None): - super(tzfile, self).__init__() - - file_opened_here = False - if isinstance(fileobj, string_types): - self._filename = fileobj - fileobj = open(fileobj, 'rb') - file_opened_here = True - elif filename is not None: - self._filename = filename - elif hasattr(fileobj, "name"): - self._filename = fileobj.name - else: - self._filename = repr(fileobj) - - if fileobj is not None: - if not file_opened_here: - fileobj = _nullcontext(fileobj) - - with fileobj as file_stream: - tzobj = self._read_tzfile(file_stream) - - self._set_tzdata(tzobj) - - def _set_tzdata(self, tzobj): - """ Set the time zone data of this object from a _tzfile object """ - # Copy the relevant attributes over as private attributes - for attr in _tzfile.attrs: - setattr(self, '_' + attr, getattr(tzobj, attr)) - - def _read_tzfile(self, fileobj): - out = _tzfile() - - # From tzfile(5): - # - # The time zone information files used by tzset(3) - # begin with the magic characters "TZif" to identify - # them as time zone information files, followed by - # sixteen bytes reserved for future use, followed by - # six four-byte values of type long, written in a - # ``standard'' byte order (the high-order byte - # of the value is written first). - if fileobj.read(4).decode() != "TZif": - raise ValueError("magic not found") - - fileobj.read(16) - - ( - # The number of UTC/local indicators stored in the file. - ttisgmtcnt, - - # The number of standard/wall indicators stored in the file. - ttisstdcnt, - - # The number of leap seconds for which data is - # stored in the file. - leapcnt, - - # The number of "transition times" for which data - # is stored in the file. - timecnt, - - # The number of "local time types" for which data - # is stored in the file (must not be zero). - typecnt, - - # The number of characters of "time zone - # abbreviation strings" stored in the file. - charcnt, - - ) = struct.unpack(">6l", fileobj.read(24)) - - # The above header is followed by tzh_timecnt four-byte - # values of type long, sorted in ascending order. - # These values are written in ``standard'' byte order. - # Each is used as a transition time (as returned by - # time(2)) at which the rules for computing local time - # change. - - if timecnt: - out.trans_list_utc = list(struct.unpack(">%dl" % timecnt, - fileobj.read(timecnt*4))) - else: - out.trans_list_utc = [] - - # Next come tzh_timecnt one-byte values of type unsigned - # char; each one tells which of the different types of - # ``local time'' types described in the file is associated - # with the same-indexed transition time. These values - # serve as indices into an array of ttinfo structures that - # appears next in the file. - - if timecnt: - out.trans_idx = struct.unpack(">%dB" % timecnt, - fileobj.read(timecnt)) - else: - out.trans_idx = [] - - # Each ttinfo structure is written as a four-byte value - # for tt_gmtoff of type long, in a standard byte - # order, followed by a one-byte value for tt_isdst - # and a one-byte value for tt_abbrind. In each - # structure, tt_gmtoff gives the number of - # seconds to be added to UTC, tt_isdst tells whether - # tm_isdst should be set by localtime(3), and - # tt_abbrind serves as an index into the array of - # time zone abbreviation characters that follow the - # ttinfo structure(s) in the file. - - ttinfo = [] - - for i in range(typecnt): - ttinfo.append(struct.unpack(">lbb", fileobj.read(6))) - - abbr = fileobj.read(charcnt).decode() - - # Then there are tzh_leapcnt pairs of four-byte - # values, written in standard byte order; the - # first value of each pair gives the time (as - # returned by time(2)) at which a leap second - # occurs; the second gives the total number of - # leap seconds to be applied after the given time. - # The pairs of values are sorted in ascending order - # by time. - - # Not used, for now (but seek for correct file position) - if leapcnt: - fileobj.seek(leapcnt * 8, os.SEEK_CUR) - - # Then there are tzh_ttisstdcnt standard/wall - # indicators, each stored as a one-byte value; - # they tell whether the transition times associated - # with local time types were specified as standard - # time or wall clock time, and are used when - # a time zone file is used in handling POSIX-style - # time zone environment variables. - - if ttisstdcnt: - isstd = struct.unpack(">%db" % ttisstdcnt, - fileobj.read(ttisstdcnt)) - - # Finally, there are tzh_ttisgmtcnt UTC/local - # indicators, each stored as a one-byte value; - # they tell whether the transition times associated - # with local time types were specified as UTC or - # local time, and are used when a time zone file - # is used in handling POSIX-style time zone envi- - # ronment variables. - - if ttisgmtcnt: - isgmt = struct.unpack(">%db" % ttisgmtcnt, - fileobj.read(ttisgmtcnt)) - - # Build ttinfo list - out.ttinfo_list = [] - for i in range(typecnt): - gmtoff, isdst, abbrind = ttinfo[i] - gmtoff = _get_supported_offset(gmtoff) - tti = _ttinfo() - tti.offset = gmtoff - tti.dstoffset = datetime.timedelta(0) - tti.delta = datetime.timedelta(seconds=gmtoff) - tti.isdst = isdst - tti.abbr = abbr[abbrind:abbr.find('\x00', abbrind)] - tti.isstd = (ttisstdcnt > i and isstd[i] != 0) - tti.isgmt = (ttisgmtcnt > i and isgmt[i] != 0) - out.ttinfo_list.append(tti) - - # Replace ttinfo indexes for ttinfo objects. - out.trans_idx = [out.ttinfo_list[idx] for idx in out.trans_idx] - - # Set standard, dst, and before ttinfos. before will be - # used when a given time is before any transitions, - # and will be set to the first non-dst ttinfo, or to - # the first dst, if all of them are dst. - out.ttinfo_std = None - out.ttinfo_dst = None - out.ttinfo_before = None - if out.ttinfo_list: - if not out.trans_list_utc: - out.ttinfo_std = out.ttinfo_first = out.ttinfo_list[0] - else: - for i in range(timecnt-1, -1, -1): - tti = out.trans_idx[i] - if not out.ttinfo_std and not tti.isdst: - out.ttinfo_std = tti - elif not out.ttinfo_dst and tti.isdst: - out.ttinfo_dst = tti - - if out.ttinfo_std and out.ttinfo_dst: - break - else: - if out.ttinfo_dst and not out.ttinfo_std: - out.ttinfo_std = out.ttinfo_dst - - for tti in out.ttinfo_list: - if not tti.isdst: - out.ttinfo_before = tti - break - else: - out.ttinfo_before = out.ttinfo_list[0] - - # Now fix transition times to become relative to wall time. - # - # I'm not sure about this. In my tests, the tz source file - # is setup to wall time, and in the binary file isstd and - # isgmt are off, so it should be in wall time. OTOH, it's - # always in gmt time. Let me know if you have comments - # about this. - lastdst = None - lastoffset = None - lastdstoffset = None - lastbaseoffset = None - out.trans_list = [] - - for i, tti in enumerate(out.trans_idx): - offset = tti.offset - dstoffset = 0 - - if lastdst is not None: - if tti.isdst: - if not lastdst: - dstoffset = offset - lastoffset - - if not dstoffset and lastdstoffset: - dstoffset = lastdstoffset - - tti.dstoffset = datetime.timedelta(seconds=dstoffset) - lastdstoffset = dstoffset - - # If a time zone changes its base offset during a DST transition, - # then you need to adjust by the previous base offset to get the - # transition time in local time. Otherwise you use the current - # base offset. Ideally, I would have some mathematical proof of - # why this is true, but I haven't really thought about it enough. - baseoffset = offset - dstoffset - adjustment = baseoffset - if (lastbaseoffset is not None and baseoffset != lastbaseoffset - and tti.isdst != lastdst): - # The base DST has changed - adjustment = lastbaseoffset - - lastdst = tti.isdst - lastoffset = offset - lastbaseoffset = baseoffset - - out.trans_list.append(out.trans_list_utc[i] + adjustment) - - out.trans_idx = tuple(out.trans_idx) - out.trans_list = tuple(out.trans_list) - out.trans_list_utc = tuple(out.trans_list_utc) - - return out - - def _find_last_transition(self, dt, in_utc=False): - # If there's no list, there are no transitions to find - if not self._trans_list: - return None - - timestamp = _datetime_to_timestamp(dt) - - # Find where the timestamp fits in the transition list - if the - # timestamp is a transition time, it's part of the "after" period. - trans_list = self._trans_list_utc if in_utc else self._trans_list - idx = bisect.bisect_right(trans_list, timestamp) - - # We want to know when the previous transition was, so subtract off 1 - return idx - 1 - - def _get_ttinfo(self, idx): - # For no list or after the last transition, default to _ttinfo_std - if idx is None or (idx + 1) >= len(self._trans_list): - return self._ttinfo_std - - # If there is a list and the time is before it, return _ttinfo_before - if idx < 0: - return self._ttinfo_before - - return self._trans_idx[idx] - - def _find_ttinfo(self, dt): - idx = self._resolve_ambiguous_time(dt) - - return self._get_ttinfo(idx) - - def fromutc(self, dt): - """ - The ``tzfile`` implementation of :py:func:`datetime.tzinfo.fromutc`. - - :param dt: - A :py:class:`datetime.datetime` object. - - :raises TypeError: - Raised if ``dt`` is not a :py:class:`datetime.datetime` object. - - :raises ValueError: - Raised if this is called with a ``dt`` which does not have this - ``tzinfo`` attached. - - :return: - Returns a :py:class:`datetime.datetime` object representing the - wall time in ``self``'s time zone. - """ - # These isinstance checks are in datetime.tzinfo, so we'll preserve - # them, even if we don't care about duck typing. - if not isinstance(dt, datetime.datetime): - raise TypeError("fromutc() requires a datetime argument") - - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") - - # First treat UTC as wall time and get the transition we're in. - idx = self._find_last_transition(dt, in_utc=True) - tti = self._get_ttinfo(idx) - - dt_out = dt + datetime.timedelta(seconds=tti.offset) - - fold = self.is_ambiguous(dt_out, idx=idx) - - return enfold(dt_out, fold=int(fold)) - - def is_ambiguous(self, dt, idx=None): - """ - Whether or not the "wall time" of a given datetime is ambiguous in this - zone. - - :param dt: - A :py:class:`datetime.datetime`, naive or time zone aware. - - - :return: - Returns ``True`` if ambiguous, ``False`` otherwise. - - .. versionadded:: 2.6.0 - """ - if idx is None: - idx = self._find_last_transition(dt) - - # Calculate the difference in offsets from current to previous - timestamp = _datetime_to_timestamp(dt) - tti = self._get_ttinfo(idx) - - if idx is None or idx <= 0: - return False - - od = self._get_ttinfo(idx - 1).offset - tti.offset - tt = self._trans_list[idx] # Transition time - - return timestamp < tt + od - - def _resolve_ambiguous_time(self, dt): - idx = self._find_last_transition(dt) - - # If we have no transitions, return the index - _fold = self._fold(dt) - if idx is None or idx == 0: - return idx - - # If it's ambiguous and we're in a fold, shift to a different index. - idx_offset = int(not _fold and self.is_ambiguous(dt, idx)) - - return idx - idx_offset - - def utcoffset(self, dt): - if dt is None: - return None - - if not self._ttinfo_std: - return ZERO - - return self._find_ttinfo(dt).delta - - def dst(self, dt): - if dt is None: - return None - - if not self._ttinfo_dst: - return ZERO - - tti = self._find_ttinfo(dt) - - if not tti.isdst: - return ZERO - - # The documentation says that utcoffset()-dst() must - # be constant for every dt. - return tti.dstoffset - - @tzname_in_python2 - def tzname(self, dt): - if not self._ttinfo_std or dt is None: - return None - return self._find_ttinfo(dt).abbr - - def __eq__(self, other): - if not isinstance(other, tzfile): - return NotImplemented - return (self._trans_list == other._trans_list and - self._trans_idx == other._trans_idx and - self._ttinfo_list == other._ttinfo_list) - - __hash__ = None - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, repr(self._filename)) - - def __reduce__(self): - return self.__reduce_ex__(None) - - def __reduce_ex__(self, protocol): - return (self.__class__, (None, self._filename), self.__dict__) - - -class tzrange(tzrangebase): - """ - The ``tzrange`` object is a time zone specified by a set of offsets and - abbreviations, equivalent to the way the ``TZ`` variable can be specified - in POSIX-like systems, but using Python delta objects to specify DST - start, end and offsets. - - :param stdabbr: - The abbreviation for standard time (e.g. ``'EST'``). - - :param stdoffset: - An integer or :class:`datetime.timedelta` object or equivalent - specifying the base offset from UTC. - - If unspecified, +00:00 is used. - - :param dstabbr: - The abbreviation for DST / "Summer" time (e.g. ``'EDT'``). - - If specified, with no other DST information, DST is assumed to occur - and the default behavior or ``dstoffset``, ``start`` and ``end`` is - used. If unspecified and no other DST information is specified, it - is assumed that this zone has no DST. - - If this is unspecified and other DST information is *is* specified, - DST occurs in the zone but the time zone abbreviation is left - unchanged. - - :param dstoffset: - A an integer or :class:`datetime.timedelta` object or equivalent - specifying the UTC offset during DST. If unspecified and any other DST - information is specified, it is assumed to be the STD offset +1 hour. - - :param start: - A :class:`relativedelta.relativedelta` object or equivalent specifying - the time and time of year that daylight savings time starts. To - specify, for example, that DST starts at 2AM on the 2nd Sunday in - March, pass: - - ``relativedelta(hours=2, month=3, day=1, weekday=SU(+2))`` - - If unspecified and any other DST information is specified, the default - value is 2 AM on the first Sunday in April. - - :param end: - A :class:`relativedelta.relativedelta` object or equivalent - representing the time and time of year that daylight savings time - ends, with the same specification method as in ``start``. One note is - that this should point to the first time in the *standard* zone, so if - a transition occurs at 2AM in the DST zone and the clocks are set back - 1 hour to 1AM, set the ``hours`` parameter to +1. - - - **Examples:** - - .. testsetup:: tzrange - - from dateutil.tz import tzrange, tzstr - - .. doctest:: tzrange - - >>> tzstr('EST5EDT') == tzrange("EST", -18000, "EDT") - True - - >>> from dateutil.relativedelta import * - >>> range1 = tzrange("EST", -18000, "EDT") - >>> range2 = tzrange("EST", -18000, "EDT", -14400, - ... relativedelta(hours=+2, month=4, day=1, - ... weekday=SU(+1)), - ... relativedelta(hours=+1, month=10, day=31, - ... weekday=SU(-1))) - >>> tzstr('EST5EDT') == range1 == range2 - True - - """ - def __init__(self, stdabbr, stdoffset=None, - dstabbr=None, dstoffset=None, - start=None, end=None): - - global relativedelta - from dateutil import relativedelta - - self._std_abbr = stdabbr - self._dst_abbr = dstabbr - - try: - stdoffset = stdoffset.total_seconds() - except (TypeError, AttributeError): - pass - - try: - dstoffset = dstoffset.total_seconds() - except (TypeError, AttributeError): - pass - - if stdoffset is not None: - self._std_offset = datetime.timedelta(seconds=stdoffset) - else: - self._std_offset = ZERO - - if dstoffset is not None: - self._dst_offset = datetime.timedelta(seconds=dstoffset) - elif dstabbr and stdoffset is not None: - self._dst_offset = self._std_offset + datetime.timedelta(hours=+1) - else: - self._dst_offset = ZERO - - if dstabbr and start is None: - self._start_delta = relativedelta.relativedelta( - hours=+2, month=4, day=1, weekday=relativedelta.SU(+1)) - else: - self._start_delta = start - - if dstabbr and end is None: - self._end_delta = relativedelta.relativedelta( - hours=+1, month=10, day=31, weekday=relativedelta.SU(-1)) - else: - self._end_delta = end - - self._dst_base_offset_ = self._dst_offset - self._std_offset - self.hasdst = bool(self._start_delta) - - def transitions(self, year): - """ - For a given year, get the DST on and off transition times, expressed - always on the standard time side. For zones with no transitions, this - function returns ``None``. - - :param year: - The year whose transitions you would like to query. - - :return: - Returns a :class:`tuple` of :class:`datetime.datetime` objects, - ``(dston, dstoff)`` for zones with an annual DST transition, or - ``None`` for fixed offset zones. - """ - if not self.hasdst: - return None - - base_year = datetime.datetime(year, 1, 1) - - start = base_year + self._start_delta - end = base_year + self._end_delta - - return (start, end) - - def __eq__(self, other): - if not isinstance(other, tzrange): - return NotImplemented - - return (self._std_abbr == other._std_abbr and - self._dst_abbr == other._dst_abbr and - self._std_offset == other._std_offset and - self._dst_offset == other._dst_offset and - self._start_delta == other._start_delta and - self._end_delta == other._end_delta) - - @property - def _dst_base_offset(self): - return self._dst_base_offset_ - - -@six.add_metaclass(_TzStrFactory) -class tzstr(tzrange): - """ - ``tzstr`` objects are time zone objects specified by a time-zone string as - it would be passed to a ``TZ`` variable on POSIX-style systems (see - the `GNU C Library: TZ Variable`_ for more details). - - There is one notable exception, which is that POSIX-style time zones use an - inverted offset format, so normally ``GMT+3`` would be parsed as an offset - 3 hours *behind* GMT. The ``tzstr`` time zone object will parse this as an - offset 3 hours *ahead* of GMT. If you would like to maintain the POSIX - behavior, pass a ``True`` value to ``posix_offset``. - - The :class:`tzrange` object provides the same functionality, but is - specified using :class:`relativedelta.relativedelta` objects. rather than - strings. - - :param s: - A time zone string in ``TZ`` variable format. This can be a - :class:`bytes` (2.x: :class:`str`), :class:`str` (2.x: - :class:`unicode`) or a stream emitting unicode characters - (e.g. :class:`StringIO`). - - :param posix_offset: - Optional. If set to ``True``, interpret strings such as ``GMT+3`` or - ``UTC+3`` as being 3 hours *behind* UTC rather than ahead, per the - POSIX standard. - - .. caution:: - - Prior to version 2.7.0, this function also supported time zones - in the format: - - * ``EST5EDT,4,0,6,7200,10,0,26,7200,3600`` - * ``EST5EDT,4,1,0,7200,10,-1,0,7200,3600`` - - This format is non-standard and has been deprecated; this function - will raise a :class:`DeprecatedTZFormatWarning` until - support is removed in a future version. - - .. _`GNU C Library: TZ Variable`: - https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html - """ - def __init__(self, s, posix_offset=False): - global parser - from dateutil.parser import _parser as parser - - self._s = s - - res = parser._parsetz(s) - if res is None or res.any_unused_tokens: - raise ValueError("unknown string format") - - # Here we break the compatibility with the TZ variable handling. - # GMT-3 actually *means* the timezone -3. - if res.stdabbr in ("GMT", "UTC") and not posix_offset: - res.stdoffset *= -1 - - # We must initialize it first, since _delta() needs - # _std_offset and _dst_offset set. Use False in start/end - # to avoid building it two times. - tzrange.__init__(self, res.stdabbr, res.stdoffset, - res.dstabbr, res.dstoffset, - start=False, end=False) - - if not res.dstabbr: - self._start_delta = None - self._end_delta = None - else: - self._start_delta = self._delta(res.start) - if self._start_delta: - self._end_delta = self._delta(res.end, isend=1) - - self.hasdst = bool(self._start_delta) - - def _delta(self, x, isend=0): - from dateutil import relativedelta - kwargs = {} - if x.month is not None: - kwargs["month"] = x.month - if x.weekday is not None: - kwargs["weekday"] = relativedelta.weekday(x.weekday, x.week) - if x.week > 0: - kwargs["day"] = 1 - else: - kwargs["day"] = 31 - elif x.day: - kwargs["day"] = x.day - elif x.yday is not None: - kwargs["yearday"] = x.yday - elif x.jyday is not None: - kwargs["nlyearday"] = x.jyday - if not kwargs: - # Default is to start on first sunday of april, and end - # on last sunday of october. - if not isend: - kwargs["month"] = 4 - kwargs["day"] = 1 - kwargs["weekday"] = relativedelta.SU(+1) - else: - kwargs["month"] = 10 - kwargs["day"] = 31 - kwargs["weekday"] = relativedelta.SU(-1) - if x.time is not None: - kwargs["seconds"] = x.time - else: - # Default is 2AM. - kwargs["seconds"] = 7200 - if isend: - # Convert to standard time, to follow the documented way - # of working with the extra hour. See the documentation - # of the tzinfo class. - delta = self._dst_offset - self._std_offset - kwargs["seconds"] -= delta.seconds + delta.days * 86400 - return relativedelta.relativedelta(**kwargs) - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, repr(self._s)) - - -class _tzicalvtzcomp(object): - def __init__(self, tzoffsetfrom, tzoffsetto, isdst, - tzname=None, rrule=None): - self.tzoffsetfrom = datetime.timedelta(seconds=tzoffsetfrom) - self.tzoffsetto = datetime.timedelta(seconds=tzoffsetto) - self.tzoffsetdiff = self.tzoffsetto - self.tzoffsetfrom - self.isdst = isdst - self.tzname = tzname - self.rrule = rrule - - -class _tzicalvtz(_tzinfo): - def __init__(self, tzid, comps=[]): - super(_tzicalvtz, self).__init__() - - self._tzid = tzid - self._comps = comps - self._cachedate = [] - self._cachecomp = [] - self._cache_lock = _thread.allocate_lock() - - def _find_comp(self, dt): - if len(self._comps) == 1: - return self._comps[0] - - dt = dt.replace(tzinfo=None) - - try: - with self._cache_lock: - return self._cachecomp[self._cachedate.index( - (dt, self._fold(dt)))] - except ValueError: - pass - - lastcompdt = None - lastcomp = None - - for comp in self._comps: - compdt = self._find_compdt(comp, dt) - - if compdt and (not lastcompdt or lastcompdt < compdt): - lastcompdt = compdt - lastcomp = comp - - if not lastcomp: - # RFC says nothing about what to do when a given - # time is before the first onset date. We'll look for the - # first standard component, or the first component, if - # none is found. - for comp in self._comps: - if not comp.isdst: - lastcomp = comp - break - else: - lastcomp = comp[0] - - with self._cache_lock: - self._cachedate.insert(0, (dt, self._fold(dt))) - self._cachecomp.insert(0, lastcomp) - - if len(self._cachedate) > 10: - self._cachedate.pop() - self._cachecomp.pop() - - return lastcomp - - def _find_compdt(self, comp, dt): - if comp.tzoffsetdiff < ZERO and self._fold(dt): - dt -= comp.tzoffsetdiff - - compdt = comp.rrule.before(dt, inc=True) - - return compdt - - def utcoffset(self, dt): - if dt is None: - return None - - return self._find_comp(dt).tzoffsetto - - def dst(self, dt): - comp = self._find_comp(dt) - if comp.isdst: - return comp.tzoffsetdiff - else: - return ZERO - - @tzname_in_python2 - def tzname(self, dt): - return self._find_comp(dt).tzname - - def __repr__(self): - return "" % repr(self._tzid) - - __reduce__ = object.__reduce__ - - -class tzical(object): - """ - This object is designed to parse an iCalendar-style ``VTIMEZONE`` structure - as set out in `RFC 5545`_ Section 4.6.5 into one or more `tzinfo` objects. - - :param `fileobj`: - A file or stream in iCalendar format, which should be UTF-8 encoded - with CRLF endings. - - .. _`RFC 5545`: https://tools.ietf.org/html/rfc5545 - """ - def __init__(self, fileobj): - global rrule - from dateutil import rrule - - if isinstance(fileobj, string_types): - self._s = fileobj - # ical should be encoded in UTF-8 with CRLF - fileobj = open(fileobj, 'r') - else: - self._s = getattr(fileobj, 'name', repr(fileobj)) - fileobj = _nullcontext(fileobj) - - self._vtz = {} - - with fileobj as fobj: - self._parse_rfc(fobj.read()) - - def keys(self): - """ - Retrieves the available time zones as a list. - """ - return list(self._vtz.keys()) - - def get(self, tzid=None): - """ - Retrieve a :py:class:`datetime.tzinfo` object by its ``tzid``. - - :param tzid: - If there is exactly one time zone available, omitting ``tzid`` - or passing :py:const:`None` value returns it. Otherwise a valid - key (which can be retrieved from :func:`keys`) is required. - - :raises ValueError: - Raised if ``tzid`` is not specified but there are either more - or fewer than 1 zone defined. - - :returns: - Returns either a :py:class:`datetime.tzinfo` object representing - the relevant time zone or :py:const:`None` if the ``tzid`` was - not found. - """ - if tzid is None: - if len(self._vtz) == 0: - raise ValueError("no timezones defined") - elif len(self._vtz) > 1: - raise ValueError("more than one timezone available") - tzid = next(iter(self._vtz)) - - return self._vtz.get(tzid) - - def _parse_offset(self, s): - s = s.strip() - if not s: - raise ValueError("empty offset") - if s[0] in ('+', '-'): - signal = (-1, +1)[s[0] == '+'] - s = s[1:] - else: - signal = +1 - if len(s) == 4: - return (int(s[:2]) * 3600 + int(s[2:]) * 60) * signal - elif len(s) == 6: - return (int(s[:2]) * 3600 + int(s[2:4]) * 60 + int(s[4:])) * signal - else: - raise ValueError("invalid offset: " + s) - - def _parse_rfc(self, s): - lines = s.splitlines() - if not lines: - raise ValueError("empty string") - - # Unfold - i = 0 - while i < len(lines): - line = lines[i].rstrip() - if not line: - del lines[i] - elif i > 0 and line[0] == " ": - lines[i-1] += line[1:] - del lines[i] - else: - i += 1 - - tzid = None - comps = [] - invtz = False - comptype = None - for line in lines: - if not line: - continue - name, value = line.split(':', 1) - parms = name.split(';') - if not parms: - raise ValueError("empty property name") - name = parms[0].upper() - parms = parms[1:] - if invtz: - if name == "BEGIN": - if value in ("STANDARD", "DAYLIGHT"): - # Process component - pass - else: - raise ValueError("unknown component: "+value) - comptype = value - founddtstart = False - tzoffsetfrom = None - tzoffsetto = None - rrulelines = [] - tzname = None - elif name == "END": - if value == "VTIMEZONE": - if comptype: - raise ValueError("component not closed: "+comptype) - if not tzid: - raise ValueError("mandatory TZID not found") - if not comps: - raise ValueError( - "at least one component is needed") - # Process vtimezone - self._vtz[tzid] = _tzicalvtz(tzid, comps) - invtz = False - elif value == comptype: - if not founddtstart: - raise ValueError("mandatory DTSTART not found") - if tzoffsetfrom is None: - raise ValueError( - "mandatory TZOFFSETFROM not found") - if tzoffsetto is None: - raise ValueError( - "mandatory TZOFFSETFROM not found") - # Process component - rr = None - if rrulelines: - rr = rrule.rrulestr("\n".join(rrulelines), - compatible=True, - ignoretz=True, - cache=True) - comp = _tzicalvtzcomp(tzoffsetfrom, tzoffsetto, - (comptype == "DAYLIGHT"), - tzname, rr) - comps.append(comp) - comptype = None - else: - raise ValueError("invalid component end: "+value) - elif comptype: - if name == "DTSTART": - # DTSTART in VTIMEZONE takes a subset of valid RRULE - # values under RFC 5545. - for parm in parms: - if parm != 'VALUE=DATE-TIME': - msg = ('Unsupported DTSTART param in ' + - 'VTIMEZONE: ' + parm) - raise ValueError(msg) - rrulelines.append(line) - founddtstart = True - elif name in ("RRULE", "RDATE", "EXRULE", "EXDATE"): - rrulelines.append(line) - elif name == "TZOFFSETFROM": - if parms: - raise ValueError( - "unsupported %s parm: %s " % (name, parms[0])) - tzoffsetfrom = self._parse_offset(value) - elif name == "TZOFFSETTO": - if parms: - raise ValueError( - "unsupported TZOFFSETTO parm: "+parms[0]) - tzoffsetto = self._parse_offset(value) - elif name == "TZNAME": - if parms: - raise ValueError( - "unsupported TZNAME parm: "+parms[0]) - tzname = value - elif name == "COMMENT": - pass - else: - raise ValueError("unsupported property: "+name) - else: - if name == "TZID": - if parms: - raise ValueError( - "unsupported TZID parm: "+parms[0]) - tzid = value - elif name in ("TZURL", "LAST-MODIFIED", "COMMENT"): - pass - else: - raise ValueError("unsupported property: "+name) - elif name == "BEGIN" and value == "VTIMEZONE": - tzid = None - comps = [] - invtz = True - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, repr(self._s)) - - -if sys.platform != "win32": - TZFILES = ["/etc/localtime", "localtime"] - TZPATHS = ["/usr/share/zoneinfo", - "/usr/lib/zoneinfo", - "/usr/share/lib/zoneinfo", - "/etc/zoneinfo"] -else: - TZFILES = [] - TZPATHS = [] - - -def __get_gettz(): - tzlocal_classes = (tzlocal,) - if tzwinlocal is not None: - tzlocal_classes += (tzwinlocal,) - - class GettzFunc(object): - """ - Retrieve a time zone object from a string representation - - This function is intended to retrieve the :py:class:`tzinfo` subclass - that best represents the time zone that would be used if a POSIX - `TZ variable`_ were set to the same value. - - If no argument or an empty string is passed to ``gettz``, local time - is returned: - - .. code-block:: python3 - - >>> gettz() - tzfile('/etc/localtime') - - This function is also the preferred way to map IANA tz database keys - to :class:`tzfile` objects: - - .. code-block:: python3 - - >>> gettz('Pacific/Kiritimati') - tzfile('/usr/share/zoneinfo/Pacific/Kiritimati') - - On Windows, the standard is extended to include the Windows-specific - zone names provided by the operating system: - - .. code-block:: python3 - - >>> gettz('Egypt Standard Time') - tzwin('Egypt Standard Time') - - Passing a GNU ``TZ`` style string time zone specification returns a - :class:`tzstr` object: - - .. code-block:: python3 - - >>> gettz('AEST-10AEDT-11,M10.1.0/2,M4.1.0/3') - tzstr('AEST-10AEDT-11,M10.1.0/2,M4.1.0/3') - - :param name: - A time zone name (IANA, or, on Windows, Windows keys), location of - a ``tzfile(5)`` zoneinfo file or ``TZ`` variable style time zone - specifier. An empty string, no argument or ``None`` is interpreted - as local time. - - :return: - Returns an instance of one of ``dateutil``'s :py:class:`tzinfo` - subclasses. - - .. versionchanged:: 2.7.0 - - After version 2.7.0, any two calls to ``gettz`` using the same - input strings will return the same object: - - .. code-block:: python3 - - >>> tz.gettz('America/Chicago') is tz.gettz('America/Chicago') - True - - In addition to improving performance, this ensures that - `"same zone" semantics`_ are used for datetimes in the same zone. - - - .. _`TZ variable`: - https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html - - .. _`"same zone" semantics`: - https://blog.ganssle.io/articles/2018/02/aware-datetime-arithmetic.html - """ - def __init__(self): - - self.__instances = weakref.WeakValueDictionary() - self.__strong_cache_size = 8 - self.__strong_cache = OrderedDict() - self._cache_lock = _thread.allocate_lock() - - def __call__(self, name=None): - with self._cache_lock: - rv = self.__instances.get(name, None) - - if rv is None: - rv = self.nocache(name=name) - if not (name is None - or isinstance(rv, tzlocal_classes) - or rv is None): - # tzlocal is slightly more complicated than the other - # time zone providers because it depends on environment - # at construction time, so don't cache that. - # - # We also cannot store weak references to None, so we - # will also not store that. - self.__instances[name] = rv - else: - # No need for strong caching, return immediately - return rv - - self.__strong_cache[name] = self.__strong_cache.pop(name, rv) - - if len(self.__strong_cache) > self.__strong_cache_size: - self.__strong_cache.popitem(last=False) - - return rv - - def set_cache_size(self, size): - with self._cache_lock: - self.__strong_cache_size = size - while len(self.__strong_cache) > size: - self.__strong_cache.popitem(last=False) - - def cache_clear(self): - with self._cache_lock: - self.__instances = weakref.WeakValueDictionary() - self.__strong_cache.clear() - - @staticmethod - def nocache(name=None): - """A non-cached version of gettz""" - tz = None - if not name: - try: - name = os.environ["TZ"] - except KeyError: - pass - if name is None or name == ":": - for filepath in TZFILES: - if not os.path.isabs(filepath): - filename = filepath - for path in TZPATHS: - filepath = os.path.join(path, filename) - if os.path.isfile(filepath): - break - else: - continue - if os.path.isfile(filepath): - try: - tz = tzfile(filepath) - break - except (IOError, OSError, ValueError): - pass - else: - tz = tzlocal() - else: - if name.startswith(":"): - name = name[1:] - if os.path.isabs(name): - if os.path.isfile(name): - tz = tzfile(name) - else: - tz = None - else: - for path in TZPATHS: - filepath = os.path.join(path, name) - if not os.path.isfile(filepath): - filepath = filepath.replace(' ', '_') - if not os.path.isfile(filepath): - continue - try: - tz = tzfile(filepath) - break - except (IOError, OSError, ValueError): - pass - else: - tz = None - if tzwin is not None: - try: - tz = tzwin(name) - except (WindowsError, UnicodeEncodeError): - # UnicodeEncodeError is for Python 2.7 compat - tz = None - - if not tz: - from dateutil.zoneinfo import get_zonefile_instance - tz = get_zonefile_instance().get(name) - - if not tz: - for c in name: - # name is not a tzstr unless it has at least - # one offset. For short values of "name", an - # explicit for loop seems to be the fastest way - # To determine if a string contains a digit - if c in "0123456789": - try: - tz = tzstr(name) - except ValueError: - pass - break - else: - if name in ("GMT", "UTC"): - tz = tzutc() - elif name in time.tzname: - tz = tzlocal() - return tz - - return GettzFunc() - - -gettz = __get_gettz() -del __get_gettz - - -def datetime_exists(dt, tz=None): - """ - Given a datetime and a time zone, determine whether or not a given datetime - would fall in a gap. - - :param dt: - A :class:`datetime.datetime` (whose time zone will be ignored if ``tz`` - is provided.) - - :param tz: - A :class:`datetime.tzinfo` with support for the ``fold`` attribute. If - ``None`` or not provided, the datetime's own time zone will be used. - - :return: - Returns a boolean value whether or not the "wall time" exists in - ``tz``. - - .. versionadded:: 2.7.0 - """ - if tz is None: - if dt.tzinfo is None: - raise ValueError('Datetime is naive and no time zone provided.') - tz = dt.tzinfo - - dt = dt.replace(tzinfo=None) - - # This is essentially a test of whether or not the datetime can survive - # a round trip to UTC. - dt_rt = dt.replace(tzinfo=tz).astimezone(tzutc()).astimezone(tz) - dt_rt = dt_rt.replace(tzinfo=None) - - return dt == dt_rt - - -def datetime_ambiguous(dt, tz=None): - """ - Given a datetime and a time zone, determine whether or not a given datetime - is ambiguous (i.e if there are two times differentiated only by their DST - status). - - :param dt: - A :class:`datetime.datetime` (whose time zone will be ignored if ``tz`` - is provided.) - - :param tz: - A :class:`datetime.tzinfo` with support for the ``fold`` attribute. If - ``None`` or not provided, the datetime's own time zone will be used. - - :return: - Returns a boolean value whether or not the "wall time" is ambiguous in - ``tz``. - - .. versionadded:: 2.6.0 - """ - if tz is None: - if dt.tzinfo is None: - raise ValueError('Datetime is naive and no time zone provided.') - - tz = dt.tzinfo - - # If a time zone defines its own "is_ambiguous" function, we'll use that. - is_ambiguous_fn = getattr(tz, 'is_ambiguous', None) - if is_ambiguous_fn is not None: - try: - return tz.is_ambiguous(dt) - except Exception: - pass - - # If it doesn't come out and tell us it's ambiguous, we'll just check if - # the fold attribute has any effect on this particular date and time. - dt = dt.replace(tzinfo=tz) - wall_0 = enfold(dt, fold=0) - wall_1 = enfold(dt, fold=1) - - same_offset = wall_0.utcoffset() == wall_1.utcoffset() - same_dst = wall_0.dst() == wall_1.dst() - - return not (same_offset and same_dst) - - -def resolve_imaginary(dt): - """ - Given a datetime that may be imaginary, return an existing datetime. - - This function assumes that an imaginary datetime represents what the - wall time would be in a zone had the offset transition not occurred, so - it will always fall forward by the transition's change in offset. - - .. doctest:: - - >>> from dateutil import tz - >>> from datetime import datetime - >>> NYC = tz.gettz('America/New_York') - >>> print(tz.resolve_imaginary(datetime(2017, 3, 12, 2, 30, tzinfo=NYC))) - 2017-03-12 03:30:00-04:00 - - >>> KIR = tz.gettz('Pacific/Kiritimati') - >>> print(tz.resolve_imaginary(datetime(1995, 1, 1, 12, 30, tzinfo=KIR))) - 1995-01-02 12:30:00+14:00 - - As a note, :func:`datetime.astimezone` is guaranteed to produce a valid, - existing datetime, so a round-trip to and from UTC is sufficient to get - an extant datetime, however, this generally "falls back" to an earlier time - rather than falling forward to the STD side (though no guarantees are made - about this behavior). - - :param dt: - A :class:`datetime.datetime` which may or may not exist. - - :return: - Returns an existing :class:`datetime.datetime`. If ``dt`` was not - imaginary, the datetime returned is guaranteed to be the same object - passed to the function. - - .. versionadded:: 2.7.0 - """ - if dt.tzinfo is not None and not datetime_exists(dt): - - curr_offset = (dt + datetime.timedelta(hours=24)).utcoffset() - old_offset = (dt - datetime.timedelta(hours=24)).utcoffset() - - dt += curr_offset - old_offset - - return dt - - -def _datetime_to_timestamp(dt): - """ - Convert a :class:`datetime.datetime` object to an epoch timestamp in - seconds since January 1, 1970, ignoring the time zone. - """ - return (dt.replace(tzinfo=None) - EPOCH).total_seconds() - - -if sys.version_info >= (3, 6): - def _get_supported_offset(second_offset): - return second_offset -else: - def _get_supported_offset(second_offset): - # For python pre-3.6, round to full-minutes if that's not the case. - # Python's datetime doesn't accept sub-minute timezones. Check - # http://python.org/sf/1447945 or https://bugs.python.org/issue5288 - # for some information. - old_offset = second_offset - calculated_offset = 60 * ((second_offset + 30) // 60) - return calculated_offset - - -try: - # Python 3.7 feature - from contextmanager import nullcontext as _nullcontext -except ImportError: - class _nullcontext(object): - """ - Class for wrapping contexts so that they are passed through in a - with statement. - """ - def __init__(self, context): - self.context = context - - def __enter__(self): - return self.context - - def __exit__(*args, **kwargs): - pass - -# vim:ts=4:sw=4:et diff --git a/src/dateutil/tz/win.py b/src/dateutil/tz/win.py deleted file mode 100644 index cde07ba7..00000000 --- a/src/dateutil/tz/win.py +++ /dev/null @@ -1,370 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module provides an interface to the native time zone data on Windows, -including :py:class:`datetime.tzinfo` implementations. - -Attempting to import this module on a non-Windows platform will raise an -:py:obj:`ImportError`. -""" -# This code was originally contributed by Jeffrey Harris. -import datetime -import struct - -from six.moves import winreg -from six import text_type - -try: - import ctypes - from ctypes import wintypes -except ValueError: - # ValueError is raised on non-Windows systems for some horrible reason. - raise ImportError("Running tzwin on non-Windows system") - -from ._common import tzrangebase - -__all__ = ["tzwin", "tzwinlocal", "tzres"] - -ONEWEEK = datetime.timedelta(7) - -TZKEYNAMENT = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones" -TZKEYNAME9X = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Time Zones" -TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation" - - -def _settzkeyname(): - handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) - try: - winreg.OpenKey(handle, TZKEYNAMENT).Close() - TZKEYNAME = TZKEYNAMENT - except WindowsError: - TZKEYNAME = TZKEYNAME9X - handle.Close() - return TZKEYNAME - - -TZKEYNAME = _settzkeyname() - - -class tzres(object): - """ - Class for accessing ``tzres.dll``, which contains timezone name related - resources. - - .. versionadded:: 2.5.0 - """ - p_wchar = ctypes.POINTER(wintypes.WCHAR) # Pointer to a wide char - - def __init__(self, tzres_loc='tzres.dll'): - # Load the user32 DLL so we can load strings from tzres - user32 = ctypes.WinDLL('user32') - - # Specify the LoadStringW function - user32.LoadStringW.argtypes = (wintypes.HINSTANCE, - wintypes.UINT, - wintypes.LPWSTR, - ctypes.c_int) - - self.LoadStringW = user32.LoadStringW - self._tzres = ctypes.WinDLL(tzres_loc) - self.tzres_loc = tzres_loc - - def load_name(self, offset): - """ - Load a timezone name from a DLL offset (integer). - - >>> from dateutil.tzwin import tzres - >>> tzr = tzres() - >>> print(tzr.load_name(112)) - 'Eastern Standard Time' - - :param offset: - A positive integer value referring to a string from the tzres dll. - - .. note:: - - Offsets found in the registry are generally of the form - ``@tzres.dll,-114``. The offset in this case is 114, not -114. - - """ - resource = self.p_wchar() - lpBuffer = ctypes.cast(ctypes.byref(resource), wintypes.LPWSTR) - nchar = self.LoadStringW(self._tzres._handle, offset, lpBuffer, 0) - return resource[:nchar] - - def name_from_string(self, tzname_str): - """ - Parse strings as returned from the Windows registry into the time zone - name as defined in the registry. - - >>> from dateutil.tzwin import tzres - >>> tzr = tzres() - >>> print(tzr.name_from_string('@tzres.dll,-251')) - 'Dateline Daylight Time' - >>> print(tzr.name_from_string('Eastern Standard Time')) - 'Eastern Standard Time' - - :param tzname_str: - A timezone name string as returned from a Windows registry key. - - :return: - Returns the localized timezone string from tzres.dll if the string - is of the form `@tzres.dll,-offset`, else returns the input string. - """ - if not tzname_str.startswith('@'): - return tzname_str - - name_splt = tzname_str.split(',-') - try: - offset = int(name_splt[1]) - except: - raise ValueError("Malformed timezone string.") - - return self.load_name(offset) - - -class tzwinbase(tzrangebase): - """tzinfo class based on win32's timezones available in the registry.""" - def __init__(self): - raise NotImplementedError('tzwinbase is an abstract base class') - - def __eq__(self, other): - # Compare on all relevant dimensions, including name. - if not isinstance(other, tzwinbase): - return NotImplemented - - return (self._std_offset == other._std_offset and - self._dst_offset == other._dst_offset and - self._stddayofweek == other._stddayofweek and - self._dstdayofweek == other._dstdayofweek and - self._stdweeknumber == other._stdweeknumber and - self._dstweeknumber == other._dstweeknumber and - self._stdhour == other._stdhour and - self._dsthour == other._dsthour and - self._stdminute == other._stdminute and - self._dstminute == other._dstminute and - self._std_abbr == other._std_abbr and - self._dst_abbr == other._dst_abbr) - - @staticmethod - def list(): - """Return a list of all time zones known to the system.""" - with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle: - with winreg.OpenKey(handle, TZKEYNAME) as tzkey: - result = [winreg.EnumKey(tzkey, i) - for i in range(winreg.QueryInfoKey(tzkey)[0])] - return result - - def display(self): - """ - Return the display name of the time zone. - """ - return self._display - - def transitions(self, year): - """ - For a given year, get the DST on and off transition times, expressed - always on the standard time side. For zones with no transitions, this - function returns ``None``. - - :param year: - The year whose transitions you would like to query. - - :return: - Returns a :class:`tuple` of :class:`datetime.datetime` objects, - ``(dston, dstoff)`` for zones with an annual DST transition, or - ``None`` for fixed offset zones. - """ - - if not self.hasdst: - return None - - dston = picknthweekday(year, self._dstmonth, self._dstdayofweek, - self._dsthour, self._dstminute, - self._dstweeknumber) - - dstoff = picknthweekday(year, self._stdmonth, self._stddayofweek, - self._stdhour, self._stdminute, - self._stdweeknumber) - - # Ambiguous dates default to the STD side - dstoff -= self._dst_base_offset - - return dston, dstoff - - def _get_hasdst(self): - return self._dstmonth != 0 - - @property - def _dst_base_offset(self): - return self._dst_base_offset_ - - -class tzwin(tzwinbase): - """ - Time zone object created from the zone info in the Windows registry - - These are similar to :py:class:`dateutil.tz.tzrange` objects in that - the time zone data is provided in the format of a single offset rule - for either 0 or 2 time zone transitions per year. - - :param: name - The name of a Windows time zone key, e.g. "Eastern Standard Time". - The full list of keys can be retrieved with :func:`tzwin.list`. - """ - - def __init__(self, name): - self._name = name - - with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle: - tzkeyname = text_type("{kn}\\{name}").format(kn=TZKEYNAME, name=name) - with winreg.OpenKey(handle, tzkeyname) as tzkey: - keydict = valuestodict(tzkey) - - self._std_abbr = keydict["Std"] - self._dst_abbr = keydict["Dlt"] - - self._display = keydict["Display"] - - # See http://ww_winreg.jsiinc.com/SUBA/tip0300/rh0398.htm - tup = struct.unpack("=3l16h", keydict["TZI"]) - stdoffset = -tup[0]-tup[1] # Bias + StandardBias * -1 - dstoffset = stdoffset-tup[2] # + DaylightBias * -1 - self._std_offset = datetime.timedelta(minutes=stdoffset) - self._dst_offset = datetime.timedelta(minutes=dstoffset) - - # for the meaning see the win32 TIME_ZONE_INFORMATION structure docs - # http://msdn.microsoft.com/en-us/library/windows/desktop/ms725481(v=vs.85).aspx - (self._stdmonth, - self._stddayofweek, # Sunday = 0 - self._stdweeknumber, # Last = 5 - self._stdhour, - self._stdminute) = tup[4:9] - - (self._dstmonth, - self._dstdayofweek, # Sunday = 0 - self._dstweeknumber, # Last = 5 - self._dsthour, - self._dstminute) = tup[12:17] - - self._dst_base_offset_ = self._dst_offset - self._std_offset - self.hasdst = self._get_hasdst() - - def __repr__(self): - return "tzwin(%s)" % repr(self._name) - - def __reduce__(self): - return (self.__class__, (self._name,)) - - -class tzwinlocal(tzwinbase): - """ - Class representing the local time zone information in the Windows registry - - While :class:`dateutil.tz.tzlocal` makes system calls (via the :mod:`time` - module) to retrieve time zone information, ``tzwinlocal`` retrieves the - rules directly from the Windows registry and creates an object like - :class:`dateutil.tz.tzwin`. - - Because Windows does not have an equivalent of :func:`time.tzset`, on - Windows, :class:`dateutil.tz.tzlocal` instances will always reflect the - time zone settings *at the time that the process was started*, meaning - changes to the machine's time zone settings during the run of a program - on Windows will **not** be reflected by :class:`dateutil.tz.tzlocal`. - Because ``tzwinlocal`` reads the registry directly, it is unaffected by - this issue. - """ - def __init__(self): - with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle: - with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey: - keydict = valuestodict(tzlocalkey) - - self._std_abbr = keydict["StandardName"] - self._dst_abbr = keydict["DaylightName"] - - try: - tzkeyname = text_type('{kn}\\{sn}').format(kn=TZKEYNAME, - sn=self._std_abbr) - with winreg.OpenKey(handle, tzkeyname) as tzkey: - _keydict = valuestodict(tzkey) - self._display = _keydict["Display"] - except OSError: - self._display = None - - stdoffset = -keydict["Bias"]-keydict["StandardBias"] - dstoffset = stdoffset-keydict["DaylightBias"] - - self._std_offset = datetime.timedelta(minutes=stdoffset) - self._dst_offset = datetime.timedelta(minutes=dstoffset) - - # For reasons unclear, in this particular key, the day of week has been - # moved to the END of the SYSTEMTIME structure. - tup = struct.unpack("=8h", keydict["StandardStart"]) - - (self._stdmonth, - self._stdweeknumber, # Last = 5 - self._stdhour, - self._stdminute) = tup[1:5] - - self._stddayofweek = tup[7] - - tup = struct.unpack("=8h", keydict["DaylightStart"]) - - (self._dstmonth, - self._dstweeknumber, # Last = 5 - self._dsthour, - self._dstminute) = tup[1:5] - - self._dstdayofweek = tup[7] - - self._dst_base_offset_ = self._dst_offset - self._std_offset - self.hasdst = self._get_hasdst() - - def __repr__(self): - return "tzwinlocal()" - - def __str__(self): - # str will return the standard name, not the daylight name. - return "tzwinlocal(%s)" % repr(self._std_abbr) - - def __reduce__(self): - return (self.__class__, ()) - - -def picknthweekday(year, month, dayofweek, hour, minute, whichweek): - """ dayofweek == 0 means Sunday, whichweek 5 means last instance """ - first = datetime.datetime(year, month, 1, hour, minute) - - # This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6), - # Because 7 % 7 = 0 - weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1) - wd = weekdayone + ((whichweek - 1) * ONEWEEK) - if (wd.month != month): - wd -= ONEWEEK - - return wd - - -def valuestodict(key): - """Convert a registry key's values to a dictionary.""" - dout = {} - size = winreg.QueryInfoKey(key)[1] - tz_res = None - - for i in range(size): - key_name, value, dtype = winreg.EnumValue(key, i) - if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN: - # If it's a DWORD (32-bit integer), it's stored as unsigned - convert - # that to a proper signed integer - if value & (1 << 31): - value = value - (1 << 32) - elif dtype == winreg.REG_SZ: - # If it's a reference to the tzres DLL, load the actual string - if value.startswith('@tzres'): - tz_res = tz_res or tzres() - value = tz_res.name_from_string(value) - - value = value.rstrip('\x00') # Remove trailing nulls - - dout[key_name] = value - - return dout diff --git a/src/dateutil/tzwin.py b/src/dateutil/tzwin.py deleted file mode 100644 index cebc673e..00000000 --- a/src/dateutil/tzwin.py +++ /dev/null @@ -1,2 +0,0 @@ -# tzwin has moved to dateutil.tz.win -from .tz.win import * diff --git a/src/dateutil/utils.py b/src/dateutil/utils.py deleted file mode 100644 index ebcce6aa..00000000 --- a/src/dateutil/utils.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module offers general convenience and utility functions for dealing with -datetimes. - -.. versionadded:: 2.7.0 -""" -from __future__ import unicode_literals - -from datetime import datetime, time - - -def today(tzinfo=None): - """ - Returns a :py:class:`datetime` representing the current day at midnight - - :param tzinfo: - The time zone to attach (also used to determine the current day). - - :return: - A :py:class:`datetime.datetime` object representing the current day - at midnight. - """ - - dt = datetime.now(tzinfo) - return datetime.combine(dt.date(), time(0, tzinfo=tzinfo)) - - -def default_tzinfo(dt, tzinfo): - """ - Sets the the ``tzinfo`` parameter on naive datetimes only - - This is useful for example when you are provided a datetime that may have - either an implicit or explicit time zone, such as when parsing a time zone - string. - - .. doctest:: - - >>> from dateutil.tz import tzoffset - >>> from dateutil.parser import parse - >>> from dateutil.utils import default_tzinfo - >>> dflt_tz = tzoffset("EST", -18000) - >>> print(default_tzinfo(parse('2014-01-01 12:30 UTC'), dflt_tz)) - 2014-01-01 12:30:00+00:00 - >>> print(default_tzinfo(parse('2014-01-01 12:30'), dflt_tz)) - 2014-01-01 12:30:00-05:00 - - :param dt: - The datetime on which to replace the time zone - - :param tzinfo: - The :py:class:`datetime.tzinfo` subclass instance to assign to - ``dt`` if (and only if) it is naive. - - :return: - Returns an aware :py:class:`datetime.datetime`. - """ - if dt.tzinfo is not None: - return dt - else: - return dt.replace(tzinfo=tzinfo) - - -def within_delta(dt1, dt2, delta): - """ - Useful for comparing two datetimes that may a negilible difference - to be considered equal. - """ - delta = abs(delta) - difference = dt1 - dt2 - return -delta <= difference <= delta diff --git a/src/dateutil/zoneinfo/__init__.py b/src/dateutil/zoneinfo/__init__.py deleted file mode 100644 index 34f11ad6..00000000 --- a/src/dateutil/zoneinfo/__init__.py +++ /dev/null @@ -1,167 +0,0 @@ -# -*- coding: utf-8 -*- -import warnings -import json - -from tarfile import TarFile -from pkgutil import get_data -from io import BytesIO - -from dateutil.tz import tzfile as _tzfile - -__all__ = ["get_zonefile_instance", "gettz", "gettz_db_metadata"] - -ZONEFILENAME = "dateutil-zoneinfo.tar.gz" -METADATA_FN = 'METADATA' - - -class tzfile(_tzfile): - def __reduce__(self): - return (gettz, (self._filename,)) - - -def getzoneinfofile_stream(): - try: - return BytesIO(get_data(__name__, ZONEFILENAME)) - except IOError as e: # TODO switch to FileNotFoundError? - warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror)) - return None - - -class ZoneInfoFile(object): - def __init__(self, zonefile_stream=None): - if zonefile_stream is not None: - with TarFile.open(fileobj=zonefile_stream) as tf: - self.zones = {zf.name: tzfile(tf.extractfile(zf), filename=zf.name) - for zf in tf.getmembers() - if zf.isfile() and zf.name != METADATA_FN} - # deal with links: They'll point to their parent object. Less - # waste of memory - links = {zl.name: self.zones[zl.linkname] - for zl in tf.getmembers() if - zl.islnk() or zl.issym()} - self.zones.update(links) - try: - metadata_json = tf.extractfile(tf.getmember(METADATA_FN)) - metadata_str = metadata_json.read().decode('UTF-8') - self.metadata = json.loads(metadata_str) - except KeyError: - # no metadata in tar file - self.metadata = None - else: - self.zones = {} - self.metadata = None - - def get(self, name, default=None): - """ - Wrapper for :func:`ZoneInfoFile.zones.get`. This is a convenience method - for retrieving zones from the zone dictionary. - - :param name: - The name of the zone to retrieve. (Generally IANA zone names) - - :param default: - The value to return in the event of a missing key. - - .. versionadded:: 2.6.0 - - """ - return self.zones.get(name, default) - - -# The current API has gettz as a module function, although in fact it taps into -# a stateful class. So as a workaround for now, without changing the API, we -# will create a new "global" class instance the first time a user requests a -# timezone. Ugly, but adheres to the api. -# -# TODO: Remove after deprecation period. -_CLASS_ZONE_INSTANCE = [] - - -def get_zonefile_instance(new_instance=False): - """ - This is a convenience function which provides a :class:`ZoneInfoFile` - instance using the data provided by the ``dateutil`` package. By default, it - caches a single instance of the ZoneInfoFile object and returns that. - - :param new_instance: - If ``True``, a new instance of :class:`ZoneInfoFile` is instantiated and - used as the cached instance for the next call. Otherwise, new instances - are created only as necessary. - - :return: - Returns a :class:`ZoneInfoFile` object. - - .. versionadded:: 2.6 - """ - if new_instance: - zif = None - else: - zif = getattr(get_zonefile_instance, '_cached_instance', None) - - if zif is None: - zif = ZoneInfoFile(getzoneinfofile_stream()) - - get_zonefile_instance._cached_instance = zif - - return zif - - -def gettz(name): - """ - This retrieves a time zone from the local zoneinfo tarball that is packaged - with dateutil. - - :param name: - An IANA-style time zone name, as found in the zoneinfo file. - - :return: - Returns a :class:`dateutil.tz.tzfile` time zone object. - - .. warning:: - It is generally inadvisable to use this function, and it is only - provided for API compatibility with earlier versions. This is *not* - equivalent to ``dateutil.tz.gettz()``, which selects an appropriate - time zone based on the inputs, favoring system zoneinfo. This is ONLY - for accessing the dateutil-specific zoneinfo (which may be out of - date compared to the system zoneinfo). - - .. deprecated:: 2.6 - If you need to use a specific zoneinfofile over the system zoneinfo, - instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call - :func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead. - - Use :func:`get_zonefile_instance` to retrieve an instance of the - dateutil-provided zoneinfo. - """ - warnings.warn("zoneinfo.gettz() will be removed in future versions, " - "to use the dateutil-provided zoneinfo files, instantiate a " - "ZoneInfoFile object and use ZoneInfoFile.zones.get() " - "instead. See the documentation for details.", - DeprecationWarning) - - if len(_CLASS_ZONE_INSTANCE) == 0: - _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream())) - return _CLASS_ZONE_INSTANCE[0].zones.get(name) - - -def gettz_db_metadata(): - """ Get the zonefile metadata - - See `zonefile_metadata`_ - - :returns: - A dictionary with the database metadata - - .. deprecated:: 2.6 - See deprecation warning in :func:`zoneinfo.gettz`. To get metadata, - query the attribute ``zoneinfo.ZoneInfoFile.metadata``. - """ - warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future " - "versions, to use the dateutil-provided zoneinfo files, " - "ZoneInfoFile object and query the 'metadata' attribute " - "instead. See the documentation for details.", - DeprecationWarning) - - if len(_CLASS_ZONE_INSTANCE) == 0: - _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream())) - return _CLASS_ZONE_INSTANCE[0].metadata diff --git a/src/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz b/src/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz deleted file mode 100644 index 124f3e14..00000000 Binary files a/src/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz and /dev/null differ diff --git a/src/dateutil/zoneinfo/rebuild.py b/src/dateutil/zoneinfo/rebuild.py deleted file mode 100644 index 78f0d1a0..00000000 --- a/src/dateutil/zoneinfo/rebuild.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -import os -import tempfile -import shutil -import json -from subprocess import check_call -from tarfile import TarFile - -from dateutil.zoneinfo import METADATA_FN, ZONEFILENAME - - -def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None): - """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar* - - filename is the timezone tarball from ``ftp.iana.org/tz``. - - """ - tmpdir = tempfile.mkdtemp() - zonedir = os.path.join(tmpdir, "zoneinfo") - moduledir = os.path.dirname(__file__) - try: - with TarFile.open(filename) as tf: - for name in zonegroups: - tf.extract(name, tmpdir) - filepaths = [os.path.join(tmpdir, n) for n in zonegroups] - try: - check_call(["zic", "-d", zonedir] + filepaths) - except OSError as e: - _print_on_nosuchfile(e) - raise - # write metadata file - with open(os.path.join(zonedir, METADATA_FN), 'w') as f: - json.dump(metadata, f, indent=4, sort_keys=True) - target = os.path.join(moduledir, ZONEFILENAME) - with TarFile.open(target, "w:%s" % format) as tf: - for entry in os.listdir(zonedir): - entrypath = os.path.join(zonedir, entry) - tf.add(entrypath, entry) - finally: - shutil.rmtree(tmpdir) - - -def _print_on_nosuchfile(e): - """Print helpful troubleshooting message - - e is an exception raised by subprocess.check_call() - - """ - if e.errno == 2: - logging.error( - "Could not find zic. Perhaps you need to install " - "libc-bin or some other package that provides it, " - "or it's not in your PATH?") diff --git a/src/dns/__init__.py b/src/dns/__init__.py deleted file mode 100644 index c1ce8e60..00000000 --- a/src/dns/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009, 2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""dnspython DNS toolkit""" - -__all__ = [ - 'dnssec', - 'e164', - 'edns', - 'entropy', - 'exception', - 'flags', - 'hash', - 'inet', - 'ipv4', - 'ipv6', - 'message', - 'name', - 'namedict', - 'node', - 'opcode', - 'query', - 'rcode', - 'rdata', - 'rdataclass', - 'rdataset', - 'rdatatype', - 'renderer', - 'resolver', - 'reversename', - 'rrset', - 'set', - 'tokenizer', - 'tsig', - 'tsigkeyring', - 'ttl', - 'rdtypes', - 'update', - 'version', - 'wiredata', - 'zone', -] diff --git a/src/dns/_compat.py b/src/dns/_compat.py deleted file mode 100644 index ca0931c2..00000000 --- a/src/dns/_compat.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys -import decimal -from decimal import Context - -PY3 = sys.version_info[0] == 3 -PY2 = sys.version_info[0] == 2 - - -if PY3: - long = int - xrange = range -else: - long = long # pylint: disable=long-builtin - xrange = xrange # pylint: disable=xrange-builtin - -# unicode / binary types -if PY3: - text_type = str - binary_type = bytes - string_types = (str,) - unichr = chr - def maybe_decode(x): - return x.decode() - def maybe_encode(x): - return x.encode() - def maybe_chr(x): - return x - def maybe_ord(x): - return x -else: - text_type = unicode # pylint: disable=unicode-builtin, undefined-variable - binary_type = str - string_types = ( - basestring, # pylint: disable=basestring-builtin, undefined-variable - ) - unichr = unichr # pylint: disable=unichr-builtin - def maybe_decode(x): - return x - def maybe_encode(x): - return x - def maybe_chr(x): - return chr(x) - def maybe_ord(x): - return ord(x) - - -def round_py2_compat(what): - """ - Python 2 and Python 3 use different rounding strategies in round(). This - function ensures that results are python2/3 compatible and backward - compatible with previous py2 releases - :param what: float - :return: rounded long - """ - d = Context( - prec=len(str(long(what))), # round to integer with max precision - rounding=decimal.ROUND_HALF_UP - ).create_decimal(str(what)) # str(): python 2.6 compat - return long(d) diff --git a/src/dns/dnssec.py b/src/dns/dnssec.py deleted file mode 100644 index 35da6b5a..00000000 --- a/src/dns/dnssec.py +++ /dev/null @@ -1,519 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Common DNSSEC-related functions and constants.""" - -from io import BytesIO -import struct -import time - -import dns.exception -import dns.name -import dns.node -import dns.rdataset -import dns.rdata -import dns.rdatatype -import dns.rdataclass -from ._compat import string_types - - -class UnsupportedAlgorithm(dns.exception.DNSException): - """The DNSSEC algorithm is not supported.""" - - -class ValidationFailure(dns.exception.DNSException): - """The DNSSEC signature is invalid.""" - - -#: RSAMD5 -RSAMD5 = 1 -#: DH -DH = 2 -#: DSA -DSA = 3 -#: ECC -ECC = 4 -#: RSASHA1 -RSASHA1 = 5 -#: DSANSEC3SHA1 -DSANSEC3SHA1 = 6 -#: RSASHA1NSEC3SHA1 -RSASHA1NSEC3SHA1 = 7 -#: RSASHA256 -RSASHA256 = 8 -#: RSASHA512 -RSASHA512 = 10 -#: ECDSAP256SHA256 -ECDSAP256SHA256 = 13 -#: ECDSAP384SHA384 -ECDSAP384SHA384 = 14 -#: INDIRECT -INDIRECT = 252 -#: PRIVATEDNS -PRIVATEDNS = 253 -#: PRIVATEOID -PRIVATEOID = 254 - -_algorithm_by_text = { - 'RSAMD5': RSAMD5, - 'DH': DH, - 'DSA': DSA, - 'ECC': ECC, - 'RSASHA1': RSASHA1, - 'DSANSEC3SHA1': DSANSEC3SHA1, - 'RSASHA1NSEC3SHA1': RSASHA1NSEC3SHA1, - 'RSASHA256': RSASHA256, - 'RSASHA512': RSASHA512, - 'INDIRECT': INDIRECT, - 'ECDSAP256SHA256': ECDSAP256SHA256, - 'ECDSAP384SHA384': ECDSAP384SHA384, - 'PRIVATEDNS': PRIVATEDNS, - 'PRIVATEOID': PRIVATEOID, -} - -# We construct the inverse mapping programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mapping not to be true inverse. - -_algorithm_by_value = {y: x for x, y in _algorithm_by_text.items()} - - -def algorithm_from_text(text): - """Convert text into a DNSSEC algorithm value. - - Returns an ``int``. - """ - - value = _algorithm_by_text.get(text.upper()) - if value is None: - value = int(text) - return value - - -def algorithm_to_text(value): - """Convert a DNSSEC algorithm value to text - - Returns a ``str``. - """ - - text = _algorithm_by_value.get(value) - if text is None: - text = str(value) - return text - - -def _to_rdata(record, origin): - s = BytesIO() - record.to_wire(s, origin=origin) - return s.getvalue() - - -def key_id(key, origin=None): - """Return the key id (a 16-bit number) for the specified key. - - Note the *origin* parameter of this function is historical and - is not needed. - - Returns an ``int`` between 0 and 65535. - """ - - rdata = _to_rdata(key, origin) - rdata = bytearray(rdata) - if key.algorithm == RSAMD5: - return (rdata[-3] << 8) + rdata[-2] - else: - total = 0 - for i in range(len(rdata) // 2): - total += (rdata[2 * i] << 8) + \ - rdata[2 * i + 1] - if len(rdata) % 2 != 0: - total += rdata[len(rdata) - 1] << 8 - total += ((total >> 16) & 0xffff) - return total & 0xffff - - -def make_ds(name, key, algorithm, origin=None): - """Create a DS record for a DNSSEC key. - - *name* is the owner name of the DS record. - - *key* is a ``dns.rdtypes.ANY.DNSKEY``. - - *algorithm* is a string describing which hash algorithm to use. The - currently supported hashes are "SHA1" and "SHA256". Case does not - matter for these strings. - - *origin* is a ``dns.name.Name`` and will be used as the origin - if *key* is a relative name. - - Returns a ``dns.rdtypes.ANY.DS``. - """ - - if algorithm.upper() == 'SHA1': - dsalg = 1 - hash = SHA1.new() - elif algorithm.upper() == 'SHA256': - dsalg = 2 - hash = SHA256.new() - else: - raise UnsupportedAlgorithm('unsupported algorithm "%s"' % algorithm) - - if isinstance(name, string_types): - name = dns.name.from_text(name, origin) - hash.update(name.canonicalize().to_wire()) - hash.update(_to_rdata(key, origin)) - digest = hash.digest() - - dsrdata = struct.pack("!HBB", key_id(key), key.algorithm, dsalg) + digest - return dns.rdata.from_wire(dns.rdataclass.IN, dns.rdatatype.DS, dsrdata, 0, - len(dsrdata)) - - -def _find_candidate_keys(keys, rrsig): - candidate_keys = [] - value = keys.get(rrsig.signer) - if value is None: - return None - if isinstance(value, dns.node.Node): - try: - rdataset = value.find_rdataset(dns.rdataclass.IN, - dns.rdatatype.DNSKEY) - except KeyError: - return None - else: - rdataset = value - for rdata in rdataset: - if rdata.algorithm == rrsig.algorithm and \ - key_id(rdata) == rrsig.key_tag: - candidate_keys.append(rdata) - return candidate_keys - - -def _is_rsa(algorithm): - return algorithm in (RSAMD5, RSASHA1, - RSASHA1NSEC3SHA1, RSASHA256, - RSASHA512) - - -def _is_dsa(algorithm): - return algorithm in (DSA, DSANSEC3SHA1) - - -def _is_ecdsa(algorithm): - return _have_ecdsa and (algorithm in (ECDSAP256SHA256, ECDSAP384SHA384)) - - -def _is_md5(algorithm): - return algorithm == RSAMD5 - - -def _is_sha1(algorithm): - return algorithm in (DSA, RSASHA1, - DSANSEC3SHA1, RSASHA1NSEC3SHA1) - - -def _is_sha256(algorithm): - return algorithm in (RSASHA256, ECDSAP256SHA256) - - -def _is_sha384(algorithm): - return algorithm == ECDSAP384SHA384 - - -def _is_sha512(algorithm): - return algorithm == RSASHA512 - - -def _make_hash(algorithm): - if _is_md5(algorithm): - return MD5.new() - if _is_sha1(algorithm): - return SHA1.new() - if _is_sha256(algorithm): - return SHA256.new() - if _is_sha384(algorithm): - return SHA384.new() - if _is_sha512(algorithm): - return SHA512.new() - raise ValidationFailure('unknown hash for algorithm %u' % algorithm) - - -def _make_algorithm_id(algorithm): - if _is_md5(algorithm): - oid = [0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05] - elif _is_sha1(algorithm): - oid = [0x2b, 0x0e, 0x03, 0x02, 0x1a] - elif _is_sha256(algorithm): - oid = [0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01] - elif _is_sha512(algorithm): - oid = [0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03] - else: - raise ValidationFailure('unknown algorithm %u' % algorithm) - olen = len(oid) - dlen = _make_hash(algorithm).digest_size - idbytes = [0x30] + [8 + olen + dlen] + \ - [0x30, olen + 4] + [0x06, olen] + oid + \ - [0x05, 0x00] + [0x04, dlen] - return struct.pack('!%dB' % len(idbytes), *idbytes) - - -def _validate_rrsig(rrset, rrsig, keys, origin=None, now=None): - """Validate an RRset against a single signature rdata - - The owner name of *rrsig* is assumed to be the same as the owner name - of *rrset*. - - *rrset* is the RRset to validate. It can be a ``dns.rrset.RRset`` or - a ``(dns.name.Name, dns.rdataset.Rdataset)`` tuple. - - *rrsig* is a ``dns.rdata.Rdata``, the signature to validate. - - *keys* is the key dictionary, used to find the DNSKEY associated with - a given name. The dictionary is keyed by a ``dns.name.Name``, and has - ``dns.node.Node`` or ``dns.rdataset.Rdataset`` values. - - *origin* is a ``dns.name.Name``, the origin to use for relative names. - - *now* is an ``int``, the time to use when validating the signatures, - in seconds since the UNIX epoch. The default is the current time. - """ - - if isinstance(origin, string_types): - origin = dns.name.from_text(origin, dns.name.root) - - candidate_keys = _find_candidate_keys(keys, rrsig) - if candidate_keys is None: - raise ValidationFailure('unknown key') - - for candidate_key in candidate_keys: - # For convenience, allow the rrset to be specified as a (name, - # rdataset) tuple as well as a proper rrset - if isinstance(rrset, tuple): - rrname = rrset[0] - rdataset = rrset[1] - else: - rrname = rrset.name - rdataset = rrset - - if now is None: - now = time.time() - if rrsig.expiration < now: - raise ValidationFailure('expired') - if rrsig.inception > now: - raise ValidationFailure('not yet valid') - - hash = _make_hash(rrsig.algorithm) - - if _is_rsa(rrsig.algorithm): - keyptr = candidate_key.key - (bytes_,) = struct.unpack('!B', keyptr[0:1]) - keyptr = keyptr[1:] - if bytes_ == 0: - (bytes_,) = struct.unpack('!H', keyptr[0:2]) - keyptr = keyptr[2:] - rsa_e = keyptr[0:bytes_] - rsa_n = keyptr[bytes_:] - try: - pubkey = CryptoRSA.construct( - (number.bytes_to_long(rsa_n), - number.bytes_to_long(rsa_e))) - except ValueError: - raise ValidationFailure('invalid public key') - sig = rrsig.signature - elif _is_dsa(rrsig.algorithm): - keyptr = candidate_key.key - (t,) = struct.unpack('!B', keyptr[0:1]) - keyptr = keyptr[1:] - octets = 64 + t * 8 - dsa_q = keyptr[0:20] - keyptr = keyptr[20:] - dsa_p = keyptr[0:octets] - keyptr = keyptr[octets:] - dsa_g = keyptr[0:octets] - keyptr = keyptr[octets:] - dsa_y = keyptr[0:octets] - pubkey = CryptoDSA.construct( - (number.bytes_to_long(dsa_y), - number.bytes_to_long(dsa_g), - number.bytes_to_long(dsa_p), - number.bytes_to_long(dsa_q))) - sig = rrsig.signature[1:] - elif _is_ecdsa(rrsig.algorithm): - # use ecdsa for NIST-384p -- not currently supported by pycryptodome - - keyptr = candidate_key.key - - if rrsig.algorithm == ECDSAP256SHA256: - curve = ecdsa.curves.NIST256p - key_len = 32 - elif rrsig.algorithm == ECDSAP384SHA384: - curve = ecdsa.curves.NIST384p - key_len = 48 - - x = number.bytes_to_long(keyptr[0:key_len]) - y = number.bytes_to_long(keyptr[key_len:key_len * 2]) - if not ecdsa.ecdsa.point_is_valid(curve.generator, x, y): - raise ValidationFailure('invalid ECDSA key') - point = ecdsa.ellipticcurve.Point(curve.curve, x, y, curve.order) - verifying_key = ecdsa.keys.VerifyingKey.from_public_point(point, - curve) - pubkey = ECKeyWrapper(verifying_key, key_len) - r = rrsig.signature[:key_len] - s = rrsig.signature[key_len:] - sig = ecdsa.ecdsa.Signature(number.bytes_to_long(r), - number.bytes_to_long(s)) - - else: - raise ValidationFailure('unknown algorithm %u' % rrsig.algorithm) - - hash.update(_to_rdata(rrsig, origin)[:18]) - hash.update(rrsig.signer.to_digestable(origin)) - - if rrsig.labels < len(rrname) - 1: - suffix = rrname.split(rrsig.labels + 1)[1] - rrname = dns.name.from_text('*', suffix) - rrnamebuf = rrname.to_digestable(origin) - rrfixed = struct.pack('!HHI', rdataset.rdtype, rdataset.rdclass, - rrsig.original_ttl) - rrlist = sorted(rdataset) - for rr in rrlist: - hash.update(rrnamebuf) - hash.update(rrfixed) - rrdata = rr.to_digestable(origin) - rrlen = struct.pack('!H', len(rrdata)) - hash.update(rrlen) - hash.update(rrdata) - - try: - if _is_rsa(rrsig.algorithm): - verifier = pkcs1_15.new(pubkey) - # will raise ValueError if verify fails: - verifier.verify(hash, sig) - elif _is_dsa(rrsig.algorithm): - verifier = DSS.new(pubkey, 'fips-186-3') - verifier.verify(hash, sig) - elif _is_ecdsa(rrsig.algorithm): - digest = hash.digest() - if not pubkey.verify(digest, sig): - raise ValueError - else: - # Raise here for code clarity; this won't actually ever happen - # since if the algorithm is really unknown we'd already have - # raised an exception above - raise ValidationFailure('unknown algorithm %u' % rrsig.algorithm) - # If we got here, we successfully verified so we can return without error - return - except ValueError: - # this happens on an individual validation failure - continue - # nothing verified -- raise failure: - raise ValidationFailure('verify failure') - - -def _validate(rrset, rrsigset, keys, origin=None, now=None): - """Validate an RRset. - - *rrset* is the RRset to validate. It can be a ``dns.rrset.RRset`` or - a ``(dns.name.Name, dns.rdataset.Rdataset)`` tuple. - - *rrsigset* is the signature RRset to be validated. It can be a - ``dns.rrset.RRset`` or a ``(dns.name.Name, dns.rdataset.Rdataset)`` tuple. - - *keys* is the key dictionary, used to find the DNSKEY associated with - a given name. The dictionary is keyed by a ``dns.name.Name``, and has - ``dns.node.Node`` or ``dns.rdataset.Rdataset`` values. - - *origin* is a ``dns.name.Name``, the origin to use for relative names. - - *now* is an ``int``, the time to use when validating the signatures, - in seconds since the UNIX epoch. The default is the current time. - """ - - if isinstance(origin, string_types): - origin = dns.name.from_text(origin, dns.name.root) - - if isinstance(rrset, tuple): - rrname = rrset[0] - else: - rrname = rrset.name - - if isinstance(rrsigset, tuple): - rrsigname = rrsigset[0] - rrsigrdataset = rrsigset[1] - else: - rrsigname = rrsigset.name - rrsigrdataset = rrsigset - - rrname = rrname.choose_relativity(origin) - rrsigname = rrsigname.choose_relativity(origin) - if rrname != rrsigname: - raise ValidationFailure("owner names do not match") - - for rrsig in rrsigrdataset: - try: - _validate_rrsig(rrset, rrsig, keys, origin, now) - return - except ValidationFailure: - pass - raise ValidationFailure("no RRSIGs validated") - - -def _need_pycrypto(*args, **kwargs): - raise NotImplementedError("DNSSEC validation requires pycryptodome/pycryptodomex") - - -try: - try: - # test we're using pycryptodome, not pycrypto (which misses SHA1 for example) - from Crypto.Hash import MD5, SHA1, SHA256, SHA384, SHA512 - from Crypto.PublicKey import RSA as CryptoRSA, DSA as CryptoDSA - from Crypto.Signature import pkcs1_15, DSS - from Crypto.Util import number - except ImportError: - from Cryptodome.Hash import MD5, SHA1, SHA256, SHA384, SHA512 - from Cryptodome.PublicKey import RSA as CryptoRSA, DSA as CryptoDSA - from Cryptodome.Signature import pkcs1_15, DSS - from Cryptodome.Util import number -except ImportError: - validate = _need_pycrypto - validate_rrsig = _need_pycrypto - _have_pycrypto = False - _have_ecdsa = False -else: - validate = _validate - validate_rrsig = _validate_rrsig - _have_pycrypto = True - - try: - import ecdsa - import ecdsa.ecdsa - import ecdsa.ellipticcurve - import ecdsa.keys - except ImportError: - _have_ecdsa = False - else: - _have_ecdsa = True - - class ECKeyWrapper(object): - - def __init__(self, key, key_len): - self.key = key - self.key_len = key_len - - def verify(self, digest, sig): - diglong = number.bytes_to_long(digest) - return self.key.pubkey.verifies(diglong, sig) diff --git a/src/dns/dnssec.pyi b/src/dns/dnssec.pyi deleted file mode 100644 index 5699b3e1..00000000 --- a/src/dns/dnssec.pyi +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Union, Dict, Tuple, Optional -from . import rdataset, rrset, exception, name, rdtypes, rdata, node -import dns.rdtypes.ANY.DS as DS -import dns.rdtypes.ANY.DNSKEY as DNSKEY - -_have_ecdsa : bool -_have_pycrypto : bool - -def validate_rrsig(rrset : Union[Tuple[name.Name, rdataset.Rdataset], rrset.RRset], rrsig : rdata.Rdata, keys : Dict[name.Name, Union[node.Node, rdataset.Rdataset]], origin : Optional[name.Name] = None, now : Optional[int] = None) -> None: - ... - -def validate(rrset: Union[Tuple[name.Name, rdataset.Rdataset], rrset.RRset], rrsigset : Union[Tuple[name.Name, rdataset.Rdataset], rrset.RRset], keys : Dict[name.Name, Union[node.Node, rdataset.Rdataset]], origin=None, now=None) -> None: - ... - -class ValidationFailure(exception.DNSException): - ... - -def make_ds(name : name.Name, key : DNSKEY.DNSKEY, algorithm : str, origin : Optional[name.Name] = None) -> DS.DS: - ... diff --git a/src/dns/e164.py b/src/dns/e164.py deleted file mode 100644 index 758c47a7..00000000 --- a/src/dns/e164.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2006-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS E.164 helpers.""" - -import dns.exception -import dns.name -import dns.resolver -from ._compat import string_types, maybe_decode - -#: The public E.164 domain. -public_enum_domain = dns.name.from_text('e164.arpa.') - - -def from_e164(text, origin=public_enum_domain): - """Convert an E.164 number in textual form into a Name object whose - value is the ENUM domain name for that number. - - Non-digits in the text are ignored, i.e. "16505551212", - "+1.650.555.1212" and "1 (650) 555-1212" are all the same. - - *text*, a ``text``, is an E.164 number in textual form. - - *origin*, a ``dns.name.Name``, the domain in which the number - should be constructed. The default is ``e164.arpa.``. - - Returns a ``dns.name.Name``. - """ - - parts = [d for d in text if d.isdigit()] - parts.reverse() - return dns.name.from_text('.'.join(parts), origin=origin) - - -def to_e164(name, origin=public_enum_domain, want_plus_prefix=True): - """Convert an ENUM domain name into an E.164 number. - - Note that dnspython does not have any information about preferred - number formats within national numbering plans, so all numbers are - emitted as a simple string of digits, prefixed by a '+' (unless - *want_plus_prefix* is ``False``). - - *name* is a ``dns.name.Name``, the ENUM domain name. - - *origin* is a ``dns.name.Name``, a domain containing the ENUM - domain name. The name is relativized to this domain before being - converted to text. If ``None``, no relativization is done. - - *want_plus_prefix* is a ``bool``. If True, add a '+' to the beginning of - the returned number. - - Returns a ``text``. - - """ - if origin is not None: - name = name.relativize(origin) - dlabels = [d for d in name.labels if d.isdigit() and len(d) == 1] - if len(dlabels) != len(name.labels): - raise dns.exception.SyntaxError('non-digit labels in ENUM domain name') - dlabels.reverse() - text = b''.join(dlabels) - if want_plus_prefix: - text = b'+' + text - return maybe_decode(text) - - -def query(number, domains, resolver=None): - """Look for NAPTR RRs for the specified number in the specified domains. - - e.g. lookup('16505551212', ['e164.dnspython.org.', 'e164.arpa.']) - - *number*, a ``text`` is the number to look for. - - *domains* is an iterable containing ``dns.name.Name`` values. - - *resolver*, a ``dns.resolver.Resolver``, is the resolver to use. If - ``None``, the default resolver is used. - """ - - if resolver is None: - resolver = dns.resolver.get_default_resolver() - e_nx = dns.resolver.NXDOMAIN() - for domain in domains: - if isinstance(domain, string_types): - domain = dns.name.from_text(domain) - qname = dns.e164.from_e164(number, domain) - try: - return resolver.query(qname, 'NAPTR') - except dns.resolver.NXDOMAIN as e: - e_nx += e - raise e_nx diff --git a/src/dns/e164.pyi b/src/dns/e164.pyi deleted file mode 100644 index 37a99fed..00000000 --- a/src/dns/e164.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Optional, Iterable -from . import name, resolver -def from_e164(text : str, origin=name.Name(".")) -> name.Name: - ... - -def to_e164(name : name.Name, origin : Optional[name.Name] = None, want_plus_prefix=True) -> str: - ... - -def query(number : str, domains : Iterable[str], resolver : Optional[resolver.Resolver] = None) -> resolver.Answer: - ... diff --git a/src/dns/edns.py b/src/dns/edns.py deleted file mode 100644 index 5660f7bb..00000000 --- a/src/dns/edns.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2009-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""EDNS Options""" - -from __future__ import absolute_import - -import math -import struct - -import dns.inet - -#: NSID -NSID = 3 -#: DAU -DAU = 5 -#: DHU -DHU = 6 -#: N3U -N3U = 7 -#: ECS (client-subnet) -ECS = 8 -#: EXPIRE -EXPIRE = 9 -#: COOKIE -COOKIE = 10 -#: KEEPALIVE -KEEPALIVE = 11 -#: PADDING -PADDING = 12 -#: CHAIN -CHAIN = 13 - -class Option(object): - - """Base class for all EDNS option types.""" - - def __init__(self, otype): - """Initialize an option. - - *otype*, an ``int``, is the option type. - """ - self.otype = otype - - def to_wire(self, file): - """Convert an option to wire format. - """ - raise NotImplementedError - - @classmethod - def from_wire(cls, otype, wire, current, olen): - """Build an EDNS option object from wire format. - - *otype*, an ``int``, is the option type. - - *wire*, a ``binary``, is the wire-format message. - - *current*, an ``int``, is the offset in *wire* of the beginning - of the rdata. - - *olen*, an ``int``, is the length of the wire-format option data - - Returns a ``dns.edns.Option``. - """ - - raise NotImplementedError - - def _cmp(self, other): - """Compare an EDNS option with another option of the same type. - - Returns < 0 if < *other*, 0 if == *other*, and > 0 if > *other*. - """ - raise NotImplementedError - - def __eq__(self, other): - if not isinstance(other, Option): - return False - if self.otype != other.otype: - return False - return self._cmp(other) == 0 - - def __ne__(self, other): - if not isinstance(other, Option): - return False - if self.otype != other.otype: - return False - return self._cmp(other) != 0 - - def __lt__(self, other): - if not isinstance(other, Option) or \ - self.otype != other.otype: - return NotImplemented - return self._cmp(other) < 0 - - def __le__(self, other): - if not isinstance(other, Option) or \ - self.otype != other.otype: - return NotImplemented - return self._cmp(other) <= 0 - - def __ge__(self, other): - if not isinstance(other, Option) or \ - self.otype != other.otype: - return NotImplemented - return self._cmp(other) >= 0 - - def __gt__(self, other): - if not isinstance(other, Option) or \ - self.otype != other.otype: - return NotImplemented - return self._cmp(other) > 0 - - -class GenericOption(Option): - - """Generic Option Class - - This class is used for EDNS option types for which we have no better - implementation. - """ - - def __init__(self, otype, data): - super(GenericOption, self).__init__(otype) - self.data = data - - def to_wire(self, file): - file.write(self.data) - - def to_text(self): - return "Generic %d" % self.otype - - @classmethod - def from_wire(cls, otype, wire, current, olen): - return cls(otype, wire[current: current + olen]) - - def _cmp(self, other): - if self.data == other.data: - return 0 - if self.data > other.data: - return 1 - return -1 - - -class ECSOption(Option): - """EDNS Client Subnet (ECS, RFC7871)""" - - def __init__(self, address, srclen=None, scopelen=0): - """*address*, a ``text``, is the client address information. - - *srclen*, an ``int``, the source prefix length, which is the - leftmost number of bits of the address to be used for the - lookup. The default is 24 for IPv4 and 56 for IPv6. - - *scopelen*, an ``int``, the scope prefix length. This value - must be 0 in queries, and should be set in responses. - """ - - super(ECSOption, self).__init__(ECS) - af = dns.inet.af_for_address(address) - - if af == dns.inet.AF_INET6: - self.family = 2 - if srclen is None: - srclen = 56 - elif af == dns.inet.AF_INET: - self.family = 1 - if srclen is None: - srclen = 24 - else: - raise ValueError('Bad ip family') - - self.address = address - self.srclen = srclen - self.scopelen = scopelen - - addrdata = dns.inet.inet_pton(af, address) - nbytes = int(math.ceil(srclen/8.0)) - - # Truncate to srclen and pad to the end of the last octet needed - # See RFC section 6 - self.addrdata = addrdata[:nbytes] - nbits = srclen % 8 - if nbits != 0: - last = struct.pack('B', ord(self.addrdata[-1:]) & (0xff << nbits)) - self.addrdata = self.addrdata[:-1] + last - - def to_text(self): - return "ECS {}/{} scope/{}".format(self.address, self.srclen, - self.scopelen) - - def to_wire(self, file): - file.write(struct.pack('!H', self.family)) - file.write(struct.pack('!BB', self.srclen, self.scopelen)) - file.write(self.addrdata) - - @classmethod - def from_wire(cls, otype, wire, cur, olen): - family, src, scope = struct.unpack('!HBB', wire[cur:cur+4]) - cur += 4 - - addrlen = int(math.ceil(src/8.0)) - - if family == 1: - af = dns.inet.AF_INET - pad = 4 - addrlen - elif family == 2: - af = dns.inet.AF_INET6 - pad = 16 - addrlen - else: - raise ValueError('unsupported family') - - addr = dns.inet.inet_ntop(af, wire[cur:cur+addrlen] + b'\x00' * pad) - return cls(addr, src, scope) - - def _cmp(self, other): - if self.addrdata == other.addrdata: - return 0 - if self.addrdata > other.addrdata: - return 1 - return -1 - -_type_to_class = { - ECS: ECSOption -} - -def get_option_class(otype): - """Return the class for the specified option type. - - The GenericOption class is used if a more specific class is not - known. - """ - - cls = _type_to_class.get(otype) - if cls is None: - cls = GenericOption - return cls - - -def option_from_wire(otype, wire, current, olen): - """Build an EDNS option object from wire format. - - *otype*, an ``int``, is the option type. - - *wire*, a ``binary``, is the wire-format message. - - *current*, an ``int``, is the offset in *wire* of the beginning - of the rdata. - - *olen*, an ``int``, is the length of the wire-format option data - - Returns an instance of a subclass of ``dns.edns.Option``. - """ - - cls = get_option_class(otype) - return cls.from_wire(otype, wire, current, olen) diff --git a/src/dns/entropy.py b/src/dns/entropy.py deleted file mode 100644 index 00c6a4b3..00000000 --- a/src/dns/entropy.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2009-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import os -import random -import time -from ._compat import long, binary_type -try: - import threading as _threading -except ImportError: - import dummy_threading as _threading - - -class EntropyPool(object): - - # This is an entropy pool for Python implementations that do not - # have a working SystemRandom. I'm not sure there are any, but - # leaving this code doesn't hurt anything as the library code - # is used if present. - - def __init__(self, seed=None): - self.pool_index = 0 - self.digest = None - self.next_byte = 0 - self.lock = _threading.Lock() - try: - import hashlib - self.hash = hashlib.sha1() - self.hash_len = 20 - except ImportError: - try: - import sha - self.hash = sha.new() - self.hash_len = 20 - except ImportError: - import md5 # pylint: disable=import-error - self.hash = md5.new() - self.hash_len = 16 - self.pool = bytearray(b'\0' * self.hash_len) - if seed is not None: - self.stir(bytearray(seed)) - self.seeded = True - self.seed_pid = os.getpid() - else: - self.seeded = False - self.seed_pid = 0 - - def stir(self, entropy, already_locked=False): - if not already_locked: - self.lock.acquire() - try: - for c in entropy: - if self.pool_index == self.hash_len: - self.pool_index = 0 - b = c & 0xff - self.pool[self.pool_index] ^= b - self.pool_index += 1 - finally: - if not already_locked: - self.lock.release() - - def _maybe_seed(self): - if not self.seeded or self.seed_pid != os.getpid(): - try: - seed = os.urandom(16) - except Exception: - try: - r = open('/dev/urandom', 'rb', 0) - try: - seed = r.read(16) - finally: - r.close() - except Exception: - seed = str(time.time()) - self.seeded = True - self.seed_pid = os.getpid() - self.digest = None - seed = bytearray(seed) - self.stir(seed, True) - - def random_8(self): - self.lock.acquire() - try: - self._maybe_seed() - if self.digest is None or self.next_byte == self.hash_len: - self.hash.update(binary_type(self.pool)) - self.digest = bytearray(self.hash.digest()) - self.stir(self.digest, True) - self.next_byte = 0 - value = self.digest[self.next_byte] - self.next_byte += 1 - finally: - self.lock.release() - return value - - def random_16(self): - return self.random_8() * 256 + self.random_8() - - def random_32(self): - return self.random_16() * 65536 + self.random_16() - - def random_between(self, first, last): - size = last - first + 1 - if size > long(4294967296): - raise ValueError('too big') - if size > 65536: - rand = self.random_32 - max = long(4294967295) - elif size > 256: - rand = self.random_16 - max = 65535 - else: - rand = self.random_8 - max = 255 - return first + size * rand() // (max + 1) - -pool = EntropyPool() - -try: - system_random = random.SystemRandom() -except Exception: - system_random = None - -def random_16(): - if system_random is not None: - return system_random.randrange(0, 65536) - else: - return pool.random_16() - -def between(first, last): - if system_random is not None: - return system_random.randrange(first, last + 1) - else: - return pool.random_between(first, last) diff --git a/src/dns/entropy.pyi b/src/dns/entropy.pyi deleted file mode 100644 index 818f805a..00000000 --- a/src/dns/entropy.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Optional -from random import SystemRandom - -system_random : Optional[SystemRandom] - -def random_16() -> int: - pass - -def between(first: int, last: int) -> int: - pass diff --git a/src/dns/exception.py b/src/dns/exception.py deleted file mode 100644 index 71ff04f1..00000000 --- a/src/dns/exception.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Common DNS Exceptions. - -Dnspython modules may also define their own exceptions, which will -always be subclasses of ``DNSException``. -""" - -class DNSException(Exception): - """Abstract base class shared by all dnspython exceptions. - - It supports two basic modes of operation: - - a) Old/compatible mode is used if ``__init__`` was called with - empty *kwargs*. In compatible mode all *args* are passed - to the standard Python Exception class as before and all *args* are - printed by the standard ``__str__`` implementation. Class variable - ``msg`` (or doc string if ``msg`` is ``None``) is returned from ``str()`` - if *args* is empty. - - b) New/parametrized mode is used if ``__init__`` was called with - non-empty *kwargs*. - In the new mode *args* must be empty and all kwargs must match - those set in class variable ``supp_kwargs``. All kwargs are stored inside - ``self.kwargs`` and used in a new ``__str__`` implementation to construct - a formatted message based on the ``fmt`` class variable, a ``string``. - - In the simplest case it is enough to override the ``supp_kwargs`` - and ``fmt`` class variables to get nice parametrized messages. - """ - - msg = None # non-parametrized message - supp_kwargs = set() # accepted parameters for _fmt_kwargs (sanity check) - fmt = None # message parametrized with results from _fmt_kwargs - - def __init__(self, *args, **kwargs): - self._check_params(*args, **kwargs) - if kwargs: - self.kwargs = self._check_kwargs(**kwargs) - self.msg = str(self) - else: - self.kwargs = dict() # defined but empty for old mode exceptions - if self.msg is None: - # doc string is better implicit message than empty string - self.msg = self.__doc__ - if args: - super(DNSException, self).__init__(*args) - else: - super(DNSException, self).__init__(self.msg) - - def _check_params(self, *args, **kwargs): - """Old exceptions supported only args and not kwargs. - - For sanity we do not allow to mix old and new behavior.""" - if args or kwargs: - assert bool(args) != bool(kwargs), \ - 'keyword arguments are mutually exclusive with positional args' - - def _check_kwargs(self, **kwargs): - if kwargs: - assert set(kwargs.keys()) == self.supp_kwargs, \ - 'following set of keyword args is required: %s' % ( - self.supp_kwargs) - return kwargs - - def _fmt_kwargs(self, **kwargs): - """Format kwargs before printing them. - - Resulting dictionary has to have keys necessary for str.format call - on fmt class variable. - """ - fmtargs = {} - for kw, data in kwargs.items(): - if isinstance(data, (list, set)): - # convert list of to list of str() - fmtargs[kw] = list(map(str, data)) - if len(fmtargs[kw]) == 1: - # remove list brackets [] from single-item lists - fmtargs[kw] = fmtargs[kw].pop() - else: - fmtargs[kw] = data - return fmtargs - - def __str__(self): - if self.kwargs and self.fmt: - # provide custom message constructed from keyword arguments - fmtargs = self._fmt_kwargs(**self.kwargs) - return self.fmt.format(**fmtargs) - else: - # print *args directly in the same way as old DNSException - return super(DNSException, self).__str__() - - -class FormError(DNSException): - """DNS message is malformed.""" - - -class SyntaxError(DNSException): - """Text input is malformed.""" - - -class UnexpectedEnd(SyntaxError): - """Text input ended unexpectedly.""" - - -class TooBig(DNSException): - """The DNS message is too big.""" - - -class Timeout(DNSException): - """The DNS operation timed out.""" - supp_kwargs = {'timeout'} - fmt = "The DNS operation timed out after {timeout} seconds" diff --git a/src/dns/exception.pyi b/src/dns/exception.pyi deleted file mode 100644 index 4b346cc4..00000000 --- a/src/dns/exception.pyi +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Set, Optional, Dict - -class DNSException(Exception): - supp_kwargs : Set[str] - kwargs : Optional[Dict] - -class SyntaxError(DNSException): ... -class FormError(DNSException): ... -class Timeout(DNSException): ... diff --git a/src/dns/flags.py b/src/dns/flags.py deleted file mode 100644 index 0119dec7..00000000 --- a/src/dns/flags.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Message Flags.""" - -# Standard DNS flags - -#: Query Response -QR = 0x8000 -#: Authoritative Answer -AA = 0x0400 -#: Truncated Response -TC = 0x0200 -#: Recursion Desired -RD = 0x0100 -#: Recursion Available -RA = 0x0080 -#: Authentic Data -AD = 0x0020 -#: Checking Disabled -CD = 0x0010 - -# EDNS flags - -#: DNSSEC answer OK -DO = 0x8000 - -_by_text = { - 'QR': QR, - 'AA': AA, - 'TC': TC, - 'RD': RD, - 'RA': RA, - 'AD': AD, - 'CD': CD -} - -_edns_by_text = { - 'DO': DO -} - - -# We construct the inverse mappings programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mappings not to be true inverses. - -_by_value = {y: x for x, y in _by_text.items()} - -_edns_by_value = {y: x for x, y in _edns_by_text.items()} - - -def _order_flags(table): - order = list(table.items()) - order.sort() - order.reverse() - return order - -_flags_order = _order_flags(_by_value) - -_edns_flags_order = _order_flags(_edns_by_value) - - -def _from_text(text, table): - flags = 0 - tokens = text.split() - for t in tokens: - flags = flags | table[t.upper()] - return flags - - -def _to_text(flags, table, order): - text_flags = [] - for k, v in order: - if flags & k != 0: - text_flags.append(v) - return ' '.join(text_flags) - - -def from_text(text): - """Convert a space-separated list of flag text values into a flags - value. - - Returns an ``int`` - """ - - return _from_text(text, _by_text) - - -def to_text(flags): - """Convert a flags value into a space-separated list of flag text - values. - - Returns a ``text``. - """ - - return _to_text(flags, _by_value, _flags_order) - - -def edns_from_text(text): - """Convert a space-separated list of EDNS flag text values into a EDNS - flags value. - - Returns an ``int`` - """ - - return _from_text(text, _edns_by_text) - - -def edns_to_text(flags): - """Convert an EDNS flags value into a space-separated list of EDNS flag - text values. - - Returns a ``text``. - """ - - return _to_text(flags, _edns_by_value, _edns_flags_order) diff --git a/src/dns/grange.py b/src/dns/grange.py deleted file mode 100644 index ffe8be7c..00000000 --- a/src/dns/grange.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2012-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS GENERATE range conversion.""" - -import dns - -def from_text(text): - """Convert the text form of a range in a ``$GENERATE`` statement to an - integer. - - *text*, a ``str``, the textual range in ``$GENERATE`` form. - - Returns a tuple of three ``int`` values ``(start, stop, step)``. - """ - - # TODO, figure out the bounds on start, stop and step. - step = 1 - cur = '' - state = 0 - # state 0 1 2 3 4 - # x - y / z - - if text and text[0] == '-': - raise dns.exception.SyntaxError("Start cannot be a negative number") - - for c in text: - if c == '-' and state == 0: - start = int(cur) - cur = '' - state = 2 - elif c == '/': - stop = int(cur) - cur = '' - state = 4 - elif c.isdigit(): - cur += c - else: - raise dns.exception.SyntaxError("Could not parse %s" % (c)) - - if state in (1, 3): - raise dns.exception.SyntaxError() - - if state == 2: - stop = int(cur) - - if state == 4: - step = int(cur) - - assert step >= 1 - assert start >= 0 - assert start <= stop - # TODO, can start == stop? - - return (start, stop, step) diff --git a/src/dns/hash.py b/src/dns/hash.py deleted file mode 100644 index 1713e628..00000000 --- a/src/dns/hash.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Hashing backwards compatibility wrapper""" - -import hashlib -import warnings - -warnings.warn( - "dns.hash module will be removed in future versions. Please use hashlib instead.", - DeprecationWarning) - -hashes = {} -hashes['MD5'] = hashlib.md5 -hashes['SHA1'] = hashlib.sha1 -hashes['SHA224'] = hashlib.sha224 -hashes['SHA256'] = hashlib.sha256 -hashes['SHA384'] = hashlib.sha384 -hashes['SHA512'] = hashlib.sha512 - - -def get(algorithm): - return hashes[algorithm.upper()] diff --git a/src/dns/inet.py b/src/dns/inet.py deleted file mode 100644 index c8d7c1b4..00000000 --- a/src/dns/inet.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Generic Internet address helper functions.""" - -import socket - -import dns.ipv4 -import dns.ipv6 - -from ._compat import maybe_ord - -# We assume that AF_INET is always defined. - -AF_INET = socket.AF_INET - -# AF_INET6 might not be defined in the socket module, but we need it. -# We'll try to use the socket module's value, and if it doesn't work, -# we'll use our own value. - -try: - AF_INET6 = socket.AF_INET6 -except AttributeError: - AF_INET6 = 9999 - - -def inet_pton(family, text): - """Convert the textual form of a network address into its binary form. - - *family* is an ``int``, the address family. - - *text* is a ``text``, the textual address. - - Raises ``NotImplementedError`` if the address family specified is not - implemented. - - Returns a ``binary``. - """ - - if family == AF_INET: - return dns.ipv4.inet_aton(text) - elif family == AF_INET6: - return dns.ipv6.inet_aton(text) - else: - raise NotImplementedError - - -def inet_ntop(family, address): - """Convert the binary form of a network address into its textual form. - - *family* is an ``int``, the address family. - - *address* is a ``binary``, the network address in binary form. - - Raises ``NotImplementedError`` if the address family specified is not - implemented. - - Returns a ``text``. - """ - - if family == AF_INET: - return dns.ipv4.inet_ntoa(address) - elif family == AF_INET6: - return dns.ipv6.inet_ntoa(address) - else: - raise NotImplementedError - - -def af_for_address(text): - """Determine the address family of a textual-form network address. - - *text*, a ``text``, the textual address. - - Raises ``ValueError`` if the address family cannot be determined - from the input. - - Returns an ``int``. - """ - - try: - dns.ipv4.inet_aton(text) - return AF_INET - except Exception: - try: - dns.ipv6.inet_aton(text) - return AF_INET6 - except: - raise ValueError - - -def is_multicast(text): - """Is the textual-form network address a multicast address? - - *text*, a ``text``, the textual address. - - Raises ``ValueError`` if the address family cannot be determined - from the input. - - Returns a ``bool``. - """ - - try: - first = maybe_ord(dns.ipv4.inet_aton(text)[0]) - return first >= 224 and first <= 239 - except Exception: - try: - first = maybe_ord(dns.ipv6.inet_aton(text)[0]) - return first == 255 - except Exception: - raise ValueError diff --git a/src/dns/inet.pyi b/src/dns/inet.pyi deleted file mode 100644 index 6d9dcc70..00000000 --- a/src/dns/inet.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from typing import Union -from socket import AddressFamily - -AF_INET6 : Union[int, AddressFamily] diff --git a/src/dns/ipv4.py b/src/dns/ipv4.py deleted file mode 100644 index 8fc4f7dc..00000000 --- a/src/dns/ipv4.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""IPv4 helper functions.""" - -import struct - -import dns.exception -from ._compat import binary_type - -def inet_ntoa(address): - """Convert an IPv4 address in binary form to text form. - - *address*, a ``binary``, the IPv4 address in binary form. - - Returns a ``text``. - """ - - if len(address) != 4: - raise dns.exception.SyntaxError - if not isinstance(address, bytearray): - address = bytearray(address) - return ('%u.%u.%u.%u' % (address[0], address[1], - address[2], address[3])) - -def inet_aton(text): - """Convert an IPv4 address in text form to binary form. - - *text*, a ``text``, the IPv4 address in textual form. - - Returns a ``binary``. - """ - - if not isinstance(text, binary_type): - text = text.encode() - parts = text.split(b'.') - if len(parts) != 4: - raise dns.exception.SyntaxError - for part in parts: - if not part.isdigit(): - raise dns.exception.SyntaxError - if len(part) > 1 and part[0] == '0': - # No leading zeros - raise dns.exception.SyntaxError - try: - bytes = [int(part) for part in parts] - return struct.pack('BBBB', *bytes) - except: - raise dns.exception.SyntaxError diff --git a/src/dns/ipv6.py b/src/dns/ipv6.py deleted file mode 100644 index 128e56c8..00000000 --- a/src/dns/ipv6.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""IPv6 helper functions.""" - -import re -import binascii - -import dns.exception -import dns.ipv4 -from ._compat import xrange, binary_type, maybe_decode - -_leading_zero = re.compile(r'0+([0-9a-f]+)') - -def inet_ntoa(address): - """Convert an IPv6 address in binary form to text form. - - *address*, a ``binary``, the IPv6 address in binary form. - - Raises ``ValueError`` if the address isn't 16 bytes long. - Returns a ``text``. - """ - - if len(address) != 16: - raise ValueError("IPv6 addresses are 16 bytes long") - hex = binascii.hexlify(address) - chunks = [] - i = 0 - l = len(hex) - while i < l: - chunk = maybe_decode(hex[i : i + 4]) - # strip leading zeros. we do this with an re instead of - # with lstrip() because lstrip() didn't support chars until - # python 2.2.2 - m = _leading_zero.match(chunk) - if not m is None: - chunk = m.group(1) - chunks.append(chunk) - i += 4 - # - # Compress the longest subsequence of 0-value chunks to :: - # - best_start = 0 - best_len = 0 - start = -1 - last_was_zero = False - for i in xrange(8): - if chunks[i] != '0': - if last_was_zero: - end = i - current_len = end - start - if current_len > best_len: - best_start = start - best_len = current_len - last_was_zero = False - elif not last_was_zero: - start = i - last_was_zero = True - if last_was_zero: - end = 8 - current_len = end - start - if current_len > best_len: - best_start = start - best_len = current_len - if best_len > 1: - if best_start == 0 and \ - (best_len == 6 or - best_len == 5 and chunks[5] == 'ffff'): - # We have an embedded IPv4 address - if best_len == 6: - prefix = '::' - else: - prefix = '::ffff:' - hex = prefix + dns.ipv4.inet_ntoa(address[12:]) - else: - hex = ':'.join(chunks[:best_start]) + '::' + \ - ':'.join(chunks[best_start + best_len:]) - else: - hex = ':'.join(chunks) - return hex - -_v4_ending = re.compile(br'(.*):(\d+\.\d+\.\d+\.\d+)$') -_colon_colon_start = re.compile(br'::.*') -_colon_colon_end = re.compile(br'.*::$') - -def inet_aton(text): - """Convert an IPv6 address in text form to binary form. - - *text*, a ``text``, the IPv6 address in textual form. - - Returns a ``binary``. - """ - - # - # Our aim here is not something fast; we just want something that works. - # - if not isinstance(text, binary_type): - text = text.encode() - - if text == b'::': - text = b'0::' - # - # Get rid of the icky dot-quad syntax if we have it. - # - m = _v4_ending.match(text) - if not m is None: - b = bytearray(dns.ipv4.inet_aton(m.group(2))) - text = (u"{}:{:02x}{:02x}:{:02x}{:02x}".format(m.group(1).decode(), - b[0], b[1], b[2], - b[3])).encode() - # - # Try to turn '::' into ':'; if no match try to - # turn '::' into ':' - # - m = _colon_colon_start.match(text) - if not m is None: - text = text[1:] - else: - m = _colon_colon_end.match(text) - if not m is None: - text = text[:-1] - # - # Now canonicalize into 8 chunks of 4 hex digits each - # - chunks = text.split(b':') - l = len(chunks) - if l > 8: - raise dns.exception.SyntaxError - seen_empty = False - canonical = [] - for c in chunks: - if c == b'': - if seen_empty: - raise dns.exception.SyntaxError - seen_empty = True - for i in xrange(0, 8 - l + 1): - canonical.append(b'0000') - else: - lc = len(c) - if lc > 4: - raise dns.exception.SyntaxError - if lc != 4: - c = (b'0' * (4 - lc)) + c - canonical.append(c) - if l < 8 and not seen_empty: - raise dns.exception.SyntaxError - text = b''.join(canonical) - - # - # Finally we can go to binary. - # - try: - return binascii.unhexlify(text) - except (binascii.Error, TypeError): - raise dns.exception.SyntaxError - -_mapped_prefix = b'\x00' * 10 + b'\xff\xff' - -def is_mapped(address): - """Is the specified address a mapped IPv4 address? - - *address*, a ``binary`` is an IPv6 address in binary form. - - Returns a ``bool``. - """ - - return address.startswith(_mapped_prefix) diff --git a/src/dns/message.py b/src/dns/message.py deleted file mode 100644 index 9d2b2f43..00000000 --- a/src/dns/message.py +++ /dev/null @@ -1,1175 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Messages""" - -from __future__ import absolute_import - -from io import StringIO -import struct -import time - -import dns.edns -import dns.exception -import dns.flags -import dns.name -import dns.opcode -import dns.entropy -import dns.rcode -import dns.rdata -import dns.rdataclass -import dns.rdatatype -import dns.rrset -import dns.renderer -import dns.tsig -import dns.wiredata - -from ._compat import long, xrange, string_types - - -class ShortHeader(dns.exception.FormError): - """The DNS packet passed to from_wire() is too short.""" - - -class TrailingJunk(dns.exception.FormError): - """The DNS packet passed to from_wire() has extra junk at the end of it.""" - - -class UnknownHeaderField(dns.exception.DNSException): - """The header field name was not recognized when converting from text - into a message.""" - - -class BadEDNS(dns.exception.FormError): - """An OPT record occurred somewhere other than the start of - the additional data section.""" - - -class BadTSIG(dns.exception.FormError): - """A TSIG record occurred somewhere other than the end of - the additional data section.""" - - -class UnknownTSIGKey(dns.exception.DNSException): - """A TSIG with an unknown key was received.""" - - -#: The question section number -QUESTION = 0 - -#: The answer section number -ANSWER = 1 - -#: The authority section number -AUTHORITY = 2 - -#: The additional section number -ADDITIONAL = 3 - -class Message(object): - """A DNS message.""" - - def __init__(self, id=None): - if id is None: - self.id = dns.entropy.random_16() - else: - self.id = id - self.flags = 0 - self.question = [] - self.answer = [] - self.authority = [] - self.additional = [] - self.edns = -1 - self.ednsflags = 0 - self.payload = 0 - self.options = [] - self.request_payload = 0 - self.keyring = None - self.keyname = None - self.keyalgorithm = dns.tsig.default_algorithm - self.request_mac = b'' - self.other_data = b'' - self.tsig_error = 0 - self.fudge = 300 - self.original_id = self.id - self.mac = b'' - self.xfr = False - self.origin = None - self.tsig_ctx = None - self.had_tsig = False - self.multi = False - self.first = True - self.index = {} - - def __repr__(self): - return '' - - def __str__(self): - return self.to_text() - - def to_text(self, origin=None, relativize=True, **kw): - """Convert the message to text. - - The *origin*, *relativize*, and any other keyword - arguments are passed to the RRset ``to_wire()`` method. - - Returns a ``text``. - """ - - s = StringIO() - s.write(u'id %d\n' % self.id) - s.write(u'opcode %s\n' % - dns.opcode.to_text(dns.opcode.from_flags(self.flags))) - rc = dns.rcode.from_flags(self.flags, self.ednsflags) - s.write(u'rcode %s\n' % dns.rcode.to_text(rc)) - s.write(u'flags %s\n' % dns.flags.to_text(self.flags)) - if self.edns >= 0: - s.write(u'edns %s\n' % self.edns) - if self.ednsflags != 0: - s.write(u'eflags %s\n' % - dns.flags.edns_to_text(self.ednsflags)) - s.write(u'payload %d\n' % self.payload) - for opt in self.options: - s.write(u'option %s\n' % opt.to_text()) - is_update = dns.opcode.is_update(self.flags) - if is_update: - s.write(u';ZONE\n') - else: - s.write(u';QUESTION\n') - for rrset in self.question: - s.write(rrset.to_text(origin, relativize, **kw)) - s.write(u'\n') - if is_update: - s.write(u';PREREQ\n') - else: - s.write(u';ANSWER\n') - for rrset in self.answer: - s.write(rrset.to_text(origin, relativize, **kw)) - s.write(u'\n') - if is_update: - s.write(u';UPDATE\n') - else: - s.write(u';AUTHORITY\n') - for rrset in self.authority: - s.write(rrset.to_text(origin, relativize, **kw)) - s.write(u'\n') - s.write(u';ADDITIONAL\n') - for rrset in self.additional: - s.write(rrset.to_text(origin, relativize, **kw)) - s.write(u'\n') - # - # We strip off the final \n so the caller can print the result without - # doing weird things to get around eccentricities in Python print - # formatting - # - return s.getvalue()[:-1] - - def __eq__(self, other): - """Two messages are equal if they have the same content in the - header, question, answer, and authority sections. - - Returns a ``bool``. - """ - - if not isinstance(other, Message): - return False - if self.id != other.id: - return False - if self.flags != other.flags: - return False - for n in self.question: - if n not in other.question: - return False - for n in other.question: - if n not in self.question: - return False - for n in self.answer: - if n not in other.answer: - return False - for n in other.answer: - if n not in self.answer: - return False - for n in self.authority: - if n not in other.authority: - return False - for n in other.authority: - if n not in self.authority: - return False - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def is_response(self, other): - """Is this message a response to *other*? - - Returns a ``bool``. - """ - - if other.flags & dns.flags.QR == 0 or \ - self.id != other.id or \ - dns.opcode.from_flags(self.flags) != \ - dns.opcode.from_flags(other.flags): - return False - if dns.rcode.from_flags(other.flags, other.ednsflags) != \ - dns.rcode.NOERROR: - return True - if dns.opcode.is_update(self.flags): - return True - for n in self.question: - if n not in other.question: - return False - for n in other.question: - if n not in self.question: - return False - return True - - def section_number(self, section): - """Return the "section number" of the specified section for use - in indexing. The question section is 0, the answer section is 1, - the authority section is 2, and the additional section is 3. - - *section* is one of the section attributes of this message. - - Raises ``ValueError`` if the section isn't known. - - Returns an ``int``. - """ - - if section is self.question: - return QUESTION - elif section is self.answer: - return ANSWER - elif section is self.authority: - return AUTHORITY - elif section is self.additional: - return ADDITIONAL - else: - raise ValueError('unknown section') - - def section_from_number(self, number): - """Return the "section number" of the specified section for use - in indexing. The question section is 0, the answer section is 1, - the authority section is 2, and the additional section is 3. - - *section* is one of the section attributes of this message. - - Raises ``ValueError`` if the section isn't known. - - Returns an ``int``. - """ - - if number == QUESTION: - return self.question - elif number == ANSWER: - return self.answer - elif number == AUTHORITY: - return self.authority - elif number == ADDITIONAL: - return self.additional - else: - raise ValueError('unknown section') - - def find_rrset(self, section, name, rdclass, rdtype, - covers=dns.rdatatype.NONE, deleting=None, create=False, - force_unique=False): - """Find the RRset with the given attributes in the specified section. - - *section*, an ``int`` section number, or one of the section - attributes of this message. This specifies the - the section of the message to search. For example:: - - my_message.find_rrset(my_message.answer, name, rdclass, rdtype) - my_message.find_rrset(dns.message.ANSWER, name, rdclass, rdtype) - - *name*, a ``dns.name.Name``, the name of the RRset. - - *rdclass*, an ``int``, the class of the RRset. - - *rdtype*, an ``int``, the type of the RRset. - - *covers*, an ``int`` or ``None``, the covers value of the RRset. - The default is ``None``. - - *deleting*, an ``int`` or ``None``, the deleting value of the RRset. - The default is ``None``. - - *create*, a ``bool``. If ``True``, create the RRset if it is not found. - The created RRset is appended to *section*. - - *force_unique*, a ``bool``. If ``True`` and *create* is also ``True``, - create a new RRset regardless of whether a matching RRset exists - already. The default is ``False``. This is useful when creating - DDNS Update messages, as order matters for them. - - Raises ``KeyError`` if the RRset was not found and create was - ``False``. - - Returns a ``dns.rrset.RRset object``. - """ - - if isinstance(section, int): - section_number = section - section = self.section_from_number(section_number) - else: - section_number = self.section_number(section) - key = (section_number, name, rdclass, rdtype, covers, deleting) - if not force_unique: - if self.index is not None: - rrset = self.index.get(key) - if rrset is not None: - return rrset - else: - for rrset in section: - if rrset.match(name, rdclass, rdtype, covers, deleting): - return rrset - if not create: - raise KeyError - rrset = dns.rrset.RRset(name, rdclass, rdtype, covers, deleting) - section.append(rrset) - if self.index is not None: - self.index[key] = rrset - return rrset - - def get_rrset(self, section, name, rdclass, rdtype, - covers=dns.rdatatype.NONE, deleting=None, create=False, - force_unique=False): - """Get the RRset with the given attributes in the specified section. - - If the RRset is not found, None is returned. - - *section*, an ``int`` section number, or one of the section - attributes of this message. This specifies the - the section of the message to search. For example:: - - my_message.get_rrset(my_message.answer, name, rdclass, rdtype) - my_message.get_rrset(dns.message.ANSWER, name, rdclass, rdtype) - - *name*, a ``dns.name.Name``, the name of the RRset. - - *rdclass*, an ``int``, the class of the RRset. - - *rdtype*, an ``int``, the type of the RRset. - - *covers*, an ``int`` or ``None``, the covers value of the RRset. - The default is ``None``. - - *deleting*, an ``int`` or ``None``, the deleting value of the RRset. - The default is ``None``. - - *create*, a ``bool``. If ``True``, create the RRset if it is not found. - The created RRset is appended to *section*. - - *force_unique*, a ``bool``. If ``True`` and *create* is also ``True``, - create a new RRset regardless of whether a matching RRset exists - already. The default is ``False``. This is useful when creating - DDNS Update messages, as order matters for them. - - Returns a ``dns.rrset.RRset object`` or ``None``. - """ - - try: - rrset = self.find_rrset(section, name, rdclass, rdtype, covers, - deleting, create, force_unique) - except KeyError: - rrset = None - return rrset - - def to_wire(self, origin=None, max_size=0, **kw): - """Return a string containing the message in DNS compressed wire - format. - - Additional keyword arguments are passed to the RRset ``to_wire()`` - method. - - *origin*, a ``dns.name.Name`` or ``None``, the origin to be appended - to any relative names. - - *max_size*, an ``int``, the maximum size of the wire format - output; default is 0, which means "the message's request - payload, if nonzero, or 65535". - - Raises ``dns.exception.TooBig`` if *max_size* was exceeded. - - Returns a ``binary``. - """ - - if max_size == 0: - if self.request_payload != 0: - max_size = self.request_payload - else: - max_size = 65535 - if max_size < 512: - max_size = 512 - elif max_size > 65535: - max_size = 65535 - r = dns.renderer.Renderer(self.id, self.flags, max_size, origin) - for rrset in self.question: - r.add_question(rrset.name, rrset.rdtype, rrset.rdclass) - for rrset in self.answer: - r.add_rrset(dns.renderer.ANSWER, rrset, **kw) - for rrset in self.authority: - r.add_rrset(dns.renderer.AUTHORITY, rrset, **kw) - if self.edns >= 0: - r.add_edns(self.edns, self.ednsflags, self.payload, self.options) - for rrset in self.additional: - r.add_rrset(dns.renderer.ADDITIONAL, rrset, **kw) - r.write_header() - if self.keyname is not None: - r.add_tsig(self.keyname, self.keyring[self.keyname], - self.fudge, self.original_id, self.tsig_error, - self.other_data, self.request_mac, - self.keyalgorithm) - self.mac = r.mac - return r.get_wire() - - def use_tsig(self, keyring, keyname=None, fudge=300, - original_id=None, tsig_error=0, other_data=b'', - algorithm=dns.tsig.default_algorithm): - """When sending, a TSIG signature using the specified keyring - and keyname should be added. - - See the documentation of the Message class for a complete - description of the keyring dictionary. - - *keyring*, a ``dict``, the TSIG keyring to use. If a - *keyring* is specified but a *keyname* is not, then the key - used will be the first key in the *keyring*. Note that the - order of keys in a dictionary is not defined, so applications - should supply a keyname when a keyring is used, unless they - know the keyring contains only one key. - - *keyname*, a ``dns.name.Name`` or ``None``, the name of the TSIG key - to use; defaults to ``None``. The key must be defined in the keyring. - - *fudge*, an ``int``, the TSIG time fudge. - - *original_id*, an ``int``, the TSIG original id. If ``None``, - the message's id is used. - - *tsig_error*, an ``int``, the TSIG error code. - - *other_data*, a ``binary``, the TSIG other data. - - *algorithm*, a ``dns.name.Name``, the TSIG algorithm to use. - """ - - self.keyring = keyring - if keyname is None: - self.keyname = list(self.keyring.keys())[0] - else: - if isinstance(keyname, string_types): - keyname = dns.name.from_text(keyname) - self.keyname = keyname - self.keyalgorithm = algorithm - self.fudge = fudge - if original_id is None: - self.original_id = self.id - else: - self.original_id = original_id - self.tsig_error = tsig_error - self.other_data = other_data - - def use_edns(self, edns=0, ednsflags=0, payload=1280, request_payload=None, - options=None): - """Configure EDNS behavior. - - *edns*, an ``int``, is the EDNS level to use. Specifying - ``None``, ``False``, or ``-1`` means "do not use EDNS", and in this case - the other parameters are ignored. Specifying ``True`` is - equivalent to specifying 0, i.e. "use EDNS0". - - *ednsflags*, an ``int``, the EDNS flag values. - - *payload*, an ``int``, is the EDNS sender's payload field, which is the - maximum size of UDP datagram the sender can handle. I.e. how big - a response to this message can be. - - *request_payload*, an ``int``, is the EDNS payload size to use when - sending this message. If not specified, defaults to the value of - *payload*. - - *options*, a list of ``dns.edns.Option`` objects or ``None``, the EDNS - options. - """ - - if edns is None or edns is False: - edns = -1 - if edns is True: - edns = 0 - if request_payload is None: - request_payload = payload - if edns < 0: - ednsflags = 0 - payload = 0 - request_payload = 0 - options = [] - else: - # make sure the EDNS version in ednsflags agrees with edns - ednsflags &= long(0xFF00FFFF) - ednsflags |= (edns << 16) - if options is None: - options = [] - self.edns = edns - self.ednsflags = ednsflags - self.payload = payload - self.options = options - self.request_payload = request_payload - - def want_dnssec(self, wanted=True): - """Enable or disable 'DNSSEC desired' flag in requests. - - *wanted*, a ``bool``. If ``True``, then DNSSEC data is - desired in the response, EDNS is enabled if required, and then - the DO bit is set. If ``False``, the DO bit is cleared if - EDNS is enabled. - """ - - if wanted: - if self.edns < 0: - self.use_edns() - self.ednsflags |= dns.flags.DO - elif self.edns >= 0: - self.ednsflags &= ~dns.flags.DO - - def rcode(self): - """Return the rcode. - - Returns an ``int``. - """ - return dns.rcode.from_flags(self.flags, self.ednsflags) - - def set_rcode(self, rcode): - """Set the rcode. - - *rcode*, an ``int``, is the rcode to set. - """ - (value, evalue) = dns.rcode.to_flags(rcode) - self.flags &= 0xFFF0 - self.flags |= value - self.ednsflags &= long(0x00FFFFFF) - self.ednsflags |= evalue - if self.ednsflags != 0 and self.edns < 0: - self.edns = 0 - - def opcode(self): - """Return the opcode. - - Returns an ``int``. - """ - return dns.opcode.from_flags(self.flags) - - def set_opcode(self, opcode): - """Set the opcode. - - *opcode*, an ``int``, is the opcode to set. - """ - self.flags &= 0x87FF - self.flags |= dns.opcode.to_flags(opcode) - - -class _WireReader(object): - - """Wire format reader. - - wire: a binary, is the wire-format message. - message: The message object being built - current: When building a message object from wire format, this - variable contains the offset from the beginning of wire of the next octet - to be read. - updating: Is the message a dynamic update? - one_rr_per_rrset: Put each RR into its own RRset? - ignore_trailing: Ignore trailing junk at end of request? - zone_rdclass: The class of the zone in messages which are - DNS dynamic updates. - """ - - def __init__(self, wire, message, question_only=False, - one_rr_per_rrset=False, ignore_trailing=False): - self.wire = dns.wiredata.maybe_wrap(wire) - self.message = message - self.current = 0 - self.updating = False - self.zone_rdclass = dns.rdataclass.IN - self.question_only = question_only - self.one_rr_per_rrset = one_rr_per_rrset - self.ignore_trailing = ignore_trailing - - def _get_question(self, qcount): - """Read the next *qcount* records from the wire data and add them to - the question section. - """ - - if self.updating and qcount > 1: - raise dns.exception.FormError - - for i in xrange(0, qcount): - (qname, used) = dns.name.from_wire(self.wire, self.current) - if self.message.origin is not None: - qname = qname.relativize(self.message.origin) - self.current = self.current + used - (rdtype, rdclass) = \ - struct.unpack('!HH', - self.wire[self.current:self.current + 4]) - self.current = self.current + 4 - self.message.find_rrset(self.message.question, qname, - rdclass, rdtype, create=True, - force_unique=True) - if self.updating: - self.zone_rdclass = rdclass - - def _get_section(self, section, count): - """Read the next I{count} records from the wire data and add them to - the specified section. - - section: the section of the message to which to add records - count: the number of records to read - """ - - if self.updating or self.one_rr_per_rrset: - force_unique = True - else: - force_unique = False - seen_opt = False - for i in xrange(0, count): - rr_start = self.current - (name, used) = dns.name.from_wire(self.wire, self.current) - absolute_name = name - if self.message.origin is not None: - name = name.relativize(self.message.origin) - self.current = self.current + used - (rdtype, rdclass, ttl, rdlen) = \ - struct.unpack('!HHIH', - self.wire[self.current:self.current + 10]) - self.current = self.current + 10 - if rdtype == dns.rdatatype.OPT: - if section is not self.message.additional or seen_opt: - raise BadEDNS - self.message.payload = rdclass - self.message.ednsflags = ttl - self.message.edns = (ttl & 0xff0000) >> 16 - self.message.options = [] - current = self.current - optslen = rdlen - while optslen > 0: - (otype, olen) = \ - struct.unpack('!HH', - self.wire[current:current + 4]) - current = current + 4 - opt = dns.edns.option_from_wire( - otype, self.wire, current, olen) - self.message.options.append(opt) - current = current + olen - optslen = optslen - 4 - olen - seen_opt = True - elif rdtype == dns.rdatatype.TSIG: - if not (section is self.message.additional and - i == (count - 1)): - raise BadTSIG - if self.message.keyring is None: - raise UnknownTSIGKey('got signed message without keyring') - secret = self.message.keyring.get(absolute_name) - if secret is None: - raise UnknownTSIGKey("key '%s' unknown" % name) - self.message.keyname = absolute_name - (self.message.keyalgorithm, self.message.mac) = \ - dns.tsig.get_algorithm_and_mac(self.wire, self.current, - rdlen) - self.message.tsig_ctx = \ - dns.tsig.validate(self.wire, - absolute_name, - secret, - int(time.time()), - self.message.request_mac, - rr_start, - self.current, - rdlen, - self.message.tsig_ctx, - self.message.multi, - self.message.first) - self.message.had_tsig = True - else: - if ttl < 0: - ttl = 0 - if self.updating and \ - (rdclass == dns.rdataclass.ANY or - rdclass == dns.rdataclass.NONE): - deleting = rdclass - rdclass = self.zone_rdclass - else: - deleting = None - if deleting == dns.rdataclass.ANY or \ - (deleting == dns.rdataclass.NONE and - section is self.message.answer): - covers = dns.rdatatype.NONE - rd = None - else: - rd = dns.rdata.from_wire(rdclass, rdtype, self.wire, - self.current, rdlen, - self.message.origin) - covers = rd.covers() - if self.message.xfr and rdtype == dns.rdatatype.SOA: - force_unique = True - rrset = self.message.find_rrset(section, name, - rdclass, rdtype, covers, - deleting, True, force_unique) - if rd is not None: - rrset.add(rd, ttl) - self.current = self.current + rdlen - - def read(self): - """Read a wire format DNS message and build a dns.message.Message - object.""" - - l = len(self.wire) - if l < 12: - raise ShortHeader - (self.message.id, self.message.flags, qcount, ancount, - aucount, adcount) = struct.unpack('!HHHHHH', self.wire[:12]) - self.current = 12 - if dns.opcode.is_update(self.message.flags): - self.updating = True - self._get_question(qcount) - if self.question_only: - return - self._get_section(self.message.answer, ancount) - self._get_section(self.message.authority, aucount) - self._get_section(self.message.additional, adcount) - if not self.ignore_trailing and self.current != l: - raise TrailingJunk - if self.message.multi and self.message.tsig_ctx and \ - not self.message.had_tsig: - self.message.tsig_ctx.update(self.wire) - - -def from_wire(wire, keyring=None, request_mac=b'', xfr=False, origin=None, - tsig_ctx=None, multi=False, first=True, - question_only=False, one_rr_per_rrset=False, - ignore_trailing=False): - """Convert a DNS wire format message into a message - object. - - *keyring*, a ``dict``, the keyring to use if the message is signed. - - *request_mac*, a ``binary``. If the message is a response to a - TSIG-signed request, *request_mac* should be set to the MAC of - that request. - - *xfr*, a ``bool``, should be set to ``True`` if this message is part of - a zone transfer. - - *origin*, a ``dns.name.Name`` or ``None``. If the message is part - of a zone transfer, *origin* should be the origin name of the - zone. - - *tsig_ctx*, a ``hmac.HMAC`` objext, the ongoing TSIG context, used - when validating zone transfers. - - *multi*, a ``bool``, should be set to ``True`` if this message - part of a multiple message sequence. - - *first*, a ``bool``, should be set to ``True`` if this message is - stand-alone, or the first message in a multi-message sequence. - - *question_only*, a ``bool``. If ``True``, read only up to - the end of the question section. - - *one_rr_per_rrset*, a ``bool``. If ``True``, put each RR into its - own RRset. - - *ignore_trailing*, a ``bool``. If ``True``, ignore trailing - junk at end of the message. - - Raises ``dns.message.ShortHeader`` if the message is less than 12 octets - long. - - Raises ``dns.messaage.TrailingJunk`` if there were octets in the message - past the end of the proper DNS message, and *ignore_trailing* is ``False``. - - Raises ``dns.message.BadEDNS`` if an OPT record was in the - wrong section, or occurred more than once. - - Raises ``dns.message.BadTSIG`` if a TSIG record was not the last - record of the additional data section. - - Returns a ``dns.message.Message``. - """ - - m = Message(id=0) - m.keyring = keyring - m.request_mac = request_mac - m.xfr = xfr - m.origin = origin - m.tsig_ctx = tsig_ctx - m.multi = multi - m.first = first - - reader = _WireReader(wire, m, question_only, one_rr_per_rrset, - ignore_trailing) - reader.read() - - return m - - -class _TextReader(object): - - """Text format reader. - - tok: the tokenizer. - message: The message object being built. - updating: Is the message a dynamic update? - zone_rdclass: The class of the zone in messages which are - DNS dynamic updates. - last_name: The most recently read name when building a message object. - """ - - def __init__(self, text, message): - self.message = message - self.tok = dns.tokenizer.Tokenizer(text) - self.last_name = None - self.zone_rdclass = dns.rdataclass.IN - self.updating = False - - def _header_line(self, section): - """Process one line from the text format header section.""" - - token = self.tok.get() - what = token.value - if what == 'id': - self.message.id = self.tok.get_int() - elif what == 'flags': - while True: - token = self.tok.get() - if not token.is_identifier(): - self.tok.unget(token) - break - self.message.flags = self.message.flags | \ - dns.flags.from_text(token.value) - if dns.opcode.is_update(self.message.flags): - self.updating = True - elif what == 'edns': - self.message.edns = self.tok.get_int() - self.message.ednsflags = self.message.ednsflags | \ - (self.message.edns << 16) - elif what == 'eflags': - if self.message.edns < 0: - self.message.edns = 0 - while True: - token = self.tok.get() - if not token.is_identifier(): - self.tok.unget(token) - break - self.message.ednsflags = self.message.ednsflags | \ - dns.flags.edns_from_text(token.value) - elif what == 'payload': - self.message.payload = self.tok.get_int() - if self.message.edns < 0: - self.message.edns = 0 - elif what == 'opcode': - text = self.tok.get_string() - self.message.flags = self.message.flags | \ - dns.opcode.to_flags(dns.opcode.from_text(text)) - elif what == 'rcode': - text = self.tok.get_string() - self.message.set_rcode(dns.rcode.from_text(text)) - else: - raise UnknownHeaderField - self.tok.get_eol() - - def _question_line(self, section): - """Process one line from the text format question section.""" - - token = self.tok.get(want_leading=True) - if not token.is_whitespace(): - self.last_name = dns.name.from_text(token.value, None) - name = self.last_name - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - # Class - try: - rdclass = dns.rdataclass.from_text(token.value) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except dns.exception.SyntaxError: - raise dns.exception.SyntaxError - except Exception: - rdclass = dns.rdataclass.IN - # Type - rdtype = dns.rdatatype.from_text(token.value) - self.message.find_rrset(self.message.question, name, - rdclass, rdtype, create=True, - force_unique=True) - if self.updating: - self.zone_rdclass = rdclass - self.tok.get_eol() - - def _rr_line(self, section): - """Process one line from the text format answer, authority, or - additional data sections. - """ - - deleting = None - # Name - token = self.tok.get(want_leading=True) - if not token.is_whitespace(): - self.last_name = dns.name.from_text(token.value, None) - name = self.last_name - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - # TTL - try: - ttl = int(token.value, 0) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except dns.exception.SyntaxError: - raise dns.exception.SyntaxError - except Exception: - ttl = 0 - # Class - try: - rdclass = dns.rdataclass.from_text(token.value) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - if rdclass == dns.rdataclass.ANY or rdclass == dns.rdataclass.NONE: - deleting = rdclass - rdclass = self.zone_rdclass - except dns.exception.SyntaxError: - raise dns.exception.SyntaxError - except Exception: - rdclass = dns.rdataclass.IN - # Type - rdtype = dns.rdatatype.from_text(token.value) - token = self.tok.get() - if not token.is_eol_or_eof(): - self.tok.unget(token) - rd = dns.rdata.from_text(rdclass, rdtype, self.tok, None) - covers = rd.covers() - else: - rd = None - covers = dns.rdatatype.NONE - rrset = self.message.find_rrset(section, name, - rdclass, rdtype, covers, - deleting, True, self.updating) - if rd is not None: - rrset.add(rd, ttl) - - def read(self): - """Read a text format DNS message and build a dns.message.Message - object.""" - - line_method = self._header_line - section = None - while 1: - token = self.tok.get(True, True) - if token.is_eol_or_eof(): - break - if token.is_comment(): - u = token.value.upper() - if u == 'HEADER': - line_method = self._header_line - elif u == 'QUESTION' or u == 'ZONE': - line_method = self._question_line - section = self.message.question - elif u == 'ANSWER' or u == 'PREREQ': - line_method = self._rr_line - section = self.message.answer - elif u == 'AUTHORITY' or u == 'UPDATE': - line_method = self._rr_line - section = self.message.authority - elif u == 'ADDITIONAL': - line_method = self._rr_line - section = self.message.additional - self.tok.get_eol() - continue - self.tok.unget(token) - line_method(section) - - -def from_text(text): - """Convert the text format message into a message object. - - *text*, a ``text``, the text format message. - - Raises ``dns.message.UnknownHeaderField`` if a header is unknown. - - Raises ``dns.exception.SyntaxError`` if the text is badly formed. - - Returns a ``dns.message.Message object`` - """ - - # 'text' can also be a file, but we don't publish that fact - # since it's an implementation detail. The official file - # interface is from_file(). - - m = Message() - - reader = _TextReader(text, m) - reader.read() - - return m - - -def from_file(f): - """Read the next text format message from the specified file. - - *f*, a ``file`` or ``text``. If *f* is text, it is treated as the - pathname of a file to open. - - Raises ``dns.message.UnknownHeaderField`` if a header is unknown. - - Raises ``dns.exception.SyntaxError`` if the text is badly formed. - - Returns a ``dns.message.Message object`` - """ - - str_type = string_types - opts = 'rU' - - if isinstance(f, str_type): - f = open(f, opts) - want_close = True - else: - want_close = False - - try: - m = from_text(f) - finally: - if want_close: - f.close() - return m - - -def make_query(qname, rdtype, rdclass=dns.rdataclass.IN, use_edns=None, - want_dnssec=False, ednsflags=None, payload=None, - request_payload=None, options=None): - """Make a query message. - - The query name, type, and class may all be specified either - as objects of the appropriate type, or as strings. - - The query will have a randomly chosen query id, and its DNS flags - will be set to dns.flags.RD. - - qname, a ``dns.name.Name`` or ``text``, the query name. - - *rdtype*, an ``int`` or ``text``, the desired rdata type. - - *rdclass*, an ``int`` or ``text``, the desired rdata class; the default - is class IN. - - *use_edns*, an ``int``, ``bool`` or ``None``. The EDNS level to use; the - default is None (no EDNS). - See the description of dns.message.Message.use_edns() for the possible - values for use_edns and their meanings. - - *want_dnssec*, a ``bool``. If ``True``, DNSSEC data is desired. - - *ednsflags*, an ``int``, the EDNS flag values. - - *payload*, an ``int``, is the EDNS sender's payload field, which is the - maximum size of UDP datagram the sender can handle. I.e. how big - a response to this message can be. - - *request_payload*, an ``int``, is the EDNS payload size to use when - sending this message. If not specified, defaults to the value of - *payload*. - - *options*, a list of ``dns.edns.Option`` objects or ``None``, the EDNS - options. - - Returns a ``dns.message.Message`` - """ - - if isinstance(qname, string_types): - qname = dns.name.from_text(qname) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if isinstance(rdclass, string_types): - rdclass = dns.rdataclass.from_text(rdclass) - m = Message() - m.flags |= dns.flags.RD - m.find_rrset(m.question, qname, rdclass, rdtype, create=True, - force_unique=True) - # only pass keywords on to use_edns if they have been set to a - # non-None value. Setting a field will turn EDNS on if it hasn't - # been configured. - kwargs = {} - if ednsflags is not None: - kwargs['ednsflags'] = ednsflags - if use_edns is None: - use_edns = 0 - if payload is not None: - kwargs['payload'] = payload - if use_edns is None: - use_edns = 0 - if request_payload is not None: - kwargs['request_payload'] = request_payload - if use_edns is None: - use_edns = 0 - if options is not None: - kwargs['options'] = options - if use_edns is None: - use_edns = 0 - kwargs['edns'] = use_edns - m.use_edns(**kwargs) - m.want_dnssec(want_dnssec) - return m - - -def make_response(query, recursion_available=False, our_payload=8192, - fudge=300): - """Make a message which is a response for the specified query. - The message returned is really a response skeleton; it has all - of the infrastructure required of a response, but none of the - content. - - The response's question section is a shallow copy of the query's - question section, so the query's question RRsets should not be - changed. - - *query*, a ``dns.message.Message``, the query to respond to. - - *recursion_available*, a ``bool``, should RA be set in the response? - - *our_payload*, an ``int``, the payload size to advertise in EDNS - responses. - - *fudge*, an ``int``, the TSIG time fudge. - - Returns a ``dns.message.Message`` object. - """ - - if query.flags & dns.flags.QR: - raise dns.exception.FormError('specified query message is not a query') - response = dns.message.Message(query.id) - response.flags = dns.flags.QR | (query.flags & dns.flags.RD) - if recursion_available: - response.flags |= dns.flags.RA - response.set_opcode(query.opcode()) - response.question = list(query.question) - if query.edns >= 0: - response.use_edns(0, 0, our_payload, query.payload) - if query.had_tsig: - response.use_tsig(query.keyring, query.keyname, fudge, None, 0, b'', - query.keyalgorithm) - response.request_mac = query.mac - return response diff --git a/src/dns/message.pyi b/src/dns/message.pyi deleted file mode 100644 index ed99b3c0..00000000 --- a/src/dns/message.pyi +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Optional, Dict, List, Tuple, Union -from . import name, rrset, tsig, rdatatype, entropy, edns, rdataclass -import hmac - -class Message: - def to_wire(self, origin : Optional[name.Name]=None, max_size=0, **kw) -> bytes: - ... - def find_rrset(self, section : List[rrset.RRset], name : name.Name, rdclass : int, rdtype : int, - covers=rdatatype.NONE, deleting : Optional[int]=None, create=False, - force_unique=False) -> rrset.RRset: - ... - def __init__(self, id : Optional[int] =None) -> None: - self.id : int - self.flags = 0 - self.question : List[rrset.RRset] = [] - self.answer : List[rrset.RRset] = [] - self.authority : List[rrset.RRset] = [] - self.additional : List[rrset.RRset] = [] - self.edns = -1 - self.ednsflags = 0 - self.payload = 0 - self.options : List[edns.Option] = [] - self.request_payload = 0 - self.keyring = None - self.keyname = None - self.keyalgorithm = tsig.default_algorithm - self.request_mac = b'' - self.other_data = b'' - self.tsig_error = 0 - self.fudge = 300 - self.original_id = self.id - self.mac = b'' - self.xfr = False - self.origin = None - self.tsig_ctx = None - self.had_tsig = False - self.multi = False - self.first = True - self.index : Dict[Tuple[rrset.RRset, name.Name, int, int, Union[int,str], int], rrset.RRset] = {} -def from_text(a : str) -> Message: - ... - -def from_wire(wire, keyring : Optional[Dict[name.Name,bytes]] = None, request_mac = b'', xfr=False, origin=None, - tsig_ctx : Optional[hmac.HMAC] = None, multi=False, first=True, - question_only=False, one_rr_per_rrset=False, - ignore_trailing=False) -> Message: - ... -def make_response(query : Message, recursion_available=False, our_payload=8192, - fudge=300) -> Message: - ... - -def make_query(qname : Union[name.Name,str], rdtype : Union[str,int], rdclass : Union[int,str] =rdataclass.IN, use_edns : Optional[bool] = None, - want_dnssec=False, ednsflags : Optional[int] = None, payload : Optional[int] = None, - request_payload : Optional[int] = None, options : Optional[List[edns.Option]] = None) -> Message: - ... diff --git a/src/dns/name.py b/src/dns/name.py deleted file mode 100644 index 0bcfd834..00000000 --- a/src/dns/name.py +++ /dev/null @@ -1,994 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Names. -""" - -from io import BytesIO -import struct -import sys -import copy -import encodings.idna -try: - import idna - have_idna_2008 = True -except ImportError: - have_idna_2008 = False - -import dns.exception -import dns.wiredata - -from ._compat import long, binary_type, text_type, unichr, maybe_decode - -try: - maxint = sys.maxint # pylint: disable=sys-max-int -except AttributeError: - maxint = (1 << (8 * struct.calcsize("P"))) // 2 - 1 - - -# fullcompare() result values - -#: The compared names have no relationship to each other. -NAMERELN_NONE = 0 -#: the first name is a superdomain of the second. -NAMERELN_SUPERDOMAIN = 1 -#: The first name is a subdomain of the second. -NAMERELN_SUBDOMAIN = 2 -#: The compared names are equal. -NAMERELN_EQUAL = 3 -#: The compared names have a common ancestor. -NAMERELN_COMMONANCESTOR = 4 - - -class EmptyLabel(dns.exception.SyntaxError): - """A DNS label is empty.""" - - -class BadEscape(dns.exception.SyntaxError): - """An escaped code in a text format of DNS name is invalid.""" - - -class BadPointer(dns.exception.FormError): - """A DNS compression pointer points forward instead of backward.""" - - -class BadLabelType(dns.exception.FormError): - """The label type in DNS name wire format is unknown.""" - - -class NeedAbsoluteNameOrOrigin(dns.exception.DNSException): - """An attempt was made to convert a non-absolute name to - wire when there was also a non-absolute (or missing) origin.""" - - -class NameTooLong(dns.exception.FormError): - """A DNS name is > 255 octets long.""" - - -class LabelTooLong(dns.exception.SyntaxError): - """A DNS label is > 63 octets long.""" - - -class AbsoluteConcatenation(dns.exception.DNSException): - """An attempt was made to append anything other than the - empty name to an absolute DNS name.""" - - -class NoParent(dns.exception.DNSException): - """An attempt was made to get the parent of the root name - or the empty name.""" - -class NoIDNA2008(dns.exception.DNSException): - """IDNA 2008 processing was requested but the idna module is not - available.""" - - -class IDNAException(dns.exception.DNSException): - """IDNA processing raised an exception.""" - - supp_kwargs = {'idna_exception'} - fmt = "IDNA processing exception: {idna_exception}" - - -class IDNACodec(object): - """Abstract base class for IDNA encoder/decoders.""" - - def __init__(self): - pass - - def encode(self, label): - raise NotImplementedError - - def decode(self, label): - # We do not apply any IDNA policy on decode; we just - downcased = label.lower() - if downcased.startswith(b'xn--'): - try: - label = downcased[4:].decode('punycode') - except Exception as e: - raise IDNAException(idna_exception=e) - else: - label = maybe_decode(label) - return _escapify(label, True) - - -class IDNA2003Codec(IDNACodec): - """IDNA 2003 encoder/decoder.""" - - def __init__(self, strict_decode=False): - """Initialize the IDNA 2003 encoder/decoder. - - *strict_decode* is a ``bool``. If `True`, then IDNA2003 checking - is done when decoding. This can cause failures if the name - was encoded with IDNA2008. The default is `False`. - """ - - super(IDNA2003Codec, self).__init__() - self.strict_decode = strict_decode - - def encode(self, label): - """Encode *label*.""" - - if label == '': - return b'' - try: - return encodings.idna.ToASCII(label) - except UnicodeError: - raise LabelTooLong - - def decode(self, label): - """Decode *label*.""" - if not self.strict_decode: - return super(IDNA2003Codec, self).decode(label) - if label == b'': - return u'' - try: - return _escapify(encodings.idna.ToUnicode(label), True) - except Exception as e: - raise IDNAException(idna_exception=e) - - -class IDNA2008Codec(IDNACodec): - """IDNA 2008 encoder/decoder. - - *uts_46* is a ``bool``. If True, apply Unicode IDNA - compatibility processing as described in Unicode Technical - Standard #46 (http://unicode.org/reports/tr46/). - If False, do not apply the mapping. The default is False. - - *transitional* is a ``bool``: If True, use the - "transitional" mode described in Unicode Technical Standard - #46. The default is False. - - *allow_pure_ascii* is a ``bool``. If True, then a label which - consists of only ASCII characters is allowed. This is less - strict than regular IDNA 2008, but is also necessary for mixed - names, e.g. a name with starting with "_sip._tcp." and ending - in an IDN suffix which would otherwise be disallowed. The - default is False. - - *strict_decode* is a ``bool``: If True, then IDNA2008 checking - is done when decoding. This can cause failures if the name - was encoded with IDNA2003. The default is False. - """ - - def __init__(self, uts_46=False, transitional=False, - allow_pure_ascii=False, strict_decode=False): - """Initialize the IDNA 2008 encoder/decoder.""" - super(IDNA2008Codec, self).__init__() - self.uts_46 = uts_46 - self.transitional = transitional - self.allow_pure_ascii = allow_pure_ascii - self.strict_decode = strict_decode - - def is_all_ascii(self, label): - for c in label: - if ord(c) > 0x7f: - return False - return True - - def encode(self, label): - if label == '': - return b'' - if self.allow_pure_ascii and self.is_all_ascii(label): - return label.encode('ascii') - if not have_idna_2008: - raise NoIDNA2008 - try: - if self.uts_46: - label = idna.uts46_remap(label, False, self.transitional) - return idna.alabel(label) - except idna.IDNAError as e: - raise IDNAException(idna_exception=e) - - def decode(self, label): - if not self.strict_decode: - return super(IDNA2008Codec, self).decode(label) - if label == b'': - return u'' - if not have_idna_2008: - raise NoIDNA2008 - try: - if self.uts_46: - label = idna.uts46_remap(label, False, False) - return _escapify(idna.ulabel(label), True) - except idna.IDNAError as e: - raise IDNAException(idna_exception=e) - -_escaped = bytearray(b'"().;\\@$') - -IDNA_2003_Practical = IDNA2003Codec(False) -IDNA_2003_Strict = IDNA2003Codec(True) -IDNA_2003 = IDNA_2003_Practical -IDNA_2008_Practical = IDNA2008Codec(True, False, True, False) -IDNA_2008_UTS_46 = IDNA2008Codec(True, False, False, False) -IDNA_2008_Strict = IDNA2008Codec(False, False, False, True) -IDNA_2008_Transitional = IDNA2008Codec(True, True, False, False) -IDNA_2008 = IDNA_2008_Practical - -def _escapify(label, unicode_mode=False): - """Escape the characters in label which need it. - @param unicode_mode: escapify only special and whitespace (<= 0x20) - characters - @returns: the escaped string - @rtype: string""" - if not unicode_mode: - text = '' - if isinstance(label, text_type): - label = label.encode() - for c in bytearray(label): - if c in _escaped: - text += '\\' + chr(c) - elif c > 0x20 and c < 0x7F: - text += chr(c) - else: - text += '\\%03d' % c - return text.encode() - - text = u'' - if isinstance(label, binary_type): - label = label.decode() - for c in label: - if c > u'\x20' and c < u'\x7f': - text += c - else: - if c >= u'\x7f': - text += c - else: - text += u'\\%03d' % ord(c) - return text - -def _validate_labels(labels): - """Check for empty labels in the middle of a label sequence, - labels that are too long, and for too many labels. - - Raises ``dns.name.NameTooLong`` if the name as a whole is too long. - - Raises ``dns.name.EmptyLabel`` if a label is empty (i.e. the root - label) and appears in a position other than the end of the label - sequence - - """ - - l = len(labels) - total = 0 - i = -1 - j = 0 - for label in labels: - ll = len(label) - total += ll + 1 - if ll > 63: - raise LabelTooLong - if i < 0 and label == b'': - i = j - j += 1 - if total > 255: - raise NameTooLong - if i >= 0 and i != l - 1: - raise EmptyLabel - - -def _maybe_convert_to_binary(label): - """If label is ``text``, convert it to ``binary``. If it is already - ``binary`` just return it. - - """ - - if isinstance(label, binary_type): - return label - if isinstance(label, text_type): - return label.encode() - raise ValueError - - -class Name(object): - - """A DNS name. - - The dns.name.Name class represents a DNS name as a tuple of - labels. Each label is a `binary` in DNS wire format. Instances - of the class are immutable. - """ - - __slots__ = ['labels'] - - def __init__(self, labels): - """*labels* is any iterable whose values are ``text`` or ``binary``. - """ - - labels = [_maybe_convert_to_binary(x) for x in labels] - super(Name, self).__setattr__('labels', tuple(labels)) - _validate_labels(self.labels) - - def __setattr__(self, name, value): - # Names are immutable - raise TypeError("object doesn't support attribute assignment") - - def __copy__(self): - return Name(self.labels) - - def __deepcopy__(self, memo): - return Name(copy.deepcopy(self.labels, memo)) - - def __getstate__(self): - # Names can be pickled - return {'labels': self.labels} - - def __setstate__(self, state): - super(Name, self).__setattr__('labels', state['labels']) - _validate_labels(self.labels) - - def is_absolute(self): - """Is the most significant label of this name the root label? - - Returns a ``bool``. - """ - - return len(self.labels) > 0 and self.labels[-1] == b'' - - def is_wild(self): - """Is this name wild? (I.e. Is the least significant label '*'?) - - Returns a ``bool``. - """ - - return len(self.labels) > 0 and self.labels[0] == b'*' - - def __hash__(self): - """Return a case-insensitive hash of the name. - - Returns an ``int``. - """ - - h = long(0) - for label in self.labels: - for c in bytearray(label.lower()): - h += (h << 3) + c - return int(h % maxint) - - def fullcompare(self, other): - """Compare two names, returning a 3-tuple - ``(relation, order, nlabels)``. - - *relation* describes the relation ship between the names, - and is one of: ``dns.name.NAMERELN_NONE``, - ``dns.name.NAMERELN_SUPERDOMAIN``, ``dns.name.NAMERELN_SUBDOMAIN``, - ``dns.name.NAMERELN_EQUAL``, or ``dns.name.NAMERELN_COMMONANCESTOR``. - - *order* is < 0 if *self* < *other*, > 0 if *self* > *other*, and == - 0 if *self* == *other*. A relative name is always less than an - absolute name. If both names have the same relativity, then - the DNSSEC order relation is used to order them. - - *nlabels* is the number of significant labels that the two names - have in common. - - Here are some examples. Names ending in "." are absolute names, - those not ending in "." are relative names. - - ============= ============= =========== ===== ======= - self other relation order nlabels - ============= ============= =========== ===== ======= - www.example. www.example. equal 0 3 - www.example. example. subdomain > 0 2 - example. www.example. superdomain < 0 2 - example1.com. example2.com. common anc. < 0 2 - example1 example2. none < 0 0 - example1. example2 none > 0 0 - ============= ============= =========== ===== ======= - """ - - sabs = self.is_absolute() - oabs = other.is_absolute() - if sabs != oabs: - if sabs: - return (NAMERELN_NONE, 1, 0) - else: - return (NAMERELN_NONE, -1, 0) - l1 = len(self.labels) - l2 = len(other.labels) - ldiff = l1 - l2 - if ldiff < 0: - l = l1 - else: - l = l2 - - order = 0 - nlabels = 0 - namereln = NAMERELN_NONE - while l > 0: - l -= 1 - l1 -= 1 - l2 -= 1 - label1 = self.labels[l1].lower() - label2 = other.labels[l2].lower() - if label1 < label2: - order = -1 - if nlabels > 0: - namereln = NAMERELN_COMMONANCESTOR - return (namereln, order, nlabels) - elif label1 > label2: - order = 1 - if nlabels > 0: - namereln = NAMERELN_COMMONANCESTOR - return (namereln, order, nlabels) - nlabels += 1 - order = ldiff - if ldiff < 0: - namereln = NAMERELN_SUPERDOMAIN - elif ldiff > 0: - namereln = NAMERELN_SUBDOMAIN - else: - namereln = NAMERELN_EQUAL - return (namereln, order, nlabels) - - def is_subdomain(self, other): - """Is self a subdomain of other? - - Note that the notion of subdomain includes equality, e.g. - "dnpython.org" is a subdomain of itself. - - Returns a ``bool``. - """ - - (nr, o, nl) = self.fullcompare(other) - if nr == NAMERELN_SUBDOMAIN or nr == NAMERELN_EQUAL: - return True - return False - - def is_superdomain(self, other): - """Is self a superdomain of other? - - Note that the notion of superdomain includes equality, e.g. - "dnpython.org" is a superdomain of itself. - - Returns a ``bool``. - """ - - (nr, o, nl) = self.fullcompare(other) - if nr == NAMERELN_SUPERDOMAIN or nr == NAMERELN_EQUAL: - return True - return False - - def canonicalize(self): - """Return a name which is equal to the current name, but is in - DNSSEC canonical form. - """ - - return Name([x.lower() for x in self.labels]) - - def __eq__(self, other): - if isinstance(other, Name): - return self.fullcompare(other)[1] == 0 - else: - return False - - def __ne__(self, other): - if isinstance(other, Name): - return self.fullcompare(other)[1] != 0 - else: - return True - - def __lt__(self, other): - if isinstance(other, Name): - return self.fullcompare(other)[1] < 0 - else: - return NotImplemented - - def __le__(self, other): - if isinstance(other, Name): - return self.fullcompare(other)[1] <= 0 - else: - return NotImplemented - - def __ge__(self, other): - if isinstance(other, Name): - return self.fullcompare(other)[1] >= 0 - else: - return NotImplemented - - def __gt__(self, other): - if isinstance(other, Name): - return self.fullcompare(other)[1] > 0 - else: - return NotImplemented - - def __repr__(self): - return '' - - def __str__(self): - return self.to_text(False) - - def to_text(self, omit_final_dot=False): - """Convert name to DNS text format. - - *omit_final_dot* is a ``bool``. If True, don't emit the final - dot (denoting the root label) for absolute names. The default - is False. - - Returns a ``text``. - """ - - if len(self.labels) == 0: - return maybe_decode(b'@') - if len(self.labels) == 1 and self.labels[0] == b'': - return maybe_decode(b'.') - if omit_final_dot and self.is_absolute(): - l = self.labels[:-1] - else: - l = self.labels - s = b'.'.join(map(_escapify, l)) - return maybe_decode(s) - - def to_unicode(self, omit_final_dot=False, idna_codec=None): - """Convert name to Unicode text format. - - IDN ACE labels are converted to Unicode. - - *omit_final_dot* is a ``bool``. If True, don't emit the final - dot (denoting the root label) for absolute names. The default - is False. - *idna_codec* specifies the IDNA encoder/decoder. If None, the - dns.name.IDNA_2003_Practical encoder/decoder is used. - The IDNA_2003_Practical decoder does - not impose any policy, it just decodes punycode, so if you - don't want checking for compliance, you can use this decoder - for IDNA2008 as well. - - Returns a ``text``. - """ - - if len(self.labels) == 0: - return u'@' - if len(self.labels) == 1 and self.labels[0] == b'': - return u'.' - if omit_final_dot and self.is_absolute(): - l = self.labels[:-1] - else: - l = self.labels - if idna_codec is None: - idna_codec = IDNA_2003_Practical - return u'.'.join([idna_codec.decode(x) for x in l]) - - def to_digestable(self, origin=None): - """Convert name to a format suitable for digesting in hashes. - - The name is canonicalized and converted to uncompressed wire - format. All names in wire format are absolute. If the name - is a relative name, then an origin must be supplied. - - *origin* is a ``dns.name.Name`` or ``None``. If the name is - relative and origin is not ``None``, then origin will be appended - to the name. - - Raises ``dns.name.NeedAbsoluteNameOrOrigin`` if the name is - relative and no origin was provided. - - Returns a ``binary``. - """ - - if not self.is_absolute(): - if origin is None or not origin.is_absolute(): - raise NeedAbsoluteNameOrOrigin - labels = list(self.labels) - labels.extend(list(origin.labels)) - else: - labels = self.labels - dlabels = [struct.pack('!B%ds' % len(x), len(x), x.lower()) - for x in labels] - return b''.join(dlabels) - - def to_wire(self, file=None, compress=None, origin=None): - """Convert name to wire format, possibly compressing it. - - *file* is the file where the name is emitted (typically a - BytesIO file). If ``None`` (the default), a ``binary`` - containing the wire name will be returned. - - *compress*, a ``dict``, is the compression table to use. If - ``None`` (the default), names will not be compressed. - - *origin* is a ``dns.name.Name`` or ``None``. If the name is - relative and origin is not ``None``, then *origin* will be appended - to it. - - Raises ``dns.name.NeedAbsoluteNameOrOrigin`` if the name is - relative and no origin was provided. - - Returns a ``binary`` or ``None``. - """ - - if file is None: - file = BytesIO() - want_return = True - else: - want_return = False - - if not self.is_absolute(): - if origin is None or not origin.is_absolute(): - raise NeedAbsoluteNameOrOrigin - labels = list(self.labels) - labels.extend(list(origin.labels)) - else: - labels = self.labels - i = 0 - for label in labels: - n = Name(labels[i:]) - i += 1 - if compress is not None: - pos = compress.get(n) - else: - pos = None - if pos is not None: - value = 0xc000 + pos - s = struct.pack('!H', value) - file.write(s) - break - else: - if compress is not None and len(n) > 1: - pos = file.tell() - if pos <= 0x3fff: - compress[n] = pos - l = len(label) - file.write(struct.pack('!B', l)) - if l > 0: - file.write(label) - if want_return: - return file.getvalue() - - def __len__(self): - """The length of the name (in labels). - - Returns an ``int``. - """ - - return len(self.labels) - - def __getitem__(self, index): - return self.labels[index] - - def __add__(self, other): - return self.concatenate(other) - - def __sub__(self, other): - return self.relativize(other) - - def split(self, depth): - """Split a name into a prefix and suffix names at the specified depth. - - *depth* is an ``int`` specifying the number of labels in the suffix - - Raises ``ValueError`` if *depth* was not >= 0 and <= the length of the - name. - - Returns the tuple ``(prefix, suffix)``. - """ - - l = len(self.labels) - if depth == 0: - return (self, dns.name.empty) - elif depth == l: - return (dns.name.empty, self) - elif depth < 0 or depth > l: - raise ValueError( - 'depth must be >= 0 and <= the length of the name') - return (Name(self[: -depth]), Name(self[-depth:])) - - def concatenate(self, other): - """Return a new name which is the concatenation of self and other. - - Raises ``dns.name.AbsoluteConcatenation`` if the name is - absolute and *other* is not the empty name. - - Returns a ``dns.name.Name``. - """ - - if self.is_absolute() and len(other) > 0: - raise AbsoluteConcatenation - labels = list(self.labels) - labels.extend(list(other.labels)) - return Name(labels) - - def relativize(self, origin): - """If the name is a subdomain of *origin*, return a new name which is - the name relative to origin. Otherwise return the name. - - For example, relativizing ``www.dnspython.org.`` to origin - ``dnspython.org.`` returns the name ``www``. Relativizing ``example.`` - to origin ``dnspython.org.`` returns ``example.``. - - Returns a ``dns.name.Name``. - """ - - if origin is not None and self.is_subdomain(origin): - return Name(self[: -len(origin)]) - else: - return self - - def derelativize(self, origin): - """If the name is a relative name, return a new name which is the - concatenation of the name and origin. Otherwise return the name. - - For example, derelativizing ``www`` to origin ``dnspython.org.`` - returns the name ``www.dnspython.org.``. Derelativizing ``example.`` - to origin ``dnspython.org.`` returns ``example.``. - - Returns a ``dns.name.Name``. - """ - - if not self.is_absolute(): - return self.concatenate(origin) - else: - return self - - def choose_relativity(self, origin=None, relativize=True): - """Return a name with the relativity desired by the caller. - - If *origin* is ``None``, then the name is returned. - Otherwise, if *relativize* is ``True`` the name is - relativized, and if *relativize* is ``False`` the name is - derelativized. - - Returns a ``dns.name.Name``. - """ - - if origin: - if relativize: - return self.relativize(origin) - else: - return self.derelativize(origin) - else: - return self - - def parent(self): - """Return the parent of the name. - - For example, the parent of ``www.dnspython.org.`` is ``dnspython.org``. - - Raises ``dns.name.NoParent`` if the name is either the root name or the - empty name, and thus has no parent. - - Returns a ``dns.name.Name``. - """ - - if self == root or self == empty: - raise NoParent - return Name(self.labels[1:]) - -#: The root name, '.' -root = Name([b'']) - -#: The empty name. -empty = Name([]) - -def from_unicode(text, origin=root, idna_codec=None): - """Convert unicode text into a Name object. - - Labels are encoded in IDN ACE form according to rules specified by - the IDNA codec. - - *text*, a ``text``, is the text to convert into a name. - - *origin*, a ``dns.name.Name``, specifies the origin to - append to non-absolute names. The default is the root name. - - *idna_codec*, a ``dns.name.IDNACodec``, specifies the IDNA - encoder/decoder. If ``None``, the default IDNA 2003 encoder/decoder - is used. - - Returns a ``dns.name.Name``. - """ - - if not isinstance(text, text_type): - raise ValueError("input to from_unicode() must be a unicode string") - if not (origin is None or isinstance(origin, Name)): - raise ValueError("origin must be a Name or None") - labels = [] - label = u'' - escaping = False - edigits = 0 - total = 0 - if idna_codec is None: - idna_codec = IDNA_2003 - if text == u'@': - text = u'' - if text: - if text == u'.': - return Name([b'']) # no Unicode "u" on this constant! - for c in text: - if escaping: - if edigits == 0: - if c.isdigit(): - total = int(c) - edigits += 1 - else: - label += c - escaping = False - else: - if not c.isdigit(): - raise BadEscape - total *= 10 - total += int(c) - edigits += 1 - if edigits == 3: - escaping = False - label += unichr(total) - elif c in [u'.', u'\u3002', u'\uff0e', u'\uff61']: - if len(label) == 0: - raise EmptyLabel - labels.append(idna_codec.encode(label)) - label = u'' - elif c == u'\\': - escaping = True - edigits = 0 - total = 0 - else: - label += c - if escaping: - raise BadEscape - if len(label) > 0: - labels.append(idna_codec.encode(label)) - else: - labels.append(b'') - - if (len(labels) == 0 or labels[-1] != b'') and origin is not None: - labels.extend(list(origin.labels)) - return Name(labels) - - -def from_text(text, origin=root, idna_codec=None): - """Convert text into a Name object. - - *text*, a ``text``, is the text to convert into a name. - - *origin*, a ``dns.name.Name``, specifies the origin to - append to non-absolute names. The default is the root name. - - *idna_codec*, a ``dns.name.IDNACodec``, specifies the IDNA - encoder/decoder. If ``None``, the default IDNA 2003 encoder/decoder - is used. - - Returns a ``dns.name.Name``. - """ - - if isinstance(text, text_type): - return from_unicode(text, origin, idna_codec) - if not isinstance(text, binary_type): - raise ValueError("input to from_text() must be a string") - if not (origin is None or isinstance(origin, Name)): - raise ValueError("origin must be a Name or None") - labels = [] - label = b'' - escaping = False - edigits = 0 - total = 0 - if text == b'@': - text = b'' - if text: - if text == b'.': - return Name([b'']) - for c in bytearray(text): - byte_ = struct.pack('!B', c) - if escaping: - if edigits == 0: - if byte_.isdigit(): - total = int(byte_) - edigits += 1 - else: - label += byte_ - escaping = False - else: - if not byte_.isdigit(): - raise BadEscape - total *= 10 - total += int(byte_) - edigits += 1 - if edigits == 3: - escaping = False - label += struct.pack('!B', total) - elif byte_ == b'.': - if len(label) == 0: - raise EmptyLabel - labels.append(label) - label = b'' - elif byte_ == b'\\': - escaping = True - edigits = 0 - total = 0 - else: - label += byte_ - if escaping: - raise BadEscape - if len(label) > 0: - labels.append(label) - else: - labels.append(b'') - if (len(labels) == 0 or labels[-1] != b'') and origin is not None: - labels.extend(list(origin.labels)) - return Name(labels) - - -def from_wire(message, current): - """Convert possibly compressed wire format into a Name. - - *message* is a ``binary`` containing an entire DNS message in DNS - wire form. - - *current*, an ``int``, is the offset of the beginning of the name - from the start of the message - - Raises ``dns.name.BadPointer`` if a compression pointer did not - point backwards in the message. - - Raises ``dns.name.BadLabelType`` if an invalid label type was encountered. - - Returns a ``(dns.name.Name, int)`` tuple consisting of the name - that was read and the number of bytes of the wire format message - which were consumed reading it. - """ - - if not isinstance(message, binary_type): - raise ValueError("input to from_wire() must be a byte string") - message = dns.wiredata.maybe_wrap(message) - labels = [] - biggest_pointer = current - hops = 0 - count = message[current] - current += 1 - cused = 1 - while count != 0: - if count < 64: - labels.append(message[current: current + count].unwrap()) - current += count - if hops == 0: - cused += count - elif count >= 192: - current = (count & 0x3f) * 256 + message[current] - if hops == 0: - cused += 1 - if current >= biggest_pointer: - raise BadPointer - biggest_pointer = current - hops += 1 - else: - raise BadLabelType - count = message[current] - current += 1 - if hops == 0: - cused += 1 - labels.append('') - return (Name(labels), cused) diff --git a/src/dns/name.pyi b/src/dns/name.pyi deleted file mode 100644 index 5a8061b2..00000000 --- a/src/dns/name.pyi +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Optional, Union, Tuple, Iterable, List - -class Name: - def is_subdomain(self, o : Name) -> bool: ... - def is_superdomain(self, o : Name) -> bool: ... - def __init__(self, labels : Iterable[Union[bytes,str]]) -> None: - self.labels : List[bytes] - def is_absolute(self) -> bool: ... - def is_wild(self) -> bool: ... - def fullcompare(self, other) -> Tuple[int,int,int]: ... - def canonicalize(self) -> Name: ... - def __lt__(self, other : Name): ... - def __le__(self, other : Name): ... - def __ge__(self, other : Name): ... - def __gt__(self, other : Name): ... - def to_text(self, omit_final_dot=False) -> str: ... - def to_unicode(self, omit_final_dot=False, idna_codec=None) -> str: ... - def to_digestable(self, origin=None) -> bytes: ... - def to_wire(self, file=None, compress=None, origin=None) -> Optional[bytes]: ... - def __add__(self, other : Name): ... - def __sub__(self, other : Name): ... - def split(self, depth) -> List[Tuple[str,str]]: ... - def concatenate(self, other : Name) -> Name: ... - def relativize(self, origin): ... - def derelativize(self, origin): ... - def choose_relativity(self, origin : Optional[Name] = None, relativize=True): ... - def parent(self) -> Name: ... - -class IDNACodec: - pass - -def from_text(text, origin : Optional[Name] = Name('.'), idna_codec : Optional[IDNACodec] = None) -> Name: - ... - -empty : Name diff --git a/src/dns/namedict.py b/src/dns/namedict.py deleted file mode 100644 index 37a13104..00000000 --- a/src/dns/namedict.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# Copyright (C) 2016 Coresec Systems AB -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND CORESEC SYSTEMS AB DISCLAIMS ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL CORESEC -# SYSTEMS AB BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION -# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS name dictionary""" - -import collections -import dns.name -from ._compat import xrange - - -class NameDict(collections.MutableMapping): - """A dictionary whose keys are dns.name.Name objects. - - In addition to being like a regular Python dictionary, this - dictionary can also get the deepest match for a given key. - """ - - __slots__ = ["max_depth", "max_depth_items", "__store"] - - def __init__(self, *args, **kwargs): - super(NameDict, self).__init__() - self.__store = dict() - #: the maximum depth of the keys that have ever been added - self.max_depth = 0 - #: the number of items of maximum depth - self.max_depth_items = 0 - self.update(dict(*args, **kwargs)) - - def __update_max_depth(self, key): - if len(key) == self.max_depth: - self.max_depth_items = self.max_depth_items + 1 - elif len(key) > self.max_depth: - self.max_depth = len(key) - self.max_depth_items = 1 - - def __getitem__(self, key): - return self.__store[key] - - def __setitem__(self, key, value): - if not isinstance(key, dns.name.Name): - raise ValueError('NameDict key must be a name') - self.__store[key] = value - self.__update_max_depth(key) - - def __delitem__(self, key): - value = self.__store.pop(key) - if len(value) == self.max_depth: - self.max_depth_items = self.max_depth_items - 1 - if self.max_depth_items == 0: - self.max_depth = 0 - for k in self.__store: - self.__update_max_depth(k) - - def __iter__(self): - return iter(self.__store) - - def __len__(self): - return len(self.__store) - - def has_key(self, key): - return key in self.__store - - def get_deepest_match(self, name): - """Find the deepest match to *fname* in the dictionary. - - The deepest match is the longest name in the dictionary which is - a superdomain of *name*. Note that *superdomain* includes matching - *name* itself. - - *name*, a ``dns.name.Name``, the name to find. - - Returns a ``(key, value)`` where *key* is the deepest - ``dns.name.Name``, and *value* is the value associated with *key*. - """ - - depth = len(name) - if depth > self.max_depth: - depth = self.max_depth - for i in xrange(-depth, 0): - n = dns.name.Name(name[i:]) - if n in self: - return (n, self[n]) - v = self[dns.name.empty] - return (dns.name.empty, v) diff --git a/src/dns/node.py b/src/dns/node.py deleted file mode 100644 index 8a7f19f5..00000000 --- a/src/dns/node.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS nodes. A node is a set of rdatasets.""" - -from io import StringIO - -import dns.rdataset -import dns.rdatatype -import dns.renderer - - -class Node(object): - - """A Node is a set of rdatasets.""" - - __slots__ = ['rdatasets'] - - def __init__(self): - #: the set of rdatsets, represented as a list. - self.rdatasets = [] - - def to_text(self, name, **kw): - """Convert a node to text format. - - Each rdataset at the node is printed. Any keyword arguments - to this method are passed on to the rdataset's to_text() method. - - *name*, a ``dns.name.Name`` or ``text``, the owner name of the rdatasets. - - Returns a ``text``. - """ - - s = StringIO() - for rds in self.rdatasets: - if len(rds) > 0: - s.write(rds.to_text(name, **kw)) - s.write(u'\n') - return s.getvalue()[:-1] - - def __repr__(self): - return '' - - def __eq__(self, other): - # - # This is inefficient. Good thing we don't need to do it much. - # - for rd in self.rdatasets: - if rd not in other.rdatasets: - return False - for rd in other.rdatasets: - if rd not in self.rdatasets: - return False - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def __len__(self): - return len(self.rdatasets) - - def __iter__(self): - return iter(self.rdatasets) - - def find_rdataset(self, rdclass, rdtype, covers=dns.rdatatype.NONE, - create=False): - """Find an rdataset matching the specified properties in the - current node. - - *rdclass*, an ``int``, the class of the rdataset. - - *rdtype*, an ``int``, the type of the rdataset. - - *covers*, an ``int``, the covered type. Usually this value is - dns.rdatatype.NONE, but if the rdtype is dns.rdatatype.SIG or - dns.rdatatype.RRSIG, then the covers value will be the rdata - type the SIG/RRSIG covers. The library treats the SIG and RRSIG - types as if they were a family of - types, e.g. RRSIG(A), RRSIG(NS), RRSIG(SOA). This makes RRSIGs much - easier to work with than if RRSIGs covering different rdata - types were aggregated into a single RRSIG rdataset. - - *create*, a ``bool``. If True, create the rdataset if it is not found. - - Raises ``KeyError`` if an rdataset of the desired type and class does - not exist and *create* is not ``True``. - - Returns a ``dns.rdataset.Rdataset``. - """ - - for rds in self.rdatasets: - if rds.match(rdclass, rdtype, covers): - return rds - if not create: - raise KeyError - rds = dns.rdataset.Rdataset(rdclass, rdtype) - self.rdatasets.append(rds) - return rds - - def get_rdataset(self, rdclass, rdtype, covers=dns.rdatatype.NONE, - create=False): - """Get an rdataset matching the specified properties in the - current node. - - None is returned if an rdataset of the specified type and - class does not exist and *create* is not ``True``. - - *rdclass*, an ``int``, the class of the rdataset. - - *rdtype*, an ``int``, the type of the rdataset. - - *covers*, an ``int``, the covered type. Usually this value is - dns.rdatatype.NONE, but if the rdtype is dns.rdatatype.SIG or - dns.rdatatype.RRSIG, then the covers value will be the rdata - type the SIG/RRSIG covers. The library treats the SIG and RRSIG - types as if they were a family of - types, e.g. RRSIG(A), RRSIG(NS), RRSIG(SOA). This makes RRSIGs much - easier to work with than if RRSIGs covering different rdata - types were aggregated into a single RRSIG rdataset. - - *create*, a ``bool``. If True, create the rdataset if it is not found. - - Returns a ``dns.rdataset.Rdataset`` or ``None``. - """ - - try: - rds = self.find_rdataset(rdclass, rdtype, covers, create) - except KeyError: - rds = None - return rds - - def delete_rdataset(self, rdclass, rdtype, covers=dns.rdatatype.NONE): - """Delete the rdataset matching the specified properties in the - current node. - - If a matching rdataset does not exist, it is not an error. - - *rdclass*, an ``int``, the class of the rdataset. - - *rdtype*, an ``int``, the type of the rdataset. - - *covers*, an ``int``, the covered type. - """ - - rds = self.get_rdataset(rdclass, rdtype, covers) - if rds is not None: - self.rdatasets.remove(rds) - - def replace_rdataset(self, replacement): - """Replace an rdataset. - - It is not an error if there is no rdataset matching *replacement*. - - Ownership of the *replacement* object is transferred to the node; - in other words, this method does not store a copy of *replacement* - at the node, it stores *replacement* itself. - - *replacement*, a ``dns.rdataset.Rdataset``. - - Raises ``ValueError`` if *replacement* is not a - ``dns.rdataset.Rdataset``. - """ - - if not isinstance(replacement, dns.rdataset.Rdataset): - raise ValueError('replacement is not an rdataset') - self.delete_rdataset(replacement.rdclass, replacement.rdtype, - replacement.covers) - self.rdatasets.append(replacement) diff --git a/src/dns/node.pyi b/src/dns/node.pyi deleted file mode 100644 index 0997edf9..00000000 --- a/src/dns/node.pyi +++ /dev/null @@ -1,17 +0,0 @@ -from typing import List, Optional, Union -from . import rdataset, rdatatype, name -class Node: - def __init__(self): - self.rdatasets : List[rdataset.Rdataset] - def to_text(self, name : Union[str,name.Name], **kw) -> str: - ... - def find_rdataset(self, rdclass : int, rdtype : int, covers=rdatatype.NONE, - create=False) -> rdataset.Rdataset: - ... - def get_rdataset(self, rdclass : int, rdtype : int, covers=rdatatype.NONE, - create=False) -> Optional[rdataset.Rdataset]: - ... - def delete_rdataset(self, rdclass : int, rdtype : int, covers=rdatatype.NONE): - ... - def replace_rdataset(self, replacement : rdataset.Rdataset) -> None: - ... diff --git a/src/dns/opcode.py b/src/dns/opcode.py deleted file mode 100644 index c0735ba4..00000000 --- a/src/dns/opcode.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Opcodes.""" - -import dns.exception - -#: Query -QUERY = 0 -#: Inverse Query (historical) -IQUERY = 1 -#: Server Status (unspecified and unimplemented anywhere) -STATUS = 2 -#: Notify -NOTIFY = 4 -#: Dynamic Update -UPDATE = 5 - -_by_text = { - 'QUERY': QUERY, - 'IQUERY': IQUERY, - 'STATUS': STATUS, - 'NOTIFY': NOTIFY, - 'UPDATE': UPDATE -} - -# We construct the inverse mapping programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mapping not to be true inverse. - -_by_value = {y: x for x, y in _by_text.items()} - - -class UnknownOpcode(dns.exception.DNSException): - """An DNS opcode is unknown.""" - - -def from_text(text): - """Convert text into an opcode. - - *text*, a ``text``, the textual opcode - - Raises ``dns.opcode.UnknownOpcode`` if the opcode is unknown. - - Returns an ``int``. - """ - - if text.isdigit(): - value = int(text) - if value >= 0 and value <= 15: - return value - value = _by_text.get(text.upper()) - if value is None: - raise UnknownOpcode - return value - - -def from_flags(flags): - """Extract an opcode from DNS message flags. - - *flags*, an ``int``, the DNS flags. - - Returns an ``int``. - """ - - return (flags & 0x7800) >> 11 - - -def to_flags(value): - """Convert an opcode to a value suitable for ORing into DNS message - flags. - - *value*, an ``int``, the DNS opcode value. - - Returns an ``int``. - """ - - return (value << 11) & 0x7800 - - -def to_text(value): - """Convert an opcode to text. - - *value*, an ``int`` the opcode value, - - Raises ``dns.opcode.UnknownOpcode`` if the opcode is unknown. - - Returns a ``text``. - """ - - text = _by_value.get(value) - if text is None: - text = str(value) - return text - - -def is_update(flags): - """Is the opcode in flags UPDATE? - - *flags*, an ``int``, the DNS message flags. - - Returns a ``bool``. - """ - - return from_flags(flags) == UPDATE diff --git a/src/dns/py.typed b/src/dns/py.typed deleted file mode 100644 index e69de29b..00000000 diff --git a/src/dns/query.py b/src/dns/query.py deleted file mode 100644 index c0c517cc..00000000 --- a/src/dns/query.py +++ /dev/null @@ -1,683 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Talk to a DNS server.""" - -from __future__ import generators - -import errno -import select -import socket -import struct -import sys -import time - -import dns.exception -import dns.inet -import dns.name -import dns.message -import dns.rcode -import dns.rdataclass -import dns.rdatatype -from ._compat import long, string_types, PY3 - -if PY3: - select_error = OSError -else: - select_error = select.error - -# Function used to create a socket. Can be overridden if needed in special -# situations. -socket_factory = socket.socket - -class UnexpectedSource(dns.exception.DNSException): - """A DNS query response came from an unexpected address or port.""" - - -class BadResponse(dns.exception.FormError): - """A DNS query response does not respond to the question asked.""" - - -class TransferError(dns.exception.DNSException): - """A zone transfer response got a non-zero rcode.""" - - def __init__(self, rcode): - message = 'Zone transfer error: %s' % dns.rcode.to_text(rcode) - super(TransferError, self).__init__(message) - self.rcode = rcode - - -def _compute_expiration(timeout): - if timeout is None: - return None - else: - return time.time() + timeout - -# This module can use either poll() or select() as the "polling backend". -# -# A backend function takes an fd, bools for readability, writablity, and -# error detection, and a timeout. - -def _poll_for(fd, readable, writable, error, timeout): - """Poll polling backend.""" - - event_mask = 0 - if readable: - event_mask |= select.POLLIN - if writable: - event_mask |= select.POLLOUT - if error: - event_mask |= select.POLLERR - - pollable = select.poll() - pollable.register(fd, event_mask) - - if timeout: - event_list = pollable.poll(long(timeout * 1000)) - else: - event_list = pollable.poll() - - return bool(event_list) - - -def _select_for(fd, readable, writable, error, timeout): - """Select polling backend.""" - - rset, wset, xset = [], [], [] - - if readable: - rset = [fd] - if writable: - wset = [fd] - if error: - xset = [fd] - - if timeout is None: - (rcount, wcount, xcount) = select.select(rset, wset, xset) - else: - (rcount, wcount, xcount) = select.select(rset, wset, xset, timeout) - - return bool((rcount or wcount or xcount)) - - -def _wait_for(fd, readable, writable, error, expiration): - # Use the selected polling backend to wait for any of the specified - # events. An "expiration" absolute time is converted into a relative - # timeout. - - done = False - while not done: - if expiration is None: - timeout = None - else: - timeout = expiration - time.time() - if timeout <= 0.0: - raise dns.exception.Timeout - try: - if not _polling_backend(fd, readable, writable, error, timeout): - raise dns.exception.Timeout - except select_error as e: - if e.args[0] != errno.EINTR: - raise e - done = True - - -def _set_polling_backend(fn): - # Internal API. Do not use. - - global _polling_backend - - _polling_backend = fn - -if hasattr(select, 'poll'): - # Prefer poll() on platforms that support it because it has no - # limits on the maximum value of a file descriptor (plus it will - # be more efficient for high values). - _polling_backend = _poll_for -else: - _polling_backend = _select_for - - -def _wait_for_readable(s, expiration): - _wait_for(s, True, False, True, expiration) - - -def _wait_for_writable(s, expiration): - _wait_for(s, False, True, True, expiration) - - -def _addresses_equal(af, a1, a2): - # Convert the first value of the tuple, which is a textual format - # address into binary form, so that we are not confused by different - # textual representations of the same address - try: - n1 = dns.inet.inet_pton(af, a1[0]) - n2 = dns.inet.inet_pton(af, a2[0]) - except dns.exception.SyntaxError: - return False - return n1 == n2 and a1[1:] == a2[1:] - - -def _destination_and_source(af, where, port, source, source_port): - # Apply defaults and compute destination and source tuples - # suitable for use in connect(), sendto(), or bind(). - if af is None: - try: - af = dns.inet.af_for_address(where) - except Exception: - af = dns.inet.AF_INET - if af == dns.inet.AF_INET: - destination = (where, port) - if source is not None or source_port != 0: - if source is None: - source = '0.0.0.0' - source = (source, source_port) - elif af == dns.inet.AF_INET6: - destination = (where, port, 0, 0) - if source is not None or source_port != 0: - if source is None: - source = '::' - source = (source, source_port, 0, 0) - return (af, destination, source) - - -def send_udp(sock, what, destination, expiration=None): - """Send a DNS message to the specified UDP socket. - - *sock*, a ``socket``. - - *what*, a ``binary`` or ``dns.message.Message``, the message to send. - - *destination*, a destination tuple appropriate for the address family - of the socket, specifying where to send the query. - - *expiration*, a ``float`` or ``None``, the absolute time at which - a timeout exception should be raised. If ``None``, no timeout will - occur. - - Returns an ``(int, float)`` tuple of bytes sent and the sent time. - """ - - if isinstance(what, dns.message.Message): - what = what.to_wire() - _wait_for_writable(sock, expiration) - sent_time = time.time() - n = sock.sendto(what, destination) - return (n, sent_time) - - -def receive_udp(sock, destination, expiration=None, - ignore_unexpected=False, one_rr_per_rrset=False, - keyring=None, request_mac=b'', ignore_trailing=False): - """Read a DNS message from a UDP socket. - - *sock*, a ``socket``. - - *destination*, a destination tuple appropriate for the address family - of the socket, specifying where the associated query was sent. - - *expiration*, a ``float`` or ``None``, the absolute time at which - a timeout exception should be raised. If ``None``, no timeout will - occur. - - *ignore_unexpected*, a ``bool``. If ``True``, ignore responses from - unexpected sources. - - *one_rr_per_rrset*, a ``bool``. If ``True``, put each RR into its own - RRset. - - *keyring*, a ``dict``, the keyring to use for TSIG. - - *request_mac*, a ``binary``, the MAC of the request (for TSIG). - - *ignore_trailing*, a ``bool``. If ``True``, ignore trailing - junk at end of the received message. - - Raises if the message is malformed, if network errors occur, of if - there is a timeout. - - Returns a ``dns.message.Message`` object. - """ - - wire = b'' - while 1: - _wait_for_readable(sock, expiration) - (wire, from_address) = sock.recvfrom(65535) - if _addresses_equal(sock.family, from_address, destination) or \ - (dns.inet.is_multicast(destination[0]) and - from_address[1:] == destination[1:]): - break - if not ignore_unexpected: - raise UnexpectedSource('got a response from ' - '%s instead of %s' % (from_address, - destination)) - received_time = time.time() - r = dns.message.from_wire(wire, keyring=keyring, request_mac=request_mac, - one_rr_per_rrset=one_rr_per_rrset, - ignore_trailing=ignore_trailing) - return (r, received_time) - -def udp(q, where, timeout=None, port=53, af=None, source=None, source_port=0, - ignore_unexpected=False, one_rr_per_rrset=False, ignore_trailing=False): - """Return the response obtained after sending a query via UDP. - - *q*, a ``dns.message.Message``, the query to send - - *where*, a ``text`` containing an IPv4 or IPv6 address, where - to send the message. - - *timeout*, a ``float`` or ``None``, the number of seconds to wait before the - query times out. If ``None``, the default, wait forever. - - *port*, an ``int``, the port send the message to. The default is 53. - - *af*, an ``int``, the address family to use. The default is ``None``, - which causes the address family to use to be inferred from the form of - *where*. If the inference attempt fails, AF_INET is used. This - parameter is historical; you need never set it. - - *source*, a ``text`` containing an IPv4 or IPv6 address, specifying - the source address. The default is the wildcard address. - - *source_port*, an ``int``, the port from which to send the message. - The default is 0. - - *ignore_unexpected*, a ``bool``. If ``True``, ignore responses from - unexpected sources. - - *one_rr_per_rrset*, a ``bool``. If ``True``, put each RR into its own - RRset. - - *ignore_trailing*, a ``bool``. If ``True``, ignore trailing - junk at end of the received message. - - Returns a ``dns.message.Message``. - """ - - wire = q.to_wire() - (af, destination, source) = _destination_and_source(af, where, port, - source, source_port) - s = socket_factory(af, socket.SOCK_DGRAM, 0) - received_time = None - sent_time = None - try: - expiration = _compute_expiration(timeout) - s.setblocking(0) - if source is not None: - s.bind(source) - (_, sent_time) = send_udp(s, wire, destination, expiration) - (r, received_time) = receive_udp(s, destination, expiration, - ignore_unexpected, one_rr_per_rrset, - q.keyring, q.mac, ignore_trailing) - finally: - if sent_time is None or received_time is None: - response_time = 0 - else: - response_time = received_time - sent_time - s.close() - r.time = response_time - if not q.is_response(r): - raise BadResponse - return r - - -def _net_read(sock, count, expiration): - """Read the specified number of bytes from sock. Keep trying until we - either get the desired amount, or we hit EOF. - A Timeout exception will be raised if the operation is not completed - by the expiration time. - """ - s = b'' - while count > 0: - _wait_for_readable(sock, expiration) - n = sock.recv(count) - if n == b'': - raise EOFError - count = count - len(n) - s = s + n - return s - - -def _net_write(sock, data, expiration): - """Write the specified data to the socket. - A Timeout exception will be raised if the operation is not completed - by the expiration time. - """ - current = 0 - l = len(data) - while current < l: - _wait_for_writable(sock, expiration) - current += sock.send(data[current:]) - - -def send_tcp(sock, what, expiration=None): - """Send a DNS message to the specified TCP socket. - - *sock*, a ``socket``. - - *what*, a ``binary`` or ``dns.message.Message``, the message to send. - - *expiration*, a ``float`` or ``None``, the absolute time at which - a timeout exception should be raised. If ``None``, no timeout will - occur. - - Returns an ``(int, float)`` tuple of bytes sent and the sent time. - """ - - if isinstance(what, dns.message.Message): - what = what.to_wire() - l = len(what) - # copying the wire into tcpmsg is inefficient, but lets us - # avoid writev() or doing a short write that would get pushed - # onto the net - tcpmsg = struct.pack("!H", l) + what - _wait_for_writable(sock, expiration) - sent_time = time.time() - _net_write(sock, tcpmsg, expiration) - return (len(tcpmsg), sent_time) - -def receive_tcp(sock, expiration=None, one_rr_per_rrset=False, - keyring=None, request_mac=b'', ignore_trailing=False): - """Read a DNS message from a TCP socket. - - *sock*, a ``socket``. - - *expiration*, a ``float`` or ``None``, the absolute time at which - a timeout exception should be raised. If ``None``, no timeout will - occur. - - *one_rr_per_rrset*, a ``bool``. If ``True``, put each RR into its own - RRset. - - *keyring*, a ``dict``, the keyring to use for TSIG. - - *request_mac*, a ``binary``, the MAC of the request (for TSIG). - - *ignore_trailing*, a ``bool``. If ``True``, ignore trailing - junk at end of the received message. - - Raises if the message is malformed, if network errors occur, of if - there is a timeout. - - Returns a ``dns.message.Message`` object. - """ - - ldata = _net_read(sock, 2, expiration) - (l,) = struct.unpack("!H", ldata) - wire = _net_read(sock, l, expiration) - received_time = time.time() - r = dns.message.from_wire(wire, keyring=keyring, request_mac=request_mac, - one_rr_per_rrset=one_rr_per_rrset, - ignore_trailing=ignore_trailing) - return (r, received_time) - -def _connect(s, address): - try: - s.connect(address) - except socket.error: - (ty, v) = sys.exc_info()[:2] - - if hasattr(v, 'errno'): - v_err = v.errno - else: - v_err = v[0] - if v_err not in [errno.EINPROGRESS, errno.EWOULDBLOCK, errno.EALREADY]: - raise v - - -def tcp(q, where, timeout=None, port=53, af=None, source=None, source_port=0, - one_rr_per_rrset=False, ignore_trailing=False): - """Return the response obtained after sending a query via TCP. - - *q*, a ``dns.message.Message``, the query to send - - *where*, a ``text`` containing an IPv4 or IPv6 address, where - to send the message. - - *timeout*, a ``float`` or ``None``, the number of seconds to wait before the - query times out. If ``None``, the default, wait forever. - - *port*, an ``int``, the port send the message to. The default is 53. - - *af*, an ``int``, the address family to use. The default is ``None``, - which causes the address family to use to be inferred from the form of - *where*. If the inference attempt fails, AF_INET is used. This - parameter is historical; you need never set it. - - *source*, a ``text`` containing an IPv4 or IPv6 address, specifying - the source address. The default is the wildcard address. - - *source_port*, an ``int``, the port from which to send the message. - The default is 0. - - *one_rr_per_rrset*, a ``bool``. If ``True``, put each RR into its own - RRset. - - *ignore_trailing*, a ``bool``. If ``True``, ignore trailing - junk at end of the received message. - - Returns a ``dns.message.Message``. - """ - - wire = q.to_wire() - (af, destination, source) = _destination_and_source(af, where, port, - source, source_port) - s = socket_factory(af, socket.SOCK_STREAM, 0) - begin_time = None - received_time = None - try: - expiration = _compute_expiration(timeout) - s.setblocking(0) - begin_time = time.time() - if source is not None: - s.bind(source) - _connect(s, destination) - send_tcp(s, wire, expiration) - (r, received_time) = receive_tcp(s, expiration, one_rr_per_rrset, - q.keyring, q.mac, ignore_trailing) - finally: - if begin_time is None or received_time is None: - response_time = 0 - else: - response_time = received_time - begin_time - s.close() - r.time = response_time - if not q.is_response(r): - raise BadResponse - return r - - -def xfr(where, zone, rdtype=dns.rdatatype.AXFR, rdclass=dns.rdataclass.IN, - timeout=None, port=53, keyring=None, keyname=None, relativize=True, - af=None, lifetime=None, source=None, source_port=0, serial=0, - use_udp=False, keyalgorithm=dns.tsig.default_algorithm): - """Return a generator for the responses to a zone transfer. - - *where*. If the inference attempt fails, AF_INET is used. This - parameter is historical; you need never set it. - - *zone*, a ``dns.name.Name`` or ``text``, the name of the zone to transfer. - - *rdtype*, an ``int`` or ``text``, the type of zone transfer. The - default is ``dns.rdatatype.AXFR``. ``dns.rdatatype.IXFR`` can be - used to do an incremental transfer instead. - - *rdclass*, an ``int`` or ``text``, the class of the zone transfer. - The default is ``dns.rdataclass.IN``. - - *timeout*, a ``float``, the number of seconds to wait for each - response message. If None, the default, wait forever. - - *port*, an ``int``, the port send the message to. The default is 53. - - *keyring*, a ``dict``, the keyring to use for TSIG. - - *keyname*, a ``dns.name.Name`` or ``text``, the name of the TSIG - key to use. - - *relativize*, a ``bool``. If ``True``, all names in the zone will be - relativized to the zone origin. It is essential that the - relativize setting matches the one specified to - ``dns.zone.from_xfr()`` if using this generator to make a zone. - - *af*, an ``int``, the address family to use. The default is ``None``, - which causes the address family to use to be inferred from the form of - *where*. If the inference attempt fails, AF_INET is used. This - parameter is historical; you need never set it. - - *lifetime*, a ``float``, the total number of seconds to spend - doing the transfer. If ``None``, the default, then there is no - limit on the time the transfer may take. - - *source*, a ``text`` containing an IPv4 or IPv6 address, specifying - the source address. The default is the wildcard address. - - *source_port*, an ``int``, the port from which to send the message. - The default is 0. - - *serial*, an ``int``, the SOA serial number to use as the base for - an IXFR diff sequence (only meaningful if *rdtype* is - ``dns.rdatatype.IXFR``). - - *use_udp*, a ``bool``. If ``True``, use UDP (only meaningful for IXFR). - - *keyalgorithm*, a ``dns.name.Name`` or ``text``, the TSIG algorithm to use. - - Raises on errors, and so does the generator. - - Returns a generator of ``dns.message.Message`` objects. - """ - - if isinstance(zone, string_types): - zone = dns.name.from_text(zone) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - q = dns.message.make_query(zone, rdtype, rdclass) - if rdtype == dns.rdatatype.IXFR: - rrset = dns.rrset.from_text(zone, 0, 'IN', 'SOA', - '. . %u 0 0 0 0' % serial) - q.authority.append(rrset) - if keyring is not None: - q.use_tsig(keyring, keyname, algorithm=keyalgorithm) - wire = q.to_wire() - (af, destination, source) = _destination_and_source(af, where, port, - source, source_port) - if use_udp: - if rdtype != dns.rdatatype.IXFR: - raise ValueError('cannot do a UDP AXFR') - s = socket_factory(af, socket.SOCK_DGRAM, 0) - else: - s = socket_factory(af, socket.SOCK_STREAM, 0) - s.setblocking(0) - if source is not None: - s.bind(source) - expiration = _compute_expiration(lifetime) - _connect(s, destination) - l = len(wire) - if use_udp: - _wait_for_writable(s, expiration) - s.send(wire) - else: - tcpmsg = struct.pack("!H", l) + wire - _net_write(s, tcpmsg, expiration) - done = False - delete_mode = True - expecting_SOA = False - soa_rrset = None - if relativize: - origin = zone - oname = dns.name.empty - else: - origin = None - oname = zone - tsig_ctx = None - first = True - while not done: - mexpiration = _compute_expiration(timeout) - if mexpiration is None or mexpiration > expiration: - mexpiration = expiration - if use_udp: - _wait_for_readable(s, expiration) - (wire, from_address) = s.recvfrom(65535) - else: - ldata = _net_read(s, 2, mexpiration) - (l,) = struct.unpack("!H", ldata) - wire = _net_read(s, l, mexpiration) - is_ixfr = (rdtype == dns.rdatatype.IXFR) - r = dns.message.from_wire(wire, keyring=q.keyring, request_mac=q.mac, - xfr=True, origin=origin, tsig_ctx=tsig_ctx, - multi=True, first=first, - one_rr_per_rrset=is_ixfr) - rcode = r.rcode() - if rcode != dns.rcode.NOERROR: - raise TransferError(rcode) - tsig_ctx = r.tsig_ctx - first = False - answer_index = 0 - if soa_rrset is None: - if not r.answer or r.answer[0].name != oname: - raise dns.exception.FormError( - "No answer or RRset not for qname") - rrset = r.answer[0] - if rrset.rdtype != dns.rdatatype.SOA: - raise dns.exception.FormError("first RRset is not an SOA") - answer_index = 1 - soa_rrset = rrset.copy() - if rdtype == dns.rdatatype.IXFR: - if soa_rrset[0].serial <= serial: - # - # We're already up-to-date. - # - done = True - else: - expecting_SOA = True - # - # Process SOAs in the answer section (other than the initial - # SOA in the first message). - # - for rrset in r.answer[answer_index:]: - if done: - raise dns.exception.FormError("answers after final SOA") - if rrset.rdtype == dns.rdatatype.SOA and rrset.name == oname: - if expecting_SOA: - if rrset[0].serial != serial: - raise dns.exception.FormError( - "IXFR base serial mismatch") - expecting_SOA = False - elif rdtype == dns.rdatatype.IXFR: - delete_mode = not delete_mode - # - # If this SOA RRset is equal to the first we saw then we're - # finished. If this is an IXFR we also check that we're seeing - # the record in the expected part of the response. - # - if rrset == soa_rrset and \ - (rdtype == dns.rdatatype.AXFR or - (rdtype == dns.rdatatype.IXFR and delete_mode)): - done = True - elif expecting_SOA: - # - # We made an IXFR request and are expecting another - # SOA RR, but saw something else, so this must be an - # AXFR response. - # - rdtype = dns.rdatatype.AXFR - expecting_SOA = False - if done and q.keyring and not r.had_tsig: - raise dns.exception.FormError("missing TSIG") - yield r - s.close() diff --git a/src/dns/query.pyi b/src/dns/query.pyi deleted file mode 100644 index fe5ef826..00000000 --- a/src/dns/query.pyi +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Optional, Union, Dict, Generator, Any -from . import message, tsig, rdatatype, rdataclass, name, message -def tcp(q : message.Message, where : str, timeout : float = None, port=53, af : Optional[int] = None, source : Optional[str] = None, source_port : int = 0, - one_rr_per_rrset=False) -> message.Message: - pass - -def xfr(where : None, zone : Union[name.Name,str], rdtype=rdatatype.AXFR, rdclass=rdataclass.IN, - timeout : Optional[float] =None, port=53, keyring : Optional[Dict[name.Name, bytes]] =None, keyname : Union[str,name.Name]=None, relativize=True, - af : Optional[int] =None, lifetime : Optional[float]=None, source : Optional[str] =None, source_port=0, serial=0, - use_udp=False, keyalgorithm=tsig.default_algorithm) -> Generator[Any,Any,message.Message]: - pass - -def udp(q : message.Message, where : str, timeout : Optional[float] = None, port=53, af : Optional[int] = None, source : Optional[str] = None, source_port=0, - ignore_unexpected=False, one_rr_per_rrset=False) -> message.Message: - ... diff --git a/src/dns/rcode.py b/src/dns/rcode.py deleted file mode 100644 index 5191e1b1..00000000 --- a/src/dns/rcode.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Result Codes.""" - -import dns.exception -from ._compat import long - -#: No error -NOERROR = 0 -#: Form error -FORMERR = 1 -#: Server failure -SERVFAIL = 2 -#: Name does not exist ("Name Error" in RFC 1025 terminology). -NXDOMAIN = 3 -#: Not implemented -NOTIMP = 4 -#: Refused -REFUSED = 5 -#: Name exists. -YXDOMAIN = 6 -#: RRset exists. -YXRRSET = 7 -#: RRset does not exist. -NXRRSET = 8 -#: Not authoritative. -NOTAUTH = 9 -#: Name not in zone. -NOTZONE = 10 -#: Bad EDNS version. -BADVERS = 16 - -_by_text = { - 'NOERROR': NOERROR, - 'FORMERR': FORMERR, - 'SERVFAIL': SERVFAIL, - 'NXDOMAIN': NXDOMAIN, - 'NOTIMP': NOTIMP, - 'REFUSED': REFUSED, - 'YXDOMAIN': YXDOMAIN, - 'YXRRSET': YXRRSET, - 'NXRRSET': NXRRSET, - 'NOTAUTH': NOTAUTH, - 'NOTZONE': NOTZONE, - 'BADVERS': BADVERS -} - -# We construct the inverse mapping programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mapping not to be a true inverse. - -_by_value = {y: x for x, y in _by_text.items()} - - -class UnknownRcode(dns.exception.DNSException): - """A DNS rcode is unknown.""" - - -def from_text(text): - """Convert text into an rcode. - - *text*, a ``text``, the textual rcode or an integer in textual form. - - Raises ``dns.rcode.UnknownRcode`` if the rcode mnemonic is unknown. - - Returns an ``int``. - """ - - if text.isdigit(): - v = int(text) - if v >= 0 and v <= 4095: - return v - v = _by_text.get(text.upper()) - if v is None: - raise UnknownRcode - return v - - -def from_flags(flags, ednsflags): - """Return the rcode value encoded by flags and ednsflags. - - *flags*, an ``int``, the DNS flags field. - - *ednsflags*, an ``int``, the EDNS flags field. - - Raises ``ValueError`` if rcode is < 0 or > 4095 - - Returns an ``int``. - """ - - value = (flags & 0x000f) | ((ednsflags >> 20) & 0xff0) - if value < 0 or value > 4095: - raise ValueError('rcode must be >= 0 and <= 4095') - return value - - -def to_flags(value): - """Return a (flags, ednsflags) tuple which encodes the rcode. - - *value*, an ``int``, the rcode. - - Raises ``ValueError`` if rcode is < 0 or > 4095. - - Returns an ``(int, int)`` tuple. - """ - - if value < 0 or value > 4095: - raise ValueError('rcode must be >= 0 and <= 4095') - v = value & 0xf - ev = long(value & 0xff0) << 20 - return (v, ev) - - -def to_text(value): - """Convert rcode into text. - - *value*, and ``int``, the rcode. - - Raises ``ValueError`` if rcode is < 0 or > 4095. - - Returns a ``text``. - """ - - if value < 0 or value > 4095: - raise ValueError('rcode must be >= 0 and <= 4095') - text = _by_value.get(value) - if text is None: - text = str(value) - return text diff --git a/src/dns/rdata.py b/src/dns/rdata.py deleted file mode 100644 index ea1971dc..00000000 --- a/src/dns/rdata.py +++ /dev/null @@ -1,456 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS rdata.""" - -from io import BytesIO -import base64 -import binascii - -import dns.exception -import dns.name -import dns.rdataclass -import dns.rdatatype -import dns.tokenizer -import dns.wiredata -from ._compat import xrange, string_types, text_type - -try: - import threading as _threading -except ImportError: - import dummy_threading as _threading - -_hex_chunksize = 32 - - -def _hexify(data, chunksize=_hex_chunksize): - """Convert a binary string into its hex encoding, broken up into chunks - of chunksize characters separated by a space. - """ - - line = binascii.hexlify(data) - return b' '.join([line[i:i + chunksize] - for i - in range(0, len(line), chunksize)]).decode() - -_base64_chunksize = 32 - - -def _base64ify(data, chunksize=_base64_chunksize): - """Convert a binary string into its base64 encoding, broken up into chunks - of chunksize characters separated by a space. - """ - - line = base64.b64encode(data) - return b' '.join([line[i:i + chunksize] - for i - in range(0, len(line), chunksize)]).decode() - -__escaped = bytearray(b'"\\') - -def _escapify(qstring): - """Escape the characters in a quoted string which need it.""" - - if isinstance(qstring, text_type): - qstring = qstring.encode() - if not isinstance(qstring, bytearray): - qstring = bytearray(qstring) - - text = '' - for c in qstring: - if c in __escaped: - text += '\\' + chr(c) - elif c >= 0x20 and c < 0x7F: - text += chr(c) - else: - text += '\\%03d' % c - return text - - -def _truncate_bitmap(what): - """Determine the index of greatest byte that isn't all zeros, and - return the bitmap that contains all the bytes less than that index. - """ - - for i in xrange(len(what) - 1, -1, -1): - if what[i] != 0: - return what[0: i + 1] - return what[0:1] - - -class Rdata(object): - """Base class for all DNS rdata types.""" - - __slots__ = ['rdclass', 'rdtype'] - - def __init__(self, rdclass, rdtype): - """Initialize an rdata. - - *rdclass*, an ``int`` is the rdataclass of the Rdata. - *rdtype*, an ``int`` is the rdatatype of the Rdata. - """ - - self.rdclass = rdclass - self.rdtype = rdtype - - def covers(self): - """Return the type a Rdata covers. - - DNS SIG/RRSIG rdatas apply to a specific type; this type is - returned by the covers() function. If the rdata type is not - SIG or RRSIG, dns.rdatatype.NONE is returned. This is useful when - creating rdatasets, allowing the rdataset to contain only RRSIGs - of a particular type, e.g. RRSIG(NS). - - Returns an ``int``. - """ - - return dns.rdatatype.NONE - - def extended_rdatatype(self): - """Return a 32-bit type value, the least significant 16 bits of - which are the ordinary DNS type, and the upper 16 bits of which are - the "covered" type, if any. - - Returns an ``int``. - """ - - return self.covers() << 16 | self.rdtype - - def to_text(self, origin=None, relativize=True, **kw): - """Convert an rdata to text format. - - Returns a ``text``. - """ - - raise NotImplementedError - - def to_wire(self, file, compress=None, origin=None): - """Convert an rdata to wire format. - - Returns a ``binary``. - """ - - raise NotImplementedError - - def to_digestable(self, origin=None): - """Convert rdata to a format suitable for digesting in hashes. This - is also the DNSSEC canonical form. - - Returns a ``binary``. - """ - - f = BytesIO() - self.to_wire(f, None, origin) - return f.getvalue() - - def validate(self): - """Check that the current contents of the rdata's fields are - valid. - - If you change an rdata by assigning to its fields, - it is a good idea to call validate() when you are done making - changes. - - Raises various exceptions if there are problems. - - Returns ``None``. - """ - - dns.rdata.from_text(self.rdclass, self.rdtype, self.to_text()) - - def __repr__(self): - covers = self.covers() - if covers == dns.rdatatype.NONE: - ctext = '' - else: - ctext = '(' + dns.rdatatype.to_text(covers) + ')' - return '' - - def __str__(self): - return self.to_text() - - def _cmp(self, other): - """Compare an rdata with another rdata of the same rdtype and - rdclass. - - Return < 0 if self < other in the DNSSEC ordering, 0 if self - == other, and > 0 if self > other. - - """ - our = self.to_digestable(dns.name.root) - their = other.to_digestable(dns.name.root) - if our == their: - return 0 - elif our > their: - return 1 - else: - return -1 - - def __eq__(self, other): - if not isinstance(other, Rdata): - return False - if self.rdclass != other.rdclass or self.rdtype != other.rdtype: - return False - return self._cmp(other) == 0 - - def __ne__(self, other): - if not isinstance(other, Rdata): - return True - if self.rdclass != other.rdclass or self.rdtype != other.rdtype: - return True - return self._cmp(other) != 0 - - def __lt__(self, other): - if not isinstance(other, Rdata) or \ - self.rdclass != other.rdclass or self.rdtype != other.rdtype: - - return NotImplemented - return self._cmp(other) < 0 - - def __le__(self, other): - if not isinstance(other, Rdata) or \ - self.rdclass != other.rdclass or self.rdtype != other.rdtype: - return NotImplemented - return self._cmp(other) <= 0 - - def __ge__(self, other): - if not isinstance(other, Rdata) or \ - self.rdclass != other.rdclass or self.rdtype != other.rdtype: - return NotImplemented - return self._cmp(other) >= 0 - - def __gt__(self, other): - if not isinstance(other, Rdata) or \ - self.rdclass != other.rdclass or self.rdtype != other.rdtype: - return NotImplemented - return self._cmp(other) > 0 - - def __hash__(self): - return hash(self.to_digestable(dns.name.root)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - raise NotImplementedError - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - raise NotImplementedError - - def choose_relativity(self, origin=None, relativize=True): - """Convert any domain names in the rdata to the specified - relativization. - """ - -class GenericRdata(Rdata): - - """Generic Rdata Class - - This class is used for rdata types for which we have no better - implementation. It implements the DNS "unknown RRs" scheme. - """ - - __slots__ = ['data'] - - def __init__(self, rdclass, rdtype, data): - super(GenericRdata, self).__init__(rdclass, rdtype) - self.data = data - - def to_text(self, origin=None, relativize=True, **kw): - return r'\# %d ' % len(self.data) + _hexify(self.data) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - token = tok.get() - if not token.is_identifier() or token.value != r'\#': - raise dns.exception.SyntaxError( - r'generic rdata does not start with \#') - length = tok.get_int() - chunks = [] - while 1: - token = tok.get() - if token.is_eol_or_eof(): - break - chunks.append(token.value.encode()) - hex = b''.join(chunks) - data = binascii.unhexlify(hex) - if len(data) != length: - raise dns.exception.SyntaxError( - 'generic rdata hex data has wrong length') - return cls(rdclass, rdtype, data) - - def to_wire(self, file, compress=None, origin=None): - file.write(self.data) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - return cls(rdclass, rdtype, wire[current: current + rdlen]) - -_rdata_modules = {} -_module_prefix = 'dns.rdtypes' -_import_lock = _threading.Lock() - -def get_rdata_class(rdclass, rdtype): - - def import_module(name): - with _import_lock: - mod = __import__(name) - components = name.split('.') - for comp in components[1:]: - mod = getattr(mod, comp) - return mod - - mod = _rdata_modules.get((rdclass, rdtype)) - rdclass_text = dns.rdataclass.to_text(rdclass) - rdtype_text = dns.rdatatype.to_text(rdtype) - rdtype_text = rdtype_text.replace('-', '_') - if not mod: - mod = _rdata_modules.get((dns.rdatatype.ANY, rdtype)) - if not mod: - try: - mod = import_module('.'.join([_module_prefix, - rdclass_text, rdtype_text])) - _rdata_modules[(rdclass, rdtype)] = mod - except ImportError: - try: - mod = import_module('.'.join([_module_prefix, - 'ANY', rdtype_text])) - _rdata_modules[(dns.rdataclass.ANY, rdtype)] = mod - except ImportError: - mod = None - if mod: - cls = getattr(mod, rdtype_text) - else: - cls = GenericRdata - return cls - - -def from_text(rdclass, rdtype, tok, origin=None, relativize=True): - """Build an rdata object from text format. - - This function attempts to dynamically load a class which - implements the specified rdata class and type. If there is no - class-and-type-specific implementation, the GenericRdata class - is used. - - Once a class is chosen, its from_text() class method is called - with the parameters to this function. - - If *tok* is a ``text``, then a tokenizer is created and the string - is used as its input. - - *rdclass*, an ``int``, the rdataclass. - - *rdtype*, an ``int``, the rdatatype. - - *tok*, a ``dns.tokenizer.Tokenizer`` or a ``text``. - - *origin*, a ``dns.name.Name`` (or ``None``), the - origin to use for relative names. - - *relativize*, a ``bool``. If true, name will be relativized to - the specified origin. - - Returns an instance of the chosen Rdata subclass. - """ - - if isinstance(tok, string_types): - tok = dns.tokenizer.Tokenizer(tok) - cls = get_rdata_class(rdclass, rdtype) - if cls != GenericRdata: - # peek at first token - token = tok.get() - tok.unget(token) - if token.is_identifier() and \ - token.value == r'\#': - # - # Known type using the generic syntax. Extract the - # wire form from the generic syntax, and then run - # from_wire on it. - # - rdata = GenericRdata.from_text(rdclass, rdtype, tok, origin, - relativize) - return from_wire(rdclass, rdtype, rdata.data, 0, len(rdata.data), - origin) - return cls.from_text(rdclass, rdtype, tok, origin, relativize) - - -def from_wire(rdclass, rdtype, wire, current, rdlen, origin=None): - """Build an rdata object from wire format - - This function attempts to dynamically load a class which - implements the specified rdata class and type. If there is no - class-and-type-specific implementation, the GenericRdata class - is used. - - Once a class is chosen, its from_wire() class method is called - with the parameters to this function. - - *rdclass*, an ``int``, the rdataclass. - - *rdtype*, an ``int``, the rdatatype. - - *wire*, a ``binary``, the wire-format message. - - *current*, an ``int``, the offset in wire of the beginning of - the rdata. - - *rdlen*, an ``int``, the length of the wire-format rdata - - *origin*, a ``dns.name.Name`` (or ``None``). If not ``None``, - then names will be relativized to this origin. - - Returns an instance of the chosen Rdata subclass. - """ - - wire = dns.wiredata.maybe_wrap(wire) - cls = get_rdata_class(rdclass, rdtype) - return cls.from_wire(rdclass, rdtype, wire, current, rdlen, origin) - - -class RdatatypeExists(dns.exception.DNSException): - """DNS rdatatype already exists.""" - supp_kwargs = {'rdclass', 'rdtype'} - fmt = "The rdata type with class {rdclass} and rdtype {rdtype} " + \ - "already exists." - - -def register_type(implementation, rdtype, rdtype_text, is_singleton=False, - rdclass=dns.rdataclass.IN): - """Dynamically register a module to handle an rdatatype. - - *implementation*, a module implementing the type in the usual dnspython - way. - - *rdtype*, an ``int``, the rdatatype to register. - - *rdtype_text*, a ``text``, the textual form of the rdatatype. - - *is_singleton*, a ``bool``, indicating if the type is a singleton (i.e. - RRsets of the type can have only one member.) - - *rdclass*, the rdataclass of the type, or ``dns.rdataclass.ANY`` if - it applies to all classes. - """ - - existing_cls = get_rdata_class(rdclass, rdtype) - if existing_cls != GenericRdata: - raise RdatatypeExists(rdclass=rdclass, rdtype=rdtype) - _rdata_modules[(rdclass, rdtype)] = implementation - dns.rdatatype.register_type(rdtype, rdtype_text, is_singleton) diff --git a/src/dns/rdata.pyi b/src/dns/rdata.pyi deleted file mode 100644 index 8663955c..00000000 --- a/src/dns/rdata.pyi +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Dict, Tuple, Any, Optional -from .name import Name -class Rdata: - def __init__(self): - self.address : str - def to_wire(self, file, compress : Optional[Dict[Name,int]], origin : Optional[Name]) -> bytes: - ... - @classmethod - def from_text(cls, rdclass : int, rdtype : int, tok, origin=None, relativize=True): - ... -_rdata_modules : Dict[Tuple[Any,Rdata],Any] - -def from_text(rdclass : int, rdtype : int, tok : Optional[str], origin : Optional[Name] = None, relativize : bool = True): - ... - -def from_wire(rdclass : int, rdtype : int, wire : bytes, current : int, rdlen : int, origin : Optional[Name] = None): - ... diff --git a/src/dns/rdataclass.py b/src/dns/rdataclass.py deleted file mode 100644 index b88aa85b..00000000 --- a/src/dns/rdataclass.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Rdata Classes.""" - -import re - -import dns.exception - -RESERVED0 = 0 -IN = 1 -CH = 3 -HS = 4 -NONE = 254 -ANY = 255 - -_by_text = { - 'RESERVED0': RESERVED0, - 'IN': IN, - 'CH': CH, - 'HS': HS, - 'NONE': NONE, - 'ANY': ANY -} - -# We construct the inverse mapping programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mapping not to be true inverse. - -_by_value = {y: x for x, y in _by_text.items()} - -# Now that we've built the inverse map, we can add class aliases to -# the _by_text mapping. - -_by_text.update({ - 'INTERNET': IN, - 'CHAOS': CH, - 'HESIOD': HS -}) - -_metaclasses = { - NONE: True, - ANY: True -} - -_unknown_class_pattern = re.compile('CLASS([0-9]+)$', re.I) - - -class UnknownRdataclass(dns.exception.DNSException): - """A DNS class is unknown.""" - - -def from_text(text): - """Convert text into a DNS rdata class value. - - The input text can be a defined DNS RR class mnemonic or - instance of the DNS generic class syntax. - - For example, "IN" and "CLASS1" will both result in a value of 1. - - Raises ``dns.rdatatype.UnknownRdataclass`` if the class is unknown. - - Raises ``ValueError`` if the rdata class value is not >= 0 and <= 65535. - - Returns an ``int``. - """ - - value = _by_text.get(text.upper()) - if value is None: - match = _unknown_class_pattern.match(text) - if match is None: - raise UnknownRdataclass - value = int(match.group(1)) - if value < 0 or value > 65535: - raise ValueError("class must be between >= 0 and <= 65535") - return value - - -def to_text(value): - """Convert a DNS rdata type value to text. - - If the value has a known mnemonic, it will be used, otherwise the - DNS generic class syntax will be used. - - Raises ``ValueError`` if the rdata class value is not >= 0 and <= 65535. - - Returns a ``str``. - """ - - if value < 0 or value > 65535: - raise ValueError("class must be between >= 0 and <= 65535") - text = _by_value.get(value) - if text is None: - text = 'CLASS' + repr(value) - return text - - -def is_metaclass(rdclass): - """True if the specified class is a metaclass. - - The currently defined metaclasses are ANY and NONE. - - *rdclass* is an ``int``. - """ - - if rdclass in _metaclasses: - return True - return False diff --git a/src/dns/rdataset.py b/src/dns/rdataset.py deleted file mode 100644 index f1afe241..00000000 --- a/src/dns/rdataset.py +++ /dev/null @@ -1,347 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS rdatasets (an rdataset is a set of rdatas of a given type and class)""" - -import random -from io import StringIO -import struct - -import dns.exception -import dns.rdatatype -import dns.rdataclass -import dns.rdata -import dns.set -from ._compat import string_types - -# define SimpleSet here for backwards compatibility -SimpleSet = dns.set.Set - - -class DifferingCovers(dns.exception.DNSException): - """An attempt was made to add a DNS SIG/RRSIG whose covered type - is not the same as that of the other rdatas in the rdataset.""" - - -class IncompatibleTypes(dns.exception.DNSException): - """An attempt was made to add DNS RR data of an incompatible type.""" - - -class Rdataset(dns.set.Set): - - """A DNS rdataset.""" - - __slots__ = ['rdclass', 'rdtype', 'covers', 'ttl'] - - def __init__(self, rdclass, rdtype, covers=dns.rdatatype.NONE, ttl=0): - """Create a new rdataset of the specified class and type. - - *rdclass*, an ``int``, the rdataclass. - - *rdtype*, an ``int``, the rdatatype. - - *covers*, an ``int``, the covered rdatatype. - - *ttl*, an ``int``, the TTL. - """ - - super(Rdataset, self).__init__() - self.rdclass = rdclass - self.rdtype = rdtype - self.covers = covers - self.ttl = ttl - - def _clone(self): - obj = super(Rdataset, self)._clone() - obj.rdclass = self.rdclass - obj.rdtype = self.rdtype - obj.covers = self.covers - obj.ttl = self.ttl - return obj - - def update_ttl(self, ttl): - """Perform TTL minimization. - - Set the TTL of the rdataset to be the lesser of the set's current - TTL or the specified TTL. If the set contains no rdatas, set the TTL - to the specified TTL. - - *ttl*, an ``int``. - """ - - if len(self) == 0: - self.ttl = ttl - elif ttl < self.ttl: - self.ttl = ttl - - def add(self, rd, ttl=None): - """Add the specified rdata to the rdataset. - - If the optional *ttl* parameter is supplied, then - ``self.update_ttl(ttl)`` will be called prior to adding the rdata. - - *rd*, a ``dns.rdata.Rdata``, the rdata - - *ttl*, an ``int``, the TTL. - - Raises ``dns.rdataset.IncompatibleTypes`` if the type and class - do not match the type and class of the rdataset. - - Raises ``dns.rdataset.DifferingCovers`` if the type is a signature - type and the covered type does not match that of the rdataset. - """ - - # - # If we're adding a signature, do some special handling to - # check that the signature covers the same type as the - # other rdatas in this rdataset. If this is the first rdata - # in the set, initialize the covers field. - # - if self.rdclass != rd.rdclass or self.rdtype != rd.rdtype: - raise IncompatibleTypes - if ttl is not None: - self.update_ttl(ttl) - if self.rdtype == dns.rdatatype.RRSIG or \ - self.rdtype == dns.rdatatype.SIG: - covers = rd.covers() - if len(self) == 0 and self.covers == dns.rdatatype.NONE: - self.covers = covers - elif self.covers != covers: - raise DifferingCovers - if dns.rdatatype.is_singleton(rd.rdtype) and len(self) > 0: - self.clear() - super(Rdataset, self).add(rd) - - def union_update(self, other): - self.update_ttl(other.ttl) - super(Rdataset, self).union_update(other) - - def intersection_update(self, other): - self.update_ttl(other.ttl) - super(Rdataset, self).intersection_update(other) - - def update(self, other): - """Add all rdatas in other to self. - - *other*, a ``dns.rdataset.Rdataset``, the rdataset from which - to update. - """ - - self.update_ttl(other.ttl) - super(Rdataset, self).update(other) - - def __repr__(self): - if self.covers == 0: - ctext = '' - else: - ctext = '(' + dns.rdatatype.to_text(self.covers) + ')' - return '' - - def __str__(self): - return self.to_text() - - def __eq__(self, other): - if not isinstance(other, Rdataset): - return False - if self.rdclass != other.rdclass or \ - self.rdtype != other.rdtype or \ - self.covers != other.covers: - return False - return super(Rdataset, self).__eq__(other) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_text(self, name=None, origin=None, relativize=True, - override_rdclass=None, **kw): - """Convert the rdataset into DNS master file format. - - See ``dns.name.Name.choose_relativity`` for more information - on how *origin* and *relativize* determine the way names - are emitted. - - Any additional keyword arguments are passed on to the rdata - ``to_text()`` method. - - *name*, a ``dns.name.Name``. If name is not ``None``, emit RRs with - *name* as the owner name. - - *origin*, a ``dns.name.Name`` or ``None``, the origin for relative - names. - - *relativize*, a ``bool``. If ``True``, names will be relativized - to *origin*. - """ - - if name is not None: - name = name.choose_relativity(origin, relativize) - ntext = str(name) - pad = ' ' - else: - ntext = '' - pad = '' - s = StringIO() - if override_rdclass is not None: - rdclass = override_rdclass - else: - rdclass = self.rdclass - if len(self) == 0: - # - # Empty rdatasets are used for the question section, and in - # some dynamic updates, so we don't need to print out the TTL - # (which is meaningless anyway). - # - s.write(u'{}{}{} {}\n'.format(ntext, pad, - dns.rdataclass.to_text(rdclass), - dns.rdatatype.to_text(self.rdtype))) - else: - for rd in self: - s.write(u'%s%s%d %s %s %s\n' % - (ntext, pad, self.ttl, dns.rdataclass.to_text(rdclass), - dns.rdatatype.to_text(self.rdtype), - rd.to_text(origin=origin, relativize=relativize, - **kw))) - # - # We strip off the final \n for the caller's convenience in printing - # - return s.getvalue()[:-1] - - def to_wire(self, name, file, compress=None, origin=None, - override_rdclass=None, want_shuffle=True): - """Convert the rdataset to wire format. - - *name*, a ``dns.name.Name`` is the owner name to use. - - *file* is the file where the name is emitted (typically a - BytesIO file). - - *compress*, a ``dict``, is the compression table to use. If - ``None`` (the default), names will not be compressed. - - *origin* is a ``dns.name.Name`` or ``None``. If the name is - relative and origin is not ``None``, then *origin* will be appended - to it. - - *override_rdclass*, an ``int``, is used as the class instead of the - class of the rdataset. This is useful when rendering rdatasets - associated with dynamic updates. - - *want_shuffle*, a ``bool``. If ``True``, then the order of the - Rdatas within the Rdataset will be shuffled before rendering. - - Returns an ``int``, the number of records emitted. - """ - - if override_rdclass is not None: - rdclass = override_rdclass - want_shuffle = False - else: - rdclass = self.rdclass - file.seek(0, 2) - if len(self) == 0: - name.to_wire(file, compress, origin) - stuff = struct.pack("!HHIH", self.rdtype, rdclass, 0, 0) - file.write(stuff) - return 1 - else: - if want_shuffle: - l = list(self) - random.shuffle(l) - else: - l = self - for rd in l: - name.to_wire(file, compress, origin) - stuff = struct.pack("!HHIH", self.rdtype, rdclass, - self.ttl, 0) - file.write(stuff) - start = file.tell() - rd.to_wire(file, compress, origin) - end = file.tell() - assert end - start < 65536 - file.seek(start - 2) - stuff = struct.pack("!H", end - start) - file.write(stuff) - file.seek(0, 2) - return len(self) - - def match(self, rdclass, rdtype, covers): - """Returns ``True`` if this rdataset matches the specified class, - type, and covers. - """ - if self.rdclass == rdclass and \ - self.rdtype == rdtype and \ - self.covers == covers: - return True - return False - - -def from_text_list(rdclass, rdtype, ttl, text_rdatas): - """Create an rdataset with the specified class, type, and TTL, and with - the specified list of rdatas in text format. - - Returns a ``dns.rdataset.Rdataset`` object. - """ - - if isinstance(rdclass, string_types): - rdclass = dns.rdataclass.from_text(rdclass) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - r = Rdataset(rdclass, rdtype) - r.update_ttl(ttl) - for t in text_rdatas: - rd = dns.rdata.from_text(r.rdclass, r.rdtype, t) - r.add(rd) - return r - - -def from_text(rdclass, rdtype, ttl, *text_rdatas): - """Create an rdataset with the specified class, type, and TTL, and with - the specified rdatas in text format. - - Returns a ``dns.rdataset.Rdataset`` object. - """ - - return from_text_list(rdclass, rdtype, ttl, text_rdatas) - - -def from_rdata_list(ttl, rdatas): - """Create an rdataset with the specified TTL, and with - the specified list of rdata objects. - - Returns a ``dns.rdataset.Rdataset`` object. - """ - - if len(rdatas) == 0: - raise ValueError("rdata list must not be empty") - r = None - for rd in rdatas: - if r is None: - r = Rdataset(rd.rdclass, rd.rdtype) - r.update_ttl(ttl) - r.add(rd) - return r - - -def from_rdata(ttl, *rdatas): - """Create an rdataset with the specified TTL, and with - the specified rdata objects. - - Returns a ``dns.rdataset.Rdataset`` object. - """ - - return from_rdata_list(ttl, rdatas) diff --git a/src/dns/rdataset.pyi b/src/dns/rdataset.pyi deleted file mode 100644 index 3efff88a..00000000 --- a/src/dns/rdataset.pyi +++ /dev/null @@ -1,58 +0,0 @@ -from typing import Optional, Dict, List, Union -from io import BytesIO -from . import exception, name, set, rdatatype, rdata, rdataset - -class DifferingCovers(exception.DNSException): - """An attempt was made to add a DNS SIG/RRSIG whose covered type - is not the same as that of the other rdatas in the rdataset.""" - - -class IncompatibleTypes(exception.DNSException): - """An attempt was made to add DNS RR data of an incompatible type.""" - - -class Rdataset(set.Set): - def __init__(self, rdclass, rdtype, covers=rdatatype.NONE, ttl=0): - self.rdclass : int = rdclass - self.rdtype : int = rdtype - self.covers : int = covers - self.ttl : int = ttl - - def update_ttl(self, ttl : int) -> None: - ... - - def add(self, rd : rdata.Rdata, ttl : Optional[int] =None): - ... - - def union_update(self, other : Rdataset): - ... - - def intersection_update(self, other : Rdataset): - ... - - def update(self, other : Rdataset): - ... - - def to_text(self, name : Optional[name.Name] =None, origin : Optional[name.Name] =None, relativize=True, - override_rdclass : Optional[int] =None, **kw) -> bytes: - ... - - def to_wire(self, name : Optional[name.Name], file : BytesIO, compress : Optional[Dict[name.Name, int]] = None, origin : Optional[name.Name] = None, - override_rdclass : Optional[int] = None, want_shuffle=True) -> int: - ... - - def match(self, rdclass : int, rdtype : int, covers : int) -> bool: - ... - - -def from_text_list(rdclass : Union[int,str], rdtype : Union[int,str], ttl : int, text_rdatas : str) -> rdataset.Rdataset: - ... - -def from_text(rdclass : Union[int,str], rdtype : Union[int,str], ttl : int, *text_rdatas : str) -> rdataset.Rdataset: - ... - -def from_rdata_list(ttl : int, rdatas : List[rdata.Rdata]) -> rdataset.Rdataset: - ... - -def from_rdata(ttl : int, *rdatas : List[rdata.Rdata]) -> rdataset.Rdataset: - ... diff --git a/src/dns/rdatatype.py b/src/dns/rdatatype.py deleted file mode 100644 index b247bc9c..00000000 --- a/src/dns/rdatatype.py +++ /dev/null @@ -1,287 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Rdata Types.""" - -import re - -import dns.exception - -NONE = 0 -A = 1 -NS = 2 -MD = 3 -MF = 4 -CNAME = 5 -SOA = 6 -MB = 7 -MG = 8 -MR = 9 -NULL = 10 -WKS = 11 -PTR = 12 -HINFO = 13 -MINFO = 14 -MX = 15 -TXT = 16 -RP = 17 -AFSDB = 18 -X25 = 19 -ISDN = 20 -RT = 21 -NSAP = 22 -NSAP_PTR = 23 -SIG = 24 -KEY = 25 -PX = 26 -GPOS = 27 -AAAA = 28 -LOC = 29 -NXT = 30 -SRV = 33 -NAPTR = 35 -KX = 36 -CERT = 37 -A6 = 38 -DNAME = 39 -OPT = 41 -APL = 42 -DS = 43 -SSHFP = 44 -IPSECKEY = 45 -RRSIG = 46 -NSEC = 47 -DNSKEY = 48 -DHCID = 49 -NSEC3 = 50 -NSEC3PARAM = 51 -TLSA = 52 -HIP = 55 -CDS = 59 -CDNSKEY = 60 -OPENPGPKEY = 61 -CSYNC = 62 -SPF = 99 -UNSPEC = 103 -EUI48 = 108 -EUI64 = 109 -TKEY = 249 -TSIG = 250 -IXFR = 251 -AXFR = 252 -MAILB = 253 -MAILA = 254 -ANY = 255 -URI = 256 -CAA = 257 -AVC = 258 -TA = 32768 -DLV = 32769 - -_by_text = { - 'NONE': NONE, - 'A': A, - 'NS': NS, - 'MD': MD, - 'MF': MF, - 'CNAME': CNAME, - 'SOA': SOA, - 'MB': MB, - 'MG': MG, - 'MR': MR, - 'NULL': NULL, - 'WKS': WKS, - 'PTR': PTR, - 'HINFO': HINFO, - 'MINFO': MINFO, - 'MX': MX, - 'TXT': TXT, - 'RP': RP, - 'AFSDB': AFSDB, - 'X25': X25, - 'ISDN': ISDN, - 'RT': RT, - 'NSAP': NSAP, - 'NSAP-PTR': NSAP_PTR, - 'SIG': SIG, - 'KEY': KEY, - 'PX': PX, - 'GPOS': GPOS, - 'AAAA': AAAA, - 'LOC': LOC, - 'NXT': NXT, - 'SRV': SRV, - 'NAPTR': NAPTR, - 'KX': KX, - 'CERT': CERT, - 'A6': A6, - 'DNAME': DNAME, - 'OPT': OPT, - 'APL': APL, - 'DS': DS, - 'SSHFP': SSHFP, - 'IPSECKEY': IPSECKEY, - 'RRSIG': RRSIG, - 'NSEC': NSEC, - 'DNSKEY': DNSKEY, - 'DHCID': DHCID, - 'NSEC3': NSEC3, - 'NSEC3PARAM': NSEC3PARAM, - 'TLSA': TLSA, - 'HIP': HIP, - 'CDS': CDS, - 'CDNSKEY': CDNSKEY, - 'OPENPGPKEY': OPENPGPKEY, - 'CSYNC': CSYNC, - 'SPF': SPF, - 'UNSPEC': UNSPEC, - 'EUI48': EUI48, - 'EUI64': EUI64, - 'TKEY': TKEY, - 'TSIG': TSIG, - 'IXFR': IXFR, - 'AXFR': AXFR, - 'MAILB': MAILB, - 'MAILA': MAILA, - 'ANY': ANY, - 'URI': URI, - 'CAA': CAA, - 'AVC': AVC, - 'TA': TA, - 'DLV': DLV, -} - -# We construct the inverse mapping programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mapping not to be true inverse. - -_by_value = {y: x for x, y in _by_text.items()} - -_metatypes = { - OPT: True -} - -_singletons = { - SOA: True, - NXT: True, - DNAME: True, - NSEC: True, - CNAME: True, -} - -_unknown_type_pattern = re.compile('TYPE([0-9]+)$', re.I) - - -class UnknownRdatatype(dns.exception.DNSException): - """DNS resource record type is unknown.""" - - -def from_text(text): - """Convert text into a DNS rdata type value. - - The input text can be a defined DNS RR type mnemonic or - instance of the DNS generic type syntax. - - For example, "NS" and "TYPE2" will both result in a value of 2. - - Raises ``dns.rdatatype.UnknownRdatatype`` if the type is unknown. - - Raises ``ValueError`` if the rdata type value is not >= 0 and <= 65535. - - Returns an ``int``. - """ - - value = _by_text.get(text.upper()) - if value is None: - match = _unknown_type_pattern.match(text) - if match is None: - raise UnknownRdatatype - value = int(match.group(1)) - if value < 0 or value > 65535: - raise ValueError("type must be between >= 0 and <= 65535") - return value - - -def to_text(value): - """Convert a DNS rdata type value to text. - - If the value has a known mnemonic, it will be used, otherwise the - DNS generic type syntax will be used. - - Raises ``ValueError`` if the rdata type value is not >= 0 and <= 65535. - - Returns a ``str``. - """ - - if value < 0 or value > 65535: - raise ValueError("type must be between >= 0 and <= 65535") - text = _by_value.get(value) - if text is None: - text = 'TYPE' + repr(value) - return text - - -def is_metatype(rdtype): - """True if the specified type is a metatype. - - *rdtype* is an ``int``. - - The currently defined metatypes are TKEY, TSIG, IXFR, AXFR, MAILA, - MAILB, ANY, and OPT. - - Returns a ``bool``. - """ - - if rdtype >= TKEY and rdtype <= ANY or rdtype in _metatypes: - return True - return False - - -def is_singleton(rdtype): - """Is the specified type a singleton type? - - Singleton types can only have a single rdata in an rdataset, or a single - RR in an RRset. - - The currently defined singleton types are CNAME, DNAME, NSEC, NXT, and - SOA. - - *rdtype* is an ``int``. - - Returns a ``bool``. - """ - - if rdtype in _singletons: - return True - return False - - -def register_type(rdtype, rdtype_text, is_singleton=False): # pylint: disable=redefined-outer-name - """Dynamically register an rdatatype. - - *rdtype*, an ``int``, the rdatatype to register. - - *rdtype_text*, a ``text``, the textual form of the rdatatype. - - *is_singleton*, a ``bool``, indicating if the type is a singleton (i.e. - RRsets of the type can have only one member.) - """ - - _by_text[rdtype_text] = rdtype - _by_value[rdtype] = rdtype_text - if is_singleton: - _singletons[rdtype] = True diff --git a/src/dns/rdtypes/ANY/AFSDB.py b/src/dns/rdtypes/ANY/AFSDB.py deleted file mode 100644 index c6a700cf..00000000 --- a/src/dns/rdtypes/ANY/AFSDB.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.mxbase - - -class AFSDB(dns.rdtypes.mxbase.UncompressedDowncasingMX): - - """AFSDB record - - @ivar subtype: the subtype value - @type subtype: int - @ivar hostname: the hostname name - @type hostname: dns.name.Name object""" - - # Use the property mechanism to make "subtype" an alias for the - # "preference" attribute, and "hostname" an alias for the "exchange" - # attribute. - # - # This lets us inherit the UncompressedMX implementation but lets - # the caller use appropriate attribute names for the rdata type. - # - # We probably lose some performance vs. a cut-and-paste - # implementation, but this way we don't copy code, and that's - # good. - - def get_subtype(self): - return self.preference - - def set_subtype(self, subtype): - self.preference = subtype - - subtype = property(get_subtype, set_subtype) - - def get_hostname(self): - return self.exchange - - def set_hostname(self, hostname): - self.exchange = hostname - - hostname = property(get_hostname, set_hostname) diff --git a/src/dns/rdtypes/ANY/AVC.py b/src/dns/rdtypes/ANY/AVC.py deleted file mode 100644 index 7f340b39..00000000 --- a/src/dns/rdtypes/ANY/AVC.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2016 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.txtbase - - -class AVC(dns.rdtypes.txtbase.TXTBase): - - """AVC record - - @see: U{http://www.iana.org/assignments/dns-parameters/AVC/avc-completed-template}""" diff --git a/src/dns/rdtypes/ANY/CAA.py b/src/dns/rdtypes/ANY/CAA.py deleted file mode 100644 index 0acf201a..00000000 --- a/src/dns/rdtypes/ANY/CAA.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.tokenizer - - -class CAA(dns.rdata.Rdata): - - """CAA (Certification Authority Authorization) record - - @ivar flags: the flags - @type flags: int - @ivar tag: the tag - @type tag: string - @ivar value: the value - @type value: string - @see: RFC 6844""" - - __slots__ = ['flags', 'tag', 'value'] - - def __init__(self, rdclass, rdtype, flags, tag, value): - super(CAA, self).__init__(rdclass, rdtype) - self.flags = flags - self.tag = tag - self.value = value - - def to_text(self, origin=None, relativize=True, **kw): - return '%u %s "%s"' % (self.flags, - dns.rdata._escapify(self.tag), - dns.rdata._escapify(self.value)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - flags = tok.get_uint8() - tag = tok.get_string().encode() - if len(tag) > 255: - raise dns.exception.SyntaxError("tag too long") - if not tag.isalnum(): - raise dns.exception.SyntaxError("tag is not alphanumeric") - value = tok.get_string().encode() - return cls(rdclass, rdtype, flags, tag, value) - - def to_wire(self, file, compress=None, origin=None): - file.write(struct.pack('!B', self.flags)) - l = len(self.tag) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.tag) - file.write(self.value) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (flags, l) = struct.unpack('!BB', wire[current: current + 2]) - current += 2 - tag = wire[current: current + l] - value = wire[current + l:current + rdlen - 2] - return cls(rdclass, rdtype, flags, tag, value) diff --git a/src/dns/rdtypes/ANY/CDNSKEY.py b/src/dns/rdtypes/ANY/CDNSKEY.py deleted file mode 100644 index 653ae1be..00000000 --- a/src/dns/rdtypes/ANY/CDNSKEY.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.dnskeybase -from dns.rdtypes.dnskeybase import flags_to_text_set, flags_from_text_set - - -__all__ = ['flags_to_text_set', 'flags_from_text_set'] - - -class CDNSKEY(dns.rdtypes.dnskeybase.DNSKEYBase): - - """CDNSKEY record""" diff --git a/src/dns/rdtypes/ANY/CDS.py b/src/dns/rdtypes/ANY/CDS.py deleted file mode 100644 index a63041dd..00000000 --- a/src/dns/rdtypes/ANY/CDS.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.dsbase - - -class CDS(dns.rdtypes.dsbase.DSBase): - - """CDS record""" diff --git a/src/dns/rdtypes/ANY/CERT.py b/src/dns/rdtypes/ANY/CERT.py deleted file mode 100644 index eea27b52..00000000 --- a/src/dns/rdtypes/ANY/CERT.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import base64 - -import dns.exception -import dns.dnssec -import dns.rdata -import dns.tokenizer - -_ctype_by_value = { - 1: 'PKIX', - 2: 'SPKI', - 3: 'PGP', - 253: 'URI', - 254: 'OID', -} - -_ctype_by_name = { - 'PKIX': 1, - 'SPKI': 2, - 'PGP': 3, - 'URI': 253, - 'OID': 254, -} - - -def _ctype_from_text(what): - v = _ctype_by_name.get(what) - if v is not None: - return v - return int(what) - - -def _ctype_to_text(what): - v = _ctype_by_value.get(what) - if v is not None: - return v - return str(what) - - -class CERT(dns.rdata.Rdata): - - """CERT record - - @ivar certificate_type: certificate type - @type certificate_type: int - @ivar key_tag: key tag - @type key_tag: int - @ivar algorithm: algorithm - @type algorithm: int - @ivar certificate: the certificate or CRL - @type certificate: string - @see: RFC 2538""" - - __slots__ = ['certificate_type', 'key_tag', 'algorithm', 'certificate'] - - def __init__(self, rdclass, rdtype, certificate_type, key_tag, algorithm, - certificate): - super(CERT, self).__init__(rdclass, rdtype) - self.certificate_type = certificate_type - self.key_tag = key_tag - self.algorithm = algorithm - self.certificate = certificate - - def to_text(self, origin=None, relativize=True, **kw): - certificate_type = _ctype_to_text(self.certificate_type) - return "%s %d %s %s" % (certificate_type, self.key_tag, - dns.dnssec.algorithm_to_text(self.algorithm), - dns.rdata._base64ify(self.certificate)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - certificate_type = _ctype_from_text(tok.get_string()) - key_tag = tok.get_uint16() - algorithm = dns.dnssec.algorithm_from_text(tok.get_string()) - if algorithm < 0 or algorithm > 255: - raise dns.exception.SyntaxError("bad algorithm type") - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - b64 = b''.join(chunks) - certificate = base64.b64decode(b64) - return cls(rdclass, rdtype, certificate_type, key_tag, - algorithm, certificate) - - def to_wire(self, file, compress=None, origin=None): - prefix = struct.pack("!HHB", self.certificate_type, self.key_tag, - self.algorithm) - file.write(prefix) - file.write(self.certificate) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - prefix = wire[current: current + 5].unwrap() - current += 5 - rdlen -= 5 - if rdlen < 0: - raise dns.exception.FormError - (certificate_type, key_tag, algorithm) = struct.unpack("!HHB", prefix) - certificate = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, certificate_type, key_tag, algorithm, - certificate) diff --git a/src/dns/rdtypes/ANY/CNAME.py b/src/dns/rdtypes/ANY/CNAME.py deleted file mode 100644 index 11d42aa7..00000000 --- a/src/dns/rdtypes/ANY/CNAME.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.nsbase - - -class CNAME(dns.rdtypes.nsbase.NSBase): - - """CNAME record - - Note: although CNAME is officially a singleton type, dnspython allows - non-singleton CNAME rdatasets because such sets have been commonly - used by BIND and other nameservers for load balancing.""" diff --git a/src/dns/rdtypes/ANY/CSYNC.py b/src/dns/rdtypes/ANY/CSYNC.py deleted file mode 100644 index 06292fb2..00000000 --- a/src/dns/rdtypes/ANY/CSYNC.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011, 2016 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.rdatatype -import dns.name -from dns._compat import xrange - -class CSYNC(dns.rdata.Rdata): - - """CSYNC record - - @ivar serial: the SOA serial number - @type serial: int - @ivar flags: the CSYNC flags - @type flags: int - @ivar windows: the windowed bitmap list - @type windows: list of (window number, string) tuples""" - - __slots__ = ['serial', 'flags', 'windows'] - - def __init__(self, rdclass, rdtype, serial, flags, windows): - super(CSYNC, self).__init__(rdclass, rdtype) - self.serial = serial - self.flags = flags - self.windows = windows - - def to_text(self, origin=None, relativize=True, **kw): - text = '' - for (window, bitmap) in self.windows: - bits = [] - for i in xrange(0, len(bitmap)): - byte = bitmap[i] - for j in xrange(0, 8): - if byte & (0x80 >> j): - bits.append(dns.rdatatype.to_text(window * 256 + - i * 8 + j)) - text += (' ' + ' '.join(bits)) - return '%d %d%s' % (self.serial, self.flags, text) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - serial = tok.get_uint32() - flags = tok.get_uint16() - rdtypes = [] - while 1: - token = tok.get().unescape() - if token.is_eol_or_eof(): - break - nrdtype = dns.rdatatype.from_text(token.value) - if nrdtype == 0: - raise dns.exception.SyntaxError("CSYNC with bit 0") - if nrdtype > 65535: - raise dns.exception.SyntaxError("CSYNC with bit > 65535") - rdtypes.append(nrdtype) - rdtypes.sort() - window = 0 - octets = 0 - prior_rdtype = 0 - bitmap = bytearray(b'\0' * 32) - windows = [] - for nrdtype in rdtypes: - if nrdtype == prior_rdtype: - continue - prior_rdtype = nrdtype - new_window = nrdtype // 256 - if new_window != window: - windows.append((window, bitmap[0:octets])) - bitmap = bytearray(b'\0' * 32) - window = new_window - offset = nrdtype % 256 - byte = offset // 8 - bit = offset % 8 - octets = byte + 1 - bitmap[byte] = bitmap[byte] | (0x80 >> bit) - - windows.append((window, bitmap[0:octets])) - return cls(rdclass, rdtype, serial, flags, windows) - - def to_wire(self, file, compress=None, origin=None): - file.write(struct.pack('!IH', self.serial, self.flags)) - for (window, bitmap) in self.windows: - file.write(struct.pack('!BB', window, len(bitmap))) - file.write(bitmap) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - if rdlen < 6: - raise dns.exception.FormError("CSYNC too short") - (serial, flags) = struct.unpack("!IH", wire[current: current + 6]) - current += 6 - rdlen -= 6 - windows = [] - while rdlen > 0: - if rdlen < 3: - raise dns.exception.FormError("CSYNC too short") - window = wire[current] - octets = wire[current + 1] - if octets == 0 or octets > 32: - raise dns.exception.FormError("bad CSYNC octets") - current += 2 - rdlen -= 2 - if rdlen < octets: - raise dns.exception.FormError("bad CSYNC bitmap length") - bitmap = bytearray(wire[current: current + octets].unwrap()) - current += octets - rdlen -= octets - windows.append((window, bitmap)) - return cls(rdclass, rdtype, serial, flags, windows) diff --git a/src/dns/rdtypes/ANY/DLV.py b/src/dns/rdtypes/ANY/DLV.py deleted file mode 100644 index 16352125..00000000 --- a/src/dns/rdtypes/ANY/DLV.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.dsbase - - -class DLV(dns.rdtypes.dsbase.DSBase): - - """DLV record""" diff --git a/src/dns/rdtypes/ANY/DNAME.py b/src/dns/rdtypes/ANY/DNAME.py deleted file mode 100644 index 2499283c..00000000 --- a/src/dns/rdtypes/ANY/DNAME.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.nsbase - - -class DNAME(dns.rdtypes.nsbase.UncompressedNS): - - """DNAME record""" - - def to_digestable(self, origin=None): - return self.target.to_digestable(origin) diff --git a/src/dns/rdtypes/ANY/DNSKEY.py b/src/dns/rdtypes/ANY/DNSKEY.py deleted file mode 100644 index e36f7bc5..00000000 --- a/src/dns/rdtypes/ANY/DNSKEY.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.dnskeybase -from dns.rdtypes.dnskeybase import flags_to_text_set, flags_from_text_set - - -__all__ = ['flags_to_text_set', 'flags_from_text_set'] - - -class DNSKEY(dns.rdtypes.dnskeybase.DNSKEYBase): - - """DNSKEY record""" diff --git a/src/dns/rdtypes/ANY/DS.py b/src/dns/rdtypes/ANY/DS.py deleted file mode 100644 index 7d457b22..00000000 --- a/src/dns/rdtypes/ANY/DS.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.dsbase - - -class DS(dns.rdtypes.dsbase.DSBase): - - """DS record""" diff --git a/src/dns/rdtypes/ANY/EUI48.py b/src/dns/rdtypes/ANY/EUI48.py deleted file mode 100644 index aa260e20..00000000 --- a/src/dns/rdtypes/ANY/EUI48.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2015 Red Hat, Inc. -# Author: Petr Spacek -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED 'AS IS' AND RED HAT DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.euibase - - -class EUI48(dns.rdtypes.euibase.EUIBase): - - """EUI48 record - - @ivar fingerprint: 48-bit Extended Unique Identifier (EUI-48) - @type fingerprint: string - @see: rfc7043.txt""" - - byte_len = 6 # 0123456789ab (in hex) - text_len = byte_len * 3 - 1 # 01-23-45-67-89-ab diff --git a/src/dns/rdtypes/ANY/EUI64.py b/src/dns/rdtypes/ANY/EUI64.py deleted file mode 100644 index 5eba350d..00000000 --- a/src/dns/rdtypes/ANY/EUI64.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2015 Red Hat, Inc. -# Author: Petr Spacek -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED 'AS IS' AND RED HAT DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.euibase - - -class EUI64(dns.rdtypes.euibase.EUIBase): - - """EUI64 record - - @ivar fingerprint: 64-bit Extended Unique Identifier (EUI-64) - @type fingerprint: string - @see: rfc7043.txt""" - - byte_len = 8 # 0123456789abcdef (in hex) - text_len = byte_len * 3 - 1 # 01-23-45-67-89-ab-cd-ef diff --git a/src/dns/rdtypes/ANY/GPOS.py b/src/dns/rdtypes/ANY/GPOS.py deleted file mode 100644 index 422822f0..00000000 --- a/src/dns/rdtypes/ANY/GPOS.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.tokenizer -from dns._compat import long, text_type - - -def _validate_float_string(what): - if what[0] == b'-'[0] or what[0] == b'+'[0]: - what = what[1:] - if what.isdigit(): - return - (left, right) = what.split(b'.') - if left == b'' and right == b'': - raise dns.exception.FormError - if not left == b'' and not left.decode().isdigit(): - raise dns.exception.FormError - if not right == b'' and not right.decode().isdigit(): - raise dns.exception.FormError - - -def _sanitize(value): - if isinstance(value, text_type): - return value.encode() - return value - - -class GPOS(dns.rdata.Rdata): - - """GPOS record - - @ivar latitude: latitude - @type latitude: string - @ivar longitude: longitude - @type longitude: string - @ivar altitude: altitude - @type altitude: string - @see: RFC 1712""" - - __slots__ = ['latitude', 'longitude', 'altitude'] - - def __init__(self, rdclass, rdtype, latitude, longitude, altitude): - super(GPOS, self).__init__(rdclass, rdtype) - if isinstance(latitude, float) or \ - isinstance(latitude, int) or \ - isinstance(latitude, long): - latitude = str(latitude) - if isinstance(longitude, float) or \ - isinstance(longitude, int) or \ - isinstance(longitude, long): - longitude = str(longitude) - if isinstance(altitude, float) or \ - isinstance(altitude, int) or \ - isinstance(altitude, long): - altitude = str(altitude) - latitude = _sanitize(latitude) - longitude = _sanitize(longitude) - altitude = _sanitize(altitude) - _validate_float_string(latitude) - _validate_float_string(longitude) - _validate_float_string(altitude) - self.latitude = latitude - self.longitude = longitude - self.altitude = altitude - - def to_text(self, origin=None, relativize=True, **kw): - return '{} {} {}'.format(self.latitude.decode(), - self.longitude.decode(), - self.altitude.decode()) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - latitude = tok.get_string() - longitude = tok.get_string() - altitude = tok.get_string() - tok.get_eol() - return cls(rdclass, rdtype, latitude, longitude, altitude) - - def to_wire(self, file, compress=None, origin=None): - l = len(self.latitude) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.latitude) - l = len(self.longitude) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.longitude) - l = len(self.altitude) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.altitude) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - l = wire[current] - current += 1 - rdlen -= 1 - if l > rdlen: - raise dns.exception.FormError - latitude = wire[current: current + l].unwrap() - current += l - rdlen -= l - l = wire[current] - current += 1 - rdlen -= 1 - if l > rdlen: - raise dns.exception.FormError - longitude = wire[current: current + l].unwrap() - current += l - rdlen -= l - l = wire[current] - current += 1 - rdlen -= 1 - if l != rdlen: - raise dns.exception.FormError - altitude = wire[current: current + l].unwrap() - return cls(rdclass, rdtype, latitude, longitude, altitude) - - def _get_float_latitude(self): - return float(self.latitude) - - def _set_float_latitude(self, value): - self.latitude = str(value) - - float_latitude = property(_get_float_latitude, _set_float_latitude, - doc="latitude as a floating point value") - - def _get_float_longitude(self): - return float(self.longitude) - - def _set_float_longitude(self, value): - self.longitude = str(value) - - float_longitude = property(_get_float_longitude, _set_float_longitude, - doc="longitude as a floating point value") - - def _get_float_altitude(self): - return float(self.altitude) - - def _set_float_altitude(self, value): - self.altitude = str(value) - - float_altitude = property(_get_float_altitude, _set_float_altitude, - doc="altitude as a floating point value") diff --git a/src/dns/rdtypes/ANY/HINFO.py b/src/dns/rdtypes/ANY/HINFO.py deleted file mode 100644 index e4e0b34a..00000000 --- a/src/dns/rdtypes/ANY/HINFO.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.tokenizer -from dns._compat import text_type - - -class HINFO(dns.rdata.Rdata): - - """HINFO record - - @ivar cpu: the CPU type - @type cpu: string - @ivar os: the OS type - @type os: string - @see: RFC 1035""" - - __slots__ = ['cpu', 'os'] - - def __init__(self, rdclass, rdtype, cpu, os): - super(HINFO, self).__init__(rdclass, rdtype) - if isinstance(cpu, text_type): - self.cpu = cpu.encode() - else: - self.cpu = cpu - if isinstance(os, text_type): - self.os = os.encode() - else: - self.os = os - - def to_text(self, origin=None, relativize=True, **kw): - return '"{}" "{}"'.format(dns.rdata._escapify(self.cpu), - dns.rdata._escapify(self.os)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - cpu = tok.get_string() - os = tok.get_string() - tok.get_eol() - return cls(rdclass, rdtype, cpu, os) - - def to_wire(self, file, compress=None, origin=None): - l = len(self.cpu) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.cpu) - l = len(self.os) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.os) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - l = wire[current] - current += 1 - rdlen -= 1 - if l > rdlen: - raise dns.exception.FormError - cpu = wire[current:current + l].unwrap() - current += l - rdlen -= l - l = wire[current] - current += 1 - rdlen -= 1 - if l != rdlen: - raise dns.exception.FormError - os = wire[current: current + l].unwrap() - return cls(rdclass, rdtype, cpu, os) diff --git a/src/dns/rdtypes/ANY/HIP.py b/src/dns/rdtypes/ANY/HIP.py deleted file mode 100644 index 7c876b2d..00000000 --- a/src/dns/rdtypes/ANY/HIP.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2010, 2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import base64 -import binascii - -import dns.exception -import dns.rdata -import dns.rdatatype - - -class HIP(dns.rdata.Rdata): - - """HIP record - - @ivar hit: the host identity tag - @type hit: string - @ivar algorithm: the public key cryptographic algorithm - @type algorithm: int - @ivar key: the public key - @type key: string - @ivar servers: the rendezvous servers - @type servers: list of dns.name.Name objects - @see: RFC 5205""" - - __slots__ = ['hit', 'algorithm', 'key', 'servers'] - - def __init__(self, rdclass, rdtype, hit, algorithm, key, servers): - super(HIP, self).__init__(rdclass, rdtype) - self.hit = hit - self.algorithm = algorithm - self.key = key - self.servers = servers - - def to_text(self, origin=None, relativize=True, **kw): - hit = binascii.hexlify(self.hit).decode() - key = base64.b64encode(self.key).replace(b'\n', b'').decode() - text = u'' - servers = [] - for server in self.servers: - servers.append(server.choose_relativity(origin, relativize)) - if len(servers) > 0: - text += (u' ' + u' '.join((x.to_unicode() for x in servers))) - return u'%u %s %s%s' % (self.algorithm, hit, key, text) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - algorithm = tok.get_uint8() - hit = binascii.unhexlify(tok.get_string().encode()) - if len(hit) > 255: - raise dns.exception.SyntaxError("HIT too long") - key = base64.b64decode(tok.get_string().encode()) - servers = [] - while 1: - token = tok.get() - if token.is_eol_or_eof(): - break - server = dns.name.from_text(token.value, origin) - server.choose_relativity(origin, relativize) - servers.append(server) - return cls(rdclass, rdtype, hit, algorithm, key, servers) - - def to_wire(self, file, compress=None, origin=None): - lh = len(self.hit) - lk = len(self.key) - file.write(struct.pack("!BBH", lh, self.algorithm, lk)) - file.write(self.hit) - file.write(self.key) - for server in self.servers: - server.to_wire(file, None, origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (lh, algorithm, lk) = struct.unpack('!BBH', - wire[current: current + 4]) - current += 4 - rdlen -= 4 - hit = wire[current: current + lh].unwrap() - current += lh - rdlen -= lh - key = wire[current: current + lk].unwrap() - current += lk - rdlen -= lk - servers = [] - while rdlen > 0: - (server, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - current += cused - rdlen -= cused - if origin is not None: - server = server.relativize(origin) - servers.append(server) - return cls(rdclass, rdtype, hit, algorithm, key, servers) - - def choose_relativity(self, origin=None, relativize=True): - servers = [] - for server in self.servers: - server = server.choose_relativity(origin, relativize) - servers.append(server) - self.servers = servers diff --git a/src/dns/rdtypes/ANY/ISDN.py b/src/dns/rdtypes/ANY/ISDN.py deleted file mode 100644 index f5f5f8b9..00000000 --- a/src/dns/rdtypes/ANY/ISDN.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.tokenizer -from dns._compat import text_type - - -class ISDN(dns.rdata.Rdata): - - """ISDN record - - @ivar address: the ISDN address - @type address: string - @ivar subaddress: the ISDN subaddress (or '' if not present) - @type subaddress: string - @see: RFC 1183""" - - __slots__ = ['address', 'subaddress'] - - def __init__(self, rdclass, rdtype, address, subaddress): - super(ISDN, self).__init__(rdclass, rdtype) - if isinstance(address, text_type): - self.address = address.encode() - else: - self.address = address - if isinstance(address, text_type): - self.subaddress = subaddress.encode() - else: - self.subaddress = subaddress - - def to_text(self, origin=None, relativize=True, **kw): - if self.subaddress: - return '"{}" "{}"'.format(dns.rdata._escapify(self.address), - dns.rdata._escapify(self.subaddress)) - else: - return '"%s"' % dns.rdata._escapify(self.address) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - address = tok.get_string() - t = tok.get() - if not t.is_eol_or_eof(): - tok.unget(t) - subaddress = tok.get_string() - else: - tok.unget(t) - subaddress = '' - tok.get_eol() - return cls(rdclass, rdtype, address, subaddress) - - def to_wire(self, file, compress=None, origin=None): - l = len(self.address) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.address) - l = len(self.subaddress) - if l > 0: - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.subaddress) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - l = wire[current] - current += 1 - rdlen -= 1 - if l > rdlen: - raise dns.exception.FormError - address = wire[current: current + l].unwrap() - current += l - rdlen -= l - if rdlen > 0: - l = wire[current] - current += 1 - rdlen -= 1 - if l != rdlen: - raise dns.exception.FormError - subaddress = wire[current: current + l].unwrap() - else: - subaddress = '' - return cls(rdclass, rdtype, address, subaddress) diff --git a/src/dns/rdtypes/ANY/LOC.py b/src/dns/rdtypes/ANY/LOC.py deleted file mode 100644 index da9bb03a..00000000 --- a/src/dns/rdtypes/ANY/LOC.py +++ /dev/null @@ -1,327 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import division - -import struct - -import dns.exception -import dns.rdata -from dns._compat import long, xrange, round_py2_compat - - -_pows = tuple(long(10**i) for i in range(0, 11)) - -# default values are in centimeters -_default_size = 100.0 -_default_hprec = 1000000.0 -_default_vprec = 1000.0 - - -def _exponent_of(what, desc): - if what == 0: - return 0 - exp = None - for i in xrange(len(_pows)): - if what // _pows[i] == long(0): - exp = i - 1 - break - if exp is None or exp < 0: - raise dns.exception.SyntaxError("%s value out of bounds" % desc) - return exp - - -def _float_to_tuple(what): - if what < 0: - sign = -1 - what *= -1 - else: - sign = 1 - what = round_py2_compat(what * 3600000) - degrees = int(what // 3600000) - what -= degrees * 3600000 - minutes = int(what // 60000) - what -= minutes * 60000 - seconds = int(what // 1000) - what -= int(seconds * 1000) - what = int(what) - return (degrees, minutes, seconds, what, sign) - - -def _tuple_to_float(what): - value = float(what[0]) - value += float(what[1]) / 60.0 - value += float(what[2]) / 3600.0 - value += float(what[3]) / 3600000.0 - return float(what[4]) * value - - -def _encode_size(what, desc): - what = long(what) - exponent = _exponent_of(what, desc) & 0xF - base = what // pow(10, exponent) & 0xF - return base * 16 + exponent - - -def _decode_size(what, desc): - exponent = what & 0x0F - if exponent > 9: - raise dns.exception.SyntaxError("bad %s exponent" % desc) - base = (what & 0xF0) >> 4 - if base > 9: - raise dns.exception.SyntaxError("bad %s base" % desc) - return long(base) * pow(10, exponent) - - -class LOC(dns.rdata.Rdata): - - """LOC record - - @ivar latitude: latitude - @type latitude: (int, int, int, int, sign) tuple specifying the degrees, minutes, - seconds, milliseconds, and sign of the coordinate. - @ivar longitude: longitude - @type longitude: (int, int, int, int, sign) tuple specifying the degrees, - minutes, seconds, milliseconds, and sign of the coordinate. - @ivar altitude: altitude - @type altitude: float - @ivar size: size of the sphere - @type size: float - @ivar horizontal_precision: horizontal precision - @type horizontal_precision: float - @ivar vertical_precision: vertical precision - @type vertical_precision: float - @see: RFC 1876""" - - __slots__ = ['latitude', 'longitude', 'altitude', 'size', - 'horizontal_precision', 'vertical_precision'] - - def __init__(self, rdclass, rdtype, latitude, longitude, altitude, - size=_default_size, hprec=_default_hprec, - vprec=_default_vprec): - """Initialize a LOC record instance. - - The parameters I{latitude} and I{longitude} may be either a 4-tuple - of integers specifying (degrees, minutes, seconds, milliseconds), - or they may be floating point values specifying the number of - degrees. The other parameters are floats. Size, horizontal precision, - and vertical precision are specified in centimeters.""" - - super(LOC, self).__init__(rdclass, rdtype) - if isinstance(latitude, int) or isinstance(latitude, long): - latitude = float(latitude) - if isinstance(latitude, float): - latitude = _float_to_tuple(latitude) - self.latitude = latitude - if isinstance(longitude, int) or isinstance(longitude, long): - longitude = float(longitude) - if isinstance(longitude, float): - longitude = _float_to_tuple(longitude) - self.longitude = longitude - self.altitude = float(altitude) - self.size = float(size) - self.horizontal_precision = float(hprec) - self.vertical_precision = float(vprec) - - def to_text(self, origin=None, relativize=True, **kw): - if self.latitude[4] > 0: - lat_hemisphere = 'N' - else: - lat_hemisphere = 'S' - if self.longitude[4] > 0: - long_hemisphere = 'E' - else: - long_hemisphere = 'W' - text = "%d %d %d.%03d %s %d %d %d.%03d %s %0.2fm" % ( - self.latitude[0], self.latitude[1], - self.latitude[2], self.latitude[3], lat_hemisphere, - self.longitude[0], self.longitude[1], self.longitude[2], - self.longitude[3], long_hemisphere, - self.altitude / 100.0 - ) - - # do not print default values - if self.size != _default_size or \ - self.horizontal_precision != _default_hprec or \ - self.vertical_precision != _default_vprec: - text += " {:0.2f}m {:0.2f}m {:0.2f}m".format( - self.size / 100.0, self.horizontal_precision / 100.0, - self.vertical_precision / 100.0 - ) - return text - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - latitude = [0, 0, 0, 0, 1] - longitude = [0, 0, 0, 0, 1] - size = _default_size - hprec = _default_hprec - vprec = _default_vprec - - latitude[0] = tok.get_int() - t = tok.get_string() - if t.isdigit(): - latitude[1] = int(t) - t = tok.get_string() - if '.' in t: - (seconds, milliseconds) = t.split('.') - if not seconds.isdigit(): - raise dns.exception.SyntaxError( - 'bad latitude seconds value') - latitude[2] = int(seconds) - if latitude[2] >= 60: - raise dns.exception.SyntaxError('latitude seconds >= 60') - l = len(milliseconds) - if l == 0 or l > 3 or not milliseconds.isdigit(): - raise dns.exception.SyntaxError( - 'bad latitude milliseconds value') - if l == 1: - m = 100 - elif l == 2: - m = 10 - else: - m = 1 - latitude[3] = m * int(milliseconds) - t = tok.get_string() - elif t.isdigit(): - latitude[2] = int(t) - t = tok.get_string() - if t == 'S': - latitude[4] = -1 - elif t != 'N': - raise dns.exception.SyntaxError('bad latitude hemisphere value') - - longitude[0] = tok.get_int() - t = tok.get_string() - if t.isdigit(): - longitude[1] = int(t) - t = tok.get_string() - if '.' in t: - (seconds, milliseconds) = t.split('.') - if not seconds.isdigit(): - raise dns.exception.SyntaxError( - 'bad longitude seconds value') - longitude[2] = int(seconds) - if longitude[2] >= 60: - raise dns.exception.SyntaxError('longitude seconds >= 60') - l = len(milliseconds) - if l == 0 or l > 3 or not milliseconds.isdigit(): - raise dns.exception.SyntaxError( - 'bad longitude milliseconds value') - if l == 1: - m = 100 - elif l == 2: - m = 10 - else: - m = 1 - longitude[3] = m * int(milliseconds) - t = tok.get_string() - elif t.isdigit(): - longitude[2] = int(t) - t = tok.get_string() - if t == 'W': - longitude[4] = -1 - elif t != 'E': - raise dns.exception.SyntaxError('bad longitude hemisphere value') - - t = tok.get_string() - if t[-1] == 'm': - t = t[0: -1] - altitude = float(t) * 100.0 # m -> cm - - token = tok.get().unescape() - if not token.is_eol_or_eof(): - value = token.value - if value[-1] == 'm': - value = value[0: -1] - size = float(value) * 100.0 # m -> cm - token = tok.get().unescape() - if not token.is_eol_or_eof(): - value = token.value - if value[-1] == 'm': - value = value[0: -1] - hprec = float(value) * 100.0 # m -> cm - token = tok.get().unescape() - if not token.is_eol_or_eof(): - value = token.value - if value[-1] == 'm': - value = value[0: -1] - vprec = float(value) * 100.0 # m -> cm - tok.get_eol() - - return cls(rdclass, rdtype, latitude, longitude, altitude, - size, hprec, vprec) - - def to_wire(self, file, compress=None, origin=None): - milliseconds = (self.latitude[0] * 3600000 + - self.latitude[1] * 60000 + - self.latitude[2] * 1000 + - self.latitude[3]) * self.latitude[4] - latitude = long(0x80000000) + milliseconds - milliseconds = (self.longitude[0] * 3600000 + - self.longitude[1] * 60000 + - self.longitude[2] * 1000 + - self.longitude[3]) * self.longitude[4] - longitude = long(0x80000000) + milliseconds - altitude = long(self.altitude) + long(10000000) - size = _encode_size(self.size, "size") - hprec = _encode_size(self.horizontal_precision, "horizontal precision") - vprec = _encode_size(self.vertical_precision, "vertical precision") - wire = struct.pack("!BBBBIII", 0, size, hprec, vprec, latitude, - longitude, altitude) - file.write(wire) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (version, size, hprec, vprec, latitude, longitude, altitude) = \ - struct.unpack("!BBBBIII", wire[current: current + rdlen]) - if latitude > long(0x80000000): - latitude = float(latitude - long(0x80000000)) / 3600000 - else: - latitude = -1 * float(long(0x80000000) - latitude) / 3600000 - if latitude < -90.0 or latitude > 90.0: - raise dns.exception.FormError("bad latitude") - if longitude > long(0x80000000): - longitude = float(longitude - long(0x80000000)) / 3600000 - else: - longitude = -1 * float(long(0x80000000) - longitude) / 3600000 - if longitude < -180.0 or longitude > 180.0: - raise dns.exception.FormError("bad longitude") - altitude = float(altitude) - 10000000.0 - size = _decode_size(size, "size") - hprec = _decode_size(hprec, "horizontal precision") - vprec = _decode_size(vprec, "vertical precision") - return cls(rdclass, rdtype, latitude, longitude, altitude, - size, hprec, vprec) - - def _get_float_latitude(self): - return _tuple_to_float(self.latitude) - - def _set_float_latitude(self, value): - self.latitude = _float_to_tuple(value) - - float_latitude = property(_get_float_latitude, _set_float_latitude, - doc="latitude as a floating point value") - - def _get_float_longitude(self): - return _tuple_to_float(self.longitude) - - def _set_float_longitude(self, value): - self.longitude = _float_to_tuple(value) - - float_longitude = property(_get_float_longitude, _set_float_longitude, - doc="longitude as a floating point value") diff --git a/src/dns/rdtypes/ANY/MX.py b/src/dns/rdtypes/ANY/MX.py deleted file mode 100644 index 0a06494f..00000000 --- a/src/dns/rdtypes/ANY/MX.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.mxbase - - -class MX(dns.rdtypes.mxbase.MXBase): - - """MX record""" diff --git a/src/dns/rdtypes/ANY/NS.py b/src/dns/rdtypes/ANY/NS.py deleted file mode 100644 index f9fcf637..00000000 --- a/src/dns/rdtypes/ANY/NS.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.nsbase - - -class NS(dns.rdtypes.nsbase.NSBase): - - """NS record""" diff --git a/src/dns/rdtypes/ANY/NSEC.py b/src/dns/rdtypes/ANY/NSEC.py deleted file mode 100644 index 4e3da729..00000000 --- a/src/dns/rdtypes/ANY/NSEC.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.rdatatype -import dns.name -from dns._compat import xrange - - -class NSEC(dns.rdata.Rdata): - - """NSEC record - - @ivar next: the next name - @type next: dns.name.Name object - @ivar windows: the windowed bitmap list - @type windows: list of (window number, string) tuples""" - - __slots__ = ['next', 'windows'] - - def __init__(self, rdclass, rdtype, next, windows): - super(NSEC, self).__init__(rdclass, rdtype) - self.next = next - self.windows = windows - - def to_text(self, origin=None, relativize=True, **kw): - next = self.next.choose_relativity(origin, relativize) - text = '' - for (window, bitmap) in self.windows: - bits = [] - for i in xrange(0, len(bitmap)): - byte = bitmap[i] - for j in xrange(0, 8): - if byte & (0x80 >> j): - bits.append(dns.rdatatype.to_text(window * 256 + - i * 8 + j)) - text += (' ' + ' '.join(bits)) - return '{}{}'.format(next, text) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - next = tok.get_name() - next = next.choose_relativity(origin, relativize) - rdtypes = [] - while 1: - token = tok.get().unescape() - if token.is_eol_or_eof(): - break - nrdtype = dns.rdatatype.from_text(token.value) - if nrdtype == 0: - raise dns.exception.SyntaxError("NSEC with bit 0") - if nrdtype > 65535: - raise dns.exception.SyntaxError("NSEC with bit > 65535") - rdtypes.append(nrdtype) - rdtypes.sort() - window = 0 - octets = 0 - prior_rdtype = 0 - bitmap = bytearray(b'\0' * 32) - windows = [] - for nrdtype in rdtypes: - if nrdtype == prior_rdtype: - continue - prior_rdtype = nrdtype - new_window = nrdtype // 256 - if new_window != window: - windows.append((window, bitmap[0:octets])) - bitmap = bytearray(b'\0' * 32) - window = new_window - offset = nrdtype % 256 - byte = offset // 8 - bit = offset % 8 - octets = byte + 1 - bitmap[byte] = bitmap[byte] | (0x80 >> bit) - - windows.append((window, bitmap[0:octets])) - return cls(rdclass, rdtype, next, windows) - - def to_wire(self, file, compress=None, origin=None): - self.next.to_wire(file, None, origin) - for (window, bitmap) in self.windows: - file.write(struct.pack('!BB', window, len(bitmap))) - file.write(bitmap) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (next, cused) = dns.name.from_wire(wire[: current + rdlen], current) - current += cused - rdlen -= cused - windows = [] - while rdlen > 0: - if rdlen < 3: - raise dns.exception.FormError("NSEC too short") - window = wire[current] - octets = wire[current + 1] - if octets == 0 or octets > 32: - raise dns.exception.FormError("bad NSEC octets") - current += 2 - rdlen -= 2 - if rdlen < octets: - raise dns.exception.FormError("bad NSEC bitmap length") - bitmap = bytearray(wire[current: current + octets].unwrap()) - current += octets - rdlen -= octets - windows.append((window, bitmap)) - if origin is not None: - next = next.relativize(origin) - return cls(rdclass, rdtype, next, windows) - - def choose_relativity(self, origin=None, relativize=True): - self.next = self.next.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/ANY/NSEC3.py b/src/dns/rdtypes/ANY/NSEC3.py deleted file mode 100644 index 1c281c4a..00000000 --- a/src/dns/rdtypes/ANY/NSEC3.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import base64 -import binascii -import string -import struct - -import dns.exception -import dns.rdata -import dns.rdatatype -from dns._compat import xrange, text_type, PY3 - -# pylint: disable=deprecated-string-function -if PY3: - b32_hex_to_normal = bytes.maketrans(b'0123456789ABCDEFGHIJKLMNOPQRSTUV', - b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567') - b32_normal_to_hex = bytes.maketrans(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', - b'0123456789ABCDEFGHIJKLMNOPQRSTUV') -else: - b32_hex_to_normal = string.maketrans('0123456789ABCDEFGHIJKLMNOPQRSTUV', - 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567') - b32_normal_to_hex = string.maketrans('ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', - '0123456789ABCDEFGHIJKLMNOPQRSTUV') -# pylint: enable=deprecated-string-function - - -# hash algorithm constants -SHA1 = 1 - -# flag constants -OPTOUT = 1 - - -class NSEC3(dns.rdata.Rdata): - - """NSEC3 record - - @ivar algorithm: the hash algorithm number - @type algorithm: int - @ivar flags: the flags - @type flags: int - @ivar iterations: the number of iterations - @type iterations: int - @ivar salt: the salt - @type salt: string - @ivar next: the next name hash - @type next: string - @ivar windows: the windowed bitmap list - @type windows: list of (window number, string) tuples""" - - __slots__ = ['algorithm', 'flags', 'iterations', 'salt', 'next', 'windows'] - - def __init__(self, rdclass, rdtype, algorithm, flags, iterations, salt, - next, windows): - super(NSEC3, self).__init__(rdclass, rdtype) - self.algorithm = algorithm - self.flags = flags - self.iterations = iterations - if isinstance(salt, text_type): - self.salt = salt.encode() - else: - self.salt = salt - self.next = next - self.windows = windows - - def to_text(self, origin=None, relativize=True, **kw): - next = base64.b32encode(self.next).translate( - b32_normal_to_hex).lower().decode() - if self.salt == b'': - salt = '-' - else: - salt = binascii.hexlify(self.salt).decode() - text = u'' - for (window, bitmap) in self.windows: - bits = [] - for i in xrange(0, len(bitmap)): - byte = bitmap[i] - for j in xrange(0, 8): - if byte & (0x80 >> j): - bits.append(dns.rdatatype.to_text(window * 256 + - i * 8 + j)) - text += (u' ' + u' '.join(bits)) - return u'%u %u %u %s %s%s' % (self.algorithm, self.flags, - self.iterations, salt, next, text) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - algorithm = tok.get_uint8() - flags = tok.get_uint8() - iterations = tok.get_uint16() - salt = tok.get_string() - if salt == u'-': - salt = b'' - else: - salt = binascii.unhexlify(salt.encode('ascii')) - next = tok.get_string().encode( - 'ascii').upper().translate(b32_hex_to_normal) - next = base64.b32decode(next) - rdtypes = [] - while 1: - token = tok.get().unescape() - if token.is_eol_or_eof(): - break - nrdtype = dns.rdatatype.from_text(token.value) - if nrdtype == 0: - raise dns.exception.SyntaxError("NSEC3 with bit 0") - if nrdtype > 65535: - raise dns.exception.SyntaxError("NSEC3 with bit > 65535") - rdtypes.append(nrdtype) - rdtypes.sort() - window = 0 - octets = 0 - prior_rdtype = 0 - bitmap = bytearray(b'\0' * 32) - windows = [] - for nrdtype in rdtypes: - if nrdtype == prior_rdtype: - continue - prior_rdtype = nrdtype - new_window = nrdtype // 256 - if new_window != window: - if octets != 0: - windows.append((window, bitmap[0:octets])) - bitmap = bytearray(b'\0' * 32) - window = new_window - offset = nrdtype % 256 - byte = offset // 8 - bit = offset % 8 - octets = byte + 1 - bitmap[byte] = bitmap[byte] | (0x80 >> bit) - if octets != 0: - windows.append((window, bitmap[0:octets])) - return cls(rdclass, rdtype, algorithm, flags, iterations, salt, next, - windows) - - def to_wire(self, file, compress=None, origin=None): - l = len(self.salt) - file.write(struct.pack("!BBHB", self.algorithm, self.flags, - self.iterations, l)) - file.write(self.salt) - l = len(self.next) - file.write(struct.pack("!B", l)) - file.write(self.next) - for (window, bitmap) in self.windows: - file.write(struct.pack("!BB", window, len(bitmap))) - file.write(bitmap) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (algorithm, flags, iterations, slen) = \ - struct.unpack('!BBHB', wire[current: current + 5]) - - current += 5 - rdlen -= 5 - salt = wire[current: current + slen].unwrap() - current += slen - rdlen -= slen - nlen = wire[current] - current += 1 - rdlen -= 1 - next = wire[current: current + nlen].unwrap() - current += nlen - rdlen -= nlen - windows = [] - while rdlen > 0: - if rdlen < 3: - raise dns.exception.FormError("NSEC3 too short") - window = wire[current] - octets = wire[current + 1] - if octets == 0 or octets > 32: - raise dns.exception.FormError("bad NSEC3 octets") - current += 2 - rdlen -= 2 - if rdlen < octets: - raise dns.exception.FormError("bad NSEC3 bitmap length") - bitmap = bytearray(wire[current: current + octets].unwrap()) - current += octets - rdlen -= octets - windows.append((window, bitmap)) - return cls(rdclass, rdtype, algorithm, flags, iterations, salt, next, - windows) diff --git a/src/dns/rdtypes/ANY/NSEC3PARAM.py b/src/dns/rdtypes/ANY/NSEC3PARAM.py deleted file mode 100644 index 87c36e56..00000000 --- a/src/dns/rdtypes/ANY/NSEC3PARAM.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import binascii - -import dns.exception -import dns.rdata -from dns._compat import text_type - - -class NSEC3PARAM(dns.rdata.Rdata): - - """NSEC3PARAM record - - @ivar algorithm: the hash algorithm number - @type algorithm: int - @ivar flags: the flags - @type flags: int - @ivar iterations: the number of iterations - @type iterations: int - @ivar salt: the salt - @type salt: string""" - - __slots__ = ['algorithm', 'flags', 'iterations', 'salt'] - - def __init__(self, rdclass, rdtype, algorithm, flags, iterations, salt): - super(NSEC3PARAM, self).__init__(rdclass, rdtype) - self.algorithm = algorithm - self.flags = flags - self.iterations = iterations - if isinstance(salt, text_type): - self.salt = salt.encode() - else: - self.salt = salt - - def to_text(self, origin=None, relativize=True, **kw): - if self.salt == b'': - salt = '-' - else: - salt = binascii.hexlify(self.salt).decode() - return '%u %u %u %s' % (self.algorithm, self.flags, self.iterations, - salt) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - algorithm = tok.get_uint8() - flags = tok.get_uint8() - iterations = tok.get_uint16() - salt = tok.get_string() - if salt == '-': - salt = '' - else: - salt = binascii.unhexlify(salt.encode()) - tok.get_eol() - return cls(rdclass, rdtype, algorithm, flags, iterations, salt) - - def to_wire(self, file, compress=None, origin=None): - l = len(self.salt) - file.write(struct.pack("!BBHB", self.algorithm, self.flags, - self.iterations, l)) - file.write(self.salt) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (algorithm, flags, iterations, slen) = \ - struct.unpack('!BBHB', - wire[current: current + 5]) - current += 5 - rdlen -= 5 - salt = wire[current: current + slen].unwrap() - current += slen - rdlen -= slen - if rdlen != 0: - raise dns.exception.FormError - return cls(rdclass, rdtype, algorithm, flags, iterations, salt) diff --git a/src/dns/rdtypes/ANY/OPENPGPKEY.py b/src/dns/rdtypes/ANY/OPENPGPKEY.py deleted file mode 100644 index a066cf98..00000000 --- a/src/dns/rdtypes/ANY/OPENPGPKEY.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2016 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import base64 - -import dns.exception -import dns.rdata -import dns.tokenizer - -class OPENPGPKEY(dns.rdata.Rdata): - - """OPENPGPKEY record - - @ivar key: the key - @type key: bytes - @see: RFC 7929 - """ - - def __init__(self, rdclass, rdtype, key): - super(OPENPGPKEY, self).__init__(rdclass, rdtype) - self.key = key - - def to_text(self, origin=None, relativize=True, **kw): - return dns.rdata._base64ify(self.key) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - b64 = b''.join(chunks) - key = base64.b64decode(b64) - return cls(rdclass, rdtype, key) - - def to_wire(self, file, compress=None, origin=None): - file.write(self.key) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - key = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, key) diff --git a/src/dns/rdtypes/ANY/PTR.py b/src/dns/rdtypes/ANY/PTR.py deleted file mode 100644 index 20cd5076..00000000 --- a/src/dns/rdtypes/ANY/PTR.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.nsbase - - -class PTR(dns.rdtypes.nsbase.NSBase): - - """PTR record""" diff --git a/src/dns/rdtypes/ANY/RP.py b/src/dns/rdtypes/ANY/RP.py deleted file mode 100644 index 8f07be90..00000000 --- a/src/dns/rdtypes/ANY/RP.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.exception -import dns.rdata -import dns.name - - -class RP(dns.rdata.Rdata): - - """RP record - - @ivar mbox: The responsible person's mailbox - @type mbox: dns.name.Name object - @ivar txt: The owner name of a node with TXT records, or the root name - if no TXT records are associated with this RP. - @type txt: dns.name.Name object - @see: RFC 1183""" - - __slots__ = ['mbox', 'txt'] - - def __init__(self, rdclass, rdtype, mbox, txt): - super(RP, self).__init__(rdclass, rdtype) - self.mbox = mbox - self.txt = txt - - def to_text(self, origin=None, relativize=True, **kw): - mbox = self.mbox.choose_relativity(origin, relativize) - txt = self.txt.choose_relativity(origin, relativize) - return "{} {}".format(str(mbox), str(txt)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - mbox = tok.get_name() - txt = tok.get_name() - mbox = mbox.choose_relativity(origin, relativize) - txt = txt.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, mbox, txt) - - def to_wire(self, file, compress=None, origin=None): - self.mbox.to_wire(file, None, origin) - self.txt.to_wire(file, None, origin) - - def to_digestable(self, origin=None): - return self.mbox.to_digestable(origin) + \ - self.txt.to_digestable(origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (mbox, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - current += cused - rdlen -= cused - if rdlen <= 0: - raise dns.exception.FormError - (txt, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused != rdlen: - raise dns.exception.FormError - if origin is not None: - mbox = mbox.relativize(origin) - txt = txt.relativize(origin) - return cls(rdclass, rdtype, mbox, txt) - - def choose_relativity(self, origin=None, relativize=True): - self.mbox = self.mbox.choose_relativity(origin, relativize) - self.txt = self.txt.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/ANY/RRSIG.py b/src/dns/rdtypes/ANY/RRSIG.py deleted file mode 100644 index d3756ece..00000000 --- a/src/dns/rdtypes/ANY/RRSIG.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import base64 -import calendar -import struct -import time - -import dns.dnssec -import dns.exception -import dns.rdata -import dns.rdatatype - - -class BadSigTime(dns.exception.DNSException): - - """Time in DNS SIG or RRSIG resource record cannot be parsed.""" - - -def sigtime_to_posixtime(what): - if len(what) != 14: - raise BadSigTime - year = int(what[0:4]) - month = int(what[4:6]) - day = int(what[6:8]) - hour = int(what[8:10]) - minute = int(what[10:12]) - second = int(what[12:14]) - return calendar.timegm((year, month, day, hour, minute, second, - 0, 0, 0)) - - -def posixtime_to_sigtime(what): - return time.strftime('%Y%m%d%H%M%S', time.gmtime(what)) - - -class RRSIG(dns.rdata.Rdata): - - """RRSIG record - - @ivar type_covered: the rdata type this signature covers - @type type_covered: int - @ivar algorithm: the algorithm used for the sig - @type algorithm: int - @ivar labels: number of labels - @type labels: int - @ivar original_ttl: the original TTL - @type original_ttl: long - @ivar expiration: signature expiration time - @type expiration: long - @ivar inception: signature inception time - @type inception: long - @ivar key_tag: the key tag - @type key_tag: int - @ivar signer: the signer - @type signer: dns.name.Name object - @ivar signature: the signature - @type signature: string""" - - __slots__ = ['type_covered', 'algorithm', 'labels', 'original_ttl', - 'expiration', 'inception', 'key_tag', 'signer', - 'signature'] - - def __init__(self, rdclass, rdtype, type_covered, algorithm, labels, - original_ttl, expiration, inception, key_tag, signer, - signature): - super(RRSIG, self).__init__(rdclass, rdtype) - self.type_covered = type_covered - self.algorithm = algorithm - self.labels = labels - self.original_ttl = original_ttl - self.expiration = expiration - self.inception = inception - self.key_tag = key_tag - self.signer = signer - self.signature = signature - - def covers(self): - return self.type_covered - - def to_text(self, origin=None, relativize=True, **kw): - return '%s %d %d %d %s %s %d %s %s' % ( - dns.rdatatype.to_text(self.type_covered), - self.algorithm, - self.labels, - self.original_ttl, - posixtime_to_sigtime(self.expiration), - posixtime_to_sigtime(self.inception), - self.key_tag, - self.signer.choose_relativity(origin, relativize), - dns.rdata._base64ify(self.signature) - ) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - type_covered = dns.rdatatype.from_text(tok.get_string()) - algorithm = dns.dnssec.algorithm_from_text(tok.get_string()) - labels = tok.get_int() - original_ttl = tok.get_ttl() - expiration = sigtime_to_posixtime(tok.get_string()) - inception = sigtime_to_posixtime(tok.get_string()) - key_tag = tok.get_int() - signer = tok.get_name() - signer = signer.choose_relativity(origin, relativize) - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - b64 = b''.join(chunks) - signature = base64.b64decode(b64) - return cls(rdclass, rdtype, type_covered, algorithm, labels, - original_ttl, expiration, inception, key_tag, signer, - signature) - - def to_wire(self, file, compress=None, origin=None): - header = struct.pack('!HBBIIIH', self.type_covered, - self.algorithm, self.labels, - self.original_ttl, self.expiration, - self.inception, self.key_tag) - file.write(header) - self.signer.to_wire(file, None, origin) - file.write(self.signature) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - header = struct.unpack('!HBBIIIH', wire[current: current + 18]) - current += 18 - rdlen -= 18 - (signer, cused) = dns.name.from_wire(wire[: current + rdlen], current) - current += cused - rdlen -= cused - if origin is not None: - signer = signer.relativize(origin) - signature = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, header[0], header[1], header[2], - header[3], header[4], header[5], header[6], signer, - signature) - - def choose_relativity(self, origin=None, relativize=True): - self.signer = self.signer.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/ANY/RT.py b/src/dns/rdtypes/ANY/RT.py deleted file mode 100644 index d0feb79e..00000000 --- a/src/dns/rdtypes/ANY/RT.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.mxbase - - -class RT(dns.rdtypes.mxbase.UncompressedDowncasingMX): - - """RT record""" diff --git a/src/dns/rdtypes/ANY/SOA.py b/src/dns/rdtypes/ANY/SOA.py deleted file mode 100644 index aec81cad..00000000 --- a/src/dns/rdtypes/ANY/SOA.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.name - - -class SOA(dns.rdata.Rdata): - - """SOA record - - @ivar mname: the SOA MNAME (master name) field - @type mname: dns.name.Name object - @ivar rname: the SOA RNAME (responsible name) field - @type rname: dns.name.Name object - @ivar serial: The zone's serial number - @type serial: int - @ivar refresh: The zone's refresh value (in seconds) - @type refresh: int - @ivar retry: The zone's retry value (in seconds) - @type retry: int - @ivar expire: The zone's expiration value (in seconds) - @type expire: int - @ivar minimum: The zone's negative caching time (in seconds, called - "minimum" for historical reasons) - @type minimum: int - @see: RFC 1035""" - - __slots__ = ['mname', 'rname', 'serial', 'refresh', 'retry', 'expire', - 'minimum'] - - def __init__(self, rdclass, rdtype, mname, rname, serial, refresh, retry, - expire, minimum): - super(SOA, self).__init__(rdclass, rdtype) - self.mname = mname - self.rname = rname - self.serial = serial - self.refresh = refresh - self.retry = retry - self.expire = expire - self.minimum = minimum - - def to_text(self, origin=None, relativize=True, **kw): - mname = self.mname.choose_relativity(origin, relativize) - rname = self.rname.choose_relativity(origin, relativize) - return '%s %s %d %d %d %d %d' % ( - mname, rname, self.serial, self.refresh, self.retry, - self.expire, self.minimum) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - mname = tok.get_name() - rname = tok.get_name() - mname = mname.choose_relativity(origin, relativize) - rname = rname.choose_relativity(origin, relativize) - serial = tok.get_uint32() - refresh = tok.get_ttl() - retry = tok.get_ttl() - expire = tok.get_ttl() - minimum = tok.get_ttl() - tok.get_eol() - return cls(rdclass, rdtype, mname, rname, serial, refresh, retry, - expire, minimum) - - def to_wire(self, file, compress=None, origin=None): - self.mname.to_wire(file, compress, origin) - self.rname.to_wire(file, compress, origin) - five_ints = struct.pack('!IIIII', self.serial, self.refresh, - self.retry, self.expire, self.minimum) - file.write(five_ints) - - def to_digestable(self, origin=None): - return self.mname.to_digestable(origin) + \ - self.rname.to_digestable(origin) + \ - struct.pack('!IIIII', self.serial, self.refresh, - self.retry, self.expire, self.minimum) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (mname, cused) = dns.name.from_wire(wire[: current + rdlen], current) - current += cused - rdlen -= cused - (rname, cused) = dns.name.from_wire(wire[: current + rdlen], current) - current += cused - rdlen -= cused - if rdlen != 20: - raise dns.exception.FormError - five_ints = struct.unpack('!IIIII', - wire[current: current + rdlen]) - if origin is not None: - mname = mname.relativize(origin) - rname = rname.relativize(origin) - return cls(rdclass, rdtype, mname, rname, - five_ints[0], five_ints[1], five_ints[2], five_ints[3], - five_ints[4]) - - def choose_relativity(self, origin=None, relativize=True): - self.mname = self.mname.choose_relativity(origin, relativize) - self.rname = self.rname.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/ANY/SPF.py b/src/dns/rdtypes/ANY/SPF.py deleted file mode 100644 index 41dee623..00000000 --- a/src/dns/rdtypes/ANY/SPF.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2006, 2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.txtbase - - -class SPF(dns.rdtypes.txtbase.TXTBase): - - """SPF record - - @see: RFC 4408""" diff --git a/src/dns/rdtypes/ANY/SSHFP.py b/src/dns/rdtypes/ANY/SSHFP.py deleted file mode 100644 index c18311e9..00000000 --- a/src/dns/rdtypes/ANY/SSHFP.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2005-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import binascii - -import dns.rdata -import dns.rdatatype - - -class SSHFP(dns.rdata.Rdata): - - """SSHFP record - - @ivar algorithm: the algorithm - @type algorithm: int - @ivar fp_type: the digest type - @type fp_type: int - @ivar fingerprint: the fingerprint - @type fingerprint: string - @see: draft-ietf-secsh-dns-05.txt""" - - __slots__ = ['algorithm', 'fp_type', 'fingerprint'] - - def __init__(self, rdclass, rdtype, algorithm, fp_type, - fingerprint): - super(SSHFP, self).__init__(rdclass, rdtype) - self.algorithm = algorithm - self.fp_type = fp_type - self.fingerprint = fingerprint - - def to_text(self, origin=None, relativize=True, **kw): - return '%d %d %s' % (self.algorithm, - self.fp_type, - dns.rdata._hexify(self.fingerprint, - chunksize=128)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - algorithm = tok.get_uint8() - fp_type = tok.get_uint8() - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - fingerprint = b''.join(chunks) - fingerprint = binascii.unhexlify(fingerprint) - return cls(rdclass, rdtype, algorithm, fp_type, fingerprint) - - def to_wire(self, file, compress=None, origin=None): - header = struct.pack("!BB", self.algorithm, self.fp_type) - file.write(header) - file.write(self.fingerprint) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - header = struct.unpack("!BB", wire[current: current + 2]) - current += 2 - rdlen -= 2 - fingerprint = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, header[0], header[1], fingerprint) diff --git a/src/dns/rdtypes/ANY/TLSA.py b/src/dns/rdtypes/ANY/TLSA.py deleted file mode 100644 index a135c2b3..00000000 --- a/src/dns/rdtypes/ANY/TLSA.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2005-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import binascii - -import dns.rdata -import dns.rdatatype - - -class TLSA(dns.rdata.Rdata): - - """TLSA record - - @ivar usage: The certificate usage - @type usage: int - @ivar selector: The selector field - @type selector: int - @ivar mtype: The 'matching type' field - @type mtype: int - @ivar cert: The 'Certificate Association Data' field - @type cert: string - @see: RFC 6698""" - - __slots__ = ['usage', 'selector', 'mtype', 'cert'] - - def __init__(self, rdclass, rdtype, usage, selector, - mtype, cert): - super(TLSA, self).__init__(rdclass, rdtype) - self.usage = usage - self.selector = selector - self.mtype = mtype - self.cert = cert - - def to_text(self, origin=None, relativize=True, **kw): - return '%d %d %d %s' % (self.usage, - self.selector, - self.mtype, - dns.rdata._hexify(self.cert, - chunksize=128)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - usage = tok.get_uint8() - selector = tok.get_uint8() - mtype = tok.get_uint8() - cert_chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - cert_chunks.append(t.value.encode()) - cert = b''.join(cert_chunks) - cert = binascii.unhexlify(cert) - return cls(rdclass, rdtype, usage, selector, mtype, cert) - - def to_wire(self, file, compress=None, origin=None): - header = struct.pack("!BBB", self.usage, self.selector, self.mtype) - file.write(header) - file.write(self.cert) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - header = struct.unpack("!BBB", wire[current: current + 3]) - current += 3 - rdlen -= 3 - cert = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, header[0], header[1], header[2], cert) diff --git a/src/dns/rdtypes/ANY/TXT.py b/src/dns/rdtypes/ANY/TXT.py deleted file mode 100644 index c5ae919c..00000000 --- a/src/dns/rdtypes/ANY/TXT.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.txtbase - - -class TXT(dns.rdtypes.txtbase.TXTBase): - - """TXT record""" diff --git a/src/dns/rdtypes/ANY/URI.py b/src/dns/rdtypes/ANY/URI.py deleted file mode 100644 index f5b65ed6..00000000 --- a/src/dns/rdtypes/ANY/URI.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# Copyright (C) 2015 Red Hat, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.name -from dns._compat import text_type - - -class URI(dns.rdata.Rdata): - - """URI record - - @ivar priority: the priority - @type priority: int - @ivar weight: the weight - @type weight: int - @ivar target: the target host - @type target: dns.name.Name object - @see: draft-faltstrom-uri-13""" - - __slots__ = ['priority', 'weight', 'target'] - - def __init__(self, rdclass, rdtype, priority, weight, target): - super(URI, self).__init__(rdclass, rdtype) - self.priority = priority - self.weight = weight - if len(target) < 1: - raise dns.exception.SyntaxError("URI target cannot be empty") - if isinstance(target, text_type): - self.target = target.encode() - else: - self.target = target - - def to_text(self, origin=None, relativize=True, **kw): - return '%d %d "%s"' % (self.priority, self.weight, - self.target.decode()) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - priority = tok.get_uint16() - weight = tok.get_uint16() - target = tok.get().unescape() - if not (target.is_quoted_string() or target.is_identifier()): - raise dns.exception.SyntaxError("URI target must be a string") - tok.get_eol() - return cls(rdclass, rdtype, priority, weight, target.value) - - def to_wire(self, file, compress=None, origin=None): - two_ints = struct.pack("!HH", self.priority, self.weight) - file.write(two_ints) - file.write(self.target) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - if rdlen < 5: - raise dns.exception.FormError('URI RR is shorter than 5 octets') - - (priority, weight) = struct.unpack('!HH', wire[current: current + 4]) - current += 4 - rdlen -= 4 - target = wire[current: current + rdlen] - current += rdlen - - return cls(rdclass, rdtype, priority, weight, target) diff --git a/src/dns/rdtypes/ANY/X25.py b/src/dns/rdtypes/ANY/X25.py deleted file mode 100644 index e530a2c2..00000000 --- a/src/dns/rdtypes/ANY/X25.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.tokenizer -from dns._compat import text_type - - -class X25(dns.rdata.Rdata): - - """X25 record - - @ivar address: the PSDN address - @type address: string - @see: RFC 1183""" - - __slots__ = ['address'] - - def __init__(self, rdclass, rdtype, address): - super(X25, self).__init__(rdclass, rdtype) - if isinstance(address, text_type): - self.address = address.encode() - else: - self.address = address - - def to_text(self, origin=None, relativize=True, **kw): - return '"%s"' % dns.rdata._escapify(self.address) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - address = tok.get_string() - tok.get_eol() - return cls(rdclass, rdtype, address) - - def to_wire(self, file, compress=None, origin=None): - l = len(self.address) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(self.address) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - l = wire[current] - current += 1 - rdlen -= 1 - if l != rdlen: - raise dns.exception.FormError - address = wire[current: current + l].unwrap() - return cls(rdclass, rdtype, address) diff --git a/src/dns/rdtypes/ANY/__init__.py b/src/dns/rdtypes/ANY/__init__.py deleted file mode 100644 index ca41ef80..00000000 --- a/src/dns/rdtypes/ANY/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Class ANY (generic) rdata type classes.""" - -__all__ = [ - 'AFSDB', - 'AVC', - 'CAA', - 'CDNSKEY', - 'CDS', - 'CERT', - 'CNAME', - 'CSYNC', - 'DLV', - 'DNAME', - 'DNSKEY', - 'DS', - 'EUI48', - 'EUI64', - 'GPOS', - 'HINFO', - 'HIP', - 'ISDN', - 'LOC', - 'MX', - 'NS', - 'NSEC', - 'NSEC3', - 'NSEC3PARAM', - 'OPENPGPKEY', - 'PTR', - 'RP', - 'RRSIG', - 'RT', - 'SOA', - 'SPF', - 'SSHFP', - 'TLSA', - 'TXT', - 'URI', - 'X25', -] diff --git a/src/dns/rdtypes/CH/A.py b/src/dns/rdtypes/CH/A.py deleted file mode 100644 index e65d192d..00000000 --- a/src/dns/rdtypes/CH/A.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.mxbase -import struct - -class A(dns.rdtypes.mxbase.MXBase): - - """A record for Chaosnet - @ivar domain: the domain of the address - @type domain: dns.name.Name object - @ivar address: the 16-bit address - @type address: int""" - - __slots__ = ['domain', 'address'] - - def __init__(self, rdclass, rdtype, address, domain): - super(A, self).__init__(rdclass, rdtype, address, domain) - self.domain = domain - self.address = address - - def to_text(self, origin=None, relativize=True, **kw): - domain = self.domain.choose_relativity(origin, relativize) - return '%s %o' % (domain, self.address) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - domain = tok.get_name() - address = tok.get_uint16(base=8) - domain = domain.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, address, domain) - - def to_wire(self, file, compress=None, origin=None): - self.domain.to_wire(file, compress, origin) - pref = struct.pack("!H", self.address) - file.write(pref) - - def to_digestable(self, origin=None): - return self.domain.to_digestable(origin) + \ - struct.pack("!H", self.address) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (domain, cused) = dns.name.from_wire(wire[: current + rdlen-2], - current) - current += cused - (address,) = struct.unpack('!H', wire[current: current + 2]) - if cused+2 != rdlen: - raise dns.exception.FormError - if origin is not None: - domain = domain.relativize(origin) - return cls(rdclass, rdtype, address, domain) - - def choose_relativity(self, origin=None, relativize=True): - self.domain = self.domain.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/CH/__init__.py b/src/dns/rdtypes/CH/__init__.py deleted file mode 100644 index 7184a733..00000000 --- a/src/dns/rdtypes/CH/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Class CH rdata type classes.""" - -__all__ = [ - 'A', -] diff --git a/src/dns/rdtypes/IN/A.py b/src/dns/rdtypes/IN/A.py deleted file mode 100644 index 89989824..00000000 --- a/src/dns/rdtypes/IN/A.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.exception -import dns.ipv4 -import dns.rdata -import dns.tokenizer - - -class A(dns.rdata.Rdata): - - """A record. - - @ivar address: an IPv4 address - @type address: string (in the standard "dotted quad" format)""" - - __slots__ = ['address'] - - def __init__(self, rdclass, rdtype, address): - super(A, self).__init__(rdclass, rdtype) - # check that it's OK - dns.ipv4.inet_aton(address) - self.address = address - - def to_text(self, origin=None, relativize=True, **kw): - return self.address - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - address = tok.get_identifier() - tok.get_eol() - return cls(rdclass, rdtype, address) - - def to_wire(self, file, compress=None, origin=None): - file.write(dns.ipv4.inet_aton(self.address)) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - address = dns.ipv4.inet_ntoa(wire[current: current + rdlen]) - return cls(rdclass, rdtype, address) diff --git a/src/dns/rdtypes/IN/AAAA.py b/src/dns/rdtypes/IN/AAAA.py deleted file mode 100644 index a77c5bf2..00000000 --- a/src/dns/rdtypes/IN/AAAA.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.exception -import dns.inet -import dns.rdata -import dns.tokenizer - - -class AAAA(dns.rdata.Rdata): - - """AAAA record. - - @ivar address: an IPv6 address - @type address: string (in the standard IPv6 format)""" - - __slots__ = ['address'] - - def __init__(self, rdclass, rdtype, address): - super(AAAA, self).__init__(rdclass, rdtype) - # check that it's OK - dns.inet.inet_pton(dns.inet.AF_INET6, address) - self.address = address - - def to_text(self, origin=None, relativize=True, **kw): - return self.address - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - address = tok.get_identifier() - tok.get_eol() - return cls(rdclass, rdtype, address) - - def to_wire(self, file, compress=None, origin=None): - file.write(dns.inet.inet_pton(dns.inet.AF_INET6, self.address)) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - address = dns.inet.inet_ntop(dns.inet.AF_INET6, - wire[current: current + rdlen]) - return cls(rdclass, rdtype, address) diff --git a/src/dns/rdtypes/IN/APL.py b/src/dns/rdtypes/IN/APL.py deleted file mode 100644 index 48faf88a..00000000 --- a/src/dns/rdtypes/IN/APL.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import binascii -import codecs -import struct - -import dns.exception -import dns.inet -import dns.rdata -import dns.tokenizer -from dns._compat import xrange, maybe_chr - - -class APLItem(object): - - """An APL list item. - - @ivar family: the address family (IANA address family registry) - @type family: int - @ivar negation: is this item negated? - @type negation: bool - @ivar address: the address - @type address: string - @ivar prefix: the prefix length - @type prefix: int - """ - - __slots__ = ['family', 'negation', 'address', 'prefix'] - - def __init__(self, family, negation, address, prefix): - self.family = family - self.negation = negation - self.address = address - self.prefix = prefix - - def __str__(self): - if self.negation: - return "!%d:%s/%s" % (self.family, self.address, self.prefix) - else: - return "%d:%s/%s" % (self.family, self.address, self.prefix) - - def to_wire(self, file): - if self.family == 1: - address = dns.inet.inet_pton(dns.inet.AF_INET, self.address) - elif self.family == 2: - address = dns.inet.inet_pton(dns.inet.AF_INET6, self.address) - else: - address = binascii.unhexlify(self.address) - # - # Truncate least significant zero bytes. - # - last = 0 - for i in xrange(len(address) - 1, -1, -1): - if address[i] != maybe_chr(0): - last = i + 1 - break - address = address[0: last] - l = len(address) - assert l < 128 - if self.negation: - l |= 0x80 - header = struct.pack('!HBB', self.family, self.prefix, l) - file.write(header) - file.write(address) - - -class APL(dns.rdata.Rdata): - - """APL record. - - @ivar items: a list of APL items - @type items: list of APL_Item - @see: RFC 3123""" - - __slots__ = ['items'] - - def __init__(self, rdclass, rdtype, items): - super(APL, self).__init__(rdclass, rdtype) - self.items = items - - def to_text(self, origin=None, relativize=True, **kw): - return ' '.join(map(str, self.items)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - items = [] - while 1: - token = tok.get().unescape() - if token.is_eol_or_eof(): - break - item = token.value - if item[0] == '!': - negation = True - item = item[1:] - else: - negation = False - (family, rest) = item.split(':', 1) - family = int(family) - (address, prefix) = rest.split('/', 1) - prefix = int(prefix) - item = APLItem(family, negation, address, prefix) - items.append(item) - - return cls(rdclass, rdtype, items) - - def to_wire(self, file, compress=None, origin=None): - for item in self.items: - item.to_wire(file) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - - items = [] - while 1: - if rdlen == 0: - break - if rdlen < 4: - raise dns.exception.FormError - header = struct.unpack('!HBB', wire[current: current + 4]) - afdlen = header[2] - if afdlen > 127: - negation = True - afdlen -= 128 - else: - negation = False - current += 4 - rdlen -= 4 - if rdlen < afdlen: - raise dns.exception.FormError - address = wire[current: current + afdlen].unwrap() - l = len(address) - if header[0] == 1: - if l < 4: - address += b'\x00' * (4 - l) - address = dns.inet.inet_ntop(dns.inet.AF_INET, address) - elif header[0] == 2: - if l < 16: - address += b'\x00' * (16 - l) - address = dns.inet.inet_ntop(dns.inet.AF_INET6, address) - else: - # - # This isn't really right according to the RFC, but it - # seems better than throwing an exception - # - address = codecs.encode(address, 'hex_codec') - current += afdlen - rdlen -= afdlen - item = APLItem(header[0], negation, address, header[1]) - items.append(item) - return cls(rdclass, rdtype, items) diff --git a/src/dns/rdtypes/IN/DHCID.py b/src/dns/rdtypes/IN/DHCID.py deleted file mode 100644 index cec64590..00000000 --- a/src/dns/rdtypes/IN/DHCID.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2006, 2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import base64 - -import dns.exception - - -class DHCID(dns.rdata.Rdata): - - """DHCID record - - @ivar data: the data (the content of the RR is opaque as far as the - DNS is concerned) - @type data: string - @see: RFC 4701""" - - __slots__ = ['data'] - - def __init__(self, rdclass, rdtype, data): - super(DHCID, self).__init__(rdclass, rdtype) - self.data = data - - def to_text(self, origin=None, relativize=True, **kw): - return dns.rdata._base64ify(self.data) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - b64 = b''.join(chunks) - data = base64.b64decode(b64) - return cls(rdclass, rdtype, data) - - def to_wire(self, file, compress=None, origin=None): - file.write(self.data) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - data = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, data) diff --git a/src/dns/rdtypes/IN/IPSECKEY.py b/src/dns/rdtypes/IN/IPSECKEY.py deleted file mode 100644 index 8f49ba13..00000000 --- a/src/dns/rdtypes/IN/IPSECKEY.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2006, 2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import base64 - -import dns.exception -import dns.inet -import dns.name - - -class IPSECKEY(dns.rdata.Rdata): - - """IPSECKEY record - - @ivar precedence: the precedence for this key data - @type precedence: int - @ivar gateway_type: the gateway type - @type gateway_type: int - @ivar algorithm: the algorithm to use - @type algorithm: int - @ivar gateway: the public key - @type gateway: None, IPv4 address, IPV6 address, or domain name - @ivar key: the public key - @type key: string - @see: RFC 4025""" - - __slots__ = ['precedence', 'gateway_type', 'algorithm', 'gateway', 'key'] - - def __init__(self, rdclass, rdtype, precedence, gateway_type, algorithm, - gateway, key): - super(IPSECKEY, self).__init__(rdclass, rdtype) - if gateway_type == 0: - if gateway != '.' and gateway is not None: - raise SyntaxError('invalid gateway for gateway type 0') - gateway = None - elif gateway_type == 1: - # check that it's OK - dns.inet.inet_pton(dns.inet.AF_INET, gateway) - elif gateway_type == 2: - # check that it's OK - dns.inet.inet_pton(dns.inet.AF_INET6, gateway) - elif gateway_type == 3: - pass - else: - raise SyntaxError( - 'invalid IPSECKEY gateway type: %d' % gateway_type) - self.precedence = precedence - self.gateway_type = gateway_type - self.algorithm = algorithm - self.gateway = gateway - self.key = key - - def to_text(self, origin=None, relativize=True, **kw): - if self.gateway_type == 0: - gateway = '.' - elif self.gateway_type == 1: - gateway = self.gateway - elif self.gateway_type == 2: - gateway = self.gateway - elif self.gateway_type == 3: - gateway = str(self.gateway.choose_relativity(origin, relativize)) - else: - raise ValueError('invalid gateway type') - return '%d %d %d %s %s' % (self.precedence, self.gateway_type, - self.algorithm, gateway, - dns.rdata._base64ify(self.key)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - precedence = tok.get_uint8() - gateway_type = tok.get_uint8() - algorithm = tok.get_uint8() - if gateway_type == 3: - gateway = tok.get_name().choose_relativity(origin, relativize) - else: - gateway = tok.get_string() - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - b64 = b''.join(chunks) - key = base64.b64decode(b64) - return cls(rdclass, rdtype, precedence, gateway_type, algorithm, - gateway, key) - - def to_wire(self, file, compress=None, origin=None): - header = struct.pack("!BBB", self.precedence, self.gateway_type, - self.algorithm) - file.write(header) - if self.gateway_type == 0: - pass - elif self.gateway_type == 1: - file.write(dns.inet.inet_pton(dns.inet.AF_INET, self.gateway)) - elif self.gateway_type == 2: - file.write(dns.inet.inet_pton(dns.inet.AF_INET6, self.gateway)) - elif self.gateway_type == 3: - self.gateway.to_wire(file, None, origin) - else: - raise ValueError('invalid gateway type') - file.write(self.key) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - if rdlen < 3: - raise dns.exception.FormError - header = struct.unpack('!BBB', wire[current: current + 3]) - gateway_type = header[1] - current += 3 - rdlen -= 3 - if gateway_type == 0: - gateway = None - elif gateway_type == 1: - gateway = dns.inet.inet_ntop(dns.inet.AF_INET, - wire[current: current + 4]) - current += 4 - rdlen -= 4 - elif gateway_type == 2: - gateway = dns.inet.inet_ntop(dns.inet.AF_INET6, - wire[current: current + 16]) - current += 16 - rdlen -= 16 - elif gateway_type == 3: - (gateway, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - current += cused - rdlen -= cused - else: - raise dns.exception.FormError('invalid IPSECKEY gateway type') - key = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, header[0], gateway_type, header[2], - gateway, key) diff --git a/src/dns/rdtypes/IN/KX.py b/src/dns/rdtypes/IN/KX.py deleted file mode 100644 index 1318a582..00000000 --- a/src/dns/rdtypes/IN/KX.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.mxbase - - -class KX(dns.rdtypes.mxbase.UncompressedMX): - - """KX record""" diff --git a/src/dns/rdtypes/IN/NAPTR.py b/src/dns/rdtypes/IN/NAPTR.py deleted file mode 100644 index 32fa4745..00000000 --- a/src/dns/rdtypes/IN/NAPTR.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.name -import dns.rdata -from dns._compat import xrange, text_type - - -def _write_string(file, s): - l = len(s) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(s) - - -def _sanitize(value): - if isinstance(value, text_type): - return value.encode() - return value - - -class NAPTR(dns.rdata.Rdata): - - """NAPTR record - - @ivar order: order - @type order: int - @ivar preference: preference - @type preference: int - @ivar flags: flags - @type flags: string - @ivar service: service - @type service: string - @ivar regexp: regular expression - @type regexp: string - @ivar replacement: replacement name - @type replacement: dns.name.Name object - @see: RFC 3403""" - - __slots__ = ['order', 'preference', 'flags', 'service', 'regexp', - 'replacement'] - - def __init__(self, rdclass, rdtype, order, preference, flags, service, - regexp, replacement): - super(NAPTR, self).__init__(rdclass, rdtype) - self.flags = _sanitize(flags) - self.service = _sanitize(service) - self.regexp = _sanitize(regexp) - self.order = order - self.preference = preference - self.replacement = replacement - - def to_text(self, origin=None, relativize=True, **kw): - replacement = self.replacement.choose_relativity(origin, relativize) - return '%d %d "%s" "%s" "%s" %s' % \ - (self.order, self.preference, - dns.rdata._escapify(self.flags), - dns.rdata._escapify(self.service), - dns.rdata._escapify(self.regexp), - replacement) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - order = tok.get_uint16() - preference = tok.get_uint16() - flags = tok.get_string() - service = tok.get_string() - regexp = tok.get_string() - replacement = tok.get_name() - replacement = replacement.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, order, preference, flags, service, - regexp, replacement) - - def to_wire(self, file, compress=None, origin=None): - two_ints = struct.pack("!HH", self.order, self.preference) - file.write(two_ints) - _write_string(file, self.flags) - _write_string(file, self.service) - _write_string(file, self.regexp) - self.replacement.to_wire(file, compress, origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (order, preference) = struct.unpack('!HH', wire[current: current + 4]) - current += 4 - rdlen -= 4 - strings = [] - for i in xrange(3): - l = wire[current] - current += 1 - rdlen -= 1 - if l > rdlen or rdlen < 0: - raise dns.exception.FormError - s = wire[current: current + l].unwrap() - current += l - rdlen -= l - strings.append(s) - (replacement, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused != rdlen: - raise dns.exception.FormError - if origin is not None: - replacement = replacement.relativize(origin) - return cls(rdclass, rdtype, order, preference, strings[0], strings[1], - strings[2], replacement) - - def choose_relativity(self, origin=None, relativize=True): - self.replacement = self.replacement.choose_relativity(origin, - relativize) diff --git a/src/dns/rdtypes/IN/NSAP.py b/src/dns/rdtypes/IN/NSAP.py deleted file mode 100644 index 336befc7..00000000 --- a/src/dns/rdtypes/IN/NSAP.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import binascii - -import dns.exception -import dns.rdata -import dns.tokenizer - - -class NSAP(dns.rdata.Rdata): - - """NSAP record. - - @ivar address: a NASP - @type address: string - @see: RFC 1706""" - - __slots__ = ['address'] - - def __init__(self, rdclass, rdtype, address): - super(NSAP, self).__init__(rdclass, rdtype) - self.address = address - - def to_text(self, origin=None, relativize=True, **kw): - return "0x%s" % binascii.hexlify(self.address).decode() - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - address = tok.get_string() - tok.get_eol() - if address[0:2] != '0x': - raise dns.exception.SyntaxError('string does not start with 0x') - address = address[2:].replace('.', '') - if len(address) % 2 != 0: - raise dns.exception.SyntaxError('hexstring has odd length') - address = binascii.unhexlify(address.encode()) - return cls(rdclass, rdtype, address) - - def to_wire(self, file, compress=None, origin=None): - file.write(self.address) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - address = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, address) diff --git a/src/dns/rdtypes/IN/NSAP_PTR.py b/src/dns/rdtypes/IN/NSAP_PTR.py deleted file mode 100644 index a5b66c80..00000000 --- a/src/dns/rdtypes/IN/NSAP_PTR.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import dns.rdtypes.nsbase - - -class NSAP_PTR(dns.rdtypes.nsbase.UncompressedNS): - - """NSAP-PTR record""" diff --git a/src/dns/rdtypes/IN/PX.py b/src/dns/rdtypes/IN/PX.py deleted file mode 100644 index 2dbaee6c..00000000 --- a/src/dns/rdtypes/IN/PX.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.name - - -class PX(dns.rdata.Rdata): - - """PX record. - - @ivar preference: the preference value - @type preference: int - @ivar map822: the map822 name - @type map822: dns.name.Name object - @ivar mapx400: the mapx400 name - @type mapx400: dns.name.Name object - @see: RFC 2163""" - - __slots__ = ['preference', 'map822', 'mapx400'] - - def __init__(self, rdclass, rdtype, preference, map822, mapx400): - super(PX, self).__init__(rdclass, rdtype) - self.preference = preference - self.map822 = map822 - self.mapx400 = mapx400 - - def to_text(self, origin=None, relativize=True, **kw): - map822 = self.map822.choose_relativity(origin, relativize) - mapx400 = self.mapx400.choose_relativity(origin, relativize) - return '%d %s %s' % (self.preference, map822, mapx400) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - preference = tok.get_uint16() - map822 = tok.get_name() - map822 = map822.choose_relativity(origin, relativize) - mapx400 = tok.get_name(None) - mapx400 = mapx400.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, preference, map822, mapx400) - - def to_wire(self, file, compress=None, origin=None): - pref = struct.pack("!H", self.preference) - file.write(pref) - self.map822.to_wire(file, None, origin) - self.mapx400.to_wire(file, None, origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (preference, ) = struct.unpack('!H', wire[current: current + 2]) - current += 2 - rdlen -= 2 - (map822, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused > rdlen: - raise dns.exception.FormError - current += cused - rdlen -= cused - if origin is not None: - map822 = map822.relativize(origin) - (mapx400, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused != rdlen: - raise dns.exception.FormError - if origin is not None: - mapx400 = mapx400.relativize(origin) - return cls(rdclass, rdtype, preference, map822, mapx400) - - def choose_relativity(self, origin=None, relativize=True): - self.map822 = self.map822.choose_relativity(origin, relativize) - self.mapx400 = self.mapx400.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/IN/SRV.py b/src/dns/rdtypes/IN/SRV.py deleted file mode 100644 index b2c1bc9f..00000000 --- a/src/dns/rdtypes/IN/SRV.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct - -import dns.exception -import dns.rdata -import dns.name - - -class SRV(dns.rdata.Rdata): - - """SRV record - - @ivar priority: the priority - @type priority: int - @ivar weight: the weight - @type weight: int - @ivar port: the port of the service - @type port: int - @ivar target: the target host - @type target: dns.name.Name object - @see: RFC 2782""" - - __slots__ = ['priority', 'weight', 'port', 'target'] - - def __init__(self, rdclass, rdtype, priority, weight, port, target): - super(SRV, self).__init__(rdclass, rdtype) - self.priority = priority - self.weight = weight - self.port = port - self.target = target - - def to_text(self, origin=None, relativize=True, **kw): - target = self.target.choose_relativity(origin, relativize) - return '%d %d %d %s' % (self.priority, self.weight, self.port, - target) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - priority = tok.get_uint16() - weight = tok.get_uint16() - port = tok.get_uint16() - target = tok.get_name(None) - target = target.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, priority, weight, port, target) - - def to_wire(self, file, compress=None, origin=None): - three_ints = struct.pack("!HHH", self.priority, self.weight, self.port) - file.write(three_ints) - self.target.to_wire(file, compress, origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (priority, weight, port) = struct.unpack('!HHH', - wire[current: current + 6]) - current += 6 - rdlen -= 6 - (target, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused != rdlen: - raise dns.exception.FormError - if origin is not None: - target = target.relativize(origin) - return cls(rdclass, rdtype, priority, weight, port, target) - - def choose_relativity(self, origin=None, relativize=True): - self.target = self.target.choose_relativity(origin, relativize) diff --git a/src/dns/rdtypes/IN/WKS.py b/src/dns/rdtypes/IN/WKS.py deleted file mode 100644 index 96f98ada..00000000 --- a/src/dns/rdtypes/IN/WKS.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import socket -import struct - -import dns.ipv4 -import dns.rdata -from dns._compat import xrange - -_proto_tcp = socket.getprotobyname('tcp') -_proto_udp = socket.getprotobyname('udp') - - -class WKS(dns.rdata.Rdata): - - """WKS record - - @ivar address: the address - @type address: string - @ivar protocol: the protocol - @type protocol: int - @ivar bitmap: the bitmap - @type bitmap: string - @see: RFC 1035""" - - __slots__ = ['address', 'protocol', 'bitmap'] - - def __init__(self, rdclass, rdtype, address, protocol, bitmap): - super(WKS, self).__init__(rdclass, rdtype) - self.address = address - self.protocol = protocol - if not isinstance(bitmap, bytearray): - self.bitmap = bytearray(bitmap) - else: - self.bitmap = bitmap - - def to_text(self, origin=None, relativize=True, **kw): - bits = [] - for i in xrange(0, len(self.bitmap)): - byte = self.bitmap[i] - for j in xrange(0, 8): - if byte & (0x80 >> j): - bits.append(str(i * 8 + j)) - text = ' '.join(bits) - return '%s %d %s' % (self.address, self.protocol, text) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - address = tok.get_string() - protocol = tok.get_string() - if protocol.isdigit(): - protocol = int(protocol) - else: - protocol = socket.getprotobyname(protocol) - bitmap = bytearray() - while 1: - token = tok.get().unescape() - if token.is_eol_or_eof(): - break - if token.value.isdigit(): - serv = int(token.value) - else: - if protocol != _proto_udp and protocol != _proto_tcp: - raise NotImplementedError("protocol must be TCP or UDP") - if protocol == _proto_udp: - protocol_text = "udp" - else: - protocol_text = "tcp" - serv = socket.getservbyname(token.value, protocol_text) - i = serv // 8 - l = len(bitmap) - if l < i + 1: - for j in xrange(l, i + 1): - bitmap.append(0) - bitmap[i] = bitmap[i] | (0x80 >> (serv % 8)) - bitmap = dns.rdata._truncate_bitmap(bitmap) - return cls(rdclass, rdtype, address, protocol, bitmap) - - def to_wire(self, file, compress=None, origin=None): - file.write(dns.ipv4.inet_aton(self.address)) - protocol = struct.pack('!B', self.protocol) - file.write(protocol) - file.write(self.bitmap) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - address = dns.ipv4.inet_ntoa(wire[current: current + 4]) - protocol, = struct.unpack('!B', wire[current + 4: current + 5]) - current += 5 - rdlen -= 5 - bitmap = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, address, protocol, bitmap) diff --git a/src/dns/rdtypes/IN/__init__.py b/src/dns/rdtypes/IN/__init__.py deleted file mode 100644 index d7e69c9f..00000000 --- a/src/dns/rdtypes/IN/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Class IN rdata type classes.""" - -__all__ = [ - 'A', - 'AAAA', - 'APL', - 'DHCID', - 'IPSECKEY', - 'KX', - 'NAPTR', - 'NSAP', - 'NSAP_PTR', - 'PX', - 'SRV', - 'WKS', -] diff --git a/src/dns/rdtypes/__init__.py b/src/dns/rdtypes/__init__.py deleted file mode 100644 index 1ac137f1..00000000 --- a/src/dns/rdtypes/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS rdata type classes""" - -__all__ = [ - 'ANY', - 'IN', - 'CH', - 'euibase', - 'mxbase', - 'nsbase', -] diff --git a/src/dns/rdtypes/dnskeybase.py b/src/dns/rdtypes/dnskeybase.py deleted file mode 100644 index 3e7e87ef..00000000 --- a/src/dns/rdtypes/dnskeybase.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2004-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import base64 -import struct - -import dns.exception -import dns.dnssec -import dns.rdata - -# wildcard import -__all__ = ["SEP", "REVOKE", "ZONE", - "flags_to_text_set", "flags_from_text_set"] - -# flag constants -SEP = 0x0001 -REVOKE = 0x0080 -ZONE = 0x0100 - -_flag_by_text = { - 'SEP': SEP, - 'REVOKE': REVOKE, - 'ZONE': ZONE -} - -# We construct the inverse mapping programmatically to ensure that we -# cannot make any mistakes (e.g. omissions, cut-and-paste errors) that -# would cause the mapping not to be true inverse. -_flag_by_value = {y: x for x, y in _flag_by_text.items()} - - -def flags_to_text_set(flags): - """Convert a DNSKEY flags value to set texts - @rtype: set([string])""" - - flags_set = set() - mask = 0x1 - while mask <= 0x8000: - if flags & mask: - text = _flag_by_value.get(mask) - if not text: - text = hex(mask) - flags_set.add(text) - mask <<= 1 - return flags_set - - -def flags_from_text_set(texts_set): - """Convert set of DNSKEY flag mnemonic texts to DNSKEY flag value - @rtype: int""" - - flags = 0 - for text in texts_set: - try: - flags += _flag_by_text[text] - except KeyError: - raise NotImplementedError( - "DNSKEY flag '%s' is not supported" % text) - return flags - - -class DNSKEYBase(dns.rdata.Rdata): - - """Base class for rdata that is like a DNSKEY record - - @ivar flags: the key flags - @type flags: int - @ivar protocol: the protocol for which this key may be used - @type protocol: int - @ivar algorithm: the algorithm used for the key - @type algorithm: int - @ivar key: the public key - @type key: string""" - - __slots__ = ['flags', 'protocol', 'algorithm', 'key'] - - def __init__(self, rdclass, rdtype, flags, protocol, algorithm, key): - super(DNSKEYBase, self).__init__(rdclass, rdtype) - self.flags = flags - self.protocol = protocol - self.algorithm = algorithm - self.key = key - - def to_text(self, origin=None, relativize=True, **kw): - return '%d %d %d %s' % (self.flags, self.protocol, self.algorithm, - dns.rdata._base64ify(self.key)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - flags = tok.get_uint16() - protocol = tok.get_uint8() - algorithm = dns.dnssec.algorithm_from_text(tok.get_string()) - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - b64 = b''.join(chunks) - key = base64.b64decode(b64) - return cls(rdclass, rdtype, flags, protocol, algorithm, key) - - def to_wire(self, file, compress=None, origin=None): - header = struct.pack("!HBB", self.flags, self.protocol, self.algorithm) - file.write(header) - file.write(self.key) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - if rdlen < 4: - raise dns.exception.FormError - header = struct.unpack('!HBB', wire[current: current + 4]) - current += 4 - rdlen -= 4 - key = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, header[0], header[1], header[2], - key) - - def flags_to_text_set(self): - """Convert a DNSKEY flags value to set texts - @rtype: set([string])""" - return flags_to_text_set(self.flags) diff --git a/src/dns/rdtypes/dnskeybase.pyi b/src/dns/rdtypes/dnskeybase.pyi deleted file mode 100644 index e102a698..00000000 --- a/src/dns/rdtypes/dnskeybase.pyi +++ /dev/null @@ -1,37 +0,0 @@ -from typing import Set, Any - -SEP : int -REVOKE : int -ZONE : int - -def flags_to_text_set(flags : int) -> Set[str]: - ... - -def flags_from_text_set(texts_set) -> int: - ... - -from .. import rdata - -class DNSKEYBase(rdata.Rdata): - def __init__(self, rdclass, rdtype, flags, protocol, algorithm, key): - self.flags : int - self.protocol : int - self.key : str - self.algorithm : int - - def to_text(self, origin : Any = None, relativize=True, **kw : Any): - ... - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - ... - - def to_wire(self, file, compress=None, origin=None): - ... - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - ... - - def flags_to_text_set(self) -> Set[str]: - ... diff --git a/src/dns/rdtypes/dsbase.py b/src/dns/rdtypes/dsbase.py deleted file mode 100644 index 26ae9d5c..00000000 --- a/src/dns/rdtypes/dsbase.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2010, 2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import struct -import binascii - -import dns.rdata -import dns.rdatatype - - -class DSBase(dns.rdata.Rdata): - - """Base class for rdata that is like a DS record - - @ivar key_tag: the key tag - @type key_tag: int - @ivar algorithm: the algorithm - @type algorithm: int - @ivar digest_type: the digest type - @type digest_type: int - @ivar digest: the digest - @type digest: int - @see: draft-ietf-dnsext-delegation-signer-14.txt""" - - __slots__ = ['key_tag', 'algorithm', 'digest_type', 'digest'] - - def __init__(self, rdclass, rdtype, key_tag, algorithm, digest_type, - digest): - super(DSBase, self).__init__(rdclass, rdtype) - self.key_tag = key_tag - self.algorithm = algorithm - self.digest_type = digest_type - self.digest = digest - - def to_text(self, origin=None, relativize=True, **kw): - return '%d %d %d %s' % (self.key_tag, self.algorithm, - self.digest_type, - dns.rdata._hexify(self.digest, - chunksize=128)) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - key_tag = tok.get_uint16() - algorithm = tok.get_uint8() - digest_type = tok.get_uint8() - chunks = [] - while 1: - t = tok.get().unescape() - if t.is_eol_or_eof(): - break - if not t.is_identifier(): - raise dns.exception.SyntaxError - chunks.append(t.value.encode()) - digest = b''.join(chunks) - digest = binascii.unhexlify(digest) - return cls(rdclass, rdtype, key_tag, algorithm, digest_type, - digest) - - def to_wire(self, file, compress=None, origin=None): - header = struct.pack("!HBB", self.key_tag, self.algorithm, - self.digest_type) - file.write(header) - file.write(self.digest) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - header = struct.unpack("!HBB", wire[current: current + 4]) - current += 4 - rdlen -= 4 - digest = wire[current: current + rdlen].unwrap() - return cls(rdclass, rdtype, header[0], header[1], header[2], digest) diff --git a/src/dns/rdtypes/euibase.py b/src/dns/rdtypes/euibase.py deleted file mode 100644 index cc5fdaa6..00000000 --- a/src/dns/rdtypes/euibase.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (C) 2015 Red Hat, Inc. -# Author: Petr Spacek -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED 'AS IS' AND RED HAT DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import binascii - -import dns.rdata -from dns._compat import xrange - - -class EUIBase(dns.rdata.Rdata): - - """EUIxx record - - @ivar fingerprint: xx-bit Extended Unique Identifier (EUI-xx) - @type fingerprint: string - @see: rfc7043.txt""" - - __slots__ = ['eui'] - # define these in subclasses - # byte_len = 6 # 0123456789ab (in hex) - # text_len = byte_len * 3 - 1 # 01-23-45-67-89-ab - - def __init__(self, rdclass, rdtype, eui): - super(EUIBase, self).__init__(rdclass, rdtype) - if len(eui) != self.byte_len: - raise dns.exception.FormError('EUI%s rdata has to have %s bytes' - % (self.byte_len * 8, self.byte_len)) - self.eui = eui - - def to_text(self, origin=None, relativize=True, **kw): - return dns.rdata._hexify(self.eui, chunksize=2).replace(' ', '-') - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - text = tok.get_string() - tok.get_eol() - if len(text) != cls.text_len: - raise dns.exception.SyntaxError( - 'Input text must have %s characters' % cls.text_len) - expected_dash_idxs = xrange(2, cls.byte_len * 3 - 1, 3) - for i in expected_dash_idxs: - if text[i] != '-': - raise dns.exception.SyntaxError('Dash expected at position %s' - % i) - text = text.replace('-', '') - try: - data = binascii.unhexlify(text.encode()) - except (ValueError, TypeError) as ex: - raise dns.exception.SyntaxError('Hex decoding error: %s' % str(ex)) - return cls(rdclass, rdtype, data) - - def to_wire(self, file, compress=None, origin=None): - file.write(self.eui) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - eui = wire[current:current + rdlen].unwrap() - return cls(rdclass, rdtype, eui) diff --git a/src/dns/rdtypes/mxbase.py b/src/dns/rdtypes/mxbase.py deleted file mode 100644 index 9a3fa623..00000000 --- a/src/dns/rdtypes/mxbase.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""MX-like base classes.""" - -from io import BytesIO -import struct - -import dns.exception -import dns.rdata -import dns.name - - -class MXBase(dns.rdata.Rdata): - - """Base class for rdata that is like an MX record. - - @ivar preference: the preference value - @type preference: int - @ivar exchange: the exchange name - @type exchange: dns.name.Name object""" - - __slots__ = ['preference', 'exchange'] - - def __init__(self, rdclass, rdtype, preference, exchange): - super(MXBase, self).__init__(rdclass, rdtype) - self.preference = preference - self.exchange = exchange - - def to_text(self, origin=None, relativize=True, **kw): - exchange = self.exchange.choose_relativity(origin, relativize) - return '%d %s' % (self.preference, exchange) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - preference = tok.get_uint16() - exchange = tok.get_name() - exchange = exchange.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, preference, exchange) - - def to_wire(self, file, compress=None, origin=None): - pref = struct.pack("!H", self.preference) - file.write(pref) - self.exchange.to_wire(file, compress, origin) - - def to_digestable(self, origin=None): - return struct.pack("!H", self.preference) + \ - self.exchange.to_digestable(origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (preference, ) = struct.unpack('!H', wire[current: current + 2]) - current += 2 - rdlen -= 2 - (exchange, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused != rdlen: - raise dns.exception.FormError - if origin is not None: - exchange = exchange.relativize(origin) - return cls(rdclass, rdtype, preference, exchange) - - def choose_relativity(self, origin=None, relativize=True): - self.exchange = self.exchange.choose_relativity(origin, relativize) - - -class UncompressedMX(MXBase): - - """Base class for rdata that is like an MX record, but whose name - is not compressed when converted to DNS wire format, and whose - digestable form is not downcased.""" - - def to_wire(self, file, compress=None, origin=None): - super(UncompressedMX, self).to_wire(file, None, origin) - - def to_digestable(self, origin=None): - f = BytesIO() - self.to_wire(f, None, origin) - return f.getvalue() - - -class UncompressedDowncasingMX(MXBase): - - """Base class for rdata that is like an MX record, but whose name - is not compressed when convert to DNS wire format.""" - - def to_wire(self, file, compress=None, origin=None): - super(UncompressedDowncasingMX, self).to_wire(file, None, origin) diff --git a/src/dns/rdtypes/nsbase.py b/src/dns/rdtypes/nsbase.py deleted file mode 100644 index 97a22326..00000000 --- a/src/dns/rdtypes/nsbase.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""NS-like base classes.""" - -from io import BytesIO - -import dns.exception -import dns.rdata -import dns.name - - -class NSBase(dns.rdata.Rdata): - - """Base class for rdata that is like an NS record. - - @ivar target: the target name of the rdata - @type target: dns.name.Name object""" - - __slots__ = ['target'] - - def __init__(self, rdclass, rdtype, target): - super(NSBase, self).__init__(rdclass, rdtype) - self.target = target - - def to_text(self, origin=None, relativize=True, **kw): - target = self.target.choose_relativity(origin, relativize) - return str(target) - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - target = tok.get_name() - target = target.choose_relativity(origin, relativize) - tok.get_eol() - return cls(rdclass, rdtype, target) - - def to_wire(self, file, compress=None, origin=None): - self.target.to_wire(file, compress, origin) - - def to_digestable(self, origin=None): - return self.target.to_digestable(origin) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - (target, cused) = dns.name.from_wire(wire[: current + rdlen], - current) - if cused != rdlen: - raise dns.exception.FormError - if origin is not None: - target = target.relativize(origin) - return cls(rdclass, rdtype, target) - - def choose_relativity(self, origin=None, relativize=True): - self.target = self.target.choose_relativity(origin, relativize) - - -class UncompressedNS(NSBase): - - """Base class for rdata that is like an NS record, but whose name - is not compressed when convert to DNS wire format, and whose - digestable form is not downcased.""" - - def to_wire(self, file, compress=None, origin=None): - super(UncompressedNS, self).to_wire(file, None, origin) - - def to_digestable(self, origin=None): - f = BytesIO() - self.to_wire(f, None, origin) - return f.getvalue() diff --git a/src/dns/rdtypes/txtbase.py b/src/dns/rdtypes/txtbase.py deleted file mode 100644 index 645a57ec..00000000 --- a/src/dns/rdtypes/txtbase.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2006-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""TXT-like base class.""" - -import struct - -import dns.exception -import dns.rdata -import dns.tokenizer -from dns._compat import binary_type, string_types - - -class TXTBase(dns.rdata.Rdata): - - """Base class for rdata that is like a TXT record - - @ivar strings: the strings - @type strings: list of binary - @see: RFC 1035""" - - __slots__ = ['strings'] - - def __init__(self, rdclass, rdtype, strings): - super(TXTBase, self).__init__(rdclass, rdtype) - if isinstance(strings, binary_type) or \ - isinstance(strings, string_types): - strings = [strings] - self.strings = [] - for string in strings: - if isinstance(string, string_types): - string = string.encode() - self.strings.append(string) - - def to_text(self, origin=None, relativize=True, **kw): - txt = '' - prefix = '' - for s in self.strings: - txt += '{}"{}"'.format(prefix, dns.rdata._escapify(s)) - prefix = ' ' - return txt - - @classmethod - def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True): - strings = [] - while 1: - token = tok.get().unescape() - if token.is_eol_or_eof(): - break - if not (token.is_quoted_string() or token.is_identifier()): - raise dns.exception.SyntaxError("expected a string") - if len(token.value) > 255: - raise dns.exception.SyntaxError("string too long") - value = token.value - if isinstance(value, binary_type): - strings.append(value) - else: - strings.append(value.encode()) - if len(strings) == 0: - raise dns.exception.UnexpectedEnd - return cls(rdclass, rdtype, strings) - - def to_wire(self, file, compress=None, origin=None): - for s in self.strings: - l = len(s) - assert l < 256 - file.write(struct.pack('!B', l)) - file.write(s) - - @classmethod - def from_wire(cls, rdclass, rdtype, wire, current, rdlen, origin=None): - strings = [] - while rdlen > 0: - l = wire[current] - current += 1 - rdlen -= 1 - if l > rdlen: - raise dns.exception.FormError - s = wire[current: current + l].unwrap() - current += l - rdlen -= l - strings.append(s) - return cls(rdclass, rdtype, strings) diff --git a/src/dns/rdtypes/txtbase.pyi b/src/dns/rdtypes/txtbase.pyi deleted file mode 100644 index af447d50..00000000 --- a/src/dns/rdtypes/txtbase.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from .. import rdata - -class TXTBase(rdata.Rdata): - ... -class TXT(TXTBase): - ... diff --git a/src/dns/renderer.py b/src/dns/renderer.py deleted file mode 100644 index d7ef8c7f..00000000 --- a/src/dns/renderer.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Help for building DNS wire format messages""" - -from io import BytesIO -import struct -import random -import time - -import dns.exception -import dns.tsig -from ._compat import long - - -QUESTION = 0 -ANSWER = 1 -AUTHORITY = 2 -ADDITIONAL = 3 - - -class Renderer(object): - """Helper class for building DNS wire-format messages. - - Most applications can use the higher-level L{dns.message.Message} - class and its to_wire() method to generate wire-format messages. - This class is for those applications which need finer control - over the generation of messages. - - Typical use:: - - r = dns.renderer.Renderer(id=1, flags=0x80, max_size=512) - r.add_question(qname, qtype, qclass) - r.add_rrset(dns.renderer.ANSWER, rrset_1) - r.add_rrset(dns.renderer.ANSWER, rrset_2) - r.add_rrset(dns.renderer.AUTHORITY, ns_rrset) - r.add_edns(0, 0, 4096) - r.add_rrset(dns.renderer.ADDTIONAL, ad_rrset_1) - r.add_rrset(dns.renderer.ADDTIONAL, ad_rrset_2) - r.write_header() - r.add_tsig(keyname, secret, 300, 1, 0, '', request_mac) - wire = r.get_wire() - - output, a BytesIO, where rendering is written - - id: the message id - - flags: the message flags - - max_size: the maximum size of the message - - origin: the origin to use when rendering relative names - - compress: the compression table - - section: an int, the section currently being rendered - - counts: list of the number of RRs in each section - - mac: the MAC of the rendered message (if TSIG was used) - """ - - def __init__(self, id=None, flags=0, max_size=65535, origin=None): - """Initialize a new renderer.""" - - self.output = BytesIO() - if id is None: - self.id = random.randint(0, 65535) - else: - self.id = id - self.flags = flags - self.max_size = max_size - self.origin = origin - self.compress = {} - self.section = QUESTION - self.counts = [0, 0, 0, 0] - self.output.write(b'\x00' * 12) - self.mac = '' - - def _rollback(self, where): - """Truncate the output buffer at offset *where*, and remove any - compression table entries that pointed beyond the truncation - point. - """ - - self.output.seek(where) - self.output.truncate() - keys_to_delete = [] - for k, v in self.compress.items(): - if v >= where: - keys_to_delete.append(k) - for k in keys_to_delete: - del self.compress[k] - - def _set_section(self, section): - """Set the renderer's current section. - - Sections must be rendered order: QUESTION, ANSWER, AUTHORITY, - ADDITIONAL. Sections may be empty. - - Raises dns.exception.FormError if an attempt was made to set - a section value less than the current section. - """ - - if self.section != section: - if self.section > section: - raise dns.exception.FormError - self.section = section - - def add_question(self, qname, rdtype, rdclass=dns.rdataclass.IN): - """Add a question to the message.""" - - self._set_section(QUESTION) - before = self.output.tell() - qname.to_wire(self.output, self.compress, self.origin) - self.output.write(struct.pack("!HH", rdtype, rdclass)) - after = self.output.tell() - if after >= self.max_size: - self._rollback(before) - raise dns.exception.TooBig - self.counts[QUESTION] += 1 - - def add_rrset(self, section, rrset, **kw): - """Add the rrset to the specified section. - - Any keyword arguments are passed on to the rdataset's to_wire() - routine. - """ - - self._set_section(section) - before = self.output.tell() - n = rrset.to_wire(self.output, self.compress, self.origin, **kw) - after = self.output.tell() - if after >= self.max_size: - self._rollback(before) - raise dns.exception.TooBig - self.counts[section] += n - - def add_rdataset(self, section, name, rdataset, **kw): - """Add the rdataset to the specified section, using the specified - name as the owner name. - - Any keyword arguments are passed on to the rdataset's to_wire() - routine. - """ - - self._set_section(section) - before = self.output.tell() - n = rdataset.to_wire(name, self.output, self.compress, self.origin, - **kw) - after = self.output.tell() - if after >= self.max_size: - self._rollback(before) - raise dns.exception.TooBig - self.counts[section] += n - - def add_edns(self, edns, ednsflags, payload, options=None): - """Add an EDNS OPT record to the message.""" - - # make sure the EDNS version in ednsflags agrees with edns - ednsflags &= long(0xFF00FFFF) - ednsflags |= (edns << 16) - self._set_section(ADDITIONAL) - before = self.output.tell() - self.output.write(struct.pack('!BHHIH', 0, dns.rdatatype.OPT, payload, - ednsflags, 0)) - if options is not None: - lstart = self.output.tell() - for opt in options: - stuff = struct.pack("!HH", opt.otype, 0) - self.output.write(stuff) - start = self.output.tell() - opt.to_wire(self.output) - end = self.output.tell() - assert end - start < 65536 - self.output.seek(start - 2) - stuff = struct.pack("!H", end - start) - self.output.write(stuff) - self.output.seek(0, 2) - lend = self.output.tell() - assert lend - lstart < 65536 - self.output.seek(lstart - 2) - stuff = struct.pack("!H", lend - lstart) - self.output.write(stuff) - self.output.seek(0, 2) - after = self.output.tell() - if after >= self.max_size: - self._rollback(before) - raise dns.exception.TooBig - self.counts[ADDITIONAL] += 1 - - def add_tsig(self, keyname, secret, fudge, id, tsig_error, other_data, - request_mac, algorithm=dns.tsig.default_algorithm): - """Add a TSIG signature to the message.""" - - s = self.output.getvalue() - (tsig_rdata, self.mac, ctx) = dns.tsig.sign(s, - keyname, - secret, - int(time.time()), - fudge, - id, - tsig_error, - other_data, - request_mac, - algorithm=algorithm) - self._write_tsig(tsig_rdata, keyname) - - def add_multi_tsig(self, ctx, keyname, secret, fudge, id, tsig_error, - other_data, request_mac, - algorithm=dns.tsig.default_algorithm): - """Add a TSIG signature to the message. Unlike add_tsig(), this can be - used for a series of consecutive DNS envelopes, e.g. for a zone - transfer over TCP [RFC2845, 4.4]. - - For the first message in the sequence, give ctx=None. For each - subsequent message, give the ctx that was returned from the - add_multi_tsig() call for the previous message.""" - - s = self.output.getvalue() - (tsig_rdata, self.mac, ctx) = dns.tsig.sign(s, - keyname, - secret, - int(time.time()), - fudge, - id, - tsig_error, - other_data, - request_mac, - ctx=ctx, - first=ctx is None, - multi=True, - algorithm=algorithm) - self._write_tsig(tsig_rdata, keyname) - return ctx - - def _write_tsig(self, tsig_rdata, keyname): - self._set_section(ADDITIONAL) - before = self.output.tell() - - keyname.to_wire(self.output, self.compress, self.origin) - self.output.write(struct.pack('!HHIH', dns.rdatatype.TSIG, - dns.rdataclass.ANY, 0, 0)) - rdata_start = self.output.tell() - self.output.write(tsig_rdata) - - after = self.output.tell() - assert after - rdata_start < 65536 - if after >= self.max_size: - self._rollback(before) - raise dns.exception.TooBig - - self.output.seek(rdata_start - 2) - self.output.write(struct.pack('!H', after - rdata_start)) - self.counts[ADDITIONAL] += 1 - self.output.seek(10) - self.output.write(struct.pack('!H', self.counts[ADDITIONAL])) - self.output.seek(0, 2) - - def write_header(self): - """Write the DNS message header. - - Writing the DNS message header is done after all sections - have been rendered, but before the optional TSIG signature - is added. - """ - - self.output.seek(0) - self.output.write(struct.pack('!HHHHHH', self.id, self.flags, - self.counts[0], self.counts[1], - self.counts[2], self.counts[3])) - self.output.seek(0, 2) - - def get_wire(self): - """Return the wire format message.""" - - return self.output.getvalue() diff --git a/src/dns/resolver.py b/src/dns/resolver.py deleted file mode 100644 index 806e5b2b..00000000 --- a/src/dns/resolver.py +++ /dev/null @@ -1,1383 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS stub resolver.""" - -import socket -import sys -import time -import random - -try: - import threading as _threading -except ImportError: - import dummy_threading as _threading - -import dns.exception -import dns.flags -import dns.ipv4 -import dns.ipv6 -import dns.message -import dns.name -import dns.query -import dns.rcode -import dns.rdataclass -import dns.rdatatype -import dns.reversename -import dns.tsig -from ._compat import xrange, string_types - -if sys.platform == 'win32': - try: - import winreg as _winreg - except ImportError: - import _winreg # pylint: disable=import-error - -class NXDOMAIN(dns.exception.DNSException): - """The DNS query name does not exist.""" - supp_kwargs = {'qnames', 'responses'} - fmt = None # we have our own __str__ implementation - - def _check_kwargs(self, qnames, responses=None): - if not isinstance(qnames, (list, tuple, set)): - raise AttributeError("qnames must be a list, tuple or set") - if len(qnames) == 0: - raise AttributeError("qnames must contain at least one element") - if responses is None: - responses = {} - elif not isinstance(responses, dict): - raise AttributeError("responses must be a dict(qname=response)") - kwargs = dict(qnames=qnames, responses=responses) - return kwargs - - def __str__(self): - if 'qnames' not in self.kwargs: - return super(NXDOMAIN, self).__str__() - qnames = self.kwargs['qnames'] - if len(qnames) > 1: - msg = 'None of DNS query names exist' - else: - msg = 'The DNS query name does not exist' - qnames = ', '.join(map(str, qnames)) - return "{}: {}".format(msg, qnames) - - def canonical_name(self): - if not 'qnames' in self.kwargs: - raise TypeError("parametrized exception required") - IN = dns.rdataclass.IN - CNAME = dns.rdatatype.CNAME - cname = None - for qname in self.kwargs['qnames']: - response = self.kwargs['responses'][qname] - for answer in response.answer: - if answer.rdtype != CNAME or answer.rdclass != IN: - continue - cname = answer.items[0].target.to_text() - if cname is not None: - return dns.name.from_text(cname) - return self.kwargs['qnames'][0] - canonical_name = property(canonical_name, doc=( - "Return the unresolved canonical name.")) - - def __add__(self, e_nx): - """Augment by results from another NXDOMAIN exception.""" - qnames0 = list(self.kwargs.get('qnames', [])) - responses0 = dict(self.kwargs.get('responses', {})) - responses1 = e_nx.kwargs.get('responses', {}) - for qname1 in e_nx.kwargs.get('qnames', []): - if qname1 not in qnames0: - qnames0.append(qname1) - if qname1 in responses1: - responses0[qname1] = responses1[qname1] - return NXDOMAIN(qnames=qnames0, responses=responses0) - - def qnames(self): - """All of the names that were tried. - - Returns a list of ``dns.name.Name``. - """ - return self.kwargs['qnames'] - - def responses(self): - """A map from queried names to their NXDOMAIN responses. - - Returns a dict mapping a ``dns.name.Name`` to a - ``dns.message.Message``. - """ - return self.kwargs['responses'] - - def response(self, qname): - """The response for query *qname*. - - Returns a ``dns.message.Message``. - """ - return self.kwargs['responses'][qname] - - -class YXDOMAIN(dns.exception.DNSException): - """The DNS query name is too long after DNAME substitution.""" - -# The definition of the Timeout exception has moved from here to the -# dns.exception module. We keep dns.resolver.Timeout defined for -# backwards compatibility. - -Timeout = dns.exception.Timeout - - -class NoAnswer(dns.exception.DNSException): - """The DNS response does not contain an answer to the question.""" - fmt = 'The DNS response does not contain an answer ' + \ - 'to the question: {query}' - supp_kwargs = {'response'} - - def _fmt_kwargs(self, **kwargs): - return super(NoAnswer, self)._fmt_kwargs( - query=kwargs['response'].question) - - -class NoNameservers(dns.exception.DNSException): - """All nameservers failed to answer the query. - - errors: list of servers and respective errors - The type of errors is - [(server IP address, any object convertible to string)]. - Non-empty errors list will add explanatory message () - """ - - msg = "All nameservers failed to answer the query." - fmt = "%s {query}: {errors}" % msg[:-1] - supp_kwargs = {'request', 'errors'} - - def _fmt_kwargs(self, **kwargs): - srv_msgs = [] - for err in kwargs['errors']: - srv_msgs.append('Server {} {} port {} answered {}'.format(err[0], - 'TCP' if err[1] else 'UDP', err[2], err[3])) - return super(NoNameservers, self)._fmt_kwargs( - query=kwargs['request'].question, errors='; '.join(srv_msgs)) - - -class NotAbsolute(dns.exception.DNSException): - """An absolute domain name is required but a relative name was provided.""" - - -class NoRootSOA(dns.exception.DNSException): - """There is no SOA RR at the DNS root name. This should never happen!""" - - -class NoMetaqueries(dns.exception.DNSException): - """DNS metaqueries are not allowed.""" - - -class Answer(object): - """DNS stub resolver answer. - - Instances of this class bundle up the result of a successful DNS - resolution. - - For convenience, the answer object implements much of the sequence - protocol, forwarding to its ``rrset`` attribute. E.g. - ``for a in answer`` is equivalent to ``for a in answer.rrset``. - ``answer[i]`` is equivalent to ``answer.rrset[i]``, and - ``answer[i:j]`` is equivalent to ``answer.rrset[i:j]``. - - Note that CNAMEs or DNAMEs in the response may mean that answer - RRset's name might not be the query name. - """ - - def __init__(self, qname, rdtype, rdclass, response, - raise_on_no_answer=True): - self.qname = qname - self.rdtype = rdtype - self.rdclass = rdclass - self.response = response - min_ttl = -1 - rrset = None - for count in xrange(0, 15): - try: - rrset = response.find_rrset(response.answer, qname, - rdclass, rdtype) - if min_ttl == -1 or rrset.ttl < min_ttl: - min_ttl = rrset.ttl - break - except KeyError: - if rdtype != dns.rdatatype.CNAME: - try: - crrset = response.find_rrset(response.answer, - qname, - rdclass, - dns.rdatatype.CNAME) - if min_ttl == -1 or crrset.ttl < min_ttl: - min_ttl = crrset.ttl - for rd in crrset: - qname = rd.target - break - continue - except KeyError: - if raise_on_no_answer: - raise NoAnswer(response=response) - if raise_on_no_answer: - raise NoAnswer(response=response) - if rrset is None and raise_on_no_answer: - raise NoAnswer(response=response) - self.canonical_name = qname - self.rrset = rrset - if rrset is None: - while 1: - # Look for a SOA RR whose owner name is a superdomain - # of qname. - try: - srrset = response.find_rrset(response.authority, qname, - rdclass, dns.rdatatype.SOA) - if min_ttl == -1 or srrset.ttl < min_ttl: - min_ttl = srrset.ttl - if srrset[0].minimum < min_ttl: - min_ttl = srrset[0].minimum - break - except KeyError: - try: - qname = qname.parent() - except dns.name.NoParent: - break - self.expiration = time.time() + min_ttl - - def __getattr__(self, attr): - if attr == 'name': - return self.rrset.name - elif attr == 'ttl': - return self.rrset.ttl - elif attr == 'covers': - return self.rrset.covers - elif attr == 'rdclass': - return self.rrset.rdclass - elif attr == 'rdtype': - return self.rrset.rdtype - else: - raise AttributeError(attr) - - def __len__(self): - return self.rrset and len(self.rrset) or 0 - - def __iter__(self): - return self.rrset and iter(self.rrset) or iter(tuple()) - - def __getitem__(self, i): - if self.rrset is None: - raise IndexError - return self.rrset[i] - - def __delitem__(self, i): - if self.rrset is None: - raise IndexError - del self.rrset[i] - - -class Cache(object): - """Simple thread-safe DNS answer cache.""" - - def __init__(self, cleaning_interval=300.0): - """*cleaning_interval*, a ``float`` is the number of seconds between - periodic cleanings. - """ - - self.data = {} - self.cleaning_interval = cleaning_interval - self.next_cleaning = time.time() + self.cleaning_interval - self.lock = _threading.Lock() - - def _maybe_clean(self): - """Clean the cache if it's time to do so.""" - - now = time.time() - if self.next_cleaning <= now: - keys_to_delete = [] - for (k, v) in self.data.items(): - if v.expiration <= now: - keys_to_delete.append(k) - for k in keys_to_delete: - del self.data[k] - now = time.time() - self.next_cleaning = now + self.cleaning_interval - - def get(self, key): - """Get the answer associated with *key*. - - Returns None if no answer is cached for the key. - - *key*, a ``(dns.name.Name, int, int)`` tuple whose values are the - query name, rdtype, and rdclass respectively. - - Returns a ``dns.resolver.Answer`` or ``None``. - """ - - try: - self.lock.acquire() - self._maybe_clean() - v = self.data.get(key) - if v is None or v.expiration <= time.time(): - return None - return v - finally: - self.lock.release() - - def put(self, key, value): - """Associate key and value in the cache. - - *key*, a ``(dns.name.Name, int, int)`` tuple whose values are the - query name, rdtype, and rdclass respectively. - - *value*, a ``dns.resolver.Answer``, the answer. - """ - - try: - self.lock.acquire() - self._maybe_clean() - self.data[key] = value - finally: - self.lock.release() - - def flush(self, key=None): - """Flush the cache. - - If *key* is not ``None``, only that item is flushed. Otherwise - the entire cache is flushed. - - *key*, a ``(dns.name.Name, int, int)`` tuple whose values are the - query name, rdtype, and rdclass respectively. - """ - - try: - self.lock.acquire() - if key is not None: - if key in self.data: - del self.data[key] - else: - self.data = {} - self.next_cleaning = time.time() + self.cleaning_interval - finally: - self.lock.release() - - -class LRUCacheNode(object): - """LRUCache node.""" - - def __init__(self, key, value): - self.key = key - self.value = value - self.prev = self - self.next = self - - def link_before(self, node): - self.prev = node.prev - self.next = node - node.prev.next = self - node.prev = self - - def link_after(self, node): - self.prev = node - self.next = node.next - node.next.prev = self - node.next = self - - def unlink(self): - self.next.prev = self.prev - self.prev.next = self.next - - -class LRUCache(object): - """Thread-safe, bounded, least-recently-used DNS answer cache. - - This cache is better than the simple cache (above) if you're - running a web crawler or other process that does a lot of - resolutions. The LRUCache has a maximum number of nodes, and when - it is full, the least-recently used node is removed to make space - for a new one. - """ - - def __init__(self, max_size=100000): - """*max_size*, an ``int``, is the maximum number of nodes to cache; - it must be greater than 0. - """ - - self.data = {} - self.set_max_size(max_size) - self.sentinel = LRUCacheNode(None, None) - self.lock = _threading.Lock() - - def set_max_size(self, max_size): - if max_size < 1: - max_size = 1 - self.max_size = max_size - - def get(self, key): - """Get the answer associated with *key*. - - Returns None if no answer is cached for the key. - - *key*, a ``(dns.name.Name, int, int)`` tuple whose values are the - query name, rdtype, and rdclass respectively. - - Returns a ``dns.resolver.Answer`` or ``None``. - """ - - try: - self.lock.acquire() - node = self.data.get(key) - if node is None: - return None - # Unlink because we're either going to move the node to the front - # of the LRU list or we're going to free it. - node.unlink() - if node.value.expiration <= time.time(): - del self.data[node.key] - return None - node.link_after(self.sentinel) - return node.value - finally: - self.lock.release() - - def put(self, key, value): - """Associate key and value in the cache. - - *key*, a ``(dns.name.Name, int, int)`` tuple whose values are the - query name, rdtype, and rdclass respectively. - - *value*, a ``dns.resolver.Answer``, the answer. - """ - - try: - self.lock.acquire() - node = self.data.get(key) - if node is not None: - node.unlink() - del self.data[node.key] - while len(self.data) >= self.max_size: - node = self.sentinel.prev - node.unlink() - del self.data[node.key] - node = LRUCacheNode(key, value) - node.link_after(self.sentinel) - self.data[key] = node - finally: - self.lock.release() - - def flush(self, key=None): - """Flush the cache. - - If *key* is not ``None``, only that item is flushed. Otherwise - the entire cache is flushed. - - *key*, a ``(dns.name.Name, int, int)`` tuple whose values are the - query name, rdtype, and rdclass respectively. - """ - - try: - self.lock.acquire() - if key is not None: - node = self.data.get(key) - if node is not None: - node.unlink() - del self.data[node.key] - else: - node = self.sentinel.next - while node != self.sentinel: - next = node.next - node.prev = None - node.next = None - node = next - self.data = {} - finally: - self.lock.release() - - -class Resolver(object): - """DNS stub resolver.""" - - def __init__(self, filename='/etc/resolv.conf', configure=True): - """*filename*, a ``text`` or file object, specifying a file - in standard /etc/resolv.conf format. This parameter is meaningful - only when *configure* is true and the platform is POSIX. - - *configure*, a ``bool``. If True (the default), the resolver - instance is configured in the normal fashion for the operating - system the resolver is running on. (I.e. by reading a - /etc/resolv.conf file on POSIX systems and from the registry - on Windows systems.) - """ - - self.domain = None - self.nameservers = None - self.nameserver_ports = None - self.port = None - self.search = None - self.timeout = None - self.lifetime = None - self.keyring = None - self.keyname = None - self.keyalgorithm = None - self.edns = None - self.ednsflags = None - self.payload = None - self.cache = None - self.flags = None - self.retry_servfail = False - self.rotate = False - - self.reset() - if configure: - if sys.platform == 'win32': - self.read_registry() - elif filename: - self.read_resolv_conf(filename) - - def reset(self): - """Reset all resolver configuration to the defaults.""" - - self.domain = \ - dns.name.Name(dns.name.from_text(socket.gethostname())[1:]) - if len(self.domain) == 0: - self.domain = dns.name.root - self.nameservers = [] - self.nameserver_ports = {} - self.port = 53 - self.search = [] - self.timeout = 2.0 - self.lifetime = 30.0 - self.keyring = None - self.keyname = None - self.keyalgorithm = dns.tsig.default_algorithm - self.edns = -1 - self.ednsflags = 0 - self.payload = 0 - self.cache = None - self.flags = None - self.retry_servfail = False - self.rotate = False - - def read_resolv_conf(self, f): - """Process *f* as a file in the /etc/resolv.conf format. If f is - a ``text``, it is used as the name of the file to open; otherwise it - is treated as the file itself.""" - - if isinstance(f, string_types): - try: - f = open(f, 'r') - except IOError: - # /etc/resolv.conf doesn't exist, can't be read, etc. - # We'll just use the default resolver configuration. - self.nameservers = ['127.0.0.1'] - return - want_close = True - else: - want_close = False - try: - for l in f: - if len(l) == 0 or l[0] == '#' or l[0] == ';': - continue - tokens = l.split() - - # Any line containing less than 2 tokens is malformed - if len(tokens) < 2: - continue - - if tokens[0] == 'nameserver': - self.nameservers.append(tokens[1]) - elif tokens[0] == 'domain': - self.domain = dns.name.from_text(tokens[1]) - elif tokens[0] == 'search': - for suffix in tokens[1:]: - self.search.append(dns.name.from_text(suffix)) - elif tokens[0] == 'options': - if 'rotate' in tokens[1:]: - self.rotate = True - finally: - if want_close: - f.close() - if len(self.nameservers) == 0: - self.nameservers.append('127.0.0.1') - - def _determine_split_char(self, entry): - # - # The windows registry irritatingly changes the list element - # delimiter in between ' ' and ',' (and vice-versa) in various - # versions of windows. - # - if entry.find(' ') >= 0: - split_char = ' ' - elif entry.find(',') >= 0: - split_char = ',' - else: - # probably a singleton; treat as a space-separated list. - split_char = ' ' - return split_char - - def _config_win32_nameservers(self, nameservers): - # we call str() on nameservers to convert it from unicode to ascii - nameservers = str(nameservers) - split_char = self._determine_split_char(nameservers) - ns_list = nameservers.split(split_char) - for ns in ns_list: - if ns not in self.nameservers: - self.nameservers.append(ns) - - def _config_win32_domain(self, domain): - # we call str() on domain to convert it from unicode to ascii - self.domain = dns.name.from_text(str(domain)) - - def _config_win32_search(self, search): - # we call str() on search to convert it from unicode to ascii - search = str(search) - split_char = self._determine_split_char(search) - search_list = search.split(split_char) - for s in search_list: - if s not in self.search: - self.search.append(dns.name.from_text(s)) - - def _config_win32_fromkey(self, key, always_try_domain): - try: - servers, rtype = _winreg.QueryValueEx(key, 'NameServer') - except WindowsError: # pylint: disable=undefined-variable - servers = None - if servers: - self._config_win32_nameservers(servers) - if servers or always_try_domain: - try: - dom, rtype = _winreg.QueryValueEx(key, 'Domain') - if dom: - self._config_win32_domain(dom) - except WindowsError: # pylint: disable=undefined-variable - pass - else: - try: - servers, rtype = _winreg.QueryValueEx(key, 'DhcpNameServer') - except WindowsError: # pylint: disable=undefined-variable - servers = None - if servers: - self._config_win32_nameservers(servers) - try: - dom, rtype = _winreg.QueryValueEx(key, 'DhcpDomain') - if dom: - self._config_win32_domain(dom) - except WindowsError: # pylint: disable=undefined-variable - pass - try: - search, rtype = _winreg.QueryValueEx(key, 'SearchList') - except WindowsError: # pylint: disable=undefined-variable - search = None - if search: - self._config_win32_search(search) - - def read_registry(self): - """Extract resolver configuration from the Windows registry.""" - - lm = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE) - want_scan = False - try: - try: - # XP, 2000 - tcp_params = _winreg.OpenKey(lm, - r'SYSTEM\CurrentControlSet' - r'\Services\Tcpip\Parameters') - want_scan = True - except EnvironmentError: - # ME - tcp_params = _winreg.OpenKey(lm, - r'SYSTEM\CurrentControlSet' - r'\Services\VxD\MSTCP') - try: - self._config_win32_fromkey(tcp_params, True) - finally: - tcp_params.Close() - if want_scan: - interfaces = _winreg.OpenKey(lm, - r'SYSTEM\CurrentControlSet' - r'\Services\Tcpip\Parameters' - r'\Interfaces') - try: - i = 0 - while True: - try: - guid = _winreg.EnumKey(interfaces, i) - i += 1 - key = _winreg.OpenKey(interfaces, guid) - if not self._win32_is_nic_enabled(lm, guid, key): - continue - try: - self._config_win32_fromkey(key, False) - finally: - key.Close() - except EnvironmentError: - break - finally: - interfaces.Close() - finally: - lm.Close() - - def _win32_is_nic_enabled(self, lm, guid, interface_key): - # Look in the Windows Registry to determine whether the network - # interface corresponding to the given guid is enabled. - # - # (Code contributed by Paul Marks, thanks!) - # - try: - # This hard-coded location seems to be consistent, at least - # from Windows 2000 through Vista. - connection_key = _winreg.OpenKey( - lm, - r'SYSTEM\CurrentControlSet\Control\Network' - r'\{4D36E972-E325-11CE-BFC1-08002BE10318}' - r'\%s\Connection' % guid) - - try: - # The PnpInstanceID points to a key inside Enum - (pnp_id, ttype) = _winreg.QueryValueEx( - connection_key, 'PnpInstanceID') - - if ttype != _winreg.REG_SZ: - raise ValueError - - device_key = _winreg.OpenKey( - lm, r'SYSTEM\CurrentControlSet\Enum\%s' % pnp_id) - - try: - # Get ConfigFlags for this device - (flags, ttype) = _winreg.QueryValueEx( - device_key, 'ConfigFlags') - - if ttype != _winreg.REG_DWORD: - raise ValueError - - # Based on experimentation, bit 0x1 indicates that the - # device is disabled. - return not flags & 0x1 - - finally: - device_key.Close() - finally: - connection_key.Close() - except (EnvironmentError, ValueError): - # Pre-vista, enabled interfaces seem to have a non-empty - # NTEContextList; this was how dnspython detected enabled - # nics before the code above was contributed. We've retained - # the old method since we don't know if the code above works - # on Windows 95/98/ME. - try: - (nte, ttype) = _winreg.QueryValueEx(interface_key, - 'NTEContextList') - return nte is not None - except WindowsError: # pylint: disable=undefined-variable - return False - - def _compute_timeout(self, start, lifetime=None): - lifetime = self.lifetime if lifetime is None else lifetime - now = time.time() - duration = now - start - if duration < 0: - if duration < -1: - # Time going backwards is bad. Just give up. - raise Timeout(timeout=duration) - else: - # Time went backwards, but only a little. This can - # happen, e.g. under vmware with older linux kernels. - # Pretend it didn't happen. - now = start - if duration >= lifetime: - raise Timeout(timeout=duration) - return min(lifetime - duration, self.timeout) - - def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN, - tcp=False, source=None, raise_on_no_answer=True, source_port=0, - lifetime=None): - """Query nameservers to find the answer to the question. - - The *qname*, *rdtype*, and *rdclass* parameters may be objects - of the appropriate type, or strings that can be converted into objects - of the appropriate type. - - *qname*, a ``dns.name.Name`` or ``text``, the query name. - - *rdtype*, an ``int`` or ``text``, the query type. - - *rdclass*, an ``int`` or ``text``, the query class. - - *tcp*, a ``bool``. If ``True``, use TCP to make the query. - - *source*, a ``text`` or ``None``. If not ``None``, bind to this IP - address when making queries. - - *raise_on_no_answer*, a ``bool``. If ``True``, raise - ``dns.resolver.NoAnswer`` if there's no answer to the question. - - *source_port*, an ``int``, the port from which to send the message. - - *lifetime*, a ``float``, how long query should run before timing out. - - Raises ``dns.exception.Timeout`` if no answers could be found - in the specified lifetime. - - Raises ``dns.resolver.NXDOMAIN`` if the query name does not exist. - - Raises ``dns.resolver.YXDOMAIN`` if the query name is too long after - DNAME substitution. - - Raises ``dns.resolver.NoAnswer`` if *raise_on_no_answer* is - ``True`` and the query name exists but has no RRset of the - desired type and class. - - Raises ``dns.resolver.NoNameservers`` if no non-broken - nameservers are available to answer the question. - - Returns a ``dns.resolver.Answer`` instance. - """ - - if isinstance(qname, string_types): - qname = dns.name.from_text(qname, None) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if dns.rdatatype.is_metatype(rdtype): - raise NoMetaqueries - if isinstance(rdclass, string_types): - rdclass = dns.rdataclass.from_text(rdclass) - if dns.rdataclass.is_metaclass(rdclass): - raise NoMetaqueries - qnames_to_try = [] - if qname.is_absolute(): - qnames_to_try.append(qname) - else: - if len(qname) > 1: - qnames_to_try.append(qname.concatenate(dns.name.root)) - if self.search: - for suffix in self.search: - qnames_to_try.append(qname.concatenate(suffix)) - else: - qnames_to_try.append(qname.concatenate(self.domain)) - all_nxdomain = True - nxdomain_responses = {} - start = time.time() - _qname = None # make pylint happy - for _qname in qnames_to_try: - if self.cache: - answer = self.cache.get((_qname, rdtype, rdclass)) - if answer is not None: - if answer.rrset is None and raise_on_no_answer: - raise NoAnswer(response=answer.response) - else: - return answer - request = dns.message.make_query(_qname, rdtype, rdclass) - if self.keyname is not None: - request.use_tsig(self.keyring, self.keyname, - algorithm=self.keyalgorithm) - request.use_edns(self.edns, self.ednsflags, self.payload) - if self.flags is not None: - request.flags = self.flags - response = None - # - # make a copy of the servers list so we can alter it later. - # - nameservers = self.nameservers[:] - errors = [] - if self.rotate: - random.shuffle(nameservers) - backoff = 0.10 - while response is None: - if len(nameservers) == 0: - raise NoNameservers(request=request, errors=errors) - for nameserver in nameservers[:]: - timeout = self._compute_timeout(start, lifetime) - port = self.nameserver_ports.get(nameserver, self.port) - try: - tcp_attempt = tcp - if tcp: - response = dns.query.tcp(request, nameserver, - timeout, port, - source=source, - source_port=source_port) - else: - response = dns.query.udp(request, nameserver, - timeout, port, - source=source, - source_port=source_port) - if response.flags & dns.flags.TC: - # Response truncated; retry with TCP. - tcp_attempt = True - timeout = self._compute_timeout(start, lifetime) - response = \ - dns.query.tcp(request, nameserver, - timeout, port, - source=source, - source_port=source_port) - except (socket.error, dns.exception.Timeout) as ex: - # - # Communication failure or timeout. Go to the - # next server - # - errors.append((nameserver, tcp_attempt, port, ex, - response)) - response = None - continue - except dns.query.UnexpectedSource as ex: - # - # Who knows? Keep going. - # - errors.append((nameserver, tcp_attempt, port, ex, - response)) - response = None - continue - except dns.exception.FormError as ex: - # - # We don't understand what this server is - # saying. Take it out of the mix and - # continue. - # - nameservers.remove(nameserver) - errors.append((nameserver, tcp_attempt, port, ex, - response)) - response = None - continue - except EOFError as ex: - # - # We're using TCP and they hung up on us. - # Probably they don't support TCP (though - # they're supposed to!). Take it out of the - # mix and continue. - # - nameservers.remove(nameserver) - errors.append((nameserver, tcp_attempt, port, ex, - response)) - response = None - continue - rcode = response.rcode() - if rcode == dns.rcode.YXDOMAIN: - ex = YXDOMAIN() - errors.append((nameserver, tcp_attempt, port, ex, - response)) - raise ex - if rcode == dns.rcode.NOERROR or \ - rcode == dns.rcode.NXDOMAIN: - break - # - # We got a response, but we're not happy with the - # rcode in it. Remove the server from the mix if - # the rcode isn't SERVFAIL. - # - if rcode != dns.rcode.SERVFAIL or not self.retry_servfail: - nameservers.remove(nameserver) - errors.append((nameserver, tcp_attempt, port, - dns.rcode.to_text(rcode), response)) - response = None - if response is not None: - break - # - # All nameservers failed! - # - if len(nameservers) > 0: - # - # But we still have servers to try. Sleep a bit - # so we don't pound them! - # - timeout = self._compute_timeout(start, lifetime) - sleep_time = min(timeout, backoff) - backoff *= 2 - time.sleep(sleep_time) - if response.rcode() == dns.rcode.NXDOMAIN: - nxdomain_responses[_qname] = response - continue - all_nxdomain = False - break - if all_nxdomain: - raise NXDOMAIN(qnames=qnames_to_try, responses=nxdomain_responses) - answer = Answer(_qname, rdtype, rdclass, response, - raise_on_no_answer) - if self.cache: - self.cache.put((_qname, rdtype, rdclass), answer) - return answer - - def use_tsig(self, keyring, keyname=None, - algorithm=dns.tsig.default_algorithm): - """Add a TSIG signature to the query. - - See the documentation of the Message class for a complete - description of the keyring dictionary. - - *keyring*, a ``dict``, the TSIG keyring to use. If a - *keyring* is specified but a *keyname* is not, then the key - used will be the first key in the *keyring*. Note that the - order of keys in a dictionary is not defined, so applications - should supply a keyname when a keyring is used, unless they - know the keyring contains only one key. - - *keyname*, a ``dns.name.Name`` or ``None``, the name of the TSIG key - to use; defaults to ``None``. The key must be defined in the keyring. - - *algorithm*, a ``dns.name.Name``, the TSIG algorithm to use. - """ - - self.keyring = keyring - if keyname is None: - self.keyname = list(self.keyring.keys())[0] - else: - self.keyname = keyname - self.keyalgorithm = algorithm - - def use_edns(self, edns, ednsflags, payload): - """Configure EDNS behavior. - - *edns*, an ``int``, is the EDNS level to use. Specifying - ``None``, ``False``, or ``-1`` means "do not use EDNS", and in this case - the other parameters are ignored. Specifying ``True`` is - equivalent to specifying 0, i.e. "use EDNS0". - - *ednsflags*, an ``int``, the EDNS flag values. - - *payload*, an ``int``, is the EDNS sender's payload field, which is the - maximum size of UDP datagram the sender can handle. I.e. how big - a response to this message can be. - """ - - if edns is None: - edns = -1 - self.edns = edns - self.ednsflags = ednsflags - self.payload = payload - - def set_flags(self, flags): - """Overrides the default flags with your own. - - *flags*, an ``int``, the message flags to use. - """ - - self.flags = flags - - -#: The default resolver. -default_resolver = None - - -def get_default_resolver(): - """Get the default resolver, initializing it if necessary.""" - if default_resolver is None: - reset_default_resolver() - return default_resolver - - -def reset_default_resolver(): - """Re-initialize default resolver. - - Note that the resolver configuration (i.e. /etc/resolv.conf on UNIX - systems) will be re-read immediately. - """ - - global default_resolver - default_resolver = Resolver() - - -def query(qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN, - tcp=False, source=None, raise_on_no_answer=True, - source_port=0, lifetime=None): - """Query nameservers to find the answer to the question. - - This is a convenience function that uses the default resolver - object to make the query. - - See ``dns.resolver.Resolver.query`` for more information on the - parameters. - """ - - return get_default_resolver().query(qname, rdtype, rdclass, tcp, source, - raise_on_no_answer, source_port, - lifetime) - - -def zone_for_name(name, rdclass=dns.rdataclass.IN, tcp=False, resolver=None): - """Find the name of the zone which contains the specified name. - - *name*, an absolute ``dns.name.Name`` or ``text``, the query name. - - *rdclass*, an ``int``, the query class. - - *tcp*, a ``bool``. If ``True``, use TCP to make the query. - - *resolver*, a ``dns.resolver.Resolver`` or ``None``, the resolver to use. - If ``None``, the default resolver is used. - - Raises ``dns.resolver.NoRootSOA`` if there is no SOA RR at the DNS - root. (This is only likely to happen if you're using non-default - root servers in your network and they are misconfigured.) - - Returns a ``dns.name.Name``. - """ - - if isinstance(name, string_types): - name = dns.name.from_text(name, dns.name.root) - if resolver is None: - resolver = get_default_resolver() - if not name.is_absolute(): - raise NotAbsolute(name) - while 1: - try: - answer = resolver.query(name, dns.rdatatype.SOA, rdclass, tcp) - if answer.rrset.name == name: - return name - # otherwise we were CNAMEd or DNAMEd and need to look higher - except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): - pass - try: - name = name.parent() - except dns.name.NoParent: - raise NoRootSOA - -# -# Support for overriding the system resolver for all python code in the -# running process. -# - -_protocols_for_socktype = { - socket.SOCK_DGRAM: [socket.SOL_UDP], - socket.SOCK_STREAM: [socket.SOL_TCP], -} - -_resolver = None -_original_getaddrinfo = socket.getaddrinfo -_original_getnameinfo = socket.getnameinfo -_original_getfqdn = socket.getfqdn -_original_gethostbyname = socket.gethostbyname -_original_gethostbyname_ex = socket.gethostbyname_ex -_original_gethostbyaddr = socket.gethostbyaddr - - -def _getaddrinfo(host=None, service=None, family=socket.AF_UNSPEC, socktype=0, - proto=0, flags=0): - if flags & (socket.AI_ADDRCONFIG | socket.AI_V4MAPPED) != 0: - raise NotImplementedError - if host is None and service is None: - raise socket.gaierror(socket.EAI_NONAME) - v6addrs = [] - v4addrs = [] - canonical_name = None - try: - # Is host None or a V6 address literal? - if host is None: - canonical_name = 'localhost' - if flags & socket.AI_PASSIVE != 0: - v6addrs.append('::') - v4addrs.append('0.0.0.0') - else: - v6addrs.append('::1') - v4addrs.append('127.0.0.1') - else: - parts = host.split('%') - if len(parts) == 2: - ahost = parts[0] - else: - ahost = host - addr = dns.ipv6.inet_aton(ahost) - v6addrs.append(host) - canonical_name = host - except Exception: - try: - # Is it a V4 address literal? - addr = dns.ipv4.inet_aton(host) - v4addrs.append(host) - canonical_name = host - except Exception: - if flags & socket.AI_NUMERICHOST == 0: - try: - if family == socket.AF_INET6 or family == socket.AF_UNSPEC: - v6 = _resolver.query(host, dns.rdatatype.AAAA, - raise_on_no_answer=False) - # Note that setting host ensures we query the same name - # for A as we did for AAAA. - host = v6.qname - canonical_name = v6.canonical_name.to_text(True) - if v6.rrset is not None: - for rdata in v6.rrset: - v6addrs.append(rdata.address) - if family == socket.AF_INET or family == socket.AF_UNSPEC: - v4 = _resolver.query(host, dns.rdatatype.A, - raise_on_no_answer=False) - host = v4.qname - canonical_name = v4.canonical_name.to_text(True) - if v4.rrset is not None: - for rdata in v4.rrset: - v4addrs.append(rdata.address) - except dns.resolver.NXDOMAIN: - raise socket.gaierror(socket.EAI_NONAME) - except Exception: - raise socket.gaierror(socket.EAI_SYSTEM) - port = None - try: - # Is it a port literal? - if service is None: - port = 0 - else: - port = int(service) - except Exception: - if flags & socket.AI_NUMERICSERV == 0: - try: - port = socket.getservbyname(service) - except Exception: - pass - if port is None: - raise socket.gaierror(socket.EAI_NONAME) - tuples = [] - if socktype == 0: - socktypes = [socket.SOCK_DGRAM, socket.SOCK_STREAM] - else: - socktypes = [socktype] - if flags & socket.AI_CANONNAME != 0: - cname = canonical_name - else: - cname = '' - if family == socket.AF_INET6 or family == socket.AF_UNSPEC: - for addr in v6addrs: - for socktype in socktypes: - for proto in _protocols_for_socktype[socktype]: - tuples.append((socket.AF_INET6, socktype, proto, - cname, (addr, port, 0, 0))) - if family == socket.AF_INET or family == socket.AF_UNSPEC: - for addr in v4addrs: - for socktype in socktypes: - for proto in _protocols_for_socktype[socktype]: - tuples.append((socket.AF_INET, socktype, proto, - cname, (addr, port))) - if len(tuples) == 0: - raise socket.gaierror(socket.EAI_NONAME) - return tuples - - -def _getnameinfo(sockaddr, flags=0): - host = sockaddr[0] - port = sockaddr[1] - if len(sockaddr) == 4: - scope = sockaddr[3] - family = socket.AF_INET6 - else: - scope = None - family = socket.AF_INET - tuples = _getaddrinfo(host, port, family, socket.SOCK_STREAM, - socket.SOL_TCP, 0) - if len(tuples) > 1: - raise socket.error('sockaddr resolved to multiple addresses') - addr = tuples[0][4][0] - if flags & socket.NI_DGRAM: - pname = 'udp' - else: - pname = 'tcp' - qname = dns.reversename.from_address(addr) - if flags & socket.NI_NUMERICHOST == 0: - try: - answer = _resolver.query(qname, 'PTR') - hostname = answer.rrset[0].target.to_text(True) - except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): - if flags & socket.NI_NAMEREQD: - raise socket.gaierror(socket.EAI_NONAME) - hostname = addr - if scope is not None: - hostname += '%' + str(scope) - else: - hostname = addr - if scope is not None: - hostname += '%' + str(scope) - if flags & socket.NI_NUMERICSERV: - service = str(port) - else: - service = socket.getservbyport(port, pname) - return (hostname, service) - - -def _getfqdn(name=None): - if name is None: - name = socket.gethostname() - try: - return _getnameinfo(_getaddrinfo(name, 80)[0][4])[0] - except Exception: - return name - - -def _gethostbyname(name): - return _gethostbyname_ex(name)[2][0] - - -def _gethostbyname_ex(name): - aliases = [] - addresses = [] - tuples = _getaddrinfo(name, 0, socket.AF_INET, socket.SOCK_STREAM, - socket.SOL_TCP, socket.AI_CANONNAME) - canonical = tuples[0][3] - for item in tuples: - addresses.append(item[4][0]) - # XXX we just ignore aliases - return (canonical, aliases, addresses) - - -def _gethostbyaddr(ip): - try: - dns.ipv6.inet_aton(ip) - sockaddr = (ip, 80, 0, 0) - family = socket.AF_INET6 - except Exception: - sockaddr = (ip, 80) - family = socket.AF_INET - (name, port) = _getnameinfo(sockaddr, socket.NI_NAMEREQD) - aliases = [] - addresses = [] - tuples = _getaddrinfo(name, 0, family, socket.SOCK_STREAM, socket.SOL_TCP, - socket.AI_CANONNAME) - canonical = tuples[0][3] - for item in tuples: - addresses.append(item[4][0]) - # XXX we just ignore aliases - return (canonical, aliases, addresses) - - -def override_system_resolver(resolver=None): - """Override the system resolver routines in the socket module with - versions which use dnspython's resolver. - - This can be useful in testing situations where you want to control - the resolution behavior of python code without having to change - the system's resolver settings (e.g. /etc/resolv.conf). - - The resolver to use may be specified; if it's not, the default - resolver will be used. - - resolver, a ``dns.resolver.Resolver`` or ``None``, the resolver to use. - """ - - if resolver is None: - resolver = get_default_resolver() - global _resolver - _resolver = resolver - socket.getaddrinfo = _getaddrinfo - socket.getnameinfo = _getnameinfo - socket.getfqdn = _getfqdn - socket.gethostbyname = _gethostbyname - socket.gethostbyname_ex = _gethostbyname_ex - socket.gethostbyaddr = _gethostbyaddr - - -def restore_system_resolver(): - """Undo the effects of prior override_system_resolver().""" - - global _resolver - _resolver = None - socket.getaddrinfo = _original_getaddrinfo - socket.getnameinfo = _original_getnameinfo - socket.getfqdn = _original_getfqdn - socket.gethostbyname = _original_gethostbyname - socket.gethostbyname_ex = _original_gethostbyname_ex - socket.gethostbyaddr = _original_gethostbyaddr diff --git a/src/dns/resolver.pyi b/src/dns/resolver.pyi deleted file mode 100644 index e839ec21..00000000 --- a/src/dns/resolver.pyi +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Union, Optional, List -from . import exception, rdataclass, name, rdatatype - -import socket -_gethostbyname = socket.gethostbyname -class NXDOMAIN(exception.DNSException): - ... -def query(qname : str, rdtype : Union[int,str] = 0, rdclass : Union[int,str] = 0, - tcp=False, source=None, raise_on_no_answer=True, - source_port=0): - ... -class LRUCache: - def __init__(self, max_size=1000): - ... - def get(self, key): - ... - def put(self, key, val): - ... -class Answer: - def __init__(self, qname, rdtype, rdclass, response, - raise_on_no_answer=True): - ... -def zone_for_name(name, rdclass : int = rdataclass.IN, tcp=False, resolver : Optional[Resolver] = None): - ... - -class Resolver: - def __init__(self, configure): - self.nameservers : List[str] - def query(self, qname : str, rdtype : Union[int,str] = rdatatype.A, rdclass : Union[int,str] = rdataclass.IN, - tcp : bool = False, source : Optional[str] = None, raise_on_no_answer=True, source_port : int = 0): - ... diff --git a/src/dns/reversename.py b/src/dns/reversename.py deleted file mode 100644 index 8f095fa9..00000000 --- a/src/dns/reversename.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2006-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Reverse Map Names.""" - -import binascii - -import dns.name -import dns.ipv6 -import dns.ipv4 - -from dns._compat import PY3 - -ipv4_reverse_domain = dns.name.from_text('in-addr.arpa.') -ipv6_reverse_domain = dns.name.from_text('ip6.arpa.') - - -def from_address(text): - """Convert an IPv4 or IPv6 address in textual form into a Name object whose - value is the reverse-map domain name of the address. - - *text*, a ``text``, is an IPv4 or IPv6 address in textual form - (e.g. '127.0.0.1', '::1') - - Raises ``dns.exception.SyntaxError`` if the address is badly formed. - - Returns a ``dns.name.Name``. - """ - - try: - v6 = dns.ipv6.inet_aton(text) - if dns.ipv6.is_mapped(v6): - if PY3: - parts = ['%d' % byte for byte in v6[12:]] - else: - parts = ['%d' % ord(byte) for byte in v6[12:]] - origin = ipv4_reverse_domain - else: - parts = [x for x in str(binascii.hexlify(v6).decode())] - origin = ipv6_reverse_domain - except Exception: - parts = ['%d' % - byte for byte in bytearray(dns.ipv4.inet_aton(text))] - origin = ipv4_reverse_domain - parts.reverse() - return dns.name.from_text('.'.join(parts), origin=origin) - - -def to_address(name): - """Convert a reverse map domain name into textual address form. - - *name*, a ``dns.name.Name``, an IPv4 or IPv6 address in reverse-map name - form. - - Raises ``dns.exception.SyntaxError`` if the name does not have a - reverse-map form. - - Returns a ``text``. - """ - - if name.is_subdomain(ipv4_reverse_domain): - name = name.relativize(ipv4_reverse_domain) - labels = list(name.labels) - labels.reverse() - text = b'.'.join(labels) - # run through inet_aton() to check syntax and make pretty. - return dns.ipv4.inet_ntoa(dns.ipv4.inet_aton(text)) - elif name.is_subdomain(ipv6_reverse_domain): - name = name.relativize(ipv6_reverse_domain) - labels = list(name.labels) - labels.reverse() - parts = [] - i = 0 - l = len(labels) - while i < l: - parts.append(b''.join(labels[i:i + 4])) - i += 4 - text = b':'.join(parts) - # run through inet_aton() to check syntax and make pretty. - return dns.ipv6.inet_ntoa(dns.ipv6.inet_aton(text)) - else: - raise dns.exception.SyntaxError('unknown reverse-map address family') diff --git a/src/dns/reversename.pyi b/src/dns/reversename.pyi deleted file mode 100644 index 97f072ea..00000000 --- a/src/dns/reversename.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from . import name -def from_address(text : str) -> name.Name: - ... - -def to_address(name : name.Name) -> str: - ... diff --git a/src/dns/rrset.py b/src/dns/rrset.py deleted file mode 100644 index a53ec324..00000000 --- a/src/dns/rrset.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS RRsets (an RRset is a named rdataset)""" - - -import dns.name -import dns.rdataset -import dns.rdataclass -import dns.renderer -from ._compat import string_types - - -class RRset(dns.rdataset.Rdataset): - - """A DNS RRset (named rdataset). - - RRset inherits from Rdataset, and RRsets can be treated as - Rdatasets in most cases. There are, however, a few notable - exceptions. RRsets have different to_wire() and to_text() method - arguments, reflecting the fact that RRsets always have an owner - name. - """ - - __slots__ = ['name', 'deleting'] - - def __init__(self, name, rdclass, rdtype, covers=dns.rdatatype.NONE, - deleting=None): - """Create a new RRset.""" - - super(RRset, self).__init__(rdclass, rdtype, covers) - self.name = name - self.deleting = deleting - - def _clone(self): - obj = super(RRset, self)._clone() - obj.name = self.name - obj.deleting = self.deleting - return obj - - def __repr__(self): - if self.covers == 0: - ctext = '' - else: - ctext = '(' + dns.rdatatype.to_text(self.covers) + ')' - if self.deleting is not None: - dtext = ' delete=' + dns.rdataclass.to_text(self.deleting) - else: - dtext = '' - return '' - - def __str__(self): - return self.to_text() - - def __eq__(self, other): - if not isinstance(other, RRset): - return False - if self.name != other.name: - return False - return super(RRset, self).__eq__(other) - - def match(self, name, rdclass, rdtype, covers, deleting=None): - """Returns ``True`` if this rrset matches the specified class, type, - covers, and deletion state. - """ - - if not super(RRset, self).match(rdclass, rdtype, covers): - return False - if self.name != name or self.deleting != deleting: - return False - return True - - def to_text(self, origin=None, relativize=True, **kw): - """Convert the RRset into DNS master file format. - - See ``dns.name.Name.choose_relativity`` for more information - on how *origin* and *relativize* determine the way names - are emitted. - - Any additional keyword arguments are passed on to the rdata - ``to_text()`` method. - - *origin*, a ``dns.name.Name`` or ``None``, the origin for relative - names. - - *relativize*, a ``bool``. If ``True``, names will be relativized - to *origin*. - """ - - return super(RRset, self).to_text(self.name, origin, relativize, - self.deleting, **kw) - - def to_wire(self, file, compress=None, origin=None, **kw): - """Convert the RRset to wire format. - - All keyword arguments are passed to ``dns.rdataset.to_wire()``; see - that function for details. - - Returns an ``int``, the number of records emitted. - """ - - return super(RRset, self).to_wire(self.name, file, compress, origin, - self.deleting, **kw) - - def to_rdataset(self): - """Convert an RRset into an Rdataset. - - Returns a ``dns.rdataset.Rdataset``. - """ - return dns.rdataset.from_rdata_list(self.ttl, list(self)) - - -def from_text_list(name, ttl, rdclass, rdtype, text_rdatas, - idna_codec=None): - """Create an RRset with the specified name, TTL, class, and type, and with - the specified list of rdatas in text format. - - Returns a ``dns.rrset.RRset`` object. - """ - - if isinstance(name, string_types): - name = dns.name.from_text(name, None, idna_codec=idna_codec) - if isinstance(rdclass, string_types): - rdclass = dns.rdataclass.from_text(rdclass) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - r = RRset(name, rdclass, rdtype) - r.update_ttl(ttl) - for t in text_rdatas: - rd = dns.rdata.from_text(r.rdclass, r.rdtype, t) - r.add(rd) - return r - - -def from_text(name, ttl, rdclass, rdtype, *text_rdatas): - """Create an RRset with the specified name, TTL, class, and type and with - the specified rdatas in text format. - - Returns a ``dns.rrset.RRset`` object. - """ - - return from_text_list(name, ttl, rdclass, rdtype, text_rdatas) - - -def from_rdata_list(name, ttl, rdatas, idna_codec=None): - """Create an RRset with the specified name and TTL, and with - the specified list of rdata objects. - - Returns a ``dns.rrset.RRset`` object. - """ - - if isinstance(name, string_types): - name = dns.name.from_text(name, None, idna_codec=idna_codec) - - if len(rdatas) == 0: - raise ValueError("rdata list must not be empty") - r = None - for rd in rdatas: - if r is None: - r = RRset(name, rd.rdclass, rd.rdtype) - r.update_ttl(ttl) - r.add(rd) - return r - - -def from_rdata(name, ttl, *rdatas): - """Create an RRset with the specified name and TTL, and with - the specified rdata objects. - - Returns a ``dns.rrset.RRset`` object. - """ - - return from_rdata_list(name, ttl, rdatas) diff --git a/src/dns/rrset.pyi b/src/dns/rrset.pyi deleted file mode 100644 index 0a81a2a0..00000000 --- a/src/dns/rrset.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import List, Optional -from . import rdataset, rdatatype - -class RRset(rdataset.Rdataset): - def __init__(self, name, rdclass : int , rdtype : int, covers=rdatatype.NONE, - deleting : Optional[int] =None) -> None: - self.name = name - self.deleting = deleting -def from_text(name : str, ttl : int, rdclass : str, rdtype : str, *text_rdatas : str): - ... diff --git a/src/dns/set.py b/src/dns/set.py deleted file mode 100644 index 81329bf4..00000000 --- a/src/dns/set.py +++ /dev/null @@ -1,261 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -class Set(object): - - """A simple set class. - - This class was originally used to deal with sets being missing in - ancient versions of python, but dnspython will continue to use it - as these sets are based on lists and are thus indexable, and this - ability is widely used in dnspython applications. - """ - - __slots__ = ['items'] - - def __init__(self, items=None): - """Initialize the set. - - *items*, an iterable or ``None``, the initial set of items. - """ - - self.items = [] - if items is not None: - for item in items: - self.add(item) - - def __repr__(self): - return "dns.simpleset.Set(%s)" % repr(self.items) - - def add(self, item): - """Add an item to the set. - """ - - if item not in self.items: - self.items.append(item) - - def remove(self, item): - """Remove an item from the set. - """ - - self.items.remove(item) - - def discard(self, item): - """Remove an item from the set if present. - """ - - try: - self.items.remove(item) - except ValueError: - pass - - def _clone(self): - """Make a (shallow) copy of the set. - - There is a 'clone protocol' that subclasses of this class - should use. To make a copy, first call your super's _clone() - method, and use the object returned as the new instance. Then - make shallow copies of the attributes defined in the subclass. - - This protocol allows us to write the set algorithms that - return new instances (e.g. union) once, and keep using them in - subclasses. - """ - - cls = self.__class__ - obj = cls.__new__(cls) - obj.items = list(self.items) - return obj - - def __copy__(self): - """Make a (shallow) copy of the set. - """ - - return self._clone() - - def copy(self): - """Make a (shallow) copy of the set. - """ - - return self._clone() - - def union_update(self, other): - """Update the set, adding any elements from other which are not - already in the set. - """ - - if not isinstance(other, Set): - raise ValueError('other must be a Set instance') - if self is other: - return - for item in other.items: - self.add(item) - - def intersection_update(self, other): - """Update the set, removing any elements from other which are not - in both sets. - """ - - if not isinstance(other, Set): - raise ValueError('other must be a Set instance') - if self is other: - return - # we make a copy of the list so that we can remove items from - # the list without breaking the iterator. - for item in list(self.items): - if item not in other.items: - self.items.remove(item) - - def difference_update(self, other): - """Update the set, removing any elements from other which are in - the set. - """ - - if not isinstance(other, Set): - raise ValueError('other must be a Set instance') - if self is other: - self.items = [] - else: - for item in other.items: - self.discard(item) - - def union(self, other): - """Return a new set which is the union of ``self`` and ``other``. - - Returns the same Set type as this set. - """ - - obj = self._clone() - obj.union_update(other) - return obj - - def intersection(self, other): - """Return a new set which is the intersection of ``self`` and - ``other``. - - Returns the same Set type as this set. - """ - - obj = self._clone() - obj.intersection_update(other) - return obj - - def difference(self, other): - """Return a new set which ``self`` - ``other``, i.e. the items - in ``self`` which are not also in ``other``. - - Returns the same Set type as this set. - """ - - obj = self._clone() - obj.difference_update(other) - return obj - - def __or__(self, other): - return self.union(other) - - def __and__(self, other): - return self.intersection(other) - - def __add__(self, other): - return self.union(other) - - def __sub__(self, other): - return self.difference(other) - - def __ior__(self, other): - self.union_update(other) - return self - - def __iand__(self, other): - self.intersection_update(other) - return self - - def __iadd__(self, other): - self.union_update(other) - return self - - def __isub__(self, other): - self.difference_update(other) - return self - - def update(self, other): - """Update the set, adding any elements from other which are not - already in the set. - - *other*, the collection of items with which to update the set, which - may be any iterable type. - """ - - for item in other: - self.add(item) - - def clear(self): - """Make the set empty.""" - self.items = [] - - def __eq__(self, other): - # Yes, this is inefficient but the sets we're dealing with are - # usually quite small, so it shouldn't hurt too much. - for item in self.items: - if item not in other.items: - return False - for item in other.items: - if item not in self.items: - return False - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def __len__(self): - return len(self.items) - - def __iter__(self): - return iter(self.items) - - def __getitem__(self, i): - return self.items[i] - - def __delitem__(self, i): - del self.items[i] - - def issubset(self, other): - """Is this set a subset of *other*? - - Returns a ``bool``. - """ - - if not isinstance(other, Set): - raise ValueError('other must be a Set instance') - for item in self.items: - if item not in other.items: - return False - return True - - def issuperset(self, other): - """Is this set a superset of *other*? - - Returns a ``bool``. - """ - - if not isinstance(other, Set): - raise ValueError('other must be a Set instance') - for item in other.items: - if item not in self.items: - return False - return True diff --git a/src/dns/tokenizer.py b/src/dns/tokenizer.py deleted file mode 100644 index 880b71ce..00000000 --- a/src/dns/tokenizer.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""Tokenize DNS master file format""" - -from io import StringIO -import sys - -import dns.exception -import dns.name -import dns.ttl -from ._compat import long, text_type, binary_type - -_DELIMITERS = { - ' ': True, - '\t': True, - '\n': True, - ';': True, - '(': True, - ')': True, - '"': True} - -_QUOTING_DELIMITERS = {'"': True} - -EOF = 0 -EOL = 1 -WHITESPACE = 2 -IDENTIFIER = 3 -QUOTED_STRING = 4 -COMMENT = 5 -DELIMITER = 6 - - -class UngetBufferFull(dns.exception.DNSException): - """An attempt was made to unget a token when the unget buffer was full.""" - - -class Token(object): - """A DNS master file format token. - - ttype: The token type - value: The token value - has_escape: Does the token value contain escapes? - """ - - def __init__(self, ttype, value='', has_escape=False): - """Initialize a token instance.""" - - self.ttype = ttype - self.value = value - self.has_escape = has_escape - - def is_eof(self): - return self.ttype == EOF - - def is_eol(self): - return self.ttype == EOL - - def is_whitespace(self): - return self.ttype == WHITESPACE - - def is_identifier(self): - return self.ttype == IDENTIFIER - - def is_quoted_string(self): - return self.ttype == QUOTED_STRING - - def is_comment(self): - return self.ttype == COMMENT - - def is_delimiter(self): - return self.ttype == DELIMITER - - def is_eol_or_eof(self): - return self.ttype == EOL or self.ttype == EOF - - def __eq__(self, other): - if not isinstance(other, Token): - return False - return (self.ttype == other.ttype and - self.value == other.value) - - def __ne__(self, other): - if not isinstance(other, Token): - return True - return (self.ttype != other.ttype or - self.value != other.value) - - def __str__(self): - return '%d "%s"' % (self.ttype, self.value) - - def unescape(self): - if not self.has_escape: - return self - unescaped = '' - l = len(self.value) - i = 0 - while i < l: - c = self.value[i] - i += 1 - if c == '\\': - if i >= l: - raise dns.exception.UnexpectedEnd - c = self.value[i] - i += 1 - if c.isdigit(): - if i >= l: - raise dns.exception.UnexpectedEnd - c2 = self.value[i] - i += 1 - if i >= l: - raise dns.exception.UnexpectedEnd - c3 = self.value[i] - i += 1 - if not (c2.isdigit() and c3.isdigit()): - raise dns.exception.SyntaxError - c = chr(int(c) * 100 + int(c2) * 10 + int(c3)) - unescaped += c - return Token(self.ttype, unescaped) - - # compatibility for old-style tuple tokens - - def __len__(self): - return 2 - - def __iter__(self): - return iter((self.ttype, self.value)) - - def __getitem__(self, i): - if i == 0: - return self.ttype - elif i == 1: - return self.value - else: - raise IndexError - - -class Tokenizer(object): - """A DNS master file format tokenizer. - - A token object is basically a (type, value) tuple. The valid - types are EOF, EOL, WHITESPACE, IDENTIFIER, QUOTED_STRING, - COMMENT, and DELIMITER. - - file: The file to tokenize - - ungotten_char: The most recently ungotten character, or None. - - ungotten_token: The most recently ungotten token, or None. - - multiline: The current multiline level. This value is increased - by one every time a '(' delimiter is read, and decreased by one every time - a ')' delimiter is read. - - quoting: This variable is true if the tokenizer is currently - reading a quoted string. - - eof: This variable is true if the tokenizer has encountered EOF. - - delimiters: The current delimiter dictionary. - - line_number: The current line number - - filename: A filename that will be returned by the where() method. - """ - - def __init__(self, f=sys.stdin, filename=None): - """Initialize a tokenizer instance. - - f: The file to tokenize. The default is sys.stdin. - This parameter may also be a string, in which case the tokenizer - will take its input from the contents of the string. - - filename: the name of the filename that the where() method - will return. - """ - - if isinstance(f, text_type): - f = StringIO(f) - if filename is None: - filename = '' - elif isinstance(f, binary_type): - f = StringIO(f.decode()) - if filename is None: - filename = '' - else: - if filename is None: - if f is sys.stdin: - filename = '' - else: - filename = '' - self.file = f - self.ungotten_char = None - self.ungotten_token = None - self.multiline = 0 - self.quoting = False - self.eof = False - self.delimiters = _DELIMITERS - self.line_number = 1 - self.filename = filename - - def _get_char(self): - """Read a character from input. - """ - - if self.ungotten_char is None: - if self.eof: - c = '' - else: - c = self.file.read(1) - if c == '': - self.eof = True - elif c == '\n': - self.line_number += 1 - else: - c = self.ungotten_char - self.ungotten_char = None - return c - - def where(self): - """Return the current location in the input. - - Returns a (string, int) tuple. The first item is the filename of - the input, the second is the current line number. - """ - - return (self.filename, self.line_number) - - def _unget_char(self, c): - """Unget a character. - - The unget buffer for characters is only one character large; it is - an error to try to unget a character when the unget buffer is not - empty. - - c: the character to unget - raises UngetBufferFull: there is already an ungotten char - """ - - if self.ungotten_char is not None: - raise UngetBufferFull - self.ungotten_char = c - - def skip_whitespace(self): - """Consume input until a non-whitespace character is encountered. - - The non-whitespace character is then ungotten, and the number of - whitespace characters consumed is returned. - - If the tokenizer is in multiline mode, then newlines are whitespace. - - Returns the number of characters skipped. - """ - - skipped = 0 - while True: - c = self._get_char() - if c != ' ' and c != '\t': - if (c != '\n') or not self.multiline: - self._unget_char(c) - return skipped - skipped += 1 - - def get(self, want_leading=False, want_comment=False): - """Get the next token. - - want_leading: If True, return a WHITESPACE token if the - first character read is whitespace. The default is False. - - want_comment: If True, return a COMMENT token if the - first token read is a comment. The default is False. - - Raises dns.exception.UnexpectedEnd: input ended prematurely - - Raises dns.exception.SyntaxError: input was badly formed - - Returns a Token. - """ - - if self.ungotten_token is not None: - token = self.ungotten_token - self.ungotten_token = None - if token.is_whitespace(): - if want_leading: - return token - elif token.is_comment(): - if want_comment: - return token - else: - return token - skipped = self.skip_whitespace() - if want_leading and skipped > 0: - return Token(WHITESPACE, ' ') - token = '' - ttype = IDENTIFIER - has_escape = False - while True: - c = self._get_char() - if c == '' or c in self.delimiters: - if c == '' and self.quoting: - raise dns.exception.UnexpectedEnd - if token == '' and ttype != QUOTED_STRING: - if c == '(': - self.multiline += 1 - self.skip_whitespace() - continue - elif c == ')': - if self.multiline <= 0: - raise dns.exception.SyntaxError - self.multiline -= 1 - self.skip_whitespace() - continue - elif c == '"': - if not self.quoting: - self.quoting = True - self.delimiters = _QUOTING_DELIMITERS - ttype = QUOTED_STRING - continue - else: - self.quoting = False - self.delimiters = _DELIMITERS - self.skip_whitespace() - continue - elif c == '\n': - return Token(EOL, '\n') - elif c == ';': - while 1: - c = self._get_char() - if c == '\n' or c == '': - break - token += c - if want_comment: - self._unget_char(c) - return Token(COMMENT, token) - elif c == '': - if self.multiline: - raise dns.exception.SyntaxError( - 'unbalanced parentheses') - return Token(EOF) - elif self.multiline: - self.skip_whitespace() - token = '' - continue - else: - return Token(EOL, '\n') - else: - # This code exists in case we ever want a - # delimiter to be returned. It never produces - # a token currently. - token = c - ttype = DELIMITER - else: - self._unget_char(c) - break - elif self.quoting: - if c == '\\': - c = self._get_char() - if c == '': - raise dns.exception.UnexpectedEnd - if c.isdigit(): - c2 = self._get_char() - if c2 == '': - raise dns.exception.UnexpectedEnd - c3 = self._get_char() - if c == '': - raise dns.exception.UnexpectedEnd - if not (c2.isdigit() and c3.isdigit()): - raise dns.exception.SyntaxError - c = chr(int(c) * 100 + int(c2) * 10 + int(c3)) - elif c == '\n': - raise dns.exception.SyntaxError('newline in quoted string') - elif c == '\\': - # - # It's an escape. Put it and the next character into - # the token; it will be checked later for goodness. - # - token += c - has_escape = True - c = self._get_char() - if c == '' or c == '\n': - raise dns.exception.UnexpectedEnd - token += c - if token == '' and ttype != QUOTED_STRING: - if self.multiline: - raise dns.exception.SyntaxError('unbalanced parentheses') - ttype = EOF - return Token(ttype, token, has_escape) - - def unget(self, token): - """Unget a token. - - The unget buffer for tokens is only one token large; it is - an error to try to unget a token when the unget buffer is not - empty. - - token: the token to unget - - Raises UngetBufferFull: there is already an ungotten token - """ - - if self.ungotten_token is not None: - raise UngetBufferFull - self.ungotten_token = token - - def next(self): - """Return the next item in an iteration. - - Returns a Token. - """ - - token = self.get() - if token.is_eof(): - raise StopIteration - return token - - __next__ = next - - def __iter__(self): - return self - - # Helpers - - def get_int(self, base=10): - """Read the next token and interpret it as an integer. - - Raises dns.exception.SyntaxError if not an integer. - - Returns an int. - """ - - token = self.get().unescape() - if not token.is_identifier(): - raise dns.exception.SyntaxError('expecting an identifier') - if not token.value.isdigit(): - raise dns.exception.SyntaxError('expecting an integer') - return int(token.value, base) - - def get_uint8(self): - """Read the next token and interpret it as an 8-bit unsigned - integer. - - Raises dns.exception.SyntaxError if not an 8-bit unsigned integer. - - Returns an int. - """ - - value = self.get_int() - if value < 0 or value > 255: - raise dns.exception.SyntaxError( - '%d is not an unsigned 8-bit integer' % value) - return value - - def get_uint16(self, base=10): - """Read the next token and interpret it as a 16-bit unsigned - integer. - - Raises dns.exception.SyntaxError if not a 16-bit unsigned integer. - - Returns an int. - """ - - value = self.get_int(base=base) - if value < 0 or value > 65535: - if base == 8: - raise dns.exception.SyntaxError( - '%o is not an octal unsigned 16-bit integer' % value) - else: - raise dns.exception.SyntaxError( - '%d is not an unsigned 16-bit integer' % value) - return value - - def get_uint32(self): - """Read the next token and interpret it as a 32-bit unsigned - integer. - - Raises dns.exception.SyntaxError if not a 32-bit unsigned integer. - - Returns an int. - """ - - token = self.get().unescape() - if not token.is_identifier(): - raise dns.exception.SyntaxError('expecting an identifier') - if not token.value.isdigit(): - raise dns.exception.SyntaxError('expecting an integer') - value = long(token.value) - if value < 0 or value > long(4294967296): - raise dns.exception.SyntaxError( - '%d is not an unsigned 32-bit integer' % value) - return value - - def get_string(self, origin=None): - """Read the next token and interpret it as a string. - - Raises dns.exception.SyntaxError if not a string. - - Returns a string. - """ - - token = self.get().unescape() - if not (token.is_identifier() or token.is_quoted_string()): - raise dns.exception.SyntaxError('expecting a string') - return token.value - - def get_identifier(self, origin=None): - """Read the next token, which should be an identifier. - - Raises dns.exception.SyntaxError if not an identifier. - - Returns a string. - """ - - token = self.get().unescape() - if not token.is_identifier(): - raise dns.exception.SyntaxError('expecting an identifier') - return token.value - - def get_name(self, origin=None): - """Read the next token and interpret it as a DNS name. - - Raises dns.exception.SyntaxError if not a name. - - Returns a dns.name.Name. - """ - - token = self.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError('expecting an identifier') - return dns.name.from_text(token.value, origin) - - def get_eol(self): - """Read the next token and raise an exception if it isn't EOL or - EOF. - - Returns a string. - """ - - token = self.get() - if not token.is_eol_or_eof(): - raise dns.exception.SyntaxError( - 'expected EOL or EOF, got %d "%s"' % (token.ttype, - token.value)) - return token.value - - def get_ttl(self): - """Read the next token and interpret it as a DNS TTL. - - Raises dns.exception.SyntaxError or dns.ttl.BadTTL if not an - identifier or badly formed. - - Returns an int. - """ - - token = self.get().unescape() - if not token.is_identifier(): - raise dns.exception.SyntaxError('expecting an identifier') - return dns.ttl.from_text(token.value) diff --git a/src/dns/tsig.py b/src/dns/tsig.py deleted file mode 100644 index 3daa3878..00000000 --- a/src/dns/tsig.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2001-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS TSIG support.""" - -import hashlib -import hmac -import struct - -import dns.exception -import dns.rdataclass -import dns.name -from ._compat import long, string_types, text_type - -class BadTime(dns.exception.DNSException): - - """The current time is not within the TSIG's validity time.""" - - -class BadSignature(dns.exception.DNSException): - - """The TSIG signature fails to verify.""" - - -class PeerError(dns.exception.DNSException): - - """Base class for all TSIG errors generated by the remote peer""" - - -class PeerBadKey(PeerError): - - """The peer didn't know the key we used""" - - -class PeerBadSignature(PeerError): - - """The peer didn't like the signature we sent""" - - -class PeerBadTime(PeerError): - - """The peer didn't like the time we sent""" - - -class PeerBadTruncation(PeerError): - - """The peer didn't like amount of truncation in the TSIG we sent""" - -# TSIG Algorithms - -HMAC_MD5 = dns.name.from_text("HMAC-MD5.SIG-ALG.REG.INT") -HMAC_SHA1 = dns.name.from_text("hmac-sha1") -HMAC_SHA224 = dns.name.from_text("hmac-sha224") -HMAC_SHA256 = dns.name.from_text("hmac-sha256") -HMAC_SHA384 = dns.name.from_text("hmac-sha384") -HMAC_SHA512 = dns.name.from_text("hmac-sha512") - -_hashes = { - HMAC_SHA224: hashlib.sha224, - HMAC_SHA256: hashlib.sha256, - HMAC_SHA384: hashlib.sha384, - HMAC_SHA512: hashlib.sha512, - HMAC_SHA1: hashlib.sha1, - HMAC_MD5: hashlib.md5, -} - -default_algorithm = HMAC_MD5 - -BADSIG = 16 -BADKEY = 17 -BADTIME = 18 -BADTRUNC = 22 - - -def sign(wire, keyname, secret, time, fudge, original_id, error, - other_data, request_mac, ctx=None, multi=False, first=True, - algorithm=default_algorithm): - """Return a (tsig_rdata, mac, ctx) tuple containing the HMAC TSIG rdata - for the input parameters, the HMAC MAC calculated by applying the - TSIG signature algorithm, and the TSIG digest context. - @rtype: (string, string, hmac.HMAC object) - @raises ValueError: I{other_data} is too long - @raises NotImplementedError: I{algorithm} is not supported - """ - - if isinstance(other_data, text_type): - other_data = other_data.encode() - (algorithm_name, digestmod) = get_algorithm(algorithm) - if first: - ctx = hmac.new(secret, digestmod=digestmod) - ml = len(request_mac) - if ml > 0: - ctx.update(struct.pack('!H', ml)) - ctx.update(request_mac) - id = struct.pack('!H', original_id) - ctx.update(id) - ctx.update(wire[2:]) - if first: - ctx.update(keyname.to_digestable()) - ctx.update(struct.pack('!H', dns.rdataclass.ANY)) - ctx.update(struct.pack('!I', 0)) - long_time = time + long(0) - upper_time = (long_time >> 32) & long(0xffff) - lower_time = long_time & long(0xffffffff) - time_mac = struct.pack('!HIH', upper_time, lower_time, fudge) - pre_mac = algorithm_name + time_mac - ol = len(other_data) - if ol > 65535: - raise ValueError('TSIG Other Data is > 65535 bytes') - post_mac = struct.pack('!HH', error, ol) + other_data - if first: - ctx.update(pre_mac) - ctx.update(post_mac) - else: - ctx.update(time_mac) - mac = ctx.digest() - mpack = struct.pack('!H', len(mac)) - tsig_rdata = pre_mac + mpack + mac + id + post_mac - if multi: - ctx = hmac.new(secret, digestmod=digestmod) - ml = len(mac) - ctx.update(struct.pack('!H', ml)) - ctx.update(mac) - else: - ctx = None - return (tsig_rdata, mac, ctx) - - -def hmac_md5(wire, keyname, secret, time, fudge, original_id, error, - other_data, request_mac, ctx=None, multi=False, first=True, - algorithm=default_algorithm): - return sign(wire, keyname, secret, time, fudge, original_id, error, - other_data, request_mac, ctx, multi, first, algorithm) - - -def validate(wire, keyname, secret, now, request_mac, tsig_start, tsig_rdata, - tsig_rdlen, ctx=None, multi=False, first=True): - """Validate the specified TSIG rdata against the other input parameters. - - @raises FormError: The TSIG is badly formed. - @raises BadTime: There is too much time skew between the client and the - server. - @raises BadSignature: The TSIG signature did not validate - @rtype: hmac.HMAC object""" - - (adcount,) = struct.unpack("!H", wire[10:12]) - if adcount == 0: - raise dns.exception.FormError - adcount -= 1 - new_wire = wire[0:10] + struct.pack("!H", adcount) + wire[12:tsig_start] - current = tsig_rdata - (aname, used) = dns.name.from_wire(wire, current) - current = current + used - (upper_time, lower_time, fudge, mac_size) = \ - struct.unpack("!HIHH", wire[current:current + 10]) - time = ((upper_time + long(0)) << 32) + (lower_time + long(0)) - current += 10 - mac = wire[current:current + mac_size] - current += mac_size - (original_id, error, other_size) = \ - struct.unpack("!HHH", wire[current:current + 6]) - current += 6 - other_data = wire[current:current + other_size] - current += other_size - if current != tsig_rdata + tsig_rdlen: - raise dns.exception.FormError - if error != 0: - if error == BADSIG: - raise PeerBadSignature - elif error == BADKEY: - raise PeerBadKey - elif error == BADTIME: - raise PeerBadTime - elif error == BADTRUNC: - raise PeerBadTruncation - else: - raise PeerError('unknown TSIG error code %d' % error) - time_low = time - fudge - time_high = time + fudge - if now < time_low or now > time_high: - raise BadTime - (junk, our_mac, ctx) = sign(new_wire, keyname, secret, time, fudge, - original_id, error, other_data, - request_mac, ctx, multi, first, aname) - if our_mac != mac: - raise BadSignature - return ctx - - -def get_algorithm(algorithm): - """Returns the wire format string and the hash module to use for the - specified TSIG algorithm - - @rtype: (string, hash constructor) - @raises NotImplementedError: I{algorithm} is not supported - """ - - if isinstance(algorithm, string_types): - algorithm = dns.name.from_text(algorithm) - - try: - return (algorithm.to_digestable(), _hashes[algorithm]) - except KeyError: - raise NotImplementedError("TSIG algorithm " + str(algorithm) + - " is not supported") - - -def get_algorithm_and_mac(wire, tsig_rdata, tsig_rdlen): - """Return the tsig algorithm for the specified tsig_rdata - @raises FormError: The TSIG is badly formed. - """ - current = tsig_rdata - (aname, used) = dns.name.from_wire(wire, current) - current = current + used - (upper_time, lower_time, fudge, mac_size) = \ - struct.unpack("!HIHH", wire[current:current + 10]) - current += 10 - mac = wire[current:current + mac_size] - current += mac_size - if current > tsig_rdata + tsig_rdlen: - raise dns.exception.FormError - return (aname, mac) diff --git a/src/dns/tsigkeyring.py b/src/dns/tsigkeyring.py deleted file mode 100644 index 5e5fe1cb..00000000 --- a/src/dns/tsigkeyring.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""A place to store TSIG keys.""" - -from dns._compat import maybe_decode, maybe_encode - -import base64 - -import dns.name - - -def from_text(textring): - """Convert a dictionary containing (textual DNS name, base64 secret) pairs - into a binary keyring which has (dns.name.Name, binary secret) pairs. - @rtype: dict""" - - keyring = {} - for keytext in textring: - keyname = dns.name.from_text(keytext) - secret = base64.decodestring(maybe_encode(textring[keytext])) - keyring[keyname] = secret - return keyring - - -def to_text(keyring): - """Convert a dictionary containing (dns.name.Name, binary secret) pairs - into a text keyring which has (textual DNS name, base64 secret) pairs. - @rtype: dict""" - - textring = {} - for keyname in keyring: - keytext = maybe_decode(keyname.to_text()) - secret = maybe_decode(base64.encodestring(keyring[keyname])) - textring[keytext] = secret - return textring diff --git a/src/dns/tsigkeyring.pyi b/src/dns/tsigkeyring.pyi deleted file mode 100644 index b5d51e15..00000000 --- a/src/dns/tsigkeyring.pyi +++ /dev/null @@ -1,7 +0,0 @@ -from typing import Dict -from . import name - -def from_text(textring : Dict[str,str]) -> Dict[name.Name,bytes]: - ... -def to_text(keyring : Dict[name.Name,bytes]) -> Dict[str, str]: - ... diff --git a/src/dns/ttl.py b/src/dns/ttl.py deleted file mode 100644 index 4be16bee..00000000 --- a/src/dns/ttl.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS TTL conversion.""" - -import dns.exception -from ._compat import long - - -class BadTTL(dns.exception.SyntaxError): - """DNS TTL value is not well-formed.""" - - -def from_text(text): - """Convert the text form of a TTL to an integer. - - The BIND 8 units syntax for TTLs (e.g. '1w6d4h3m10s') is supported. - - *text*, a ``text``, the textual TTL. - - Raises ``dns.ttl.BadTTL`` if the TTL is not well-formed. - - Returns an ``int``. - """ - - if text.isdigit(): - total = long(text) - else: - if not text[0].isdigit(): - raise BadTTL - total = long(0) - current = long(0) - for c in text: - if c.isdigit(): - current *= 10 - current += long(c) - else: - c = c.lower() - if c == 'w': - total += current * long(604800) - elif c == 'd': - total += current * long(86400) - elif c == 'h': - total += current * long(3600) - elif c == 'm': - total += current * long(60) - elif c == 's': - total += current - else: - raise BadTTL("unknown unit '%s'" % c) - current = 0 - if not current == 0: - raise BadTTL("trailing integer") - if total < long(0) or total > long(2147483647): - raise BadTTL("TTL should be between 0 and 2^31 - 1 (inclusive)") - return total diff --git a/src/dns/update.py b/src/dns/update.py deleted file mode 100644 index 96a00d5d..00000000 --- a/src/dns/update.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Dynamic Update Support""" - - -import dns.message -import dns.name -import dns.opcode -import dns.rdata -import dns.rdataclass -import dns.rdataset -import dns.tsig -from ._compat import string_types - - -class Update(dns.message.Message): - - def __init__(self, zone, rdclass=dns.rdataclass.IN, keyring=None, - keyname=None, keyalgorithm=dns.tsig.default_algorithm): - """Initialize a new DNS Update object. - - See the documentation of the Message class for a complete - description of the keyring dictionary. - - *zone*, a ``dns.name.Name`` or ``text``, the zone which is being - updated. - - *rdclass*, an ``int`` or ``text``, the class of the zone. - - *keyring*, a ``dict``, the TSIG keyring to use. If a - *keyring* is specified but a *keyname* is not, then the key - used will be the first key in the *keyring*. Note that the - order of keys in a dictionary is not defined, so applications - should supply a keyname when a keyring is used, unless they - know the keyring contains only one key. - - *keyname*, a ``dns.name.Name`` or ``None``, the name of the TSIG key - to use; defaults to ``None``. The key must be defined in the keyring. - - *keyalgorithm*, a ``dns.name.Name``, the TSIG algorithm to use. - """ - super(Update, self).__init__() - self.flags |= dns.opcode.to_flags(dns.opcode.UPDATE) - if isinstance(zone, string_types): - zone = dns.name.from_text(zone) - self.origin = zone - if isinstance(rdclass, string_types): - rdclass = dns.rdataclass.from_text(rdclass) - self.zone_rdclass = rdclass - self.find_rrset(self.question, self.origin, rdclass, dns.rdatatype.SOA, - create=True, force_unique=True) - if keyring is not None: - self.use_tsig(keyring, keyname, algorithm=keyalgorithm) - - def _add_rr(self, name, ttl, rd, deleting=None, section=None): - """Add a single RR to the update section.""" - - if section is None: - section = self.authority - covers = rd.covers() - rrset = self.find_rrset(section, name, self.zone_rdclass, rd.rdtype, - covers, deleting, True, True) - rrset.add(rd, ttl) - - def _add(self, replace, section, name, *args): - """Add records. - - *replace* is the replacement mode. If ``False``, - RRs are added to an existing RRset; if ``True``, the RRset - is replaced with the specified contents. The second - argument is the section to add to. The third argument - is always a name. The other arguments can be: - - - rdataset... - - - ttl, rdata... - - - ttl, rdtype, string... - """ - - if isinstance(name, string_types): - name = dns.name.from_text(name, None) - if isinstance(args[0], dns.rdataset.Rdataset): - for rds in args: - if replace: - self.delete(name, rds.rdtype) - for rd in rds: - self._add_rr(name, rds.ttl, rd, section=section) - else: - args = list(args) - ttl = int(args.pop(0)) - if isinstance(args[0], dns.rdata.Rdata): - if replace: - self.delete(name, args[0].rdtype) - for rd in args: - self._add_rr(name, ttl, rd, section=section) - else: - rdtype = args.pop(0) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if replace: - self.delete(name, rdtype) - for s in args: - rd = dns.rdata.from_text(self.zone_rdclass, rdtype, s, - self.origin) - self._add_rr(name, ttl, rd, section=section) - - def add(self, name, *args): - """Add records. - - The first argument is always a name. The other - arguments can be: - - - rdataset... - - - ttl, rdata... - - - ttl, rdtype, string... - """ - - self._add(False, self.authority, name, *args) - - def delete(self, name, *args): - """Delete records. - - The first argument is always a name. The other - arguments can be: - - - *empty* - - - rdataset... - - - rdata... - - - rdtype, [string...] - """ - - if isinstance(name, string_types): - name = dns.name.from_text(name, None) - if len(args) == 0: - self.find_rrset(self.authority, name, dns.rdataclass.ANY, - dns.rdatatype.ANY, dns.rdatatype.NONE, - dns.rdatatype.ANY, True, True) - elif isinstance(args[0], dns.rdataset.Rdataset): - for rds in args: - for rd in rds: - self._add_rr(name, 0, rd, dns.rdataclass.NONE) - else: - args = list(args) - if isinstance(args[0], dns.rdata.Rdata): - for rd in args: - self._add_rr(name, 0, rd, dns.rdataclass.NONE) - else: - rdtype = args.pop(0) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if len(args) == 0: - self.find_rrset(self.authority, name, - self.zone_rdclass, rdtype, - dns.rdatatype.NONE, - dns.rdataclass.ANY, - True, True) - else: - for s in args: - rd = dns.rdata.from_text(self.zone_rdclass, rdtype, s, - self.origin) - self._add_rr(name, 0, rd, dns.rdataclass.NONE) - - def replace(self, name, *args): - """Replace records. - - The first argument is always a name. The other - arguments can be: - - - rdataset... - - - ttl, rdata... - - - ttl, rdtype, string... - - Note that if you want to replace the entire node, you should do - a delete of the name followed by one or more calls to add. - """ - - self._add(True, self.authority, name, *args) - - def present(self, name, *args): - """Require that an owner name (and optionally an rdata type, - or specific rdataset) exists as a prerequisite to the - execution of the update. - - The first argument is always a name. - The other arguments can be: - - - rdataset... - - - rdata... - - - rdtype, string... - """ - - if isinstance(name, string_types): - name = dns.name.from_text(name, None) - if len(args) == 0: - self.find_rrset(self.answer, name, - dns.rdataclass.ANY, dns.rdatatype.ANY, - dns.rdatatype.NONE, None, - True, True) - elif isinstance(args[0], dns.rdataset.Rdataset) or \ - isinstance(args[0], dns.rdata.Rdata) or \ - len(args) > 1: - if not isinstance(args[0], dns.rdataset.Rdataset): - # Add a 0 TTL - args = list(args) - args.insert(0, 0) - self._add(False, self.answer, name, *args) - else: - rdtype = args[0] - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - self.find_rrset(self.answer, name, - dns.rdataclass.ANY, rdtype, - dns.rdatatype.NONE, None, - True, True) - - def absent(self, name, rdtype=None): - """Require that an owner name (and optionally an rdata type) does - not exist as a prerequisite to the execution of the update.""" - - if isinstance(name, string_types): - name = dns.name.from_text(name, None) - if rdtype is None: - self.find_rrset(self.answer, name, - dns.rdataclass.NONE, dns.rdatatype.ANY, - dns.rdatatype.NONE, None, - True, True) - else: - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - self.find_rrset(self.answer, name, - dns.rdataclass.NONE, rdtype, - dns.rdatatype.NONE, None, - True, True) - - def to_wire(self, origin=None, max_size=65535): - """Return a string containing the update in DNS compressed wire - format. - - *origin*, a ``dns.name.Name`` or ``None``, the origin to be - appended to any relative names. If *origin* is ``None``, then - the origin of the ``dns.update.Update`` message object is used - (i.e. the *zone* parameter passed when the Update object was - created). - - *max_size*, an ``int``, the maximum size of the wire format - output; default is 0, which means "the message's request - payload, if nonzero, or 65535". - - Returns a ``binary``. - """ - - if origin is None: - origin = self.origin - return super(Update, self).to_wire(origin, max_size) diff --git a/src/dns/update.pyi b/src/dns/update.pyi deleted file mode 100644 index eeac0591..00000000 --- a/src/dns/update.pyi +++ /dev/null @@ -1,21 +0,0 @@ -from typing import Optional,Dict,Union,Any - -from . import message, tsig, rdataclass, name - -class Update(message.Message): - def __init__(self, zone : Union[name.Name, str], rdclass : Union[int,str] = rdataclass.IN, keyring : Optional[Dict[name.Name,bytes]] = None, - keyname : Optional[name.Name] = None, keyalgorithm : Optional[name.Name] = tsig.default_algorithm) -> None: - self.id : int - def add(self, name : Union[str,name.Name], *args : Any): - ... - def delete(self, name, *args : Any): - ... - def replace(self, name : Union[str,name.Name], *args : Any): - ... - def present(self, name : Union[str,name.Name], *args : Any): - ... - def absent(self, name : Union[str,name.Name], rdtype=None): - """Require that an owner name (and optionally an rdata type) does - not exist as a prerequisite to the execution of the update.""" - def to_wire(self, origin : Optional[name.Name] = None, max_size=65535, **kw) -> bytes: - ... diff --git a/src/dns/version.py b/src/dns/version.py deleted file mode 100644 index f116904b..00000000 --- a/src/dns/version.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""dnspython release version information.""" - -#: MAJOR -MAJOR = 1 -#: MINOR -MINOR = 16 -#: MICRO -MICRO = 0 -#: RELEASELEVEL -RELEASELEVEL = 0x0f -#: SERIAL -SERIAL = 0 - -if RELEASELEVEL == 0x0f: - #: version - version = '%d.%d.%d' % (MAJOR, MINOR, MICRO) -elif RELEASELEVEL == 0x00: - version = '%d.%d.%dx%d' % \ - (MAJOR, MINOR, MICRO, SERIAL) -else: - version = '%d.%d.%d%x%d' % \ - (MAJOR, MINOR, MICRO, RELEASELEVEL, SERIAL) - -#: hexversion -hexversion = MAJOR << 24 | MINOR << 16 | MICRO << 8 | RELEASELEVEL << 4 | \ - SERIAL diff --git a/src/dns/wiredata.py b/src/dns/wiredata.py deleted file mode 100644 index ea3c1e67..00000000 --- a/src/dns/wiredata.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2011,2017 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Wire Data Helper""" - -import dns.exception -from ._compat import binary_type, string_types, PY2 - -# Figure out what constant python passes for an unspecified slice bound. -# It's supposed to be sys.maxint, yet on 64-bit windows sys.maxint is 2^31 - 1 -# but Python uses 2^63 - 1 as the constant. Rather than making pointless -# extra comparisons, duplicating code, or weakening WireData, we just figure -# out what constant Python will use. - - -class _SliceUnspecifiedBound(binary_type): - - def __getitem__(self, key): - return key.stop - - if PY2: - def __getslice__(self, i, j): # pylint: disable=getslice-method - return self.__getitem__(slice(i, j)) - -_unspecified_bound = _SliceUnspecifiedBound()[1:] - - -class WireData(binary_type): - # WireData is a binary type with stricter slicing - - def __getitem__(self, key): - try: - if isinstance(key, slice): - # make sure we are not going outside of valid ranges, - # do stricter control of boundaries than python does - # by default - start = key.start - stop = key.stop - - if PY2: - if stop == _unspecified_bound: - # handle the case where the right bound is unspecified - stop = len(self) - - if start < 0 or stop < 0: - raise dns.exception.FormError - # If it's not an empty slice, access left and right bounds - # to make sure they're valid - if start != stop: - super(WireData, self).__getitem__(start) - super(WireData, self).__getitem__(stop - 1) - else: - for index in (start, stop): - if index is None: - continue - elif abs(index) > len(self): - raise dns.exception.FormError - - return WireData(super(WireData, self).__getitem__( - slice(start, stop))) - return bytearray(self.unwrap())[key] - except IndexError: - raise dns.exception.FormError - - if PY2: - def __getslice__(self, i, j): # pylint: disable=getslice-method - return self.__getitem__(slice(i, j)) - - def __iter__(self): - i = 0 - while 1: - try: - yield self[i] - i += 1 - except dns.exception.FormError: - raise StopIteration - - def unwrap(self): - return binary_type(self) - - -def maybe_wrap(wire): - if isinstance(wire, WireData): - return wire - elif isinstance(wire, binary_type): - return WireData(wire) - elif isinstance(wire, string_types): - return WireData(wire.encode()) - raise ValueError("unhandled type %s" % type(wire)) diff --git a/src/dns/zone.py b/src/dns/zone.py deleted file mode 100644 index 1e2fe781..00000000 --- a/src/dns/zone.py +++ /dev/null @@ -1,1127 +0,0 @@ -# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license - -# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc. -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose with or without fee is hereby granted, -# provided that the above copyright notice and this permission notice -# appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -"""DNS Zones.""" - -from __future__ import generators - -import sys -import re -import os -from io import BytesIO - -import dns.exception -import dns.name -import dns.node -import dns.rdataclass -import dns.rdatatype -import dns.rdata -import dns.rdtypes.ANY.SOA -import dns.rrset -import dns.tokenizer -import dns.ttl -import dns.grange -from ._compat import string_types, text_type, PY3 - - -class BadZone(dns.exception.DNSException): - - """The DNS zone is malformed.""" - - -class NoSOA(BadZone): - - """The DNS zone has no SOA RR at its origin.""" - - -class NoNS(BadZone): - - """The DNS zone has no NS RRset at its origin.""" - - -class UnknownOrigin(BadZone): - - """The DNS zone's origin is unknown.""" - - -class Zone(object): - - """A DNS zone. - - A Zone is a mapping from names to nodes. The zone object may be - treated like a Python dictionary, e.g. zone[name] will retrieve - the node associated with that name. The I{name} may be a - dns.name.Name object, or it may be a string. In the either case, - if the name is relative it is treated as relative to the origin of - the zone. - - @ivar rdclass: The zone's rdata class; the default is class IN. - @type rdclass: int - @ivar origin: The origin of the zone. - @type origin: dns.name.Name object - @ivar nodes: A dictionary mapping the names of nodes in the zone to the - nodes themselves. - @type nodes: dict - @ivar relativize: should names in the zone be relativized? - @type relativize: bool - @cvar node_factory: the factory used to create a new node - @type node_factory: class or callable - """ - - node_factory = dns.node.Node - - __slots__ = ['rdclass', 'origin', 'nodes', 'relativize'] - - def __init__(self, origin, rdclass=dns.rdataclass.IN, relativize=True): - """Initialize a zone object. - - @param origin: The origin of the zone. - @type origin: dns.name.Name object - @param rdclass: The zone's rdata class; the default is class IN. - @type rdclass: int""" - - if origin is not None: - if isinstance(origin, string_types): - origin = dns.name.from_text(origin) - elif not isinstance(origin, dns.name.Name): - raise ValueError("origin parameter must be convertible to a " - "DNS name") - if not origin.is_absolute(): - raise ValueError("origin parameter must be an absolute name") - self.origin = origin - self.rdclass = rdclass - self.nodes = {} - self.relativize = relativize - - def __eq__(self, other): - """Two zones are equal if they have the same origin, class, and - nodes. - @rtype: bool - """ - - if not isinstance(other, Zone): - return False - if self.rdclass != other.rdclass or \ - self.origin != other.origin or \ - self.nodes != other.nodes: - return False - return True - - def __ne__(self, other): - """Are two zones not equal? - @rtype: bool - """ - - return not self.__eq__(other) - - def _validate_name(self, name): - if isinstance(name, string_types): - name = dns.name.from_text(name, None) - elif not isinstance(name, dns.name.Name): - raise KeyError("name parameter must be convertible to a DNS name") - if name.is_absolute(): - if not name.is_subdomain(self.origin): - raise KeyError( - "name parameter must be a subdomain of the zone origin") - if self.relativize: - name = name.relativize(self.origin) - return name - - def __getitem__(self, key): - key = self._validate_name(key) - return self.nodes[key] - - def __setitem__(self, key, value): - key = self._validate_name(key) - self.nodes[key] = value - - def __delitem__(self, key): - key = self._validate_name(key) - del self.nodes[key] - - def __iter__(self): - return self.nodes.__iter__() - - def iterkeys(self): - if PY3: - return self.nodes.keys() # pylint: disable=dict-keys-not-iterating - else: - return self.nodes.iterkeys() # pylint: disable=dict-iter-method - - def keys(self): - return self.nodes.keys() # pylint: disable=dict-keys-not-iterating - - def itervalues(self): - if PY3: - return self.nodes.values() # pylint: disable=dict-values-not-iterating - else: - return self.nodes.itervalues() # pylint: disable=dict-iter-method - - def values(self): - return self.nodes.values() # pylint: disable=dict-values-not-iterating - - def items(self): - return self.nodes.items() # pylint: disable=dict-items-not-iterating - - iteritems = items - - def get(self, key): - key = self._validate_name(key) - return self.nodes.get(key) - - def __contains__(self, other): - return other in self.nodes - - def find_node(self, name, create=False): - """Find a node in the zone, possibly creating it. - - @param name: the name of the node to find - @type name: dns.name.Name object or string - @param create: should the node be created if it doesn't exist? - @type create: bool - @raises KeyError: the name is not known and create was not specified. - @rtype: dns.node.Node object - """ - - name = self._validate_name(name) - node = self.nodes.get(name) - if node is None: - if not create: - raise KeyError - node = self.node_factory() - self.nodes[name] = node - return node - - def get_node(self, name, create=False): - """Get a node in the zone, possibly creating it. - - This method is like L{find_node}, except it returns None instead - of raising an exception if the node does not exist and creation - has not been requested. - - @param name: the name of the node to find - @type name: dns.name.Name object or string - @param create: should the node be created if it doesn't exist? - @type create: bool - @rtype: dns.node.Node object or None - """ - - try: - node = self.find_node(name, create) - except KeyError: - node = None - return node - - def delete_node(self, name): - """Delete the specified node if it exists. - - It is not an error if the node does not exist. - """ - - name = self._validate_name(name) - if name in self.nodes: - del self.nodes[name] - - def find_rdataset(self, name, rdtype, covers=dns.rdatatype.NONE, - create=False): - """Look for rdata with the specified name and type in the zone, - and return an rdataset encapsulating it. - - The I{name}, I{rdtype}, and I{covers} parameters may be - strings, in which case they will be converted to their proper - type. - - The rdataset returned is not a copy; changes to it will change - the zone. - - KeyError is raised if the name or type are not found. - Use L{get_rdataset} if you want to have None returned instead. - - @param name: the owner name to look for - @type name: DNS.name.Name object or string - @param rdtype: the rdata type desired - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - @param create: should the node and rdataset be created if they do not - exist? - @type create: bool - @raises KeyError: the node or rdata could not be found - @rtype: dns.rdataset.Rdataset object - """ - - name = self._validate_name(name) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if isinstance(covers, string_types): - covers = dns.rdatatype.from_text(covers) - node = self.find_node(name, create) - return node.find_rdataset(self.rdclass, rdtype, covers, create) - - def get_rdataset(self, name, rdtype, covers=dns.rdatatype.NONE, - create=False): - """Look for rdata with the specified name and type in the zone, - and return an rdataset encapsulating it. - - The I{name}, I{rdtype}, and I{covers} parameters may be - strings, in which case they will be converted to their proper - type. - - The rdataset returned is not a copy; changes to it will change - the zone. - - None is returned if the name or type are not found. - Use L{find_rdataset} if you want to have KeyError raised instead. - - @param name: the owner name to look for - @type name: DNS.name.Name object or string - @param rdtype: the rdata type desired - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - @param create: should the node and rdataset be created if they do not - exist? - @type create: bool - @rtype: dns.rdataset.Rdataset object or None - """ - - try: - rdataset = self.find_rdataset(name, rdtype, covers, create) - except KeyError: - rdataset = None - return rdataset - - def delete_rdataset(self, name, rdtype, covers=dns.rdatatype.NONE): - """Delete the rdataset matching I{rdtype} and I{covers}, if it - exists at the node specified by I{name}. - - The I{name}, I{rdtype}, and I{covers} parameters may be - strings, in which case they will be converted to their proper - type. - - It is not an error if the node does not exist, or if there is no - matching rdataset at the node. - - If the node has no rdatasets after the deletion, it will itself - be deleted. - - @param name: the owner name to look for - @type name: DNS.name.Name object or string - @param rdtype: the rdata type desired - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - """ - - name = self._validate_name(name) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if isinstance(covers, string_types): - covers = dns.rdatatype.from_text(covers) - node = self.get_node(name) - if node is not None: - node.delete_rdataset(self.rdclass, rdtype, covers) - if len(node) == 0: - self.delete_node(name) - - def replace_rdataset(self, name, replacement): - """Replace an rdataset at name. - - It is not an error if there is no rdataset matching I{replacement}. - - Ownership of the I{replacement} object is transferred to the zone; - in other words, this method does not store a copy of I{replacement} - at the node, it stores I{replacement} itself. - - If the I{name} node does not exist, it is created. - - @param name: the owner name - @type name: DNS.name.Name object or string - @param replacement: the replacement rdataset - @type replacement: dns.rdataset.Rdataset - """ - - if replacement.rdclass != self.rdclass: - raise ValueError('replacement.rdclass != zone.rdclass') - node = self.find_node(name, True) - node.replace_rdataset(replacement) - - def find_rrset(self, name, rdtype, covers=dns.rdatatype.NONE): - """Look for rdata with the specified name and type in the zone, - and return an RRset encapsulating it. - - The I{name}, I{rdtype}, and I{covers} parameters may be - strings, in which case they will be converted to their proper - type. - - This method is less efficient than the similar - L{find_rdataset} because it creates an RRset instead of - returning the matching rdataset. It may be more convenient - for some uses since it returns an object which binds the owner - name to the rdata. - - This method may not be used to create new nodes or rdatasets; - use L{find_rdataset} instead. - - KeyError is raised if the name or type are not found. - Use L{get_rrset} if you want to have None returned instead. - - @param name: the owner name to look for - @type name: DNS.name.Name object or string - @param rdtype: the rdata type desired - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - @raises KeyError: the node or rdata could not be found - @rtype: dns.rrset.RRset object - """ - - name = self._validate_name(name) - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if isinstance(covers, string_types): - covers = dns.rdatatype.from_text(covers) - rdataset = self.nodes[name].find_rdataset(self.rdclass, rdtype, covers) - rrset = dns.rrset.RRset(name, self.rdclass, rdtype, covers) - rrset.update(rdataset) - return rrset - - def get_rrset(self, name, rdtype, covers=dns.rdatatype.NONE): - """Look for rdata with the specified name and type in the zone, - and return an RRset encapsulating it. - - The I{name}, I{rdtype}, and I{covers} parameters may be - strings, in which case they will be converted to their proper - type. - - This method is less efficient than the similar L{get_rdataset} - because it creates an RRset instead of returning the matching - rdataset. It may be more convenient for some uses since it - returns an object which binds the owner name to the rdata. - - This method may not be used to create new nodes or rdatasets; - use L{find_rdataset} instead. - - None is returned if the name or type are not found. - Use L{find_rrset} if you want to have KeyError raised instead. - - @param name: the owner name to look for - @type name: DNS.name.Name object or string - @param rdtype: the rdata type desired - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - @rtype: dns.rrset.RRset object - """ - - try: - rrset = self.find_rrset(name, rdtype, covers) - except KeyError: - rrset = None - return rrset - - def iterate_rdatasets(self, rdtype=dns.rdatatype.ANY, - covers=dns.rdatatype.NONE): - """Return a generator which yields (name, rdataset) tuples for - all rdatasets in the zone which have the specified I{rdtype} - and I{covers}. If I{rdtype} is dns.rdatatype.ANY, the default, - then all rdatasets will be matched. - - @param rdtype: int or string - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - """ - - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if isinstance(covers, string_types): - covers = dns.rdatatype.from_text(covers) - for (name, node) in self.iteritems(): # pylint: disable=dict-iter-method - for rds in node: - if rdtype == dns.rdatatype.ANY or \ - (rds.rdtype == rdtype and rds.covers == covers): - yield (name, rds) - - def iterate_rdatas(self, rdtype=dns.rdatatype.ANY, - covers=dns.rdatatype.NONE): - """Return a generator which yields (name, ttl, rdata) tuples for - all rdatas in the zone which have the specified I{rdtype} - and I{covers}. If I{rdtype} is dns.rdatatype.ANY, the default, - then all rdatas will be matched. - - @param rdtype: int or string - @type rdtype: int or string - @param covers: the covered type (defaults to None) - @type covers: int or string - """ - - if isinstance(rdtype, string_types): - rdtype = dns.rdatatype.from_text(rdtype) - if isinstance(covers, string_types): - covers = dns.rdatatype.from_text(covers) - for (name, node) in self.iteritems(): # pylint: disable=dict-iter-method - for rds in node: - if rdtype == dns.rdatatype.ANY or \ - (rds.rdtype == rdtype and rds.covers == covers): - for rdata in rds: - yield (name, rds.ttl, rdata) - - def to_file(self, f, sorted=True, relativize=True, nl=None): - """Write a zone to a file. - - @param f: file or string. If I{f} is a string, it is treated - as the name of a file to open. - @param sorted: if True, the file will be written with the - names sorted in DNSSEC order from least to greatest. Otherwise - the names will be written in whatever order they happen to have - in the zone's dictionary. - @param relativize: if True, domain names in the output will be - relativized to the zone's origin (if possible). - @type relativize: bool - @param nl: The end of line string. If not specified, the - output will use the platform's native end-of-line marker (i.e. - LF on POSIX, CRLF on Windows, CR on Macintosh). - @type nl: string or None - """ - - if isinstance(f, string_types): - f = open(f, 'wb') - want_close = True - else: - want_close = False - - # must be in this way, f.encoding may contain None, or even attribute - # may not be there - file_enc = getattr(f, 'encoding', None) - if file_enc is None: - file_enc = 'utf-8' - - if nl is None: - nl_b = os.linesep.encode(file_enc) # binary mode, '\n' is not enough - nl = u'\n' - elif isinstance(nl, string_types): - nl_b = nl.encode(file_enc) - else: - nl_b = nl - nl = nl.decode() - - try: - if sorted: - names = list(self.keys()) - names.sort() - else: - names = self.iterkeys() # pylint: disable=dict-iter-method - for n in names: - l = self[n].to_text(n, origin=self.origin, - relativize=relativize) - if isinstance(l, text_type): - l_b = l.encode(file_enc) - else: - l_b = l - l = l.decode() - - try: - f.write(l_b) - f.write(nl_b) - except TypeError: # textual mode - f.write(l) - f.write(nl) - finally: - if want_close: - f.close() - - def to_text(self, sorted=True, relativize=True, nl=None): - """Return a zone's text as though it were written to a file. - - @param sorted: if True, the file will be written with the - names sorted in DNSSEC order from least to greatest. Otherwise - the names will be written in whatever order they happen to have - in the zone's dictionary. - @param relativize: if True, domain names in the output will be - relativized to the zone's origin (if possible). - @type relativize: bool - @param nl: The end of line string. If not specified, the - output will use the platform's native end-of-line marker (i.e. - LF on POSIX, CRLF on Windows, CR on Macintosh). - @type nl: string or None - """ - temp_buffer = BytesIO() - self.to_file(temp_buffer, sorted, relativize, nl) - return_value = temp_buffer.getvalue() - temp_buffer.close() - return return_value - - def check_origin(self): - """Do some simple checking of the zone's origin. - - @raises dns.zone.NoSOA: there is no SOA RR - @raises dns.zone.NoNS: there is no NS RRset - @raises KeyError: there is no origin node - """ - if self.relativize: - name = dns.name.empty - else: - name = self.origin - if self.get_rdataset(name, dns.rdatatype.SOA) is None: - raise NoSOA - if self.get_rdataset(name, dns.rdatatype.NS) is None: - raise NoNS - - -class _MasterReader(object): - - """Read a DNS master file - - @ivar tok: The tokenizer - @type tok: dns.tokenizer.Tokenizer object - @ivar last_ttl: The last seen explicit TTL for an RR - @type last_ttl: int - @ivar last_ttl_known: Has last TTL been detected - @type last_ttl_known: bool - @ivar default_ttl: The default TTL from a $TTL directive or SOA RR - @type default_ttl: int - @ivar default_ttl_known: Has default TTL been detected - @type default_ttl_known: bool - @ivar last_name: The last name read - @type last_name: dns.name.Name object - @ivar current_origin: The current origin - @type current_origin: dns.name.Name object - @ivar relativize: should names in the zone be relativized? - @type relativize: bool - @ivar zone: the zone - @type zone: dns.zone.Zone object - @ivar saved_state: saved reader state (used when processing $INCLUDE) - @type saved_state: list of (tokenizer, current_origin, last_name, file, - last_ttl, last_ttl_known, default_ttl, default_ttl_known) tuples. - @ivar current_file: the file object of the $INCLUDed file being parsed - (None if no $INCLUDE is active). - @ivar allow_include: is $INCLUDE allowed? - @type allow_include: bool - @ivar check_origin: should sanity checks of the origin node be done? - The default is True. - @type check_origin: bool - """ - - def __init__(self, tok, origin, rdclass, relativize, zone_factory=Zone, - allow_include=False, check_origin=True): - if isinstance(origin, string_types): - origin = dns.name.from_text(origin) - self.tok = tok - self.current_origin = origin - self.relativize = relativize - self.last_ttl = 0 - self.last_ttl_known = False - self.default_ttl = 0 - self.default_ttl_known = False - self.last_name = self.current_origin - self.zone = zone_factory(origin, rdclass, relativize=relativize) - self.saved_state = [] - self.current_file = None - self.allow_include = allow_include - self.check_origin = check_origin - - def _eat_line(self): - while 1: - token = self.tok.get() - if token.is_eol_or_eof(): - break - - def _rr_line(self): - """Process one line from a DNS master file.""" - # Name - if self.current_origin is None: - raise UnknownOrigin - token = self.tok.get(want_leading=True) - if not token.is_whitespace(): - self.last_name = dns.name.from_text( - token.value, self.current_origin) - else: - token = self.tok.get() - if token.is_eol_or_eof(): - # treat leading WS followed by EOL/EOF as if they were EOL/EOF. - return - self.tok.unget(token) - name = self.last_name - if not name.is_subdomain(self.zone.origin): - self._eat_line() - return - if self.relativize: - name = name.relativize(self.zone.origin) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - # TTL - try: - ttl = dns.ttl.from_text(token.value) - self.last_ttl = ttl - self.last_ttl_known = True - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except dns.ttl.BadTTL: - if not (self.last_ttl_known or self.default_ttl_known): - raise dns.exception.SyntaxError("Missing default TTL value") - if self.default_ttl_known: - ttl = self.default_ttl - else: - ttl = self.last_ttl - # Class - try: - rdclass = dns.rdataclass.from_text(token.value) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except dns.exception.SyntaxError: - raise dns.exception.SyntaxError - except Exception: - rdclass = self.zone.rdclass - if rdclass != self.zone.rdclass: - raise dns.exception.SyntaxError("RR class is not zone's class") - # Type - try: - rdtype = dns.rdatatype.from_text(token.value) - except: - raise dns.exception.SyntaxError( - "unknown rdatatype '%s'" % token.value) - n = self.zone.nodes.get(name) - if n is None: - n = self.zone.node_factory() - self.zone.nodes[name] = n - try: - rd = dns.rdata.from_text(rdclass, rdtype, self.tok, - self.current_origin, False) - except dns.exception.SyntaxError: - # Catch and reraise. - (ty, va) = sys.exc_info()[:2] - raise va - except: - # All exceptions that occur in the processing of rdata - # are treated as syntax errors. This is not strictly - # correct, but it is correct almost all of the time. - # We convert them to syntax errors so that we can emit - # helpful filename:line info. - (ty, va) = sys.exc_info()[:2] - raise dns.exception.SyntaxError( - "caught exception {}: {}".format(str(ty), str(va))) - - if not self.default_ttl_known and isinstance(rd, dns.rdtypes.ANY.SOA.SOA): - # The pre-RFC2308 and pre-BIND9 behavior inherits the zone default - # TTL from the SOA minttl if no $TTL statement is present before the - # SOA is parsed. - self.default_ttl = rd.minimum - self.default_ttl_known = True - - rd.choose_relativity(self.zone.origin, self.relativize) - covers = rd.covers() - rds = n.find_rdataset(rdclass, rdtype, covers, True) - rds.add(rd, ttl) - - def _parse_modify(self, side): - # Here we catch everything in '{' '}' in a group so we can replace it - # with ''. - is_generate1 = re.compile("^.*\$({(\+|-?)(\d+),(\d+),(.)}).*$") - is_generate2 = re.compile("^.*\$({(\+|-?)(\d+)}).*$") - is_generate3 = re.compile("^.*\$({(\+|-?)(\d+),(\d+)}).*$") - # Sometimes there are modifiers in the hostname. These come after - # the dollar sign. They are in the form: ${offset[,width[,base]]}. - # Make names - g1 = is_generate1.match(side) - if g1: - mod, sign, offset, width, base = g1.groups() - if sign == '': - sign = '+' - g2 = is_generate2.match(side) - if g2: - mod, sign, offset = g2.groups() - if sign == '': - sign = '+' - width = 0 - base = 'd' - g3 = is_generate3.match(side) - if g3: - mod, sign, offset, width = g1.groups() - if sign == '': - sign = '+' - width = g1.groups()[2] - base = 'd' - - if not (g1 or g2 or g3): - mod = '' - sign = '+' - offset = 0 - width = 0 - base = 'd' - - if base != 'd': - raise NotImplementedError() - - return mod, sign, offset, width, base - - def _generate_line(self): - # range lhs [ttl] [class] type rhs [ comment ] - """Process one line containing the GENERATE statement from a DNS - master file.""" - if self.current_origin is None: - raise UnknownOrigin - - token = self.tok.get() - # Range (required) - try: - start, stop, step = dns.grange.from_text(token.value) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except: - raise dns.exception.SyntaxError - - # lhs (required) - try: - lhs = token.value - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except: - raise dns.exception.SyntaxError - - # TTL - try: - ttl = dns.ttl.from_text(token.value) - self.last_ttl = ttl - self.last_ttl_known = True - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except dns.ttl.BadTTL: - if not (self.last_ttl_known or self.default_ttl_known): - raise dns.exception.SyntaxError("Missing default TTL value") - if self.default_ttl_known: - ttl = self.default_ttl - else: - ttl = self.last_ttl - # Class - try: - rdclass = dns.rdataclass.from_text(token.value) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except dns.exception.SyntaxError: - raise dns.exception.SyntaxError - except Exception: - rdclass = self.zone.rdclass - if rdclass != self.zone.rdclass: - raise dns.exception.SyntaxError("RR class is not zone's class") - # Type - try: - rdtype = dns.rdatatype.from_text(token.value) - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError - except Exception: - raise dns.exception.SyntaxError("unknown rdatatype '%s'" % - token.value) - - # lhs (required) - try: - rhs = token.value - except: - raise dns.exception.SyntaxError - - lmod, lsign, loffset, lwidth, lbase = self._parse_modify(lhs) - rmod, rsign, roffset, rwidth, rbase = self._parse_modify(rhs) - for i in range(start, stop + 1, step): - # +1 because bind is inclusive and python is exclusive - - if lsign == u'+': - lindex = i + int(loffset) - elif lsign == u'-': - lindex = i - int(loffset) - - if rsign == u'-': - rindex = i - int(roffset) - elif rsign == u'+': - rindex = i + int(roffset) - - lzfindex = str(lindex).zfill(int(lwidth)) - rzfindex = str(rindex).zfill(int(rwidth)) - - name = lhs.replace(u'$%s' % (lmod), lzfindex) - rdata = rhs.replace(u'$%s' % (rmod), rzfindex) - - self.last_name = dns.name.from_text(name, self.current_origin) - name = self.last_name - if not name.is_subdomain(self.zone.origin): - self._eat_line() - return - if self.relativize: - name = name.relativize(self.zone.origin) - - n = self.zone.nodes.get(name) - if n is None: - n = self.zone.node_factory() - self.zone.nodes[name] = n - try: - rd = dns.rdata.from_text(rdclass, rdtype, rdata, - self.current_origin, False) - except dns.exception.SyntaxError: - # Catch and reraise. - (ty, va) = sys.exc_info()[:2] - raise va - except: - # All exceptions that occur in the processing of rdata - # are treated as syntax errors. This is not strictly - # correct, but it is correct almost all of the time. - # We convert them to syntax errors so that we can emit - # helpful filename:line info. - (ty, va) = sys.exc_info()[:2] - raise dns.exception.SyntaxError("caught exception %s: %s" % - (str(ty), str(va))) - - rd.choose_relativity(self.zone.origin, self.relativize) - covers = rd.covers() - rds = n.find_rdataset(rdclass, rdtype, covers, True) - rds.add(rd, ttl) - - def read(self): - """Read a DNS master file and build a zone object. - - @raises dns.zone.NoSOA: No SOA RR was found at the zone origin - @raises dns.zone.NoNS: No NS RRset was found at the zone origin - """ - - try: - while 1: - token = self.tok.get(True, True) - if token.is_eof(): - if self.current_file is not None: - self.current_file.close() - if len(self.saved_state) > 0: - (self.tok, - self.current_origin, - self.last_name, - self.current_file, - self.last_ttl, - self.last_ttl_known, - self.default_ttl, - self.default_ttl_known) = self.saved_state.pop(-1) - continue - break - elif token.is_eol(): - continue - elif token.is_comment(): - self.tok.get_eol() - continue - elif token.value[0] == u'$': - c = token.value.upper() - if c == u'$TTL': - token = self.tok.get() - if not token.is_identifier(): - raise dns.exception.SyntaxError("bad $TTL") - self.default_ttl = dns.ttl.from_text(token.value) - self.default_ttl_known = True - self.tok.get_eol() - elif c == u'$ORIGIN': - self.current_origin = self.tok.get_name() - self.tok.get_eol() - if self.zone.origin is None: - self.zone.origin = self.current_origin - elif c == u'$INCLUDE' and self.allow_include: - token = self.tok.get() - filename = token.value - token = self.tok.get() - if token.is_identifier(): - new_origin =\ - dns.name.from_text(token.value, - self.current_origin) - self.tok.get_eol() - elif not token.is_eol_or_eof(): - raise dns.exception.SyntaxError( - "bad origin in $INCLUDE") - else: - new_origin = self.current_origin - self.saved_state.append((self.tok, - self.current_origin, - self.last_name, - self.current_file, - self.last_ttl, - self.last_ttl_known, - self.default_ttl, - self.default_ttl_known)) - self.current_file = open(filename, 'r') - self.tok = dns.tokenizer.Tokenizer(self.current_file, - filename) - self.current_origin = new_origin - elif c == u'$GENERATE': - self._generate_line() - else: - raise dns.exception.SyntaxError( - "Unknown master file directive '" + c + "'") - continue - self.tok.unget(token) - self._rr_line() - except dns.exception.SyntaxError as detail: - (filename, line_number) = self.tok.where() - if detail is None: - detail = "syntax error" - raise dns.exception.SyntaxError( - "%s:%d: %s" % (filename, line_number, detail)) - - # Now that we're done reading, do some basic checking of the zone. - if self.check_origin: - self.zone.check_origin() - - -def from_text(text, origin=None, rdclass=dns.rdataclass.IN, - relativize=True, zone_factory=Zone, filename=None, - allow_include=False, check_origin=True): - """Build a zone object from a master file format string. - - @param text: the master file format input - @type text: string. - @param origin: The origin of the zone; if not specified, the first - $ORIGIN statement in the master file will determine the origin of the - zone. - @type origin: dns.name.Name object or string - @param rdclass: The zone's rdata class; the default is class IN. - @type rdclass: int - @param relativize: should names be relativized? The default is True - @type relativize: bool - @param zone_factory: The zone factory to use - @type zone_factory: function returning a Zone - @param filename: The filename to emit when describing where an error - occurred; the default is ''. - @type filename: string - @param allow_include: is $INCLUDE allowed? - @type allow_include: bool - @param check_origin: should sanity checks of the origin node be done? - The default is True. - @type check_origin: bool - @raises dns.zone.NoSOA: No SOA RR was found at the zone origin - @raises dns.zone.NoNS: No NS RRset was found at the zone origin - @rtype: dns.zone.Zone object - """ - - # 'text' can also be a file, but we don't publish that fact - # since it's an implementation detail. The official file - # interface is from_file(). - - if filename is None: - filename = '' - tok = dns.tokenizer.Tokenizer(text, filename) - reader = _MasterReader(tok, origin, rdclass, relativize, zone_factory, - allow_include=allow_include, - check_origin=check_origin) - reader.read() - return reader.zone - - -def from_file(f, origin=None, rdclass=dns.rdataclass.IN, - relativize=True, zone_factory=Zone, filename=None, - allow_include=True, check_origin=True): - """Read a master file and build a zone object. - - @param f: file or string. If I{f} is a string, it is treated - as the name of a file to open. - @param origin: The origin of the zone; if not specified, the first - $ORIGIN statement in the master file will determine the origin of the - zone. - @type origin: dns.name.Name object or string - @param rdclass: The zone's rdata class; the default is class IN. - @type rdclass: int - @param relativize: should names be relativized? The default is True - @type relativize: bool - @param zone_factory: The zone factory to use - @type zone_factory: function returning a Zone - @param filename: The filename to emit when describing where an error - occurred; the default is '', or the value of I{f} if I{f} is a - string. - @type filename: string - @param allow_include: is $INCLUDE allowed? - @type allow_include: bool - @param check_origin: should sanity checks of the origin node be done? - The default is True. - @type check_origin: bool - @raises dns.zone.NoSOA: No SOA RR was found at the zone origin - @raises dns.zone.NoNS: No NS RRset was found at the zone origin - @rtype: dns.zone.Zone object - """ - - str_type = string_types - if PY3: - opts = 'r' - else: - opts = 'rU' - - if isinstance(f, str_type): - if filename is None: - filename = f - f = open(f, opts) - want_close = True - else: - if filename is None: - filename = '' - want_close = False - - try: - z = from_text(f, origin, rdclass, relativize, zone_factory, - filename, allow_include, check_origin) - finally: - if want_close: - f.close() - return z - - -def from_xfr(xfr, zone_factory=Zone, relativize=True, check_origin=True): - """Convert the output of a zone transfer generator into a zone object. - - @param xfr: The xfr generator - @type xfr: generator of dns.message.Message objects - @param relativize: should names be relativized? The default is True. - It is essential that the relativize setting matches the one specified - to dns.query.xfr(). - @type relativize: bool - @param check_origin: should sanity checks of the origin node be done? - The default is True. - @type check_origin: bool - @raises dns.zone.NoSOA: No SOA RR was found at the zone origin - @raises dns.zone.NoNS: No NS RRset was found at the zone origin - @rtype: dns.zone.Zone object - """ - - z = None - for r in xfr: - if z is None: - if relativize: - origin = r.origin - else: - origin = r.answer[0].name - rdclass = r.answer[0].rdclass - z = zone_factory(origin, rdclass, relativize=relativize) - for rrset in r.answer: - znode = z.nodes.get(rrset.name) - if not znode: - znode = z.node_factory() - z.nodes[rrset.name] = znode - zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype, - rrset.covers, True) - zrds.update_ttl(rrset.ttl) - for rd in rrset: - rd.choose_relativity(z.origin, relativize) - zrds.add(rd) - if check_origin: - z.check_origin() - return z diff --git a/src/dns/zone.pyi b/src/dns/zone.pyi deleted file mode 100644 index 911d7a01..00000000 --- a/src/dns/zone.pyi +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Generator, Optional, Union, Tuple, Iterable, Callable, Any, Iterator, TextIO, BinaryIO, Dict -from . import rdata, zone, rdataclass, name, rdataclass, message, rdatatype, exception, node, rdataset, rrset, rdatatype - -class BadZone(exception.DNSException): ... -class NoSOA(BadZone): ... -class NoNS(BadZone): ... -class UnknownOrigin(BadZone): ... - -class Zone: - def __getitem__(self, key : str) -> node.Node: - ... - def __init__(self, origin : Union[str,name.Name], rdclass : int = rdataclass.IN, relativize : bool = True) -> None: - self.nodes : Dict[str,node.Node] - self.origin = origin - def values(self): - return self.nodes.values() - def iterate_rdatas(self, rdtype : Union[int,str] = rdatatype.ANY, covers : Union[int,str] = None) -> Iterable[Tuple[name.Name, int, rdata.Rdata]]: - ... - def __iter__(self) -> Iterator[str]: - ... - def get_node(self, name : Union[name.Name,str], create=False) -> Optional[node.Node]: - ... - def find_rrset(self, name : Union[str,name.Name], rdtype : Union[int,str], covers=rdatatype.NONE) -> rrset.RRset: - ... - def find_rdataset(self, name : Union[str,name.Name], rdtype : Union[str,int], covers=rdatatype.NONE, - create=False) -> rdataset.Rdataset: - ... - def get_rdataset(self, name : Union[str,name.Name], rdtype : Union[str,int], covers=rdatatype.NONE, create=False) -> Optional[rdataset.Rdataset]: - ... - def get_rrset(self, name : Union[str,name.Name], rdtype : Union[str,int], covers=rdatatype.NONE) -> Optional[rrset.RRset]: - ... - def replace_rdataset(self, name : Union[str,name.Name], replacement : rdataset.Rdataset) -> None: - ... - def delete_rdataset(self, name : Union[str,name.Name], rdtype : Union[str,int], covers=rdatatype.NONE) -> None: - ... - def iterate_rdatasets(self, rdtype : Union[str,int] =rdatatype.ANY, - covers : Union[str,int] =rdatatype.NONE): - ... - def to_file(self, f : Union[TextIO, BinaryIO, str], sorted=True, relativize=True, nl : Optional[bytes] = None): - ... - def to_text(self, sorted=True, relativize=True, nl : Optional[bytes] = None) -> bytes: - ... - -def from_xfr(xfr : Generator[Any,Any,message.Message], zone_factory : Callable[..., zone.Zone] = zone.Zone, relativize=True, check_origin=True): - ... - -def from_text(text : str, origin : Optional[Union[str,name.Name]] = None, rdclass : int = rdataclass.IN, - relativize=True, zone_factory : Callable[...,zone.Zone] = zone.Zone, filename : Optional[str] = None, - allow_include=False, check_origin=True) -> zone.Zone: - ... - -def from_file(f, origin : Optional[Union[str,name.Name]] = None, rdclass=rdataclass.IN, - relativize=True, zone_factory : Callable[..., zone.Zone] = Zone, filename : Optional[str] = None, - allow_include=True, check_origin=True) -> zone.Zone: - ... diff --git a/src/gam-install.sh b/src/gam-install.sh index 9aaadfd1..0c037294 100755 --- a/src/gam-install.sh +++ b/src/gam-install.sh @@ -8,7 +8,7 @@ GAM installation script. OPTIONS: -h show help. -d Directory where gam folder will be installed. Default is \$HOME/bin/ - -a Architecture to install (i386, x86_64, arm, arm64). Default is to detect your arch with "uname -m". + -a Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m". -o OS we are running (linux, macos). Default is to detect your OS with "uname -s". -l Just upgrade GAM to latest version. Skips project creation and auth. -p Profile update (true, false). Should script add gam command to environment. Default is true. @@ -26,6 +26,8 @@ upgrade_only=false gamversion="latest" adminuser="" regularuser="" +gam_glibc_ver="2.23" # Ubuntu 16.04 Xenial + while getopts "hd:a:o:lp:u:r:v:" OPTION do case $OPTION in @@ -75,11 +77,23 @@ echo -e "\x1B[1;33m$1" echo -e '\x1B[0m' } +version_gt() +{ +test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1" +} + case $gamos in [lL]inux) gamos="linux" + this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}') + echo "This Linux distribution uses glibc $this_glibc_ver" + if version_gt $gam_glibc_ver $this_glibc_ver; then + echo_yellow "NOTICE: You are running an older Linux distro than the one GAM was compiled on. A legacy GAM version that should be compatible with your system but may run slower will be installed. For best performance, upgrade to a newer Linux distribution like Debian 9 stable, Ubuntu Xenial 16.04, Fedora 24+ or RedHat Enterprise Linux 8." + gamarch=x86_64_legacy + fi case $gamarch in x86_64) gamfile="linux-x86_64.tar.xz";; + x86_64_legacy) gamfile="linux-x86_64-legacy.tar.xz";; i?86) gamfile="linux-i686.tar.xz";; arm|armv7l) gamfile="linux-armv7l.tar.xz";; arm64|aarch64) gamfile="linux-aarch64.tar.xz";; diff --git a/src/gam.py b/src/gam.py index 69eed83b..aa5ca851 100755 --- a/src/gam.py +++ b/src/gam.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # GAM @@ -16,7 +16,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -u"""GAM is a command line tool which allows Administrators to control their G Suite domain and accounts. +"""GAM is a command line tool which allows Administrators to control their G Suite domain and accounts. With GAM you can programatically create users, turn on/off services for users like POP and Forwarding and much more. For more information, see https://git.io/gam @@ -28,11 +28,11 @@ import string import time import base64 import codecs -import ConfigParser +import configparser import csv import datetime import hashlib -import httplib +import http.client import json import mimetypes import platform @@ -40,9 +40,9 @@ import random import re import signal import socket -import StringIO +import io import struct -from urllib import urlencode +from urllib.parse import urlencode import uuid import webbrowser import zipfile @@ -64,12 +64,22 @@ import google_auth_httplib2 import oauth2client.client import oauth2client.file import oauth2client.tools -from passlib.handlers.sha2_crypt import sha512_crypt +from passlib.hash import sha512_crypt from oauth2client.contrib.dictionary_storage import DictionaryStorage import utils from var import * +# Nasty hack to support StaticX. +# - we do this in gam.py because if we do it in var.py StaticX can't get right path at all. +# - StaticX is frozen but it seems to mix up the path checking results. +if os.environ.get('GAM_REAL_PATH', False): + GM_Globals[GM_GAM_PATH] = os.environ['GAM_REAL_PATH'] +elif getattr(sys, 'frozen', False): + GM_Globals[GM_GAM_PATH] = os.path.dirname(sys.executable) +else: + GM_Globals[GM_GAM_PATH] = os.path.dirname(os.path.realpath(__file__)) + # Override some oauth2client.tools strings saving us a few GAM-specific mods to oauth2client oauth2client.tools._FAILED_START_MESSAGE = """ Failed to start a local webserver listening on either port 8080 @@ -128,7 +138,7 @@ google_auth_httplib2.AuthorizedHttp.request = _request_with_user_agent( def showUsage(): doGAMVersion(checkForArgs=False) - print u''' + print(''' Usage: gam [OPTIONS]... GAM. Retrieve or set G Suite domain, @@ -142,15 +152,15 @@ gam update user jsmith suspended on gam.exe update group announcements add member jsmith ... -''' +''') # # Error handling # def stderrErrorMsg(message): - sys.stderr.write(utils.convertUTF8(u'\n{0}{1}\n'.format(ERROR_PREFIX, message))) + sys.stderr.write(utils.convertUTF8('\n{0}{1}\n'.format(ERROR_PREFIX, message))) def stderrWarningMsg(message): - sys.stderr.write(utils.convertUTF8(u'\n{0}{1}\n'.format(WARNING_PREFIX, message))) + sys.stderr.write(utils.convertUTF8('\n{0}{1}\n'.format(WARNING_PREFIX, message))) def systemErrorExit(sysRC, message): if message: @@ -164,13 +174,13 @@ def noPythonSSLExit(): systemErrorExit(8, MESSAGE_NO_PYTHON_SSL) def currentCount(i, count): - return u' ({0}/{1})'.format(i, count) if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else u'' + return ' ({0}/{1})'.format(i, count) if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else '' def currentCountNL(i, count): - return u' ({0}/{1})\n'.format(i, count) if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else u'\n' + return ' ({0}/{1})\n'.format(i, count) if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n' def formatHTTPError(http_status, reason, message): - return u'{0}: {1} - {2}'.format(http_status, reason, message) + return '{0}: {1} - {2}'.format(http_status, reason, message) def getHTTPError(responses, http_status, reason, message): if reason in responses: @@ -179,29 +189,29 @@ def getHTTPError(responses, http_status, reason, message): def printGettingAllItems(items, query): if query: - sys.stderr.write(u"Getting all {0} in G Suite account that match query ({1}) (may take some time on a large account)...\n".format(items, query)) + sys.stderr.write("Getting all {0} in G Suite account that match query ({1}) (may take some time on a large account)...\n".format(items, query)) else: - sys.stderr.write(u"Getting all {0} in G Suite account (may take some time on a large account)...\n".format(items)) + sys.stderr.write("Getting all {0} in G Suite account (may take some time on a large account)...\n".format(items)) def entityServiceNotApplicableWarning(entityType, entityName, i, count): - sys.stderr.write(u'{0}: {1}, Service not applicable/Does not exist{2}'.format(entityType, entityName, currentCountNL(i, count))) + sys.stderr.write('{0}: {1}, Service not applicable/Does not exist{2}'.format(entityType, entityName, currentCountNL(i, count))) def entityDoesNotExistWarning(entityType, entityName, i, count): - sys.stderr.write(u'{0}: {1}, Does not exist{2}'.format(entityType, entityName, currentCountNL(i, count))) + sys.stderr.write('{0}: {1}, Does not exist{2}'.format(entityType, entityName, currentCountNL(i, count))) def entityUnknownWarning(entityType, entityName, i, count): domain = getEmailAddressDomain(entityName) - if (domain == GC_Values[GC_DOMAIN]) or (domain.endswith(u'google.com')): + if (domain == GC_Values[GC_DOMAIN]) or (domain.endswith('google.com')): entityDoesNotExistWarning(entityType, entityName, i, count) else: entityServiceNotApplicableWarning(entityType, entityName, i, count) # Invalid CSV ~Header or ~~Header~~ def csvFieldErrorExit(fieldName, fieldNames): - systemErrorExit(2, MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(fieldName, u','.join(fieldNames))) + systemErrorExit(2, MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(fieldName, ','.join(fieldNames))) def printLine(message): - sys.stdout.write(message+u'\n') + sys.stdout.write(message+'\n') # def getBoolean(value, item): value = value.lower() @@ -209,10 +219,10 @@ def getBoolean(value, item): return True if value in false_values: return False - systemErrorExit(2, u'Value for {0} must be {1} or {2}; got {3}'.format(item, u'|'.join(true_values), u'|'.join(false_values), value)) + systemErrorExit(2, 'Value for {0} must be {1} or {2}; got {3}'.format(item, '|'.join(true_values), '|'.join(false_values), value)) def getCharSet(i): - if (i == len(sys.argv)) or (sys.argv[i].lower() != u'charset'): + if (i == len(sys.argv)) or (sys.argv[i].lower() != 'charset'): return (i, GC_Values.get(GC_CHARSET, GM_Globals[GM_SYS_ENCODING])) return (i+2, sys.argv[i+1]) @@ -260,7 +270,7 @@ def getColor(color): tg = COLORHEX_PATTERN.match(color) if tg: return tg.group(0) - systemErrorExit(2, u'A color must be a valid name or # and six hex characters (#012345); got {0}'.format(color)) + systemErrorExit(2, 'A color must be a valid name or # and six hex characters (#012345); got {0}'.format(color)) def getLabelColor(color): color = color.lower().strip() @@ -269,17 +279,17 @@ def getLabelColor(color): color = tg.group(0) if color in LABEL_COLORS: return color - systemErrorExit(2, u'A label color must be in the list: {0}; got {1}'.format(u'|'.join(LABEL_COLORS), color)) - systemErrorExit(2, u'A label color must be # and six hex characters (#012345); got {0}'.format(color)) + systemErrorExit(2, 'A label color must be in the list: {0}; got {1}'.format('|'.join(LABEL_COLORS), color)) + systemErrorExit(2, 'A label color must be # and six hex characters (#012345); got {0}'.format(color)) -def integerLimits(minVal, maxVal, item=u'integer'): +def integerLimits(minVal, maxVal, item='integer'): if (minVal is not None) and (maxVal is not None): - return u'{0} {1}<=x<={2}'.format(item, minVal, maxVal) + return '{0} {1}<=x<={2}'.format(item, minVal, maxVal) if minVal is not None: - return u'{0} x>={1}'.format(item, minVal) + return '{0} x>={1}'.format(item, minVal) if maxVal is not None: - return u'{0} x<={1}'.format(item, maxVal) - return u'{0} x'.format(item) + return '{0} x<={1}'.format(item, maxVal) + return '{0} x'.format(item) def getInteger(value, item, minVal=None, maxVal=None): try: @@ -288,16 +298,16 @@ def getInteger(value, item, minVal=None, maxVal=None): return number except ValueError: pass - systemErrorExit(2, u'expected {0} in range <{1}>, got {2}'.format(item, integerLimits(minVal, maxVal), value)) + systemErrorExit(2, 'expected {0} in range <{1}>, got {2}'.format(item, integerLimits(minVal, maxVal), value)) def removeCourseIdScope(courseId): - if courseId.startswith(u'd:'): + if courseId.startswith('d:'): return courseId[2:] return courseId def addCourseIdScope(courseId): - if not courseId.isdigit() and courseId[:2] != u'd:': - return u'd:{0}'.format(courseId) + if not courseId.isdigit() and courseId[:2] != 'd:': + return 'd:{0}'.format(courseId) return courseId def getString(i, item, optional=False, minLen=1, maxLen=None): @@ -306,52 +316,52 @@ def getString(i, item, optional=False, minLen=1, maxLen=None): if argstr: if (len(argstr) >= minLen) and ((maxLen is None) or (len(argstr) <= maxLen)): return argstr - systemErrorExit(2, u'expected <{0} for {1}>'.format(integerLimits(minLen, maxLen, u'string length'), item)) + systemErrorExit(2, 'expected <{0} for {1}>'.format(integerLimits(minLen, maxLen, 'string length'), item)) if optional or (minLen == 0): - return u'' - systemErrorExit(2, u'expected a Non-empty <{0}>'.format(item)) + return '' + systemErrorExit(2, 'expected a Non-empty <{0}>'.format(item)) elif optional: - return u'' - systemErrorExit(2, u'expected a <{0}>'.format(item)) + return '' + systemErrorExit(2, 'expected a <{0}>'.format(item)) def getDelta(argstr, pattern, formatRequired): tg = pattern.match(argstr.lower()) if tg is None: - systemErrorExit(2, u'expected a <{0}>; got {1}'.format(formatRequired, argstr)) + systemErrorExit(2, 'expected a <{0}>; got {1}'.format(formatRequired, argstr)) sign = tg.group(1) delta = int(tg.group(2)) unit = tg.group(3) - if unit == u'w': + if unit == 'w': deltaTime = datetime.timedelta(weeks=delta) - elif unit == u'd': + elif unit == 'd': deltaTime = datetime.timedelta(days=delta) - elif unit == u'h': + elif unit == 'h': deltaTime = datetime.timedelta(hours=delta) - elif unit == u'm': + elif unit == 'm': deltaTime = datetime.timedelta(minutes=delta) - if sign == u'-': + if sign == '-': return -deltaTime return deltaTime DELTA_DATE_PATTERN = re.compile(r'^([+-])(\d+)([dw])$') -DELTA_DATE_FORMAT_REQUIRED = u'(+|-)(d|w)' +DELTA_DATE_FORMAT_REQUIRED = '(+|-)(d|w)' def getDeltaDate(argstr): return getDelta(argstr, DELTA_DATE_PATTERN, DELTA_DATE_FORMAT_REQUIRED) DELTA_TIME_PATTERN = re.compile(r'^([+-])(\d+)([mhdw])$') -DELTA_TIME_FORMAT_REQUIRED = u'(+|-)(m|h|d|w)' +DELTA_TIME_FORMAT_REQUIRED = '(+|-)(m|h|d|w)' def getDeltaTime(argstr): return getDelta(argstr, DELTA_TIME_PATTERN, DELTA_TIME_FORMAT_REQUIRED) -YYYYMMDD_FORMAT = u'%Y-%m-%d' -YYYYMMDD_FORMAT_REQUIRED = u'yyyy-mm-dd' +YYYYMMDD_FORMAT = '%Y-%m-%d' +YYYYMMDD_FORMAT_REQUIRED = 'yyyy-mm-dd' def getYYYYMMDD(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False): argstr = argstr.strip() if argstr: - if argstr[0] in [u'+', u'-']: + if argstr[0] in ['+', '-']: today = datetime.date.today() argstr = (datetime.datetime(today.year, today.month, today.day)+getDeltaDate(argstr)).strftime(YYYYMMDD_FORMAT) try: @@ -362,12 +372,12 @@ def getYYYYMMDD(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False): return dateTime return argstr except ValueError: - systemErrorExit(2, u'expected a <{0}>; got {1}'.format(YYYYMMDD_FORMAT_REQUIRED, argstr)) + systemErrorExit(2, 'expected a <{0}>; got {1}'.format(YYYYMMDD_FORMAT_REQUIRED, argstr)) elif minLen == 0: - return u'' - systemErrorExit(2, u'expected a <{0}>'.format(YYYYMMDD_FORMAT_REQUIRED)) + return '' + systemErrorExit(2, 'expected a <{0}>'.format(YYYYMMDD_FORMAT_REQUIRED)) -YYYYMMDDTHHMMSS_FORMAT_REQUIRED = u'yyyy-mm-ddThh:mm:ss[.fff](Z|(+|-(hh:mm)))' +YYYYMMDDTHHMMSS_FORMAT_REQUIRED = 'yyyy-mm-ddThh:mm:ss[.fff](Z|(+|-(hh:mm)))' def getTimeOrDeltaFromNow(time_string): """Get an ISO 8601 date/time or a positive/negative delta applied to now. @@ -380,10 +390,10 @@ def getTimeOrDeltaFromNow(time_string): """ time_string = time_string.strip().upper() if time_string: - if time_string[0] not in [u'+', u'-']: + if time_string[0] not in ['+', '-']: return time_string - return (datetime.datetime.utcnow() + getDeltaTime(time_string)).isoformat() + u'Z' - systemErrorExit(2, u'expected a <{0}>'.format(YYYYMMDDTHHMMSS_FORMAT_REQUIRED)) + return (datetime.datetime.utcnow() + getDeltaTime(time_string)).isoformat() + 'Z' + systemErrorExit(2, 'expected a <{0}>'.format(YYYYMMDDTHHMMSS_FORMAT_REQUIRED)) YYYYMMDD_PATTERN = re.compile(r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$') @@ -391,20 +401,20 @@ def getDateZeroTimeOrFullTime(time_string): time_string = time_string.strip() if time_string: if YYYYMMDD_PATTERN.match(time_string): - return getYYYYMMDD(time_string)+u'T00:00:00.000Z' + return getYYYYMMDD(time_string)+'T00:00:00.000Z' return getTimeOrDeltaFromNow(time_string) - systemErrorExit(2, u'expected a <{0}>'.format(YYYYMMDDTHHMMSS_FORMAT_REQUIRED)) + systemErrorExit(2, 'expected a <{0}>'.format(YYYYMMDDTHHMMSS_FORMAT_REQUIRED)) # Get domain from email address def getEmailAddressDomain(emailAddress): - atLoc = emailAddress.find(u'@') + atLoc = emailAddress.find('@') if atLoc == -1: return GC_Values[GC_DOMAIN].lower() return emailAddress[atLoc+1:].lower() # Split email address unto user and domain def splitEmailAddress(emailAddress): - atLoc = emailAddress.find(u'@') + atLoc = emailAddress.find('@') if atLoc == -1: return (emailAddress.lower(), GC_Values[GC_DOMAIN].lower()) return (emailAddress[:atLoc].lower(), emailAddress[atLoc+1:].lower()) @@ -424,14 +434,14 @@ def normalizeEmailAddressOrUID(emailAddressOrUID, noUid=False, checkForCustomerI cg = UID_PATTERN.match(emailAddressOrUID) if cg: return cg.group(1) - atLoc = emailAddressOrUID.find(u'@') + atLoc = emailAddressOrUID.find('@') if atLoc == 0: return emailAddressOrUID[1:].lower() if not noLower else emailAddressOrUID[1:] if (atLoc == -1) or (atLoc == len(emailAddressOrUID)-1) and GC_Values[GC_DOMAIN]: if atLoc == -1: - emailAddressOrUID = u'{0}@{1}'.format(emailAddressOrUID, GC_Values[GC_DOMAIN]) + emailAddressOrUID = '{0}@{1}'.format(emailAddressOrUID, GC_Values[GC_DOMAIN]) else: - emailAddressOrUID = u'{0}{1}'.format(emailAddressOrUID, GC_Values[GC_DOMAIN]) + emailAddressOrUID = '{0}{1}'.format(emailAddressOrUID, GC_Values[GC_DOMAIN]) return emailAddressOrUID.lower() if not noLower else emailAddressOrUID # Normalize student/guardian email address/uid @@ -439,18 +449,18 @@ def normalizeEmailAddressOrUID(emailAddressOrUID, noUid=False, checkForCustomerI # - -> - # Otherwise, same results as normalizeEmailAddressOrUID def normalizeStudentGuardianEmailAddressOrUID(emailAddressOrUID): - if emailAddressOrUID.isdigit() or emailAddressOrUID == u'-': + if emailAddressOrUID.isdigit() or emailAddressOrUID == '-': return emailAddressOrUID return normalizeEmailAddressOrUID(emailAddressOrUID) # # Open a file # -def openFile(filename, mode=u'rU'): +def openFile(filename, mode='rU'): try: - if filename != u'-': + if filename != '-': return open(os.path.expanduser(filename), mode) - if mode.startswith(u'r'): - return StringIO.StringIO(unicode(sys.stdin.read())) + if mode.startswith('r'): + return io.StringIO(str(sys.stdin.read())) return sys.stdout except IOError as e: systemErrorExit(6, e) @@ -467,9 +477,9 @@ def closeFile(f): # # Read a file # -def readFile(filename, mode=u'rb', continueOnError=False, displayError=True, encoding=None): +def readFile(filename, mode='rb', continueOnError=False, displayError=True, encoding=None): try: - if filename != u'-': + if filename != '-': if not encoding: with open(os.path.expanduser(filename), mode) as f: return f.read() @@ -479,7 +489,7 @@ def readFile(filename, mode=u'rb', continueOnError=False, displayError=True, enc if not content.startswith(codecs.BOM_UTF8): return content return content[3:] - return unicode(sys.stdin.read()) + return str(sys.stdin.read()) except IOError as e: if continueOnError: if displayError: @@ -491,7 +501,7 @@ def readFile(filename, mode=u'rb', continueOnError=False, displayError=True, enc # # Write a file # -def writeFile(filename, data, mode=u'wb', continueOnError=False, displayError=True): +def writeFile(filename, data, mode='w', continueOnError=False, displayError=True): try: with open(os.path.expanduser(filename), mode) as f: f.write(data) @@ -513,8 +523,8 @@ class UTF8Recoder(object): def __iter__(self): return self - def next(self): - return self.reader.next().encode(u'utf-8') + def __next__(self): + return self.reader.next().encode('utf-8') class UnicodeDictReader(object): """ @@ -522,11 +532,11 @@ class UnicodeDictReader(object): which is encoded in the given encoding. """ - def __init__(self, f, dialect=csv.excel, encoding=u'utf-8', **kwds): + def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds): self.encoding = encoding try: - self.reader = csv.reader(UTF8Recoder(f, encoding) if self.encoding != u'utf-8' else f, dialect=dialect, **kwds) - self.fieldnames = self.reader.next() + self.reader = csv.reader(UTF8Recoder(f, encoding) if self.encoding != 'utf-8' else f, dialect=dialect, **kwds) + self.fieldnames = next(self.reader) if len(self.fieldnames) > 0 and self.fieldnames[0].startswith(codecs.BOM_UTF8): self.fieldnames[0] = self.fieldnames[0].replace(codecs.BOM_UTF8, '', 1) except (csv.Error, StopIteration): @@ -538,12 +548,12 @@ class UnicodeDictReader(object): def __iter__(self): return self - def next(self): - row = self.reader.next() + def __next__(self): + row = next(self.reader) l = len(row) if l < self.numfields: row += ['']*(self.numfields-l) # Must be '', not u'' - return dict((self.fieldnames[x], unicode(row[x], u'utf-8')) for x in range(self.numfields)) + return dict((self.fieldnames[x], str(row[x], 'utf-8')) for x in range(self.numfields)) # # Set global variables # Check for GAM updates based on status of noupdatecheck.txt @@ -578,39 +588,39 @@ def SetGlobalVariables(): return value GC_Defaults[GC_CONFIG_DIR] = GM_Globals[GM_GAM_PATH] - GC_Defaults[GC_CACHE_DIR] = os.path.join(GM_Globals[GM_GAM_PATH], u'gamcache') + GC_Defaults[GC_CACHE_DIR] = os.path.join(GM_Globals[GM_GAM_PATH], 'gamcache') GC_Defaults[GC_DRIVE_DIR] = GM_Globals[GM_GAM_PATH] GC_Defaults[GC_SITE_DIR] = GM_Globals[GM_GAM_PATH] - _getOldEnvVar(GC_CONFIG_DIR, u'GAMUSERCONFIGDIR') - _getOldEnvVar(GC_SITE_DIR, u'GAMSITECONFIGDIR') - _getOldEnvVar(GC_CACHE_DIR, u'GAMCACHEDIR') - _getOldEnvVar(GC_DRIVE_DIR, u'GAMDRIVEDIR') - _getOldEnvVar(GC_OAUTH2_TXT, u'OAUTHFILE') - _getOldEnvVar(GC_OAUTH2SERVICE_JSON, u'OAUTHSERVICEFILE') - if GC_Defaults[GC_OAUTH2SERVICE_JSON].find(u'.') == -1: - GC_Defaults[GC_OAUTH2SERVICE_JSON] += u'.json' - _getOldEnvVar(GC_CLIENT_SECRETS_JSON, u'CLIENTSECRETS') - _getOldEnvVar(GC_DOMAIN, u'GA_DOMAIN') - _getOldEnvVar(GC_CUSTOMER_ID, u'CUSTOMER_ID') - _getOldEnvVar(GC_CHARSET, u'GAM_CHARSET') - _getOldEnvVar(GC_NUM_THREADS, u'GAM_THREADS') - _getOldEnvVar(GC_ACTIVITY_MAX_RESULTS, u'GAM_ACTIVITY_MAX_RESULTS') - _getOldEnvVar(GC_AUTO_BATCH_MIN, u'GAM_AUTOBATCH') - _getOldEnvVar(GC_BATCH_SIZE, u'GAM_BATCH_SIZE') - _getOldEnvVar(GC_DEVICE_MAX_RESULTS, u'GAM_DEVICE_MAX_RESULTS') - _getOldEnvVar(GC_DRIVE_MAX_RESULTS, u'GAM_DRIVE_MAX_RESULTS') - _getOldEnvVar(GC_MEMBER_MAX_RESULTS, u'GAM_MEMBER_MAX_RESULTS') - _getOldEnvVar(GC_USER_MAX_RESULTS, u'GAM_USER_MAX_RESULTS') - _getOldEnvVar(GC_CSV_HEADER_FILTER, u'GAM_CSV_HEADER_FILTER') - _getOldEnvVar(GC_CSV_ROW_FILTER, u'GAM_CSV_ROW_FILTER') - _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', filePresentValue=4, fileAbsentValue=0) - _getOldSignalFile(GC_NO_VERIFY_SSL, u'noverifyssl.txt') - _getOldSignalFile(GC_NO_BROWSER, u'nobrowser.txt') + _getOldEnvVar(GC_CONFIG_DIR, 'GAMUSERCONFIGDIR') + _getOldEnvVar(GC_SITE_DIR, 'GAMSITECONFIGDIR') + _getOldEnvVar(GC_CACHE_DIR, 'GAMCACHEDIR') + _getOldEnvVar(GC_DRIVE_DIR, 'GAMDRIVEDIR') + _getOldEnvVar(GC_OAUTH2_TXT, 'OAUTHFILE') + _getOldEnvVar(GC_OAUTH2SERVICE_JSON, 'OAUTHSERVICEFILE') + if GC_Defaults[GC_OAUTH2SERVICE_JSON].find('.') == -1: + GC_Defaults[GC_OAUTH2SERVICE_JSON] += '.json' + _getOldEnvVar(GC_CLIENT_SECRETS_JSON, 'CLIENTSECRETS') + _getOldEnvVar(GC_DOMAIN, 'GA_DOMAIN') + _getOldEnvVar(GC_CUSTOMER_ID, 'CUSTOMER_ID') + _getOldEnvVar(GC_CHARSET, 'GAM_CHARSET') + _getOldEnvVar(GC_NUM_THREADS, 'GAM_THREADS') + _getOldEnvVar(GC_ACTIVITY_MAX_RESULTS, 'GAM_ACTIVITY_MAX_RESULTS') + _getOldEnvVar(GC_AUTO_BATCH_MIN, 'GAM_AUTOBATCH') + _getOldEnvVar(GC_BATCH_SIZE, 'GAM_BATCH_SIZE') + _getOldEnvVar(GC_DEVICE_MAX_RESULTS, 'GAM_DEVICE_MAX_RESULTS') + _getOldEnvVar(GC_DRIVE_MAX_RESULTS, 'GAM_DRIVE_MAX_RESULTS') + _getOldEnvVar(GC_MEMBER_MAX_RESULTS, 'GAM_MEMBER_MAX_RESULTS') + _getOldEnvVar(GC_USER_MAX_RESULTS, 'GAM_USER_MAX_RESULTS') + _getOldEnvVar(GC_CSV_HEADER_FILTER, 'GAM_CSV_HEADER_FILTER') + _getOldEnvVar(GC_CSV_ROW_FILTER, 'GAM_CSV_ROW_FILTER') + _getOldSignalFile(GC_DEBUG_LEVEL, 'debug.gam', filePresentValue=4, fileAbsentValue=0) + _getOldSignalFile(GC_NO_VERIFY_SSL, 'noverifyssl.txt') + _getOldSignalFile(GC_NO_BROWSER, 'nobrowser.txt') # _getOldSignalFile(GC_NO_CACHE, u'nocache.txt') # _getOldSignalFile(GC_CACHE_DISCOVERY_ONLY, u'allcache.txt', filePresentValue=False, fileAbsentValue=True) - _getOldSignalFile(GC_NO_CACHE, u'allcache.txt', filePresentValue=False, fileAbsentValue=True) - _getOldSignalFile(GC_NO_UPDATE_CHECK, u'noupdatecheck.txt') + _getOldSignalFile(GC_NO_CACHE, 'allcache.txt', filePresentValue=False, fileAbsentValue=True) + _getOldSignalFile(GC_NO_UPDATE_CHECK, 'noupdatecheck.txt') # Assign directories first for itemName in GC_VAR_INFO: if GC_VAR_INFO[itemName][GC_VAR_TYPE] == GC_TYPE_DIRECTORY: @@ -627,13 +637,13 @@ def SetGlobalVariables(): # Globals derived from config file values GM_Globals[GM_OAUTH2SERVICE_JSON_DATA] = None GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = None - GM_Globals[GM_EXTRA_ARGS_DICT] = {u'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} + GM_Globals[GM_EXTRA_ARGS_DICT] = {'prettyPrint': GC_Values[GC_DEBUG_LEVEL] > 0} httplib2.debuglevel = GC_Values[GC_DEBUG_LEVEL] if os.path.isfile(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)): - ea_config = ConfigParser.ConfigParser() + ea_config = configparser.ConfigParser() ea_config.optionxform = str ea_config.read(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)) - GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) + GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items('extra-args'))) if GC_Values[GC_NO_CACHE]: GM_Globals[GM_CACHE_DIR] = None GM_Globals[GM_CACHE_DISCOVERY_ONLY] = False @@ -647,7 +657,7 @@ def doGAMCheckForUpdates(forceCheck=False): def _gamLatestVersionNotAvailable(): if forceCheck: - systemErrorExit(4, u'GAM Latest Version information not available') + systemErrorExit(4, 'GAM Latest Version information not available') current_version = gam_version now_time = int(time.time()) @@ -659,41 +669,41 @@ def doGAMCheckForUpdates(forceCheck=False): if last_check_time > now_time-604800: return check_url = GAM_LATEST_RELEASE # latest full release - headers = {u'Accept': u'application/vnd.github.v3.text+json'} + headers = {'Accept': 'application/vnd.github.v3.text+json'} simplehttp = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) try: - (_, c) = simplehttp.request(check_url, u'GET', headers=headers) + (_, c) = simplehttp.request(check_url, 'GET', headers=headers) try: - release_data = json.loads(c) + release_data = json.loads(c.decode('utf-8')) except ValueError: _gamLatestVersionNotAvailable() return if isinstance(release_data, list): release_data = release_data[0] # only care about latest release - if not isinstance(release_data, dict) or u'tag_name' not in release_data: + if not isinstance(release_data, dict) or 'tag_name' not in release_data: _gamLatestVersionNotAvailable() return - latest_version = release_data[u'tag_name'] - if latest_version[0].lower() == u'v': + latest_version = release_data['tag_name'] + if latest_version[0].lower() == 'v': latest_version = latest_version[1:] if forceCheck or (latest_version > current_version): - print u'Version Check:\n Current: {0}\n Latest: {1}'.format(current_version, latest_version) + print('Version Check:\n Current: {0}\n Latest: {1}'.format(current_version, latest_version)) if latest_version <= current_version: writeFile(GM_Globals[GM_LAST_UPDATE_CHECK_TXT], str(now_time), continueOnError=True, displayError=forceCheck) return - announcement = release_data.get(u'body_text', u'No details about this release') - sys.stderr.write(u'\nGAM %s release notes:\n\n' % latest_version) + announcement = release_data.get('body_text', 'No details about this release') + sys.stderr.write('\nGAM %s release notes:\n\n' % latest_version) sys.stderr.write(announcement) try: printLine(MESSAGE_HIT_CONTROL_C_TO_UPDATE) time.sleep(15) except KeyboardInterrupt: - webbrowser.open(release_data[u'html_url']) + webbrowser.open(release_data['html_url']) printLine(MESSAGE_GAM_EXITING_FOR_UPDATE) sys.exit(0) writeFile(GM_Globals[GM_LAST_UPDATE_CHECK_TXT], str(now_time), continueOnError=True, displayError=forceCheck) return - except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError, httplib2.CertificateValidationUnsupported): + except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError): return def doGAMVersion(checkForArgs=True): @@ -702,37 +712,37 @@ def doGAMVersion(checkForArgs=True): if checkForArgs: i = 2 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'check': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'check': force_check = True i += 1 - elif myarg == u'simple': + elif myarg == 'simple': simple = True i += 1 else: - systemErrorExit(2, u'%s is not a valid argument for "gam version"' % sys.argv[i]) + systemErrorExit(2, '%s is not a valid argument for "gam version"' % sys.argv[i]) if simple: sys.stdout.write(gam_version) return - version_data = u'GAM {0} - {1}\n{2}\nPython {3}.{4}.{5} {6}-bit {7}\ngoogle-api-python-client {8}\noauth2client {9}\n{10} {11}\nPath: {12}' - print version_data.format(gam_version, GAM_URL, gam_author, sys.version_info[0], - sys.version_info[1], sys.version_info[2], struct.calcsize(u'P')*8, + version_data = 'GAM {0} - {1}\n{2}\nPython {3}.{4}.{5} {6}-bit {7}\ngoogle-api-python-client {8}\noauth2client {9}\n{10} {11}\nPath: {12}' + print(version_data.format(gam_version, GAM_URL, gam_author, sys.version_info[0], + sys.version_info[1], sys.version_info[2], struct.calcsize('P')*8, sys.version_info[3], googleapiclient.__version__, oauth2client.__version__, platform.platform(), - platform.machine(), GM_Globals[GM_GAM_PATH]) + platform.machine(), GM_Globals[GM_GAM_PATH])) if force_check: doGAMCheckForUpdates(forceCheck=True) def handleOAuthTokenError(e, soft_errors): - if e.replace(u'.', u'') in OAUTH2_TOKEN_ERRORS or e.startswith(u'Invalid response'): + if e.replace('.', '') in OAUTH2_TOKEN_ERRORS or e.startswith('Invalid response'): if soft_errors: return None if not GM_Globals[GM_CURRENT_API_USER]: stderrErrorMsg(MESSAGE_API_ACCESS_DENIED.format(GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], - u','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) + ','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) systemErrorExit(12, MESSAGE_API_ACCESS_CONFIG) else: systemErrorExit(19, MESSAGE_SERVICE_NOT_APPLICABLE.format(GM_Globals[GM_CURRENT_API_USER])) - systemErrorExit(18, u'Authentication Token Error - {0}'.format(e)) + systemErrorExit(18, 'Authentication Token Error - {0}'.format(e)) def getSvcAcctCredentials(scopes, act_as): try: @@ -754,24 +764,24 @@ def getSvcAcctCredentials(scopes, act_as): def waitOnFailure(n, retries, errMsg): wait_on_fail = min(2 ** n, 60) + float(random.randint(1, 1000)) / 1000 if n > 3: - sys.stderr.write(u'Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'.format(errMsg, int(wait_on_fail), n, retries)) + sys.stderr.write('Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'.format(errMsg, int(wait_on_fail), n, retries)) sys.stderr.flush() time.sleep(wait_on_fail) def checkGAPIError(e, soft_errors=False, silent_errors=False, retryOnHttpError=False, service=None): try: - error = json.loads(e.content) + error = json.loads(e.content.decode('utf-8')) except ValueError: - if (e.resp[u'status'] == u'503') and (e.content == u'Quota exceeded for the current request'): - return (e.resp[u'status'], GAPI_QUOTA_EXCEEDED, e.content) - if (e.resp[u'status'] == u'403') and (e.content.startswith(u'Request rate higher than configured')): - return (e.resp[u'status'], GAPI_QUOTA_EXCEEDED, e.content) - if (e.resp[u'status'] == u'403') and (u'Invalid domain.' in e.content): - error = {u'error': {u'code': 403, u'errors': [{u'reason': GAPI_NOT_FOUND, u'message': u'Domain not found'}]}} - elif (e.resp[u'status'] == u'400') and (u'InvalidSsoSigningKey' in e.content): - error = {u'error': {u'code': 400, u'errors': [{u'reason': GAPI_INVALID, u'message': u'InvalidSsoSigningKey'}]}} - elif (e.resp[u'status'] == u'400') and (u'UnknownError' in e.content): - error = {u'error': {u'code': 400, u'errors': [{u'reason': GAPI_INVALID, u'message': u'UnknownError'}]}} + if (e.resp['status'] == '503') and (e.content == 'Quota exceeded for the current request'): + return (e.resp['status'], GAPI_QUOTA_EXCEEDED, e.content) + if (e.resp['status'] == '403') and (e.content.startswith('Request rate higher than configured')): + return (e.resp['status'], GAPI_QUOTA_EXCEEDED, e.content) + if (e.resp['status'] == '403') and ('Invalid domain.' in e.content): + error = {'error': {'code': 403, 'errors': [{'reason': GAPI_NOT_FOUND, 'message': 'Domain not found'}]}} + elif (e.resp['status'] == '400') and ('InvalidSsoSigningKey' in e.content): + error = {'error': {'code': 400, 'errors': [{'reason': GAPI_INVALID, 'message': 'InvalidSsoSigningKey'}]}} + elif (e.resp['status'] == '400') and ('UnknownError' in e.content): + error = {'error': {'code': 400, 'errors': [{'reason': GAPI_INVALID, 'message': 'UnknownError'}]}} elif retryOnHttpError: service._http.request.credentials.refresh(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL])) return (-1, None, None) @@ -781,53 +791,53 @@ def checkGAPIError(e, soft_errors=False, silent_errors=False, retryOnHttpError=F return (0, None, None) else: systemErrorExit(5, e.content) - if u'error' in error: - http_status = error[u'error'][u'code'] + if 'error' in error: + http_status = error['error']['code'] try: - message = error[u'error'][u'errors'][0][u'message'] + message = error['error']['errors'][0]['message'] except KeyError: - message = error[u'error'][u'message'] + message = error['error']['message'] else: - if u'error_description' in error: - if error[u'error_description'] == u'Invalid Value': - message = error[u'error_description'] + if 'error_description' in error: + if error['error_description'] == 'Invalid Value': + message = error['error_description'] http_status = 400 - error = {u'error': {u'errors': [{u'reason': GAPI_INVALID, u'message': message}]}} + error = {'error': {'errors': [{'reason': GAPI_INVALID, 'message': message}]}} else: systemErrorExit(4, str(error)) else: systemErrorExit(4, str(error)) try: - reason = error[u'error'][u'errors'][0][u'reason'] - if reason == u'notFound': - if u'userKey' in message: + reason = error['error']['errors'][0]['reason'] + if reason == 'notFound': + if 'userKey' in message: reason = GAPI_USER_NOT_FOUND - elif u'groupKey' in message: + elif 'groupKey' in message: reason = GAPI_GROUP_NOT_FOUND - elif u'memberKey' in message: + elif 'memberKey' in message: reason = GAPI_MEMBER_NOT_FOUND - elif u'Domain not found' in message: + elif 'Domain not found' in message: reason = GAPI_DOMAIN_NOT_FOUND - elif u'Resource Not Found' in message: + elif 'Resource Not Found' in message: reason = GAPI_RESOURCE_NOT_FOUND - elif reason == u'invalid': - if u'userId' in message: + elif reason == 'invalid': + if 'userId' in message: reason = GAPI_USER_NOT_FOUND - elif u'memberKey' in message: + elif 'memberKey' in message: reason = GAPI_INVALID_MEMBER - elif reason == u'failedPrecondition': - if u'Bad Request' in message: + elif reason == 'failedPrecondition': + if 'Bad Request' in message: reason = GAPI_BAD_REQUEST - elif u'Mail service not enabled' in message: + elif 'Mail service not enabled' in message: reason = GAPI_SERVICE_NOT_AVAILABLE - elif reason == u'required': - if u'memberKey' in message: + elif reason == 'required': + if 'memberKey' in message: reason = GAPI_MEMBER_NOT_FOUND - elif reason == u'conditionNotMet': - if u'Cyclic memberships not allowed' in message: + elif reason == 'conditionNotMet': + if 'Cyclic memberships not allowed' in message: reason = GAPI_CYCLIC_MEMBERSHIPS_NOT_ALLOWED except KeyError: - reason = u'{0}'.format(http_status) + reason = '{0}'.format(http_status) return (http_status, reason, message) class GAPI_aborted(Exception): @@ -926,7 +936,7 @@ def callGAPI(service, function, method = getattr(service, function) retries = 10 - parameters = dict(kwargs.items() + GM_Globals[GM_EXTRA_ARGS_DICT].items()) + parameters = dict(list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items())) for n in range(1, retries+1): try: return method(**parameters).execute() @@ -944,14 +954,14 @@ def callGAPI(service, function, waitOnFailure(n, retries, reason) continue if soft_errors: - stderrErrorMsg(u'{0}: {1} - {2}{3}'.format(http_status, message, reason, [u'', u': Giving up.'][n > 1])) + stderrErrorMsg('{0}: {1} - {2}{3}'.format(http_status, message, reason, ['', ': Giving up.'][n > 1])) return None - systemErrorExit(int(http_status), u'{0}: {1} - {2}'.format(http_status, message, reason)) + systemErrorExit(int(http_status), '{0}: {1} - {2}'.format(http_status, message, reason)) except oauth2client.client.AccessTokenRefreshError as e: handleOAuthTokenError(str(e), soft_errors or GAPI_SERVICE_NOT_AVAILABLE in throw_reasons) if GAPI_SERVICE_NOT_AVAILABLE in throw_reasons: raise GAPI_serviceNotAvailable(str(e)) - stderrErrorMsg(u'User {0}: {1}'.format(GM_Globals[GM_CURRENT_API_USER], str(e))) + stderrErrorMsg('User {0}: {1}'.format(GM_Globals[GM_CURRENT_API_USER], str(e))) return None except httplib2.CertificateValidationUnsupported: noPythonSSLExit() @@ -963,7 +973,7 @@ def callGAPI(service, function, except (TypeError, httplib2.ServerNotFoundError) as e: systemErrorExit(4, str(e)) -def callGAPIpages(service, function, items=u'items', +def callGAPIpages(service, function, items='items', page_message=None, message_attribute=None, soft_errors=False, throw_reasons=None, retry_reasons=None, **kwargs): @@ -1017,7 +1027,7 @@ def callGAPIpages(service, function, items=u'items', pageToken=page_token, **kwargs) if page: - page_token = page.get(u'nextPageToken') + page_token = page.get('nextPageToken') page_items = page.get(items, []) num_page_items = len(page_items) total_items += num_page_items @@ -1028,25 +1038,25 @@ def callGAPIpages(service, function, items=u'items', # Show a paging message to the user that indicates paging progress if page_message: - show_message = page_message.replace(u'%%num_items%%', str(num_page_items)) - show_message = show_message.replace(u'%%total_items%%', str(total_items)) + show_message = page_message.replace('%%num_items%%', str(num_page_items)) + show_message = show_message.replace('%%total_items%%', str(total_items)) if message_attribute: first_item = page_items[0] if num_page_items > 0 else {} last_item = page_items[-1] if num_page_items > 1 else first_item - show_message = show_message.replace(u'%%first_item%%', str(first_item.get(message_attribute, u''))) - show_message = show_message.replace(u'%%last_item%%', str(last_item.get(message_attribute, u''))) - sys.stderr.write(u'\r') + show_message = show_message.replace('%%first_item%%', str(first_item.get(message_attribute, ''))) + show_message = show_message.replace('%%last_item%%', str(last_item.get(message_attribute, ''))) + sys.stderr.write('\r') sys.stderr.flush() sys.stderr.write(show_message) if not page_token: # End the paging status message and return all items. - if page_message and (page_message[-1] != u'\n'): - sys.stderr.write(u'\r\n') + if page_message and (page_message[-1] != '\n'): + sys.stderr.write('\r\n') sys.stderr.flush() return all_items -def callGAPIitems(service, function, items=u'items', +def callGAPIitems(service, function, items='items', throw_reasons=None, retry_reasons=None, **kwargs): """Gets a single page of items from a Google service function that is paged. @@ -1077,17 +1087,17 @@ def callGAPIitems(service, function, items=u'items', return [] def getAPIVersion(api): - version = API_VER_MAPPING.get(api, u'v1') - if api in [u'directory', u'reports', u'datatransfer']: - api = u'admin' - elif api == u'drive3': - api = u'drive' - return (api, version, u'{0}-{1}'.format(api, version)) + version = API_VER_MAPPING.get(api, 'v1') + if api in ['directory', 'reports', 'datatransfer']: + api = 'admin' + elif api == 'drive3': + api = 'drive' + return (api, version, '{0}-{1}'.format(api, version)) def readDiscoveryFile(api_version): - disc_filename = u'%s.json' % (api_version) + disc_filename = '%s.json' % (api_version) disc_file = os.path.join(GM_Globals[GM_GAM_PATH], disc_filename) - if hasattr(sys, u'_MEIPASS'): + if hasattr(sys, '_MEIPASS'): pyinstaller_disc_file = os.path.join(sys._MEIPASS, disc_filename) else: pyinstaller_disc_file = None @@ -1154,7 +1164,7 @@ def getService(api, http): waitOnFailure(n, retries, str(e)) continue systemErrorExit(17, str(e)) - except (httplib.ResponseNotReady, httplib2.SSLHandshakeError, socket.error) as e: + except (http.client.ResponseNotReady, httplib2.SSLHandshakeError, socket.error) as e: if n != retries: waitOnFailure(n, retries, str(e)) continue @@ -1181,75 +1191,75 @@ def buildGAPIObject(api): service = getService(api, http) if GC_Values[GC_DOMAIN]: if not GC_Values[GC_CUSTOMER_ID]: - resp, result = service._http.request(u'https://www.googleapis.com/admin/directory/v1/users?domain={0}&maxResults=1&fields=users(customerId)'.format(GC_Values[GC_DOMAIN])) + resp, result = service._http.request('https://www.googleapis.com/admin/directory/v1/users?domain={0}&maxResults=1&fields=users(customerId)'.format(GC_Values[GC_DOMAIN])) try: resultObj = json.loads(result) except ValueError: - systemErrorExit(8, u'Unexpected response: {0}'.format(result)) - if resp[u'status'] in [u'403', u'404']: + systemErrorExit(8, 'Unexpected response: {0}'.format(result)) + if resp['status'] in ['403', '404']: try: - message = resultObj[u'error'][u'errors'][0][u'message'] + message = resultObj['error']['errors'][0]['message'] except KeyError: - message = resultObj[u'error'][u'message'] - systemErrorExit(8, u'{0} - {1}'.format(message, GC_Values[GC_DOMAIN])) + message = resultObj['error']['message'] + systemErrorExit(8, '{0} - {1}'.format(message, GC_Values[GC_DOMAIN])) try: - GC_Values[GC_CUSTOMER_ID] = resultObj[u'users'][0][u'customerId'] + GC_Values[GC_CUSTOMER_ID] = resultObj['users'][0]['customerId'] except KeyError: GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER else: - GC_Values[GC_DOMAIN] = _getValueFromOAuth(u'hd', credentials=credentials) + GC_Values[GC_DOMAIN] = _getValueFromOAuth('hd', credentials=credentials) if not GC_Values[GC_CUSTOMER_ID]: GC_Values[GC_CUSTOMER_ID] = MY_CUSTOMER return service # Convert UID to email address -def convertUIDtoEmailAddress(emailAddressOrUID, cd=None, email_type=u'user'): +def convertUIDtoEmailAddress(emailAddressOrUID, cd=None, email_type='user'): normalizedEmailAddressOrUID = normalizeEmailAddressOrUID(emailAddressOrUID) - if normalizedEmailAddressOrUID.find(u'@') > 0: + if normalizedEmailAddressOrUID.find('@') > 0: return normalizedEmailAddressOrUID if not cd: - cd = buildGAPIObject(u'directory') - if email_type == u'user': + cd = buildGAPIObject('directory') + if email_type == 'user': try: - result = callGAPI(cd.users(), u'get', + result = callGAPI(cd.users(), 'get', throw_reasons=[GAPI_USER_NOT_FOUND], - userKey=normalizedEmailAddressOrUID, fields=u'primaryEmail') - if u'primaryEmail' in result: - return result[u'primaryEmail'].lower() + userKey=normalizedEmailAddressOrUID, fields='primaryEmail') + if 'primaryEmail' in result: + return result['primaryEmail'].lower() except GAPI_userNotFound: pass else: try: - result = callGAPI(cd.groups(), u'get', + result = callGAPI(cd.groups(), 'get', throw_reasons=[GAPI_GROUP_NOT_FOUND], - groupKey=normalizedEmailAddressOrUID, fields=u'email') - if u'email' in result: - return result[u'email'].lower() + groupKey=normalizedEmailAddressOrUID, fields='email') + if 'email' in result: + return result['email'].lower() except GAPI_groupNotFound: pass return normalizedEmailAddressOrUID # Convert email address to UID -def convertEmailAddressToUID(emailAddressOrUID, cd=None, email_type=u'user'): +def convertEmailAddressToUID(emailAddressOrUID, cd=None, email_type='user'): normalizedEmailAddressOrUID = normalizeEmailAddressOrUID(emailAddressOrUID) - if normalizedEmailAddressOrUID.find(u'@') > 0: + if normalizedEmailAddressOrUID.find('@') > 0: if not cd: - cd = buildGAPIObject(u'directory') - if email_type != u'group': + cd = buildGAPIObject('directory') + if email_type != 'group': try: - result = callGAPI(cd.users(), u'get', + result = callGAPI(cd.users(), 'get', throw_reasons=[GAPI_USER_NOT_FOUND], - userKey=normalizedEmailAddressOrUID, fields=u'id') - if u'id' in result: - return result[u'id'] + userKey=normalizedEmailAddressOrUID, fields='id') + if 'id' in result: + return result['id'] except GAPI_userNotFound: pass try: - result = callGAPI(cd.groups(), u'get', + result = callGAPI(cd.groups(), 'get', throw_reasons=[GAPI_NOT_FOUND], - groupKey=normalizedEmailAddressOrUID, fields=u'id') - if u'id' in result: - return result[u'id'] + groupKey=normalizedEmailAddressOrUID, fields='id') + if 'id' in result: + return result['id'] except GAPI_notFound: pass return None @@ -1270,77 +1280,77 @@ def buildGAPIServiceObject(api, act_as, showAuthError=True): systemErrorExit(4, e) except google.auth.exceptions.RefreshError as e: if showAuthError: - stderrErrorMsg(u'User {0}: {1}'.format(GM_Globals[GM_CURRENT_API_USER], str(e[0]))) - return handleOAuthTokenError(str(e[0]), True) + stderrErrorMsg('User {0}: {1}'.format(GM_Globals[GM_CURRENT_API_USER], str(e))) + return handleOAuthTokenError(str(e), True) return service def buildAlertCenterGAPIObject(user): userEmail = convertUIDtoEmailAddress(user) - return (userEmail, buildGAPIServiceObject(u'alertcenter', userEmail)) + return (userEmail, buildGAPIServiceObject('alertcenter', userEmail)) def buildActivityGAPIObject(user): userEmail = convertUIDtoEmailAddress(user) - return (userEmail, buildGAPIServiceObject(u'appsactivity', userEmail)) + return (userEmail, buildGAPIServiceObject('appsactivity', userEmail)) def normalizeCalendarId(calname, checkPrimary=False): calname = calname.lower() - if checkPrimary and calname == u'primary': + if checkPrimary and calname == 'primary': return calname if not GC_Values[GC_DOMAIN]: - GC_Values[GC_DOMAIN] = _getValueFromOAuth(u'hd') + GC_Values[GC_DOMAIN] = _getValueFromOAuth('hd') return convertUIDtoEmailAddress(calname) def buildCalendarGAPIObject(calname): calendarId = normalizeCalendarId(calname) - return (calendarId, buildGAPIServiceObject(u'calendar', calendarId)) + return (calendarId, buildGAPIServiceObject('calendar', calendarId)) def buildCalendarDataGAPIObject(calname): calendarId = normalizeCalendarId(calname) # Force service account token request. If we fail fall back to using admin for authentication - cal = buildGAPIServiceObject(u'calendar', calendarId, False) + cal = buildGAPIServiceObject('calendar', calendarId, False) if cal is None: - _, cal = buildCalendarGAPIObject(_getValueFromOAuth(u'email')) + _, cal = buildCalendarGAPIObject(_getValueFromOAuth('email')) return (calendarId, cal) def buildDriveGAPIObject(user): userEmail = convertUIDtoEmailAddress(user) - return (userEmail, buildGAPIServiceObject(u'drive', userEmail)) + return (userEmail, buildGAPIServiceObject('drive', userEmail)) def buildDrive3GAPIObject(user): userEmail = convertUIDtoEmailAddress(user) - return (userEmail, buildGAPIServiceObject(u'drive3', userEmail)) + return (userEmail, buildGAPIServiceObject('drive3', userEmail)) def buildGmailGAPIObject(user): userEmail = convertUIDtoEmailAddress(user) - return (userEmail, buildGAPIServiceObject(u'gmail', userEmail)) + return (userEmail, buildGAPIServiceObject('gmail', userEmail)) def doCheckServiceAccount(users): all_scopes = [] - for _, scopes in API_SCOPE_MAPPING.items(): + for _, scopes in list(API_SCOPE_MAPPING.items()): for scope in scopes: if scope not in all_scopes: all_scopes.append(scope) all_scopes.sort() for user in users: all_scopes_pass = True - print u'User: %s' % (user) + print('User: %s' % (user)) for scope in all_scopes: try: credentials = getSvcAcctCredentials([scope], user) request = google_auth_httplib2.Request(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL])) credentials.refresh(request) - result = u'PASS' + result = 'PASS' except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) except google.auth.exceptions.RefreshError: - result = u'FAIL' + result = 'FAIL' all_scopes_pass = False - print u' Scope: {0:60} {1}'.format(scope, result) + print(' Scope: {0:60} {1}'.format(scope, result)) service_account = GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] if all_scopes_pass: - print u'\nAll scopes passed!\nService account %s is fully authorized.' % service_account + print('\nAll scopes passed!\nService account %s is fully authorized.' % service_account) return - user_domain = user[user.find(u'@')+1:] + user_domain = user[user.find('@')+1:] scopes_failed = '''Some scopes failed! Please go to: https://admin.google.com/%s/AdminHome?#OGX:ManageOauthClients @@ -1361,32 +1371,32 @@ RI_JCOUNT = 2 RI_ITEM = 3 RI_ROLE = 4 -def batchRequestID(entityName, j, jcount, item, role=u''): - return u'{0}\n{1}\n{2}\n{3}\n{4}'.format(entityName, j, jcount, item, role) +def batchRequestID(entityName, j, jcount, item, role=''): + return '{0}\n{1}\n{2}\n{3}\n{4}'.format(entityName, j, jcount, item, role) def _adjustDate(errMsg): - match_date = re.match(u'Data for dates later than (.*) is not yet available. Please check back later', errMsg) + match_date = re.match('Data for dates later than (.*) is not yet available. Please check back later', errMsg) if not match_date: - match_date = re.match(u'Start date can not be later than (.*)', errMsg) + match_date = re.match('Start date can not be later than (.*)', errMsg) if not match_date: systemErrorExit(4, errMsg) - return unicode(match_date.group(1)) + return str(match_date.group(1)) def _checkFullDataAvailable(warnings, tryDate, fullDataRequired): for warning in warnings: - if warning[u'code'] == u'PARTIAL_DATA_AVAILABLE': - for app in warning[u'data']: - if app[u'key'] == u'application' and app[u'value'] != u'docs' and (not fullDataRequired or app[u'value'] in fullDataRequired): + if warning['code'] == 'PARTIAL_DATA_AVAILABLE': + for app in warning['data']: + if app['key'] == 'application' and app['value'] != 'docs' and (not fullDataRequired or app['value'] in fullDataRequired): tryDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)-datetime.timedelta(days=1) return (0, tryDateTime.strftime(YYYYMMDD_FORMAT)) - elif warning[u'code'] == u'DATA_NOT_AVAILABLE': - for app in warning[u'data']: - if app[u'key'] == u'application' and app[u'value'] != u'docs' and (not fullDataRequired or app[u'value'] in fullDataRequired): + elif warning['code'] == 'DATA_NOT_AVAILABLE': + for app in warning['data']: + if app['key'] == 'application' and app['value'] != 'docs' and (not fullDataRequired or app['value'] in fullDataRequired): return (-1, tryDate) return (1, tryDate) def showReport(): - rep = buildGAPIObject(u'reports') + rep = buildGAPIObject('reports') report = sys.argv[2].lower() customerId = GC_Values[GC_CUSTOMER_ID] if customerId == MY_CUSTOMER: @@ -1394,80 +1404,80 @@ def showReport(): filters = parameters = actorIpAddress = startTime = endTime = eventName = orgUnitId = None tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT) to_drive = False - userKey = u'all' + userKey = 'all' fullDataRequired = None i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'date': + if myarg == 'date': tryDate = getYYYYMMDD(sys.argv[i+1]) i += 2 - elif myarg in [u'orgunit', u'org', u'ou']: + elif myarg in ['orgunit', 'org', 'ou']: _, orgUnitId = getOrgUnitId(sys.argv[i+1]) i += 2 - elif myarg == u'fulldatarequired': + elif myarg == 'fulldatarequired': fullDataRequired = [] fdr = sys.argv[i+1].lower() - if len(fdr) > 0 and fdr != u'all': - fullDataRequired = fdr.replace(u',', u' ').split() + if len(fdr) > 0 and fdr != 'all': + fullDataRequired = fdr.replace(',', ' ').split() i += 2 - elif myarg == u'start': + elif myarg == 'start': startTime = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg == u'end': + elif myarg == 'end': endTime = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg == u'event': + elif myarg == 'event': eventName = sys.argv[i+1] i += 2 - elif myarg == u'user': + elif myarg == 'user': userKey = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg in [u'filter', u'filters']: + elif myarg in ['filter', 'filters']: filters = sys.argv[i+1] i += 2 - elif myarg in [u'fields', u'parameters']: + elif myarg in ['fields', 'parameters']: parameters = sys.argv[i+1] i += 2 - elif myarg == u'ip': + elif myarg == 'ip': actorIpAddress = sys.argv[i+1] i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': to_drive = True i += 1 else: - systemErrorExit(2, u'%s is not a valid argument to "gam report"' % sys.argv[i]) - if report in [u'users', u'user']: + systemErrorExit(2, '%s is not a valid argument to "gam report"' % sys.argv[i]) + if report in ['users', 'user']: while True: try: if fullDataRequired is not None: - warnings = callGAPIitems(rep.userUsageReport(), u'get', u'warnings', + warnings = callGAPIitems(rep.userUsageReport(), 'get', 'warnings', throw_reasons=[GAPI_INVALID], - date=tryDate, userKey=userKey, customerId=customerId, orgUnitID=orgUnitId, fields=u'warnings') + date=tryDate, userKey=userKey, customerId=customerId, orgUnitID=orgUnitId, fields='warnings') fullData, tryDate = _checkFullDataAvailable(warnings, tryDate, fullDataRequired) if fullData < 0: - print u'No user report available.' + print('No user report available.') sys.exit(1) if fullData == 0: continue - page_message = u'Got %%num_items%% Users\n' - usage = callGAPIpages(rep.userUsageReport(), u'get', u'usageReports', page_message=page_message, throw_reasons=[GAPI_INVALID], + page_message = 'Got %%num_items%% Users\n' + usage = callGAPIpages(rep.userUsageReport(), 'get', 'usageReports', page_message=page_message, throw_reasons=[GAPI_INVALID], date=tryDate, userKey=userKey, customerId=customerId, orgUnitID=orgUnitId, filters=filters, parameters=parameters) break except GAPI_invalid as e: tryDate = _adjustDate(str(e)) if not usage: - print u'No user report available.' + print('No user report available.') sys.exit(1) - titles = [u'email', u'date'] + titles = ['email', 'date'] csvRows = [] for user_report in usage: - if u'entity' not in user_report: + if 'entity' not in user_report: continue - row = {u'email': user_report[u'entity'][u'userEmail'], u'date': tryDate} + row = {'email': user_report['entity']['userEmail'], 'date': tryDate} try: - for report_item in user_report[u'parameters']: - items = report_item.values() + for report_item in user_report['parameters']: + items = list(report_item.values()) if len(items) < 2: continue name = items[1] @@ -1478,182 +1488,182 @@ def showReport(): except KeyError: pass csvRows.append(row) - writeCSVfile(csvRows, titles, u'User Reports - %s' % tryDate, to_drive) - elif report in [u'customer', u'customers', u'domain']: + writeCSVfile(csvRows, titles, 'User Reports - %s' % tryDate, to_drive) + elif report in ['customer', 'customers', 'domain']: while True: try: if fullDataRequired is not None: - warnings = callGAPIitems(rep.customerUsageReports(), u'get', u'warnings', + warnings = callGAPIitems(rep.customerUsageReports(), 'get', 'warnings', throw_reasons=[GAPI_INVALID], - customerId=customerId, date=tryDate, fields=u'warnings') + customerId=customerId, date=tryDate, fields='warnings') fullData, tryDate = _checkFullDataAvailable(warnings, tryDate, fullDataRequired) if fullData < 0: - print u'No customer report available.' + print('No customer report available.') sys.exit(1) if fullData == 0: continue - usage = callGAPIpages(rep.customerUsageReports(), u'get', u'usageReports', throw_reasons=[GAPI_INVALID], + usage = callGAPIpages(rep.customerUsageReports(), 'get', 'usageReports', throw_reasons=[GAPI_INVALID], customerId=customerId, date=tryDate, parameters=parameters) break except GAPI_invalid as e: tryDate = _adjustDate(str(e)) if not usage: - print u'No customer report available.' + print('No customer report available.') sys.exit(1) - titles = [u'name', u'value', u'client_id'] + titles = ['name', 'value', 'client_id'] csvRows = [] auth_apps = list() - for item in usage[0][u'parameters']: - if u'name' not in item: + for item in usage[0]['parameters']: + if 'name' not in item: continue - name = item[u'name'] - if u'intValue' in item: - value = item[u'intValue'] - elif u'msgValue' in item: - if name == u'accounts:authorized_apps': - for subitem in item[u'msgValue']: + name = item['name'] + if 'intValue' in item: + value = item['intValue'] + elif 'msgValue' in item: + if name == 'accounts:authorized_apps': + for subitem in item['msgValue']: app = {} for an_item in subitem: - if an_item == u'client_name': - app[u'name'] = u'App: %s' % subitem[an_item].replace(u'\n', u'\\n') - elif an_item == u'num_users': - app[u'value'] = u'%s users' % subitem[an_item] - elif an_item == u'client_id': - app[u'client_id'] = subitem[an_item] + if an_item == 'client_name': + app['name'] = 'App: %s' % subitem[an_item].replace('\n', '\\n') + elif an_item == 'num_users': + app['value'] = '%s users' % subitem[an_item] + elif an_item == 'client_id': + app['client_id'] = subitem[an_item] auth_apps.append(app) continue else: values = [] - for subitem in item[u'msgValue']: - if u'count' in subitem: + for subitem in item['msgValue']: + if 'count' in subitem: mycount = myvalue = None - for key, value in subitem.items(): - if key == u'count': + for key, value in list(subitem.items()): + if key == 'count': mycount = value else: myvalue = value if mycount and myvalue: - values.append(u'%s:%s' % (myvalue, mycount)) - value = u' '.join(values) - elif u'version_number' in subitem and u'num_devices' in subitem: - values.append(u'%s:%s' % (subitem[u'version_number'], subitem[u'num_devices'])) + values.append('%s:%s' % (myvalue, mycount)) + value = ' '.join(values) + elif 'version_number' in subitem and 'num_devices' in subitem: + values.append('%s:%s' % (subitem['version_number'], subitem['num_devices'])) else: continue - value = u' '.join(sorted(values, reverse=True)) - csvRows.append({u'name': name, u'value': value}) + value = ' '.join(sorted(values, reverse=True)) + csvRows.append({'name': name, 'value': value}) for app in auth_apps: # put apps at bottom csvRows.append(app) - writeCSVfile(csvRows, titles, u'Customer Report - %s' % tryDate, todrive=to_drive) + writeCSVfile(csvRows, titles, 'Customer Report - %s' % tryDate, todrive=to_drive) else: - if report in [u'doc', u'docs']: - report = u'drive' - elif report in [u'calendars']: - report = u'calendar' - elif report == u'logins': - report = u'login' - elif report == u'tokens': - report = u'token' - elif report == u'group': - report = u'groups' - page_message = u'Got %%num_items%% items\n' - activities = callGAPIpages(rep.activities(), u'list', u'items', page_message=page_message, applicationName=report, + if report in ['doc', 'docs']: + report = 'drive' + elif report in ['calendars']: + report = 'calendar' + elif report == 'logins': + report = 'login' + elif report == 'tokens': + report = 'token' + elif report == 'group': + report = 'groups' + page_message = 'Got %%num_items%% items\n' + activities = callGAPIpages(rep.activities(), 'list', 'items', page_message=page_message, applicationName=report, userKey=userKey, customerId=customerId, actorIpAddress=actorIpAddress, startTime=startTime, endTime=endTime, eventName=eventName, filters=filters) if len(activities) > 0: - titles = [u'name'] + titles = ['name'] csvRows = [] for activity in activities: - events = activity[u'events'] - del activity[u'events'] + events = activity['events'] + del activity['events'] activity_row = flatten_json(activity) for event in events: - for item in event.get(u'parameters', []): - if item[u'name'] in [u'start_time', u'end_time']: - val = item.get(u'intValue') + for item in event.get('parameters', []): + if item['name'] in ['start_time', 'end_time']: + val = item.get('intValue') if val is not None: val = int(val) if val >= 62135683200: - item[u'dateTimeValue'] = datetime.datetime.fromtimestamp(val-62135683200).isoformat() - item.pop(u'intValue') + item['dateTimeValue'] = datetime.datetime.fromtimestamp(val-62135683200).isoformat() + item.pop('intValue') row = flatten_json(event) row.update(activity_row) for item in row: if item not in titles: titles.append(item) csvRows.append(row) - sortCSVTitles([u'name',], titles) - writeCSVfile(csvRows, titles, u'%s Activity Report' % report.capitalize(), to_drive) + sortCSVTitles(['name',], titles) + writeCSVfile(csvRows, titles, '%s Activity Report' % report.capitalize(), to_drive) def watchGmail(users): - cs_data = readFile(GC_Values[GC_CLIENT_SECRETS_JSON], mode=u'rb', continueOnError=True, displayError=True, encoding=None) + cs_data = readFile(GC_Values[GC_CLIENT_SECRETS_JSON], mode='rb', continueOnError=True, displayError=True, encoding=None) cs_json = json.loads(cs_data) - project = u'projects/{0}'.format(cs_json[u'installed'][u'project_id']) - gamTopics = project+u'/topics/gam-pubsub-gmail-' - gamSubscriptions = project+u'/subscriptions/gam-pubsub-gmail-' - pubsub = buildGAPIObject(u'pubsub') - topics = callGAPIpages(pubsub.projects().topics(), u'list', items=u'topics', project=project) + project = 'projects/{0}'.format(cs_json['installed']['project_id']) + gamTopics = project+'/topics/gam-pubsub-gmail-' + gamSubscriptions = project+'/subscriptions/gam-pubsub-gmail-' + pubsub = buildGAPIObject('pubsub') + topics = callGAPIpages(pubsub.projects().topics(), 'list', items='topics', project=project) for atopic in topics: - if atopic[u'name'].startswith(gamTopics): - topic = atopic[u'name'] + if atopic['name'].startswith(gamTopics): + topic = atopic['name'] break else: topic = gamTopics+uuid.uuid4() - callGAPI(pubsub.projects().topics(), u'create', name=topic, body={}) - body = {u'policy': {u'bindings': [{u'members': [u'serviceAccount:gmail-api-push@system.gserviceaccount.com'], u'role': u'roles/pubsub.editor'}]}} - callGAPI(pubsub.projects().topics(), u'setIamPolicy', resource=topic, body=body) - subscriptions = callGAPIpages(pubsub.projects().topics().subscriptions(), u'list', items=u'subscriptions', topic=topic) + callGAPI(pubsub.projects().topics(), 'create', name=topic, body={}) + body = {'policy': {'bindings': [{'members': ['serviceAccount:gmail-api-push@system.gserviceaccount.com'], 'role': 'roles/pubsub.editor'}]}} + callGAPI(pubsub.projects().topics(), 'setIamPolicy', resource=topic, body=body) + subscriptions = callGAPIpages(pubsub.projects().topics().subscriptions(), 'list', items='subscriptions', topic=topic) for asubscription in subscriptions: if asubscription.startswith(gamSubscriptions): subscription = asubscription break else: subscription = gamSubscriptions+uuid.uuid4() - callGAPI(pubsub.projects().subscriptions(), u'create', name=subscription, body={u'topic': topic}) + callGAPI(pubsub.projects().subscriptions(), 'create', name=subscription, body={'topic': topic}) gmails = {} for user in users: - gmails[user] = {u'g': buildGmailGAPIObject(user)[1]} - callGAPI(gmails[user][u'g'].users(), u'watch', userId=u'me', body={u'topicName': topic}) - gmails[user]['seen_historyId'] = callGAPI(gmails[user][u'g'].users(), u'getProfile', userId=u'me', fields=u'historyId')[u'historyId'] - print 'Watching for events...' + gmails[user] = {'g': buildGmailGAPIObject(user)[1]} + callGAPI(gmails[user]['g'].users(), 'watch', userId='me', body={'topicName': topic}) + gmails[user]['seen_historyId'] = callGAPI(gmails[user]['g'].users(), 'getProfile', userId='me', fields='historyId')['historyId'] + print('Watching for events...') while True: - results = callGAPI(pubsub.projects().subscriptions(), u'pull', subscription=subscription, body={u'maxMessages': 100}) - if u'receivedMessages' in results: + results = callGAPI(pubsub.projects().subscriptions(), 'pull', subscription=subscription, body={'maxMessages': 100}) + if 'receivedMessages' in results: ackIds = [] update_history = [] - for message in results[u'receivedMessages']: - if u'data' in message[u'message']: - decoded_message = json.loads(base64.b64decode(message[u'message'][u'data'])) - if u'historyId' in decoded_message: - update_history.append(decoded_message[u'emailAddress']) - if u'ackId' in message: - ackIds.append(message[u'ackId']) + for message in results['receivedMessages']: + if 'data' in message['message']: + decoded_message = json.loads(base64.b64decode(message['message']['data'])) + if 'historyId' in decoded_message: + update_history.append(decoded_message['emailAddress']) + if 'ackId' in message: + ackIds.append(message['ackId']) if ackIds: - callGAPI(pubsub.projects().subscriptions(), u'acknowledge', subscription=subscription, body={u'ackIds': ackIds}) + callGAPI(pubsub.projects().subscriptions(), 'acknowledge', subscription=subscription, body={'ackIds': ackIds}) if update_history: for a_user in update_history: - results = callGAPI(gmails[a_user][u'g'].users().history(), u'list', userId=u'me', startHistoryId=gmails[a_user][u'seen_historyId']) - if u'history' in results: - for history in results[u'history']: - if history.keys() == [u'messages', u'id']: + results = callGAPI(gmails[a_user]['g'].users().history(), 'list', userId='me', startHistoryId=gmails[a_user]['seen_historyId']) + if 'history' in results: + for history in results['history']: + if list(history.keys()) == ['messages', 'id']: continue - if u'labelsAdded' in history: - for labelling in history[u'labelsAdded']: - print u'%s labels %s added to %s' % (a_user, u', '.join(labelling[u'labelIds']), labelling[u'message'][u'id']) - if u'labelsRemoved' in history: - for labelling in history[u'labelsRemoved']: - print u'%s labels %s removed from %s' % (a_user, u', '.join(labelling[u'labelIds']), labelling[u'message'][u'id']) - if u'messagesDeleted' in history: - for deleting in history[u'messagesDeleted']: - print u'%s permanently deleted message %s' % (a_user, deleting[u'message'][u'id']) - if u'messagesAdded' in history: - for adding in history[u'messagesAdded']: - print u'%s created message %s with labels %s' % (a_user, adding[u'message'][u'id'], u', '.join(adding[u'message'][u'labelIds'])) - gmails[a_user][u'seen_historyId'] = results[u'historyId'] + if 'labelsAdded' in history: + for labelling in history['labelsAdded']: + print('%s labels %s added to %s' % (a_user, ', '.join(labelling['labelIds']), labelling['message']['id'])) + if 'labelsRemoved' in history: + for labelling in history['labelsRemoved']: + print('%s labels %s removed from %s' % (a_user, ', '.join(labelling['labelIds']), labelling['message']['id'])) + if 'messagesDeleted' in history: + for deleting in history['messagesDeleted']: + print('%s permanently deleted message %s' % (a_user, deleting['message']['id'])) + if 'messagesAdded' in history: + for adding in history['messagesAdded']: + print('%s created message %s with labels %s' % (a_user, adding['message']['id'], ', '.join(adding['message']['labelIds']))) + gmails[a_user]['seen_historyId'] = results['historyId'] def addDelegates(users, i): if i == 4: - if sys.argv[i].lower() != u'to': - systemErrorExit(2, u'%s is not a valid argument for "gam delegate", expected to' % sys.argv[i]) + if sys.argv[i].lower() != 'to': + systemErrorExit(2, '%s is not a valid argument for "gam delegate", expected to' % sys.argv[i]) i += 1 delegate = normalizeEmailAddressOrUID(sys.argv[i], noUid=True) i = 0 @@ -1663,53 +1673,53 @@ def addDelegates(users, i): delegator, gmail = buildGmailGAPIObject(delegator) if not gmail: continue - print u"Giving %s delegate access to %s (%s/%s)" % (delegate, delegator, i, count) - callGAPI(gmail.users().settings().delegates(), u'create', soft_errors=True, userId=u'me', body={u'delegateEmail': delegate}) + print("Giving %s delegate access to %s (%s/%s)" % (delegate, delegator, i, count)) + callGAPI(gmail.users().settings().delegates(), 'create', soft_errors=True, userId='me', body={'delegateEmail': delegate}) def gen_sha512_hash(password): - return sha512_crypt.encrypt(password, rounds=5000) + return sha512_crypt.hash(password, rounds=5000) def printShowDelegates(users, csvFormat): if csvFormat: todrive = False csvRows = [] - titles = [u'User', u'delegateAddress', u'delegationStatus'] + titles = ['User', 'delegateAddress', 'delegationStatus'] else: csvStyle = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if not csvFormat and myarg == u'csv': + if not csvFormat and myarg == 'csv': csvStyle = True i += 1 - elif csvFormat and myarg == u'todrive': + elif csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, u'%s is not a valid argument for "gam show delegates"' % sys.argv[i]) + systemErrorExit(2, '%s is not a valid argument for "gam show delegates"' % sys.argv[i]) count = len(users) i = 1 for user in users: user, gmail = buildGmailGAPIObject(user) if not gmail: continue - sys.stderr.write(u"Getting delegates for %s (%s/%s)...\n" % (user, i, count)) + sys.stderr.write("Getting delegates for %s (%s/%s)...\n" % (user, i, count)) i += 1 - delegates = callGAPI(gmail.users().settings().delegates(), u'list', soft_errors=True, userId=u'me') - if delegates and u'delegates' in delegates: - for delegate in delegates[u'delegates']: - delegateAddress = delegate[u'delegateEmail'] - status = delegate[u'verificationStatus'] + delegates = callGAPI(gmail.users().settings().delegates(), 'list', soft_errors=True, userId='me') + if delegates and 'delegates' in delegates: + for delegate in delegates['delegates']: + delegateAddress = delegate['delegateEmail'] + status = delegate['verificationStatus'] if csvFormat: - row = {u'User': user, u'delegateAddress': delegateAddress, u'delegationStatus': status} + row = {'User': user, 'delegateAddress': delegateAddress, 'delegationStatus': status} csvRows.append(row) else: if csvStyle: - print u'%s,%s,%s' % (user, delegateAddress, status) + print('%s,%s,%s' % (user, delegateAddress, status)) else: - print utils.convertUTF8(u"Delegator: %s\n Status: %s\n Delegate Email: %s\n" % (user, status, delegateAddress)) + print(utils.convertUTF8("Delegator: %s\n Status: %s\n Delegate Email: %s\n" % (user, status, delegateAddress))) if csvFormat: - writeCSVfile(csvRows, titles, u'Delegates', todrive) + writeCSVfile(csvRows, titles, 'Delegates', todrive) def deleteDelegate(users): delegate = normalizeEmailAddressOrUID(sys.argv[5], noUid=True) @@ -1720,29 +1730,29 @@ def deleteDelegate(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Deleting %s delegate access to %s (%s/%s)" % (delegate, user, i, count) - callGAPI(gmail.users().settings().delegates(), u'delete', soft_errors=True, userId=u'me', delegateEmail=delegate) + print("Deleting %s delegate access to %s (%s/%s)" % (delegate, user, i, count)) + callGAPI(gmail.users().settings().delegates(), 'delete', soft_errors=True, userId='me', delegateEmail=delegate) def doAddCourseParticipant(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') courseId = addCourseIdScope(sys.argv[2]) noScopeCourseId = removeCourseIdScope(courseId) participant_type = sys.argv[4].lower() new_id = sys.argv[5] - if participant_type in [u'student', u'students']: + if participant_type in ['student', 'students']: new_id = normalizeEmailAddressOrUID(new_id) - callGAPI(croom.courses().students(), u'create', courseId=courseId, body={u'userId': new_id}) - print u'Added %s as a student of course %s' % (new_id, noScopeCourseId) - elif participant_type in [u'teacher', u'teachers']: + callGAPI(croom.courses().students(), 'create', courseId=courseId, body={'userId': new_id}) + print('Added %s as a student of course %s' % (new_id, noScopeCourseId)) + elif participant_type in ['teacher', 'teachers']: new_id = normalizeEmailAddressOrUID(new_id) - callGAPI(croom.courses().teachers(), u'create', courseId=courseId, body={u'userId': new_id}) - print u'Added %s as a teacher of course %s' % (new_id, noScopeCourseId) - elif participant_type in [u'alias']: + callGAPI(croom.courses().teachers(), 'create', courseId=courseId, body={'userId': new_id}) + print('Added %s as a teacher of course %s' % (new_id, noScopeCourseId)) + elif participant_type in ['alias']: new_id = addCourseIdScope(new_id) - callGAPI(croom.courses().aliases(), u'create', courseId=courseId, body={u'alias': new_id}) - print u'Added %s as an alias of course %s' % (removeCourseIdScope(new_id), noScopeCourseId) + callGAPI(croom.courses().aliases(), 'create', courseId=courseId, body={'alias': new_id}) + print('Added %s as an alias of course %s' % (removeCourseIdScope(new_id), noScopeCourseId)) else: - systemErrorExit(2, u'%s is not a valid argument to "gam course ID add"' % participant_type) + systemErrorExit(2, '%s is not a valid argument to "gam course ID add"' % participant_type) def doSyncCourseParticipants(): courseId = addCourseIdScope(sys.argv[2]) @@ -1750,400 +1760,400 @@ def doSyncCourseParticipants(): diff_entity_type = sys.argv[5].lower() diff_entity = sys.argv[6] current_course_users = getUsersToModify(entity_type=participant_type, entity=courseId) - print + print() current_course_users = [x.lower() for x in current_course_users] - if diff_entity_type == u'courseparticipants': + if diff_entity_type == 'courseparticipants': diff_entity_type = participant_type diff_against_users = getUsersToModify(entity_type=diff_entity_type, entity=diff_entity) - print + print() diff_against_users = [x.lower() for x in diff_against_users] to_add = list(set(diff_against_users) - set(current_course_users)) to_remove = list(set(current_course_users) - set(diff_against_users)) gam_commands = [] for add_email in to_add: - gam_commands.append([u'gam', u'course', courseId, u'add', participant_type, add_email]) + gam_commands.append(['gam', 'course', courseId, 'add', participant_type, add_email]) for remove_email in to_remove: - gam_commands.append([u'gam', u'course', courseId, u'remove', participant_type, remove_email]) + gam_commands.append(['gam', 'course', courseId, 'remove', participant_type, remove_email]) run_batch(gam_commands) def doDelCourseParticipant(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') courseId = addCourseIdScope(sys.argv[2]) noScopeCourseId = removeCourseIdScope(courseId) participant_type = sys.argv[4].lower() remove_id = sys.argv[5] - if participant_type in [u'student', u'students']: + if participant_type in ['student', 'students']: remove_id = normalizeEmailAddressOrUID(remove_id) - callGAPI(croom.courses().students(), u'delete', courseId=courseId, userId=remove_id) - print u'Removed %s as a student of course %s' % (remove_id, noScopeCourseId) - elif participant_type in [u'teacher', u'teachers']: + callGAPI(croom.courses().students(), 'delete', courseId=courseId, userId=remove_id) + print('Removed %s as a student of course %s' % (remove_id, noScopeCourseId)) + elif participant_type in ['teacher', 'teachers']: remove_id = normalizeEmailAddressOrUID(remove_id) - callGAPI(croom.courses().teachers(), u'delete', courseId=courseId, userId=remove_id) - print u'Removed %s as a teacher of course %s' % (remove_id, noScopeCourseId) - elif participant_type in [u'alias']: + callGAPI(croom.courses().teachers(), 'delete', courseId=courseId, userId=remove_id) + print('Removed %s as a teacher of course %s' % (remove_id, noScopeCourseId)) + elif participant_type in ['alias']: remove_id = addCourseIdScope(remove_id) - callGAPI(croom.courses().aliases(), u'delete', courseId=courseId, alias=remove_id) - print u'Removed %s as an alias of course %s' % (removeCourseIdScope(remove_id), noScopeCourseId) + callGAPI(croom.courses().aliases(), 'delete', courseId=courseId, alias=remove_id) + print('Removed %s as an alias of course %s' % (removeCourseIdScope(remove_id), noScopeCourseId)) else: - systemErrorExit(2, u'%s is not a valid argument to "gam course ID delete"' % participant_type) + systemErrorExit(2, '%s is not a valid argument to "gam course ID delete"' % participant_type) def doDelCourse(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') courseId = addCourseIdScope(sys.argv[3]) - callGAPI(croom.courses(), u'delete', id=courseId) - print u'Deleted Course %s' % courseId + callGAPI(croom.courses(), 'delete', id=courseId) + print('Deleted Course %s' % courseId) def _getValidCourseStates(croom): - return [state for state in croom._rootDesc[u'schemas'][u'Course'][u'properties'][u'courseState'][u'enum'] if state != u'COURSE_STATE_UNSPECIFIED'] + return [state for state in croom._rootDesc['schemas']['Course']['properties']['courseState']['enum'] if state != 'COURSE_STATE_UNSPECIFIED'] def _getValidatedState(state, validStates): state = state.upper() if state not in validStates: - systemErrorExit(2, u'course state must be one of: %s. Got %s' % (u', '.join(validStates).lower(), state.lower())) + systemErrorExit(2, 'course state must be one of: %s. Got %s' % (', '.join(validStates).lower(), state.lower())) return state def getCourseAttribute(myarg, value, body, croom, function): - if myarg == u'name': - body[u'name'] = value - elif myarg == u'section': - body[u'section'] = value - elif myarg == u'heading': - body[u'descriptionHeading'] = value - elif myarg == u'description': - body[u'description'] = value.replace(u'\\n', u'\n') - elif myarg == u'room': - body[u'room'] = value - elif myarg in [u'owner', u'ownerid', u'teacher']: - body[u'ownerId'] = normalizeEmailAddressOrUID(value) - elif myarg in [u'state', u'status']: + if myarg == 'name': + body['name'] = value + elif myarg == 'section': + body['section'] = value + elif myarg == 'heading': + body['descriptionHeading'] = value + elif myarg == 'description': + body['description'] = value.replace('\\n', '\n') + elif myarg == 'room': + body['room'] = value + elif myarg in ['owner', 'ownerid', 'teacher']: + body['ownerId'] = normalizeEmailAddressOrUID(value) + elif myarg in ['state', 'status']: validStates = _getValidCourseStates(croom) - body[u'courseState'] = _getValidatedState(value, validStates) + body['courseState'] = _getValidatedState(value, validStates) else: systemErrorExit(2, '%s is not a valid argument to "gam %s course"' % (myarg, function)) def _getCourseStates(croom, value, courseStates): validStates = _getValidCourseStates(croom) - for state in value.replace(u',', u' ').split(): + for state in value.replace(',', ' ').split(): courseStates.append(_getValidatedState(state, validStates)) def doUpdateCourse(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') courseId = addCourseIdScope(sys.argv[3]) body = {} i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - getCourseAttribute(myarg, sys.argv[i+1], body, croom, u'update') + getCourseAttribute(myarg, sys.argv[i+1], body, croom, 'update') i += 2 - updateMask = u','.join(body.keys()) - body[u'id'] = courseId - result = callGAPI(croom.courses(), u'patch', id=courseId, body=body, updateMask=updateMask) - print u'Updated Course %s' % result[u'id'] + updateMask = ','.join(list(body.keys())) + body['id'] = courseId + result = callGAPI(croom.courses(), 'patch', id=courseId, body=body, updateMask=updateMask) + print('Updated Course %s' % result['id']) def doCreateDomain(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') domain_name = sys.argv[3] - body = {u'domainName': domain_name} - callGAPI(cd.domains(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) - print u'Added domain %s' % domain_name + body = {'domainName': domain_name} + callGAPI(cd.domains(), 'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) + print('Added domain %s' % domain_name) def doCreateDomainAlias(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') body = {} - body[u'domainAliasName'] = sys.argv[3] - body[u'parentDomainName'] = sys.argv[4] - callGAPI(cd.domainAliases(), u'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) + body['domainAliasName'] = sys.argv[3] + body['parentDomainName'] = sys.argv[4] + callGAPI(cd.domainAliases(), 'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doUpdateDomain(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') domain_name = sys.argv[3] i = 4 body = {} while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'primary': - body[u'customerDomain'] = domain_name + if myarg == 'primary': + body['customerDomain'] = domain_name i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam update domain"' % sys.argv[i]) - callGAPI(cd.customers(), u'update', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) - print u'%s is now the primary domain.' % domain_name + callGAPI(cd.customers(), 'update', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) + print('%s is now the primary domain.' % domain_name) def doGetDomainInfo(): - if (len(sys.argv) < 4) or (sys.argv[3] == u'logo'): + if (len(sys.argv) < 4) or (sys.argv[3] == 'logo'): doGetCustomerInfo() return - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') domainName = sys.argv[3] - result = callGAPI(cd.domains(), u'get', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) - if u'creationTime' in result: - result[u'creationTime'] = unicode(datetime.datetime.fromtimestamp(int(result[u'creationTime'])/1000)) - if u'domainAliases' in result: - for i in range(0, len(result[u'domainAliases'])): - if u'creationTime' in result[u'domainAliases'][i]: - result[u'domainAliases'][i][u'creationTime'] = unicode(datetime.datetime.fromtimestamp(int(result[u'domainAliases'][i][u'creationTime'])/1000)) + result = callGAPI(cd.domains(), 'get', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) + if 'creationTime' in result: + result['creationTime'] = str(datetime.datetime.fromtimestamp(int(result['creationTime'])/1000)) + if 'domainAliases' in result: + for i in range(0, len(result['domainAliases'])): + if 'creationTime' in result['domainAliases'][i]: + result['domainAliases'][i]['creationTime'] = str(datetime.datetime.fromtimestamp(int(result['domainAliases'][i]['creationTime'])/1000)) print_json(None, result) def doGetDomainAliasInfo(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') alias = sys.argv[3] - result = callGAPI(cd.domainAliases(), u'get', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=alias) - if u'creationTime' in result: - result[u'creationTime'] = unicode(datetime.datetime.fromtimestamp(int(result[u'creationTime'])/1000)) + result = callGAPI(cd.domainAliases(), 'get', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=alias) + if 'creationTime' in result: + result['creationTime'] = str(datetime.datetime.fromtimestamp(int(result['creationTime'])/1000)) print_json(None, result) def doGetCustomerInfo(): - cd = buildGAPIObject(u'directory') - customer_info = callGAPI(cd.customers(), u'get', customerKey=GC_Values[GC_CUSTOMER_ID]) - print u'Customer ID: %s' % customer_info[u'id'] - print u'Primary Domain: %s' % customer_info[u'customerDomain'] - result = callGAPI(cd.domains(), u'get', - customer=customer_info[u'id'], domainName=customer_info[u'customerDomain'], fields=u'verified') - print u'Primary Domain Verified: %s' % result[u'verified'] - print u'Customer Creation Time: %s' % customer_info[u'customerCreationTime'] - print u'Default Language: %s' % customer_info.get(u'language', u'Unset (defaults to en)') - if u'postalAddress' in customer_info: - print u'Address:' + cd = buildGAPIObject('directory') + customer_info = callGAPI(cd.customers(), 'get', customerKey=GC_Values[GC_CUSTOMER_ID]) + print('Customer ID: %s' % customer_info['id']) + print('Primary Domain: %s' % customer_info['customerDomain']) + result = callGAPI(cd.domains(), 'get', + customer=customer_info['id'], domainName=customer_info['customerDomain'], fields='verified') + print('Primary Domain Verified: %s' % result['verified']) + print('Customer Creation Time: %s' % customer_info['customerCreationTime']) + print('Default Language: %s' % customer_info.get('language', 'Unset (defaults to en)')) + if 'postalAddress' in customer_info: + print('Address:') for field in ADDRESS_FIELDS_PRINT_ORDER: - if field in customer_info[u'postalAddress']: - print u' %s: %s' % (field, customer_info[u'postalAddress'][field]) - if u'phoneNumber' in customer_info: - print u'Phone: %s' % customer_info[u'phoneNumber'] - print u'Admin Secondary Email: %s' % customer_info[u'alternateEmail'] + if field in customer_info['postalAddress']: + print(' %s: %s' % (field, customer_info['postalAddress'][field])) + if 'phoneNumber' in customer_info: + print('Phone: %s' % customer_info['phoneNumber']) + print('Admin Secondary Email: %s' % customer_info['alternateEmail']) user_counts_map = { - u'accounts:num_users': u'Total Users', - u'accounts:gsuite_basic_total_licenses': u'G Suite Basic Licenses', - u'accounts:gsuite_basic_used_licenses': u'G Suite Basic Users', - u'accounts:gsuite_enterprise_total_licenses': u'G Suite Enterprise Licenses', - u'accounts:gsuite_enterprise_used_licenses': u'G Suite Enterprise Users', - u'accounts:gsuite_unlimited_total_licenses': u'G Suite Business Licenses', - u'accounts:gsuite_unlimited_used_licenses': u'G Suite Business Users' + 'accounts:num_users': 'Total Users', + 'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses', + 'accounts:gsuite_basic_used_licenses': 'G Suite Basic Users', + 'accounts:gsuite_enterprise_total_licenses': 'G Suite Enterprise Licenses', + 'accounts:gsuite_enterprise_used_licenses': 'G Suite Enterprise Users', + 'accounts:gsuite_unlimited_total_licenses': 'G Suite Business Licenses', + 'accounts:gsuite_unlimited_used_licenses': 'G Suite Business Users' } - parameters = u','.join(user_counts_map.keys()) + parameters = ','.join(list(user_counts_map.keys())) tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT) customerId = GC_Values[GC_CUSTOMER_ID] if customerId == MY_CUSTOMER: customerId = None - rep = buildGAPIObject(u'reports') + rep = buildGAPIObject('reports') usage = None while True: try: - usage = callGAPIpages(rep.customerUsageReports(), u'get', u'usageReports', throw_reasons=[GAPI_INVALID], + usage = callGAPIpages(rep.customerUsageReports(), 'get', 'usageReports', throw_reasons=[GAPI_INVALID], customerId=customerId, date=tryDate, parameters=parameters) break except GAPI_invalid as e: tryDate = _adjustDate(str(e)) if not usage: - print u'No user count data available.' + print('No user count data available.') return - print u'User counts as of %s:' % tryDate - for item in usage[0][u'parameters']: - api_name = user_counts_map.get(item[u'name']) - api_value = int(item.get(u'intValue', 0)) + print('User counts as of %s:' % tryDate) + for item in usage[0]['parameters']: + api_name = user_counts_map.get(item['name']) + api_value = int(item.get('intValue', 0)) if api_name and api_value: - print u' {}: {:,}'.format(api_name, api_value) + print(' {}: {:,}'.format(api_name, api_value)) def doUpdateCustomer(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') body = {} i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') + myarg = sys.argv[i].lower().replace('_', '') if myarg in ADDRESS_FIELDS_ARGUMENT_MAP: - body.setdefault(u'postalAddress', {}) - body[u'postalAddress'][ADDRESS_FIELDS_ARGUMENT_MAP[myarg]] = sys.argv[i+1] + body.setdefault('postalAddress', {}) + body['postalAddress'][ADDRESS_FIELDS_ARGUMENT_MAP[myarg]] = sys.argv[i+1] i += 2 - elif myarg in [u'adminsecondaryemail', u'alternateemail']: - body[u'alternateEmail'] = sys.argv[i+1] + elif myarg in ['adminsecondaryemail', 'alternateemail']: + body['alternateEmail'] = sys.argv[i+1] i += 2 - elif myarg in [u'phone', u'phonenumber']: - body[u'phoneNumber'] = sys.argv[i+1] + elif myarg in ['phone', 'phonenumber']: + body['phoneNumber'] = sys.argv[i+1] i += 2 - elif myarg == u'language': - body[u'language'] = sys.argv[i+1] + elif myarg == 'language': + body['language'] = sys.argv[i+1] i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam update customer"' % myarg) if not body: systemErrorExit(2, 'no arguments specified for "gam update customer"') - callGAPI(cd.customers(), u'patch', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) - print u'Updated customer' + callGAPI(cd.customers(), 'patch', customerKey=GC_Values[GC_CUSTOMER_ID], body=body) + print('Updated customer') def doDelDomain(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') domainName = sys.argv[3] - callGAPI(cd.domains(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) + callGAPI(cd.domains(), 'delete', customer=GC_Values[GC_CUSTOMER_ID], domainName=domainName) def doDelDomainAlias(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') domainAliasName = sys.argv[3] - callGAPI(cd.domainAliases(), u'delete', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=domainAliasName) + callGAPI(cd.domainAliases(), 'delete', customer=GC_Values[GC_CUSTOMER_ID], domainAliasName=domainAliasName) def doPrintDomains(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False - titles = [u'domainName',] + titles = ['domainName',] csvRows = [] i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print domains".' % sys.argv[i]) - results = callGAPI(cd.domains(), u'list', customer=GC_Values[GC_CUSTOMER_ID]) - for domain in results[u'domains']: + results = callGAPI(cd.domains(), 'list', customer=GC_Values[GC_CUSTOMER_ID]) + for domain in results['domains']: domain_attributes = {} - domain[u'type'] = [u'secondary', u'primary'][domain[u'isPrimary']] + domain['type'] = ['secondary', 'primary'][domain['isPrimary']] for attr in domain: - if attr in [u'kind', u'etag', u'domainAliases', u'isPrimary']: + if attr in ['kind', 'etag', 'domainAliases', 'isPrimary']: continue - if attr in [u'creationTime',]: - domain[attr] = unicode(datetime.datetime.fromtimestamp(int(domain[attr])/1000)) + if attr in ['creationTime',]: + domain[attr] = str(datetime.datetime.fromtimestamp(int(domain[attr])/1000)) if attr not in titles: titles.append(attr) domain_attributes[attr] = domain[attr] csvRows.append(domain_attributes) - if u'domainAliases' in domain: - for aliasdomain in domain[u'domainAliases']: - aliasdomain[u'domainName'] = aliasdomain[u'domainAliasName'] - del aliasdomain[u'domainAliasName'] - aliasdomain[u'type'] = u'alias' + if 'domainAliases' in domain: + for aliasdomain in domain['domainAliases']: + aliasdomain['domainName'] = aliasdomain['domainAliasName'] + del aliasdomain['domainAliasName'] + aliasdomain['type'] = 'alias' aliasdomain_attributes = {} for attr in aliasdomain: - if attr in [u'kind', u'etag']: + if attr in ['kind', 'etag']: continue - if attr in [u'creationTime',]: - aliasdomain[attr] = unicode(datetime.datetime.fromtimestamp(int(aliasdomain[attr])/1000)) + if attr in ['creationTime',]: + aliasdomain[attr] = str(datetime.datetime.fromtimestamp(int(aliasdomain[attr])/1000)) if attr not in titles: titles.append(attr) aliasdomain_attributes[attr] = aliasdomain[attr] csvRows.append(aliasdomain_attributes) - writeCSVfile(csvRows, titles, u'Domains', todrive) + writeCSVfile(csvRows, titles, 'Domains', todrive) def doPrintDomainAliases(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False - titles = [u'domainAliasName',] + titles = ['domainAliasName',] csvRows = [] i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print domainaliases".' % sys.argv[i]) - results = callGAPI(cd.domainAliases(), u'list', customer=GC_Values[GC_CUSTOMER_ID]) - for domainAlias in results[u'domainAliases']: + results = callGAPI(cd.domainAliases(), 'list', customer=GC_Values[GC_CUSTOMER_ID]) + for domainAlias in results['domainAliases']: domainAlias_attributes = {} for attr in domainAlias: - if attr in [u'kind', u'etag']: + if attr in ['kind', 'etag']: continue - if attr == u'creationTime': - domainAlias[attr] = unicode(datetime.datetime.fromtimestamp(int(domainAlias[attr])/1000)) + if attr == 'creationTime': + domainAlias[attr] = str(datetime.datetime.fromtimestamp(int(domainAlias[attr])/1000)) if attr not in titles: titles.append(attr) domainAlias_attributes[attr] = domainAlias[attr] csvRows.append(domainAlias_attributes) - writeCSVfile(csvRows, titles, u'Domains', todrive) + writeCSVfile(csvRows, titles, 'Domains', todrive) def doDelAdmin(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') roleAssignmentId = sys.argv[3] - print u'Deleting Admin Role Assignment %s' % roleAssignmentId - callGAPI(cd.roleAssignments(), u'delete', + print('Deleting Admin Role Assignment %s' % roleAssignmentId) + callGAPI(cd.roleAssignments(), 'delete', customer=GC_Values[GC_CUSTOMER_ID], roleAssignmentId=roleAssignmentId) def doCreateAdmin(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') user = normalizeEmailAddressOrUID(sys.argv[3]) - body = {u'assignedTo': convertEmailAddressToUID(user, cd)} + body = {'assignedTo': convertEmailAddressToUID(user, cd)} role = sys.argv[4] - body[u'roleId'] = getRoleId(role) - body[u'scopeType'] = sys.argv[5].upper() - if body[u'scopeType'] not in [u'CUSTOMER', u'ORG_UNIT']: - systemErrorExit(3, 'scope type must be customer or org_unit; got %s' % body[u'scopeType']) - if body[u'scopeType'] == u'ORG_UNIT': + body['roleId'] = getRoleId(role) + body['scopeType'] = sys.argv[5].upper() + if body['scopeType'] not in ['CUSTOMER', 'ORG_UNIT']: + systemErrorExit(3, 'scope type must be customer or org_unit; got %s' % body['scopeType']) + if body['scopeType'] == 'ORG_UNIT': orgUnit, orgUnitId = getOrgUnitId(sys.argv[6], cd) - body[u'orgUnitId'] = orgUnitId[3:] - scope = u'ORG_UNIT {0}'.format(orgUnit) + body['orgUnitId'] = orgUnitId[3:] + scope = 'ORG_UNIT {0}'.format(orgUnit) else: - scope = u'CUSTOMER' - print u'Giving %s admin role %s for %s' % (user, role, scope) - callGAPI(cd.roleAssignments(), u'insert', + scope = 'CUSTOMER' + print('Giving %s admin role %s for %s' % (user, role, scope)) + callGAPI(cd.roleAssignments(), 'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doPrintAdminRoles(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False - titles = [u'roleId', u'roleName', u'roleDescription', u'isSuperAdminRole', u'isSystemRole'] - fields = u'nextPageToken,items({0})'.format(u','.join(titles)) + titles = ['roleId', 'roleName', 'roleDescription', 'isSuperAdminRole', 'isSystemRole'] + fields = 'nextPageToken,items({0})'.format(','.join(titles)) csvRows = [] i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print adminroles".' % sys.argv[i]) - roles = callGAPIpages(cd.roles(), u'list', u'items', + roles = callGAPIpages(cd.roles(), 'list', 'items', customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for role in roles: role_attrib = {} - for key, value in role.items(): + for key, value in list(role.items()): role_attrib[key] = value csvRows.append(role_attrib) - writeCSVfile(csvRows, titles, u'Admin Roles', todrive) + writeCSVfile(csvRows, titles, 'Admin Roles', todrive) def doPrintAdmins(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') roleId = None userKey = None todrive = False - fields = u'nextPageToken,items({0})'.format(u','.join([u'roleAssignmentId', u'roleId', u'assignedTo', u'scopeType', u'orgUnitId'])) - titles = [u'roleAssignmentId', u'roleId', u'role', u'assignedTo', u'assignedToUser', u'scopeType', u'orgUnitId', u'orgUnit'] + fields = 'nextPageToken,items({0})'.format(','.join(['roleAssignmentId', 'roleId', 'assignedTo', 'scopeType', 'orgUnitId'])) + titles = ['roleAssignmentId', 'roleId', 'role', 'assignedTo', 'assignedToUser', 'scopeType', 'orgUnitId', 'orgUnit'] csvRows = [] i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'user': + if myarg == 'user': userKey = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg == u'role': + elif myarg == 'role': roleId = getRoleId(sys.argv[i+1]) i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print admins".' % sys.argv[i]) - admins = callGAPIpages(cd.roleAssignments(), u'list', u'items', + admins = callGAPIpages(cd.roleAssignments(), 'list', 'items', customer=GC_Values[GC_CUSTOMER_ID], userKey=userKey, roleId=roleId, fields=fields) for admin in admins: admin_attrib = {} - for key, value in admin.items(): - if key == u'assignedTo': - admin_attrib[u'assignedToUser'] = user_from_userid(value) - elif key == u'roleId': - admin_attrib[u'role'] = role_from_roleid(value) - elif key == u'orgUnitId': - value = u'id:{0}'.format(value) - admin_attrib[u'orgUnit'] = orgunit_from_orgunitid(value) + for key, value in list(admin.items()): + if key == 'assignedTo': + admin_attrib['assignedToUser'] = user_from_userid(value) + elif key == 'roleId': + admin_attrib['role'] = role_from_roleid(value) + elif key == 'orgUnitId': + value = 'id:{0}'.format(value) + admin_attrib['orgUnit'] = orgunit_from_orgunitid(value) admin_attrib[key] = value csvRows.append(admin_attrib) - writeCSVfile(csvRows, titles, u'Admins', todrive) + writeCSVfile(csvRows, titles, 'Admins', todrive) def buildOrgUnitIdToNameMap(): - cd = buildGAPIObject(u'directory') - result = callGAPI(cd.orgunits(), u'list', + cd = buildGAPIObject('directory') + result = callGAPI(cd.orgunits(), 'list', customerId=GC_Values[GC_CUSTOMER_ID], - fields=u'organizationUnits(orgUnitPath,orgUnitId)', type=u'all') + fields='organizationUnits(orgUnitPath,orgUnitId)', type='all') GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME] = {} - for orgUnit in result[u'organizationUnits']: - GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][orgUnit[u'orgUnitId']] = orgUnit[u'orgUnitPath'] + for orgUnit in result['organizationUnits']: + GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME][orgUnit['orgUnitId']] = orgUnit['orgUnitPath'] def orgunit_from_orgunitid(orgunitid): if not GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME]: @@ -2151,16 +2161,16 @@ def orgunit_from_orgunitid(orgunitid): return GM_Globals[GM_MAP_ORGUNIT_ID_TO_NAME].get(orgunitid, orgunitid) def buildRoleIdToNameToIdMap(): - cd = buildGAPIObject(u'directory') - result = callGAPIpages(cd.roles(), u'list', u'items', + cd = buildGAPIObject('directory') + result = callGAPIpages(cd.roles(), 'list', 'items', customer=GC_Values[GC_CUSTOMER_ID], - fields=u'nextPageToken,items(roleId,roleName)', + fields='nextPageToken,items(roleId,roleName)', maxResults=100) GM_Globals[GM_MAP_ROLE_ID_TO_NAME] = {} GM_Globals[GM_MAP_ROLE_NAME_TO_ID] = {} for role in result: - GM_Globals[GM_MAP_ROLE_ID_TO_NAME][role[u'roleId']] = role[u'roleName'] - GM_Globals[GM_MAP_ROLE_NAME_TO_ID][role[u'roleName']] = role[u'roleId'] + GM_Globals[GM_MAP_ROLE_ID_TO_NAME][role['roleId']] = role['roleName'] + GM_Globals[GM_MAP_ROLE_NAME_TO_ID][role['roleName']] = role['roleId'] def role_from_roleid(roleid): if not GM_Globals[GM_MAP_ROLE_ID_TO_NAME]: @@ -2183,64 +2193,64 @@ def getRoleId(role): return roleId def buildUserIdToNameMap(): - cd = buildGAPIObject(u'directory') - result = callGAPIpages(cd.users(), u'list', u'users', + cd = buildGAPIObject('directory') + result = callGAPIpages(cd.users(), 'list', 'users', customer=GC_Values[GC_CUSTOMER_ID], - fields=u'nextPageToken,users(id,primaryEmail)', + fields='nextPageToken,users(id,primaryEmail)', maxResults=GC_Values[GC_USER_MAX_RESULTS]) GM_Globals[GM_MAP_USER_ID_TO_NAME] = {} for user in result: - GM_Globals[GM_MAP_USER_ID_TO_NAME][user[u'id']] = user[u'primaryEmail'] + GM_Globals[GM_MAP_USER_ID_TO_NAME][user['id']] = user['primaryEmail'] def user_from_userid(userid): if not GM_Globals[GM_MAP_USER_ID_TO_NAME]: buildUserIdToNameMap() - return GM_Globals[GM_MAP_USER_ID_TO_NAME].get(userid, u'') + return GM_Globals[GM_MAP_USER_ID_TO_NAME].get(userid, '') def appID2app(dt, appID): - for serviceName, serviceID in SERVICE_NAME_TO_ID_MAP.items(): + for serviceName, serviceID in list(SERVICE_NAME_TO_ID_MAP.items()): if appID == serviceID: return serviceName - online_services = callGAPIpages(dt.applications(), u'list', u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + online_services = callGAPIpages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for online_service in online_services: - if appID == online_service[u'id']: - return online_service[u'name'] - return u'applicationId: {0}'.format(appID) + if appID == online_service['id']: + return online_service['name'] + return 'applicationId: {0}'.format(appID) def app2appID(dt, app): serviceName = app.lower() if serviceName in SERVICE_NAME_CHOICES_MAP: return (SERVICE_NAME_CHOICES_MAP[serviceName], SERVICE_NAME_TO_ID_MAP[SERVICE_NAME_CHOICES_MAP[serviceName]]) - online_services = callGAPIpages(dt.applications(), u'list', u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + online_services = callGAPIpages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for online_service in online_services: - if serviceName == online_service[u'name'].lower(): - return (online_service[u'name'], online_service[u'id']) + if serviceName == online_service['name'].lower(): + return (online_service['name'], online_service['id']) systemErrorExit(2, '%s is not a valid service for data transfer.' % app) def convertToUserID(user): cg = UID_PATTERN.match(user) if cg: return cg.group(1) - cd = buildGAPIObject(u'directory') - if user.find(u'@') == -1: - user = u'%s@%s' % (user, GC_Values[GC_DOMAIN]) + cd = buildGAPIObject('directory') + if user.find('@') == -1: + user = '%s@%s' % (user, GC_Values[GC_DOMAIN]) try: - return callGAPI(cd.users(), u'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_BAD_REQUEST, GAPI_FORBIDDEN], userKey=user, fields=u'id')[u'id'] + return callGAPI(cd.users(), 'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_BAD_REQUEST, GAPI_FORBIDDEN], userKey=user, fields='id')['id'] except (GAPI_userNotFound, GAPI_badRequest, GAPI_forbidden): systemErrorExit(3, 'no such user %s' % user) def convertUserIDtoEmail(uid): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') try: - return callGAPI(cd.users(), u'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_BAD_REQUEST, GAPI_FORBIDDEN], userKey=uid, fields=u'primaryEmail')[u'primaryEmail'] + return callGAPI(cd.users(), 'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_BAD_REQUEST, GAPI_FORBIDDEN], userKey=uid, fields='primaryEmail')['primaryEmail'] except (GAPI_userNotFound, GAPI_badRequest, GAPI_forbidden): - return u'uid:{0}'.format(uid) + return 'uid:{0}'.format(uid) def doCreateDataTransfer(): - dt = buildGAPIObject(u'datatransfer') + dt = buildGAPIObject('datatransfer') body = {} old_owner = sys.argv[3] - body[u'oldOwnerUserId'] = convertToUserID(old_owner) + body['oldOwnerUserId'] = convertToUserID(old_owner) apps = sys.argv[4].split(",") appNameList = [] appIDList = [] @@ -2248,216 +2258,216 @@ def doCreateDataTransfer(): while i < len(apps): serviceName, serviceID = app2appID(dt, apps[i]) appNameList.append(serviceName) - appIDList.append({u'applicationId': serviceID}) + appIDList.append({'applicationId': serviceID}) i += 1 - body[u'applicationDataTransfers'] = (appIDList) + body['applicationDataTransfers'] = (appIDList) new_owner = sys.argv[5] - body[u'newOwnerUserId'] = convertToUserID(new_owner) + body['newOwnerUserId'] = convertToUserID(new_owner) parameters = {} i = 6 while i < len(sys.argv): - parameters[sys.argv[i].upper()] = sys.argv[i+1].upper().split(u',') + parameters[sys.argv[i].upper()] = sys.argv[i+1].upper().split(',') i += 2 i = 0 - for key, value in parameters.items(): - body[u'applicationDataTransfers'][i].setdefault(u'applicationTransferParams', []) - body[u'applicationDataTransfers'][i][u'applicationTransferParams'].append({u'key': key, u'value': value}) + for key, value in list(parameters.items()): + body['applicationDataTransfers'][i].setdefault('applicationTransferParams', []) + body['applicationDataTransfers'][i]['applicationTransferParams'].append({'key': key, 'value': value}) i += 1 - result = callGAPI(dt.transfers(), u'insert', body=body, fields=u'id')[u'id'] - print u'Submitted request id %s to transfer %s from %s to %s' % (result, ','.join(map(str, appNameList)), old_owner, new_owner) + result = callGAPI(dt.transfers(), 'insert', body=body, fields='id')['id'] + print('Submitted request id %s to transfer %s from %s to %s' % (result, ','.join(map(str, appNameList)), old_owner, new_owner)) def doPrintTransferApps(): - dt = buildGAPIObject(u'datatransfer') - apps = callGAPIpages(dt.applications(), u'list', u'applications', customerId=GC_Values[GC_CUSTOMER_ID]) + dt = buildGAPIObject('datatransfer') + apps = callGAPIpages(dt.applications(), 'list', 'applications', customerId=GC_Values[GC_CUSTOMER_ID]) for app in apps: print_json(None, app) - print + print() def doPrintDataTransfers(): - dt = buildGAPIObject(u'datatransfer') + dt = buildGAPIObject('datatransfer') i = 3 newOwnerUserId = None oldOwnerUserId = None status = None todrive = False - titles = [u'id',] + titles = ['id',] csvRows = [] while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg in [u'olduser', u'oldowner']: + myarg = sys.argv[i].lower().replace('_', '') + if myarg in ['olduser', 'oldowner']: oldOwnerUserId = convertToUserID(sys.argv[i+1]) i += 2 - elif myarg in [u'newuser', u'newowner']: + elif myarg in ['newuser', 'newowner']: newOwnerUserId = convertToUserID(sys.argv[i+1]) i += 2 - elif myarg == u'status': + elif myarg == 'status': status = sys.argv[i+1] i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print transfers"' % sys.argv[i]) - transfers = callGAPIpages(dt.transfers(), u'list', u'dataTransfers', + transfers = callGAPIpages(dt.transfers(), 'list', 'dataTransfers', customerId=GC_Values[GC_CUSTOMER_ID], status=status, newOwnerUserId=newOwnerUserId, oldOwnerUserId=oldOwnerUserId) for transfer in transfers: - for i in range(0, len(transfer[u'applicationDataTransfers'])): + for i in range(0, len(transfer['applicationDataTransfers'])): a_transfer = {} - a_transfer[u'oldOwnerUserEmail'] = convertUserIDtoEmail(transfer[u'oldOwnerUserId']) - a_transfer[u'newOwnerUserEmail'] = convertUserIDtoEmail(transfer[u'newOwnerUserId']) - a_transfer[u'requestTime'] = transfer[u'requestTime'] - a_transfer[u'applicationId'] = transfer[u'applicationDataTransfers'][i][u'applicationId'] - a_transfer[u'application'] = appID2app(dt, a_transfer[u'applicationId']) - a_transfer[u'status'] = transfer[u'applicationDataTransfers'][i][u'applicationTransferStatus'] - a_transfer[u'id'] = transfer[u'id'] - if u'applicationTransferParams' in transfer[u'applicationDataTransfers'][i]: - for param in transfer[u'applicationDataTransfers'][i][u'applicationTransferParams']: - a_transfer[param[u'key']] = u','.join(param.get(u'value', [])) + a_transfer['oldOwnerUserEmail'] = convertUserIDtoEmail(transfer['oldOwnerUserId']) + a_transfer['newOwnerUserEmail'] = convertUserIDtoEmail(transfer['newOwnerUserId']) + a_transfer['requestTime'] = transfer['requestTime'] + a_transfer['applicationId'] = transfer['applicationDataTransfers'][i]['applicationId'] + a_transfer['application'] = appID2app(dt, a_transfer['applicationId']) + a_transfer['status'] = transfer['applicationDataTransfers'][i]['applicationTransferStatus'] + a_transfer['id'] = transfer['id'] + if 'applicationTransferParams' in transfer['applicationDataTransfers'][i]: + for param in transfer['applicationDataTransfers'][i]['applicationTransferParams']: + a_transfer[param['key']] = ','.join(param.get('value', [])) for title in a_transfer: if title not in titles: titles.append(title) csvRows.append(a_transfer) - writeCSVfile(csvRows, titles, u'Data Transfers', todrive) + writeCSVfile(csvRows, titles, 'Data Transfers', todrive) def doGetDataTransferInfo(): - dt = buildGAPIObject(u'datatransfer') + dt = buildGAPIObject('datatransfer') dtId = sys.argv[3] - transfer = callGAPI(dt.transfers(), u'get', dataTransferId=dtId) - print u'Old Owner: %s' % convertUserIDtoEmail(transfer[u'oldOwnerUserId']) - print u'New Owner: %s' % convertUserIDtoEmail(transfer[u'newOwnerUserId']) - print u'Request Time: %s' % transfer[u'requestTime'] - for app in transfer[u'applicationDataTransfers']: - print u'Application: %s' % appID2app(dt, app[u'applicationId']) - print u'Status: %s' % app[u'applicationTransferStatus'] - print u'Parameters:' - if u'applicationTransferParams' in app: - for param in app[u'applicationTransferParams']: - print u' %s: %s' % (param[u'key'], u','.join(param.get(u'value', []))) + transfer = callGAPI(dt.transfers(), 'get', dataTransferId=dtId) + print('Old Owner: %s' % convertUserIDtoEmail(transfer['oldOwnerUserId'])) + print('New Owner: %s' % convertUserIDtoEmail(transfer['newOwnerUserId'])) + print('Request Time: %s' % transfer['requestTime']) + for app in transfer['applicationDataTransfers']: + print('Application: %s' % appID2app(dt, app['applicationId'])) + print('Status: %s' % app['applicationTransferStatus']) + print('Parameters:') + if 'applicationTransferParams' in app: + for param in app['applicationTransferParams']: + print(' %s: %s' % (param['key'], ','.join(param.get('value', [])))) else: - print u' None' - print + print(' None') + print() def doPrintShowGuardians(csvFormat): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') invitedEmailAddress = None - studentIds = [u'-',] + studentIds = ['-',] states = None service = croom.userProfiles().guardians() - items = u'guardians' + items = 'guardians' itemName = 'Guardians' if csvFormat: csvRows = [] todrive = False - titles = [u'studentEmail', u'studentId', u'invitedEmailAddress', u'guardianId'] + titles = ['studentEmail', 'studentId', 'invitedEmailAddress', 'guardianId'] i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 - elif myarg == u'invitedguardian': + elif myarg == 'invitedguardian': invitedEmailAddress = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg == u'student': + elif myarg == 'student': studentIds = [normalizeStudentGuardianEmailAddressOrUID(sys.argv[i+1])] i += 2 - elif myarg == u'invitations': + elif myarg == 'invitations': service = croom.userProfiles().guardianInvitations() - items = u'guardianInvitations' + items = 'guardianInvitations' itemName = 'Guardian Invitations' - titles = [u'studentEmail', u'studentId', u'invitedEmailAddress', u'invitationId'] + titles = ['studentEmail', 'studentId', 'invitedEmailAddress', 'invitationId'] if states is None: - states = [u'COMPLETE', u'PENDING', u'GUARDIAN_INVITATION_STATE_UNSPECIFIED'] + states = ['COMPLETE', 'PENDING', 'GUARDIAN_INVITATION_STATE_UNSPECIFIED'] i += 1 - elif myarg == u'states': - states = sys.argv[i+1].upper().replace(u',', u' ').split() + elif myarg == 'states': + states = sys.argv[i+1].upper().replace(',', ' ').split() i += 2 elif myarg in usergroup_types: studentIds = getUsersToModify(entity_type=myarg, entity=sys.argv[i+1]) i += 2 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s guardians"' % (sys.argv[i], [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s guardians"' % (sys.argv[i], ['show', 'print'][csvFormat])) i = 0 count = len(studentIds) for studentId in studentIds: i += 1 studentId = normalizeStudentGuardianEmailAddressOrUID(studentId) - kwargs = {u'invitedEmailAddress': invitedEmailAddress, u'studentId': studentId} - if items == u'guardianInvitations': - kwargs[u'states'] = states - if studentId != u'-': + kwargs = {'invitedEmailAddress': invitedEmailAddress, 'studentId': studentId} + if items == 'guardianInvitations': + kwargs['states'] = states + if studentId != '-': if csvFormat: sys.stderr.write('\r') sys.stderr.flush() - sys.stderr.write(u'Getting %s for %s%s%s' % (itemName, studentId, currentCount(i, count), u' ' * 40)) - guardians = callGAPIpages(service, u'list', items, soft_errors=True, **kwargs) + sys.stderr.write('Getting %s for %s%s%s' % (itemName, studentId, currentCount(i, count), ' ' * 40)) + guardians = callGAPIpages(service, 'list', items, soft_errors=True, **kwargs) if not csvFormat: - print u'Student: {0}, {1}:{2}'.format(studentId, itemName, currentCount(i, count)) + print('Student: {0}, {1}:{2}'.format(studentId, itemName, currentCount(i, count))) for guardian in guardians: - print_json(None, guardian, spacing=u' ') + print_json(None, guardian, spacing=' ') else: for guardian in guardians: - guardian[u'studentEmail'] = studentId + guardian['studentEmail'] = studentId addRowTitlesToCSVfile(flatten_json(guardian), csvRows, titles) if csvFormat: - sys.stderr.write(u'\n') + sys.stderr.write('\n') writeCSVfile(csvRows, titles, itemName, todrive) def doInviteGuardian(): - croom = buildGAPIObject(u'classroom') - body = {u'invitedEmailAddress': normalizeEmailAddressOrUID(sys.argv[3])} + croom = buildGAPIObject('classroom') + body = {'invitedEmailAddress': normalizeEmailAddressOrUID(sys.argv[3])} studentId = normalizeStudentGuardianEmailAddressOrUID(sys.argv[4]) - result = callGAPI(croom.userProfiles().guardianInvitations(), u'create', studentId=studentId, body=body) - print u'Invited email %s as guardian of %s. Invite ID %s' % (result[u'invitedEmailAddress'], studentId, result[u'invitationId']) + result = callGAPI(croom.userProfiles().guardianInvitations(), 'create', studentId=studentId, body=body) + print('Invited email %s as guardian of %s. Invite ID %s' % (result['invitedEmailAddress'], studentId, result['invitationId'])) def _cancelGuardianInvitation(croom, studentId, invitationId): try: - result = callGAPI(croom.userProfiles().guardianInvitations(), u'patch', + result = callGAPI(croom.userProfiles().guardianInvitations(), 'patch', throw_reasons=[GAPI_FAILED_PRECONDITION, GAPI_FORBIDDEN, GAPI_NOT_FOUND], - studentId=studentId, invitationId=invitationId, updateMask=u'state', body={u'state': u'COMPLETE'}) - print u'Cancelled PENDING guardian invitation for %s as guardian of %s' % (result[u'invitedEmailAddress'], studentId) + studentId=studentId, invitationId=invitationId, updateMask='state', body={'state': 'COMPLETE'}) + print('Cancelled PENDING guardian invitation for %s as guardian of %s' % (result['invitedEmailAddress'], studentId)) return True except GAPI_failedPrecondition: - stderrErrorMsg(u'Guardian invitation %s for %s status is not PENDING' % (invitationId, studentId)) + stderrErrorMsg('Guardian invitation %s for %s status is not PENDING' % (invitationId, studentId)) GM_Globals[GM_SYSEXITRC] = 3 return True except GAPI_forbidden: - entityUnknownWarning(u'Student', studentId, 0, 0) + entityUnknownWarning('Student', studentId, 0, 0) sys.exit(3) except GAPI_notFound: return False def doCancelGuardianInvitation(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') invitationId = sys.argv[3] studentId = normalizeStudentGuardianEmailAddressOrUID(sys.argv[4]) if not _cancelGuardianInvitation(croom, studentId, invitationId): - systemErrorExit(3, u'Guardian invitation %s for %s does not exist' % (invitationId, studentId)) + systemErrorExit(3, 'Guardian invitation %s for %s does not exist' % (invitationId, studentId)) def _deleteGuardian(croom, studentId, guardianId, guardianEmail): try: - callGAPI(croom.userProfiles().guardians(), u'delete', + callGAPI(croom.userProfiles().guardians(), 'delete', throw_reasons=[GAPI_FORBIDDEN, GAPI_NOT_FOUND], studentId=studentId, guardianId=guardianId) - print u'Deleted %s as a guardian of %s' % (guardianEmail, studentId) + print('Deleted %s as a guardian of %s' % (guardianEmail, studentId)) return True except GAPI_forbidden: - entityUnknownWarning(u'Student', studentId, 0, 0) + entityUnknownWarning('Student', studentId, 0, 0) sys.exit(3) except GAPI_notFound: return False def doDeleteGuardian(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') invitationsOnly = False guardianId = normalizeStudentGuardianEmailAddressOrUID(sys.argv[3]) - guardianIdIsEmail = guardianId.find(u'@') != -1 + guardianIdIsEmail = guardianId.find('@') != -1 studentId = normalizeStudentGuardianEmailAddressOrUID(sys.argv[4]) i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg in [u'invitation', u'invitations']: + if myarg in ['invitation', 'invitations']: invitationsOnly = True i += 1 else: @@ -2465,16 +2475,16 @@ def doDeleteGuardian(): if not invitationsOnly: if guardianIdIsEmail: try: - results = callGAPIpages(croom.userProfiles().guardians(), u'list', u'guardians', + results = callGAPIpages(croom.userProfiles().guardians(), 'list', 'guardians', throw_reasons=[GAPI_FORBIDDEN], studentId=studentId, invitedEmailAddress=guardianId, - fields=u'nextPageToken,guardians(studentId,guardianId)') + fields='nextPageToken,guardians(studentId,guardianId)') if len(results) > 0: for result in results: - _deleteGuardian(croom, result[u'studentId'], result[u'guardianId'], guardianId) + _deleteGuardian(croom, result['studentId'], result['guardianId'], guardianId) return except GAPI_forbidden: - entityUnknownWarning(u'Student', studentId, 0, 0) + entityUnknownWarning('Student', studentId, 0, 0) sys.exit(3) else: if _deleteGuardian(croom, studentId, guardianId, guardianId): @@ -2482,16 +2492,16 @@ def doDeleteGuardian(): # See if there's a pending invitation if guardianIdIsEmail: try: - results = callGAPIpages(croom.userProfiles().guardianInvitations(), u'list', u'guardianInvitations', + results = callGAPIpages(croom.userProfiles().guardianInvitations(), 'list', 'guardianInvitations', throw_reasons=[GAPI_FORBIDDEN], - studentId=studentId, invitedEmailAddress=guardianId, states=[u'PENDING',], - fields=u'nextPageToken,guardianInvitations(studentId,invitationId)') + studentId=studentId, invitedEmailAddress=guardianId, states=['PENDING',], + fields='nextPageToken,guardianInvitations(studentId,invitationId)') if len(results) > 0: for result in results: - status = _cancelGuardianInvitation(croom, result[u'studentId'], result[u'invitationId']) + status = _cancelGuardianInvitation(croom, result['studentId'], result['invitationId']) sys.exit(status) except GAPI_forbidden: - entityUnknownWarning(u'Student', studentId, 0, 0) + entityUnknownWarning('Student', studentId, 0, 0) sys.exit(3) else: if _cancelGuardianInvitation(croom, studentId, guardianId): @@ -2499,81 +2509,81 @@ def doDeleteGuardian(): systemErrorExit(3, '%s is not a guardian of %s and no invitation exists.' % (guardianId, studentId)) def doCreateCourse(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') body = {} i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg in [u'alias', u'id']: - body[u'id'] = u'd:%s' % sys.argv[i+1] + if myarg in ['alias', 'id']: + body['id'] = 'd:%s' % sys.argv[i+1] i += 2 else: - getCourseAttribute(myarg, sys.argv[i+1], body, croom, u'create') + getCourseAttribute(myarg, sys.argv[i+1], body, croom, 'create') i += 2 - if u'ownerId' not in body: + if 'ownerId' not in body: systemErrorExit(2, 'expected teacher )') - if u'name' not in body: + if 'name' not in body: systemErrorExit(2, 'expected name )') - result = callGAPI(croom.courses(), u'create', body=body) - print u'Created course %s' % result[u'id'] + result = callGAPI(croom.courses(), 'create', body=body) + print('Created course %s' % result['id']) def doGetCourseInfo(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') courseId = addCourseIdScope(sys.argv[3]) - info = callGAPI(croom.courses(), u'get', id=courseId) - info['ownerEmail'] = convertUIDtoEmailAddress(u'uid:%s' % info['ownerId']) + info = callGAPI(croom.courses(), 'get', id=courseId) + info['ownerEmail'] = convertUIDtoEmailAddress('uid:%s' % info['ownerId']) print_json(None, info) - teachers = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', courseId=courseId) - students = callGAPIpages(croom.courses().students(), u'list', u'students', courseId=courseId) + teachers = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', courseId=courseId) + students = callGAPIpages(croom.courses().students(), 'list', 'students', courseId=courseId) try: - aliases = callGAPIpages(croom.courses().aliases(), u'list', u'aliases', throw_reasons=[GAPI_NOT_IMPLEMENTED], courseId=courseId) + aliases = callGAPIpages(croom.courses().aliases(), 'list', 'aliases', throw_reasons=[GAPI_NOT_IMPLEMENTED], courseId=courseId) except GAPI_notImplemented: aliases = [] if aliases: - print u'Aliases:' + print('Aliases:') for alias in aliases: - print u' %s' % alias[u'alias'][2:] - print u'Participants:' - print u' Teachers:' + print(' %s' % alias['alias'][2:]) + print('Participants:') + print(' Teachers:') for teacher in teachers: try: - print utils.convertUTF8(u' %s - %s' % (teacher[u'profile'][u'name'][u'fullName'], teacher[u'profile'][u'emailAddress'])) + print(utils.convertUTF8(' %s - %s' % (teacher['profile']['name']['fullName'], teacher['profile']['emailAddress']))) except KeyError: - print utils.convertUTF8(u' %s' % teacher[u'profile'][u'name'][u'fullName']) - print u' Students:' + print(utils.convertUTF8(' %s' % teacher['profile']['name']['fullName'])) + print(' Students:') for student in students: try: - print utils.convertUTF8(u' %s - %s' % (student[u'profile'][u'name'][u'fullName'], student[u'profile'][u'emailAddress'])) + print(utils.convertUTF8(' %s - %s' % (student['profile']['name']['fullName'], student['profile']['emailAddress']))) except KeyError: - print utils.convertUTF8(u' %s' % student[u'profile'][u'name'][u'fullName']) + print(utils.convertUTF8(' %s' % student['profile']['name']['fullName'])) COURSE_ARGUMENT_TO_PROPERTY_MAP = { - u'alternatelink': u'alternateLink', - u'coursegroupemail': u'courseGroupEmail', - u'coursematerialsets': u'courseMaterialSets', - u'coursestate': u'courseState', - u'creationtime': u'creationTime', - u'description': u'description', - u'descriptionheading': u'descriptionHeading', - u'enrollmentcode': u'enrollmentCode', - u'guardiansenabled': u'guardiansEnabled', - u'id': u'id', - u'name': u'name', - u'ownerid': u'ownerId', - u'room': u'room', - u'section': u'section', - u'teacherfolder': u'teacherFolder', - u'teachergroupemail': u'teacherGroupEmail', - u'updatetime': u'updateTime', + 'alternatelink': 'alternateLink', + 'coursegroupemail': 'courseGroupEmail', + 'coursematerialsets': 'courseMaterialSets', + 'coursestate': 'courseState', + 'creationtime': 'creationTime', + 'description': 'description', + 'descriptionheading': 'descriptionHeading', + 'enrollmentcode': 'enrollmentCode', + 'guardiansenabled': 'guardiansEnabled', + 'id': 'id', + 'name': 'name', + 'ownerid': 'ownerId', + 'room': 'room', + 'section': 'section', + 'teacherfolder': 'teacherFolder', + 'teachergroupemail': 'teacherGroupEmail', + 'updatetime': 'updateTime', } def doPrintCourses(): def _processFieldsList(myarg, i, fList): fieldNameList = sys.argv[i+1] - for field in fieldNameList.lower().replace(u',', u' ').split(): + for field in fieldNameList.lower().replace(',', ' ').split(): if field in COURSE_ARGUMENT_TO_PROPERTY_MAP: - if field != u'id': + if field != 'id': fList.append(COURSE_ARGUMENT_TO_PROPERTY_MAP[field]) else: systemErrorExit(2, '%s is not a valid argument for "gam print courses %s"' % (field, myarg)) @@ -2587,198 +2597,198 @@ def doPrintCourses(): j = 0 for member in participants: memberTitles = [] - prefix = u'{0}.{1}.'.format(role, j) - profile = member[u'profile'] - emailAddress = profile.get(u'emailAddress') + prefix = '{0}.{1}.'.format(role, j) + profile = member['profile'] + emailAddress = profile.get('emailAddress') if emailAddress: - memberTitle = prefix+u'emailAddress' + memberTitle = prefix+'emailAddress' course[memberTitle] = emailAddress memberTitles.append(memberTitle) - memberId = profile.get(u'id') + memberId = profile.get('id') if memberId: - memberTitle = prefix+u'id' + memberTitle = prefix+'id' course[memberTitle] = memberId memberTitles.append(memberTitle) - fullName = profile.get(u'name', {}).get(u'fullName') + fullName = profile.get('name', {}).get('fullName') if fullName: - memberTitle = prefix+u'name.fullName' + memberTitle = prefix+'name.fullName' course[memberTitle] = fullName memberTitles.append(memberTitle) addTitlesToCSVfile(memberTitles, titles) j += 1 - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') todrive = False fieldsList = [] skipFieldsList = [] - titles = [u'id',] + titles = ['id',] csvRows = [] ownerEmails = studentId = teacherId = None courseStates = [] countsOnly = showAliases = False - delimiter = u' ' - showMembers = u'' + delimiter = ' ' + showMembers = '' i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'teacher': + if myarg == 'teacher': teacherId = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg == u'student': + elif myarg == 'student': studentId = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg in [u'state', u'states', u'status']: + elif myarg in ['state', 'states', 'status']: _getCourseStates(croom, sys.argv[i+1], courseStates) i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'alias', u'aliases']: + elif myarg in ['alias', 'aliases']: showAliases = True i += 1 - elif myarg == u'countsonly': + elif myarg == 'countsonly': countsOnly = True i += 1 - elif myarg == u'delimiter': + elif myarg == 'delimiter': delimiter = sys.argv[i+1] i += 2 - elif myarg == u'show': + elif myarg == 'show': showMembers = sys.argv[i+1].lower() - if showMembers not in [u'all', u'students', u'teachers']: + if showMembers not in ['all', 'students', 'teachers']: systemErrorExit(2, 'show must be all, students or teachers; got %s' % showMembers) i += 2 - elif myarg == u'fields': + elif myarg == 'fields': if not fieldsList: - fieldsList = [u'id',] + fieldsList = ['id',] _processFieldsList(myarg, i, fieldsList) i += 2 - elif myarg == u'skipfields': + elif myarg == 'skipfields': _processFieldsList(myarg, i, skipFieldsList) i += 2 - elif myarg == u'owneremail': + elif myarg == 'owneremail': ownerEmails = {} - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print courses"' % sys.argv[i]) if ownerEmails is not None and fieldsList: - fieldsList.append(u'ownerId') - fields = u'nextPageToken,courses({0})'.format(u','.join(set(fieldsList))) if fieldsList else None - printGettingAllItems(u'Courses', None) - page_message = u'Got %%num_items%% Courses...\n' - all_courses = callGAPIpages(croom.courses(), u'list', u'courses', page_message=page_message, teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields=fields) + fieldsList.append('ownerId') + fields = 'nextPageToken,courses({0})'.format(','.join(set(fieldsList))) if fieldsList else None + printGettingAllItems('Courses', None) + page_message = 'Got %%num_items%% Courses...\n' + all_courses = callGAPIpages(croom.courses(), 'list', 'courses', page_message=page_message, teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields=fields) for course in all_courses: if ownerEmails is not None: - ownerId = course[u'ownerId'] + ownerId = course['ownerId'] if ownerId not in ownerEmails: - ownerEmails[ownerId] = convertUIDtoEmailAddress(u'uid:%s' % ownerId, cd=cd) - course[u'ownerEmail'] = ownerEmails[ownerId] + ownerEmails[ownerId] = convertUIDtoEmailAddress('uid:%s' % ownerId, cd=cd) + course['ownerEmail'] = ownerEmails[ownerId] for field in skipFieldsList: course.pop(field, None) addRowTitlesToCSVfile(flatten_json(course), csvRows, titles) if showAliases or showMembers: if showAliases: - titles.append(u'Aliases') + titles.append('Aliases') if showMembers: if countsOnly: - teachersFields = u'nextPageToken,teachers(profile(id))' - studentsFields = u'nextPageToken,students(profile(id))' + teachersFields = 'nextPageToken,teachers(profile(id))' + studentsFields = 'nextPageToken,students(profile(id))' else: - teachersFields = u'nextPageToken,teachers(profile)' - studentsFields = u'nextPageToken,students(profile)' + teachersFields = 'nextPageToken,teachers(profile)' + studentsFields = 'nextPageToken,students(profile)' i = 0 count = len(csvRows) for course in csvRows: i += 1 - courseId = course[u'id'] + courseId = course['id'] if showAliases: - alias_message = u' Got %%%%num_items%%%% Aliases for course %s%s' % (courseId, currentCount(i, count)) - course_aliases = callGAPIpages(croom.courses().aliases(), u'list', u'aliases', + alias_message = ' Got %%%%num_items%%%% Aliases for course %s%s' % (courseId, currentCount(i, count)) + course_aliases = callGAPIpages(croom.courses().aliases(), 'list', 'aliases', page_message=alias_message, courseId=courseId) - course[u'Aliases'] = delimiter.join([alias[u'alias'][2:] for alias in course_aliases]) + course['Aliases'] = delimiter.join([alias['alias'][2:] for alias in course_aliases]) if showMembers: - if showMembers != u'students': - teacher_message = u' Got %%%%num_items%%%% Teachers for course %s%s' % (courseId, currentCount(i, count)) - results = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', + if showMembers != 'students': + teacher_message = ' Got %%%%num_items%%%% Teachers for course %s%s' % (courseId, currentCount(i, count)) + results = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', page_message=teacher_message, courseId=courseId, fields=teachersFields) - _saveParticipants(course, results, u'teachers') - if showMembers != u'teachers': - student_message = u' Got %%%%num_items%%%% Students for course %s%s' % (courseId, currentCount(i, count)) - results = callGAPIpages(croom.courses().students(), u'list', u'students', + _saveParticipants(course, results, 'teachers') + if showMembers != 'teachers': + student_message = ' Got %%%%num_items%%%% Students for course %s%s' % (courseId, currentCount(i, count)) + results = callGAPIpages(croom.courses().students(), 'list', 'students', page_message=student_message, courseId=courseId, fields=studentsFields) - _saveParticipants(course, results, u'students') - sortCSVTitles([u'id', u'name'], titles) - writeCSVfile(csvRows, titles, u'Courses', todrive) + _saveParticipants(course, results, 'students') + sortCSVTitles(['id', 'name'], titles) + writeCSVfile(csvRows, titles, 'Courses', todrive) def doPrintCourseParticipants(): - croom = buildGAPIObject(u'classroom') + croom = buildGAPIObject('classroom') todrive = False - titles = [u'courseId',] + titles = ['courseId',] csvRows = [] courses = [] teacherId = None studentId = None courseStates = [] - showMembers = u'all' + showMembers = 'all' i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg in [u'course', u'class']: + if myarg in ['course', 'class']: courses.append(addCourseIdScope(sys.argv[i+1])) i += 2 - elif myarg == u'teacher': + elif myarg == 'teacher': teacherId = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg == u'student': + elif myarg == 'student': studentId = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg in [u'state', u'states', u'status']: + elif myarg in ['state', 'states', 'status']: _getCourseStates(croom, sys.argv[i+1], courseStates) i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 - elif myarg == u'show': + elif myarg == 'show': showMembers = sys.argv[i+1].lower() - if showMembers not in [u'all', u'students', u'teachers']: + if showMembers not in ['all', 'students', 'teachers']: systemErrorExit(2, 'show must be all, students or teachers; got %s' % showMembers) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam print course-participants"' % sys.argv[i]) if len(courses) == 0: - printGettingAllItems(u'Courses', None) - page_message = u'Got %%num_items%% Courses...\n' - all_courses = callGAPIpages(croom.courses(), u'list', u'courses', page_message=page_message, - teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields=u'nextPageToken,courses(id,name)') + printGettingAllItems('Courses', None) + page_message = 'Got %%num_items%% Courses...\n' + all_courses = callGAPIpages(croom.courses(), 'list', 'courses', page_message=page_message, + teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields='nextPageToken,courses(id,name)') else: all_courses = [] for course in courses: - all_courses.append(callGAPI(croom.courses(), u'get', id=course, fields=u'id,name')) + all_courses.append(callGAPI(croom.courses(), 'get', id=course, fields='id,name')) i = 0 count = len(all_courses) for course in all_courses: i += 1 - courseId = course[u'id'] - if showMembers != u'students': - page_message = u' Got %%%%num_items%%%% Teachers for course %s (%s/%s)' % (courseId, i, count) - teachers = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', page_message=page_message, courseId=courseId) + courseId = course['id'] + if showMembers != 'students': + page_message = ' Got %%%%num_items%%%% Teachers for course %s (%s/%s)' % (courseId, i, count) + teachers = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', page_message=page_message, courseId=courseId) for teacher in teachers: - addRowTitlesToCSVfile(flatten_json(teacher, flattened={u'courseId': courseId, u'courseName': course[u'name'], u'userRole': u'TEACHER'}), csvRows, titles) - if showMembers != u'teachers': - page_message = u' Got %%%%num_items%%%% Students for course %s (%s/%s)' % (courseId, i, count) - students = callGAPIpages(croom.courses().students(), u'list', u'students', page_message=page_message, courseId=courseId) + addRowTitlesToCSVfile(flatten_json(teacher, flattened={'courseId': courseId, 'courseName': course['name'], 'userRole': 'TEACHER'}), csvRows, titles) + if showMembers != 'teachers': + page_message = ' Got %%%%num_items%%%% Students for course %s (%s/%s)' % (courseId, i, count) + students = callGAPIpages(croom.courses().students(), 'list', 'students', page_message=page_message, courseId=courseId) for student in students: - addRowTitlesToCSVfile(flatten_json(student, flattened={u'courseId': courseId, u'courseName': course[u'name'], u'userRole': u'STUDENT'}), csvRows, titles) - sortCSVTitles([u'courseId', u'courseName', u'userRole', u'userId'], titles) - writeCSVfile(csvRows, titles, u'Course Participants', todrive) + addRowTitlesToCSVfile(flatten_json(student, flattened={'courseId': courseId, 'courseName': course['name'], 'userRole': 'STUDENT'}), csvRows, titles) + sortCSVTitles(['courseId', 'courseName', 'userRole', 'userId'], titles) + writeCSVfile(csvRows, titles, 'Course Participants', todrive) def doPrintPrintJobs(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') todrive = False - titles = [u'printerid', u'id'] + titles = ['printerid', 'id'] csvRows = [] printerid = None owner = None @@ -2791,53 +2801,53 @@ def doPrintPrintJobs(): jobLimit = PRINTJOBS_DEFAULT_JOB_LIMIT i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'olderthan', u'newerthan']: - if myarg == u'olderthan': - older_or_newer = u'older' + elif myarg in ['olderthan', 'newerthan']: + if myarg == 'olderthan': + older_or_newer = 'older' else: - older_or_newer = u'newer' + older_or_newer = 'newer' age_number = sys.argv[i+1][:-1] if not age_number.isdigit(): systemErrorExit(2, 'expected a number; got %s' % age_number) age_unit = sys.argv[i+1][-1].lower() - if age_unit == u'm': + if age_unit == 'm': age = int(time.time()) - (int(age_number) * 60) - elif age_unit == u'h': + elif age_unit == 'h': age = int(time.time()) - (int(age_number) * 60 * 60) - elif age_unit == u'd': + elif age_unit == 'd': age = int(time.time()) - (int(age_number) * 60 * 60 * 24) else: systemErrorExit(2, 'expected m (minutes), h (hours) or d (days); got %s' % age_unit) i += 2 - elif myarg == u'query': + elif myarg == 'query': query = sys.argv[i+1] i += 2 - elif myarg == u'status': + elif myarg == 'status': status = sys.argv[i+1] i += 2 - elif myarg == u'ascending': + elif myarg == 'ascending': descending = False i += 1 - elif myarg == u'descending': + elif myarg == 'descending': descending = True i += 1 - elif myarg == u'orderby': - sortorder = sys.argv[i+1].lower().replace(u'_', u'') + elif myarg == 'orderby': + sortorder = sys.argv[i+1].lower().replace('_', '') if sortorder not in PRINTJOB_ASCENDINGORDER_MAP: - systemErrorExit(2, 'orderby must be one of %s; got %s' % (u', '.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder)) + systemErrorExit(2, 'orderby must be one of %s; got %s' % (', '.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder)) sortorder = PRINTJOB_ASCENDINGORDER_MAP[sortorder] i += 2 - elif myarg in [u'printer', u'printerid']: + elif myarg in ['printer', 'printerid']: printerid = sys.argv[i+1] i += 2 - elif myarg in [u'owner', u'user']: + elif myarg in ['owner', 'user']: owner = sys.argv[i+1] i += 2 - elif myarg == u'limit': + elif myarg == 'limit': jobLimit = getInteger(sys.argv[i+1], myarg, minVal=0) i += 2 else: @@ -2845,12 +2855,12 @@ def doPrintPrintJobs(): if sortorder and descending: sortorder = PRINTJOB_DESCENDINGORDER_MAP[sortorder] if printerid: - result = callGAPI(cp.printers(), u'get', + result = callGAPI(cp.printers(), 'get', printerid=printerid) checkCloudPrintResult(result) - if ((not sortorder) or (sortorder == u'CREATE_TIME_DESC')) and (older_or_newer == u'newer'): + if ((not sortorder) or (sortorder == 'CREATE_TIME_DESC')) and (older_or_newer == 'newer'): timeExit = True - elif (sortorder == u'CREATE_TIME') and (older_or_newer == u'older'): + elif (sortorder == 'CREATE_TIME') and (older_or_newer == 'older'): timeExit = True else: timeExit = False @@ -2862,44 +2872,44 @@ def doPrintPrintJobs(): limit = min(PRINTJOBS_DEFAULT_MAX_RESULTS, jobLimit-jobCount) if limit == 0: break - result = callGAPI(cp.jobs(), u'list', + result = callGAPI(cp.jobs(), 'list', printerid=printerid, q=query, status=status, sortorder=sortorder, owner=owner, offset=offset, limit=limit) checkCloudPrintResult(result) - newJobs = result[u'range'][u'jobsCount'] - totalJobs = int(result[u'range'][u'jobsTotal']) + newJobs = result['range']['jobsCount'] + totalJobs = int(result['range']['jobsTotal']) if GC_Values[GC_DEBUG_LEVEL] > 0: - sys.stderr.write(u'Debug: jobCount: {0}, jobLimit: {1}, jobsCount: {2}, jobsTotal: {3}\n'.format(jobCount, jobLimit, newJobs, totalJobs)) + sys.stderr.write('Debug: jobCount: {0}, jobLimit: {1}, jobsCount: {2}, jobsTotal: {3}\n'.format(jobCount, jobLimit, newJobs, totalJobs)) if newJobs == 0: break jobCount += newJobs offset += newJobs - for job in result[u'jobs']: - createTime = int(job[u'createTime'])/1000 + for job in result['jobs']: + createTime = int(job['createTime'])/1000 if older_or_newer: - if older_or_newer == u'older' and createTime > age: + if older_or_newer == 'older' and createTime > age: if timeExit: jobCount = totalJobs break continue - elif older_or_newer == u'newer' and createTime < age: + elif older_or_newer == 'newer' and createTime < age: if timeExit: jobCount = totalJobs break continue - updateTime = int(job[u'updateTime'])/1000 - job[u'createTime'] = datetime.datetime.fromtimestamp(createTime).strftime(u'%Y-%m-%d %H:%M:%S') - job[u'updateTime'] = datetime.datetime.fromtimestamp(updateTime).strftime(u'%Y-%m-%d %H:%M:%S') - job[u'tags'] = u' '.join(job[u'tags']) + updateTime = int(job['updateTime'])/1000 + job['createTime'] = datetime.datetime.fromtimestamp(createTime).strftime('%Y-%m-%d %H:%M:%S') + job['updateTime'] = datetime.datetime.fromtimestamp(updateTime).strftime('%Y-%m-%d %H:%M:%S') + job['tags'] = ' '.join(job['tags']) addRowTitlesToCSVfile(flatten_json(job), csvRows, titles) if jobCount >= totalJobs: break - writeCSVfile(csvRows, titles, u'Print Jobs', todrive) + writeCSVfile(csvRows, titles, 'Print Jobs', todrive) def doPrintPrinters(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') todrive = False - titles = [u'id',] + titles = ['id',] csvRows = [] queries = [None] printer_type = None @@ -2907,37 +2917,37 @@ def doPrintPrinters(): extra_fields = None i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg in [u'query', u'queries']: + myarg = sys.argv[i].lower().replace('_', '') + if myarg in ['query', 'queries']: queries = getQueries(myarg, sys.argv[i+1]) i += 2 - elif myarg == u'type': + elif myarg == 'type': printer_type = sys.argv[i+1] i += 2 - elif myarg == u'status': + elif myarg == 'status': connection_status = sys.argv[i+1] i += 2 - elif myarg == u'extrafields': + elif myarg == 'extrafields': extra_fields = sys.argv[i+1] i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print printers"' % sys.argv[i]) for query in queries: - printers = callGAPI(cp.printers(), u'list', q=query, type=printer_type, connection_status=connection_status, extra_fields=extra_fields) + printers = callGAPI(cp.printers(), 'list', q=query, type=printer_type, connection_status=connection_status, extra_fields=extra_fields) checkCloudPrintResult(printers) - for printer in printers[u'printers']: - createTime = int(printer[u'createTime'])/1000 - accessTime = int(printer[u'accessTime'])/1000 - updateTime = int(printer[u'updateTime'])/1000 - printer[u'createTime'] = datetime.datetime.fromtimestamp(createTime).strftime(u'%Y-%m-%d %H:%M:%S') - printer[u'accessTime'] = datetime.datetime.fromtimestamp(accessTime).strftime(u'%Y-%m-%d %H:%M:%S') - printer[u'updateTime'] = datetime.datetime.fromtimestamp(updateTime).strftime(u'%Y-%m-%d %H:%M:%S') - printer[u'tags'] = u' '.join(printer[u'tags']) + for printer in printers['printers']: + createTime = int(printer['createTime'])/1000 + accessTime = int(printer['accessTime'])/1000 + updateTime = int(printer['updateTime'])/1000 + printer['createTime'] = datetime.datetime.fromtimestamp(createTime).strftime('%Y-%m-%d %H:%M:%S') + printer['accessTime'] = datetime.datetime.fromtimestamp(accessTime).strftime('%Y-%m-%d %H:%M:%S') + printer['updateTime'] = datetime.datetime.fromtimestamp(updateTime).strftime('%Y-%m-%d %H:%M:%S') + printer['tags'] = ' '.join(printer['tags']) addRowTitlesToCSVfile(flatten_json(printer), csvRows, titles) - writeCSVfile(csvRows, titles, u'Printers', todrive) + writeCSVfile(csvRows, titles, 'Printers', todrive) def changeCalendarAttendees(users): do_it = True @@ -2946,19 +2956,19 @@ def changeCalendarAttendees(users): start_date = end_date = None while len(sys.argv) > i: myarg = sys.argv[i].lower() - if myarg == u'csv': + if myarg == 'csv': csv_file = sys.argv[i+1] i += 2 - elif myarg == u'dryrun': + elif myarg == 'dryrun': do_it = False i += 1 - elif myarg == u'start': + elif myarg == 'start': start_date = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg == u'end': + elif myarg == 'end': end_date = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg == u'allevents': + elif myarg == 'allevents': allevents = True i += 1 else: @@ -2970,38 +2980,38 @@ def changeCalendarAttendees(users): attendee_map[row[0].lower()] = row[1].lower() closeFile(f) for user in users: - sys.stdout.write(u'Checking user %s\n' % user) + sys.stdout.write('Checking user %s\n' % user) user, cal = buildCalendarGAPIObject(user) if not cal: continue page_token = None while True: - events_page = callGAPI(cal.events(), u'list', calendarId=user, pageToken=page_token, timeMin=start_date, timeMax=end_date, showDeleted=False, showHiddenInvitations=False) - print u'Got %s items' % len(events_page.get(u'items', [])) - for event in events_page.get(u'items', []): - if event[u'status'] == u'cancelled': + events_page = callGAPI(cal.events(), 'list', calendarId=user, pageToken=page_token, timeMin=start_date, timeMax=end_date, showDeleted=False, showHiddenInvitations=False) + print('Got %s items' % len(events_page.get('items', []))) + for event in events_page.get('items', []): + if event['status'] == 'cancelled': #print u' skipping cancelled event' continue try: - event_summary = utils.convertUTF8(event[u'summary']) + event_summary = utils.convertUTF8(event['summary']) except (KeyError, UnicodeEncodeError, UnicodeDecodeError): - event_summary = event[u'id'] + event_summary = event['id'] try: - if not allevents and event[u'organizer'][u'email'].lower() != user: + if not allevents and event['organizer']['email'].lower() != user: #print u' skipping not-my-event %s' % event_summary continue except KeyError: pass # no email for organizer needs_update = False try: - for attendee in event[u'attendees']: + for attendee in event['attendees']: try: - if attendee[u'email'].lower() in attendee_map: - old_email = attendee[u'email'].lower() - new_email = attendee_map[attendee[u'email'].lower()] - print u' SWITCHING attendee %s to %s for %s' % (old_email, new_email, event_summary) - event[u'attendees'].remove(attendee) - event[u'attendees'].append({u'email': new_email}) + if attendee['email'].lower() in attendee_map: + old_email = attendee['email'].lower() + new_email = attendee_map[attendee['email'].lower()] + print(' SWITCHING attendee %s to %s for %s' % (old_email, new_email, event_summary)) + event['attendees'].remove(attendee) + event['attendees'].append({'email': new_email}) needs_update = True except KeyError: # no email for that attendee pass @@ -3009,16 +3019,16 @@ def changeCalendarAttendees(users): continue # no attendees if needs_update: body = {} - body[u'attendees'] = event[u'attendees'] - print u'UPDATING %s' % event_summary + body['attendees'] = event['attendees'] + print('UPDATING %s' % event_summary) if do_it: - callGAPI(cal.events(), u'patch', calendarId=user, eventId=event[u'id'], sendNotifications=False, body=body) + callGAPI(cal.events(), 'patch', calendarId=user, eventId=event['id'], sendNotifications=False, body=body) else: - print u' not pulling the trigger.' + print(' not pulling the trigger.') #else: # print u' no update needed for %s' % event_summary try: - page_token = events_page[u'nextPageToken'] + page_token = events_page['nextPageToken'] except KeyError: break @@ -3028,7 +3038,7 @@ def deleteCalendar(users): user, cal = buildCalendarGAPIObject(user) if not cal: continue - callGAPI(cal.calendarList(), u'delete', soft_errors=True, calendarId=calendarId) + callGAPI(cal.calendarList(), 'delete', soft_errors=True, calendarId=calendarId) CALENDAR_REMINDER_MAX_MINUTES = 40320 @@ -3041,48 +3051,48 @@ CALENDAR_EVENT_MAX_COLOR_INDEX = 11 def getCalendarAttributes(i, body, function): colorRgbFormat = False while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'selected': - body[u'selected'] = getBoolean(sys.argv[i+1], myarg) + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'selected': + body['selected'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'hidden': - body[u'hidden'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'hidden': + body['hidden'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'summary': - body[u'summaryOverride'] = sys.argv[i+1] + elif myarg == 'summary': + body['summaryOverride'] = sys.argv[i+1] i += 2 - elif myarg == u'colorindex': - body[u'colorId'] = getInteger(sys.argv[i+1], myarg, minVal=CALENDAR_MIN_COLOR_INDEX, maxVal=CALENDAR_MAX_COLOR_INDEX) + elif myarg == 'colorindex': + body['colorId'] = getInteger(sys.argv[i+1], myarg, minVal=CALENDAR_MIN_COLOR_INDEX, maxVal=CALENDAR_MAX_COLOR_INDEX) i += 2 - elif myarg == u'backgroundcolor': - body[u'backgroundColor'] = getColor(sys.argv[i+1]) + elif myarg == 'backgroundcolor': + body['backgroundColor'] = getColor(sys.argv[i+1]) colorRgbFormat = True i += 2 - elif myarg == u'foregroundcolor': - body[u'foregroundColor'] = getColor(sys.argv[i+1]) + elif myarg == 'foregroundcolor': + body['foregroundColor'] = getColor(sys.argv[i+1]) colorRgbFormat = True i += 2 - elif myarg == u'reminder': - body.setdefault(u'defaultReminders', []) + elif myarg == 'reminder': + body.setdefault('defaultReminders', []) method = sys.argv[i+1].lower() if method not in CLEAR_NONE_ARGUMENT: if method not in CALENDAR_REMINDER_METHODS: - systemErrorExit(2, 'Method must be one of %s; got %s' % (u', '.join(CALENDAR_REMINDER_METHODS+CLEAR_NONE_ARGUMENT), method)) + systemErrorExit(2, 'Method must be one of %s; got %s' % (', '.join(CALENDAR_REMINDER_METHODS+CLEAR_NONE_ARGUMENT), method)) minutes = getInteger(sys.argv[i+2], myarg, minVal=0, maxVal=CALENDAR_REMINDER_MAX_MINUTES) - body[u'defaultReminders'].append({u'method': method, u'minutes': minutes}) + body['defaultReminders'].append({'method': method, 'minutes': minutes}) i += 3 else: i += 2 - elif myarg == u'notification': - body.setdefault(u'notificationSettings', {u'notifications': []}) + elif myarg == 'notification': + body.setdefault('notificationSettings', {'notifications': []}) method = sys.argv[i+1].lower() if method not in CLEAR_NONE_ARGUMENT: if method not in CALENDAR_NOTIFICATION_METHODS: - systemErrorExit(2, 'Method must be one of %s; got %s' % (u', '.join(CALENDAR_NOTIFICATION_METHODS+CLEAR_NONE_ARGUMENT), method)) + systemErrorExit(2, 'Method must be one of %s; got %s' % (', '.join(CALENDAR_NOTIFICATION_METHODS+CLEAR_NONE_ARGUMENT), method)) eventType = sys.argv[i+2].lower() if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP: - systemErrorExit(2, 'Event must be one of %s; got %s' % (u', '.join(CALENDAR_NOTIFICATION_TYPES_MAP), eventType)) - body[u'notificationSettings'][u'notifications'].append({u'method': method, u'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]}) + systemErrorExit(2, 'Event must be one of %s; got %s' % (', '.join(CALENDAR_NOTIFICATION_TYPES_MAP), eventType)) + body['notificationSettings']['notifications'].append({'method': method, 'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]}) i += 3 else: i += 2 @@ -3092,8 +3102,8 @@ def getCalendarAttributes(i, body, function): def addCalendar(users): calendarId = normalizeCalendarId(sys.argv[5]) - body = {u'id': calendarId, u'selected': True, u'hidden': False} - colorRgbFormat = getCalendarAttributes(6, body, u'add') + body = {'id': calendarId, 'selected': True, 'hidden': False} + colorRgbFormat = getCalendarAttributes(6, body, 'add') i = 0 count = len(users) for user in users: @@ -3101,13 +3111,13 @@ def addCalendar(users): user, cal = buildCalendarGAPIObject(user) if not cal: continue - print u"Subscribing %s to %s calendar (%s/%s)" % (user, calendarId, i, count) - callGAPI(cal.calendarList(), u'insert', soft_errors=True, body=body, colorRgbFormat=colorRgbFormat) + print("Subscribing %s to %s calendar (%s/%s)" % (user, calendarId, i, count)) + callGAPI(cal.calendarList(), 'insert', soft_errors=True, body=body, colorRgbFormat=colorRgbFormat) def updateCalendar(users): calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True) body = {} - colorRgbFormat = getCalendarAttributes(6, body, u'update') + colorRgbFormat = getCalendarAttributes(6, body, 'update') i = 0 count = len(users) for user in users: @@ -3115,62 +3125,62 @@ def updateCalendar(users): user, cal = buildCalendarGAPIObject(user) if not cal: continue - print u"Updating %s's subscription to calendar %s (%s/%s)" % (user, calendarId, i, count) - calId = calendarId if calendarId != u'primary' else user - callGAPI(cal.calendarList(), u'patch', soft_errors=True, calendarId=calId, body=body, colorRgbFormat=colorRgbFormat) + print("Updating %s's subscription to calendar %s (%s/%s)" % (user, calendarId, i, count)) + calId = calendarId if calendarId != 'primary' else user + callGAPI(cal.calendarList(), 'patch', soft_errors=True, calendarId=calId, body=body, colorRgbFormat=colorRgbFormat) def doPrinterShowACL(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') show_printer = sys.argv[2] - printer_info = callGAPI(cp.printers(), u'get', printerid=show_printer) + printer_info = callGAPI(cp.printers(), 'get', printerid=show_printer) checkCloudPrintResult(printer_info) - for acl in printer_info[u'printers'][0][u'access']: - if u'key' in acl: - acl[u'accessURL'] = u'https://www.google.com/cloudprint/addpublicprinter.html?printerid=%s&key=%s' % (show_printer, acl[u'key']) + for acl in printer_info['printers'][0]['access']: + if 'key' in acl: + acl['accessURL'] = 'https://www.google.com/cloudprint/addpublicprinter.html?printerid=%s&key=%s' % (show_printer, acl['key']) print_json(None, acl) - print + print() def doPrinterAddACL(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printer = sys.argv[2] role = sys.argv[4].upper() scope = sys.argv[5] - notify = True if len(sys.argv) > 6 and sys.argv[6].lower() == u'notify' else False + notify = True if len(sys.argv) > 6 and sys.argv[6].lower() == 'notify' else False public = None skip_notification = True - if scope.lower() == u'public': + if scope.lower() == 'public': public = True scope = None role = None skip_notification = None - elif scope.find(u'@') == -1: - scope = u'/hd/domain/%s' % scope + elif scope.find('@') == -1: + scope = '/hd/domain/%s' % scope else: skip_notification = not notify - result = callGAPI(cp.printers(), u'share', printerid=printer, role=role, scope=scope, public=public, skip_notification=skip_notification) + result = callGAPI(cp.printers(), 'share', printerid=printer, role=role, scope=scope, public=public, skip_notification=skip_notification) checkCloudPrintResult(result) who = scope if who is None: - who = u'public' - role = u'user' - print u'Added %s %s' % (role, who) + who = 'public' + role = 'user' + print('Added %s %s' % (role, who)) def doPrinterDelACL(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printer = sys.argv[2] scope = sys.argv[4] public = None - if scope.lower() == u'public': + if scope.lower() == 'public': public = True scope = None - elif scope.find(u'@') == -1: - scope = u'/hd/domain/%s' % scope - result = callGAPI(cp.printers(), u'unshare', printerid=printer, scope=scope, public=public) + elif scope.find('@') == -1: + scope = '/hd/domain/%s' % scope + result = callGAPI(cp.printers(), 'unshare', printerid=printer, scope=scope, public=public) checkCloudPrintResult(result) who = scope if who is None: - who = u'public' - print u'Removed %s' % who + who = 'public' + print('Removed %s' % who) def encode_multipart(fields, files, boundary=None): def escape_quote(s): @@ -3182,21 +3192,21 @@ def encode_multipart(fields, files, boundary=None): if boundary is None: boundary = ''.join(random.choice(string.digits + string.ascii_letters) for i in range(30)) lines = [] - for name, value in fields.items(): - if name == u'tags': + for name, value in list(fields.items()): + if name == 'tags': for tag in value: lines.extend(getFormDataLine('tag', tag, boundary)) else: lines.extend(getFormDataLine(name, value, boundary)) - for name, value in files.items(): - filename = value[u'filename'] - mimetype = value[u'mimetype'] + for name, value in list(files.items()): + filename = value['filename'] + mimetype = value['mimetype'] lines.extend(( '--{0}'.format(boundary), 'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(escape_quote(name), escape_quote(filename)), 'Content-Type: {0}'.format(mimetype), '', - value[u'content'], + value['content'], )) lines.extend(( '--{0}--'.format(boundary), @@ -3210,9 +3220,9 @@ def encode_multipart(fields, files, boundary=None): return (body, headers) def doPrintJobFetch(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printerid = sys.argv[2] - if printerid == u'any': + if printerid == 'any': printerid = None owner = None status = None @@ -3225,53 +3235,53 @@ def doPrintJobFetch(): targetFolder = os.getcwd() i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg in [u'olderthan', u'newerthan']: - if myarg == u'olderthan': - older_or_newer = u'older' + myarg = sys.argv[i].lower().replace('_', '') + if myarg in ['olderthan', 'newerthan']: + if myarg == 'olderthan': + older_or_newer = 'older' else: - older_or_newer = u'newer' + older_or_newer = 'newer' age_number = sys.argv[i+1][:-1] if not age_number.isdigit(): systemErrorExit(2, 'expected a number; got %s' % age_number) age_unit = sys.argv[i+1][-1].lower() - if age_unit == u'm': + if age_unit == 'm': age = int(time.time()) - (int(age_number) * 60) - elif age_unit == u'h': + elif age_unit == 'h': age = int(time.time()) - (int(age_number) * 60 * 60) - elif age_unit == u'd': + elif age_unit == 'd': age = int(time.time()) - (int(age_number) * 60 * 60 * 24) else: systemErrorExit(2, 'expected m (minutes), h (hours) or d (days); got %s' % age_unit) i += 2 - elif myarg == u'query': + elif myarg == 'query': query = sys.argv[i+1] i += 2 - elif myarg == u'status': + elif myarg == 'status': status = sys.argv[i+1] i += 2 - elif myarg == u'ascending': + elif myarg == 'ascending': descending = False i += 1 - elif myarg == u'descending': + elif myarg == 'descending': descending = True i += 1 - elif myarg == u'orderby': - sortorder = sys.argv[i+1].lower().replace(u'_', u'') + elif myarg == 'orderby': + sortorder = sys.argv[i+1].lower().replace('_', '') if sortorder not in PRINTJOB_ASCENDINGORDER_MAP: - systemErrorExit(2, 'orderby must be one of %s; got %s' % (u', '.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder)) + systemErrorExit(2, 'orderby must be one of %s; got %s' % (', '.join(PRINTJOB_ASCENDINGORDER_MAP), sortorder)) sortorder = PRINTJOB_ASCENDINGORDER_MAP[sortorder] i += 2 - elif myarg in [u'owner', u'user']: + elif myarg in ['owner', 'user']: owner = sys.argv[i+1] i += 2 - elif myarg == u'limit': + elif myarg == 'limit': jobLimit = getInteger(sys.argv[i+1], myarg, minVal=0) i += 2 - elif myarg == u'drivedir': + elif myarg == 'drivedir': targetFolder = GC_Values[GC_DRIVE_DIR] i += 1 - elif myarg == u'targetfolder': + elif myarg == 'targetfolder': targetFolder = os.path.expanduser(sys.argv[i+1]) if not os.path.isdir(targetFolder): os.makedirs(targetFolder) @@ -3281,14 +3291,14 @@ def doPrintJobFetch(): if sortorder and descending: sortorder = PRINTJOB_DESCENDINGORDER_MAP[sortorder] if printerid: - result = callGAPI(cp.printers(), u'get', + result = callGAPI(cp.printers(), 'get', printerid=printerid) checkCloudPrintResult(result) - valid_chars = u'-_.() abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - ssd = u'{"state": {"type": "DONE"}}' - if ((not sortorder) or (sortorder == u'CREATE_TIME_DESC')) and (older_or_newer == u'newer'): + valid_chars = '-_.() abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + ssd = '{"state": {"type": "DONE"}}' + if ((not sortorder) or (sortorder == 'CREATE_TIME_DESC')) and (older_or_newer == 'newer'): timeExit = True - elif (sortorder == u'CREATE_TIME') and (older_or_newer == u'older'): + elif (sortorder == 'CREATE_TIME') and (older_or_newer == 'older'): timeExit = True else: timeExit = False @@ -3300,86 +3310,86 @@ def doPrintJobFetch(): limit = min(PRINTJOBS_DEFAULT_MAX_RESULTS, jobLimit-jobCount) if limit == 0: break - result = callGAPI(cp.jobs(), u'list', + result = callGAPI(cp.jobs(), 'list', printerid=printerid, q=query, status=status, sortorder=sortorder, owner=owner, offset=offset, limit=limit) checkCloudPrintResult(result) - newJobs = result[u'range'][u'jobsCount'] - totalJobs = int(result[u'range'][u'jobsTotal']) + newJobs = result['range']['jobsCount'] + totalJobs = int(result['range']['jobsTotal']) if newJobs == 0: break jobCount += newJobs offset += newJobs - for job in result[u'jobs']: - createTime = int(job[u'createTime'])/1000 + for job in result['jobs']: + createTime = int(job['createTime'])/1000 if older_or_newer: - if older_or_newer == u'older' and createTime > age: + if older_or_newer == 'older' and createTime > age: if timeExit: jobCount = totalJobs break continue - elif older_or_newer == u'newer' and createTime < age: + elif older_or_newer == 'newer' and createTime < age: if timeExit: jobCount = totalJobs break continue - fileUrl = job[u'fileUrl'] - jobid = job[u'id'] - fileName = os.path.join(targetFolder, u'{0}-{1}'.format(u''.join(c if c in valid_chars else u'_' for c in job[u'title']), jobid)) + fileUrl = job['fileUrl'] + jobid = job['id'] + fileName = os.path.join(targetFolder, '{0}-{1}'.format(''.join(c if c in valid_chars else '_' for c in job['title']), jobid)) _, content = cp._http.request(uri=fileUrl, method='GET') if writeFile(fileName, content, continueOnError=True): # ticket = callGAPI(cp.jobs(), u'getticket', jobid=jobid, use_cjt=True) - result = callGAPI(cp.jobs(), u'update', jobid=jobid, semantic_state_diff=ssd) + result = callGAPI(cp.jobs(), 'update', jobid=jobid, semantic_state_diff=ssd) checkCloudPrintResult(result) - print u'Printed job %s to %s' % (jobid, fileName) + print('Printed job %s to %s' % (jobid, fileName)) if jobCount >= totalJobs: break if jobCount == 0: - print u'No print jobs.' + print('No print jobs.') def doDelPrinter(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printerid = sys.argv[3] - result = callGAPI(cp.printers(), u'delete', printerid=printerid) + result = callGAPI(cp.printers(), 'delete', printerid=printerid) checkCloudPrintResult(result) def doGetPrinterInfo(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printerid = sys.argv[3] everything = False i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'everything': + if myarg == 'everything': everything = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam info printer"' % sys.argv[i]) - result = callGAPI(cp.printers(), u'get', printerid=printerid) + result = callGAPI(cp.printers(), 'get', printerid=printerid) checkCloudPrintResult(result) - printer_info = result[u'printers'][0] - createTime = int(printer_info[u'createTime'])/1000 - accessTime = int(printer_info[u'accessTime'])/1000 - updateTime = int(printer_info[u'updateTime'])/1000 - printer_info[u'createTime'] = datetime.datetime.fromtimestamp(createTime).strftime(u'%Y-%m-%d %H:%M:%S') - printer_info[u'accessTime'] = datetime.datetime.fromtimestamp(accessTime).strftime(u'%Y-%m-%d %H:%M:%S') - printer_info[u'updateTime'] = datetime.datetime.fromtimestamp(updateTime).strftime(u'%Y-%m-%d %H:%M:%S') - printer_info[u'tags'] = u' '.join(printer_info[u'tags']) + printer_info = result['printers'][0] + createTime = int(printer_info['createTime'])/1000 + accessTime = int(printer_info['accessTime'])/1000 + updateTime = int(printer_info['updateTime'])/1000 + printer_info['createTime'] = datetime.datetime.fromtimestamp(createTime).strftime('%Y-%m-%d %H:%M:%S') + printer_info['accessTime'] = datetime.datetime.fromtimestamp(accessTime).strftime('%Y-%m-%d %H:%M:%S') + printer_info['updateTime'] = datetime.datetime.fromtimestamp(updateTime).strftime('%Y-%m-%d %H:%M:%S') + printer_info['tags'] = ' '.join(printer_info['tags']) if not everything: - del printer_info[u'capabilities'] - del printer_info[u'access'] + del printer_info['capabilities'] + del printer_info['access'] print_json(None, printer_info) def doUpdatePrinter(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printerid = sys.argv[3] kwargs = {} i = 4 - update_items = [u'isTosAccepted', u'gcpVersion', u'setupUrl', - u'quotaEnabled', u'id', u'supportUrl', u'firmware', - u'currentQuota', u'type', u'public', u'status', u'description', - u'defaultDisplayName', u'proxy', u'dailyQuota', u'manufacturer', - u'displayName', u'name', u'uuid', u'updateUrl', u'ownerId', u'model'] + update_items = ['isTosAccepted', 'gcpVersion', 'setupUrl', + 'quotaEnabled', 'id', 'supportUrl', 'firmware', + 'currentQuota', 'type', 'public', 'status', 'description', + 'defaultDisplayName', 'proxy', 'dailyQuota', 'manufacturer', + 'displayName', 'name', 'uuid', 'updateUrl', 'ownerId', 'model'] while i < len(sys.argv): arg_in_item = False for item in update_items: @@ -3390,25 +3400,25 @@ def doUpdatePrinter(): break if not arg_in_item: systemErrorExit(2, '%s is not a valid argument for "gam update printer"' % sys.argv[i]) - result = callGAPI(cp.printers(), u'update', printerid=printerid, **kwargs) + result = callGAPI(cp.printers(), 'update', printerid=printerid, **kwargs) checkCloudPrintResult(result) - print u'Updated printer %s' % printerid + print('Updated printer %s' % printerid) def doPrinterRegister(): - cp = buildGAPIObject(u'cloudprint') - form_fields = {u'name': u'GAM', - u'proxy': u'GAM', - u'uuid': _getValueFromOAuth(u'sub'), - u'manufacturer': gam_author, - u'model': u'cp1', - u'gcp_version': u'2.0', - u'setup_url': GAM_URL, - u'support_url': u'https://groups.google.com/forum/#!forum/google-apps-manager', - u'update_url': GAM_RELEASES, - u'firmware': gam_version, - u'semantic_state': {"version": "1.0", "printer": {"state": "IDLE",}}, - u'use_cdd': True, - u'capabilities': {"version": "1.0", + cp = buildGAPIObject('cloudprint') + form_fields = {'name': 'GAM', + 'proxy': 'GAM', + 'uuid': _getValueFromOAuth('sub'), + 'manufacturer': gam_author, + 'model': 'cp1', + 'gcp_version': '2.0', + 'setup_url': GAM_URL, + 'support_url': 'https://groups.google.com/forum/#!forum/google-apps-manager', + 'update_url': GAM_RELEASES, + 'firmware': gam_version, + 'semantic_state': {"version": "1.0", "printer": {"state": "IDLE",}}, + 'use_cdd': True, + 'capabilities': {"version": "1.0", "printer": {"supported_content_type": [{"content_type": "application/pdf", "min_version": "1.5"}, {"content_type": "image/jpeg"}, {"content_type": "text/plain"} @@ -3421,83 +3431,83 @@ def doPrinterRegister(): }, }, }, - u'tags': [u'GAM', GAM_URL], + 'tags': ['GAM', GAM_URL], } body, headers = encode_multipart(form_fields, {}) #Get the printer first to make sure our OAuth access token is fresh - callGAPI(cp.printers(), u'list') + callGAPI(cp.printers(), 'list') _, result = cp._http.request(uri='https://www.google.com/cloudprint/register', method='POST', body=body, headers=headers) result = json.loads(result) checkCloudPrintResult(result) - print u'Created printer %s' % result[u'printers'][0][u'id'] + print('Created printer %s' % result['printers'][0]['id']) def doPrintJobResubmit(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') jobid = sys.argv[2] printerid = sys.argv[4] - ssd = u'{"state": {"type": "HELD"}}' - result = callGAPI(cp.jobs(), u'update', jobid=jobid, semantic_state_diff=ssd) + ssd = '{"state": {"type": "HELD"}}' + result = callGAPI(cp.jobs(), 'update', jobid=jobid, semantic_state_diff=ssd) checkCloudPrintResult(result) - ticket = callGAPI(cp.jobs(), u'getticket', jobid=jobid, use_cjt=True) - result = callGAPI(cp.jobs(), u'resubmit', printerid=printerid, jobid=jobid, ticket=ticket) + ticket = callGAPI(cp.jobs(), 'getticket', jobid=jobid, use_cjt=True) + result = callGAPI(cp.jobs(), 'resubmit', printerid=printerid, jobid=jobid, ticket=ticket) checkCloudPrintResult(result) - print u'Success resubmitting %s as job %s to printer %s' % (jobid, result[u'job'][u'id'], printerid) + print('Success resubmitting %s as job %s to printer %s' % (jobid, result['job']['id'], printerid)) def doPrintJobSubmit(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') printer = sys.argv[2] content = sys.argv[4] - form_fields = {u'printerid': printer, - u'title': content, - u'ticket': u'{"version": "1.0"}', - u'tags': [u'GAM', GAM_URL]} + form_fields = {'printerid': printer, + 'title': content, + 'ticket': '{"version": "1.0"}', + 'tags': ['GAM', GAM_URL]} i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'tag': - form_fields[u'tags'].append(sys.argv[i+1]) + if myarg == 'tag': + form_fields['tags'].append(sys.argv[i+1]) i += 2 - elif myarg in [u'name', u'title']: - form_fields[u'title'] = sys.argv[i+1] + elif myarg in ['name', 'title']: + form_fields['title'] = sys.argv[i+1] i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam printer ... print"' % sys.argv[i]) form_files = {} - if content[:4] == u'http': - form_fields[u'content'] = content - form_fields[u'contentType'] = u'url' + if content[:4] == 'http': + form_fields['content'] = content + form_fields['contentType'] = 'url' else: filepath = content content = os.path.basename(content) mimetype = mimetypes.guess_type(filepath)[0] if mimetype is None: - mimetype = u'application/octet-stream' + mimetype = 'application/octet-stream' filecontent = readFile(filepath) - form_files[u'content'] = {u'filename': content, u'content': filecontent, u'mimetype': mimetype} + form_files['content'] = {'filename': content, 'content': filecontent, 'mimetype': mimetype} #result = callGAPI(cp.printers(), u'submit', body=body) body, headers = encode_multipart(form_fields, form_files) #Get the printer first to make sure our OAuth access token is fresh - callGAPI(cp.printers(), u'get', printerid=printer) + callGAPI(cp.printers(), 'get', printerid=printer) _, result = cp._http.request(uri='https://www.google.com/cloudprint/submit', method='POST', body=body, headers=headers) checkCloudPrintResult(result) if isinstance(result, str): result = json.loads(result) - print u'Submitted print job %s' % result[u'job'][u'id'] + print('Submitted print job %s' % result['job']['id']) def doDeletePrintJob(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') job = sys.argv[2] - result = callGAPI(cp.jobs(), u'delete', jobid=job) + result = callGAPI(cp.jobs(), 'delete', jobid=job) checkCloudPrintResult(result) - print u'Print Job %s deleted' % job + print('Print Job %s deleted' % job) def doCancelPrintJob(): - cp = buildGAPIObject(u'cloudprint') + cp = buildGAPIObject('cloudprint') job = sys.argv[2] - ssd = u'{"state": {"type": "ABORTED", "user_action_cause": {"action_code": "CANCELLED"}}}' - result = callGAPI(cp.jobs(), u'update', jobid=job, semantic_state_diff=ssd) + ssd = '{"state": {"type": "ABORTED", "user_action_cause": {"action_code": "CANCELLED"}}}' + result = callGAPI(cp.jobs(), 'update', jobid=job, semantic_state_diff=ssd) checkCloudPrintResult(result) - print u'Print Job %s cancelled' % job + print('Print Job %s cancelled' % job) def checkCloudPrintResult(result): if isinstance(result, str): @@ -3505,94 +3515,94 @@ def checkCloudPrintResult(result): result = json.loads(result) except ValueError: systemErrorExit(3, 'unexpected response: %s' % result) - if not result[u'success']: - systemErrorExit(result[u'errorCode'], '%s: %s' % (result[u'errorCode'], result[u'message'])) + if not result['success']: + systemErrorExit(result['errorCode'], '%s: %s' % (result['errorCode'], result['message'])) def formatACLScope(rule): - if rule[u'scope'][u'type'] != u'default': - return u'(Scope: {0}:{1})'.format(rule[u'scope'][u'type'], rule[u'scope'][u'value']) - return u'(Scope: {0})'.format(rule[u'scope'][u'type']) + if rule['scope']['type'] != 'default': + return '(Scope: {0}:{1})'.format(rule['scope']['type'], rule['scope']['value']) + return '(Scope: {0})'.format(rule['scope']['type']) def formatACLRule(rule): - if rule[u'scope'][u'type'] != u'default': - return u'(Scope: {0}:{1}, Role: {2})'.format(rule[u'scope'][u'type'], rule[u'scope'][u'value'], rule[u'role']) - return u'(Scope: {0}, Role: {1})'.format(rule[u'scope'][u'type'], rule[u'role']) + if rule['scope']['type'] != 'default': + return '(Scope: {0}:{1}, Role: {2})'.format(rule['scope']['type'], rule['scope']['value'], rule['role']) + return '(Scope: {0}, Role: {1})'.format(rule['scope']['type'], rule['role']) def doCalendarShowACL(): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) if not cal: return - acls = callGAPIpages(cal.acl(), u'list', u'items', calendarId=calendarId, fields=u'nextPageToken,items(role,scope)') + acls = callGAPIpages(cal.acl(), 'list', 'items', calendarId=calendarId, fields='nextPageToken,items(role,scope)') i = 0 count = len(acls) for rule in acls: i += 1 - print u'Calendar: {0}, ACL: {1}{2}'.format(calendarId, formatACLRule(rule), currentCount(i, count)) + print('Calendar: {0}, ACL: {1}{2}'.format(calendarId, formatACLRule(rule), currentCount(i, count))) def _getCalendarACLScope(i, body): - body[u'scope'] = {} + body['scope'] = {} myarg = sys.argv[i].lower() - body[u'scope'][u'type'] = myarg + body['scope']['type'] = myarg i += 1 - if myarg in [u'user', u'group']: - body[u'scope'][u'value'] = normalizeEmailAddressOrUID(sys.argv[i], noUid=True) + if myarg in ['user', 'group']: + body['scope']['value'] = normalizeEmailAddressOrUID(sys.argv[i], noUid=True) i += 1 - elif myarg == u'domain': - if i < len(sys.argv) and sys.argv[i].lower().replace(u'_', u'') != u'sendnotifications': - body[u'scope'][u'value'] = sys.argv[i].lower() + elif myarg == 'domain': + if i < len(sys.argv) and sys.argv[i].lower().replace('_', '') != 'sendnotifications': + body['scope']['value'] = sys.argv[i].lower() i += 1 else: - body[u'scope'][u'value'] = GC_Values[GC_DOMAIN] - elif myarg != u'default': - body[u'scope'][u'type'] = u'user' - body[u'scope'][u'value'] = normalizeEmailAddressOrUID(myarg, noUid=True) + body['scope']['value'] = GC_Values[GC_DOMAIN] + elif myarg != 'default': + body['scope']['type'] = 'user' + body['scope']['value'] = normalizeEmailAddressOrUID(myarg, noUid=True) return i CALENDAR_ACL_ROLES_MAP = { - u'editor': u'writer', - u'freebusy': u'freeBusyReader', - u'freebusyreader': u'freeBusyReader', - u'owner': u'owner', - u'read': u'reader', - u'reader': u'reader', - u'writer': u'writer', - u'none': u'none', + 'editor': 'writer', + 'freebusy': 'freeBusyReader', + 'freebusyreader': 'freeBusyReader', + 'owner': 'owner', + 'read': 'reader', + 'reader': 'reader', + 'writer': 'writer', + 'none': 'none', } def doCalendarAddACL(function): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) if not cal: return - myarg = sys.argv[4].lower().replace(u'_', u'') + myarg = sys.argv[4].lower().replace('_', '') if myarg not in CALENDAR_ACL_ROLES_MAP: - systemErrorExit(2, 'Role must be one of %s; got %s' % (u', '.join(sorted(CALENDAR_ACL_ROLES_MAP.keys())), myarg)) - body = {u'role': CALENDAR_ACL_ROLES_MAP[myarg]} + systemErrorExit(2, 'Role must be one of %s; got %s' % (', '.join(sorted(CALENDAR_ACL_ROLES_MAP.keys())), myarg)) + body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]} i = _getCalendarACLScope(5, body) sendNotifications = True while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'sendnotifications': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'sendnotifications': sendNotifications = getBoolean(sys.argv[i+1], myarg) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam calendar %s"' % (sys.argv[i], function.lower())) - print u'Calendar: {0}, {1} ACL: {2}'.format(calendarId, function, formatACLRule(body)) - callGAPI(cal.acl(), u'insert', calendarId=calendarId, body=body, sendNotifications=sendNotifications) + print('Calendar: {0}, {1} ACL: {2}'.format(calendarId, function, formatACLRule(body))) + callGAPI(cal.acl(), 'insert', calendarId=calendarId, body=body, sendNotifications=sendNotifications) def doCalendarDelACL(): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) if not cal: return - body = {u'role': u'none'} + body = {'role': 'none'} _getCalendarACLScope(5, body) - print u'Calendar: {0}, {1} ACL: {2}'.format(calendarId, u'Delete', formatACLScope(body)) - callGAPI(cal.acl(), u'insert', calendarId=calendarId, body=body, sendNotifications=False) + print('Calendar: {0}, {1} ACL: {2}'.format(calendarId, 'Delete', formatACLScope(body))) + callGAPI(cal.acl(), 'insert', calendarId=calendarId, body=body, sendNotifications=False) def doCalendarWipeData(): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) if not cal: return - callGAPI(cal.calendars(), u'clear', calendarId=calendarId) + callGAPI(cal.calendars(), 'clear', calendarId=calendarId) def doCalendarDeleteEvent(): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) @@ -3603,32 +3613,32 @@ def doCalendarDeleteEvent(): doit = False i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'notifyattendees': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'notifyattendees': sendNotifications = True i += 1 - elif myarg in [u'id', u'eventid']: + elif myarg in ['id', 'eventid']: events.append(sys.argv[i+1]) i += 2 - elif myarg in [u'query', u'eventquery']: + elif myarg in ['query', 'eventquery']: query = sys.argv[i+1] - result = callGAPIpages(cal.events(), u'list', u'items', calendarId=calendarId, q=query) + result = callGAPIpages(cal.events(), 'list', 'items', calendarId=calendarId, q=query) for event in result: - if u'id' in event and event[u'id'] not in events: - events.append(event[u'id']) + if 'id' in event and event['id'] not in events: + events.append(event['id']) i += 2 - elif myarg == u'doit': + elif myarg == 'doit': doit = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam calendar deleteevent"' % sys.argv[i]) if doit: for eventId in events: - print u' deleting eventId %s' % eventId - callGAPI(cal.events(), u'delete', calendarId=calendarId, eventId=eventId, sendNotifications=sendNotifications) + print(' deleting eventId %s' % eventId) + callGAPI(cal.events(), 'delete', calendarId=calendarId, eventId=eventId, sendNotifications=sendNotifications) else: for eventId in events: - print u' would delete eventId %s. Add doit to command to actually delete event' % eventId + print(' would delete eventId %s. Add doit to command to actually delete event' % eventId) def doCalendarAddEvent(): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) @@ -3638,106 +3648,106 @@ def doCalendarAddEvent(): i = 4 body = {} while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'notifyattendees': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'notifyattendees': sendNotifications = True i += 1 - elif myarg == u'attendee': - body.setdefault(u'attendees', []) - body[u'attendees'].append({u'email': sys.argv[i+1]}) + elif myarg == 'attendee': + body.setdefault('attendees', []) + body['attendees'].append({'email': sys.argv[i+1]}) i += 2 - elif myarg == u'optionalattendee': - body.setdefault(u'attendees', []) - body[u'attendees'].append({u'email': sys.argv[i+1], u'optional': True}) + elif myarg == 'optionalattendee': + body.setdefault('attendees', []) + body['attendees'].append({'email': sys.argv[i+1], 'optional': True}) i += 2 - elif myarg == u'anyonecanaddself': - body[u'anyoneCanAddSelf'] = True + elif myarg == 'anyonecanaddself': + body['anyoneCanAddSelf'] = True i += 1 - elif myarg == u'description': - body[u'description'] = sys.argv[i+1].replace(u'\\n', u'\n') + elif myarg == 'description': + body['description'] = sys.argv[i+1].replace('\\n', '\n') i += 2 - elif myarg == u'start': - if sys.argv[i+1].lower() == u'allday': - body[u'start'] = {u'date': getYYYYMMDD(sys.argv[i+2])} + elif myarg == 'start': + if sys.argv[i+1].lower() == 'allday': + body['start'] = {'date': getYYYYMMDD(sys.argv[i+2])} i += 3 else: - body[u'start'] = {u'dateTime': getTimeOrDeltaFromNow(sys.argv[i+1])} + body['start'] = {'dateTime': getTimeOrDeltaFromNow(sys.argv[i+1])} i += 2 - elif myarg == u'end': - if sys.argv[i+1].lower() == u'allday': - body[u'end'] = {u'date': getYYYYMMDD(sys.argv[i+2])} + elif myarg == 'end': + if sys.argv[i+1].lower() == 'allday': + body['end'] = {'date': getYYYYMMDD(sys.argv[i+2])} i += 3 else: - body[u'end'] = {u'dateTime': getTimeOrDeltaFromNow(sys.argv[i+1])} + body['end'] = {'dateTime': getTimeOrDeltaFromNow(sys.argv[i+1])} i += 2 - elif myarg == u'guestscantinviteothers': - body[u'guestsCanInviteOthers'] = False + elif myarg == 'guestscantinviteothers': + body['guestsCanInviteOthers'] = False i += 1 - elif myarg == u'guestscantseeothers': - body[u'guestsCanSeeOtherGuests'] = False + elif myarg == 'guestscantseeothers': + body['guestsCanSeeOtherGuests'] = False i += 1 - elif myarg == u'id': - body[u'id'] = sys.argv[i+1] + elif myarg == 'id': + body['id'] = sys.argv[i+1] i += 2 - elif myarg == u'summary': - body[u'summary'] = sys.argv[i+1] + elif myarg == 'summary': + body['summary'] = sys.argv[i+1] i += 2 - elif myarg == u'location': - body[u'location'] = sys.argv[i+1] + elif myarg == 'location': + body['location'] = sys.argv[i+1] i += 2 - elif myarg == u'available': - body[u'transparency'] = u'transparent' + elif myarg == 'available': + body['transparency'] = 'transparent' i += 1 - elif myarg == u'visibility': - if sys.argv[i+1].lower() in [u'default', u'public', u'private']: - body[u'visibility'] = sys.argv[i+1].lower() + elif myarg == 'visibility': + if sys.argv[i+1].lower() in ['default', 'public', 'private']: + body['visibility'] = sys.argv[i+1].lower() else: systemErrorExit(2, 'visibility must be one of default, public, private; got %s' % sys.argv[i+1]) i += 2 - elif myarg == u'tentative': - body[u'status'] = u'tentative' + elif myarg == 'tentative': + body['status'] = 'tentative' i += 1 - elif myarg == u'source': - body[u'source'] = {u'title': sys.argv[i+1], u'url': sys.argv[i+2]} + elif myarg == 'source': + body['source'] = {'title': sys.argv[i+1], 'url': sys.argv[i+2]} i += 3 - elif myarg == u'noreminders': - body[u'reminders'] = {u'useDefault': False} + elif myarg == 'noreminders': + body['reminders'] = {'useDefault': False} i += 1 - elif myarg == u'reminder': - body.setdefault(u'reminders', {u'overrides': [], u'useDefault': False}) - body[u'reminders'][u'overrides'].append({u'minutes': getInteger(sys.argv[i+1], myarg, minVal=0, maxVal=CALENDAR_REMINDER_MAX_MINUTES), - u'method': sys.argv[i+2]}) + elif myarg == 'reminder': + body.setdefault('reminders', {'overrides': [], 'useDefault': False}) + body['reminders']['overrides'].append({'minutes': getInteger(sys.argv[i+1], myarg, minVal=0, maxVal=CALENDAR_REMINDER_MAX_MINUTES), + 'method': sys.argv[i+2]}) i += 3 - elif myarg == u'recurrence': - body.setdefault(u'recurrence', []) - body[u'recurrence'].append(sys.argv[i+1]) + elif myarg == 'recurrence': + body.setdefault('recurrence', []) + body['recurrence'].append(sys.argv[i+1]) i += 2 - elif myarg == u'timezone': + elif myarg == 'timezone': timeZone = sys.argv[i+1] i += 2 - elif myarg == u'privateproperty': - if u'extendedProperties' not in body: - body[u'extendedProperties'] = {u'private': {}, u'shared': {}} - body[u'extendedProperties'][u'private'][sys.argv[i+1]] = sys.argv[i+2] + elif myarg == 'privateproperty': + if 'extendedProperties' not in body: + body['extendedProperties'] = {'private': {}, 'shared': {}} + body['extendedProperties']['private'][sys.argv[i+1]] = sys.argv[i+2] i += 3 - elif myarg == u'sharedproperty': - if u'extendedProperties' not in body: - body[u'extendedProperties'] = {u'private': {}, u'shared': {}} - body[u'extendedProperties'][u'shared'][sys.argv[i+1]] = sys.argv[i+2] + elif myarg == 'sharedproperty': + if 'extendedProperties' not in body: + body['extendedProperties'] = {'private': {}, 'shared': {}} + body['extendedProperties']['shared'][sys.argv[i+1]] = sys.argv[i+2] i += 3 - elif myarg == u'colorindex': - body[u'colorId'] = getInteger(sys.argv[i+1], myarg, CALENDAR_EVENT_MIN_COLOR_INDEX, CALENDAR_EVENT_MAX_COLOR_INDEX) + elif myarg == 'colorindex': + body['colorId'] = getInteger(sys.argv[i+1], myarg, CALENDAR_EVENT_MIN_COLOR_INDEX, CALENDAR_EVENT_MAX_COLOR_INDEX) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam calendar addevent"' % sys.argv[i]) - if (u'recurrence' in body) and ((u'start' in body) or (u'end' in body)): + if ('recurrence' in body) and (('start' in body) or ('end' in body)): if not timeZone: - timeZone = callGAPI(cal.calendars(), u'get', calendarId=calendarId, fields=u'timeZone')[u'timeZone'] - if u'start' in body: - body[u'start'][u'timeZone'] = timeZone - if u'end' in body: - body[u'end'][u'timeZone'] = timeZone - callGAPI(cal.events(), u'insert', calendarId=calendarId, sendNotifications=sendNotifications, body=body) + timeZone = callGAPI(cal.calendars(), 'get', calendarId=calendarId, fields='timeZone')['timeZone'] + if 'start' in body: + body['start']['timeZone'] = timeZone + if 'end' in body: + body['end']['timeZone'] = timeZone + callGAPI(cal.events(), 'insert', calendarId=calendarId, sendNotifications=sendNotifications, body=body) def doCalendarModifySettings(): calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2]) @@ -3746,93 +3756,93 @@ def doCalendarModifySettings(): body = {} i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'description': - body[u'description'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'description': + body['description'] = sys.argv[i+1] i += 2 - elif myarg == u'location': - body[u'location'] = sys.argv[i+1] + elif myarg == 'location': + body['location'] = sys.argv[i+1] i += 2 - elif myarg == u'summary': - body[u'summary'] = sys.argv[i+1] + elif myarg == 'summary': + body['summary'] = sys.argv[i+1] i += 2 - elif myarg == u'timezone': - body[u'timeZone'] = sys.argv[i+1] + elif myarg == 'timezone': + body['timeZone'] = sys.argv[i+1] i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam calendar modify"' % sys.argv[i]) - callGAPI(cal.calendars(), u'patch', calendarId=calendarId, body=body) + callGAPI(cal.calendars(), 'patch', calendarId=calendarId, body=body) def doProfile(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') myarg = sys.argv[4].lower() - if myarg in [u'share', u'shared']: - body = {u'includeInGlobalAddressList': True} - elif myarg in [u'unshare', u'unshared']: - body = {u'includeInGlobalAddressList': False} + if myarg in ['share', 'shared']: + body = {'includeInGlobalAddressList': True} + elif myarg in ['unshare', 'unshared']: + body = {'includeInGlobalAddressList': False} else: systemErrorExit(2, 'value for "gam profile" must be true or false; got %s' % sys.argv[4]) i = 0 count = len(users) for user in users: i += 1 - print u'Setting Profile Sharing to %s for %s (%s/%s)' % (body[u'includeInGlobalAddressList'], user, i, count) - callGAPI(cd.users(), u'update', soft_errors=True, userKey=user, body=body) + print('Setting Profile Sharing to %s for %s (%s/%s)' % (body['includeInGlobalAddressList'], user, i, count)) + callGAPI(cd.users(), 'update', soft_errors=True, userKey=user, body=body) def showProfile(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i = 0 count = len(users) for user in users: i += 1 - result = callGAPI(cd.users(), u'get', userKey=user, fields=u'includeInGlobalAddressList') + result = callGAPI(cd.users(), 'get', userKey=user, fields='includeInGlobalAddressList') try: - print u'User: %s Profile Shared: %s (%s/%s)' % (user, result[u'includeInGlobalAddressList'], i, count) + print('User: %s Profile Shared: %s (%s/%s)' % (user, result['includeInGlobalAddressList'], i, count)) except IndexError: pass def doPhoto(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i = 0 count = len(users) for user in users: i += 1 - filename = sys.argv[5].replace(u'#user#', user) - filename = filename.replace(u'#email#', user) - filename = filename.replace(u'#username#', user[:user.find(u'@')]) - print u"Updating photo for %s with %s (%s/%s)" % (user, filename, i, count) - if re.match(u'^(ht|f)tps?://.*$', filename): + filename = sys.argv[5].replace('#user#', user) + filename = filename.replace('#email#', user) + filename = filename.replace('#username#', user[:user.find('@')]) + print("Updating photo for %s with %s (%s/%s)" % (user, filename, i, count)) + if re.match('^(ht|f)tps?://.*$', filename): simplehttp = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) try: - (_, f) = simplehttp.request(filename, u'GET') + (_, f) = simplehttp.request(filename, 'GET') image_data = str(f) except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError, httplib2.CertificateValidationUnsupported) as e: - print e + print(e) continue else: image_data = readFile(filename, continueOnError=True, displayError=True) if image_data is None: continue image_data = base64.urlsafe_b64encode(image_data) - body = {u'photoData': image_data} - callGAPI(cd.users().photos(), u'update', soft_errors=True, userKey=user, body=body) + body = {'photoData': image_data} + callGAPI(cd.users().photos(), 'update', soft_errors=True, userKey=user, body=body) def getPhoto(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') targetFolder = os.getcwd() showPhotoData = True i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'drivedir': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'drivedir': targetFolder = GC_Values[GC_DRIVE_DIR] i += 1 - elif myarg == u'targetfolder': + elif myarg == 'targetfolder': targetFolder = os.path.expanduser(sys.argv[i+1]) if not os.path.isdir(targetFolder): os.makedirs(targetFolder) i += 2 - elif myarg == u'noshow': + elif myarg == 'noshow': showPhotoData = False i += 1 else: @@ -3841,52 +3851,52 @@ def getPhoto(users): count = len(users) for user in users: i += 1 - filename = os.path.join(targetFolder, u'{0}.jpg'.format(user)) - print u"Saving photo to %s (%s/%s)" % (filename, i, count) + filename = os.path.join(targetFolder, '{0}.jpg'.format(user)) + print("Saving photo to %s (%s/%s)" % (filename, i, count)) try: - photo = callGAPI(cd.users().photos(), u'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_RESOURCE_NOT_FOUND], userKey=user) + photo = callGAPI(cd.users().photos(), 'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_RESOURCE_NOT_FOUND], userKey=user) except GAPI_userNotFound: - print u' unknown user %s' % user + print(' unknown user %s' % user) continue except GAPI_resourceNotFound: - print u' no photo for %s' % user + print(' no photo for %s' % user) continue try: - photo_data = str(photo[u'photoData']) + photo_data = str(photo['photoData']) if showPhotoData: - print photo_data + print(photo_data) photo_data = base64.urlsafe_b64decode(photo_data) except KeyError: - print u' no photo for %s' % user + print(' no photo for %s' % user) continue writeFile(filename, photo_data, continueOnError=True) def deletePhoto(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i = 0 count = len(users) for user in users: i += 1 - print u"Deleting photo for %s (%s/%s)" % (user, i, count) - callGAPI(cd.users().photos(), u'delete', userKey=user) + print("Deleting photo for %s (%s/%s)" % (user, i, count)) + callGAPI(cd.users().photos(), 'delete', userKey=user) def _showCalendar(userCalendar, j, jcount): - print u' Calendar: {0} ({1}/{2})'.format(userCalendar[u'id'], j, jcount) - print utils.convertUTF8(u' Summary: {0}'.format(userCalendar.get(u'summaryOverride', userCalendar[u'summary']))) - print utils.convertUTF8(u' Description: {0}'.format(userCalendar.get(u'description', u''))) - print u' Access Level: {0}'.format(userCalendar[u'accessRole']) - print u' Timezone: {0}'.format(userCalendar[u'timeZone']) - print utils.convertUTF8(u' Location: {0}'.format(userCalendar.get(u'location', u''))) - print u' Hidden: {0}'.format(userCalendar.get(u'hidden', u'False')) - print u' Selected: {0}'.format(userCalendar.get(u'selected', u'False')) - print u' Color ID: {0}, Background Color: {1}, Foreground Color: {2}'.format(userCalendar[u'colorId'], userCalendar[u'backgroundColor'], userCalendar[u'foregroundColor']) - print u' Default Reminders:' - for reminder in userCalendar.get(u'defaultReminders', []): - print u' Method: {0}, Minutes: {1}'.format(reminder[u'method'], reminder[u'minutes']) - print u' Notifications:' - if u'notificationSettings' in userCalendar: - for notification in userCalendar[u'notificationSettings'].get(u'notifications', []): - print u' Method: {0}, Type: {1}'.format(notification[u'method'], notification[u'type']) + print(' Calendar: {0} ({1}/{2})'.format(userCalendar['id'], j, jcount)) + print(utils.convertUTF8(' Summary: {0}'.format(userCalendar.get('summaryOverride', userCalendar['summary'])))) + print(utils.convertUTF8(' Description: {0}'.format(userCalendar.get('description', '')))) + print(' Access Level: {0}'.format(userCalendar['accessRole'])) + print(' Timezone: {0}'.format(userCalendar['timeZone'])) + print(utils.convertUTF8(' Location: {0}'.format(userCalendar.get('location', '')))) + print(' Hidden: {0}'.format(userCalendar.get('hidden', 'False'))) + print(' Selected: {0}'.format(userCalendar.get('selected', 'False'))) + print(' Color ID: {0}, Background Color: {1}, Foreground Color: {2}'.format(userCalendar['colorId'], userCalendar['backgroundColor'], userCalendar['foregroundColor'])) + print(' Default Reminders:') + for reminder in userCalendar.get('defaultReminders', []): + print(' Method: {0}, Minutes: {1}'.format(reminder['method'], reminder['minutes'])) + print(' Notifications:') + if 'notificationSettings' in userCalendar: + for notification in userCalendar['notificationSettings'].get('notifications', []): + print(' Method: {0}, Type: {1}'.format(notification['method'], notification['type'])) def infoCalendar(users): calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True) @@ -3897,11 +3907,11 @@ def infoCalendar(users): user, cal = buildCalendarGAPIObject(user) if not cal: continue - result = callGAPI(cal.calendarList(), u'get', + result = callGAPI(cal.calendarList(), 'get', soft_errors=True, calendarId=calendarId) if result: - print u'User: {0}, Calendar: ({1}/{2})'.format(user, i, count) + print('User: {0}, Calendar: ({1}/{2})'.format(user, i, count)) _showCalendar(result, 1, 1) def printShowCalendars(users, csvFormat): @@ -3912,11 +3922,11 @@ def printShowCalendars(users, csvFormat): i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s calendars"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s calendars"' % (myarg, ['show', 'print'][csvFormat])) i = 0 count = len(users) for user in users: @@ -3924,10 +3934,10 @@ def printShowCalendars(users, csvFormat): user, cal = buildCalendarGAPIObject(user) if not cal: continue - result = callGAPIpages(cal.calendarList(), u'list', u'items', soft_errors=True) + result = callGAPIpages(cal.calendarList(), 'list', 'items', soft_errors=True) jcount = len(result) if not csvFormat: - print u'User: {0}, Calendars: ({1}/{2})'.format(user, i, count) + print('User: {0}, Calendars: ({1}/{2})'.format(user, i, count)) if jcount == 0: continue j = 0 @@ -3938,11 +3948,11 @@ def printShowCalendars(users, csvFormat): if jcount == 0: continue for userCalendar in result: - row = {u'primaryEmail': user} + row = {'primaryEmail': user} addRowTitlesToCSVfile(flatten_json(userCalendar, flattened=row), csvRows, titles) if csvFormat: - sortCSVTitles([u'primaryEmail', u'id'], titles) - writeCSVfile(csvRows, titles, u'Calendars', todrive) + sortCSVTitles(['primaryEmail', 'id'], titles) + writeCSVfile(csvRows, titles, 'Calendars', todrive) def showCalSettings(users): i = 0 @@ -3952,28 +3962,28 @@ def showCalSettings(users): user, cal = buildCalendarGAPIObject(user) if not cal: continue - feed = callGAPIpages(cal.settings(), u'list', u'items', soft_errors=True) + feed = callGAPIpages(cal.settings(), 'list', 'items', soft_errors=True) if feed: - print u'User: {0}, Calendar Settings: ({1}/{2})'.format(user, i, count) + print('User: {0}, Calendar Settings: ({1}/{2})'.format(user, i, count)) settings = {} for setting in feed: - settings[setting[u'id']] = setting[u'value'] + settings[setting['id']] = setting['value'] for attr, value in sorted(settings.items()): - print u' {0}: {1}'.format(attr, value) + print(' {0}: {1}'.format(attr, value)) def printDriveSettings(users): todrive = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam show drivesettings"' % sys.argv[i]) - dont_show = [u'kind', u'exportFormats', u'importFormats', u'maxUploadSize', u'maxImportSizes', u'user', u'appInstalled'] + dont_show = ['kind', 'exportFormats', 'importFormats', 'maxUploadSize', 'maxImportSizes', 'user', 'appInstalled'] csvRows = [] - titles = [u'email',] + titles = ['email',] i = 0 count = len(users) for user in users: @@ -3981,17 +3991,17 @@ def printDriveSettings(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - sys.stderr.write(u'Getting Drive settings for %s (%s/%s)\n' % (user, i, count)) - feed = callGAPI(drive.about(), u'get', fields=u'*', soft_errors=True) + sys.stderr.write('Getting Drive settings for %s (%s/%s)\n' % (user, i, count)) + feed = callGAPI(drive.about(), 'get', fields='*', soft_errors=True) if feed is None: continue - row = {u'email': user} + row = {'email': user} for setting in feed: if setting in dont_show: continue - if setting == u'storageQuota': - for subsetting, value in feed[setting].iteritems(): - row[subsetting] = u'%smb' % (int(value) / 1024 / 1024) + if setting == 'storageQuota': + for subsetting, value in feed[setting].items(): + row[subsetting] = '%smb' % (int(value) / 1024 / 1024) if subsetting not in titles: titles.append(subsetting) continue @@ -3999,37 +4009,37 @@ def printDriveSettings(users): if setting not in titles: titles.append(setting) csvRows.append(row) - writeCSVfile(csvRows, titles, u'User Drive Settings', todrive) + writeCSVfile(csvRows, titles, 'User Drive Settings', todrive) def getTeamDriveThemes(users): for user in users: user, drive = buildDrive3GAPIObject(user) if not drive: continue - themes = callGAPI(drive.about(), u'get', fields=u'teamDriveThemes', soft_errors=True) - if themes is None or u'teamDriveThemes' not in themes: + themes = callGAPI(drive.about(), 'get', fields='teamDriveThemes', soft_errors=True) + if themes is None or 'teamDriveThemes' not in themes: continue - print u'theme' - for theme in themes[u'teamDriveThemes']: - print theme[u'id'] + print('theme') + for theme in themes['teamDriveThemes']: + print(theme['id']) def printDriveActivity(users): - drive_ancestorId = u'root' + drive_ancestorId = 'root' drive_fileId = None todrive = False - titles = [u'user.name', u'user.permissionId', u'target.id', u'target.name', u'target.mimeType'] + titles = ['user.name', 'user.permissionId', 'target.id', 'target.name', 'target.mimeType'] csvRows = [] i = 5 while i < len(sys.argv): - activity_object = sys.argv[i].lower().replace(u'_', u'') - if activity_object == u'fileid': + activity_object = sys.argv[i].lower().replace('_', '') + if activity_object == 'fileid': drive_fileId = sys.argv[i+1] drive_ancestorId = None i += 2 - elif activity_object == u'folderid': + elif activity_object == 'folderid': drive_ancestorId = sys.argv[i+1] i += 2 - elif activity_object == u'todrive': + elif activity_object == 'todrive': todrive = True i += 1 else: @@ -4038,37 +4048,37 @@ def printDriveActivity(users): user, activity = buildActivityGAPIObject(user) if not activity: continue - page_message = u'Got %%%%total_items%%%% activities for %s' % user - feed = callGAPIpages(activity.activities(), u'list', u'activities', - page_message=page_message, source=u'drive.google.com', userId=u'me', - drive_ancestorId=drive_ancestorId, groupingStrategy=u'none', + page_message = 'Got %%%%total_items%%%% activities for %s' % user + feed = callGAPIpages(activity.activities(), 'list', 'activities', + page_message=page_message, source='drive.google.com', userId='me', + drive_ancestorId=drive_ancestorId, groupingStrategy='none', drive_fileId=drive_fileId, pageSize=GC_Values[GC_ACTIVITY_MAX_RESULTS]) for item in feed: - addRowTitlesToCSVfile(flatten_json(item[u'combinedEvent']), csvRows, titles) - writeCSVfile(csvRows, titles, u'Drive Activity', todrive) + addRowTitlesToCSVfile(flatten_json(item['combinedEvent']), csvRows, titles) + writeCSVfile(csvRows, titles, 'Drive Activity', todrive) def printPermission(permission): - if u'name' in permission: - print utils.convertUTF8(permission[u'name']) - elif u'id' in permission: - if permission[u'id'] == u'anyone': - print u'Anyone' - elif permission[u'id'] == u'anyoneWithLink': - print u'Anyone with Link' + if 'name' in permission: + print(utils.convertUTF8(permission['name'])) + elif 'id' in permission: + if permission['id'] == 'anyone': + print('Anyone') + elif permission['id'] == 'anyoneWithLink': + print('Anyone with Link') else: - print permission[u'id'] + print(permission['id']) for key in permission: - if key in [u'name', u'kind', u'etag', u'selfLink',]: + if key in ['name', 'kind', 'etag', 'selfLink',]: continue - print utils.convertUTF8(u' %s: %s' % (key, permission[key])) + print(utils.convertUTF8(' %s: %s' % (key, permission[key]))) def showDriveFileACL(users): fileId = sys.argv[5] useDomainAdminAccess = False i = 6 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'asadmin': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'asadmin': useDomainAdminAccess = True i += 1 else: @@ -4077,12 +4087,12 @@ def showDriveFileACL(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - feed = callGAPIpages(drive.permissions(), u'list', u'permissions', - fileId=fileId, fields=u'*', supportsTeamDrives=True, + feed = callGAPIpages(drive.permissions(), 'list', 'permissions', + fileId=fileId, fields='*', supportsTeamDrives=True, useDomainAdminAccess=useDomainAdminAccess) for permission in feed: printPermission(permission) - print u'' + print('') def getPermissionId(argstr): permissionId = argstr.strip() @@ -4090,16 +4100,16 @@ def getPermissionId(argstr): if cg: return cg.group(1) permissionId = argstr.lower() - if permissionId == u'anyone': - return u'anyone' - if permissionId == u'anyonewithlink': - return u'anyoneWithLink' - if permissionId.find(u'@') == -1: - permissionId = u'%s@%s' % (permissionId, GC_Values[GC_DOMAIN].lower()) + if permissionId == 'anyone': + return 'anyone' + if permissionId == 'anyonewithlink': + return 'anyoneWithLink' + if permissionId.find('@') == -1: + permissionId = '%s@%s' % (permissionId, GC_Values[GC_DOMAIN].lower()) # We have to use v2 here since v3 has no permissions.getIdForEmail equivalent # https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=4313 - _, drive2 = buildDriveGAPIObject(_getValueFromOAuth(u'email')) - return callGAPI(drive2.permissions(), u'getIdForEmail', email=permissionId, fields=u'id')[u'id'] + _, drive2 = buildDriveGAPIObject(_getValueFromOAuth('email')) + return callGAPI(drive2.permissions(), 'getIdForEmail', email=permissionId, fields='id')['id'] def delDriveFileACL(users): fileId = sys.argv[5] @@ -4107,8 +4117,8 @@ def delDriveFileACL(users): useDomainAdminAccess = False i = 7 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'asadmin': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'asadmin': useDomainAdminAccess = True i += 1 else: @@ -4117,68 +4127,68 @@ def delDriveFileACL(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - print u'Removing permission for %s from %s' % (permissionId, fileId) - callGAPI(drive.permissions(), u'delete', fileId=fileId, + print('Removing permission for %s from %s' % (permissionId, fileId)) + callGAPI(drive.permissions(), 'delete', fileId=fileId, permissionId=permissionId, supportsTeamDrives=True, useDomainAdminAccess=useDomainAdminAccess) DRIVEFILE_ACL_ROLES_MAP = { - u'commenter': u'commenter', - u'contentmanager': u'fileOrganizer', - u'editor': u'writer', - u'fileorganizer': u'fileOrganizer', - u'organizer': u'organizer', - u'owner': u'owner', - u'read': u'reader', - u'reader': u'reader', - u'writer': u'writer', + 'commenter': 'commenter', + 'contentmanager': 'fileOrganizer', + 'editor': 'writer', + 'fileorganizer': 'fileOrganizer', + 'organizer': 'organizer', + 'owner': 'owner', + 'read': 'reader', + 'reader': 'reader', + 'writer': 'writer', } def addDriveFileACL(users): fileId = sys.argv[5] - body = {u'type': sys.argv[6].lower()} + body = {'type': sys.argv[6].lower()} sendNotificationEmail = False emailMessage = None transferOwnership = None useDomainAdminAccess = False - if body[u'type'] == u'anyone': + if body['type'] == 'anyone': i = 7 - elif body[u'type'] in [u'user', u'group']: - body[u'emailAddress'] = normalizeEmailAddressOrUID(sys.argv[7]) + elif body['type'] in ['user', 'group']: + body['emailAddress'] = normalizeEmailAddressOrUID(sys.argv[7]) i = 8 - elif body[u'type'] == u'domain': - body[u'domain'] = sys.argv[7] + elif body['type'] == 'domain': + body['domain'] = sys.argv[7] i = 8 else: - systemErrorExit(5, 'permission type must be user, group domain or anyone; got %s' % body[u'type']) + systemErrorExit(5, 'permission type must be user, group domain or anyone; got %s' % body['type']) while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'withlink': - body[u'allowFileDiscovery'] = False + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'withlink': + body['allowFileDiscovery'] = False i += 1 - elif myarg == u'discoverable': - body[u'allowFileDiscovery'] = True + elif myarg == 'discoverable': + body['allowFileDiscovery'] = True i += 1 - elif myarg == u'role': + elif myarg == 'role': role = sys.argv[i+1].lower() if role not in DRIVEFILE_ACL_ROLES_MAP: - systemErrorExit(2, 'role must be {0}; got {1}'.format(u', '.join(DRIVEFILE_ACL_ROLES_MAP), role)) - body[u'role'] = DRIVEFILE_ACL_ROLES_MAP[role] - if body[u'role'] == u'owner': + systemErrorExit(2, 'role must be {0}; got {1}'.format(', '.join(DRIVEFILE_ACL_ROLES_MAP), role)) + body['role'] = DRIVEFILE_ACL_ROLES_MAP[role] + if body['role'] == 'owner': sendNotificationEmail = True transferOwnership = True i += 2 - elif myarg == u'sendemail': + elif myarg == 'sendemail': sendNotificationEmail = True i += 1 - elif myarg == u'emailmessage': + elif myarg == 'emailmessage': sendNotificationEmail = True emailMessage = sys.argv[i+1] i += 2 - elif myarg == u'expires': - body[u'expirationTime'] = getTimeOrDeltaFromNow(sys.argv[i+1]) + elif myarg == 'expires': + body['expirationTime'] = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg == u'asadmin': + elif myarg == 'asadmin': useDomainAdminAccess = True i += 1 else: @@ -4187,7 +4197,7 @@ def addDriveFileACL(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - result = callGAPI(drive.permissions(), u'create', fields=u'*', + result = callGAPI(drive.permissions(), 'create', fields='*', fileId=fileId, sendNotificationEmail=sendNotificationEmail, emailMessage=emailMessage, body=body, supportsTeamDrives=True, transferOwnership=transferOwnership, @@ -4203,19 +4213,19 @@ def updateDriveFileACL(users): body = {} i = 7 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'removeexpiration': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'removeexpiration': removeExpiration = True i += 1 - elif myarg == u'role': + elif myarg == 'role': role = sys.argv[i+1].lower() if role not in DRIVEFILE_ACL_ROLES_MAP: - systemErrorExit(2, 'role must be {0}; got {1}'.format(u', '.join(DRIVEFILE_ACL_ROLES_MAP), role)) - body[u'role'] = DRIVEFILE_ACL_ROLES_MAP[role] - if body[u'role'] == u'owner': + systemErrorExit(2, 'role must be {0}; got {1}'.format(', '.join(DRIVEFILE_ACL_ROLES_MAP), role)) + body['role'] = DRIVEFILE_ACL_ROLES_MAP[role] + if body['role'] == 'owner': transferOwnership = True i += 2 - elif myarg == u'asadmin': + elif myarg == 'asadmin': useDomainAdminAccess = True i += 1 else: @@ -4224,8 +4234,8 @@ def updateDriveFileACL(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - print u'updating permissions for %s to file %s' % (permissionId, fileId) - result = callGAPI(drive.permissions(), u'update', fields=u'*', + print('updating permissions for %s to file %s' % (permissionId, fileId)) + result = callGAPI(drive.permissions(), 'update', fields='*', fileId=fileId, permissionId=permissionId, removeExpiration=removeExpiration, transferOwnership=transferOwnership, body=body, supportsTeamDrives=True, useDomainAdminAccess=useDomainAdminAccess) @@ -4234,10 +4244,10 @@ def updateDriveFileACL(users): def _stripMeInOwners(query): if not query: return query - if query == u"'me' in owners": + if query == "'me' in owners": return None - if query.startswith(u"'me' in owners and "): - return query[len(u"'me' in owners and "):] + if query.startswith("'me' in owners and "): + return query[len("'me' in owners and "):] return query def printDriveFileList(users): @@ -4246,42 +4256,42 @@ def printDriveFileList(users): fieldsTitles = {} labelsList = [] orderByList = [] - titles = [u'Owner',] + titles = ['Owner',] csvRows = [] - query = u"'me' in owners" + query = "'me' in owners" i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'orderby': + elif myarg == 'orderby': fieldName = sys.argv[i+1].lower() i += 2 if fieldName in DRIVEFILE_ORDERBY_CHOICES_MAP: fieldName = DRIVEFILE_ORDERBY_CHOICES_MAP[fieldName] - orderBy = u'' + orderBy = '' if i < len(sys.argv): orderBy = sys.argv[i].lower() if orderBy in SORTORDER_CHOICES_MAP: orderBy = SORTORDER_CHOICES_MAP[orderBy] i += 1 - if orderBy != u'DESCENDING': + if orderBy != 'DESCENDING': orderByList.append(fieldName) else: - orderByList.append(u'{0} desc'.format(fieldName)) + orderByList.append('{0} desc'.format(fieldName)) else: - systemErrorExit(2, 'orderby must be one of {0}; got {1}'.format(u', '.join(sorted(DRIVEFILE_ORDERBY_CHOICES_MAP.keys())), fieldName)) - elif myarg == u'query': - query += u' and %s' % sys.argv[i+1] + systemErrorExit(2, 'orderby must be one of {0}; got {1}'.format(', '.join(sorted(DRIVEFILE_ORDERBY_CHOICES_MAP.keys())), fieldName)) + elif myarg == 'query': + query += ' and %s' % sys.argv[i+1] i += 2 - elif myarg == u'fullquery': + elif myarg == 'fullquery': query = sys.argv[i+1] i += 2 - elif myarg == u'anyowner': + elif myarg == 'anyowner': anyowner = True i += 1 - elif myarg == u'allfields': + elif myarg == 'allfields': fieldsList = [] allfields = True i += 1 @@ -4294,22 +4304,22 @@ def printDriveFileList(users): else: systemErrorExit(2, '%s is not a valid argument for "gam show filelist"' % myarg) if fieldsList or labelsList: - fields = u'nextPageToken,items(' + fields = 'nextPageToken,items(' if fieldsList: - fields += u','.join(set(fieldsList)) + fields += ','.join(set(fieldsList)) if labelsList: - fields += u',' + fields += ',' if labelsList: - fields += u'labels({0})'.format(u','.join(set(labelsList))) - fields += u')' + fields += 'labels({0})'.format(','.join(set(labelsList))) + fields += ')' elif not allfields: - for field in [u'name', u'alternatelink']: + for field in ['name', 'alternatelink']: addFieldToCSVfile(field, {field: [DRIVEFILE_FIELDS_CHOICES_MAP[field]]}, fieldsList, fieldsTitles, titles) - fields = u'nextPageToken,items({0})'.format(u','.join(set(fieldsList))) + fields = 'nextPageToken,items({0})'.format(','.join(set(fieldsList))) else: - fields = u'*' + fields = '*' if orderByList: - orderBy = u','.join(orderByList) + orderBy = ','.join(orderByList) else: orderBy = None if anyowner: @@ -4318,97 +4328,97 @@ def printDriveFileList(users): user, drive = buildDriveGAPIObject(user) if not drive: continue - sys.stderr.write(u'Getting files for %s...\n' % user) - page_message = u' Got %%%%total_items%%%% files for %s...\n' % user - feed = callGAPIpages(drive.files(), u'list', u'items', + sys.stderr.write('Getting files for %s...\n' % user) + page_message = ' Got %%%%total_items%%%% files for %s...\n' % user + feed = callGAPIpages(drive.files(), 'list', 'items', page_message=page_message, soft_errors=True, q=query, orderBy=orderBy, fields=fields, maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) for f_file in feed: - a_file = {u'Owner': user} + a_file = {'Owner': user} for attrib in f_file: - if attrib in [u'kind', u'etag']: + if attrib in ['kind', 'etag']: continue if not isinstance(f_file[attrib], dict): if isinstance(f_file[attrib], list): if f_file[attrib]: - if isinstance(f_file[attrib][0], (str, unicode, int, bool)): + if isinstance(f_file[attrib][0], (str, int, bool)): if attrib not in titles: titles.append(attrib) - a_file[attrib] = u' '.join(f_file[attrib]) + a_file[attrib] = ' '.join(f_file[attrib]) else: for j, l_attrib in enumerate(f_file[attrib]): for list_attrib in l_attrib: - if list_attrib in [u'kind', u'etag', u'selfLink']: + if list_attrib in ['kind', 'etag', 'selfLink']: continue - x_attrib = u'{0}.{1}.{2}'.format(attrib, j, list_attrib) + x_attrib = '{0}.{1}.{2}'.format(attrib, j, list_attrib) if x_attrib not in titles: titles.append(x_attrib) a_file[x_attrib] = l_attrib[list_attrib] - elif isinstance(f_file[attrib], (str, unicode, int, bool)): + elif isinstance(f_file[attrib], (str, int, bool)): if attrib not in titles: titles.append(attrib) a_file[attrib] = f_file[attrib] else: - sys.stderr.write(u'File ID: {0}, Attribute: {1}, Unknown type: {2}\n'.format(f_file[u'id'], attrib, type(f_file[attrib]))) - elif attrib == u'labels': + sys.stderr.write('File ID: {0}, Attribute: {1}, Unknown type: {2}\n'.format(f_file['id'], attrib, type(f_file[attrib]))) + elif attrib == 'labels': for dict_attrib in f_file[attrib]: if dict_attrib not in titles: titles.append(dict_attrib) a_file[dict_attrib] = f_file[attrib][dict_attrib] else: for dict_attrib in f_file[attrib]: - if dict_attrib in [u'kind', u'etag']: + if dict_attrib in ['kind', 'etag']: continue - x_attrib = u'{0}.{1}'.format(attrib, dict_attrib) + x_attrib = '{0}.{1}'.format(attrib, dict_attrib) if x_attrib not in titles: titles.append(x_attrib) a_file[x_attrib] = f_file[attrib][dict_attrib] csvRows.append(a_file) if allfields: - sortCSVTitles([u'Owner', u'id', u'title'], titles) - writeCSVfile(csvRows, titles, u'%s %s Drive Files' % (sys.argv[1], sys.argv[2]), todrive) + sortCSVTitles(['Owner', 'id', 'title'], titles) + writeCSVfile(csvRows, titles, '%s %s Drive Files' % (sys.argv[1], sys.argv[2]), todrive) def doDriveSearch(drive, query=None, quiet=False): if not quiet: - print u'Searching for files with query: "%s"...' % query - page_message = u' Got %%total_items%% files...\n' + print('Searching for files with query: "%s"...' % query) + page_message = ' Got %%total_items%% files...\n' else: page_message = None - files = callGAPIpages(drive.files(), u'list', u'items', + files = callGAPIpages(drive.files(), 'list', 'items', page_message=page_message, - q=query, fields=u'nextPageToken,items(id)', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) + q=query, fields='nextPageToken,items(id)', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) ids = list() for f_file in files: - ids.append(f_file[u'id']) + ids.append(f_file['id']) return ids def getFileIdFromAlternateLink(altLink): - loc = altLink.find(u'/d/') + loc = altLink.find('/d/') if loc > 0: fileId = altLink[loc+3:] - loc = fileId.find(u'/') + loc = fileId.find('/') if loc != -1: return fileId[:loc] else: - loc = altLink.find(u'/folderview?id=') + loc = altLink.find('/folderview?id=') if loc > 0: fileId = altLink[loc+15:] - loc = fileId.find(u'&') + loc = fileId.find('&') if loc != -1: return fileId[:loc] systemErrorExit(2, '%s is not a valid Drive File alternateLink' % altLink) def deleteDriveFile(users): fileIds = sys.argv[5] - function = u'trash' + function = 'trash' i = 6 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'purge': - function = u'delete' + if myarg == 'purge': + function = 'delete' i += 1 - elif myarg == u'untrash': - function = u'untrash' + elif myarg == 'untrash': + function = 'untrash' i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam delete drivefile"' % sys.argv[i]) @@ -4417,60 +4427,60 @@ def deleteDriveFile(users): user, drive = buildDriveGAPIObject(user) if not drive: continue - if fileIds[:6].lower() == u'query:': + if fileIds[:6].lower() == 'query:': file_ids = doDriveSearch(drive, query=fileIds[6:]) else: - if fileIds[:8].lower() == u'https://' or fileIds[:7].lower() == u'http://': + if fileIds[:8].lower() == 'https://' or fileIds[:7].lower() == 'http://': fileIds = getFileIdFromAlternateLink(fileIds) file_ids = [fileIds,] if not file_ids: - print u'No files to %s for %s' % (function, user) + print('No files to %s for %s' % (function, user)) i = 0 for fileId in file_ids: i += 1 - print u'%s %s for %s (%s/%s)' % (action, fileId, user, i, len(file_ids)) + print('%s %s for %s (%s/%s)' % (action, fileId, user, i, len(file_ids))) callGAPI(drive.files(), function, fileId=fileId, supportsTeamDrives=True) def printDriveFolderContents(feed, folderId, indent): for f_file in feed: - for parent in f_file[u'parents']: - if folderId == parent[u'id']: - print u' ' * indent, utils.convertUTF8(f_file[u'title']) - if f_file[u'mimeType'] == u'application/vnd.google-apps.folder': - printDriveFolderContents(feed, f_file[u'id'], indent+1) + for parent in f_file['parents']: + if folderId == parent['id']: + print(' ' * indent, utils.convertUTF8(f_file['title'])) + if f_file['mimeType'] == 'application/vnd.google-apps.folder': + printDriveFolderContents(feed, f_file['id'], indent+1) break def showDriveFileTree(users): anyowner = False orderByList = [] - query = u"'me' in owners" + query = "'me' in owners" i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'anyowner': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'anyowner': anyowner = True i += 1 - elif myarg == u'orderby': + elif myarg == 'orderby': fieldName = sys.argv[i+1].lower() i += 2 if fieldName in DRIVEFILE_ORDERBY_CHOICES_MAP: fieldName = DRIVEFILE_ORDERBY_CHOICES_MAP[fieldName] - orderBy = u'' + orderBy = '' if i < len(sys.argv): orderBy = sys.argv[i].lower() if orderBy in SORTORDER_CHOICES_MAP: orderBy = SORTORDER_CHOICES_MAP[orderBy] i += 1 - if orderBy != u'DESCENDING': + if orderBy != 'DESCENDING': orderByList.append(fieldName) else: - orderByList.append(u'{0} desc'.format(fieldName)) + orderByList.append('{0} desc'.format(fieldName)) else: - systemErrorExit(2, 'orderby must be one of {0}; got {1}'.format(u', '.join(sorted(DRIVEFILE_ORDERBY_CHOICES_MAP.keys())), fieldName)) + systemErrorExit(2, 'orderby must be one of {0}; got {1}'.format(', '.join(sorted(DRIVEFILE_ORDERBY_CHOICES_MAP.keys())), fieldName)) else: systemErrorExit(2, '%s is not a valid argument for "gam show filetree"' % myarg) if orderByList: - orderBy = u','.join(orderByList) + orderBy = ','.join(orderByList) else: orderBy = None if anyowner: @@ -4479,53 +4489,53 @@ def showDriveFileTree(users): user, drive = buildDriveGAPIObject(user) if not drive: continue - root_folder = callGAPI(drive.about(), u'get', fields=u'rootFolderId')[u'rootFolderId'] - sys.stderr.write(u'Getting all files for %s...\n' % user) - page_message = u' Got %%%%total_items%%%% files for %s...\n' % user - feed = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, - q=query, orderBy=orderBy, fields=u'items(id,title,parents(id),mimeType),nextPageToken', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) + root_folder = callGAPI(drive.about(), 'get', fields='rootFolderId')['rootFolderId'] + sys.stderr.write('Getting all files for %s...\n' % user) + page_message = ' Got %%%%total_items%%%% files for %s...\n' % user + feed = callGAPIpages(drive.files(), 'list', 'items', page_message=page_message, + q=query, orderBy=orderBy, fields='items(id,title,parents(id),mimeType),nextPageToken', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) printDriveFolderContents(feed, root_folder, 0) def deleteEmptyDriveFolders(users): - query = u'"me" in owners and mimeType = "application/vnd.google-apps.folder"' + query = '"me" in owners and mimeType = "application/vnd.google-apps.folder"' for user in users: user, drive = buildDriveGAPIObject(user) if not drive: continue deleted_empty = True while deleted_empty: - sys.stderr.write(u'Getting folders for %s...\n' % user) - page_message = u' Got %%%%total_items%%%% folders for %s...\n' % user - feed = callGAPIpages(drive.files(), u'list', u'items', page_message=page_message, - q=query, fields=u'items(title,id),nextPageToken', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) + sys.stderr.write('Getting folders for %s...\n' % user) + page_message = ' Got %%%%total_items%%%% folders for %s...\n' % user + feed = callGAPIpages(drive.files(), 'list', 'items', page_message=page_message, + q=query, fields='items(title,id),nextPageToken', maxResults=GC_Values[GC_DRIVE_MAX_RESULTS]) deleted_empty = False for folder in feed: - children = callGAPI(drive.children(), u'list', - folderId=folder[u'id'], fields=u'items(id)', maxResults=1) - if not u'items' in children or len(children[u'items']) == 0: - print utils.convertUTF8(u' deleting empty folder %s...' % folder[u'title']) - callGAPI(drive.files(), u'delete', fileId=folder[u'id']) + children = callGAPI(drive.children(), 'list', + folderId=folder['id'], fields='items(id)', maxResults=1) + if not 'items' in children or len(children['items']) == 0: + print(utils.convertUTF8(' deleting empty folder %s...' % folder['title'])) + callGAPI(drive.files(), 'delete', fileId=folder['id']) deleted_empty = True else: - print utils.convertUTF8(u' not deleting folder %s because it contains at least 1 item (%s)' % (folder[u'title'], children[u'items'][0][u'id'])) + print(utils.convertUTF8(' not deleting folder %s because it contains at least 1 item (%s)' % (folder['title'], children['items'][0]['id']))) def doEmptyDriveTrash(users): for user in users: user, drive = buildDrive3GAPIObject(user) if not drive: continue - print u'Emptying Drive trash for %s' % user - callGAPI(drive.files(), u'emptyTrash') + print('Emptying Drive trash for %s' % user) + callGAPI(drive.files(), 'emptyTrash') def escapeDriveFileName(filename): - if filename.find(u"'") == -1 and filename.find(u'\\') == -1: + if filename.find("'") == -1 and filename.find('\\') == -1: return filename - encfilename = u'' + encfilename = '' for c in filename: - if c == u"'": - encfilename += u"\\'" - elif c == u'\\': - encfilename += u'\\\\' + if c == "'": + encfilename += "\\'" + elif c == '\\': + encfilename += '\\\\' else: encfilename += c return encfilename @@ -4534,93 +4544,93 @@ def initializeDriveFileAttributes(): return ({}, {DFA_LOCALFILEPATH: None, DFA_LOCALFILENAME: None, DFA_LOCALMIMETYPE: None, DFA_CONVERT: None, DFA_OCR: None, DFA_OCRLANGUAGE: None, DFA_PARENTQUERY: None}) def getDriveFileAttribute(i, body, parameters, myarg, update=False): - if myarg == u'localfile': + if myarg == 'localfile': parameters[DFA_LOCALFILEPATH] = sys.argv[i+1] parameters[DFA_LOCALFILENAME] = os.path.basename(parameters[DFA_LOCALFILEPATH]) - body.setdefault(u'title', parameters[DFA_LOCALFILENAME]) - body[u'mimeType'] = mimetypes.guess_type(parameters[DFA_LOCALFILEPATH])[0] - if body[u'mimeType'] is None: - body[u'mimeType'] = u'application/octet-stream' - parameters[DFA_LOCALMIMETYPE] = body[u'mimeType'] + body.setdefault('title', parameters[DFA_LOCALFILENAME]) + body['mimeType'] = mimetypes.guess_type(parameters[DFA_LOCALFILEPATH])[0] + if body['mimeType'] is None: + body['mimeType'] = 'application/octet-stream' + parameters[DFA_LOCALMIMETYPE] = body['mimeType'] i += 2 - elif myarg == u'convert': + elif myarg == 'convert': parameters[DFA_CONVERT] = True i += 1 - elif myarg == u'ocr': + elif myarg == 'ocr': parameters[DFA_OCR] = True i += 1 - elif myarg == u'ocrlanguage': + elif myarg == 'ocrlanguage': parameters[DFA_OCRLANGUAGE] = LANGUAGE_CODES_MAP.get(sys.argv[i+1].lower(), sys.argv[i+1]) i += 2 elif myarg in DRIVEFILE_LABEL_CHOICES_MAP: - body.setdefault(u'labels', {}) + body.setdefault('labels', {}) if update: - body[u'labels'][DRIVEFILE_LABEL_CHOICES_MAP[myarg]] = getBoolean(sys.argv[i+1], myarg) + body['labels'][DRIVEFILE_LABEL_CHOICES_MAP[myarg]] = getBoolean(sys.argv[i+1], myarg) i += 2 else: - body[u'labels'][DRIVEFILE_LABEL_CHOICES_MAP[myarg]] = True + body['labels'][DRIVEFILE_LABEL_CHOICES_MAP[myarg]] = True i += 1 - elif myarg in [u'lastviewedbyme', u'lastviewedbyuser', u'lastviewedbymedate', u'lastviewedbymetime']: - body[u'lastViewedByMeDate'] = getTimeOrDeltaFromNow(sys.argv[i+1]) + elif myarg in ['lastviewedbyme', 'lastviewedbyuser', 'lastviewedbymedate', 'lastviewedbymetime']: + body['lastViewedByMeDate'] = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg in [u'modifieddate', u'modifiedtime']: - body[u'modifiedDate'] = getTimeOrDeltaFromNow(sys.argv[i+1]) + elif myarg in ['modifieddate', 'modifiedtime']: + body['modifiedDate'] = getTimeOrDeltaFromNow(sys.argv[i+1]) i += 2 - elif myarg == u'description': - body[u'description'] = sys.argv[i+1] + elif myarg == 'description': + body['description'] = sys.argv[i+1] i += 2 - elif myarg == u'mimetype': + elif myarg == 'mimetype': mimeType = sys.argv[i+1] if mimeType in MIMETYPE_CHOICES_MAP: - body[u'mimeType'] = MIMETYPE_CHOICES_MAP[mimeType] + body['mimeType'] = MIMETYPE_CHOICES_MAP[mimeType] else: - systemErrorExit(2, 'mimetype must be one of %s; got %s"' % (u', '.join(MIMETYPE_CHOICES_MAP), mimeType)) + systemErrorExit(2, 'mimetype must be one of %s; got %s"' % (', '.join(MIMETYPE_CHOICES_MAP), mimeType)) i += 2 - elif myarg == u'parentid': - body.setdefault(u'parents', []) - body[u'parents'].append({u'id': sys.argv[i+1]}) + elif myarg == 'parentid': + body.setdefault('parents', []) + body['parents'].append({'id': sys.argv[i+1]}) i += 2 - elif myarg == u'parentname': - parameters[DFA_PARENTQUERY] = u"'me' in owners and mimeType = '%s' and title = '%s'" % (MIMETYPE_GA_FOLDER, escapeDriveFileName(sys.argv[i+1])) + elif myarg == 'parentname': + parameters[DFA_PARENTQUERY] = "'me' in owners and mimeType = '%s' and title = '%s'" % (MIMETYPE_GA_FOLDER, escapeDriveFileName(sys.argv[i+1])) i += 2 - elif myarg in [u'anyownerparentname']: - parameters[DFA_PARENTQUERY] = u"mimeType = '%s' and title = '%s'" % (MIMETYPE_GA_FOLDER, escapeDriveFileName(sys.argv[i+1])) + elif myarg in ['anyownerparentname']: + parameters[DFA_PARENTQUERY] = "mimeType = '%s' and title = '%s'" % (MIMETYPE_GA_FOLDER, escapeDriveFileName(sys.argv[i+1])) i += 2 - elif myarg == u'writerscantshare': - body[u'writersCanShare'] = False + elif myarg == 'writerscantshare': + body['writersCanShare'] = False i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s drivefile"' % (myarg, [u'add', u'update'][update])) + systemErrorExit(2, '%s is not a valid argument for "gam %s drivefile"' % (myarg, ['add', 'update'][update])) return i def doUpdateDriveFile(users): - fileIdSelection = {u'fileIds': [], u'query': None} + fileIdSelection = {'fileIds': [], 'query': None} media_body = None - operation = u'update' + operation = 'update' body, parameters = initializeDriveFileAttributes() i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'copy': - operation = u'copy' + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'copy': + operation = 'copy' i += 1 - elif myarg == u'newfilename': - body[u'title'] = sys.argv[i+1] + elif myarg == 'newfilename': + body['title'] = sys.argv[i+1] i += 2 - elif myarg == u'id': - fileIdSelection[u'fileIds'] = [sys.argv[i+1],] + elif myarg == 'id': + fileIdSelection['fileIds'] = [sys.argv[i+1],] i += 2 - elif myarg == u'query': - fileIdSelection[u'query'] = sys.argv[i+1] + elif myarg == 'query': + fileIdSelection['query'] = sys.argv[i+1] i += 2 - elif myarg == u'drivefilename': - fileIdSelection[u'query'] = u"'me' in owners and title = '{0}'".format(sys.argv[i+1]) + elif myarg == 'drivefilename': + fileIdSelection['query'] = "'me' in owners and title = '{0}'".format(sys.argv[i+1]) i += 2 else: i = getDriveFileAttribute(i, body, parameters, myarg, True) - if not fileIdSelection[u'query'] and not fileIdSelection[u'fileIds']: + if not fileIdSelection['query'] and not fileIdSelection['fileIds']: systemErrorExit(2, 'you need to specify either id, query or drivefilename in order to determine the file(s) to update') - if fileIdSelection[u'query'] and fileIdSelection[u'fileIds']: + if fileIdSelection['query'] and fileIdSelection['fileIds']: systemErrorExit(2, 'you cannot specify multiple file identifiers. Choose one of id, drivefilename, query.') for user in users: user, drive = buildDriveGAPIObject(user) @@ -4628,58 +4638,58 @@ def doUpdateDriveFile(users): continue if parameters[DFA_PARENTQUERY]: more_parents = doDriveSearch(drive, query=parameters[DFA_PARENTQUERY]) - body.setdefault(u'parents', []) + body.setdefault('parents', []) for a_parent in more_parents: - body[u'parents'].append({u'id': a_parent}) - if fileIdSelection[u'query']: - fileIdSelection[u'fileIds'] = doDriveSearch(drive, query=fileIdSelection[u'query']) - if not fileIdSelection[u'fileIds']: - print u'No files to %s for %s' % (operation, user) + body['parents'].append({'id': a_parent}) + if fileIdSelection['query']: + fileIdSelection['fileIds'] = doDriveSearch(drive, query=fileIdSelection['query']) + if not fileIdSelection['fileIds']: + print('No files to %s for %s' % (operation, user)) continue - if operation == u'update': + if operation == 'update': if parameters[DFA_LOCALFILEPATH]: media_body = googleapiclient.http.MediaFileUpload(parameters[DFA_LOCALFILEPATH], mimetype=parameters[DFA_LOCALMIMETYPE], resumable=True) - for fileId in fileIdSelection[u'fileIds']: + for fileId in fileIdSelection['fileIds']: if media_body: - result = callGAPI(drive.files(), u'update', + result = callGAPI(drive.files(), 'update', fileId=fileId, convert=parameters[DFA_CONVERT], ocr=parameters[DFA_OCR], ocrLanguage=parameters[DFA_OCRLANGUAGE], - media_body=media_body, body=body, fields=u'id', + media_body=media_body, body=body, fields='id', supportsTeamDrives=True) - print u'Successfully updated %s drive file with content from %s' % (result[u'id'], parameters[DFA_LOCALFILENAME]) + print('Successfully updated %s drive file with content from %s' % (result['id'], parameters[DFA_LOCALFILENAME])) else: - result = callGAPI(drive.files(), u'patch', + result = callGAPI(drive.files(), 'patch', fileId=fileId, convert=parameters[DFA_CONVERT], ocr=parameters[DFA_OCR], ocrLanguage=parameters[DFA_OCRLANGUAGE], body=body, - fields=u'id', supportsTeamDrives=True) - print u'Successfully updated drive file/folder ID %s' % (result[u'id']) + fields='id', supportsTeamDrives=True) + print('Successfully updated drive file/folder ID %s' % (result['id'])) else: - for fileId in fileIdSelection[u'fileIds']: - result = callGAPI(drive.files(), u'copy', + for fileId in fileIdSelection['fileIds']: + result = callGAPI(drive.files(), 'copy', fileId=fileId, convert=parameters[DFA_CONVERT], ocr=parameters[DFA_OCR], ocrLanguage=parameters[DFA_OCRLANGUAGE], - body=body, fields=u'id', supportsTeamDrives=True) - print u'Successfully copied %s to %s' % (fileId, result[u'id']) + body=body, fields='id', supportsTeamDrives=True) + print('Successfully copied %s to %s' % (fileId, result['id'])) def createDriveFile(users): csv_output = to_drive = False csv_rows = [] - csv_titles = [u'User', u'title', u'id'] + csv_titles = ['User', 'title', 'id'] media_body = None body, parameters = initializeDriveFileAttributes() i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'drivefilename': - body[u'title'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'drivefilename': + body['title'] = sys.argv[i+1] i += 2 - elif myarg == u'csv': + elif myarg == 'csv': csv_output = True i += 1 - elif myarg == u'todrive': + elif myarg == 'todrive': to_drive = True i += 1 else: @@ -4690,34 +4700,34 @@ def createDriveFile(users): continue if parameters[DFA_PARENTQUERY]: more_parents = doDriveSearch(drive, query=parameters[DFA_PARENTQUERY]) - body.setdefault(u'parents', []) + body.setdefault('parents', []) for a_parent in more_parents: - body[u'parents'].append({u'id': a_parent}) + body['parents'].append({'id': a_parent}) if parameters[DFA_LOCALFILEPATH]: media_body = googleapiclient.http.MediaFileUpload(parameters[DFA_LOCALFILEPATH], mimetype=parameters[DFA_LOCALMIMETYPE], resumable=True) - result = callGAPI(drive.files(), u'insert', + result = callGAPI(drive.files(), 'insert', convert=parameters[DFA_CONVERT], ocr=parameters[DFA_OCR], ocrLanguage=parameters[DFA_OCRLANGUAGE], - media_body=media_body, body=body, fields=u'id,title,mimeType', + media_body=media_body, body=body, fields='id,title,mimeType', supportsTeamDrives=True) - titleInfo = u'{0}({1})'.format(result[u'title'], result[u'id']) + titleInfo = '{0}({1})'.format(result['title'], result['id']) if csv_output: - csv_rows.append({u'User': user, u'title': result[u'title'], u'id': result[u'id']}) + csv_rows.append({'User': user, 'title': result['title'], 'id': result['id']}) else: if parameters[DFA_LOCALFILENAME]: - print u'Successfully uploaded %s to Drive File %s' % (parameters[DFA_LOCALFILENAME], titleInfo) + print('Successfully uploaded %s to Drive File %s' % (parameters[DFA_LOCALFILENAME], titleInfo)) else: - print u'Successfully created Drive %s %s' % ([u'Folder', u'File'][result[u'mimeType'] != MIMETYPE_GA_FOLDER], titleInfo) + print('Successfully created Drive %s %s' % (['Folder', 'File'][result['mimeType'] != MIMETYPE_GA_FOLDER], titleInfo)) if csv_output: - writeCSVfile(csv_rows, csv_titles, u'Files', to_drive) + writeCSVfile(csv_rows, csv_titles, 'Files', to_drive) HTTP_ERROR_PATTERN = re.compile(r'^.*returned "(.*)">$') def downloadDriveFile(users): i = 5 - fileIdSelection = {u'fileIds': [], u'query': None} + fileIdSelection = {'fileIds': [], 'query': None} csvSheetTitle = revisionId = None - exportFormatName = u'openoffice' + exportFormatName = 'openoffice' exportFormatChoices = [exportFormatName] exportFormats = DOCUMENT_FORMATS_MAP[exportFormatName] targetFolder = GC_Values[GC_DRIVE_DIR] @@ -4725,55 +4735,55 @@ def downloadDriveFile(users): overwrite = showProgress = targetStdout = False safe_filename_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'id': - fileIdSelection[u'fileIds'] = [sys.argv[i+1],] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'id': + fileIdSelection['fileIds'] = [sys.argv[i+1],] i += 2 - elif myarg == u'query': - fileIdSelection[u'query'] = sys.argv[i+1] + elif myarg == 'query': + fileIdSelection['query'] = sys.argv[i+1] i += 2 - elif myarg == u'drivefilename': - fileIdSelection[u'query'] = u"'me' in owners and title = '{0}'".format(sys.argv[i+1]) + elif myarg == 'drivefilename': + fileIdSelection['query'] = "'me' in owners and title = '{0}'".format(sys.argv[i+1]) i += 2 - elif myarg == u'revision': + elif myarg == 'revision': revisionId = getInteger(sys.argv[i+1], myarg, minVal=1) i += 2 - elif myarg == u'csvsheet': + elif myarg == 'csvsheet': csvSheetTitle = sys.argv[i+1] csvSheetTitleLower = csvSheetTitle.lower() i += 2 - elif myarg == u'format': - exportFormatChoices = sys.argv[i+1].replace(u',', u' ').lower().split() + elif myarg == 'format': + exportFormatChoices = sys.argv[i+1].replace(',', ' ').lower().split() exportFormats = [] for exportFormat in exportFormatChoices: if exportFormat in DOCUMENT_FORMATS_MAP: exportFormats.extend(DOCUMENT_FORMATS_MAP[exportFormat]) else: - systemErrorExit(2, 'format must be one of {0}; got {1}'.format(u', '.join(DOCUMENT_FORMATS_MAP), exportFormat)) + systemErrorExit(2, 'format must be one of {0}; got {1}'.format(', '.join(DOCUMENT_FORMATS_MAP), exportFormat)) i += 2 - elif myarg == u'targetfolder': + elif myarg == 'targetfolder': targetFolder = os.path.expanduser(sys.argv[i+1]) if not os.path.isdir(targetFolder): os.makedirs(targetFolder) i += 2 - elif myarg == u'targetname': + elif myarg == 'targetname': targetName = sys.argv[i+1] - targetStdout = targetName == u'-' + targetStdout = targetName == '-' i += 2 - elif myarg == u'overwrite': + elif myarg == 'overwrite': overwrite = True i += 1 - elif myarg == u'showprogress': + elif myarg == 'showprogress': showProgress = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam get drivefile"' % sys.argv[i]) - if not fileIdSelection[u'query'] and not fileIdSelection[u'fileIds']: + if not fileIdSelection['query'] and not fileIdSelection['fileIds']: systemErrorExit(2, 'you need to specify either id, query or drivefilename in order to determine the file(s) to download') - if fileIdSelection[u'query'] and fileIdSelection[u'fileIds']: + if fileIdSelection['query'] and fileIdSelection['fileIds']: systemErrorExit(2, 'you cannot specify multiple file identifiers. Choose one of id, drivefilename, query.') if csvSheetTitle: - exportFormatName = u'csv' + exportFormatName = 'csv' exportFormatChoices = [exportFormatName] exportFormats = DOCUMENT_FORMATS_MAP[exportFormatName] for user in users: @@ -4781,55 +4791,55 @@ def downloadDriveFile(users): if not drive: continue if csvSheetTitle: - sheet = buildGAPIServiceObject(u'sheets', user) + sheet = buildGAPIServiceObject('sheets', user) if not sheet: continue - if fileIdSelection[u'query']: - fileIdSelection[u'fileIds'] = doDriveSearch(drive, query=fileIdSelection[u'query'], quiet=targetStdout) + if fileIdSelection['query']: + fileIdSelection['fileIds'] = doDriveSearch(drive, query=fileIdSelection['query'], quiet=targetStdout) else: - fileId = fileIdSelection[u'fileIds'][0] - if fileId[:8].lower() == u'https://' or fileId[:7].lower() == u'http://': - fileIdSelection[u'fileIds'][0] = getFileIdFromAlternateLink(fileId) - if not fileIdSelection[u'fileIds']: - print u'No files to download for %s' % user + fileId = fileIdSelection['fileIds'][0] + if fileId[:8].lower() == 'https://' or fileId[:7].lower() == 'http://': + fileIdSelection['fileIds'][0] = getFileIdFromAlternateLink(fileId) + if not fileIdSelection['fileIds']: + print('No files to download for %s' % user) i = 0 - for fileId in fileIdSelection[u'fileIds']: + for fileId in fileIdSelection['fileIds']: fileExtension = None - result = callGAPI(drive.files(), u'get', - fileId=fileId, fields=u'fileExtension,fileSize,mimeType,title', supportsTeamDrives=True) - fileExtension = result.get(u'fileExtension') - mimeType = result[u'mimeType'] + result = callGAPI(drive.files(), 'get', + fileId=fileId, fields='fileExtension,fileSize,mimeType,title', supportsTeamDrives=True) + fileExtension = result.get('fileExtension') + mimeType = result['mimeType'] if mimeType == MIMETYPE_GA_FOLDER: - print utils.convertUTF8(u'Skipping download of folder %s' % result[u'title']) + print(utils.convertUTF8('Skipping download of folder %s' % result['title'])) continue if mimeType in NON_DOWNLOADABLE_MIMETYPES: - print utils.convertUTF8(u'Format of file %s not downloadable' % result[u'title']) + print(utils.convertUTF8('Format of file %s not downloadable' % result['title'])) continue validExtensions = GOOGLEDOC_VALID_EXTENSIONS_MAP.get(mimeType) if validExtensions: - my_line = u'Downloading Google Doc: %s' + my_line = 'Downloading Google Doc: %s' if csvSheetTitle: - my_line += u', Sheet: %s' % csvSheetTitle + my_line += ', Sheet: %s' % csvSheetTitle googleDoc = True else: - if u'fileSize' in result: - my_line = u'Downloading: %%s of %s bytes' % utils.formatFileSize(int(result[u'fileSize'])) + if 'fileSize' in result: + my_line = 'Downloading: %%s of %s bytes' % utils.formatFileSize(int(result['fileSize'])) else: - my_line = u'Downloading: %s of unknown size' + my_line = 'Downloading: %s of unknown size' googleDoc = False - my_line += u' to %s' + my_line += ' to %s' csvSheetNotFound = fileDownloaded = fileDownloadFailed = False for exportFormat in exportFormats: - extension = fileExtension or exportFormat[u'ext'] + extension = fileExtension or exportFormat['ext'] if googleDoc and (extension not in validExtensions): continue if targetStdout: - filename = u'stdout' + filename = 'stdout' else: if targetName: safe_file_title = targetName else: - safe_file_title = u''.join(c for c in result[u'title'] if c in safe_filename_chars) + safe_file_title = ''.join(c for c in result['title'] if c in safe_filename_chars) if len(safe_file_title) < 1: safe_file_title = fileId filename = os.path.join(targetFolder, safe_file_title) @@ -4840,43 +4850,43 @@ def downloadDriveFile(users): if overwrite or not os.path.isfile(filename): break y += 1 - filename = os.path.join(targetFolder, u'({0})-{1}'.format(y, safe_file_title)) - print utils.convertUTF8(my_line % (result[u'title'], filename)) + filename = os.path.join(targetFolder, '({0})-{1}'.format(y, safe_file_title)) + print(utils.convertUTF8(my_line % (result['title'], filename))) spreadsheetUrl = None if googleDoc: if csvSheetTitle is None or mimeType != MIMETYPE_GA_SPREADSHEET: - request = drive.files().export_media(fileId=fileId, mimeType=exportFormat[u'mime']) + request = drive.files().export_media(fileId=fileId, mimeType=exportFormat['mime']) if revisionId: - request.uri = u'{0}&revision={1}'.format(request.uri, revisionId) + request.uri = '{0}&revision={1}'.format(request.uri, revisionId) else: - spreadsheet = callGAPI(sheet.spreadsheets(), u'get', - spreadsheetId=fileId, fields=u'spreadsheetUrl,sheets(properties(sheetId,title))') - for sheet in spreadsheet[u'sheets']: - if sheet[u'properties'][u'title'].lower() == csvSheetTitleLower: - spreadsheetUrl = u'{0}?format=csv&id={1}&gid={2}'.format(re.sub(u'/edit$', u'/export', spreadsheet[u'spreadsheetUrl']), - fileId, sheet[u'properties'][u'sheetId']) + spreadsheet = callGAPI(sheet.spreadsheets(), 'get', + spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))') + for sheet in spreadsheet['sheets']: + if sheet['properties']['title'].lower() == csvSheetTitleLower: + spreadsheetUrl = '{0}?format=csv&id={1}&gid={2}'.format(re.sub('/edit$', '/export', spreadsheet['spreadsheetUrl']), + fileId, sheet['properties']['sheetId']) break else: - stderrErrorMsg(u'Google Doc: %s, Sheet: %s, does not exist' % (result[u'title'], csvSheetTitle)) + stderrErrorMsg('Google Doc: %s, Sheet: %s, does not exist' % (result['title'], csvSheetTitle)) csvSheetNotFound = True continue else: request = drive.files().get_media(fileId=fileId, revisionId=revisionId) fh = None try: - fh = open(filename, u'wb') if not targetStdout else sys.stdout + fh = open(filename, 'wb') if not targetStdout else sys.stdout if not spreadsheetUrl: downloader = googleapiclient.http.MediaIoBaseDownload(fh, request) done = False while not done: status, done = downloader.next_chunk() if showProgress: - print u'Downloaded: {0:>7.2%}'.format(status.progress()) + print('Downloaded: {0:>7.2%}'.format(status.progress())) else: _, content = drive._http.request(uri=spreadsheetUrl, method='GET') fh.write(content) - if targetStdout and content[-1] != u'\n': - fh.write(u'\n') + if targetStdout and content[-1] != '\n': + fh.write('\n') if not targetStdout: closeFile(fh) fileDownloaded = True @@ -4898,7 +4908,7 @@ def downloadDriveFile(users): closeFile(fh) os.remove(filename) if not fileDownloaded and not fileDownloadFailed and not csvSheetNotFound: - stderrErrorMsg(u'Format ({0}) not available'.format(u','.join(exportFormatChoices))) + stderrErrorMsg('Format ({0}) not available'.format(','.join(exportFormatChoices))) GM_Globals[GM_SYSEXITRC] = 51 def showDriveFileInfo(users): @@ -4907,8 +4917,8 @@ def showDriveFileInfo(users): fileId = sys.argv[5] i = 6 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'allfields': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'allfields': fieldsList = [] i += 1 elif myarg in DRIVEFILE_FIELDS_CHOICES_MAP: @@ -4920,17 +4930,17 @@ def showDriveFileInfo(users): else: systemErrorExit(2, '%s is not a valid argument for "gam show fileinfo"' % myarg) if fieldsList or labelsList: - fieldsList.append(u'title') - fields = u','.join(set(fieldsList)) + fieldsList.append('title') + fields = ','.join(set(fieldsList)) if labelsList: - fields += u',labels({0})'.format(u','.join(set(labelsList))) + fields += ',labels({0})'.format(','.join(set(labelsList))) else: - fields = u'*' + fields = '*' for user in users: user, drive = buildDriveGAPIObject(user) if not drive: continue - feed = callGAPI(drive.files(), u'get', fileId=fileId, fields=fields, supportsTeamDrives=True) + feed = callGAPI(drive.files(), 'get', fileId=fileId, fields=fields, supportsTeamDrives=True) if feed: print_json(None, feed) @@ -4940,7 +4950,7 @@ def showDriveFileRevisions(users): user, drive = buildDriveGAPIObject(user) if not drive: continue - feed = callGAPI(drive.revisions(), u'list', fileId=fileId) + feed = callGAPI(drive.revisions(), 'list', fileId=fileId) if feed: print_json(None, feed) @@ -4949,11 +4959,11 @@ def transferSecCals(users): remove_source_user = sendNotifications = True i = 6 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'keepuser': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'keepuser': remove_source_user = False i += 1 - elif myarg == u'sendnotifications': + elif myarg == 'sendnotifications': sendNotifications = getBoolean(sys.argv[i+1], myarg) i += 2 else: @@ -4966,16 +4976,16 @@ def transferSecCals(users): user, source_cal = buildCalendarGAPIObject(user) if not source_cal: continue - calendars = callGAPIpages(source_cal.calendarList(), u'list', u'items', soft_errors=True, - minAccessRole=u'owner', showHidden=True, fields=u'items(id),nextPageToken') + calendars = callGAPIpages(source_cal.calendarList(), 'list', 'items', soft_errors=True, + minAccessRole='owner', showHidden=True, fields='items(id),nextPageToken') for calendar in calendars: - calendarId = calendar[u'id'] - if calendarId.find(u'@group.calendar.google.com') != -1: - callGAPI(source_cal.acl(), u'insert', calendarId=calendarId, - body={u'role': u'owner', u'scope': {u'type': u'user', u'value': target_user}}, sendNotifications=sendNotifications) + calendarId = calendar['id'] + if calendarId.find('@group.calendar.google.com') != -1: + callGAPI(source_cal.acl(), 'insert', calendarId=calendarId, + body={'role': 'owner', 'scope': {'type': 'user', 'value': target_user}}, sendNotifications=sendNotifications) if remove_source_user: - callGAPI(target_cal.acl(), u'insert', calendarId=calendarId, - body={u'role': u'none', u'scope': {u'type': u'user', u'value': user}}, sendNotifications=sendNotifications) + callGAPI(target_cal.acl(), 'insert', calendarId=calendarId, + body={'role': 'none', 'scope': {'type': 'user', 'value': user}}, sendNotifications=sendNotifications) def transferDriveFiles(users): target_user = sys.argv[5] @@ -4983,7 +4993,7 @@ def transferDriveFiles(users): i = 6 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'keepuser': + if myarg == 'keepuser': remove_source_user = False i += 1 else: @@ -4991,9 +5001,9 @@ def transferDriveFiles(users): target_user, target_drive = buildDriveGAPIObject(target_user) if not target_drive: return - target_about = callGAPI(target_drive.about(), u'get', fields=u'quotaType,quotaBytesTotal,quotaBytesUsed') - if target_about[u'quotaType'] != u'UNLIMITED': - target_drive_free = int(target_about[u'quotaBytesTotal']) - int(target_about[u'quotaBytesUsed']) + target_about = callGAPI(target_drive.about(), 'get', fields='quotaType,quotaBytesTotal,quotaBytesUsed') + if target_about['quotaType'] != 'UNLIMITED': + target_drive_free = int(target_about['quotaBytesTotal']) - int(target_about['quotaBytesUsed']) else: target_drive_free = None for user in users: @@ -5001,103 +5011,103 @@ def transferDriveFiles(users): if not source_drive: continue counter = 0 - source_about = callGAPI(source_drive.about(), u'get', fields=u'quotaBytesTotal,quotaBytesUsed,rootFolderId,permissionId') - source_drive_size = int(source_about[u'quotaBytesUsed']) + source_about = callGAPI(source_drive.about(), 'get', fields='quotaBytesTotal,quotaBytesUsed,rootFolderId,permissionId') + source_drive_size = int(source_about['quotaBytesUsed']) if target_drive_free is not None: if target_drive_free < source_drive_size: systemErrorExit(4, MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE.format(source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024)) - print u'Source drive size: %smb Target drive free: %smb' % (source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024) + print('Source drive size: %smb Target drive free: %smb' % (source_drive_size / 1024 / 1024, target_drive_free / 1024 / 1024)) target_drive_free = target_drive_free - source_drive_size # prep target_drive_free for next user else: - print u'Source drive size: %smb Target drive free: UNLIMITED' % (source_drive_size / 1024 / 1024) - source_root = source_about[u'rootFolderId'] - source_permissionid = source_about[u'permissionId'] - print u"Getting file list for source user: %s..." % user - page_message = u' Got %%total_items%% files\n' - source_drive_files = callGAPIpages(source_drive.files(), u'list', u'items', page_message=page_message, - q=u"'me' in owners and trashed = false", fields=u'items(id,parents,mimeType),nextPageToken') + print('Source drive size: %smb Target drive free: UNLIMITED' % (source_drive_size / 1024 / 1024)) + source_root = source_about['rootFolderId'] + source_permissionid = source_about['permissionId'] + print("Getting file list for source user: %s..." % user) + page_message = ' Got %%total_items%% files\n' + source_drive_files = callGAPIpages(source_drive.files(), 'list', 'items', page_message=page_message, + q="'me' in owners and trashed = false", fields='items(id,parents,mimeType),nextPageToken') all_source_file_ids = [] for source_drive_file in source_drive_files: - all_source_file_ids.append(source_drive_file[u'id']) + all_source_file_ids.append(source_drive_file['id']) total_count = len(source_drive_files) - print u"Getting folder list for target user: %s..." % target_user - page_message = u' Got %%total_items%% folders\n' - target_folders = callGAPIpages(target_drive.files(), u'list', u'items', page_message=page_message, - q=u"'me' in owners and mimeType = 'application/vnd.google-apps.folder'", fields=u'items(id,title),nextPageToken') + print("Getting folder list for target user: %s..." % target_user) + page_message = ' Got %%total_items%% folders\n' + target_folders = callGAPIpages(target_drive.files(), 'list', 'items', page_message=page_message, + q="'me' in owners and mimeType = 'application/vnd.google-apps.folder'", fields='items(id,title),nextPageToken') got_top_folder = False all_target_folder_ids = [] for target_folder in target_folders: - all_target_folder_ids.append(target_folder[u'id']) - if (not got_top_folder) and target_folder[u'title'] == u'%s old files' % user: - target_top_folder = target_folder[u'id'] + all_target_folder_ids.append(target_folder['id']) + if (not got_top_folder) and target_folder['title'] == '%s old files' % user: + target_top_folder = target_folder['id'] got_top_folder = True if not got_top_folder: - create_folder = callGAPI(target_drive.files(), u'insert', body={u'title': u'%s old files' % user, u'mimeType': u'application/vnd.google-apps.folder'}, fields=u'id') - target_top_folder = create_folder[u'id'] + create_folder = callGAPI(target_drive.files(), 'insert', body={'title': '%s old files' % user, 'mimeType': 'application/vnd.google-apps.folder'}, fields='id') + target_top_folder = create_folder['id'] transferred_files = [] while True: # we loop thru, skipping files until all of their parents are done skipped_files = False for drive_file in source_drive_files: - file_id = drive_file[u'id'] + file_id = drive_file['id'] if file_id in transferred_files: continue - source_parents = drive_file[u'parents'] + source_parents = drive_file['parents'] skip_file_for_now = False for source_parent in source_parents: - if source_parent[u'id'] not in all_source_file_ids and source_parent[u'id'] not in all_target_folder_ids: + if source_parent['id'] not in all_source_file_ids and source_parent['id'] not in all_target_folder_ids: continue # means this parent isn't owned by source or target, shouldn't matter - if source_parent[u'id'] not in transferred_files and source_parent[u'id'] != source_root: + if source_parent['id'] not in transferred_files and source_parent['id'] != source_root: #print u'skipping %s' % file_id skipped_files = skip_file_for_now = True break if skip_file_for_now: continue else: - transferred_files.append(drive_file[u'id']) + transferred_files.append(drive_file['id']) counter += 1 - print u'Changing owner for file %s (%s/%s)' % (drive_file[u'id'], counter, total_count) - body = {u'role': u'owner', u'type': u'user', u'value': target_user} - callGAPI(source_drive.permissions(), u'insert', soft_errors=True, fileId=file_id, sendNotificationEmails=False, body=body) + print('Changing owner for file %s (%s/%s)' % (drive_file['id'], counter, total_count)) + body = {'role': 'owner', 'type': 'user', 'value': target_user} + callGAPI(source_drive.permissions(), 'insert', soft_errors=True, fileId=file_id, sendNotificationEmails=False, body=body) target_parents = [] for parent in source_parents: try: - if parent[u'isRoot']: - target_parents.append({u'id': target_top_folder}) + if parent['isRoot']: + target_parents.append({'id': target_top_folder}) else: - target_parents.append({u'id': parent[u'id']}) + target_parents.append({'id': parent['id']}) except TypeError: pass if not target_parents: - target_parents.append({u'id': target_top_folder}) - callGAPI(target_drive.files(), u'patch', soft_errors=True, retry_reasons=[u'notFound'], fileId=file_id, body={u'parents': target_parents}) + target_parents.append({'id': target_top_folder}) + callGAPI(target_drive.files(), 'patch', soft_errors=True, retry_reasons=['notFound'], fileId=file_id, body={'parents': target_parents}) if remove_source_user: - callGAPI(target_drive.permissions(), u'delete', soft_errors=True, fileId=file_id, permissionId=source_permissionid) + callGAPI(target_drive.permissions(), 'delete', soft_errors=True, fileId=file_id, permissionId=source_permissionid) if not skipped_files: break def doImap(users): - enable = getBoolean(sys.argv[4], u'gam imap') - body = {u'enabled': enable, u'autoExpunge': True, u'expungeBehavior': u'archive', u'maxFolderSize': 0} + enable = getBoolean(sys.argv[4], 'gam imap') + body = {'enabled': enable, 'autoExpunge': True, 'expungeBehavior': 'archive', 'maxFolderSize': 0} i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'noautoexpunge': - body[u'autoExpunge'] = False + if myarg == 'noautoexpunge': + body['autoExpunge'] = False i += 1 - elif myarg == u'expungebehavior': + elif myarg == 'expungebehavior': opt = sys.argv[i+1].lower() if opt in EMAILSETTINGS_IMAP_EXPUNGE_BEHAVIOR_CHOICES_MAP: - body[u'expungeBehavior'] = EMAILSETTINGS_IMAP_EXPUNGE_BEHAVIOR_CHOICES_MAP[opt] + body['expungeBehavior'] = EMAILSETTINGS_IMAP_EXPUNGE_BEHAVIOR_CHOICES_MAP[opt] i += 2 else: - systemErrorExit(2, 'value for "gam imap expungebehavior" must be one of %s; got %s' % (u', '.join(EMAILSETTINGS_IMAP_EXPUNGE_BEHAVIOR_CHOICES_MAP), opt)) - elif myarg == u'maxfoldersize': + systemErrorExit(2, 'value for "gam imap expungebehavior" must be one of %s; got %s' % (', '.join(EMAILSETTINGS_IMAP_EXPUNGE_BEHAVIOR_CHOICES_MAP), opt)) + elif myarg == 'maxfoldersize': opt = sys.argv[i+1].lower() if opt in EMAILSETTINGS_IMAP_MAX_FOLDER_SIZE_CHOICES: - body[u'maxFolderSize'] = int(opt) + body['maxFolderSize'] = int(opt) i += 2 else: - systemErrorExit(2, 'value for "gam imap maxfoldersize" must be one of %s; got %s' % (u'|'.join(EMAILSETTINGS_IMAP_MAX_FOLDER_SIZE_CHOICES), opt)) + systemErrorExit(2, 'value for "gam imap maxfoldersize" must be one of %s; got %s' % ('|'.join(EMAILSETTINGS_IMAP_MAX_FOLDER_SIZE_CHOICES), opt)) else: systemErrorExit(2, '%s is not a valid argument for "gam imap"' % myarg) i = 0 @@ -5107,10 +5117,10 @@ def doImap(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Setting IMAP Access to %s for %s (%s/%s)" % (str(enable), user, i, count) - callGAPI(gmail.users().settings(), u'updateImap', + print("Setting IMAP Access to %s for %s (%s/%s)" % (str(enable), user, i, count)) + callGAPI(gmail.users().settings(), 'updateImap', soft_errors=True, - userId=u'me', body=body) + userId='me', body=body) def getImap(users): i = 0 @@ -5120,74 +5130,74 @@ def getImap(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings(), u'getImap', + result = callGAPI(gmail.users().settings(), 'getImap', soft_errors=True, - userId=u'me') + userId='me') if result: - enabled = result[u'enabled'] + enabled = result['enabled'] if enabled: - print u'User: {0}, IMAP Enabled: {1}, autoExpunge: {2}, expungeBehavior: {3}, maxFolderSize:{4} ({5}/{6})'.format(user, enabled, result[u'autoExpunge'], result[u'expungeBehavior'], result[u'maxFolderSize'], i, count) + print('User: {0}, IMAP Enabled: {1}, autoExpunge: {2}, expungeBehavior: {3}, maxFolderSize:{4} ({5}/{6})'.format(user, enabled, result['autoExpunge'], result['expungeBehavior'], result['maxFolderSize'], i, count)) else: - print u'User: {0}, IMAP Enabled: {1} ({2}/{3})'.format(user, enabled, i, count) + print('User: {0}, IMAP Enabled: {1} ({2}/{3})'.format(user, enabled, i, count)) def getProductAndSKU(sku): - l_sku = sku.lower().replace(u'-', u'').replace(u' ', u'') - for a_sku, sku_values in SKUS.items(): - if l_sku == a_sku.lower().replace(u'-', u'') or l_sku in sku_values[u'aliases'] or l_sku == sku_values[u'displayName'].lower().replace(u' ', u''): - return (sku_values[u'product'], a_sku) + l_sku = sku.lower().replace('-', '').replace(' ', '') + for a_sku, sku_values in list(SKUS.items()): + if l_sku == a_sku.lower().replace('-', '') or l_sku in sku_values['aliases'] or l_sku == sku_values['displayName'].lower().replace(' ', ''): + return (sku_values['product'], a_sku) try: - product = re.search(u'^([A-Z,a-z]*-[A-Z,a-z]*)', sku).group(1) + product = re.search('^([A-Z,a-z]*-[A-Z,a-z]*)', sku).group(1) except AttributeError: product = sku return (product, sku) def doLicense(users, operation): - lic = buildGAPIObject(u'licensing') + lic = buildGAPIObject('licensing') sku = sys.argv[5] productId, skuId = getProductAndSKU(sku) i = 6 - if len(sys.argv) > 6 and sys.argv[i].lower() in [u'product', u'productid']: + if len(sys.argv) > 6 and sys.argv[i].lower() in ['product', 'productid']: productId = sys.argv[i+1] i += 2 for user in users: - if operation == u'delete': - print u'Removing license %s from user %s' % (_formatSKUIdDisplayName(skuId), user) + if operation == 'delete': + print('Removing license %s from user %s' % (_formatSKUIdDisplayName(skuId), user)) callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=skuId, userId=user) - elif operation == u'insert': - print u'Adding license %s to user %s' % (_formatSKUIdDisplayName(skuId), user) - callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=skuId, body={u'userId': user}) - elif operation == u'patch': + elif operation == 'insert': + print('Adding license %s to user %s' % (_formatSKUIdDisplayName(skuId), user)) + callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=skuId, body={'userId': user}) + elif operation == 'patch': try: old_sku = sys.argv[i] - if old_sku.lower() == u'from': + if old_sku.lower() == 'from': old_sku = sys.argv[i+1] except KeyError: systemErrorExit(2, 'You need to specify the user\'s old SKU as the last argument') _, old_sku = getProductAndSKU(old_sku) - print u'Changing user %s from license %s to %s' % (user, _formatSKUIdDisplayName(old_sku), _formatSKUIdDisplayName(skuId)) - callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=old_sku, userId=user, body={u'skuId': skuId}) + print('Changing user %s from license %s to %s' % (user, _formatSKUIdDisplayName(old_sku), _formatSKUIdDisplayName(skuId))) + callGAPI(lic.licenseAssignments(), operation, soft_errors=True, productId=productId, skuId=old_sku, userId=user, body={'skuId': skuId}) def doPop(users): - enable = getBoolean(sys.argv[4], u'gam pop') - body = {u'accessWindow': [u'disabled', u'allMail'][enable], u'disposition': u'leaveInInbox'} + enable = getBoolean(sys.argv[4], 'gam pop') + body = {'accessWindow': ['disabled', 'allMail'][enable], 'disposition': 'leaveInInbox'} i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'for': + if myarg == 'for': opt = sys.argv[i+1].lower() if opt in EMAILSETTINGS_POP_ENABLE_FOR_CHOICES_MAP: - body[u'accessWindow'] = EMAILSETTINGS_POP_ENABLE_FOR_CHOICES_MAP[opt] + body['accessWindow'] = EMAILSETTINGS_POP_ENABLE_FOR_CHOICES_MAP[opt] i += 2 else: - systemErrorExit(2, 'value for "gam pop for" must be one of %s; got %s' % (u', '.join(EMAILSETTINGS_POP_ENABLE_FOR_CHOICES_MAP), opt)) - elif myarg == u'action': + systemErrorExit(2, 'value for "gam pop for" must be one of %s; got %s' % (', '.join(EMAILSETTINGS_POP_ENABLE_FOR_CHOICES_MAP), opt)) + elif myarg == 'action': opt = sys.argv[i+1].lower() if opt in EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP: - body[u'disposition'] = EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP[opt] + body['disposition'] = EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP[opt] i += 2 else: - systemErrorExit(2, 'value for "gam pop action" must be one of %s; got %s' % (u', '.join(EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP), opt)) - elif myarg == u'confirm': + systemErrorExit(2, 'value for "gam pop action" must be one of %s; got %s' % (', '.join(EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP), opt)) + elif myarg == 'confirm': i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam pop"' % myarg) @@ -5198,10 +5208,10 @@ def doPop(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Setting POP Access to %s for %s (%s/%s)" % (str(enable), user, i, count) - callGAPI(gmail.users().settings(), u'updatePop', + print("Setting POP Access to %s for %s (%s/%s)" % (str(enable), user, i, count)) + callGAPI(gmail.users().settings(), 'updatePop', soft_errors=True, - userId=u'me', body=body) + userId='me', body=body) def getPop(users): i = 0 @@ -5211,43 +5221,43 @@ def getPop(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings(), u'getPop', + result = callGAPI(gmail.users().settings(), 'getPop', soft_errors=True, - userId=u'me') + userId='me') if result: - enabled = result[u'accessWindow'] != u'disabled' + enabled = result['accessWindow'] != 'disabled' if enabled: - print u'User: {0}, POP Enabled: {1}, For: {2}, Action: {3} ({4}/{5})'.format(user, enabled, result[u'accessWindow'], result[u'disposition'], i, count) + print('User: {0}, POP Enabled: {1}, For: {2}, Action: {3} ({4}/{5})'.format(user, enabled, result['accessWindow'], result['disposition'], i, count)) else: - print u'User: {0}, POP Enabled: {1} ({2}/{3})'.format(user, enabled, i, count) + print('User: {0}, POP Enabled: {1} ({2}/{3})'.format(user, enabled, i, count)) -SMTPMSA_DISPLAY_FIELDS = [u'host', u'port', u'securityMode'] +SMTPMSA_DISPLAY_FIELDS = ['host', 'port', 'securityMode'] def _showSendAs(result, j, jcount, formatSig): - if result[u'displayName']: - print utils.convertUTF8(u'SendAs Address: {0} <{1}>{2}'.format(result[u'displayName'], result[u'sendAsEmail'], currentCount(j, jcount))) + if result['displayName']: + print(utils.convertUTF8('SendAs Address: {0} <{1}>{2}'.format(result['displayName'], result['sendAsEmail'], currentCount(j, jcount)))) else: - print utils.convertUTF8(u'SendAs Address: <{0}>{1}'.format(result[u'sendAsEmail'], currentCount(j, jcount))) - if result.get(u'replyToAddress'): - print u' ReplyTo: {0}'.format(result[u'replyToAddress']) - print u' IsPrimary: {0}'.format(result.get(u'isPrimary', False)) - print u' Default: {0}'.format(result.get(u'isDefault', False)) - if not result.get(u'isPrimary', False): - print u' TreatAsAlias: {0}'.format(result.get(u'treatAsAlias', False)) - if u'smtpMsa' in result: + print(utils.convertUTF8('SendAs Address: <{0}>{1}'.format(result['sendAsEmail'], currentCount(j, jcount)))) + if result.get('replyToAddress'): + print(' ReplyTo: {0}'.format(result['replyToAddress'])) + print(' IsPrimary: {0}'.format(result.get('isPrimary', False))) + print(' Default: {0}'.format(result.get('isDefault', False))) + if not result.get('isPrimary', False): + print(' TreatAsAlias: {0}'.format(result.get('treatAsAlias', False))) + if 'smtpMsa' in result: for field in SMTPMSA_DISPLAY_FIELDS: - if field in result[u'smtpMsa']: - print u' smtpMsa.{0}: {1}'.format(field, result[u'smtpMsa'][field]) - if u'verificationStatus' in result: - print u' Verification Status: {0}'.format(result[u'verificationStatus']) - sys.stdout.write(u' Signature:\n ') - signature = result.get(u'signature') + if field in result['smtpMsa']: + print(' smtpMsa.{0}: {1}'.format(field, result['smtpMsa'][field])) + if 'verificationStatus' in result: + print(' Verification Status: {0}'.format(result['verificationStatus'])) + sys.stdout.write(' Signature:\n ') + signature = result.get('signature') if not signature: - signature = u'None' + signature = 'None' if formatSig: - print utils.convertUTF8(utils.indentMultiLineText(utils.dehtml(signature), n=4)) + print(utils.convertUTF8(utils.indentMultiLineText(utils.dehtml(signature), n=4))) else: - print utils.convertUTF8(utils.indentMultiLineText(signature, n=4)) + print(utils.convertUTF8(utils.indentMultiLineText(signature, n=4))) def _processTags(tagReplacements, message): while True: @@ -5255,61 +5265,61 @@ def _processTags(tagReplacements, message): if not match: break if tagReplacements.get(match.group(1)): - message = RT_OPEN_PATTERN.sub(u'', message, count=1) - message = RT_CLOSE_PATTERN.sub(u'', message, count=1) + message = RT_OPEN_PATTERN.sub('', message, count=1) + message = RT_CLOSE_PATTERN.sub('', message, count=1) else: - message = RT_STRIP_PATTERN.sub(u'', message, count=1) + message = RT_STRIP_PATTERN.sub('', message, count=1) while True: match = RT_TAG_REPLACE_PATTERN.search(message) if not match: break - message = re.sub(match.group(0), tagReplacements.get(match.group(1), u''), message) + message = re.sub(match.group(0), tagReplacements.get(match.group(1), ''), message) return message def _processSignature(tagReplacements, signature, html): if signature: - signature = signature.replace(u'\r', u'').replace(u'\\n', u'
') + signature = signature.replace('\r', '').replace('\\n', '
') if tagReplacements: signature = _processTags(tagReplacements, signature) if not html: - signature = signature.replace(u'\n', u'
') + signature = signature.replace('\n', '
') return signature def getSendAsAttributes(i, myarg, body, tagReplacements, command): - if myarg == u'replace': - matchTag = getString(i+1, u'Tag') - matchReplacement = getString(i+2, u'String', minLen=0) + if myarg == 'replace': + matchTag = getString(i+1, 'Tag') + matchReplacement = getString(i+2, 'String', minLen=0) tagReplacements[matchTag] = matchReplacement i += 3 - elif myarg == u'name': - body[u'displayName'] = sys.argv[i+1] + elif myarg == 'name': + body['displayName'] = sys.argv[i+1] i += 2 - elif myarg == u'replyto': - body[u'replyToAddress'] = sys.argv[i+1] + elif myarg == 'replyto': + body['replyToAddress'] = sys.argv[i+1] i += 2 - elif myarg == u'default': - body[u'isDefault'] = True + elif myarg == 'default': + body['isDefault'] = True i += 1 - elif myarg == u'treatasalias': - body[u'treatAsAlias'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'treatasalias': + body['treatAsAlias'] = getBoolean(sys.argv[i+1], myarg) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam %s"' % (sys.argv[i], command)) return i -SMTPMSA_PORTS = [u'25', u'465', u'587'] -SMTPMSA_SECURITY_MODES = [u'none', u'ssl', u'starttls'] -SMTPMSA_REQUIRED_FIELDS = [u'host', u'port', u'username', u'password'] +SMTPMSA_PORTS = ['25', '465', '587'] +SMTPMSA_SECURITY_MODES = ['none', 'ssl', 'starttls'] +SMTPMSA_REQUIRED_FIELDS = ['host', 'port', 'username', 'password'] def addUpdateSendAs(users, i, addCmd): emailAddress = normalizeEmailAddressOrUID(sys.argv[i], noUid=True) i += 1 if addCmd: - command = [u'sendas', u'add sendas'][i == 6] - body = {u'sendAsEmail': emailAddress, u'displayName': sys.argv[i]} + command = ['sendas', 'add sendas'][i == 6] + body = {'sendAsEmail': emailAddress, 'displayName': sys.argv[i]} i += 1 else: - command = u'update sendas' + command = 'update sendas' body = {} signature = None smtpMsa = {} @@ -5317,52 +5327,52 @@ def addUpdateSendAs(users, i, addCmd): html = False while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg in [u'signature', u'sig']: + if myarg in ['signature', 'sig']: signature = sys.argv[i+1] i += 2 - if signature.lower() == u'file': + if signature.lower() == 'file': filename = sys.argv[i] i, encoding = getCharSet(i+1) signature = readFile(filename, encoding=encoding) - elif myarg == u'html': + elif myarg == 'html': html = True i += 1 - elif addCmd and myarg.startswith(u'smtpmsa.'): - if myarg == u'smtpmsa.host': - smtpMsa[u'host'] = sys.argv[i+1] + elif addCmd and myarg.startswith('smtpmsa.'): + if myarg == 'smtpmsa.host': + smtpMsa['host'] = sys.argv[i+1] i += 2 - elif myarg == u'smtpmsa.port': + elif myarg == 'smtpmsa.port': value = sys.argv[i+1].lower() if value not in SMTPMSA_PORTS: - systemErrorExit(2, '{0} must be {1}; got {2}'.format(myarg, u', '.join(SMTPMSA_PORTS), value)) - smtpMsa[u'port'] = int(value) + systemErrorExit(2, '{0} must be {1}; got {2}'.format(myarg, ', '.join(SMTPMSA_PORTS), value)) + smtpMsa['port'] = int(value) i += 2 - elif myarg == u'smtpmsa.username': - smtpMsa[u'username'] = sys.argv[i+1] + elif myarg == 'smtpmsa.username': + smtpMsa['username'] = sys.argv[i+1] i += 2 - elif myarg == u'smtpmsa.password': - smtpMsa[u'password'] = sys.argv[i+1] + elif myarg == 'smtpmsa.password': + smtpMsa['password'] = sys.argv[i+1] i += 2 - elif myarg == u'smtpmsa.securitymode': + elif myarg == 'smtpmsa.securitymode': value = sys.argv[i+1].lower() if value not in SMTPMSA_SECURITY_MODES: - systemErrorExit(2, '{0} must be {1}; got {2}'.format(myarg, u', '.join(SMTPMSA_SECURITY_MODES), value)) - smtpMsa[u'securityMode'] = value + systemErrorExit(2, '{0} must be {1}; got {2}'.format(myarg, ', '.join(SMTPMSA_SECURITY_MODES), value)) + smtpMsa['securityMode'] = value i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam %s"' % (sys.argv[i], command)) else: i = getSendAsAttributes(i, myarg, body, tagReplacements, command) if signature is not None: - body[u'signature'] = _processSignature(tagReplacements, signature, html) + body['signature'] = _processSignature(tagReplacements, signature, html) if smtpMsa: for field in SMTPMSA_REQUIRED_FIELDS: if field not in smtpMsa: systemErrorExit(2, 'smtpmsa.{0} is required.'.format(field)) - body[u'smtpMsa'] = smtpMsa - kwargs = {u'body': body} + body['smtpMsa'] = smtpMsa + kwargs = {'body': body} if not addCmd: - kwargs[u'sendAsEmail'] = emailAddress + kwargs['sendAsEmail'] = emailAddress i = 0 count = len(users) for user in users: @@ -5370,10 +5380,10 @@ def addUpdateSendAs(users, i, addCmd): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Allowing %s to send as %s (%s/%s)" % (user, emailAddress, i, count) - callGAPI(gmail.users().settings().sendAs(), [u'patch', u'create'][addCmd], + print("Allowing %s to send as %s (%s/%s)" % (user, emailAddress, i, count)) + callGAPI(gmail.users().settings().sendAs(), ['patch', 'create'][addCmd], soft_errors=True, - userId=u'me', **kwargs) + userId='me', **kwargs) def deleteSendAs(users): emailAddress = normalizeEmailAddressOrUID(sys.argv[5], noUid=True) @@ -5384,10 +5394,10 @@ def deleteSendAs(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Disallowing %s to send as %s (%s/%s)" % (user, emailAddress, i, count) - callGAPI(gmail.users().settings().sendAs(), u'delete', + print("Disallowing %s to send as %s (%s/%s)" % (user, emailAddress, i, count)) + callGAPI(gmail.users().settings().sendAs(), 'delete', soft_errors=True, - userId=u'me', sendAsEmail=emailAddress) + userId='me', sendAsEmail=emailAddress) def updateSmime(users): smimeIdBase = None @@ -5396,19 +5406,19 @@ def updateSmime(users): i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'id': + if myarg == 'id': smimeIdBase = sys.argv[i+1] i += 2 - elif myarg in [u'sendas', u'sendasemail']: + elif myarg in ['sendas', 'sendasemail']: sendAsEmailBase = sys.argv[i+1] i += 2 - elif myarg in [u'default']: + elif myarg in ['default']: make_default = True i += 1 else: systemErrorExit(3, '%s is not a valid argument to "gam update smime"' % myarg) if not make_default: - print u'Nothing to update for smime.' + print('Nothing to update for smime.') sys.exit(0) for user in users: user, gmail = buildGmailGAPIObject(user) @@ -5416,17 +5426,17 @@ def updateSmime(users): continue sendAsEmail = sendAsEmailBase if sendAsEmailBase else user if not smimeIdBase: - result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'list', userId=u'me', sendAsEmail=sendAsEmail, fields=u'smimeInfo(id)') - smimes = result.get(u'smimeInfo', []) + result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'list', userId='me', sendAsEmail=sendAsEmail, fields='smimeInfo(id)') + smimes = result.get('smimeInfo', []) if len(smimes) == 0: systemErrorExit(3, '%s has no S/MIME certificates for sendas address %s' % (user, sendAsEmail)) elif len(smimes) > 1: - systemErrorExit(3, u'%s has more than one S/MIME certificate. Please specify a cert to update:\n %s' % (user, u'\n '.join([smime[u'id'] for smime in smimes]))) - smimeId = smimes[0][u'id'] + systemErrorExit(3, '%s has more than one S/MIME certificate. Please specify a cert to update:\n %s' % (user, '\n '.join([smime['id'] for smime in smimes]))) + smimeId = smimes[0]['id'] else: smimeId = smimeIdBase - print u'Setting smime id %s as default for user %s and sendas %s' % (smimeId, user, sendAsEmail) - callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'setDefault', userId=u'me', sendAsEmail=sendAsEmail, id=smimeId) + print('Setting smime id %s as default for user %s and sendas %s' % (smimeId, user, sendAsEmail)) + callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'setDefault', userId='me', sendAsEmail=sendAsEmail, id=smimeId) def deleteSmime(users): smimeIdBase = None @@ -5434,10 +5444,10 @@ def deleteSmime(users): i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'id': + if myarg == 'id': smimeIdBase = sys.argv[i+1] i += 2 - elif myarg in [u'sendas', u'sendasemail']: + elif myarg in ['sendas', 'sendasemail']: sendAsEmailBase = sys.argv[i+1] i += 2 else: @@ -5448,35 +5458,35 @@ def deleteSmime(users): continue sendAsEmail = sendAsEmailBase if sendAsEmailBase else user if not smimeIdBase: - result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'list', userId=u'me', sendAsEmail=sendAsEmail, fields=u'smimeInfo(id)') - smimes = result.get(u'smimeInfo', []) + result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'list', userId='me', sendAsEmail=sendAsEmail, fields='smimeInfo(id)') + smimes = result.get('smimeInfo', []) if len(smimes) == 0: systemErrorExit(3, '%s has no S/MIME certificates for sendas address %s' % (user, sendAsEmail)) elif len(smimes) > 1: - systemErrorExit(3, u'%s has more than one S/MIME certificate. Please specify a cert to delete:\n %s' % (user, u'\n '.join([smime[u'id'] for smime in smimes]))) - smimeId = smimes[0][u'id'] + systemErrorExit(3, '%s has more than one S/MIME certificate. Please specify a cert to delete:\n %s' % (user, '\n '.join([smime['id'] for smime in smimes]))) + smimeId = smimes[0]['id'] else: smimeId = smimeIdBase - print u'Deleting smime id %s for user %s and sendas %s' % (smimeId, user, sendAsEmail) - callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'delete', userId=u'me', sendAsEmail=sendAsEmail, id=smimeId) + print('Deleting smime id %s for user %s and sendas %s' % (smimeId, user, sendAsEmail)) + callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'delete', userId='me', sendAsEmail=sendAsEmail, id=smimeId) def printShowSmime(users, csvFormat): if csvFormat: todrive = False - titles = [u'User'] + titles = ['User'] csvRows = [] primaryonly = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 - elif myarg == u'primaryonly': + elif myarg == 'primaryonly': primaryonly = True i += 1 else: - systemErrorExit(3, '%s is not a valid argument for "gam %s smime"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(3, '%s is not a valid argument for "gam %s smime"' % (myarg, ['show', 'print'][csvFormat])) i = 0 count = len(users) for user in users: @@ -5487,40 +5497,40 @@ def printShowSmime(users, csvFormat): if primaryonly: sendAsEmails = [user] else: - result = callGAPI(gmail.users().settings().sendAs(), u'list', userId=u'me', fields=u'sendAs(sendAsEmail)') + result = callGAPI(gmail.users().settings().sendAs(), 'list', userId='me', fields='sendAs(sendAsEmail)') sendAsEmails = [] - for sendAs in result[u'sendAs']: - sendAsEmails.append(sendAs[u'sendAsEmail']) + for sendAs in result['sendAs']: + sendAsEmails.append(sendAs['sendAsEmail']) for sendAsEmail in sendAsEmails: - result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'list', sendAsEmail=sendAsEmail, userId=u'me') - smimes = result.get(u'smimeInfo', []) + result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'list', sendAsEmail=sendAsEmail, userId='me') + smimes = result.get('smimeInfo', []) for j, _ in enumerate(smimes): - smimes[j][u'expiration'] = datetime.datetime.fromtimestamp(int(smimes[j][u'expiration'])/1000).strftime('%Y-%m-%d %H:%M:%S') + smimes[j]['expiration'] = datetime.datetime.fromtimestamp(int(smimes[j]['expiration'])/1000).strftime('%Y-%m-%d %H:%M:%S') if csvFormat: for smime in smimes: - addRowTitlesToCSVfile(flatten_json(smime, flattened={u'User': user}), csvRows, titles) + addRowTitlesToCSVfile(flatten_json(smime, flattened={'User': user}), csvRows, titles) else: print_json(None, smimes) if csvFormat: - writeCSVfile(csvRows, titles, u'S/MIME', todrive) + writeCSVfile(csvRows, titles, 'S/MIME', todrive) def printShowSendAs(users, csvFormat): if csvFormat: todrive = False - titles = [u'User', u'displayName', u'sendAsEmail', u'replyToAddress', u'isPrimary', u'isDefault', u'treatAsAlias', u'verificationStatus'] + titles = ['User', 'displayName', 'sendAsEmail', 'replyToAddress', 'isPrimary', 'isDefault', 'treatAsAlias', 'verificationStatus'] csvRows = [] formatSig = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 - elif not csvFormat and myarg == u'format': + elif not csvFormat and myarg == 'format': formatSig = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s sendas"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s sendas"' % (myarg, ['show', 'print'][csvFormat])) i = 0 count = len(users) for user in users: @@ -5528,38 +5538,38 @@ def printShowSendAs(users, csvFormat): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings().sendAs(), u'list', + result = callGAPI(gmail.users().settings().sendAs(), 'list', soft_errors=True, - userId=u'me') - jcount = len(result.get(u'sendAs', [])) if (result) else 0 + userId='me') + jcount = len(result.get('sendAs', [])) if (result) else 0 if not csvFormat: - print u'User: {0}, SendAs Addresses: ({1}/{2})'.format(user, i, count) + print('User: {0}, SendAs Addresses: ({1}/{2})'.format(user, i, count)) if jcount == 0: continue j = 0 - for sendas in result[u'sendAs']: + for sendas in result['sendAs']: j += 1 _showSendAs(sendas, j, jcount, formatSig) else: if jcount == 0: continue - for sendas in result[u'sendAs']: - row = {u'User': user, u'isPrimary': False} + for sendas in result['sendAs']: + row = {'User': user, 'isPrimary': False} for item in sendas: - if item != u'smtpMsa': + if item != 'smtpMsa': if item not in titles: titles.append(item) row[item] = sendas[item] else: for field in SMTPMSA_DISPLAY_FIELDS: if field in sendas[item]: - title = u'smtpMsa.{0}'.format(field) + title = 'smtpMsa.{0}'.format(field) if title not in titles: titles.append(title) row[title] = sendas[item][field] csvRows.append(row) if csvFormat: - writeCSVfile(csvRows, titles, u'SendAs', todrive) + writeCSVfile(csvRows, titles, 'SendAs', todrive) def infoSendAs(users): emailAddress = normalizeEmailAddressOrUID(sys.argv[5], noUid=True) @@ -5567,7 +5577,7 @@ def infoSendAs(users): i = 6 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'format': + if myarg == 'format': formatSig = True i += 1 else: @@ -5579,10 +5589,10 @@ def infoSendAs(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u'User: {0}, Show SendAs Address:{1}'.format(user, currentCount(i, count)) - result = callGAPI(gmail.users().settings().sendAs(), u'get', + print('User: {0}, Show SendAs Address:{1}'.format(user, currentCount(i, count))) + result = callGAPI(gmail.users().settings().sendAs(), 'get', soft_errors=True, - userId=u'me', sendAsEmail=emailAddress) + userId='me', sendAsEmail=emailAddress) if result: _showSendAs(result, i, count, formatSig) @@ -5593,22 +5603,22 @@ def addSmime(users): i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'file': + if myarg == 'file': smimefile = sys.argv[i+1] - body[u'pkcs12'] = base64.urlsafe_b64encode(readFile(smimefile, mode=u'rb')) + body['pkcs12'] = base64.urlsafe_b64encode(readFile(smimefile, mode='rb')) i += 2 - elif myarg == u'password': - body[u'encryptedKeyPassword'] = sys.argv[i+1] + elif myarg == 'password': + body['encryptedKeyPassword'] = sys.argv[i+1] i += 2 - elif myarg == u'default': + elif myarg == 'default': setDefault = True i += 1 - elif myarg in [u'sendas', u'sendasemail']: + elif myarg in ['sendas', 'sendasemail']: sendAsEmailBase = sys.argv[i+1] i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam add smime"' % myarg) - if u'pkcs12' not in body: + if 'pkcs12' not in body: systemErrorExit(3, 'you must specify a file to upload') i = 0 count = len(users) @@ -5618,46 +5628,46 @@ def addSmime(users): if not gmail: continue sendAsEmail = sendAsEmailBase if sendAsEmailBase else user - result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'insert', userId=u'me', sendAsEmail=sendAsEmail, body=body) + result = callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'insert', userId='me', sendAsEmail=sendAsEmail, body=body) if setDefault: - callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'setDefault', userId=u'me', sendAsEmail=sendAsEmail, id=result[u'id']) - print u'Added S/MIME certificate for user %s sendas %s issued by %s' % (user, sendAsEmail, result[u'issuerCn']) + callGAPI(gmail.users().settings().sendAs().smimeInfo(), 'setDefault', userId='me', sendAsEmail=sendAsEmail, id=result['id']) + print('Added S/MIME certificate for user %s sendas %s issued by %s' % (user, sendAsEmail, result['issuerCn'])) def getLabelAttributes(i, myarg, body): - if myarg == u'labellistvisibility': - value = sys.argv[i+1].lower().replace(u'_', u'') - if value == u'hide': - body[u'labelListVisibility'] = u'labelHide' - elif value == u'show': - body[u'labelListVisibility'] = u'labelShow' - elif value == u'showifunread': - body[u'labelListVisibility'] = u'labelShowIfUnread' + if myarg == 'labellistvisibility': + value = sys.argv[i+1].lower().replace('_', '') + if value == 'hide': + body['labelListVisibility'] = 'labelHide' + elif value == 'show': + body['labelListVisibility'] = 'labelShow' + elif value == 'showifunread': + body['labelListVisibility'] = 'labelShowIfUnread' else: systemErrorExit(2, 'label_list_visibility must be one of hide, show, show_if_unread; got %s' % value) i += 2 - elif myarg == u'messagelistvisibility': - value = sys.argv[i+1].lower().replace(u'_', u'') - if value not in [u'hide', u'show']: + elif myarg == 'messagelistvisibility': + value = sys.argv[i+1].lower().replace('_', '') + if value not in ['hide', 'show']: systemErrorExit(2, 'message_list_visibility must be show or hide; got %s' % value) - body[u'messageListVisibility'] = value + body['messageListVisibility'] = value i += 2 - elif myarg == u'backgroundcolor': - body.setdefault(u'color', {}) - body[u'color']['backgroundColor'] = getLabelColor(sys.argv[i+1]) + elif myarg == 'backgroundcolor': + body.setdefault('color', {}) + body['color']['backgroundColor'] = getLabelColor(sys.argv[i+1]) i += 2 - elif myarg == u'textcolor': - body.setdefault(u'color', {}) - body[u'color']['textColor'] = getLabelColor(sys.argv[i+1]) + elif myarg == 'textcolor': + body.setdefault('color', {}) + body['color']['textColor'] = getLabelColor(sys.argv[i+1]) i += 2 else: systemErrorExit(2, '%s is not a valid argument for this command.' % myarg) return i def checkLabelColor(body): - if u'color' not in body: + if 'color' not in body: return - if u'backgroundColor' in body[u'color']: - if u'textColor' in body[u'color']: + if 'backgroundColor' in body['color']: + if 'textColor' in body['color']: return systemErrorExit(2, 'textcolor is required.') systemErrorExit(2, 'backgroundcolor is required.') @@ -5665,9 +5675,9 @@ def checkLabelColor(body): def doLabel(users, i): label = sys.argv[i] i += 1 - body = {u'name': label} + body = {'name': label} while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') + myarg = sys.argv[i].lower().replace('_', '') i = getLabelAttributes(i, myarg, body) checkLabelColor(body) i = 0 @@ -5677,81 +5687,81 @@ def doLabel(users, i): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Creating label %s for %s (%s/%s)" % (label, user, i, count) - callGAPI(gmail.users().labels(), u'create', soft_errors=True, userId=user, body=body) + print("Creating label %s for %s (%s/%s)" % (label, user, i, count)) + callGAPI(gmail.users().labels(), 'create', soft_errors=True, userId=user, body=body) -PROCESS_MESSAGE_FUNCTION_TO_ACTION_MAP = {u'delete': u'deleted', u'trash': u'trashed', u'untrash': u'untrashed', u'modify': u'modified'} +PROCESS_MESSAGE_FUNCTION_TO_ACTION_MAP = {'delete': 'deleted', 'trash': 'trashed', 'untrash': 'untrashed', 'modify': 'modified'} def labelsToLabelIds(gmail, labels): allLabels = { - u'INBOX': u'INBOX', u'SPAM': u'SPAM', u'TRASH': u'TRASH', - u'UNREAD': u'UNREAD', u'STARRED': u'STARRED', u'IMPORTANT': u'IMPORTANT', - u'SENT': u'SENT', u'DRAFT': u'DRAFT', - u'CATEGORY_PERSONAL': u'CATEGORY_PERSONAL', - u'CATEGORY_SOCIAL': u'CATEGORY_SOCIAL', - u'CATEGORY_PROMOTIONS': u'CATEGORY_PROMOTIONS', - u'CATEGORY_UPDATES': u'CATEGORY_UPDATES', - u'CATEGORY_FORUMS': u'CATEGORY_FORUMS', + 'INBOX': 'INBOX', 'SPAM': 'SPAM', 'TRASH': 'TRASH', + 'UNREAD': 'UNREAD', 'STARRED': 'STARRED', 'IMPORTANT': 'IMPORTANT', + 'SENT': 'SENT', 'DRAFT': 'DRAFT', + 'CATEGORY_PERSONAL': 'CATEGORY_PERSONAL', + 'CATEGORY_SOCIAL': 'CATEGORY_SOCIAL', + 'CATEGORY_PROMOTIONS': 'CATEGORY_PROMOTIONS', + 'CATEGORY_UPDATES': 'CATEGORY_UPDATES', + 'CATEGORY_FORUMS': 'CATEGORY_FORUMS', } labelIds = list() for label in labels: if label not in allLabels: # first refresh labels in user mailbox - label_results = callGAPI(gmail.users().labels(), u'list', - userId=u'me', fields=u'labels(id,name,type)') - for a_label in label_results[u'labels']: - if a_label[u'type'] == u'system': - allLabels[a_label[u'id']] = a_label[u'id'] + label_results = callGAPI(gmail.users().labels(), 'list', + userId='me', fields='labels(id,name,type)') + for a_label in label_results['labels']: + if a_label['type'] == 'system': + allLabels[a_label['id']] = a_label['id'] else: - allLabels[a_label[u'name']] = a_label[u'id'] + allLabels[a_label['name']] = a_label['id'] if label not in allLabels: # if still not there, create it - label_results = callGAPI(gmail.users().labels(), u'create', - body={u'labelListVisibility': u'labelShow', - u'messageListVisibility': u'show', u'name': label}, - userId=u'me', fields=u'id') - allLabels[label] = label_results[u'id'] + label_results = callGAPI(gmail.users().labels(), 'create', + body={'labelListVisibility': 'labelShow', + 'messageListVisibility': 'show', 'name': label}, + userId='me', fields='id') + allLabels[label] = label_results['id'] try: labelIds.append(allLabels[label]) except KeyError: pass - if label.find(u'/') != -1: + if label.find('/') != -1: # make sure to create parent labels for proper nesting - parent_label = label[:label.rfind(u'/')] + parent_label = label[:label.rfind('/')] while True: if not parent_label in allLabels: - label_result = callGAPI(gmail.users().labels(), u'create', - userId=u'me', body={u'name': parent_label}) - allLabels[parent_label] = label_result[u'id'] - if parent_label.find(u'/') == -1: + label_result = callGAPI(gmail.users().labels(), 'create', + userId='me', body={'name': parent_label}) + allLabels[parent_label] = label_result['id'] + if parent_label.find('/') == -1: break - parent_label = parent_label[:parent_label.rfind(u'/')] + parent_label = parent_label[:parent_label.rfind('/')] return labelIds -def doProcessMessagesOrThreads(users, function, unit=u'messages'): +def doProcessMessagesOrThreads(users, function, unit='messages'): query = None doIt = False maxToProcess = 1 body = {} i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'query': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'query': query = sys.argv[i+1] i += 2 - elif myarg == u'doit': + elif myarg == 'doit': doIt = True i += 1 - elif myarg in [u'maxtodelete', u'maxtotrash', u'maxtomodify', u'maxtountrash']: + elif myarg in ['maxtodelete', 'maxtotrash', 'maxtomodify', 'maxtountrash']: maxToProcess = getInteger(sys.argv[i+1], myarg, minVal=0) i += 2 - elif (function == u'modify') and (myarg == u'addlabel'): - body.setdefault(u'addLabelIds', []) - body[u'addLabelIds'].append(sys.argv[i+1]) + elif (function == 'modify') and (myarg == 'addlabel'): + body.setdefault('addLabelIds', []) + body['addLabelIds'].append(sys.argv[i+1]) i += 2 - elif (function == u'modify') and (myarg == u'removelabel'): - body.setdefault(u'removeLabelIds', []) - body[u'removeLabelIds'].append(sys.argv[i+1]) + elif (function == 'modify') and (myarg == 'removelabel'): + body.setdefault('removeLabelIds', []) + body['removeLabelIds'].append(sys.argv[i+1]) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam %s %s"' % (sys.argv[i], function, unit)) @@ -5762,46 +5772,46 @@ def doProcessMessagesOrThreads(users, function, unit=u'messages'): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u'Searching %s for %s' % (unit, user) + print('Searching %s for %s' % (unit, user)) unitmethod = getattr(gmail.users(), unit) - page_message = u'Got %%%%total_items%%%% %s for user %s' % (unit, user) - listResult = callGAPIpages(unitmethod(), u'list', unit, page_message=page_message, - userId=u'me', q=query, includeSpamTrash=True, soft_errors=True, fields=u'nextPageToken,{0}(id)'.format(unit)) + page_message = 'Got %%%%total_items%%%% %s for user %s' % (unit, user) + listResult = callGAPIpages(unitmethod(), 'list', unit, page_message=page_message, + userId='me', q=query, includeSpamTrash=True, soft_errors=True, fields='nextPageToken,{0}(id)'.format(unit)) result_count = len(listResult) if not doIt or result_count == 0: - print u'would try to %s %s messages for user %s (max %s)\n' % (function, result_count, user, maxToProcess) + print('would try to %s %s messages for user %s (max %s)\n' % (function, result_count, user, maxToProcess)) continue elif result_count > maxToProcess: - print u'WARNING: refusing to %s ANY messages for %s since max messages to process is %s and messages to be %s is %s\n' % (function, user, maxToProcess, action, result_count) + print('WARNING: refusing to %s ANY messages for %s since max messages to process is %s and messages to be %s is %s\n' % (function, user, maxToProcess, action, result_count)) continue - kwargs = {u'body': {}} + kwargs = {'body': {}} for my_key in body: - kwargs[u'body'][my_key] = labelsToLabelIds(gmail, body[my_key]) + kwargs['body'][my_key] = labelsToLabelIds(gmail, body[my_key]) i = 0 - if unit == u'messages' and function in [u'delete', u'modify']: - batchFunction = u'batch%s' % function.title() + if unit == 'messages' and function in ['delete', 'modify']: + batchFunction = 'batch%s' % function.title() id_batches = [[]] for a_unit in listResult: - id_batches[i].append(a_unit[u'id']) + id_batches[i].append(a_unit['id']) if len(id_batches[i]) == 1000: i += 1 id_batches.append([]) processed_messages = 0 for id_batch in id_batches: - kwargs[u'body'][u'ids'] = id_batch - print u'%s %s messages' % (function, len(id_batch)) + kwargs['body']['ids'] = id_batch + print('%s %s messages' % (function, len(id_batch))) callGAPI(unitmethod(), batchFunction, - userId=u'me', **kwargs) + userId='me', **kwargs) processed_messages += len(id_batch) - print u'%s %s of %s messages' % (function, processed_messages, result_count) + print('%s %s of %s messages' % (function, processed_messages, result_count)) continue - if not kwargs[u'body']: - del kwargs[u'body'] + if not kwargs['body']: + del kwargs['body'] for a_unit in listResult: i += 1 - print u' %s %s %s for user %s (%s/%s)' % (function, unit, a_unit[u'id'], user, i, result_count) + print(' %s %s %s for user %s (%s/%s)' % (function, unit, a_unit['id'], user, i, result_count)) callGAPI(unitmethod(), function, - id=a_unit[u'id'], userId=u'me', **kwargs) + id=a_unit['id'], userId='me', **kwargs) def doDeleteLabel(users): label = sys.argv[5] @@ -5810,29 +5820,29 @@ def doDeleteLabel(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u'Getting all labels for %s...' % user - labels = callGAPI(gmail.users().labels(), u'list', userId=user, fields=u'labels(id,name,type)') + print('Getting all labels for %s...' % user) + labels = callGAPI(gmail.users().labels(), 'list', userId=user, fields='labels(id,name,type)') del_labels = [] - if label == u'--ALL_LABELS--': - for del_label in labels[u'labels']: - if del_label[u'type'] == u'system': + if label == '--ALL_LABELS--': + for del_label in labels['labels']: + if del_label['type'] == 'system': continue del_labels.append(del_label) - elif label[:6].lower() == u'regex:': + elif label[:6].lower() == 'regex:': regex = label[6:] p = re.compile(regex) - for del_label in labels[u'labels']: - if del_label[u'type'] == u'system': + for del_label in labels['labels']: + if del_label['type'] == 'system': continue - elif p.match(del_label[u'name']): + elif p.match(del_label['name']): del_labels.append(del_label) else: - for del_label in labels[u'labels']: - if label_name_lower == del_label[u'name'].lower(): + for del_label in labels['labels']: + if label_name_lower == del_label['name'].lower(): del_labels.append(del_label) break else: - print u' Error: no such label for %s' % user + print(' Error: no such label for %s' % user) continue bcount = 0 j = 0 @@ -5840,8 +5850,8 @@ def doDeleteLabel(users): dbatch = gmail.new_batch_http_request(callback=gmail_del_result) for del_me in del_labels: j += 1 - print u' deleting label %s (%s/%s)' % (del_me[u'name'], j, del_me_count) - dbatch.add(gmail.users().labels().delete(userId=user, id=del_me[u'id'])) + print(' deleting label %s (%s/%s)' % (del_me['name'], j, del_me_count)) + dbatch.add(gmail.users().labels().delete(userId=user, id=del_me['id'])) bcount += 1 if bcount == 10: dbatch.execute() @@ -5852,17 +5862,17 @@ def doDeleteLabel(users): def gmail_del_result(request_id, response, exception): if exception: - print exception + print(exception) def showLabels(users): i = 5 onlyUser = showCounts = False while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'onlyuser': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'onlyuser': onlyUser = True i += 1 - elif myarg == u'showcounts': + elif myarg == 'showcounts': showCounts = True i += 1 else: @@ -5871,36 +5881,36 @@ def showLabels(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - labels = callGAPI(gmail.users().labels(), u'list', userId=user, soft_errors=True) + labels = callGAPI(gmail.users().labels(), 'list', userId=user, soft_errors=True) if labels: - for label in labels[u'labels']: - if onlyUser and (label[u'type'] == u'system'): + for label in labels['labels']: + if onlyUser and (label['type'] == 'system'): continue - print utils.convertUTF8(label[u'name']) + print(utils.convertUTF8(label['name'])) for a_key in label: - if a_key == u'name': + if a_key == 'name': continue - print u' %s: %s' % (a_key, label[a_key]) + print(' %s: %s' % (a_key, label[a_key])) if showCounts: - counts = callGAPI(gmail.users().labels(), u'get', - userId=user, id=label[u'id'], - fields=u'messagesTotal,messagesUnread,threadsTotal,threadsUnread') + counts = callGAPI(gmail.users().labels(), 'get', + userId=user, id=label['id'], + fields='messagesTotal,messagesUnread,threadsTotal,threadsUnread') for a_key in counts: - print u' %s: %s' % (a_key, counts[a_key]) - print u'' + print(' %s: %s' % (a_key, counts[a_key])) + print('') def showGmailProfile(users): todrive = False i = 6 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for gam show gmailprofile' % sys.argv[i]) csvRows = [] - titles = [u'emailAddress'] + titles = ['emailAddress'] i = 0 count = len(users) for user in users: @@ -5908,20 +5918,20 @@ def showGmailProfile(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - sys.stderr.write(u'Getting Gmail profile for %s\n' % user) + sys.stderr.write('Getting Gmail profile for %s\n' % user) try: - results = callGAPI(gmail.users(), u'getProfile', + results = callGAPI(gmail.users(), 'getProfile', throw_reasons=GAPI_GMAIL_THROW_REASONS, - userId=u'me') + userId='me') if results: for item in results: if item not in titles: titles.append(item) csvRows.append(results) except GAPI_serviceNotAvailable: - entityServiceNotApplicableWarning(u'User', user, i, count) - sortCSVTitles([u'emailAddress',], titles) - writeCSVfile(csvRows, titles, list_type=u'Gmail Profiles', todrive=todrive) + entityServiceNotApplicableWarning('User', user, i, count) + sortCSVTitles(['emailAddress',], titles) + writeCSVfile(csvRows, titles, list_type='Gmail Profiles', todrive=todrive) def updateLabels(users): label_name = sys.argv[5] @@ -5929,9 +5939,9 @@ def updateLabels(users): body = {} i = 6 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'name'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 else: i = getLabelAttributes(i, myarg, body) @@ -5940,29 +5950,29 @@ def updateLabels(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - labels = callGAPI(gmail.users().labels(), u'list', userId=user, fields=u'labels(id,name)') - for label in labels[u'labels']: - if label[u'name'].lower() == label_name_lower: - callGAPI(gmail.users().labels(), u'patch', soft_errors=True, - userId=user, id=label[u'id'], body=body) + labels = callGAPI(gmail.users().labels(), 'list', userId=user, fields='labels(id,name)') + for label in labels['labels']: + if label['name'].lower() == label_name_lower: + callGAPI(gmail.users().labels(), 'patch', soft_errors=True, + userId=user, id=label['id'], body=body) break else: - print u'Error: user does not have a label named %s' % label_name + print('Error: user does not have a label named %s' % label_name) def renameLabels(users): - search = u'^Inbox/(.*)$' - replace = u'%s' + search = '^Inbox/(.*)$' + replace = '%s' merge = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'search': + if myarg == 'search': search = sys.argv[i+1] i += 2 - elif myarg == u'replace': + elif myarg == 'replace': replace = sys.argv[i+1] i += 2 - elif myarg == u'merge': + elif myarg == 'merge': merge = True i += 1 else: @@ -5972,124 +5982,124 @@ def renameLabels(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - labels = callGAPI(gmail.users().labels(), u'list', userId=user) - for label in labels[u'labels']: - if label[u'type'] == u'system': + labels = callGAPI(gmail.users().labels(), 'list', userId=user) + for label in labels['labels']: + if label['type'] == 'system': continue - match_result = re.search(pattern, label[u'name']) + match_result = re.search(pattern, label['name']) if match_result is not None: try: new_label_name = replace % match_result.groups() except TypeError: - systemErrorExit(2, 'The number of subfields ({0}) in search "{1}" does not match the number of subfields ({2}) in replace "{3}"'.format(len(match_result.groups()), search, replace.count(u'%s'), replace)) - print u' Renaming "%s" to "%s"' % (label[u'name'], new_label_name) + systemErrorExit(2, 'The number of subfields ({0}) in search "{1}" does not match the number of subfields ({2}) in replace "{3}"'.format(len(match_result.groups()), search, replace.count('%s'), replace)) + print(' Renaming "%s" to "%s"' % (label['name'], new_label_name)) try: - callGAPI(gmail.users().labels(), u'patch', soft_errors=True, throw_reasons=[GAPI_ABORTED], id=label[u'id'], userId=user, body={u'name': new_label_name}) + callGAPI(gmail.users().labels(), 'patch', soft_errors=True, throw_reasons=[GAPI_ABORTED], id=label['id'], userId=user, body={'name': new_label_name}) except GAPI_aborted: if merge: - print u' Merging %s label to existing %s label' % (label[u'name'], new_label_name) - messages_to_relabel = callGAPIpages(gmail.users().messages(), u'list', u'messages', - userId=user, q=u'label:%s' % label[u'name'].lower().replace(u'/', u'-').replace(u' ', u'-')) + print(' Merging %s label to existing %s label' % (label['name'], new_label_name)) + messages_to_relabel = callGAPIpages(gmail.users().messages(), 'list', 'messages', + userId=user, q='label:%s' % label['name'].lower().replace('/', '-').replace(' ', '-')) if len(messages_to_relabel) > 0: - for new_label in labels[u'labels']: - if new_label[u'name'].lower() == new_label_name.lower(): - new_label_id = new_label[u'id'] - body = {u'addLabelIds': [new_label_id]} + for new_label in labels['labels']: + if new_label['name'].lower() == new_label_name.lower(): + new_label_id = new_label['id'] + body = {'addLabelIds': [new_label_id]} break j = 1 for message_to_relabel in messages_to_relabel: - print u' relabeling message %s (%s/%s)' % (message_to_relabel[u'id'], j, len(messages_to_relabel)) - callGAPI(gmail.users().messages(), u'modify', userId=user, id=message_to_relabel[u'id'], body=body) + print(' relabeling message %s (%s/%s)' % (message_to_relabel['id'], j, len(messages_to_relabel))) + callGAPI(gmail.users().messages(), 'modify', userId=user, id=message_to_relabel['id'], body=body) j += 1 else: - print u' no messages with %s label' % label[u'name'] - print u' Deleting label %s' % label[u'name'] - callGAPI(gmail.users().labels(), u'delete', id=label[u'id'], userId=user) + print(' no messages with %s label' % label['name']) + print(' Deleting label %s' % label['name']) + callGAPI(gmail.users().labels(), 'delete', id=label['id'], userId=user) else: - print u' Error: looks like %s already exists, not renaming. Use the "merge" argument to merge the labels' % new_label_name + print(' Error: looks like %s already exists, not renaming. Use the "merge" argument to merge the labels' % new_label_name) def _getUserGmailLabels(gmail, user, i, count, **kwargs): try: - labels = callGAPI(gmail.users().labels(), u'list', + labels = callGAPI(gmail.users().labels(), 'list', throw_reasons=GAPI_GMAIL_THROW_REASONS, - userId=u'me', **kwargs) + userId='me', **kwargs) if not labels: - labels = {u'labels': []} + labels = {'labels': []} return labels except GAPI_serviceNotAvailable: - entityServiceNotApplicableWarning(u'User', user, i, count) + entityServiceNotApplicableWarning('User', user, i, count) return None def _getLabelId(labels, labelName): - for label in labels[u'labels']: - if label[u'id'] == labelName or label[u'name'] == labelName: - return label[u'id'] + for label in labels['labels']: + if label['id'] == labelName or label['name'] == labelName: + return label['id'] return None def _getLabelName(labels, labelId): - for label in labels[u'labels']: - if label[u'id'] == labelId: - return label[u'name'] + for label in labels['labels']: + if label['id'] == labelId: + return label['name'] return labelId def _printFilter(user, userFilter, labels): - row = {u'User': user, u'id': userFilter[u'id']} - if u'criteria' in userFilter: - for item in userFilter[u'criteria']: - if item in [u'hasAttachment', u'excludeChats']: + row = {'User': user, 'id': userFilter['id']} + if 'criteria' in userFilter: + for item in userFilter['criteria']: + if item in ['hasAttachment', 'excludeChats']: row[item] = item - elif item == u'size': - row[item] = u'size {0} {1}'.format(userFilter[u'criteria'][u'sizeComparison'], userFilter[u'criteria'][item]) - elif item == u'sizeComparison': + elif item == 'size': + row[item] = 'size {0} {1}'.format(userFilter['criteria']['sizeComparison'], userFilter['criteria'][item]) + elif item == 'sizeComparison': pass else: - row[item] = u'{0} {1}'.format(item, userFilter[u'criteria'][item]) + row[item] = '{0} {1}'.format(item, userFilter['criteria'][item]) else: - row[u'error'] = u'NoCriteria' - if u'action' in userFilter: - for labelId in userFilter[u'action'].get(u'addLabelIds', []): + row['error'] = 'NoCriteria' + if 'action' in userFilter: + for labelId in userFilter['action'].get('addLabelIds', []): if labelId in FILTER_ADD_LABEL_TO_ARGUMENT_MAP: row[FILTER_ADD_LABEL_TO_ARGUMENT_MAP[labelId]] = FILTER_ADD_LABEL_TO_ARGUMENT_MAP[labelId] else: - row[u'label'] = u'label {0}'.format(_getLabelName(labels, labelId)) - for labelId in userFilter[u'action'].get(u'removeLabelIds', []): + row['label'] = 'label {0}'.format(_getLabelName(labels, labelId)) + for labelId in userFilter['action'].get('removeLabelIds', []): if labelId in FILTER_REMOVE_LABEL_TO_ARGUMENT_MAP: row[FILTER_REMOVE_LABEL_TO_ARGUMENT_MAP[labelId]] = FILTER_REMOVE_LABEL_TO_ARGUMENT_MAP[labelId] - if userFilter[u'action'].get(u'forward'): - row[u'forward'] = u'forward {0}'.format(userFilter[u'action'][u'forward']) + if userFilter['action'].get('forward'): + row['forward'] = 'forward {0}'.format(userFilter['action']['forward']) else: - row[u'error'] = u'NoActions' + row['error'] = 'NoActions' return row def _showFilter(userFilter, j, jcount, labels): - print u' Filter: {0}{1}'.format(userFilter[u'id'], currentCount(j, jcount)) - print u' Criteria:' - if u'criteria' in userFilter: - for item in userFilter[u'criteria']: - if item in [u'hasAttachment', u'excludeChats']: - print u' {0}'.format(item) - elif item == u'size': - print u' {0} {1} {2}'.format(item, userFilter[u'criteria'][u'sizeComparison'], userFilter[u'criteria'][item]) - elif item == u'sizeComparison': + print(' Filter: {0}{1}'.format(userFilter['id'], currentCount(j, jcount))) + print(' Criteria:') + if 'criteria' in userFilter: + for item in userFilter['criteria']: + if item in ['hasAttachment', 'excludeChats']: + print(' {0}'.format(item)) + elif item == 'size': + print(' {0} {1} {2}'.format(item, userFilter['criteria']['sizeComparison'], userFilter['criteria'][item])) + elif item == 'sizeComparison': pass else: - print utils.convertUTF8(u' {0} "{1}"'.format(item, userFilter[u'criteria'][item])) + print(utils.convertUTF8(' {0} "{1}"'.format(item, userFilter['criteria'][item]))) else: - print u' ERROR: No Filter criteria' - print u' Actions:' - if u'action' in userFilter: - for labelId in userFilter[u'action'].get(u'addLabelIds', []): + print(' ERROR: No Filter criteria') + print(' Actions:') + if 'action' in userFilter: + for labelId in userFilter['action'].get('addLabelIds', []): if labelId in FILTER_ADD_LABEL_TO_ARGUMENT_MAP: - print u' {0}'.format(FILTER_ADD_LABEL_TO_ARGUMENT_MAP[labelId]) + print(' {0}'.format(FILTER_ADD_LABEL_TO_ARGUMENT_MAP[labelId])) else: - print utils.convertUTF8(u' label "{0}"'.format(_getLabelName(labels, labelId))) - for labelId in userFilter[u'action'].get(u'removeLabelIds', []): + print(utils.convertUTF8(' label "{0}"'.format(_getLabelName(labels, labelId)))) + for labelId in userFilter['action'].get('removeLabelIds', []): if labelId in FILTER_REMOVE_LABEL_TO_ARGUMENT_MAP: - print u' {0}'.format(FILTER_REMOVE_LABEL_TO_ARGUMENT_MAP[labelId]) - if userFilter[u'action'].get(u'forward'): - print u' Forwarding Address: {0}'.format(userFilter[u'action'][u'forward']) + print(' {0}'.format(FILTER_REMOVE_LABEL_TO_ARGUMENT_MAP[labelId])) + if userFilter['action'].get('forward'): + print(' Forwarding Address: {0}'.format(userFilter['action']['forward'])) else: - print u' ERROR: No Filter actions' + print(' ERROR: No Filter actions') def addFilter(users, i): body = {} @@ -6100,66 +6110,66 @@ def addFilter(users, i): myarg = sys.argv[i].lower() if myarg in FILTER_CRITERIA_CHOICES_MAP: myarg = FILTER_CRITERIA_CHOICES_MAP[myarg] - body.setdefault(u'criteria', {}) - if myarg == u'from': - body[u'criteria'][myarg] = sys.argv[i+1] + body.setdefault('criteria', {}) + if myarg == 'from': + body['criteria'][myarg] = sys.argv[i+1] i += 2 - elif myarg == u'to': - body[u'criteria'][myarg] = sys.argv[i+1] + elif myarg == 'to': + body['criteria'][myarg] = sys.argv[i+1] i += 2 - elif myarg in [u'subject', u'query', u'negatedQuery']: - body[u'criteria'][myarg] = sys.argv[i+1] + elif myarg in ['subject', 'query', 'negatedQuery']: + body['criteria'][myarg] = sys.argv[i+1] i += 2 - elif myarg in [u'hasAttachment', u'excludeChats']: - body[u'criteria'][myarg] = True + elif myarg in ['hasAttachment', 'excludeChats']: + body['criteria'][myarg] = True i += 1 - elif myarg == u'size': - body[u'criteria'][u'sizeComparison'] = sys.argv[i+1].lower() - if body[u'criteria'][u'sizeComparison'] not in [u'larger', u'smaller']: + elif myarg == 'size': + body['criteria']['sizeComparison'] = sys.argv[i+1].lower() + if body['criteria']['sizeComparison'] not in ['larger', 'smaller']: systemErrorExit(2, 'size must be followed by larger or smaller; got %s' % sys.argv[i+1].lower()) - body[u'criteria'][myarg] = sys.argv[i+2] + body['criteria'][myarg] = sys.argv[i+2] i += 3 elif myarg in FILTER_ACTION_CHOICES: - body.setdefault(u'action', {}) - if myarg == u'label': + body.setdefault('action', {}) + if myarg == 'label': addLabelName = sys.argv[i+1] i += 2 - elif myarg == u'important': - addLabelIds.append(u'IMPORTANT') - if u'IMPORTANT' in removeLabelIds: - removeLabelIds.remove(u'IMPORTANT') + elif myarg == 'important': + addLabelIds.append('IMPORTANT') + if 'IMPORTANT' in removeLabelIds: + removeLabelIds.remove('IMPORTANT') i += 1 - elif myarg == u'star': - addLabelIds.append(u'STARRED') + elif myarg == 'star': + addLabelIds.append('STARRED') i += 1 - elif myarg == u'trash': - addLabelIds.append(u'TRASH') + elif myarg == 'trash': + addLabelIds.append('TRASH') i += 1 - elif myarg == u'notimportant': - removeLabelIds.append(u'IMPORTANT') - if u'IMPORTANT' in addLabelIds: - addLabelIds.remove(u'IMPORTANT') + elif myarg == 'notimportant': + removeLabelIds.append('IMPORTANT') + if 'IMPORTANT' in addLabelIds: + addLabelIds.remove('IMPORTANT') i += 1 - elif myarg == u'markread': - removeLabelIds.append(u'UNREAD') + elif myarg == 'markread': + removeLabelIds.append('UNREAD') i += 1 - elif myarg == u'archive': - removeLabelIds.append(u'INBOX') + elif myarg == 'archive': + removeLabelIds.append('INBOX') i += 1 - elif myarg == u'neverspam': - removeLabelIds.append(u'SPAM') + elif myarg == 'neverspam': + removeLabelIds.append('SPAM') i += 1 - elif myarg == u'forward': - body[u'action'][u'forward'] = sys.argv[i+1] + elif myarg == 'forward': + body['action']['forward'] = sys.argv[i+1] i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam filter"' % sys.argv[i]) - if u'criteria' not in body: - systemErrorExit(2, 'you must specify a crtieria <{0}> for "gam filter"'.format(u'|'.join(FILTER_CRITERIA_CHOICES_MAP))) - if u'action' not in body: - systemErrorExit(2, 'you must specify an action <{0}> for "gam filter"'.format(u'|'.join(FILTER_ACTION_CHOICES))) + if 'criteria' not in body: + systemErrorExit(2, 'you must specify a crtieria <{0}> for "gam filter"'.format('|'.join(FILTER_CRITERIA_CHOICES_MAP))) + if 'action' not in body: + systemErrorExit(2, 'you must specify an action <{0}> for "gam filter"'.format('|'.join(FILTER_ACTION_CHOICES))) if removeLabelIds: - body[u'action'][u'removeLabelIds'] = removeLabelIds + body['action']['removeLabelIds'] = removeLabelIds i = 0 count = len(users) for user in users: @@ -6167,29 +6177,29 @@ def addFilter(users, i): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - labels = _getUserGmailLabels(gmail, user, i, count, fields=u'labels(id,name)') + labels = _getUserGmailLabels(gmail, user, i, count, fields='labels(id,name)') if not labels: continue if addLabelIds: - body[u'action'][u'addLabelIds'] = addLabelIds[:] + body['action']['addLabelIds'] = addLabelIds[:] if addLabelName: if not addLabelIds: - body[u'action'][u'addLabelIds'] = [] + body['action']['addLabelIds'] = [] addLabelId = _getLabelId(labels, addLabelName) if not addLabelId: - result = callGAPI(gmail.users().labels(), u'create', + result = callGAPI(gmail.users().labels(), 'create', soft_errors=True, - userId=u'me', body={u'name': addLabelName}, fields=u'id') + userId='me', body={'name': addLabelName}, fields='id') if not result: continue - addLabelId = result[u'id'] - body[u'action'][u'addLabelIds'].append(addLabelId) - print u"Adding filter for %s (%s/%s)" % (user, i, count) - result = callGAPI(gmail.users().settings().filters(), u'create', + addLabelId = result['id'] + body['action']['addLabelIds'].append(addLabelId) + print("Adding filter for %s (%s/%s)" % (user, i, count)) + result = callGAPI(gmail.users().settings().filters(), 'create', soft_errors=True, - userId=u'me', body=body) + userId='me', body=body) if result: - print u"User: %s, Filter: %s, Added (%s/%s)" % (user, result[u'id'], i, count) + print("User: %s, Filter: %s, Added (%s/%s)" % (user, result['id'], i, count)) def deleteFilters(users): filterId = sys.argv[5] @@ -6200,24 +6210,24 @@ def deleteFilters(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Deleting filter %s for %s (%s/%s)" % (filterId, user, i, count) - callGAPI(gmail.users().settings().filters(), u'delete', + print("Deleting filter %s for %s (%s/%s)" % (filterId, user, i, count)) + callGAPI(gmail.users().settings().filters(), 'delete', soft_errors=True, - userId=u'me', id=filterId) + userId='me', id=filterId) def printShowFilters(users, csvFormat): if csvFormat: todrive = False csvRows = [] - titles = [u'User', u'id'] + titles = ['User', 'id'] i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s filter"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s filter"' % (myarg, ['show', 'print'][csvFormat])) i = 0 count = len(users) for user in users: @@ -6225,35 +6235,35 @@ def printShowFilters(users, csvFormat): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - labels = callGAPI(gmail.users().labels(), u'list', + labels = callGAPI(gmail.users().labels(), 'list', soft_errors=True, - userId=u'me', fields=u'labels(id,name)') + userId='me', fields='labels(id,name)') if not labels: - labels = {u'labels': []} - result = callGAPI(gmail.users().settings().filters(), u'list', + labels = {'labels': []} + result = callGAPI(gmail.users().settings().filters(), 'list', soft_errors=True, - userId=u'me') - jcount = len(result.get(u'filter', [])) if (result) else 0 + userId='me') + jcount = len(result.get('filter', [])) if (result) else 0 if not csvFormat: - print u'User: {0}, Filters: ({1}/{2})'.format(user, i, count) + print('User: {0}, Filters: ({1}/{2})'.format(user, i, count)) if jcount == 0: continue j = 0 - for userFilter in result[u'filter']: + for userFilter in result['filter']: j += 1 _showFilter(userFilter, j, jcount, labels) else: if jcount == 0: continue - for userFilter in result[u'filter']: + for userFilter in result['filter']: row = _printFilter(user, userFilter, labels) for item in row: if item not in titles: titles.append(item) csvRows.append(row) if csvFormat: - sortCSVTitles([u'User', u'id'], titles) - writeCSVfile(csvRows, titles, u'Filters', todrive) + sortCSVTitles(['User', 'id'], titles) + writeCSVfile(csvRows, titles, 'Filters', todrive) def infoFilters(users): filterId = sys.argv[5] @@ -6264,35 +6274,35 @@ def infoFilters(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - labels = callGAPI(gmail.users().labels(), u'list', + labels = callGAPI(gmail.users().labels(), 'list', soft_errors=True, - userId=u'me', fields=u'labels(id,name)') + userId='me', fields='labels(id,name)') if not labels: - labels = {u'labels': []} - result = callGAPI(gmail.users().settings().filters(), u'get', + labels = {'labels': []} + result = callGAPI(gmail.users().settings().filters(), 'get', soft_errors=True, - userId=u'me', id=filterId) + userId='me', id=filterId) if result: - print u'User: {0}, Filter: ({1}/{2})'.format(user, i, count) + print('User: {0}, Filter: ({1}/{2})'.format(user, i, count)) _showFilter(result, 1, 1, labels) def doForward(users): - enable = getBoolean(sys.argv[4], u'gam forward') - body = {u'enabled': enable} + enable = getBoolean(sys.argv[4], 'gam forward') + body = {'enabled': enable} i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() if myarg in EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP: - body[u'disposition'] = EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP[myarg] + body['disposition'] = EMAILSETTINGS_FORWARD_POP_ACTION_CHOICES_MAP[myarg] i += 1 - elif myarg == u'confirm': + elif myarg == 'confirm': i += 1 - elif myarg.find(u'@') != -1: - body[u'emailAddress'] = sys.argv[i] + elif myarg.find('@') != -1: + body['emailAddress'] = sys.argv[i] i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam forward"' % myarg) - if enable and (not body.get(u'disposition') or not body.get(u'emailAddress')): + if enable and (not body.get('disposition') or not body.get('emailAddress')): systemErrorExit(2, 'you must specify an action and a forwarding address for "gam forward') i = 0 count = len(users) @@ -6302,53 +6312,53 @@ def doForward(users): if not gmail: continue if enable: - print u"User: %s, Forward Enabled: %s, Forwarding Address: %s, Action: %s (%s/%s)" % (user, enable, body[u'emailAddress'], body[u'disposition'], i, count) + print("User: %s, Forward Enabled: %s, Forwarding Address: %s, Action: %s (%s/%s)" % (user, enable, body['emailAddress'], body['disposition'], i, count)) else: - print u"User: %s, Forward Enabled: %s (%s/%s)" % (user, enable, i, count) - callGAPI(gmail.users().settings(), u'updateAutoForwarding', + print("User: %s, Forward Enabled: %s (%s/%s)" % (user, enable, i, count)) + callGAPI(gmail.users().settings(), 'updateAutoForwarding', soft_errors=True, - userId=u'me', body=body) + userId='me', body=body) def printShowForward(users, csvFormat): def _showForward(user, i, count, result): - if u'enabled' in result: - enabled = result[u'enabled'] + if 'enabled' in result: + enabled = result['enabled'] if enabled: - print u"User: %s, Forward Enabled: %s, Forwarding Address: %s, Action: %s (%s/%s)" % (user, enabled, result[u'emailAddress'], result[u'disposition'], i, count) + print("User: %s, Forward Enabled: %s, Forwarding Address: %s, Action: %s (%s/%s)" % (user, enabled, result['emailAddress'], result['disposition'], i, count)) else: - print u"User: %s, Forward Enabled: %s (%s/%s)" % (user, enabled, i, count) + print("User: %s, Forward Enabled: %s (%s/%s)" % (user, enabled, i, count)) else: - enabled = result[u'enable'] == u'true' + enabled = result['enable'] == 'true' if enabled: - print u"User: %s, Forward Enabled: %s, Forwarding Address: %s, Action: %s (%s/%s)" % (user, enabled, result[u'forwardTo'], EMAILSETTINGS_OLD_NEW_OLD_FORWARD_ACTION_MAP[result[u'action']], i, count) + print("User: %s, Forward Enabled: %s, Forwarding Address: %s, Action: %s (%s/%s)" % (user, enabled, result['forwardTo'], EMAILSETTINGS_OLD_NEW_OLD_FORWARD_ACTION_MAP[result['action']], i, count)) else: - print u"User: %s, Forward Enabled: %s (%s/%s)" % (user, enabled, i, count) + print("User: %s, Forward Enabled: %s (%s/%s)" % (user, enabled, i, count)) def _printForward(user, result): - if u'enabled' in result: - row = {u'User': user, u'forwardEnabled': result[u'enabled']} - if result[u'enabled']: - row[u'forwardTo'] = result[u'emailAddress'] - row[u'disposition'] = result[u'disposition'] + if 'enabled' in result: + row = {'User': user, 'forwardEnabled': result['enabled']} + if result['enabled']: + row['forwardTo'] = result['emailAddress'] + row['disposition'] = result['disposition'] else: - row = {u'User': user, u'forwardEnabled': result[u'enable']} - if result[u'enable'] == u'true': - row[u'forwardTo'] = result[u'forwardTo'] - row[u'disposition'] = EMAILSETTINGS_OLD_NEW_OLD_FORWARD_ACTION_MAP[result[u'action']] + row = {'User': user, 'forwardEnabled': result['enable']} + if result['enable'] == 'true': + row['forwardTo'] = result['forwardTo'] + row['disposition'] = EMAILSETTINGS_OLD_NEW_OLD_FORWARD_ACTION_MAP[result['action']] csvRows.append(row) if csvFormat: todrive = False csvRows = [] - titles = [u'User', u'forwardEnabled', u'forwardTo', u'disposition'] + titles = ['User', 'forwardEnabled', 'forwardTo', 'disposition'] i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s forward"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s forward"' % (myarg, ['show', 'print'][csvFormat])) i = 0 count = len(users) for user in users: @@ -6356,20 +6366,20 @@ def printShowForward(users, csvFormat): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings(), u'getAutoForwarding', + result = callGAPI(gmail.users().settings(), 'getAutoForwarding', soft_errors=True, - userId=u'me') + userId='me') if result: if not csvFormat: _showForward(user, i, count, result) else: _printForward(user, result) if csvFormat: - writeCSVfile(csvRows, titles, u'Forward', todrive) + writeCSVfile(csvRows, titles, 'Forward', todrive) def addForwardingAddresses(users): emailAddress = normalizeEmailAddressOrUID(sys.argv[5], noUid=True) - body = {u'forwardingEmail': emailAddress} + body = {'forwardingEmail': emailAddress} i = 0 count = len(users) for user in users: @@ -6377,10 +6387,10 @@ def addForwardingAddresses(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Adding Forwarding Address %s for %s (%s/%s)" % (emailAddress, user, i, count) - callGAPI(gmail.users().settings().forwardingAddresses(), u'create', + print("Adding Forwarding Address %s for %s (%s/%s)" % (emailAddress, user, i, count)) + callGAPI(gmail.users().settings().forwardingAddresses(), 'create', soft_errors=True, - userId=u'me', body=body) + userId='me', body=body) def deleteForwardingAddresses(users): emailAddress = normalizeEmailAddressOrUID(sys.argv[5], noUid=True) @@ -6391,24 +6401,24 @@ def deleteForwardingAddresses(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Deleting Forwarding Address %s for %s (%s/%s)" % (emailAddress, user, i, count) - callGAPI(gmail.users().settings().forwardingAddresses(), u'delete', + print("Deleting Forwarding Address %s for %s (%s/%s)" % (emailAddress, user, i, count)) + callGAPI(gmail.users().settings().forwardingAddresses(), 'delete', soft_errors=True, - userId=u'me', forwardingEmail=emailAddress) + userId='me', forwardingEmail=emailAddress) def printShowForwardingAddresses(users, csvFormat): if csvFormat: todrive = False csvRows = [] - titles = [u'User', u'forwardingEmail', u'verificationStatus'] + titles = ['User', 'forwardingEmail', 'verificationStatus'] i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s forwardingaddresses"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s forwardingaddresses"' % (myarg, ['show', 'print'][csvFormat])) i = 0 count = len(users) for user in users: @@ -6416,26 +6426,26 @@ def printShowForwardingAddresses(users, csvFormat): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings().forwardingAddresses(), u'list', + result = callGAPI(gmail.users().settings().forwardingAddresses(), 'list', soft_errors=True, - userId=u'me') - jcount = len(result.get(u'forwardingAddresses', [])) if (result) else 0 + userId='me') + jcount = len(result.get('forwardingAddresses', [])) if (result) else 0 if not csvFormat: - print u'User: {0}, Forwarding Addresses: ({1}/{2})'.format(user, i, count) + print('User: {0}, Forwarding Addresses: ({1}/{2})'.format(user, i, count)) if jcount == 0: continue j = 0 - for forward in result[u'forwardingAddresses']: + for forward in result['forwardingAddresses']: j += 1 - print u' Forwarding Address: {0}, Verification Status: {1} ({2}/{3})'.format(forward[u'forwardingEmail'], forward[u'verificationStatus'], j, jcount) + print(' Forwarding Address: {0}, Verification Status: {1} ({2}/{3})'.format(forward['forwardingEmail'], forward['verificationStatus'], j, jcount)) else: if jcount == 0: continue - for forward in result[u'forwardingAddresses']: - row = {u'User': user, u'forwardingEmail': forward[u'forwardingEmail'], u'verificationStatus': forward[u'verificationStatus']} + for forward in result['forwardingAddresses']: + row = {'User': user, 'forwardingEmail': forward['forwardingEmail'], 'verificationStatus': forward['verificationStatus']} csvRows.append(row) if csvFormat: - writeCSVfile(csvRows, titles, u'Forwarding Addresses', todrive) + writeCSVfile(csvRows, titles, 'Forwarding Addresses', todrive) def infoForwardingAddresses(users): emailAddress = normalizeEmailAddressOrUID(sys.argv[5], noUid=True) @@ -6446,32 +6456,32 @@ def infoForwardingAddresses(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - forward = callGAPI(gmail.users().settings().forwardingAddresses(), u'get', + forward = callGAPI(gmail.users().settings().forwardingAddresses(), 'get', soft_errors=True, - userId=u'me', forwardingEmail=emailAddress) + userId='me', forwardingEmail=emailAddress) if forward: - print u'User: {0}, Forwarding Address: {1}, Verification Status: {2} ({3}/{4})'.format(user, forward[u'forwardingEmail'], forward[u'verificationStatus'], i, count) + print('User: {0}, Forwarding Address: {1}, Verification Status: {2} ({3}/{4})'.format(user, forward['forwardingEmail'], forward['verificationStatus'], i, count)) def doSignature(users): tagReplacements = {} i = 4 - if sys.argv[i].lower() == u'file': + if sys.argv[i].lower() == 'file': filename = sys.argv[i+1] i, encoding = getCharSet(i+2) signature = readFile(filename, encoding=encoding) else: - signature = getString(i, u'String', minLen=0) + signature = getString(i, 'String', minLen=0) i += 1 body = {} html = False while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'html': + if myarg == 'html': html = True i += 1 else: - i = getSendAsAttributes(i, myarg, body, tagReplacements, u'signature') - body[u'signature'] = _processSignature(tagReplacements, signature, html) + i = getSendAsAttributes(i, myarg, body, tagReplacements, 'signature') + body['signature'] = _processSignature(tagReplacements, signature, html) i = 0 count = len(users) for user in users: @@ -6479,17 +6489,17 @@ def doSignature(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u'Setting Signature for {0} ({1}/{2})'.format(user, i, count) - callGAPI(gmail.users().settings().sendAs(), u'patch', + print('Setting Signature for {0} ({1}/{2})'.format(user, i, count)) + callGAPI(gmail.users().settings().sendAs(), 'patch', soft_errors=True, - userId=u'me', body=body, sendAsEmail=user) + userId='me', body=body, sendAsEmail=user) def getSignature(users): formatSig = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'format': + if myarg == 'format': formatSig = True i += 1 else: @@ -6501,63 +6511,63 @@ def getSignature(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings().sendAs(), u'get', + result = callGAPI(gmail.users().settings().sendAs(), 'get', soft_errors=True, - userId=u'me', sendAsEmail=user) + userId='me', sendAsEmail=user) if result: _showSendAs(result, i, count, formatSig) def doVacation(users): - enable = getBoolean(sys.argv[4], u'gam vacation') - body = {u'enableAutoReply': enable} + enable = getBoolean(sys.argv[4], 'gam vacation') + body = {'enableAutoReply': enable} if enable: - responseBodyType = u'responseBodyPlainText' + responseBodyType = 'responseBodyPlainText' message = None tagReplacements = {} i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'subject': - body[u'responseSubject'] = sys.argv[i+1] + if myarg == 'subject': + body['responseSubject'] = sys.argv[i+1] i += 2 - elif myarg == u'message': + elif myarg == 'message': message = sys.argv[i+1] i += 2 - elif myarg == u'file': + elif myarg == 'file': filename = sys.argv[i+1] i, encoding = getCharSet(i+2) message = readFile(filename, encoding=encoding) - elif myarg == u'replace': - matchTag = getString(i+1, u'Tag') - matchReplacement = getString(i+2, u'String', minLen=0) + elif myarg == 'replace': + matchTag = getString(i+1, 'Tag') + matchReplacement = getString(i+2, 'String', minLen=0) tagReplacements[matchTag] = matchReplacement i += 3 - elif myarg == u'html': - responseBodyType = u'responseBodyHtml' + elif myarg == 'html': + responseBodyType = 'responseBodyHtml' i += 1 - elif myarg == u'contactsonly': - body[u'restrictToContacts'] = True + elif myarg == 'contactsonly': + body['restrictToContacts'] = True i += 1 - elif myarg == u'domainonly': - body[u'restrictToDomain'] = True + elif myarg == 'domainonly': + body['restrictToDomain'] = True i += 1 - elif myarg == u'startdate': - body[u'startTime'] = getYYYYMMDD(sys.argv[i+1], returnTimeStamp=True) + elif myarg == 'startdate': + body['startTime'] = getYYYYMMDD(sys.argv[i+1], returnTimeStamp=True) i += 2 - elif myarg == u'enddate': - body[u'endTime'] = getYYYYMMDD(sys.argv[i+1], returnTimeStamp=True) + elif myarg == 'enddate': + body['endTime'] = getYYYYMMDD(sys.argv[i+1], returnTimeStamp=True) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam vacation"' % sys.argv[i]) if message: - if responseBodyType == u'responseBodyHtml': - message = message.replace(u'\r', u'').replace(u'\\n', u'
') + if responseBodyType == 'responseBodyHtml': + message = message.replace('\r', '').replace('\\n', '
') else: - message = message.replace(u'\r', u'').replace(u'\\n', u'\n') + message = message.replace('\r', '').replace('\\n', '\n') if tagReplacements: message = _processTags(tagReplacements, message) body[responseBodyType] = message - if not message and not body.get(u'responseSubject'): + if not message and not body.get('responseSubject'): systemErrorExit(2, 'You must specify a non-blank subject or message!') i = 0 count = len(users) @@ -6566,17 +6576,17 @@ def doVacation(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - print u"Setting Vacation for %s (%s/%s)" % (user, i, count) - callGAPI(gmail.users().settings(), u'updateVacation', + print("Setting Vacation for %s (%s/%s)" % (user, i, count)) + callGAPI(gmail.users().settings(), 'updateVacation', soft_errors=True, - userId=u'me', body=body) + userId='me', body=body) def getVacation(users): formatReply = False i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'format': + if myarg == 'format': formatReply = True i += 1 else: @@ -6588,95 +6598,95 @@ def getVacation(users): user, gmail = buildGmailGAPIObject(user) if not gmail: continue - result = callGAPI(gmail.users().settings(), u'getVacation', + result = callGAPI(gmail.users().settings(), 'getVacation', soft_errors=True, - userId=u'me') + userId='me') if result: - enabled = result[u'enableAutoReply'] - print u'User: {0}, Vacation: ({1}/{2})'.format(user, i, count) - print u' Enabled: {0}'.format(enabled) + enabled = result['enableAutoReply'] + print('User: {0}, Vacation: ({1}/{2})'.format(user, i, count)) + print(' Enabled: {0}'.format(enabled)) if enabled: - print u' Contacts Only: {0}'.format(result[u'restrictToContacts']) - print u' Domain Only: {0}'.format(result[u'restrictToDomain']) - if u'startTime' in result: - print u' Start Date: {0}'.format(datetime.datetime.fromtimestamp(int(result[u'startTime'])/1000).strftime('%Y-%m-%d')) + print(' Contacts Only: {0}'.format(result['restrictToContacts'])) + print(' Domain Only: {0}'.format(result['restrictToDomain'])) + if 'startTime' in result: + print(' Start Date: {0}'.format(datetime.datetime.fromtimestamp(int(result['startTime'])/1000).strftime('%Y-%m-%d'))) else: - print u' Start Date: Started' - if u'endTime' in result: - print u' End Date: {0}'.format(datetime.datetime.fromtimestamp(int(result[u'endTime'])/1000).strftime('%Y-%m-%d')) + print(' Start Date: Started') + if 'endTime' in result: + print(' End Date: {0}'.format(datetime.datetime.fromtimestamp(int(result['endTime'])/1000).strftime('%Y-%m-%d'))) else: - print u' End Date: Not specified' - print utils.convertUTF8(u' Subject: {0}'.format(result.get(u'responseSubject', u'None'))) - sys.stdout.write(u' Message:\n ') - if result.get(u'responseBodyPlainText'): - print utils.convertUTF8(utils.indentMultiLineText(result[u'responseBodyPlainText'], n=4)) - elif result.get(u'responseBodyHtml'): + print(' End Date: Not specified') + print(utils.convertUTF8(' Subject: {0}'.format(result.get('responseSubject', 'None')))) + sys.stdout.write(' Message:\n ') + if result.get('responseBodyPlainText'): + print(utils.convertUTF8(utils.indentMultiLineText(result['responseBodyPlainText'], n=4))) + elif result.get('responseBodyHtml'): if formatReply: - print utils.convertUTF8(utils.indentMultiLineText(utils.dehtml(result[u'responseBodyHtml']), n=4)) + print(utils.convertUTF8(utils.indentMultiLineText(utils.dehtml(result['responseBodyHtml']), n=4))) else: - print utils.convertUTF8(utils.indentMultiLineText(result[u'responseBodyHtml'], n=4)) + print(utils.convertUTF8(utils.indentMultiLineText(result['responseBodyHtml'], n=4))) else: - print u'None' + print('None') def doDelSchema(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') schemaKey = sys.argv[3] - callGAPI(cd.schemas(), u'delete', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) - print u'Deleted schema %s' % schemaKey + callGAPI(cd.schemas(), 'delete', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) + print('Deleted schema %s' % schemaKey) def doCreateOrUpdateUserSchema(updateCmd): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') schemaKey = sys.argv[3] if updateCmd: - cmd = u'update' + cmd = 'update' try: - body = callGAPI(cd.schemas(), u'get', throw_reasons=[GAPI_NOT_FOUND], customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) + body = callGAPI(cd.schemas(), 'get', throw_reasons=[GAPI_NOT_FOUND], customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) except GAPI_notFound: systemErrorExit(3, 'Schema %s does not exist.' % schemaKey) else: # create - cmd = u'create' - body = {u'schemaName': schemaKey, u'fields': []} + cmd = 'create' + body = {'schemaName': schemaKey, 'fields': []} i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'field': + if myarg == 'field': if updateCmd: # clear field if it exists on update - for n, field in enumerate(body[u'fields']): - if field[u'fieldName'].lower() == sys.argv[i+1].lower(): - del body[u'fields'][n] + for n, field in enumerate(body['fields']): + if field['fieldName'].lower() == sys.argv[i+1].lower(): + del body['fields'][n] break - a_field = {u'fieldName': sys.argv[i+1]} + a_field = {'fieldName': sys.argv[i+1]} i += 2 while True: myarg = sys.argv[i].lower() - if myarg == u'type': - a_field[u'fieldType'] = sys.argv[i+1].upper() - if a_field[u'fieldType'] not in [u'BOOL', u'DOUBLE', u'EMAIL', u'INT64', u'PHONE', u'STRING']: - systemErrorExit(2, 'type must be one of bool, double, email, int64, phone, string; got %s' % a_field[u'fieldType']) + if myarg == 'type': + a_field['fieldType'] = sys.argv[i+1].upper() + if a_field['fieldType'] not in ['BOOL', 'DOUBLE', 'EMAIL', 'INT64', 'PHONE', 'STRING']: + systemErrorExit(2, 'type must be one of bool, double, email, int64, phone, string; got %s' % a_field['fieldType']) i += 2 - elif myarg == u'multivalued': - a_field[u'multiValued'] = True + elif myarg == 'multivalued': + a_field['multiValued'] = True i += 1 - elif myarg == u'indexed': - a_field[u'indexed'] = True + elif myarg == 'indexed': + a_field['indexed'] = True i += 1 - elif myarg == u'restricted': - a_field[u'readAccessType'] = u'ADMINS_AND_SELF' + elif myarg == 'restricted': + a_field['readAccessType'] = 'ADMINS_AND_SELF' i += 1 - elif myarg == u'range': - a_field[u'numericIndexingSpec'] = {u'minValue': getInteger(sys.argv[i+1], myarg), - u'maxValue': getInteger(sys.argv[i+2], myarg)} + elif myarg == 'range': + a_field['numericIndexingSpec'] = {'minValue': getInteger(sys.argv[i+1], myarg), + 'maxValue': getInteger(sys.argv[i+2], myarg)} i += 3 - elif myarg == u'endfield': - body[u'fields'].append(a_field) + elif myarg == 'endfield': + body['fields'].append(a_field) i += 1 break else: systemErrorExit(2, '%s is not a valid argument for "gam %s schema"' % (sys.argv[i], cmd)) - elif updateCmd and myarg == u'deletefield': - for n, field in enumerate(body[u'fields']): - if field[u'fieldName'].lower() == sys.argv[i+1].lower(): - del body[u'fields'][n] + elif updateCmd and myarg == 'deletefield': + for n, field in enumerate(body['fields']): + if field['fieldName'].lower() == sys.argv[i+1].lower(): + del body['fields'][n] break else: systemErrorExit(3, 'field %s not found in schema %s' % (sys.argv[i+1], schemaKey)) @@ -6684,25 +6694,25 @@ def doCreateOrUpdateUserSchema(updateCmd): else: systemErrorExit(2, '%s is not a valid argument for "gam %s schema"' % (sys.argv[i], cmd)) if updateCmd: - result = callGAPI(cd.schemas(), u'update', customerId=GC_Values[GC_CUSTOMER_ID], body=body, schemaKey=schemaKey) - print u'Updated user schema %s' % result[u'schemaName'] + result = callGAPI(cd.schemas(), 'update', customerId=GC_Values[GC_CUSTOMER_ID], body=body, schemaKey=schemaKey) + print('Updated user schema %s' % result['schemaName']) else: - result = callGAPI(cd.schemas(), u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) - print u'Created user schema %s' % result[u'schemaName'] + result = callGAPI(cd.schemas(), 'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) + print('Created user schema %s' % result['schemaName']) def _showSchema(schema): - print u'Schema: %s' % schema[u'schemaName'] + print('Schema: %s' % schema['schemaName']) for a_key in schema: - if a_key not in [u'schemaName', u'fields', u'etag', u'kind']: - print u' %s: %s' % (a_key, schema[a_key]) - for field in schema[u'fields']: - print u' Field: %s' % field[u'fieldName'] + if a_key not in ['schemaName', 'fields', 'etag', 'kind']: + print(' %s: %s' % (a_key, schema[a_key])) + for field in schema['fields']: + print(' Field: %s' % field['fieldName']) for a_key in field: - if a_key not in [u'fieldName', u'kind', u'etag']: - print u' %s: %s' % (a_key, field[a_key]) + if a_key not in ['fieldName', 'kind', 'etag']: + print(' %s: %s' % (a_key, field[a_key])) def doPrintShowUserSchemas(csvFormat): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') if csvFormat: todrive = False csvRows = [] @@ -6710,32 +6720,32 @@ def doPrintShowUserSchemas(csvFormat): i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s schemas"' % (myarg, [u'show', u'print'][csvFormat])) - schemas = callGAPI(cd.schemas(), u'list', customerId=GC_Values[GC_CUSTOMER_ID]) - if not schemas or u'schemas' not in schemas: + systemErrorExit(2, '%s is not a valid argument for "gam %s schemas"' % (myarg, ['show', 'print'][csvFormat])) + schemas = callGAPI(cd.schemas(), 'list', customerId=GC_Values[GC_CUSTOMER_ID]) + if not schemas or 'schemas' not in schemas: return - for schema in schemas[u'schemas']: + for schema in schemas['schemas']: if not csvFormat: _showSchema(schema) else: - row = {u'fields.Count': len(schema[u'fields'])} + row = {'fields.Count': len(schema['fields'])} addRowTitlesToCSVfile(flatten_json(schema, flattened=row), csvRows, titles) if csvFormat: - sortCSVTitles([u'schemaId', u'schemaName', u'fields.Count'], titles) - writeCSVfile(csvRows, titles, u'User Schemas', todrive) + sortCSVTitles(['schemaId', 'schemaName', 'fields.Count'], titles) + writeCSVfile(csvRows, titles, 'User Schemas', todrive) def doGetUserSchema(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') schemaKey = sys.argv[3] - schema = callGAPI(cd.schemas(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) + schema = callGAPI(cd.schemas(), 'get', customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaKey) _showSchema(schema) def getUserAttributes(i, cd, updateCmd): - def getEntryType(i, entry, entryTypes, setTypeCustom=True, customKeyword=u'custom', customTypeKeyword=u'customType'): + def getEntryType(i, entry, entryTypes, setTypeCustom=True, customKeyword='custom', customTypeKeyword='customType'): """ Get attribute entry type entryTypes is list of pre-defined types, a|b|c Allow a|b|c|, a|b|c|custom @@ -6752,18 +6762,18 @@ def getUserAttributes(i, cd, updateCmd): utype = sys.argv[i] ltype = utype.lower() if ltype in entryTypes: - entry[u'type'] = ltype + entry['type'] = ltype entry.pop(customTypeKeyword, None) else: entry[customTypeKeyword] = utype if setTypeCustom: - entry[u'type'] = customKeyword + entry['type'] = customKeyword else: - entry.pop(u'type', None) + entry.pop('type', None) return i+1 def checkClearBodyList(i, body, itemName): - if sys.argv[i].lower() == u'clear': + if sys.argv[i].lower() == 'clear': body.pop(itemName, None) body[itemName] = None return True @@ -6774,16 +6784,16 @@ def getUserAttributes(i, cd, updateCmd): del body[itemName] body.setdefault(itemName, []) # Throw an error if multiple items are marked primary - if itemValue.get(u'primary', False): + if itemValue.get('primary', False): for citem in body[itemName]: - if citem.get(u'primary', False): - if not checkSystemId or itemValue.get(u'systemId') == citem.get(u'systemId'): + if citem.get('primary', False): + if not checkSystemId or itemValue.get('systemId') == citem.get('systemId'): systemErrorExit(2, 'Multiple {0} are marked primary, only one can be primary'.format(itemName)) body[itemName].append(itemValue) def _splitSchemaNameDotFieldName(sn_fn, fnRequired=True): - if sn_fn.find(u'.') != -1: - schemaName, fieldName = sn_fn.split(u'.', 1) + if sn_fn.find('.') != -1: + schemaName, fieldName = sn_fn.split('.', 1) schemaName = schemaName.strip() fieldName = fieldName.strip() if schemaName and fieldName: @@ -6798,464 +6808,464 @@ def getUserAttributes(i, cd, updateCmd): body = {} need_password = False else: - body = {u'name': {u'givenName': u'Unknown', u'familyName': u'Unknown'}} - body[u'primaryEmail'] = normalizeEmailAddressOrUID(sys.argv[i], noUid=True) + body = {'name': {'givenName': 'Unknown', 'familyName': 'Unknown'}} + body['primaryEmail'] = normalizeEmailAddressOrUID(sys.argv[i], noUid=True) i += 1 need_password = True need_to_hash_password = True while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg in [u'firstname', u'givenname']: - body.setdefault(u'name', {}) - body[u'name'][u'givenName'] = sys.argv[i+1] + if myarg in ['firstname', 'givenname']: + body.setdefault('name', {}) + body['name']['givenName'] = sys.argv[i+1] i += 2 - elif myarg in [u'lastname', u'familyname']: - body.setdefault(u'name', {}) - body[u'name'][u'familyName'] = sys.argv[i+1] + elif myarg in ['lastname', 'familyname']: + body.setdefault('name', {}) + body['name']['familyName'] = sys.argv[i+1] i += 2 - elif myarg in [u'username', u'email', u'primaryemail'] and updateCmd: - body[u'primaryEmail'] = normalizeEmailAddressOrUID(sys.argv[i+1], noUid=True) + elif myarg in ['username', 'email', 'primaryemail'] and updateCmd: + body['primaryEmail'] = normalizeEmailAddressOrUID(sys.argv[i+1], noUid=True) i += 2 - elif myarg == u'customerid' and updateCmd: - body[u'customerId'] = sys.argv[i+1] + elif myarg == 'customerid' and updateCmd: + body['customerId'] = sys.argv[i+1] i += 2 - elif myarg == u'password': + elif myarg == 'password': need_password = False - body[u'password'] = sys.argv[i+1] - if body[u'password'].lower() == u'random': + body['password'] = sys.argv[i+1] + if body['password'].lower() == 'random': need_password = True i += 2 - elif myarg == u'admin': + elif myarg == 'admin': value = getBoolean(sys.argv[i+1], myarg) if updateCmd or value: - systemErrorExit(2, '%s %s is not a valid argument for "gam %s user"' % (sys.argv[i], value, [u'create', u'update'][updateCmd])) + systemErrorExit(2, '%s %s is not a valid argument for "gam %s user"' % (sys.argv[i], value, ['create', 'update'][updateCmd])) i += 2 - elif myarg == u'suspended': - body[u'suspended'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'suspended': + body['suspended'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'archived': - body[u'archived'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'archived': + body['archived'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'gal': - body[u'includeInGlobalAddressList'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'gal': + body['includeInGlobalAddressList'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg in [u'sha', u'sha1', u'sha-1']: - body[u'hashFunction'] = u'SHA-1' + elif myarg in ['sha', 'sha1', 'sha-1']: + body['hashFunction'] = 'SHA-1' need_to_hash_password = False i += 1 - elif myarg == u'md5': - body[u'hashFunction'] = u'MD5' + elif myarg == 'md5': + body['hashFunction'] = 'MD5' need_to_hash_password = False i += 1 - elif myarg == u'crypt': - body[u'hashFunction'] = u'crypt' + elif myarg == 'crypt': + body['hashFunction'] = 'crypt' need_to_hash_password = False i += 1 - elif myarg == u'nohash': + elif myarg == 'nohash': need_to_hash_password = False i += 1 - elif myarg == u'changepassword': - body[u'changePasswordAtNextLogin'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'changepassword': + body['changePasswordAtNextLogin'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'ipwhitelisted': - body[u'ipWhitelisted'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'ipwhitelisted': + body['ipWhitelisted'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'agreedtoterms': - body[u'agreedToTerms'] = getBoolean(sys.argv[i+1], myarg) + elif myarg == 'agreedtoterms': + body['agreedToTerms'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg in [u'org', u'ou']: - body[u'orgUnitPath'] = getOrgUnitItem(sys.argv[i+1], pathOnly=True) + elif myarg in ['org', 'ou']: + body['orgUnitPath'] = getOrgUnitItem(sys.argv[i+1], pathOnly=True) i += 2 - elif myarg in [u'language', u'languages']: + elif myarg in ['language', 'languages']: i += 1 - if checkClearBodyList(i, body, u'languages'): + if checkClearBodyList(i, body, 'languages'): i += 1 continue - for language in sys.argv[i].replace(u',', u' ').split(): + for language in sys.argv[i].replace(',', ' ').split(): if language.lower() in LANGUAGE_CODES_MAP: - appendItemToBodyList(body, u'languages', {u'languageCode': LANGUAGE_CODES_MAP[language.lower()]}) + appendItemToBodyList(body, 'languages', {'languageCode': LANGUAGE_CODES_MAP[language.lower()]}) else: - appendItemToBodyList(body, u'languages', {u'customLanguage': language}) + appendItemToBodyList(body, 'languages', {'customLanguage': language}) i += 1 - elif myarg == u'gender': + elif myarg == 'gender': i += 1 - if checkClearBodyList(i, body, u'gender'): + if checkClearBodyList(i, body, 'gender'): i += 1 continue gender = {} - i = getEntryType(i, gender, USER_GENDER_TYPES, customKeyword=u'other', customTypeKeyword=u'customGender') - if (i < len(sys.argv)) and (sys.argv[i].lower() == u'addressmeas'): - gender[u'addressMeAs'] = getString(i+1, u'String') + i = getEntryType(i, gender, USER_GENDER_TYPES, customKeyword='other', customTypeKeyword='customGender') + if (i < len(sys.argv)) and (sys.argv[i].lower() == 'addressmeas'): + gender['addressMeAs'] = getString(i+1, 'String') i += 2 - body[u'gender'] = gender - elif myarg in [u'address', u'addresses']: + body['gender'] = gender + elif myarg in ['address', 'addresses']: i += 1 - if checkClearBodyList(i, body, u'addresses'): + if checkClearBodyList(i, body, 'addresses'): i += 1 continue address = {} - if sys.argv[i].lower() != u'type': + if sys.argv[i].lower() != 'type': systemErrorExit(2, 'wrong format for account address details. Expected type got %s' % sys.argv[i]) i = getEntryType(i+1, address, USER_ADDRESS_TYPES) - if sys.argv[i].lower() in [u'unstructured', u'formatted']: + if sys.argv[i].lower() in ['unstructured', 'formatted']: i += 1 - address[u'sourceIsStructured'] = False - address[u'formatted'] = sys.argv[i].replace(u'\\n', u'\n') + address['sourceIsStructured'] = False + address['formatted'] = sys.argv[i].replace('\\n', '\n') i += 1 while True: myopt = sys.argv[i].lower() - if myopt == u'pobox': - address[u'poBox'] = sys.argv[i+1] + if myopt == 'pobox': + address['poBox'] = sys.argv[i+1] i += 2 - elif myopt == u'extendedaddress': - address[u'extendedAddress'] = sys.argv[i+1] + elif myopt == 'extendedaddress': + address['extendedAddress'] = sys.argv[i+1] i += 2 - elif myopt == u'streetaddress': - address[u'streetAddress'] = sys.argv[i+1] + elif myopt == 'streetaddress': + address['streetAddress'] = sys.argv[i+1] i += 2 - elif myopt == u'locality': - address[u'locality'] = sys.argv[i+1] + elif myopt == 'locality': + address['locality'] = sys.argv[i+1] i += 2 - elif myopt == u'region': - address[u'region'] = sys.argv[i+1] + elif myopt == 'region': + address['region'] = sys.argv[i+1] i += 2 - elif myopt == u'postalcode': - address[u'postalCode'] = sys.argv[i+1] + elif myopt == 'postalcode': + address['postalCode'] = sys.argv[i+1] i += 2 - elif myopt == u'country': - address[u'country'] = sys.argv[i+1] + elif myopt == 'country': + address['country'] = sys.argv[i+1] i += 2 - elif myopt == u'countrycode': - address[u'countryCode'] = sys.argv[i+1] + elif myopt == 'countrycode': + address['countryCode'] = sys.argv[i+1] i += 2 - elif myopt in [u'notprimary', u'primary']: - address[u'primary'] = myopt == u'primary' + elif myopt in ['notprimary', 'primary']: + address['primary'] = myopt == 'primary' i += 1 break else: systemErrorExit(2, 'invalid argument (%s) for account address details' % sys.argv[i]) - appendItemToBodyList(body, u'addresses', address) - elif myarg in [u'emails', u'otheremail', u'otheremails']: + appendItemToBodyList(body, 'addresses', address) + elif myarg in ['emails', 'otheremail', 'otheremails']: i += 1 - if checkClearBodyList(i, body, u'emails'): + if checkClearBodyList(i, body, 'emails'): i += 1 continue an_email = {} i = getEntryType(i, an_email, USER_EMAIL_TYPES) - an_email[u'address'] = sys.argv[i] + an_email['address'] = sys.argv[i] i += 1 - appendItemToBodyList(body, u'emails', an_email) - elif myarg in [u'im', u'ims']: + appendItemToBodyList(body, 'emails', an_email) + elif myarg in ['im', 'ims']: i += 1 - if checkClearBodyList(i, body, u'ims'): + if checkClearBodyList(i, body, 'ims'): i += 1 continue im = {} - if sys.argv[i].lower() != u'type': + if sys.argv[i].lower() != 'type': systemErrorExit(2, 'wrong format for account im details. Expected type got %s' % sys.argv[i]) i = getEntryType(i+1, im, USER_IM_TYPES) - if sys.argv[i].lower() != u'protocol': + if sys.argv[i].lower() != 'protocol': systemErrorExit(2, 'wrong format for account details. Expected protocol got %s' % sys.argv[i]) i += 1 - im[u'protocol'] = sys.argv[i].lower() - if im[u'protocol'] not in [u'custom_protocol', u'aim', u'gtalk', u'icq', u'jabber', u'msn', u'net_meeting', u'qq', u'skype', u'yahoo']: - systemErrorExit(2, 'protocol must be one of custom_protocol, aim, gtalk, icq, jabber, msn, net_meeting, qq, skype, yahoo; got %s' % im[u'protocol']) - if im[u'protocol'] == u'custom_protocol': + im['protocol'] = sys.argv[i].lower() + if im['protocol'] not in ['custom_protocol', 'aim', 'gtalk', 'icq', 'jabber', 'msn', 'net_meeting', 'qq', 'skype', 'yahoo']: + systemErrorExit(2, 'protocol must be one of custom_protocol, aim, gtalk, icq, jabber, msn, net_meeting, qq, skype, yahoo; got %s' % im['protocol']) + if im['protocol'] == 'custom_protocol': i += 1 - im[u'customProtocol'] = sys.argv[i] + im['customProtocol'] = sys.argv[i] i += 1 # Backwards compatability: notprimary|primary on either side of IM address myopt = sys.argv[i].lower() - if myopt in [u'notprimary', u'primary']: - im[u'primary'] = myopt == u'primary' + if myopt in ['notprimary', 'primary']: + im['primary'] = myopt == 'primary' i += 1 - im[u'im'] = sys.argv[i] + im['im'] = sys.argv[i] i += 1 myopt = sys.argv[i].lower() - if myopt in [u'notprimary', u'primary']: - im[u'primary'] = myopt == u'primary' + if myopt in ['notprimary', 'primary']: + im['primary'] = myopt == 'primary' i += 1 - appendItemToBodyList(body, u'ims', im) - elif myarg in [u'organization', u'organizations']: + appendItemToBodyList(body, 'ims', im) + elif myarg in ['organization', 'organizations']: i += 1 - if checkClearBodyList(i, body, u'organizations'): + if checkClearBodyList(i, body, 'organizations'): i += 1 continue organization = {} while True: myopt = sys.argv[i].lower() - if myopt == u'name': - organization[u'name'] = sys.argv[i+1] + if myopt == 'name': + organization['name'] = sys.argv[i+1] i += 2 - elif myopt == u'title': - organization[u'title'] = sys.argv[i+1] + elif myopt == 'title': + organization['title'] = sys.argv[i+1] i += 2 - elif myopt == u'customtype': - organization[u'customType'] = sys.argv[i+1] - organization.pop(u'type', None) + elif myopt == 'customtype': + organization['customType'] = sys.argv[i+1] + organization.pop('type', None) i += 2 - elif myopt == u'type': + elif myopt == 'type': i = getEntryType(i+1, organization, USER_ORGANIZATION_TYPES, setTypeCustom=False) - elif myopt == u'department': - organization[u'department'] = sys.argv[i+1] + elif myopt == 'department': + organization['department'] = sys.argv[i+1] i += 2 - elif myopt == u'symbol': - organization[u'symbol'] = sys.argv[i+1] + elif myopt == 'symbol': + organization['symbol'] = sys.argv[i+1] i += 2 - elif myopt == u'costcenter': - organization[u'costCenter'] = sys.argv[i+1] + elif myopt == 'costcenter': + organization['costCenter'] = sys.argv[i+1] i += 2 - elif myopt == u'location': - organization[u'location'] = sys.argv[i+1] + elif myopt == 'location': + organization['location'] = sys.argv[i+1] i += 2 - elif myopt == u'description': - organization[u'description'] = sys.argv[i+1] + elif myopt == 'description': + organization['description'] = sys.argv[i+1] i += 2 - elif myopt == u'domain': - organization[u'domain'] = sys.argv[i+1] + elif myopt == 'domain': + organization['domain'] = sys.argv[i+1] i += 2 - elif myopt in [u'notprimary', u'primary']: - organization[u'primary'] = myopt == u'primary' + elif myopt in ['notprimary', 'primary']: + organization['primary'] = myopt == 'primary' i += 1 break else: systemErrorExit(2, 'invalid argument (%s) for account organization details' % sys.argv[i]) - appendItemToBodyList(body, u'organizations', organization) - elif myarg in [u'phone', u'phones']: + appendItemToBodyList(body, 'organizations', organization) + elif myarg in ['phone', 'phones']: i += 1 - if checkClearBodyList(i, body, u'phones'): + if checkClearBodyList(i, body, 'phones'): i += 1 continue phone = {} while True: myopt = sys.argv[i].lower() - if myopt == u'value': - phone[u'value'] = sys.argv[i+1] + if myopt == 'value': + phone['value'] = sys.argv[i+1] i += 2 - elif myopt == u'type': + elif myopt == 'type': i = getEntryType(i+1, phone, USER_PHONE_TYPES) - elif myopt in [u'notprimary', u'primary']: - phone[u'primary'] = myopt == u'primary' + elif myopt in ['notprimary', 'primary']: + phone['primary'] = myopt == 'primary' i += 1 break else: systemErrorExit(2, 'invalid argument (%s) for account phone details' % sys.argv[i]) - appendItemToBodyList(body, u'phones', phone) - elif myarg in [u'relation', u'relations']: + appendItemToBodyList(body, 'phones', phone) + elif myarg in ['relation', 'relations']: i += 1 - if checkClearBodyList(i, body, u'relations'): + if checkClearBodyList(i, body, 'relations'): i += 1 continue relation = {} i = getEntryType(i, relation, USER_RELATION_TYPES) - relation[u'value'] = sys.argv[i] + relation['value'] = sys.argv[i] i += 1 - appendItemToBodyList(body, u'relations', relation) - elif myarg in [u'externalid', u'externalids']: + appendItemToBodyList(body, 'relations', relation) + elif myarg in ['externalid', 'externalids']: i += 1 - if checkClearBodyList(i, body, u'externalIds'): + if checkClearBodyList(i, body, 'externalIds'): i += 1 continue externalid = {} i = getEntryType(i, externalid, USER_EXTERNALID_TYPES) - externalid[u'value'] = sys.argv[i] + externalid['value'] = sys.argv[i] i += 1 - appendItemToBodyList(body, u'externalIds', externalid) - elif myarg in [u'website', u'websites']: + appendItemToBodyList(body, 'externalIds', externalid) + elif myarg in ['website', 'websites']: i += 1 - if checkClearBodyList(i, body, u'websites'): + if checkClearBodyList(i, body, 'websites'): i += 1 continue website = {} i = getEntryType(i, website, USER_WEBSITE_TYPES) - website[u'value'] = sys.argv[i] + website['value'] = sys.argv[i] i += 1 myopt = sys.argv[i].lower() - if myopt in [u'notprimary', u'primary']: - website[u'primary'] = myopt == u'primary' + if myopt in ['notprimary', 'primary']: + website['primary'] = myopt == 'primary' i += 1 - appendItemToBodyList(body, u'websites', website) - elif myarg in [u'note', u'notes']: + appendItemToBodyList(body, 'websites', website) + elif myarg in ['note', 'notes']: i += 1 - if checkClearBodyList(i, body, u'notes'): + if checkClearBodyList(i, body, 'notes'): i += 1 continue note = {} - if sys.argv[i].lower() in [u'text_plain', u'text_html']: - note[u'contentType'] = sys.argv[i].lower() + if sys.argv[i].lower() in ['text_plain', 'text_html']: + note['contentType'] = sys.argv[i].lower() i += 1 - if sys.argv[i].lower() == u'file': + if sys.argv[i].lower() == 'file': filename = sys.argv[i+1] i, encoding = getCharSet(i+2) - note[u'value'] = readFile(filename, encoding=encoding) + note['value'] = readFile(filename, encoding=encoding) else: - note[u'value'] = sys.argv[i].replace(u'\\n', u'\n') + note['value'] = sys.argv[i].replace('\\n', '\n') i += 1 - body[u'notes'] = note - elif myarg in [u'location', u'locations']: + body['notes'] = note + elif myarg in ['location', 'locations']: i += 1 - if checkClearBodyList(i, body, u'locations'): + if checkClearBodyList(i, body, 'locations'): i += 1 continue - location = {u'type': u'desk', u'area': u''} + location = {'type': 'desk', 'area': ''} while True: myopt = sys.argv[i].lower() - if myopt == u'type': + if myopt == 'type': i = getEntryType(i+1, location, USER_LOCATION_TYPES) - elif myopt == u'area': - location[u'area'] = sys.argv[i+1] + elif myopt == 'area': + location['area'] = sys.argv[i+1] i += 2 - elif myopt in [u'building', u'buildingid']: - location[u'buildingId'] = _getBuildingByNameOrId(cd, sys.argv[i+1]) + elif myopt in ['building', 'buildingid']: + location['buildingId'] = _getBuildingByNameOrId(cd, sys.argv[i+1]) i += 2 - elif myopt in [u'desk', u'deskcode']: - location[u'deskCode'] = sys.argv[i+1] + elif myopt in ['desk', 'deskcode']: + location['deskCode'] = sys.argv[i+1] i += 2 - elif myopt in [u'floor', u'floorname']: - location[u'floorName'] = sys.argv[i+1] + elif myopt in ['floor', 'floorname']: + location['floorName'] = sys.argv[i+1] i += 2 - elif myopt in [u'section', u'floorsection']: - location[u'floorSection'] = sys.argv[i+1] + elif myopt in ['section', 'floorsection']: + location['floorSection'] = sys.argv[i+1] i += 2 - elif myopt in [u'endlocation']: + elif myopt in ['endlocation']: i += 1 break else: systemErrorExit(3, '%s is not a valid argument for user location details. Make sure user location details end with an endlocation argument') - appendItemToBodyList(body, u'locations', location) - elif myarg in [u'ssh', u'sshkeys', u'sshpublickeys']: + appendItemToBodyList(body, 'locations', location) + elif myarg in ['ssh', 'sshkeys', 'sshpublickeys']: i += 1 - if checkClearBodyList(i, body, u'sshPublicKeys'): + if checkClearBodyList(i, body, 'sshPublicKeys'): i += 1 continue ssh = {} while True: myopt = sys.argv[i].lower() - if myopt == u'expires': - ssh[u'expirationTimeUsec'] = getInteger(sys.argv[i+1], myopt, minVal=0) + if myopt == 'expires': + ssh['expirationTimeUsec'] = getInteger(sys.argv[i+1], myopt, minVal=0) i += 2 - elif myopt == u'key': - ssh[u'key'] = sys.argv[i+1] + elif myopt == 'key': + ssh['key'] = sys.argv[i+1] i += 2 - elif myopt in [u'endssh']: + elif myopt in ['endssh']: i += 1 break else: systemErrorExit(3, '%s is not a valid argument for user ssh details. Make sure user ssh details end with an endssh argument') - appendItemToBodyList(body, u'sshPublicKeys', ssh) - elif myarg in [u'posix', u'posixaccounts']: + appendItemToBodyList(body, 'sshPublicKeys', ssh) + elif myarg in ['posix', 'posixaccounts']: i += 1 - if checkClearBodyList(i, body, u'posixAccounts'): + if checkClearBodyList(i, body, 'posixAccounts'): i += 1 continue posix = {} while True: myopt = sys.argv[i].lower() - if myopt == u'gecos': - posix[u'gecos'] = sys.argv[i+1] + if myopt == 'gecos': + posix['gecos'] = sys.argv[i+1] i += 2 - elif myopt == u'gid': - posix[u'gid'] = getInteger(sys.argv[i+1], myopt, minVal=0) + elif myopt == 'gid': + posix['gid'] = getInteger(sys.argv[i+1], myopt, minVal=0) i += 2 - elif myopt == u'uid': - posix[u'uid'] = getInteger(sys.argv[i+1], myopt, minVal=1000) + elif myopt == 'uid': + posix['uid'] = getInteger(sys.argv[i+1], myopt, minVal=1000) i += 2 - elif myopt in [u'home', u'homedirectory']: - posix[u'homeDirectory'] = sys.argv[i+1] + elif myopt in ['home', 'homedirectory']: + posix['homeDirectory'] = sys.argv[i+1] i += 2 - elif myopt in [u'primary']: - posix[u'primary'] = getBoolean(sys.argv[i+1], myopt) + elif myopt in ['primary']: + posix['primary'] = getBoolean(sys.argv[i+1], myopt) i += 2 - elif myopt in [u'shell']: - posix[u'shell'] = sys.argv[i+1] + elif myopt in ['shell']: + posix['shell'] = sys.argv[i+1] i += 2 - elif myopt in [u'system', u'systemid']: - posix[u'systemId'] = sys.argv[i+1] + elif myopt in ['system', 'systemid']: + posix['systemId'] = sys.argv[i+1] i += 2 - elif myopt in [u'username', u'name']: - posix[u'username'] = sys.argv[i+1] + elif myopt in ['username', 'name']: + posix['username'] = sys.argv[i+1] i += 2 - elif myopt in [u'os', u'operatingsystemtype']: - posix[u'operatingSystemType'] = sys.argv[i+1] + elif myopt in ['os', 'operatingsystemtype']: + posix['operatingSystemType'] = sys.argv[i+1] i += 2 - elif myopt in [u'endposix']: + elif myopt in ['endposix']: i += 1 break else: systemErrorExit(3, '%s is not a valid argument for user posix details. Make sure user posix details end with an endposix argument') - appendItemToBodyList(body, u'posixAccounts', posix, checkSystemId=True) - elif myarg in [u'keyword', u'keywords']: + appendItemToBodyList(body, 'posixAccounts', posix, checkSystemId=True) + elif myarg in ['keyword', 'keywords']: i += 1 - if checkClearBodyList(i, body, u'keywords'): + if checkClearBodyList(i, body, 'keywords'): i += 1 continue keyword = {} i = getEntryType(i, keyword, USER_KEYWORD_TYPES) - keyword[u'value'] = sys.argv[i] + keyword['value'] = sys.argv[i] i += 1 - appendItemToBodyList(body, u'keywords', keyword) - elif myarg == u'clearschema': + appendItemToBodyList(body, 'keywords', keyword) + elif myarg == 'clearschema': if not updateCmd: systemErrorExit(2, '%s is not a valid create user argument.' % sys.argv[i]) schemaName, fieldName = _splitSchemaNameDotFieldName(sys.argv[i+1], False) - up = u'customSchemas' + up = 'customSchemas' body.setdefault(up, {}) body[up].setdefault(schemaName, {}) if fieldName is None: - schema = callGAPI(cd.schemas(), u'get', + schema = callGAPI(cd.schemas(), 'get', soft_errors=True, - customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaName, fields=u'fields(fieldName)') + customerId=GC_Values[GC_CUSTOMER_ID], schemaKey=schemaName, fields='fields(fieldName)') if not schema: sys.exit(2) - for field in schema[u'fields']: - body[up][schemaName][field[u'fieldName']] = None + for field in schema['fields']: + body[up][schemaName][field['fieldName']] = None else: body[up][schemaName][fieldName] = None i += 2 - elif myarg.find(u'.') >= 0: + elif myarg.find('.') >= 0: schemaName, fieldName = _splitSchemaNameDotFieldName(sys.argv[i]) - up = u'customSchemas' + up = 'customSchemas' body.setdefault(up, {}) body[up].setdefault(schemaName, {}) i += 1 multivalue = sys.argv[i].lower() - if multivalue in [u'multivalue', u'multivalued', u'value', u'multinonempty']: + if multivalue in ['multivalue', 'multivalued', 'value', 'multinonempty']: i += 1 body[up][schemaName].setdefault(fieldName, []) schemaValue = {} - if sys.argv[i].lower() == u'type': + if sys.argv[i].lower() == 'type': i += 1 - schemaValue[u'type'] = sys.argv[i].lower() - if schemaValue[u'type'] not in [u'custom', u'home', u'other', u'work']: - systemErrorExit(2, 'wrong type must be one of custom, home, other, work; got %s' % schemaValue[u'type']) + schemaValue['type'] = sys.argv[i].lower() + if schemaValue['type'] not in ['custom', 'home', 'other', 'work']: + systemErrorExit(2, 'wrong type must be one of custom, home, other, work; got %s' % schemaValue['type']) i += 1 - if schemaValue[u'type'] == u'custom': - schemaValue[u'customType'] = sys.argv[i] + if schemaValue['type'] == 'custom': + schemaValue['customType'] = sys.argv[i] i += 1 - schemaValue[u'value'] = sys.argv[i] - if schemaValue[u'value'] or multivalue != u'multinonempty': + schemaValue['value'] = sys.argv[i] + if schemaValue['value'] or multivalue != 'multinonempty': body[up][schemaName][fieldName].append(schemaValue) else: body[up][schemaName][fieldName] = sys.argv[i] i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s user"' % (sys.argv[i], [u'create', u'update'][updateCmd])) + systemErrorExit(2, '%s is not a valid argument for "gam %s user"' % (sys.argv[i], ['create', 'update'][updateCmd])) if need_password: - body[u'password'] = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789~`!@#$%^&*()-=_+:;"\'{}[]\\|', 25)) - if u'password' in body and need_to_hash_password: - body[u'password'] = gen_sha512_hash(body[u'password']) - body[u'hashFunction'] = u'crypt' + body['password'] = ''.join(random.sample('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789~`!@#$%^&*()-=_+:;"\'{}[]\\|', 25)) + if 'password' in body and need_to_hash_password: + body['password'] = gen_sha512_hash(body['password']) + body['hashFunction'] = 'crypt' return body def getCRMService(login_hint): - scope = u'https://www.googleapis.com/auth/cloud-platform' - client_id = u'297408095146-fug707qsjv4ikron0hugpevbrjhkmsk7.apps.googleusercontent.com' - client_secret = u'qM3dP8f_4qedwzWQE1VR4zzU' + scope = 'https://www.googleapis.com/auth/cloud-platform' + client_id = '297408095146-fug707qsjv4ikron0hugpevbrjhkmsk7.apps.googleusercontent.com' + client_secret = 'qM3dP8f_4qedwzWQE1VR4zzU' flow = oauth2client.client.OAuth2WebServerFlow(client_id=client_id, client_secret=client_secret, scope=scope, redirect_uri=oauth2client.client.OOB_CALLBACK_URN, - user_agent=GAM_INFO, access_type=u'online', response_type=u'code', login_hint=login_hint) + user_agent=GAM_INFO, access_type='online', response_type='code', login_hint=login_hint) storage_dict = {} - storage = DictionaryStorage(storage_dict, u'credentials') + storage = DictionaryStorage(storage_dict, 'credentials') flags = cmd_flags(noLocalWebserver=GC_Values[GC_NO_BROWSER]) http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) try: @@ -7264,120 +7274,120 @@ def getCRMService(login_hint): noPythonSSLExit() credentials.user_agent = GAM_INFO http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], cache=None)) - return (googleapiclient.discovery.build(u'cloudresourcemanager', u'v1', + return (googleapiclient.discovery.build('cloudresourcemanager', 'v1', http=http, cache_discovery=False, discoveryServiceUrl=googleapiclient.discovery.V2_DISCOVERY_URI), http) def getGAMProjectAPIs(): httpObj = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]) - _, c = httpObj.request(GAM_PROJECT_APIS, u'GET') + _, c = httpObj.request(GAM_PROJECT_APIS, 'GET') return httpObj, c.splitlines() def enableGAMProjectAPIs(GAMProjectAPIs, httpObj, projectId, checkEnabled, i=0, count=0): apis = GAMProjectAPIs[:] - project_name = u'project:{0}'.format(projectId) - serveman = googleapiclient.discovery.build(u'servicemanagement', u'v1', + project_name = 'project:{0}'.format(projectId) + serveman = googleapiclient.discovery.build('servicemanagement', 'v1', http=httpObj, cache_discovery=False, discoveryServiceUrl=googleapiclient.discovery.V2_DISCOVERY_URI) status = True if checkEnabled: try: - services = callGAPIpages(serveman.services(), u'list', u'services', + services = callGAPIpages(serveman.services(), 'list', 'services', throw_reasons=[GAPI_NOT_FOUND], - consumerId=project_name, fields=u'nextPageToken,services(serviceName)') + consumerId=project_name, fields='nextPageToken,services(serviceName)') jcount = len(services) - print u' Project: {0}, Check {1} APIs{2}'.format(projectId, jcount, currentCount(i, count)) + print(' Project: {0}, Check {1} APIs{2}'.format(projectId, jcount, currentCount(i, count))) j = 0 - for service in sorted(services, key=lambda k: k[u'serviceName']): + for service in sorted(services, key=lambda k: k['serviceName']): j += 1 - if u'serviceName' in service: - if service[u'serviceName'] in apis: - print u' API: {0}, Already enabled{1}'.format(service[u'serviceName'], currentCount(j, jcount)) - apis.remove(service[u'serviceName']) + if 'serviceName' in service: + if service['serviceName'] in apis: + print(' API: {0}, Already enabled{1}'.format(service['serviceName'], currentCount(j, jcount))) + apis.remove(service['serviceName']) else: - print u' API: {0}, Already enabled (non-GAM which is fine){1}'.format(service[u'serviceName'], currentCount(j, jcount)) + print(' API: {0}, Already enabled (non-GAM which is fine){1}'.format(service['serviceName'], currentCount(j, jcount))) except GAPI_notFound as e: - print u' Project: {0}, Update Failed: {1}{2}'.format(projectId, str(e), currentCount(i, count)) + print(' Project: {0}, Update Failed: {1}{2}'.format(projectId, str(e), currentCount(i, count))) status = False jcount = len(apis) if status and jcount > 0: - print u' Project: {0}, Enable {1} APIs{2}'.format(projectId, jcount, currentCount(i, count)) + print(' Project: {0}, Enable {1} APIs{2}'.format(projectId, jcount, currentCount(i, count))) j = 0 for api in apis: j += 1 while True: try: - callGAPI(serveman.services(), u'enable', + callGAPI(serveman.services(), 'enable', throw_reasons=[GAPI_FAILED_PRECONDITION, GAPI_FORBIDDEN, GAPI_PERMISSION_DENIED], - serviceName=api, body={u'consumerId': project_name}) - print u' API: {0}, Enabled{1}'.format(api, currentCount(j, jcount)) + serviceName=api, body={'consumerId': project_name}) + print(' API: {0}, Enabled{1}'.format(api, currentCount(j, jcount))) break except GAPI_failedPrecondition as e: - print u'\nThere was an error enabling %s. Please resolve error as described below:' % api - print - print u'\n%s\n' % e - print - raw_input(u'Press enter once resolved and we will try enabling the API again.') + print('\nThere was an error enabling %s. Please resolve error as described below:' % api) + print() + print('\n%s\n' % e) + print() + input('Press enter once resolved and we will try enabling the API again.') except (GAPI_forbidden, GAPI_permissionDenied) as e: - print u' API: {0}, Enable Failed: {1}{2}'.format(api, str(e), currentCount(j, jcount)) + print(' API: {0}, Enable Failed: {1}{2}'.format(api, str(e), currentCount(j, jcount))) status = False return status def _createClientSecretsOauth2service(httpObj, projectId): def _checkClientAndSecret(simplehttp, client_id, client_secret): - url = u'https://www.googleapis.com/oauth2/v4/token' - post_data = {u'client_id': client_id, u'client_secret': client_secret, - u'code': u'ThisIsAnInvalidCodeOnlyBeingUsedToTestIfClientAndSecretAreValid', - u'redirect_uri': u'urn:ietf:wg:oauth:2.0:oob', u'grant_type': u'authorization_code'} + url = 'https://www.googleapis.com/oauth2/v4/token' + post_data = {'client_id': client_id, 'client_secret': client_secret, + 'code': 'ThisIsAnInvalidCodeOnlyBeingUsedToTestIfClientAndSecretAreValid', + 'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob', 'grant_type': 'authorization_code'} headers = {'Content-type': 'application/x-www-form-urlencoded'} - _, content = simplehttp.request(url, u'POST', urlencode(post_data), headers=headers) + _, content = simplehttp.request(url, 'POST', urlencode(post_data), headers=headers) try: content = json.loads(content) except ValueError: - print u'Unknown error: %s' % content + print('Unknown error: %s' % content) return False - if not u'error' in content or not u'error_description' in content: - print u'Unknown error: %s' % content + if not 'error' in content or not 'error_description' in content: + print('Unknown error: %s' % content) return False - if content[u'error'] == u'invalid_grant': + if content['error'] == 'invalid_grant': return True - if content[u'error_description'] == u'The OAuth client was not found.': - print u'Ooops!!\n\n%s\n\nIs not a valid client ID. Please make sure you are following the directions exactly and that there are no extra spaces in your client ID.' % client_id + if content['error_description'] == 'The OAuth client was not found.': + print('Ooops!!\n\n%s\n\nIs not a valid client ID. Please make sure you are following the directions exactly and that there are no extra spaces in your client ID.' % client_id) return False - if content[u'error_description'] == u'Unauthorized': - print u'Ooops!!\n\n%s\n\nIs not a valid client secret. Please make sure you are following the directions exactly and that there are no extra spaces in your client secret.' % client_secret + if content['error_description'] == 'Unauthorized': + print('Ooops!!\n\n%s\n\nIs not a valid client secret. Please make sure you are following the directions exactly and that there are no extra spaces in your client secret.' % client_secret) return False - print u'Unknown error: %s' % content + print('Unknown error: %s' % content) return False simplehttp, GAMProjectAPIs = getGAMProjectAPIs() enableGAMProjectAPIs(GAMProjectAPIs, httpObj, projectId, False) - iam = googleapiclient.discovery.build(u'iam', u'v1', + iam = googleapiclient.discovery.build('iam', 'v1', http=httpObj, cache_discovery=False, discoveryServiceUrl=googleapiclient.discovery.V2_DISCOVERY_URI) - sa_list = callGAPI(iam.projects().serviceAccounts(), u'list', - name=u'projects/%s' % projectId) + sa_list = callGAPI(iam.projects().serviceAccounts(), 'list', + name='projects/%s' % projectId) service_account = None - if u'accounts' in sa_list: - for account in sa_list[u'accounts']: - sa_email = u'%s@%s.iam.gserviceaccount.com' % (projectId, projectId) - if sa_email in account[u'name']: + if 'accounts' in sa_list: + for account in sa_list['accounts']: + sa_email = '%s@%s.iam.gserviceaccount.com' % (projectId, projectId) + if sa_email in account['name']: service_account = account break if not service_account: - print u'Creating Service Account' - service_account = callGAPI(iam.projects().serviceAccounts(), u'create', - name=u'projects/%s' % projectId, - body={u'accountId': projectId, u'serviceAccount': {u'displayName': u'GAM Project'}}) - key = callGAPI(iam.projects().serviceAccounts().keys(), u'create', - name=service_account[u'name'], body={u'privateKeyType': u'TYPE_GOOGLE_CREDENTIALS_FILE', u'keyAlgorithm': u'KEY_ALG_RSA_2048'}) - oauth2service_data = base64.b64decode(key[u'privateKeyData']) + print('Creating Service Account') + service_account = callGAPI(iam.projects().serviceAccounts(), 'create', + name='projects/%s' % projectId, + body={'accountId': projectId, 'serviceAccount': {'displayName': 'GAM Project'}}) + key = callGAPI(list(iam.projects().serviceAccounts().keys()), 'create', + name=service_account['name'], body={'privateKeyType': 'TYPE_GOOGLE_CREDENTIALS_FILE', 'keyAlgorithm': 'KEY_ALG_RSA_2048'}) + oauth2service_data = base64.b64decode(key['privateKeyData']) writeFile(GC_Values[GC_OAUTH2SERVICE_JSON], oauth2service_data, continueOnError=False) - console_credentials_url = u'https://console.developers.google.com/apis/credentials?project=%s' % projectId + console_credentials_url = 'https://console.developers.google.com/apis/credentials?project=%s' % projectId while True: - print u'''Please go to: + print('''Please go to: %s @@ -7387,22 +7397,22 @@ def _createClientSecretsOauth2service(httpObj, projectId): 3. Choose "Other". Enter a desired value for "Name". Click the blue "Create" button. 4. Copy your "client ID" value. -''' % console_credentials_url +''' % console_credentials_url) # If you use Firefox to copy the Client ID and Secret, the data has leading and trailing newlines # The first raw_input will get the leading newline, thus we have to issue another raw_input to get the data # If the newlines are not present, the data is correctly read with the first raw_input - client_id = raw_input(u'Enter your Client ID: ').strip() + client_id = input('Enter your Client ID: ').strip() if not client_id: - client_id = raw_input().strip() - print u'\nNow go back to your browser and copy your client secret.' - client_secret = raw_input(u'Enter your Client Secret: ').strip() + client_id = input().strip() + print('\nNow go back to your browser and copy your client secret.') + client_secret = input('Enter your Client Secret: ').strip() if not client_secret: - client_secret = raw_input().strip() + client_secret = input().strip() client_valid = _checkClientAndSecret(simplehttp, client_id, client_secret) if client_valid: break - print - cs_data = u'''{ + print() + cs_data = '''{ "installed": { "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -7417,38 +7427,38 @@ def _createClientSecretsOauth2service(httpObj, projectId): } }''' % (client_id, client_secret, projectId) writeFile(GC_Values[GC_CLIENT_SECRETS_JSON], cs_data, continueOnError=False) - print u'''Almost there! Now please switch back to your browser and: + print('''Almost there! Now please switch back to your browser and: 1. Click OK to close "OAuth client" popup if it's still open. 2. Click "Manage service accounts" on the right of the screen. 3. Click the 3 dots to the right of your service account. 4. Choose Edit. 5. Click "Show Domain-Wide Delegation". Check "Enable G Suite Domain-wide Delegation", Click Save. -''' - raw_input(u'Press Enter when done...') - print u'That\'s it! Your GAM Project is created and ready to use.' +''') + input('Press Enter when done...') + print('That\'s it! Your GAM Project is created and ready to use.') VALIDEMAIL_PATTERN = re.compile(r'^[^@]+@[^@]+\.[^@]+$') def _getValidateLoginHint(login_hint): while True: if not login_hint: - login_hint = raw_input(u'\nWhat is your G Suite admin email address? ').strip() - if login_hint.find(u'@') == -1 and GC_Values[GC_DOMAIN]: - login_hint = u'{0}@{1}'.format(login_hint, GC_Values[GC_DOMAIN].lower()) + login_hint = input('\nWhat is your G Suite admin email address? ').strip() + if login_hint.find('@') == -1 and GC_Values[GC_DOMAIN]: + login_hint = '{0}@{1}'.format(login_hint, GC_Values[GC_DOMAIN].lower()) if VALIDEMAIL_PATTERN.match(login_hint): return login_hint - print '{0}Invalid email address: {1}'.format(ERROR_PREFIX, login_hint) + print('{0}Invalid email address: {1}'.format(ERROR_PREFIX, login_hint)) login_hint = None def _getProjects(crm, pfilter): try: - return callGAPIpages(crm.projects(), u'list', u'projects', throw_reasons=[GAPI_BAD_REQUEST], filter=pfilter) + return callGAPIpages(crm.projects(), 'list', 'projects', throw_reasons=[GAPI_BAD_REQUEST], filter=pfilter) except GAPI_badRequest as e: - systemErrorExit(2, u'Project: {0}, {1}'.format(pfilter, str(e))) + systemErrorExit(2, 'Project: {0}, {1}'.format(pfilter, str(e))) PROJECTID_PATTERN = re.compile(r'^[a-z][a-z0-9-]{4,28}[a-z0-9]$') -PROJECTID_FORMAT_REQUIRED = u'[a-z][a-z0-9-]{4,28}[a-z0-9]' +PROJECTID_FORMAT_REQUIRED = '[a-z][a-z0-9-]{4,28}[a-z0-9]' def _getLoginHintProjectId(createCmd): login_hint = None @@ -7463,19 +7473,19 @@ def _getLoginHintProjectId(createCmd): if not PROJECTID_PATTERN.match(projectId): systemErrorExit(2, 'Invalid Project ID: {0}, expected <{1}>'.format(projectId, PROJECTID_FORMAT_REQUIRED)) elif createCmd: - projectId = u'gam-project' + projectId = 'gam-project' for _ in range(3): - projectId += u'-{0}'.format(u''.join(random.choice(string.digits + string.ascii_lowercase) for _ in range(3))) + projectId += '-{0}'.format(''.join(random.choice(string.digits + string.ascii_lowercase) for _ in range(3))) else: - projectId = raw_input(u'\nWhat is your API project ID? ').strip() + projectId = input('\nWhat is your API project ID? ').strip() if not PROJECTID_PATTERN.match(projectId): systemErrorExit(2, 'Invalid Project ID: {0}, expected <{1}>'.format(projectId, PROJECTID_FORMAT_REQUIRED)) crm, httpObj = getCRMService(login_hint) - projects = _getProjects(crm, u'id:{0}'.format(projectId)) + projects = _getProjects(crm, 'id:{0}'.format(projectId)) if not createCmd: if not projects: systemErrorExit(2, 'User: {0}, Project ID: {1}, Does not exist'.format(login_hint, projectId)) - if projects[0][u'lifecycleState'] != u'ACTIVE': + if projects[0]['lifecycleState'] != 'ACTIVE': systemErrorExit(2, 'User: {0}, Project ID: {1}, Not active'.format(login_hint, projectId)) else: if projects: @@ -7496,29 +7506,29 @@ def _getLoginHintProjects(printShowCmd): except IndexError: pass if not pfilter: - pfilter = u'current' if not printShowCmd else u'id:gam-project-*' - elif printShowCmd and pfilter.lower() == u'all': + pfilter = 'current' if not printShowCmd else 'id:gam-project-*' + elif printShowCmd and pfilter.lower() == 'all': pfilter = None - elif pfilter.lower() == u'gam': - pfilter = u'id:gam-project-*' - elif pfilter.lower() == u'filter': + elif pfilter.lower() == 'gam': + pfilter = 'id:gam-project-*' + elif pfilter.lower() == 'filter': pfilter = sys.argv[i] i += 1 elif PROJECTID_PATTERN.match(pfilter): - pfilter = u'id:{0}'.format(pfilter) + pfilter = 'id:{0}'.format(pfilter) else: - systemErrorExit(2, 'Invalid Project ID: {0}, expected <{1}{2}>'.format(pfilter, [u'', u'all|'][printShowCmd], PROJECTID_FILTER_REQUIRED)) + systemErrorExit(2, 'Invalid Project ID: {0}, expected <{1}{2}>'.format(pfilter, ['', 'all|'][printShowCmd], PROJECTID_FILTER_REQUIRED)) login_hint = _getValidateLoginHint(login_hint) crm, httpObj = getCRMService(login_hint) - if pfilter == u'current': - cs_data = readFile(GC_Values[GC_CLIENT_SECRETS_JSON], mode=u'rb', continueOnError=True, displayError=True, encoding=None) + if pfilter == 'current': + cs_data = readFile(GC_Values[GC_CLIENT_SECRETS_JSON], mode='rb', continueOnError=True, displayError=True, encoding=None) if not cs_data: - systemErrorExit(14, u'Your client secrets file:\n\n%s\n\nis missing. Please recreate the file.' % GC_Values[GC_CLIENT_SECRETS_JSON]) + systemErrorExit(14, 'Your client secrets file:\n\n%s\n\nis missing. Please recreate the file.' % GC_Values[GC_CLIENT_SECRETS_JSON]) try: cs_json = json.loads(cs_data) - projects = [{u'projectId': cs_json[u'installed'][u'project_id']}] + projects = [{'projectId': cs_json['installed']['project_id']}] except (ValueError, IndexError, KeyError): - systemErrorExit(3, u'The format of your client secrets file:\n\n%s\n\nis incorrect. Please recreate the file.' % GC_Values[GC_CLIENT_SECRETS_JSON]) + systemErrorExit(3, 'The format of your client secrets file:\n\n%s\n\nis incorrect. Please recreate the file.' % GC_Values[GC_CLIENT_SECRETS_JSON]) else: projects = _getProjects(crm, pfilter) return (crm, httpObj, login_hint, projects, i) @@ -7531,75 +7541,75 @@ def _checkForExistingProjectFiles(): def doCreateProject(): _checkForExistingProjectFiles() crm, httpObj, login_hint, projectId = _getLoginHintProjectId(True) - login_domain = login_hint[login_hint.find(u'@')+1:] - body = {u'projectId': projectId, u'name': u'GAM Project'} + login_domain = login_hint[login_hint.find('@')+1:] + body = {'projectId': projectId, 'name': 'GAM Project'} while True: create_again = False - print u'Creating project "%s"...' % body[u'name'] - create_operation = callGAPI(crm.projects(), u'create', body=body) - operation_name = create_operation[u'name'] + print('Creating project "%s"...' % body['name']) + create_operation = callGAPI(crm.projects(), 'create', body=body) + operation_name = create_operation['name'] time.sleep(5) # Google recommends always waiting at least 5 seconds for i in range(1, 5): - print u'Checking project status...' - status = callGAPI(crm.operations(), u'get', + print('Checking project status...') + status = callGAPI(crm.operations(), 'get', name=operation_name) - if u'error' in status: - if status[u'error'].get(u'message', u'') == u'No permission to create project in organization': - print u'Hmm... Looks like you have no rights to your Google Cloud Organization.' - print u'Attempting to fix that...' - getorg = callGAPI(crm.organizations(), u'search', - body={u'filter': u'domain:%s' % login_domain}) + if 'error' in status: + if status['error'].get('message', '') == 'No permission to create project in organization': + print('Hmm... Looks like you have no rights to your Google Cloud Organization.') + print('Attempting to fix that...') + getorg = callGAPI(crm.organizations(), 'search', + body={'filter': 'domain:%s' % login_domain}) try: - organization = getorg[u'organizations'][0][u'name'] - print u'Your organization name is %s' % organization + organization = getorg['organizations'][0]['name'] + print('Your organization name is %s' % organization) except (KeyError, IndexError): systemErrorExit(3, 'you have no rights to create projects for your organization and you don\'t seem to be a super admin! Sorry, there\'s nothing more I can do.') - org_policy = callGAPI(crm.organizations(), u'getIamPolicy', + org_policy = callGAPI(crm.organizations(), 'getIamPolicy', resource=organization, body={}) - if u'bindings' not in org_policy: - org_policy[u'bindings'] = [] - print u'Looks like no one has rights to your Google Cloud Organization. Attempting to give you create rights...' + if 'bindings' not in org_policy: + org_policy['bindings'] = [] + print('Looks like no one has rights to your Google Cloud Organization. Attempting to give you create rights...') else: - print u'The following rights seem to exist:' - for a_policy in org_policy[u'bindings']: - if u'role' in a_policy: - print u' Role: %s' % a_policy[u'role'] - if u'members' in a_policy: - print u' Members:' - for member in a_policy[u'members']: - print u' %s' % member - print - my_role = u'roles/resourcemanager.projectCreator' - print u'Giving %s the role of %s...' % (login_hint, my_role) - org_policy[u'bindings'].append({u'role': my_role, u'members': [u'user:%s' % login_hint]}) - callGAPI(crm.organizations(), u'setIamPolicy', - resource=organization, body={u'policy': org_policy}) + print('The following rights seem to exist:') + for a_policy in org_policy['bindings']: + if 'role' in a_policy: + print(' Role: %s' % a_policy['role']) + if 'members' in a_policy: + print(' Members:') + for member in a_policy['members']: + print(' %s' % member) + print() + my_role = 'roles/resourcemanager.projectCreator' + print('Giving %s the role of %s...' % (login_hint, my_role)) + org_policy['bindings'].append({'role': my_role, 'members': ['user:%s' % login_hint]}) + callGAPI(crm.organizations(), 'setIamPolicy', + resource=organization, body={'policy': org_policy}) create_again = True break try: - if status[u'error'][u'details'][0][u'violations'][0][u'description'] == u'Callers must accept Terms of Service': - print u'''Please go to: + if status['error']['details'][0]['violations'][0]['description'] == 'Callers must accept Terms of Service': + print('''Please go to: https://console.cloud.google.com/start -and accept the Terms of Service (ToS). As soon as you've accepted the ToS popup, you can return here and press enter.''' - raw_input() +and accept the Terms of Service (ToS). As soon as you've accepted the ToS popup, you can return here and press enter.''') + input() create_again = True break except (IndexError, KeyError): pass systemErrorExit(1, status) - if status.get(u'done', False): + if status.get('done', False): break sleep_time = i ** 2 - print u'Project still being created. Sleeping %s seconds' % sleep_time + print('Project still being created. Sleeping %s seconds' % sleep_time) time.sleep(sleep_time) if create_again: continue - if not status.get(u'done', False): - systemErrorExit(1, u'Failed to create project: %s' % status) - elif u'error' in status: - systemErrorExit(2, status[u'error']) + if not status.get('done', False): + systemErrorExit(1, 'Failed to create project: %s' % status) + elif 'error' in status: + systemErrorExit(2, status['error']) break _createClientSecretsOauth2service(httpObj, projectId) @@ -7612,26 +7622,26 @@ def doUpdateProjects(): _, httpObj, login_hint, projects, _ = _getLoginHintProjects(False) _, GAMProjectAPIs = getGAMProjectAPIs() count = len(projects) - print u'User: {0}, Update {1} Projects'.format(login_hint, count) + print('User: {0}, Update {1} Projects'.format(login_hint, count)) i = 0 for project in projects: i += 1 - projectId = project[u'projectId'] + projectId = project['projectId'] enableGAMProjectAPIs(GAMProjectAPIs, httpObj, projectId, True, i, count) def doDelProjects(): crm, _, login_hint, projects, _ = _getLoginHintProjects(False) count = len(projects) - print u'User: {0}, Delete {1} Projects'.format(login_hint, count) + print('User: {0}, Delete {1} Projects'.format(login_hint, count)) i = 0 for project in projects: i += 1 - projectId = project[u'projectId'] + projectId = project['projectId'] try: - callGAPI(crm.projects(), u'delete', throw_reasons=[GAPI_FORBIDDEN], projectId=projectId) - print u' Project: {0} Deleted{1}'.format(projectId, currentCount(i, count)) + callGAPI(crm.projects(), 'delete', throw_reasons=[GAPI_FORBIDDEN], projectId=projectId) + print(' Project: {0} Deleted{1}'.format(projectId, currentCount(i, count))) except GAPI_forbidden as e: - print u' Project: {0} Delete Failed: {1}{2}'.format(projectId, str(e), currentCount(i, count)) + print(' Project: {0} Delete Failed: {1}{2}'.format(projectId, str(e), currentCount(i, count))) def doPrintShowProjects(csvFormat): @@ -7639,83 +7649,83 @@ def doPrintShowProjects(csvFormat): if csvFormat: csvRows = [] todrive = False - titles = [u'User', u'projectId', u'projectNumber', u'name', u'createTime', u'lifecycleState'] + titles = ['User', 'projectId', 'projectNumber', 'name', 'createTime', 'lifecycleState'] while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s projects"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s projects"' % (myarg, ['show', 'print'][csvFormat])) if not csvFormat: count = len(projects) - print u'User: {0}, Show {1} Projects'.format(login_hint, count) + print('User: {0}, Show {1} Projects'.format(login_hint, count)) i = 0 for project in projects: i += 1 - print u' Project: {0}{1}'.format(project[u'projectId'], currentCount(i, count)) - print u' projectNumber: {0}'.format(project[u'projectNumber']) - print u' name: {0}'.format(project[u'name']) - print u' createTime: {0}'.format(project[u'createTime']) - print u' lifecycleState: {0}'.format(project[u'lifecycleState']) - jcount = len(project.get(u'labels', [])) + print(' Project: {0}{1}'.format(project['projectId'], currentCount(i, count))) + print(' projectNumber: {0}'.format(project['projectNumber'])) + print(' name: {0}'.format(project['name'])) + print(' createTime: {0}'.format(project['createTime'])) + print(' lifecycleState: {0}'.format(project['lifecycleState'])) + jcount = len(project.get('labels', [])) if jcount > 0: - print u' labels:' - for k, v in project[u'labels'].items(): - print u' {0}: {1}'.format(k, v) - if u'parent' in project: - print u' parent:' - print u' type: {0}'.format(project[u'parent'][u'type']) - print u' id: {0}'.format(project[u'parent'][u'id']) + print(' labels:') + for k, v in list(project['labels'].items()): + print(' {0}: {1}'.format(k, v)) + if 'parent' in project: + print(' parent:') + print(' type: {0}'.format(project['parent']['type'])) + print(' id: {0}'.format(project['parent']['id'])) else: for project in projects: - addRowTitlesToCSVfile(flatten_json(project, flattened={u'User': login_hint}), csvRows, titles) - writeCSVfile(csvRows, titles, u'Projects', todrive) + addRowTitlesToCSVfile(flatten_json(project, flattened={'User': login_hint}), csvRows, titles) + writeCSVfile(csvRows, titles, 'Projects', todrive) def doGetTeamDriveInfo(users): teamDriveId = sys.argv[5] useDomainAdminAccess = False i = 6 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'asadmin': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'asadmin': useDomainAdminAccess = True i += 1 else: systemErrorExit(3, '%s is not a valid command for "gam show teamdrive"' % sys.argv[i]) for user in users: - drive = buildGAPIServiceObject(u'drive3', user) + drive = buildGAPIServiceObject('drive3', user) if not drive: - print u'Failed to access Drive as %s' % user + print('Failed to access Drive as %s' % user) continue - result = callGAPI(drive.teamdrives(), u'get', teamDriveId=teamDriveId, - useDomainAdminAccess=useDomainAdminAccess, fields=u'*') + result = callGAPI(drive.teamdrives(), 'get', teamDriveId=teamDriveId, + useDomainAdminAccess=useDomainAdminAccess, fields='*') print_json(None, result) def doCreateTeamDrive(users): - body = {u'name': sys.argv[5]} + body = {'name': sys.argv[5]} i = 6 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'theme': - body[u'themeId'] = sys.argv[i+1] + if myarg == 'theme': + body['themeId'] = sys.argv[i+1] i += 2 else: systemErrorExit(3, '%s is not a valid argument to "gam create teamdrive"' % sys.argv[i]) for user in users: - drive = buildGAPIServiceObject(u'drive3', user) + drive = buildGAPIServiceObject('drive3', user) if not drive: - print u'Failed to access Drive as %s' % user + print('Failed to access Drive as %s' % user) continue - requestId = unicode(uuid.uuid4()) - result = callGAPI(drive.teamdrives(), u'create', requestId=requestId, body=body, fields=u'id') - print u'Created Team Drive %s with id %s' % (body[u'name'], result[u'id']) + requestId = str(uuid.uuid4()) + result = callGAPI(drive.teamdrives(), 'create', requestId=requestId, body=body, fields='id') + print('Created Team Drive %s with id %s' % (body['name'], result['id'])) TEAMDRIVE_RESTRICTIONS_MAP = { - u'adminmanagedrestrictions': u'adminManagedRestrictions', - u'copyrequireswriterpermission': u'copyRequiresWriterPermission', - u'domainusersonly': u'domainUsersOnly', - u'teammembersonly': u'teamMembersOnly', + 'adminmanagedrestrictions': 'adminManagedRestrictions', + 'copyrequireswriterpermission': 'copyRequiresWriterPermission', + 'domainusersonly': 'domainUsersOnly', + 'teammembersonly': 'teamMembersOnly', } def doUpdateTeamDrive(users): @@ -7724,30 +7734,30 @@ def doUpdateTeamDrive(users): useDomainAdminAccess = False i = 6 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'name'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 - elif myarg == u'theme': - body[u'themeId'] = sys.argv[i+1] + elif myarg == 'theme': + body['themeId'] = sys.argv[i+1] i += 2 - elif myarg == u'customtheme': - body[u'backgroundImageFile'] = { - u'id': sys.argv[i+1], - u'xCoordinate': float(sys.argv[i+2]), - u'yCoordinate': float(sys.argv[i+3]), - u'width': float(sys.argv[i+4]) + elif myarg == 'customtheme': + body['backgroundImageFile'] = { + 'id': sys.argv[i+1], + 'xCoordinate': float(sys.argv[i+2]), + 'yCoordinate': float(sys.argv[i+3]), + 'width': float(sys.argv[i+4]) } i += 5 - elif myarg == u'color': - body[u'colorRgb'] = getColor(sys.argv[i+1]) + elif myarg == 'color': + body['colorRgb'] = getColor(sys.argv[i+1]) i += 2 - elif myarg == u'asadmin': + elif myarg == 'asadmin': useDomainAdminAccess = True i += 1 elif myarg in TEAMDRIVE_RESTRICTIONS_MAP: - body.setdefault(u'restrictions', {}) - body[u'restrictions'][TEAMDRIVE_RESTRICTIONS_MAP[myarg]] = getBoolean(sys.argv[i+1], myarg) + body.setdefault('restrictions', {}) + body['restrictions'][TEAMDRIVE_RESTRICTIONS_MAP[myarg]] = getBoolean(sys.argv[i+1], myarg) i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam update teamdrive"' % sys.argv[i]) @@ -7757,11 +7767,11 @@ def doUpdateTeamDrive(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - result = callGAPI(drive.teamdrives(), u'update', - useDomainAdminAccess=useDomainAdminAccess, body=body, teamDriveId=teamDriveId, fields=u'id', soft_errors=True) + result = callGAPI(drive.teamdrives(), 'update', + useDomainAdminAccess=useDomainAdminAccess, body=body, teamDriveId=teamDriveId, fields='id', soft_errors=True) if not result: continue - print u'Updated Team Drive %s' % (teamDriveId) + print('Updated Team Drive %s' % (teamDriveId)) def printShowTeamDrives(users, csvFormat): todrive = False @@ -7769,44 +7779,44 @@ def printShowTeamDrives(users, csvFormat): q = None i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'asadmin': + elif myarg == 'asadmin': useDomainAdminAccess = True i += 1 - elif myarg == u'query': + elif myarg == 'query': q = sys.argv[i+1] i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam print|show teamdrives"') tds = [] for user in users: - sys.stderr.write(u'Getting Team Drives for %s\n' % user) + sys.stderr.write('Getting Team Drives for %s\n' % user) user, drive = buildDrive3GAPIObject(user) if not drive: continue - results = callGAPIpages(drive.teamdrives(), u'list', u'teamDrives', - useDomainAdminAccess=useDomainAdminAccess, fields=u'*', + results = callGAPIpages(drive.teamdrives(), 'list', 'teamDrives', + useDomainAdminAccess=useDomainAdminAccess, fields='*', q=q, soft_errors=True) if not results: continue for td in results: - if u'id' not in td: + if 'id' not in td: continue - if u'name' not in td: - td[u'name'] = u'' - this_td = {u'id': td[u'id'], u'name': td[u'name']} + if 'name' not in td: + td['name'] = '' + this_td = {'id': td['id'], 'name': td['name']} if this_td in tds: continue - tds.append({u'id': td[u'id'], u'name': td[u'name']}) + tds.append({'id': td['id'], 'name': td['name']}) if csvFormat: - titles = [u'name', u'id'] - writeCSVfile(tds, titles, u'Team Drives', todrive) + titles = ['name', 'id'] + writeCSVfile(tds, titles, 'Team Drives', todrive) else: for td in tds: - print u'Name: %s ID: %s' % (td[u'name'], td[u'id']) + print('Name: %s ID: %s' % (td['name'], td['id'])) def doDeleteTeamDrive(users): teamDriveId = sys.argv[5] @@ -7814,224 +7824,224 @@ def doDeleteTeamDrive(users): user, drive = buildDrive3GAPIObject(user) if not drive: continue - print u'Deleting Team Drive %s' % (teamDriveId) - callGAPI(drive.teamdrives(), u'delete', teamDriveId=teamDriveId, soft_errors=True) + print('Deleting Team Drive %s' % (teamDriveId)) + callGAPI(drive.teamdrives(), 'delete', teamDriveId=teamDriveId, soft_errors=True) def validateCollaborators(collaboratorList, cd): collaborators = [] - for collaborator in collaboratorList.split(u','): + for collaborator in collaboratorList.split(','): collaborator_id = convertEmailAddressToUID(collaborator, cd) if not collaborator_id: systemErrorExit(4, 'failed to get a UID for %s. Please make sure this is a real user.' % collaborator) - collaborators.append({u'email': collaborator, u'id': collaborator_id}) + collaborators.append({'email': collaborator, 'id': collaborator_id}) return collaborators def doCreateVaultMatter(): - v = buildGAPIObject(u'vault') - body = {u'name': u'New Matter - %s' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")} + v = buildGAPIObject('vault') + body = {'name': 'New Matter - %s' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")} collaborators = [] cd = None i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'name'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 - elif myarg == u'description': - body[u'description'] = sys.argv[i+1] + elif myarg == 'description': + body['description'] = sys.argv[i+1] i += 2 - elif myarg in [u'collaborator', u'collaborators']: + elif myarg in ['collaborator', 'collaborators']: if not cd: - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') collaborators.extend(validateCollaborators(sys.argv[i+1], cd)) i += 2 else: systemErrorExit(3, '%s is not a valid argument to "gam create matter"' % sys.argv[i]) - matterId = callGAPI(v.matters(), u'create', body=body, fields=u'matterId')[u'matterId'] - print u'Created matter %s' % matterId + matterId = callGAPI(v.matters(), 'create', body=body, fields='matterId')['matterId'] + print('Created matter %s' % matterId) for collaborator in collaborators: - print u' adding collaborator %s' % collaborator[u'email'] - callGAPI(v.matters(), u'addPermissions', matterId=matterId, body={u'matterPermission': {u'role': u'COLLABORATOR', u'accountId': collaborator[u'id']}}) + print(' adding collaborator %s' % collaborator['email']) + callGAPI(v.matters(), 'addPermissions', matterId=matterId, body={'matterPermission': {'role': 'COLLABORATOR', 'accountId': collaborator['id']}}) VAULT_SEARCH_METHODS_MAP = { - u'account': u'ACCOUNT', - u'accounts': u'ACCOUNT', - u'entireorg': u'ENTIRE_ORG', - u'everyone': u'ENTIRE_ORG', - u'orgunit': u'ORG_UNIT', - u'ou': u'ORG_UNIT', - u'room': u'ROOM', - u'rooms': u'ROOM', - u'teamdrive': u'TEAM_DRIVE', - u'teamdrives': u'TEAM_DRIVE', + 'account': 'ACCOUNT', + 'accounts': 'ACCOUNT', + 'entireorg': 'ENTIRE_ORG', + 'everyone': 'ENTIRE_ORG', + 'orgunit': 'ORG_UNIT', + 'ou': 'ORG_UNIT', + 'room': 'ROOM', + 'rooms': 'ROOM', + 'teamdrive': 'TEAM_DRIVE', + 'teamdrives': 'TEAM_DRIVE', } -VAULT_SEARCH_METHODS_LIST = [u'accounts', u'orgunit', u'teamdrives', u'rooms', u'everyone'] +VAULT_SEARCH_METHODS_LIST = ['accounts', 'orgunit', 'teamdrives', 'rooms', 'everyone'] def doCreateVaultExport(): - v = buildGAPIObject(u'vault') - allowed_corpuses = v._rootDesc[u'schemas'][u'Query'][u'properties'][u'corpus'][u'enum'] + v = buildGAPIObject('vault') + allowed_corpuses = v._rootDesc['schemas']['Query']['properties']['corpus']['enum'] try: - allowed_corpuses.remove(u'CORPUS_TYPE_UNSPECIFIED') + allowed_corpuses.remove('CORPUS_TYPE_UNSPECIFIED') except ValueError: pass - allowed_scopes = v._rootDesc[u'schemas'][u'Query'][u'properties'][u'dataScope'][u'enum'] + allowed_scopes = v._rootDesc['schemas']['Query']['properties']['dataScope']['enum'] try: - allowed_scopes.remove(u'DATA_SCOPE_UNSPECIFIED') + allowed_scopes.remove('DATA_SCOPE_UNSPECIFIED') except ValueError: pass - allowed_formats = [u'MBOX', u'PST'] - export_format = u'MBOX' + allowed_formats = ['MBOX', 'PST'] + export_format = 'MBOX' matterId = None - body = {u'query': {u'dataScope': u'ALL_DATA'}, u'exportOptions': {}} + body = {'query': {'dataScope': 'ALL_DATA'}, 'exportOptions': {}} i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'matter': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'matter': matterId = getMatterItem(v, sys.argv[i+1]) - body[u'matterId'] = matterId + body['matterId'] = matterId i += 2 - elif myarg == u'name': - body[u'name'] = sys.argv[i+1] + elif myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 - elif myarg == u'corpus': - body[u'query'][u'corpus'] = sys.argv[i+1].upper() - if body[u'query'][u'corpus'] not in allowed_corpuses: - systemErrorExit(3, 'corpus must be one of %s. Got %s' % (u', '.join(allowed_corpuses), sys.argv[i+1])) + elif myarg == 'corpus': + body['query']['corpus'] = sys.argv[i+1].upper() + if body['query']['corpus'] not in allowed_corpuses: + systemErrorExit(3, 'corpus must be one of %s. Got %s' % (', '.join(allowed_corpuses), sys.argv[i+1])) i += 2 elif myarg in VAULT_SEARCH_METHODS_MAP: - if body[u'query'].get(u'searchMethod'): - systemErrorExit(3, 'Multiple search methods ({0}) specified, only one is allowed'.format(u', '.join(VAULT_SEARCH_METHODS_LIST))) + if body['query'].get('searchMethod'): + systemErrorExit(3, 'Multiple search methods ({0}) specified, only one is allowed'.format(', '.join(VAULT_SEARCH_METHODS_LIST))) searchMethod = VAULT_SEARCH_METHODS_MAP[myarg] - body[u'query'][u'searchMethod'] = searchMethod - if searchMethod == u'ACCOUNT': - body[u'query'][u'accountInfo'] = {u'emails': sys.argv[i+1].split(u',')} + body['query']['searchMethod'] = searchMethod + if searchMethod == 'ACCOUNT': + body['query']['accountInfo'] = {'emails': sys.argv[i+1].split(',')} i += 2 - elif searchMethod == u'ORG_UNIT': - body[u'query'][u'orgUnitInfo'] = {u'orgUnitId': getOrgUnitId(sys.argv[i+1])[1]} + elif searchMethod == 'ORG_UNIT': + body['query']['orgUnitInfo'] = {'orgUnitId': getOrgUnitId(sys.argv[i+1])[1]} i += 2 - elif searchMethod == u'TEAM_DRIVE': - body[u'query'][u'teamDriveInfo'] = {u'teamDriveIds': sys.argv[i+1].split(u',')} + elif searchMethod == 'TEAM_DRIVE': + body['query']['teamDriveInfo'] = {'teamDriveIds': sys.argv[i+1].split(',')} i += 2 - elif searchMethod == u'ROOM': - body[u'query'][u'hangoutsChatInfo'] = {u'roomId': sys.argv[i+1].split(u',')} + elif searchMethod == 'ROOM': + body['query']['hangoutsChatInfo'] = {'roomId': sys.argv[i+1].split(',')} i += 2 else: i += 1 - elif myarg == u'scope': - body[u'query'][u'dataScope'] = sys.argv[i+1].upper() - if body[u'query']['dataScope'] not in allowed_scopes: - systemErrorExit(3, 'scope must be one of %s. Got %s' % (u', '.join(allowed_scopes), sys.argv[i+1])) + elif myarg == 'scope': + body['query']['dataScope'] = sys.argv[i+1].upper() + if body['query']['dataScope'] not in allowed_scopes: + systemErrorExit(3, 'scope must be one of %s. Got %s' % (', '.join(allowed_scopes), sys.argv[i+1])) i += 2 - elif myarg in [u'terms']: - body[u'query'][u'terms'] = sys.argv[i+1] + elif myarg in ['terms']: + body['query']['terms'] = sys.argv[i+1] i += 2 - elif myarg in [u'start', u'starttime']: - body[u'query'][u'startTime'] = getDateZeroTimeOrFullTime(sys.argv[i+1]) + elif myarg in ['start', 'starttime']: + body['query']['startTime'] = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg in [u'end', u'endtime']: - body[u'query'][u'endTime'] = getDateZeroTimeOrFullTime(sys.argv[i+1]) + elif myarg in ['end', 'endtime']: + body['query']['endTime'] = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg in [u'timezone']: - body[u'query'][u'timeZone'] = sys.argv[i+1] + elif myarg in ['timezone']: + body['query']['timeZone'] = sys.argv[i+1] i += 2 - elif myarg in [u'excludedrafts']: - body[u'query'][u'mailOptions'] = {u'excludeDrafts': getBoolean(sys.argv[i+1], myarg)} + elif myarg in ['excludedrafts']: + body['query']['mailOptions'] = {'excludeDrafts': getBoolean(sys.argv[i+1], myarg)} i += 2 - elif myarg in [u'driveversiondate']: - body[u'query'].setdefault(u'driveOptions', {})[u'versionDate'] = getDateZeroTimeOrFullTime(sys.argv[i+1]) + elif myarg in ['driveversiondate']: + body['query'].setdefault('driveOptions', {})['versionDate'] = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg in [u'includeteamdrives']: - body[u'query'].setdefault(u'driveOptions', {})[u'includeTeamDrives'] = getBoolean(sys.argv[i+1], myarg) + elif myarg in ['includeteamdrives']: + body['query'].setdefault('driveOptions', {})['includeTeamDrives'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg in [u'includerooms']: - body[u'query'][u'hangoutsChatOptions'] = {u'includeRooms': getBoolean(sys.argv[i+1], myarg)} + elif myarg in ['includerooms']: + body['query']['hangoutsChatOptions'] = {'includeRooms': getBoolean(sys.argv[i+1], myarg)} i += 2 - elif myarg in [u'format']: + elif myarg in ['format']: export_format = sys.argv[i+1].upper() if export_format not in allowed_formats: - print u'ERROR: export format can be one of %s, got %s' % (u', '.join(allowed_formats), export_format) + print('ERROR: export format can be one of %s, got %s' % (', '.join(allowed_formats), export_format)) sys.exit(3) i += 2 - elif myarg in [u'includeaccessinfo']: - body[u'exportOptions'].setdefault(u'driveOptions', {})[u'includeAccessInfo'] = getBoolean(sys.argv[i+1], myarg) + elif myarg in ['includeaccessinfo']: + body['exportOptions'].setdefault('driveOptions', {})['includeAccessInfo'] = getBoolean(sys.argv[i+1], myarg) i += 2 else: systemErrorExit(3, '%s is not a valid argument to "gam create export"' % sys.argv[i]) if not matterId: systemErrorExit(3, 'you must specify a matter for the new export.') - if u'corpus' not in body[u'query']: - systemErrorExit(3, 'you must specify a corpus for the new export. Choose one of %s' % u', '.join(allowed_corpuses)) - if u'searchMethod' not in body[u'query']: - systemErrorExit(3, 'you must specify a search method for the new export. Choose one of %s' % u', '.join(VAULT_SEARCH_METHODS_LIST)) - if u'name' not in body: - body[u'name'] = u'GAM %s export - %s' % (body[u'query'][u'corpus'], datetime.datetime.now()) + if 'corpus' not in body['query']: + systemErrorExit(3, 'you must specify a corpus for the new export. Choose one of %s' % ', '.join(allowed_corpuses)) + if 'searchMethod' not in body['query']: + systemErrorExit(3, 'you must specify a search method for the new export. Choose one of %s' % ', '.join(VAULT_SEARCH_METHODS_LIST)) + if 'name' not in body: + body['name'] = 'GAM %s export - %s' % (body['query']['corpus'], datetime.datetime.now()) options_field = None - if body[u'query'][u'corpus'] == u'MAIL': - options_field = u'mailOptions' - elif body[u'query'][u'corpus'] == u'GROUPS': - options_field = u'groupsOptions' - elif body[u'query'][u'corpus'] == u'HANGOUTS_CHAT': - options_field = u'hangoutsChatOptions' + if body['query']['corpus'] == 'MAIL': + options_field = 'mailOptions' + elif body['query']['corpus'] == 'GROUPS': + options_field = 'groupsOptions' + elif body['query']['corpus'] == 'HANGOUTS_CHAT': + options_field = 'hangoutsChatOptions' if options_field: - body[u'exportOptions'].pop(u'driveOptions', None) - body[u'exportOptions'][options_field] = {u'exportFormat': export_format} - results = callGAPI(v.matters().exports(), u'create', matterId=matterId, body=body) - print u'Created export %s' % results[u'id'] + body['exportOptions'].pop('driveOptions', None) + body['exportOptions'][options_field] = {'exportFormat': export_format} + results = callGAPI(v.matters().exports(), 'create', matterId=matterId, body=body) + print('Created export %s' % results['id']) print_json(None, results) def doDeleteVaultExport(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') matterId = getMatterItem(v, sys.argv[3]) exportId = convertExportNameToID(v, sys.argv[4], matterId) - print u'Deleting export %s / %s' % (sys.argv[4], exportId) - callGAPI(v.matters().exports(), u'delete', matterId=matterId, exportId=exportId) + print('Deleting export %s / %s' % (sys.argv[4], exportId)) + callGAPI(v.matters().exports(), 'delete', matterId=matterId, exportId=exportId) def doGetVaultExportInfo(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') matterId = getMatterItem(v, sys.argv[3]) exportId = convertExportNameToID(v, sys.argv[4], matterId) - export = callGAPI(v.matters().exports(), u'get', matterId=matterId, exportId=exportId) + export = callGAPI(v.matters().exports(), 'get', matterId=matterId, exportId=exportId) print_json(None, export) def doDownloadVaultExport(): verifyFiles = True extractFiles = True - v = buildGAPIObject(u'vault') - s = buildGAPIObject(u'storage') + v = buildGAPIObject('vault') + s = buildGAPIObject('storage') matterId = getMatterItem(v, sys.argv[3]) exportId = convertExportNameToID(v, sys.argv[4], matterId) targetFolder = GC_Values[GC_DRIVE_DIR] i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'targetfolder': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'targetfolder': targetFolder = os.path.expanduser(sys.argv[i+1]) if not os.path.isdir(targetFolder): os.makedirs(targetFolder) i += 2 - elif myarg == u'noverify': + elif myarg == 'noverify': verifyFiles = False i += 1 - elif myarg == u'noextract': + elif myarg == 'noextract': extractFiles = False i += 1 else: systemErrorExit(3, '%s is not a valid argument to "gam download export"' % sys.argv[i]) - export = callGAPI(v.matters().exports(), u'get', matterId=matterId, exportId=exportId) - for s_file in export[u'cloudStorageSink']['files']: + export = callGAPI(v.matters().exports(), 'get', matterId=matterId, exportId=exportId) + for s_file in export['cloudStorageSink']['files']: bucket = s_file['bucketName'] s_object = s_file['objectName'] - filename = os.path.join(targetFolder, s_object.replace(u'/', u'-')) - print u'saving to %s' % filename + filename = os.path.join(targetFolder, s_object.replace('/', '-')) + print('saving to %s' % filename) request = s.objects().get_media(bucket=bucket, object=s_object) f = openFile(filename, 'wb') downloader = googleapiclient.http.MediaIoBaseDownload(f, request) done = False while not done: status, done = downloader.next_chunk() - sys.stdout.write(u' Downloaded: {0:>7.2%}\r'.format(status.progress())) + sys.stdout.write(' Downloaded: {0:>7.2%}\r'.format(status.progress())) sys.stdout.flush() - sys.stdout.write(u'\n Download complete. Flushing to disk...\n') + sys.stdout.write('\n Download complete. Flushing to disk...\n') # Necessary to make sure file is flushed by both Python and OS # https://stackoverflow.com/a/13762137/1503886 f.flush() @@ -8040,39 +8050,39 @@ def doDownloadVaultExport(): f = openFile(filename, 'rb') if verifyFiles: expected_hash = s_file['md5Hash'] - sys.stdout.write(u' Verifying file hash is %s...' % expected_hash) + sys.stdout.write(' Verifying file hash is %s...' % expected_hash) sys.stdout.flush() hash_md5 = hashlib.md5() for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) actual_hash = hash_md5.hexdigest() if actual_hash == expected_hash: - print u'VERIFIED' + print('VERIFIED') else: - print u'ERROR: actual hash was %s. Exiting on corrupt file.' % actual_hash + print('ERROR: actual hash was %s. Exiting on corrupt file.' % actual_hash) sys.exit(6) closeFile(f) if extractFiles and re.search(r'\.zip$', filename): extract_nested_zip(filename, targetFolder) -def extract_nested_zip(zippedFile, toFolder, spacing=u' '): +def extract_nested_zip(zippedFile, toFolder, spacing=' '): """ Extract a zip file including any nested zip files Delete the zip file(s) after extraction """ - print u'%sextracting %s' % (spacing, zippedFile) + print('%sextracting %s' % (spacing, zippedFile)) with zipfile.ZipFile(zippedFile, 'r') as zfile: inner_files = zfile.infolist() for inner_file in inner_files: - print u'%s %s' % (spacing, inner_file.filename) + print('%s %s' % (spacing, inner_file.filename)) inner_file_path = zfile.extract(inner_file, toFolder) if re.search(r'\.zip$', inner_file.filename): - extract_nested_zip(inner_file_path, toFolder, spacing=spacing+u' ') + extract_nested_zip(inner_file_path, toFolder, spacing=spacing+' ') os.remove(zippedFile) def doCreateVaultHold(): - v = buildGAPIObject(u'vault') - allowed_corpuses = v._rootDesc[u'schemas'][u'Hold'][u'properties'][u'corpus'][u'enum'] - body = {u'query': {}} + v = buildGAPIObject('vault') + allowed_corpuses = v._rootDesc['schemas']['Hold']['properties']['corpus']['enum'] + body = {'query': {}} i = 3 query = None start_time = None @@ -8080,76 +8090,76 @@ def doCreateVaultHold(): matterId = None accounts = [] while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'name'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 - elif myarg == u'query': + elif myarg == 'query': query = sys.argv[i+1] i += 2 - elif myarg == u'corpus': - body[u'corpus'] = sys.argv[i+1].upper() - if body[u'corpus'] not in allowed_corpuses: - systemErrorExit(3, 'corpus must be one of %s. Got %s' % (u', '.join(allowed_corpuses), sys.argv[i+1])) + elif myarg == 'corpus': + body['corpus'] = sys.argv[i+1].upper() + if body['corpus'] not in allowed_corpuses: + systemErrorExit(3, 'corpus must be one of %s. Got %s' % (', '.join(allowed_corpuses), sys.argv[i+1])) i += 2 - elif myarg in [u'accounts', u'users', u'groups']: - accounts = sys.argv[i+1].split(u',') + elif myarg in ['accounts', 'users', 'groups']: + accounts = sys.argv[i+1].split(',') i += 2 - elif myarg in [u'orgunit', u'ou']: - body[u'orgUnit'] = {u'orgUnitId': getOrgUnitId(sys.argv[i+1])[1]} + elif myarg in ['orgunit', 'ou']: + body['orgUnit'] = {'orgUnitId': getOrgUnitId(sys.argv[i+1])[1]} i += 2 - elif myarg in [u'start', u'starttime']: + elif myarg in ['start', 'starttime']: start_time = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg in [u'end', u'endtime']: + elif myarg in ['end', 'endtime']: end_time = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg == u'matter': + elif myarg == 'matter': matterId = getMatterItem(v, sys.argv[i+1]) i += 2 else: systemErrorExit(3, '%s is not a valid argument to "gam create hold"' % sys.argv[i]) if not matterId: systemErrorExit(3, 'you must specify a matter for the new hold.') - if not body.get(u'name'): + if not body.get('name'): systemErrorExit(3, 'you must specify a name for the new hold.') - if not body.get(u'corpus'): - systemErrorExit(3, 'you must specify a corpus for the new hold. Choose one of %s' % (u', '.join(allowed_corpuses))) - if body[u'corpus'] == u'HANGOUTS_CHAT': - query_type = u'hangoutsChatQuery' + if not body.get('corpus'): + systemErrorExit(3, 'you must specify a corpus for the new hold. Choose one of %s' % (', '.join(allowed_corpuses))) + if body['corpus'] == 'HANGOUTS_CHAT': + query_type = 'hangoutsChatQuery' else: - query_type = u'%sQuery' % body[u'corpus'].lower() - body[u'query'][query_type] = {} - if body[u'corpus'] == u'DRIVE': + query_type = '%sQuery' % body['corpus'].lower() + body['query'][query_type] = {} + if body['corpus'] == 'DRIVE': if query: try: - body[u'query'][query_type] = json.loads(query) + body['query'][query_type] = json.loads(query) except ValueError as e: systemErrorExit(3, '{0}, query: {1}'.format(str(e), query)) - elif body[u'corpus'] in [u'GROUPS', u'MAIL']: + elif body['corpus'] in ['GROUPS', 'MAIL']: if query: - body[u'query'][query_type] = {u'terms': query} + body['query'][query_type] = {'terms': query} if start_time: - body[u'query'][query_type][u'startTime'] = start_time + body['query'][query_type]['startTime'] = start_time if end_time: - body[u'query'][query_type][u'endTime'] = end_time + body['query'][query_type]['endTime'] = end_time if accounts: - body[u'accounts'] = [] - cd = buildGAPIObject(u'directory') - account_type = u'group' if body[u'corpus'] == u'GROUPS' else u'user' + body['accounts'] = [] + cd = buildGAPIObject('directory') + account_type = 'group' if body['corpus'] == 'GROUPS' else 'user' for account in accounts: - body[u'accounts'].append({u'accountId': convertEmailAddressToUID(account, cd, account_type)}) - holdId = callGAPI(v.matters().holds(), u'create', matterId=matterId, body=body, fields=u'holdId')[u'holdId'] - print u'Created hold %s' % holdId + body['accounts'].append({'accountId': convertEmailAddressToUID(account, cd, account_type)}) + holdId = callGAPI(v.matters().holds(), 'create', matterId=matterId, body=body, fields='holdId')['holdId'] + print('Created hold %s' % holdId) def doDeleteVaultHold(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') hold = sys.argv[3] matterId = None i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'matter': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'matter': matterId = getMatterItem(v, sys.argv[i+1]) holdId = convertHoldNameToID(v, hold, matterId) i += 2 @@ -8157,17 +8167,17 @@ def doDeleteVaultHold(): systemErrorExit(3, '%s is not a valid argument to "gam delete hold"' % myarg) if not matterId: systemErrorExit(3, 'you must specify a matter for the hold.') - print u'Deleting hold %s / %s' % (hold, holdId) - callGAPI(v.matters().holds(), u'delete', matterId=matterId, holdId=holdId) + print('Deleting hold %s / %s' % (hold, holdId)) + callGAPI(v.matters().holds(), 'delete', matterId=matterId, holdId=holdId) def doGetVaultHoldInfo(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') hold = sys.argv[3] matterId = None i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'matter': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'matter': matterId = getMatterItem(v, sys.argv[i+1]) holdId = convertHoldNameToID(v, hold, matterId) i += 2 @@ -8175,16 +8185,16 @@ def doGetVaultHoldInfo(): systemErrorExit(3, '%s is not a valid argument for "gam info hold"' % myarg) if not matterId: systemErrorExit(3, 'you must specify a matter for the hold.') - results = callGAPI(v.matters().holds(), u'get', matterId=matterId, holdId=holdId) - cd = buildGAPIObject(u'directory') - if u'accounts' in results: - account_type = u'group' if results[u'corpus'] == u'GROUPS' else u'user' - for i in range(0, len(results[u'accounts'])): - uid = u'uid:%s' % results[u'accounts'][i][u'accountId'] + results = callGAPI(v.matters().holds(), 'get', matterId=matterId, holdId=holdId) + cd = buildGAPIObject('directory') + if 'accounts' in results: + account_type = 'group' if results['corpus'] == 'GROUPS' else 'user' + for i in range(0, len(results['accounts'])): + uid = 'uid:%s' % results['accounts'][i]['accountId'] acct_email = convertUIDtoEmailAddress(uid, cd, account_type) - results[u'accounts'][i][u'email'] = acct_email - if u'orgUnit' in results: - results[u'orgUnit'][u'orgUnitPath'] = doGetOrgInfo(results[u'orgUnit'][u'orgUnitId'], return_attrib=u'orgUnitPath') + results['accounts'][i]['email'] = acct_email + if 'orgUnit' in results: + results['orgUnit']['orgUnitPath'] = doGetOrgInfo(results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath') print_json(None, results) def convertExportNameToID(v, nameOrID, matterId): @@ -8192,10 +8202,10 @@ def convertExportNameToID(v, nameOrID, matterId): cg = UID_PATTERN.match(nameOrID) if cg: return cg.group(1) - exports = callGAPIpages(v.matters().exports(), u'list', u'exports', matterId=matterId, fields=u'exports(id,name),nextPageToken') + exports = callGAPIpages(v.matters().exports(), 'list', 'exports', matterId=matterId, fields='exports(id,name),nextPageToken') for export in exports: - if export[u'name'].lower() == nameOrID: - return export[u'id'] + if export['name'].lower() == nameOrID: + return export['id'] systemErrorExit(4, 'could not find export name %s in matter %s' % (nameOrID, matterId)) def convertHoldNameToID(v, nameOrID, matterId): @@ -8203,10 +8213,10 @@ def convertHoldNameToID(v, nameOrID, matterId): cg = UID_PATTERN.match(nameOrID) if cg: return cg.group(1) - holds = callGAPIpages(v.matters().holds(), u'list', u'holds', matterId=matterId, fields=u'holds(holdId,name),nextPageToken') + holds = callGAPIpages(v.matters().holds(), 'list', 'holds', matterId=matterId, fields='holds(holdId,name),nextPageToken') for hold in holds: - if hold[u'name'].lower() == nameOrID: - return hold[u'holdId'] + if hold['name'].lower() == nameOrID: + return hold['holdId'] systemErrorExit(4, 'could not find hold name %s in matter %s' % (nameOrID, matterId)) def convertMatterNameToID(v, nameOrID): @@ -8214,10 +8224,10 @@ def convertMatterNameToID(v, nameOrID): cg = UID_PATTERN.match(nameOrID) if cg: return cg.group(1) - matters = callGAPIpages(v.matters(), u'list', u'matters', view=u'BASIC', fields=u'matters(matterId,name),nextPageToken') + matters = callGAPIpages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,name),nextPageToken') for matter in matters: - if matter[u'name'].lower() == nameOrID: - return matter[u'matterId'] + if matter['name'].lower() == nameOrID: + return matter['matterId'] return None def getMatterItem(v, nameOrID): @@ -8227,7 +8237,7 @@ def getMatterItem(v, nameOrID): return matterId def doUpdateVaultHold(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') hold = sys.argv[3] matterId = None body = {} @@ -8238,356 +8248,356 @@ def doUpdateVaultHold(): end_time = None i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'matter': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'matter': matterId = getMatterItem(v, sys.argv[i+1]) holdId = convertHoldNameToID(v, hold, matterId) i += 2 - elif myarg == u'query': + elif myarg == 'query': query = sys.argv[i+1] i += 2 - elif myarg in [u'orgunit', u'ou']: - body[u'orgUnit'] = {u'orgUnitId': getOrgUnitId(sys.argv[i+1])[1]} + elif myarg in ['orgunit', 'ou']: + body['orgUnit'] = {'orgUnitId': getOrgUnitId(sys.argv[i+1])[1]} i += 2 - elif myarg in [u'start', u'starttime']: + elif myarg in ['start', 'starttime']: start_time = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg in [u'end', u'endtime']: + elif myarg in ['end', 'endtime']: end_time = getDateZeroTimeOrFullTime(sys.argv[i+1]) i += 2 - elif myarg in [u'addusers', u'addaccounts', u'addgroups']: - add_accounts = sys.argv[i+1].split(u',') + elif myarg in ['addusers', 'addaccounts', 'addgroups']: + add_accounts = sys.argv[i+1].split(',') i += 2 - elif myarg in [u'removeusers', u'removeaccounts', u'removegroups']: - del_accounts = sys.argv[i+1].split(u',') + elif myarg in ['removeusers', 'removeaccounts', 'removegroups']: + del_accounts = sys.argv[i+1].split(',') i += 2 else: systemErrorExit(3, '%s is not a valid argument to "gam update hold"' % myarg) if not matterId: systemErrorExit(3, 'you must specify a matter for the hold.') - if query or start_time or end_time or body.get(u'orgUnit'): - old_body = callGAPI(v.matters().holds(), u'get', matterId=matterId, holdId=holdId, fields=u'corpus,query,orgUnit') - body[u'query'] = old_body[u'query'] - body[u'corpus'] = old_body[u'corpus'] - if u'orgUnit' in old_body and u'orgUnit' not in body: + if query or start_time or end_time or body.get('orgUnit'): + old_body = callGAPI(v.matters().holds(), 'get', matterId=matterId, holdId=holdId, fields='corpus,query,orgUnit') + body['query'] = old_body['query'] + body['corpus'] = old_body['corpus'] + if 'orgUnit' in old_body and 'orgUnit' not in body: # bah, API requires this to be sent on update even when it's not changing - body[u'orgUnit'] = old_body[u'orgUnit'] - query_type = '%sQuery' % body[u'corpus'].lower() - if body[u'corpus'] == u'DRIVE': + body['orgUnit'] = old_body['orgUnit'] + query_type = '%sQuery' % body['corpus'].lower() + if body['corpus'] == 'DRIVE': if query: try: - body[u'query'][query_type] = json.loads(query) + body['query'][query_type] = json.loads(query) except ValueError as e: systemErrorExit(3, '{0}, query: {1}'.format(str(e), query)) - elif body[u'corpus'] in [u'GROUPS', u'MAIL']: + elif body['corpus'] in ['GROUPS', 'MAIL']: if query: - body[u'query'][query_type][u'terms'] = query + body['query'][query_type]['terms'] = query if start_time: - body[u'query'][query_type][u'startTime'] = start_time + body['query'][query_type]['startTime'] = start_time if end_time: - body[u'query'][query_type][u'endTime'] = end_time + body['query'][query_type]['endTime'] = end_time if body: - print u'Updating hold %s / %s' % (hold, holdId) - callGAPI(v.matters().holds(), u'update', matterId=matterId, holdId=holdId, body=body) + print('Updating hold %s / %s' % (hold, holdId)) + callGAPI(v.matters().holds(), 'update', matterId=matterId, holdId=holdId, body=body) if add_accounts or del_accounts: - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for account in add_accounts: - print u'adding %s to hold.' % account - add_body = {u'accountId': convertEmailAddressToUID(account, cd)} - callGAPI(v.matters().holds().accounts(), u'create', matterId=matterId, holdId=holdId, body=add_body) + print('adding %s to hold.' % account) + add_body = {'accountId': convertEmailAddressToUID(account, cd)} + callGAPI(v.matters().holds().accounts(), 'create', matterId=matterId, holdId=holdId, body=add_body) for account in del_accounts: - print u'removing %s from hold.' % account + print('removing %s from hold.' % account) accountId = convertEmailAddressToUID(account, cd) - callGAPI(v.matters().holds().accounts(), u'delete', matterId=matterId, holdId=holdId, accountId=accountId) + callGAPI(v.matters().holds().accounts(), 'delete', matterId=matterId, holdId=holdId, accountId=accountId) def doUpdateVaultMatter(action=None): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') matterId = getMatterItem(v, sys.argv[3]) body = {} - action_kwargs = {u'body': {}} + action_kwargs = {'body': {}} add_collaborators = [] remove_collaborators = [] cd = None i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'action': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'action': action = sys.argv[i+1].lower() if action not in VAULT_MATTER_ACTIONS: - systemErrorExit(3, 'allowed actions are %s, got %s' % (u', '.join(VAULT_MATTER_ACTIONS), action)) + systemErrorExit(3, 'allowed actions are %s, got %s' % (', '.join(VAULT_MATTER_ACTIONS), action)) i += 2 - elif myarg == u'name': - body[u'name'] = sys.argv[i+1] + elif myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 - elif myarg == u'description': - body[u'description'] = sys.argv[i+1] + elif myarg == 'description': + body['description'] = sys.argv[i+1] i += 2 - elif myarg in [u'addcollaborator', u'addcollaborators']: + elif myarg in ['addcollaborator', 'addcollaborators']: if not cd: - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') add_collaborators.extend(validateCollaborators(sys.argv[i+1], cd)) i += 2 - elif myarg in [u'removecollaborator', u'removecollaborators']: + elif myarg in ['removecollaborator', 'removecollaborators']: if not cd: - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') remove_collaborators.extend(validateCollaborators(sys.argv[i+1], cd)) i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam update matter"' % sys.argv[i]) - if action == u'delete': + if action == 'delete': action_kwargs = {} if body: - print u'Updating matter %s...' % sys.argv[3] - if u'name' not in body or u'description' not in body: + print('Updating matter %s...' % sys.argv[3]) + if 'name' not in body or 'description' not in body: # bah, API requires name/description to be sent on update even when it's not changing - result = callGAPI(v.matters(), u'get', matterId=matterId, view=u'BASIC') - body.setdefault(u'name', result[u'name']) - body.setdefault(u'description', result.get(u'description')) - callGAPI(v.matters(), u'update', body=body, matterId=matterId) + result = callGAPI(v.matters(), 'get', matterId=matterId, view='BASIC') + body.setdefault('name', result['name']) + body.setdefault('description', result.get('description')) + callGAPI(v.matters(), 'update', body=body, matterId=matterId) if action: - print u'Performing %s on matter %s' % (action, sys.argv[3]) + print('Performing %s on matter %s' % (action, sys.argv[3])) callGAPI(v.matters(), action, matterId=matterId, **action_kwargs) for collaborator in add_collaborators: - print u' adding collaborator %s' % collaborator[u'email'] - callGAPI(v.matters(), u'addPermissions', matterId=matterId, body={u'matterPermission': {u'role': u'COLLABORATOR', u'accountId': collaborator[u'id']}}) + print(' adding collaborator %s' % collaborator['email']) + callGAPI(v.matters(), 'addPermissions', matterId=matterId, body={'matterPermission': {'role': 'COLLABORATOR', 'accountId': collaborator['id']}}) for collaborator in remove_collaborators: - print u' removing collaborator %s' % collaborator[u'email'] - callGAPI(v.matters(), u'removePermissions', matterId=matterId, body={u'accountId': collaborator[u'id']}) + print(' removing collaborator %s' % collaborator['email']) + callGAPI(v.matters(), 'removePermissions', matterId=matterId, body={'accountId': collaborator['id']}) def doGetVaultMatterInfo(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') matterId = getMatterItem(v, sys.argv[3]) - result = callGAPI(v.matters(), u'get', matterId=matterId, view=u'FULL') - if u'matterPermissions' in result: - cd = buildGAPIObject(u'directory') - for i in range(0, len(result[u'matterPermissions'])): - uid = u'uid:%s' % result[u'matterPermissions'][i][u'accountId'] + result = callGAPI(v.matters(), 'get', matterId=matterId, view='FULL') + if 'matterPermissions' in result: + cd = buildGAPIObject('directory') + for i in range(0, len(result['matterPermissions'])): + uid = 'uid:%s' % result['matterPermissions'][i]['accountId'] user_email = convertUIDtoEmailAddress(uid, cd) - result[u'matterPermissions'][i][u'email'] = user_email + result['matterPermissions'][i]['email'] = user_email print_json(None, result) def doCreateUser(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') body = getUserAttributes(3, cd, False) - print u"Creating account for %s" % body[u'primaryEmail'] - callGAPI(cd.users(), u'insert', body=body, fields=u'primaryEmail') + print("Creating account for %s" % body['primaryEmail']) + callGAPI(cd.users(), 'insert', body=body, fields='primaryEmail') def GroupIsAbuseOrPostmaster(emailAddr): - return emailAddr.startswith(u'abuse@') or emailAddr.startswith(u'postmaster@') + return emailAddr.startswith('abuse@') or emailAddr.startswith('postmaster@') def mapCollaborativeACL(myarg, value): - value = value.lower().replace(u'_', u'') + value = value.lower().replace('_', '') if value in COLLABORATIVE_ACL_CHOICES: return COLLABORATIVE_ACL_CHOICES[value] - systemErrorExit(3, 'allowed choices for %s are %s, got %s' % (myarg, u', '.join(sorted(COLLABORATIVE_ACL_CHOICES)), value)) + systemErrorExit(3, 'allowed choices for %s are %s, got %s' % (myarg, ', '.join(sorted(COLLABORATIVE_ACL_CHOICES)), value)) def getGroupAttrValue(myarg, value, gs_object, gs_body, function): - if myarg == u'collaborative': + if myarg == 'collaborative': value = mapCollaborativeACL(myarg, value) - for attrName, attrValue in COLLABORATIVE_INBOX_ATTRIBUTES.items(): - if attrValue == u'acl': + for attrName, attrValue in list(COLLABORATIVE_INBOX_ATTRIBUTES.items()): + if attrValue == 'acl': gs_body[attrName] = value else: gs_body[attrName] = attrValue return - for (attrib, params) in gs_object[u'schemas'][u'Groups'][u'properties'].items(): - if attrib in [u'kind', u'etag', u'email']: + for (attrib, params) in list(gs_object['schemas']['Groups']['properties'].items()): + if attrib in ['kind', 'etag', 'email']: continue if myarg == attrib.lower(): - if params[u'type'] == u'integer': + if params['type'] == 'integer': try: - if value[-1:].upper() == u'M': + if value[-1:].upper() == 'M': value = int(value[:-1]) * 1024 * 1024 - elif value[-1:].upper() == u'K': + elif value[-1:].upper() == 'K': value = int(value[:-1]) * 1024 - elif value[-1].upper() == u'B': + elif value[-1].upper() == 'B': value = int(value[:-1]) else: value = int(value) except ValueError: systemErrorExit(2, '%s must be a number ending with M (megabytes), K (kilobytes) or nothing (bytes); got %s' % value) - elif params[u'type'] == u'string': - if attrib == u'description': - value = value.replace(u'\\n', u'\n') - elif attrib == u'primaryLanguage': + elif params['type'] == 'string': + if attrib == 'description': + value = value.replace('\\n', '\n') + elif attrib == 'primaryLanguage': value = LANGUAGE_CODES_MAP.get(value.lower(), value) - elif COLLABORATIVE_INBOX_ATTRIBUTES.get(attrib) == u'acl': + elif COLLABORATIVE_INBOX_ATTRIBUTES.get(attrib) == 'acl': value = mapCollaborativeACL(myarg, value) - elif params[u'description'].find(value.upper()) != -1: # ugly hack because API wants some values uppercased. + elif params['description'].find(value.upper()) != -1: # ugly hack because API wants some values uppercased. value = value.upper() elif value.lower() in true_values: - value = u'true' + value = 'true' elif value.lower() in false_values: - value = u'false' + value = 'false' # Another ugly hack because Groups Settings API doesn't have proper enumerator values set in discovery file. - if u'description' in params and params[u'description'].find(u'Possible values are: ') != -1: - possible_values = params[u'description'][params[u'description'].find(u'Possible values are: ')+21:].split(u' ') + if 'description' in params and params['description'].find('Possible values are: ') != -1: + possible_values = params['description'][params['description'].find('Possible values are: ')+21:].split(' ') if value not in possible_values: - systemErrorExit(2, u'value for %s must be one of %s. Got %s.' % (attrib, u', '.join(possible_values), value)) + systemErrorExit(2, 'value for %s must be one of %s. Got %s.' % (attrib, ', '.join(possible_values), value)) gs_body[attrib] = value return systemErrorExit(2, '%s is not a valid argument for "gam %s group"' % (myarg, function)) def doCreateGroup(): - cd = buildGAPIObject(u'directory') - body = {u'email': normalizeEmailAddressOrUID(sys.argv[3], noUid=True)} + cd = buildGAPIObject('directory') + body = {'email': normalizeEmailAddressOrUID(sys.argv[3], noUid=True)} gs_get_before_update = got_name = False i = 4 gs_body = {} gs = None while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'name'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'name': + body['name'] = sys.argv[i+1] got_name = True i += 2 - elif myarg == u'description': - description = sys.argv[i+1].replace(u'\\n', u'\n') + elif myarg == 'description': + description = sys.argv[i+1].replace('\\n', '\n') # The Directory API Groups insert method can not handle any of these characters ('\n<>=') in the description field # If any of these characters are present, use the Group Settings API to set the description - for c in u'\n<>=': + for c in '\n<>=': if description.find(c) != -1: - gs_body[u'description'] = description + gs_body['description'] = description if not gs: - gs = buildGAPIObject(u'groupssettings') + gs = buildGAPIObject('groupssettings') gs_object = gs._rootDesc break else: - body[u'description'] = description + body['description'] = description i += 2 - elif myarg == u'getbeforeupdate': + elif myarg == 'getbeforeupdate': gs_get_before_update = True i += 1 else: if not gs: - gs = buildGAPIObject(u'groupssettings') + gs = buildGAPIObject('groupssettings') gs_object = gs._rootDesc - getGroupAttrValue(myarg, sys.argv[i+1], gs_object, gs_body, u'create') + getGroupAttrValue(myarg, sys.argv[i+1], gs_object, gs_body, 'create') i += 2 if not got_name: - body[u'name'] = body[u'email'] - print u"Creating group %s" % body[u'email'] - callGAPI(cd.groups(), u'insert', body=body, fields=u'email') - if gs and not GroupIsAbuseOrPostmaster(body[u'email']): + body['name'] = body['email'] + print("Creating group %s" % body['email']) + callGAPI(cd.groups(), 'insert', body=body, fields='email') + if gs and not GroupIsAbuseOrPostmaster(body['email']): if gs_get_before_update: - current_settings = callGAPI(gs.groups(), u'get', - retry_reasons=[u'serviceLimit'], - groupUniqueId=body[u'email'], fields=u'*') + current_settings = callGAPI(gs.groups(), 'get', + retry_reasons=['serviceLimit'], + groupUniqueId=body['email'], fields='*') if current_settings is not None: - gs_body = dict(current_settings.items() + gs_body.items()) + gs_body = dict(list(current_settings.items()) + list(gs_body.items())) if gs_body: - callGAPI(gs.groups(), u'update', retry_reasons=[u'serviceLimit'], groupUniqueId=body[u'email'], body=gs_body) + callGAPI(gs.groups(), 'update', retry_reasons=['serviceLimit'], groupUniqueId=body['email'], body=gs_body) def doCreateAlias(): - cd = buildGAPIObject(u'directory') - body = {u'alias': normalizeEmailAddressOrUID(sys.argv[3], noUid=True, noLower=True)} + cd = buildGAPIObject('directory') + body = {'alias': normalizeEmailAddressOrUID(sys.argv[3], noUid=True, noLower=True)} target_type = sys.argv[4].lower() - if target_type not in [u'user', u'group', u'target']: + if target_type not in ['user', 'group', 'target']: systemErrorExit(2, 'type of target must be user or group; got %s' % target_type) targetKey = normalizeEmailAddressOrUID(sys.argv[5]) - print u'Creating alias %s for %s %s' % (body[u'alias'], target_type, targetKey) - if target_type == u'user': - callGAPI(cd.users().aliases(), u'insert', userKey=targetKey, body=body) - elif target_type == u'group': - callGAPI(cd.groups().aliases(), u'insert', groupKey=targetKey, body=body) - elif target_type == u'target': + print('Creating alias %s for %s %s' % (body['alias'], target_type, targetKey)) + if target_type == 'user': + callGAPI(cd.users().aliases(), 'insert', userKey=targetKey, body=body) + elif target_type == 'group': + callGAPI(cd.groups().aliases(), 'insert', groupKey=targetKey, body=body) + elif target_type == 'target': try: - callGAPI(cd.users().aliases(), u'insert', throw_reasons=[GAPI_INVALID, GAPI_BAD_REQUEST], userKey=targetKey, body=body) + callGAPI(cd.users().aliases(), 'insert', throw_reasons=[GAPI_INVALID, GAPI_BAD_REQUEST], userKey=targetKey, body=body) except (GAPI_invalid, GAPI_badRequest): - callGAPI(cd.groups().aliases(), u'insert', groupKey=targetKey, body=body) + callGAPI(cd.groups().aliases(), 'insert', groupKey=targetKey, body=body) def doCreateOrg(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') name = getOrgUnitItem(sys.argv[3], pathOnly=True, absolutePath=False) - parent = u'' + parent = '' body = {} i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'description': - body[u'description'] = sys.argv[i+1].replace(u'\\n', u'\n') + if myarg == 'description': + body['description'] = sys.argv[i+1].replace('\\n', '\n') i += 2 - elif myarg == u'parent': + elif myarg == 'parent': parent = getOrgUnitItem(sys.argv[i+1]) i += 2 - elif myarg == u'noinherit': - body[u'blockInheritance'] = True + elif myarg == 'noinherit': + body['blockInheritance'] = True i += 1 - elif myarg == u'inherit': - body[u'blockInheritance'] = False + elif myarg == 'inherit': + body['blockInheritance'] = False i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam create org"' % sys.argv[i]) - if parent.startswith(u'id:'): - parent = callGAPI(cd.orgunits(), u'get', - customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=parent, fields=u'orgUnitPath')[u'orgUnitPath'] - if parent == u'/': + if parent.startswith('id:'): + parent = callGAPI(cd.orgunits(), 'get', + customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=parent, fields='orgUnitPath')['orgUnitPath'] + if parent == '/': orgUnitPath = parent+name else: - orgUnitPath = parent+u'/'+name - if orgUnitPath.count(u'/') > 1: - body[u'parentOrgUnitPath'], body[u'name'] = orgUnitPath.rsplit(u'/', 1) + orgUnitPath = parent+'/'+name + if orgUnitPath.count('/') > 1: + body['parentOrgUnitPath'], body['name'] = orgUnitPath.rsplit('/', 1) else: - body[u'parentOrgUnitPath'] = u'/' - body[u'name'] = orgUnitPath[1:] - parent = body[u'parentOrgUnitPath'] - callGAPI(cd.orgunits(), u'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) + body['parentOrgUnitPath'] = '/' + body['name'] = orgUnitPath[1:] + parent = body['parentOrgUnitPath'] + callGAPI(cd.orgunits(), 'insert', customerId=GC_Values[GC_CUSTOMER_ID], body=body) def _getBuildingAttributes(args, body={}): i = 0 while i < len(args): - myarg = args[i].lower().replace(u'_', u'') - if myarg == u'id': - body[u'buildingId'] = args[i+1] + myarg = args[i].lower().replace('_', '') + if myarg == 'id': + body['buildingId'] = args[i+1] i += 2 - elif myarg == u'name': - body[u'buildingName'] = args[i+1] + elif myarg == 'name': + body['buildingName'] = args[i+1] i += 2 - elif myarg in [u'lat', u'latitude']: - if u'coordinates' not in body: - body[u'coordinates'] = {} - body[u'coordinates'][u'latitude'] = args[i+1] + elif myarg in ['lat', 'latitude']: + if 'coordinates' not in body: + body['coordinates'] = {} + body['coordinates']['latitude'] = args[i+1] i += 2 - elif myarg in [u'long', u'lng', u'longitude']: - if u'coordinates' not in body: - body[u'coordinates'] = {} - body[u'coordinates'][u'longitude'] = args[i+1] + elif myarg in ['long', 'lng', 'longitude']: + if 'coordinates' not in body: + body['coordinates'] = {} + body['coordinates']['longitude'] = args[i+1] i += 2 - elif myarg == u'description': - body[u'description'] = args[i+1] + elif myarg == 'description': + body['description'] = args[i+1] i += 2 - elif myarg == u'floors': - body[u'floorNames'] = args[i+1].split(u',') + elif myarg == 'floors': + body['floorNames'] = args[i+1].split(',') i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam create|update building"' % myarg) return body def doCreateBuilding(): - cd = buildGAPIObject(u'directory') - body = {u'floorNames': [u'1'], - u'buildingId': unicode(uuid.uuid4()), - u'buildingName': sys.argv[3]} + cd = buildGAPIObject('directory') + body = {'floorNames': ['1'], + 'buildingId': str(uuid.uuid4()), + 'buildingName': sys.argv[3]} body = _getBuildingAttributes(sys.argv[4:], body) - print u'Creating building %s...' % body[u'buildingId'] - callGAPI(cd.resources().buildings(), u'insert', + print('Creating building %s...' % body['buildingId']) + callGAPI(cd.resources().buildings(), 'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def _makeBuildingIdNameMap(cd): - buildings = callGAPIpages(cd.resources().buildings(), u'list', u'buildings', + buildings = callGAPIpages(cd.resources().buildings(), 'list', 'buildings', customer=GC_Values[GC_CUSTOMER_ID], - fields=u'nextPageToken,buildings(buildingId,buildingName)') + fields='nextPageToken,buildings(buildingId,buildingName)') GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {} GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {} for building in buildings: - GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][building[u'buildingId']] = building[u'buildingName'] - GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building[u'buildingName']] = building[u'buildingId'] + GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][building['buildingId']] = building['buildingName'] + GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building['buildingName']] = building['buildingId'] def _getBuildingByNameOrId(cd, which_building, minLen=1): - if not which_building or (minLen == 0 and which_building in [u'id:', u'uid:']): + if not which_building or (minLen == 0 and which_building in ['id:', 'uid:']): if minLen == 0: - return u'' - systemErrorExit(3, u'Building id/name is empty') + return '' + systemErrorExit(3, 'Building id/name is empty') cg = UID_PATTERN.match(which_building) if cg: return cg.group(1) @@ -8599,12 +8609,12 @@ def _getBuildingByNameOrId(cd, which_building, minLen=1): # No exact name match, check for case insensitive name matches which_building_lower = which_building.lower() ci_matches = [] - for buildingName, buildingId in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID].iteritems(): + for buildingName, buildingId in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID].items(): if buildingName.lower() == which_building_lower: - ci_matches.append({u'buildingName': buildingName, u'buildingId': buildingId}) + ci_matches.append({'buildingName': buildingName, 'buildingId': buildingId}) # One match, return ID if len(ci_matches) == 1: - return ci_matches[0][u'buildingId'] + return ci_matches[0]['buildingId'] # No or multiple name matches, try ID # Exact ID match, return ID if which_building in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]: @@ -8616,10 +8626,10 @@ def _getBuildingByNameOrId(cd, which_building, minLen=1): return buildingId # Multiple name matches if len(ci_matches) > 1: - message = u'Multiple buildings with same name:\n' + message = 'Multiple buildings with same name:\n' for building in ci_matches: - message += u' Name:%s id:%s\n' % (building[u'buildingName'], building[u'buildingId']) - message += u'\nPlease specify building name by exact case or by id.' + message += ' Name:%s id:%s\n' % (building['buildingName'], building['buildingId']) + message += '\nPlease specify building name by exact case or by id.' systemErrorExit(3, message) # No matches else: @@ -8628,210 +8638,210 @@ def _getBuildingByNameOrId(cd, which_building, minLen=1): def _getBuildingNameById(cd, buildingId): if GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] is None: _makeBuildingIdNameMap(cd) - return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, u'UNKNOWN') + return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, 'UNKNOWN') def doUpdateBuilding(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') buildingId = _getBuildingByNameOrId(cd, sys.argv[3]) body = _getBuildingAttributes(sys.argv[4:]) - print u'Updating building %s...' % buildingId - callGAPI(cd.resources().buildings(), u'patch', + print('Updating building %s...' % buildingId) + callGAPI(cd.resources().buildings(), 'patch', customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId, body=body) def doGetBuildingInfo(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') buildingId = _getBuildingByNameOrId(cd, sys.argv[3]) - building = callGAPI(cd.resources().buildings(), u'get', + building = callGAPI(cd.resources().buildings(), 'get', customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId) - if u'buildingId' in building: - building[u'buildingId'] = u'id:{0}'.format(building[u'buildingId']) - if u'floorNames' in building: - building[u'floorNames'] = u','.join(building[u'floorNames']) - if u'buildingName' in building: - sys.stdout.write(building.pop(u'buildingName')) + if 'buildingId' in building: + building['buildingId'] = 'id:{0}'.format(building['buildingId']) + if 'floorNames' in building: + building['floorNames'] = ','.join(building['floorNames']) + if 'buildingName' in building: + sys.stdout.write(building.pop('buildingName')) print_json(None, building) def doDeleteBuilding(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') buildingId = _getBuildingByNameOrId(cd, sys.argv[3]) - print u'Deleting building %s...' % buildingId - callGAPI(cd.resources().buildings(), u'delete', + print('Deleting building %s...' % buildingId) + callGAPI(cd.resources().buildings(), 'delete', customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId) def _getFeatureAttributes(args, body={}): i = 0 while i < len(args): - myarg = args[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'name'] = args[i+1] + myarg = args[i].lower().replace('_', '') + if myarg == 'name': + body['name'] = args[i+1] i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam create|update feature"') return body def doCreateFeature(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') body = _getFeatureAttributes(sys.argv[3:]) - print u'Creating feature %s...' % body[u'name'] - callGAPI(cd.resources().features(), u'insert', + print('Creating feature %s...' % body['name']) + callGAPI(cd.resources().features(), 'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doUpdateFeature(): # update does not work for name and name is only field to be updated # if additional writable fields are added to feature in the future # we'll add support for update as well as rename - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') oldName = sys.argv[3] - body = {u'newName': sys.argv[5:]} - print u'Updating feature %s...' % oldName - callGAPI(cd.resources().features(), u'rename', + body = {'newName': sys.argv[5:]} + print('Updating feature %s...' % oldName) + callGAPI(cd.resources().features(), 'rename', customer=GC_Values[GC_CUSTOMER_ID], oldName=oldName, body=body) def doDeleteFeature(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') featureKey = sys.argv[3] - print u'Deleting feature %s...' % featureKey - callGAPI(cd.resources().features(), u'delete', + print('Deleting feature %s...' % featureKey) + callGAPI(cd.resources().features(), 'delete', customer=GC_Values[GC_CUSTOMER_ID], featureKey=featureKey) def _getResourceCalendarAttributes(cd, args, body={}): i = 0 while i < len(args): - myarg = args[i].lower().replace(u'_', u'') - if myarg == u'name': - body[u'resourceName'] = args[i+1] + myarg = args[i].lower().replace('_', '') + if myarg == 'name': + body['resourceName'] = args[i+1] i += 2 - elif myarg == u'description': - body[u'resourceDescription'] = args[i+1].replace(u'\\n', u'\n') + elif myarg == 'description': + body['resourceDescription'] = args[i+1].replace('\\n', '\n') i += 2 - elif myarg == u'type': - body[u'resourceType'] = args[i+1] + elif myarg == 'type': + body['resourceType'] = args[i+1] i += 2 - elif myarg in [u'building', u'buildingid']: - body[u'buildingId'] = _getBuildingByNameOrId(cd, args[i+1], minLen=0) + elif myarg in ['building', 'buildingid']: + body['buildingId'] = _getBuildingByNameOrId(cd, args[i+1], minLen=0) i += 2 - elif myarg in [u'capacity']: - body[u'capacity'] = getInteger(args[i+1], myarg, minVal=0) + elif myarg in ['capacity']: + body['capacity'] = getInteger(args[i+1], myarg, minVal=0) i += 2 - elif myarg in [u'feature', u'features']: - features = args[i+1].split(u',') - body[u'featureInstances'] = [] + elif myarg in ['feature', 'features']: + features = args[i+1].split(',') + body['featureInstances'] = [] for feature in features: - body[u'featureInstances'].append({u'feature': {u'name': feature}}) + body['featureInstances'].append({'feature': {'name': feature}}) i += 2 - elif myarg in [u'floor', u'floorname']: - body[u'floorName'] = args[i+1] + elif myarg in ['floor', 'floorname']: + body['floorName'] = args[i+1] i += 2 - elif myarg in [u'floorsection']: - body[u'floorSection'] = args[i+1] + elif myarg in ['floorsection']: + body['floorSection'] = args[i+1] i += 2 - elif myarg in [u'category']: - body[u'resourceCategory'] = args[i+1].upper() - if body[u'resourceCategory'] == u'ROOM': - body[u'resourceCategory'] = u'CONFERENCE_ROOM' + elif myarg in ['category']: + body['resourceCategory'] = args[i+1].upper() + if body['resourceCategory'] == 'ROOM': + body['resourceCategory'] = 'CONFERENCE_ROOM' i += 2 - elif myarg in [u'uservisibledescription', u'userdescription']: - body[u'userVisibleDescription'] = args[i+1] + elif myarg in ['uservisibledescription', 'userdescription']: + body['userVisibleDescription'] = args[i+1] i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam create|update resource"' % args[i]) return body def doCreateResourceCalendar(): - cd = buildGAPIObject(u'directory') - body = {u'resourceId': sys.argv[3], - u'resourceName': sys.argv[4]} + cd = buildGAPIObject('directory') + body = {'resourceId': sys.argv[3], + 'resourceName': sys.argv[4]} body = _getResourceCalendarAttributes(cd, sys.argv[5:], body) - print u'Creating resource %s...' % body[u'resourceId'] - callGAPI(cd.resources().calendars(), u'insert', + print('Creating resource %s...' % body['resourceId']) + callGAPI(cd.resources().calendars(), 'insert', customer=GC_Values[GC_CUSTOMER_ID], body=body) def doUpdateResourceCalendar(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') resId = sys.argv[3] body = _getResourceCalendarAttributes(cd, sys.argv[4:]) # Use patch since it seems to work better. # update requires name to be set. - callGAPI(cd.resources().calendars(), u'patch', + callGAPI(cd.resources().calendars(), 'patch', customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId, body=body, - fields=u'') - print u'updated resource %s' % resId + fields='') + print('updated resource %s' % resId) def doUpdateUser(users, i): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') if users is None: users = [normalizeEmailAddressOrUID(sys.argv[3])] body = getUserAttributes(i, cd, True) - vfe = u'primaryEmail' in body and body[u'primaryEmail'][:4].lower() == u'vfe@' + vfe = 'primaryEmail' in body and body['primaryEmail'][:4].lower() == 'vfe@' for user in users: userKey = user if vfe: - user_primary = callGAPI(cd.users(), u'get', userKey=userKey, fields=u'primaryEmail,id') - userKey = user_primary[u'id'] - user_primary = user_primary[u'primaryEmail'] + user_primary = callGAPI(cd.users(), 'get', userKey=userKey, fields='primaryEmail,id') + userKey = user_primary['id'] + user_primary = user_primary['primaryEmail'] user_name, user_domain = splitEmailAddress(user_primary) - body[u'primaryEmail'] = u'vfe.%s.%05d@%s' % (user_name, random.randint(1, 99999), user_domain) - body[u'emails'] = [{u'type': u'custom', u'customType': u'former_employee', u'primary': False, u'address': user_primary}] - sys.stdout.write(u'updating user %s...\n' % user) + body['primaryEmail'] = 'vfe.%s.%05d@%s' % (user_name, random.randint(1, 99999), user_domain) + body['emails'] = [{'type': 'custom', 'customType': 'former_employee', 'primary': False, 'address': user_primary}] + sys.stdout.write('updating user %s...\n' % user) if body: - callGAPI(cd.users(), u'update', userKey=userKey, body=body) + callGAPI(cd.users(), 'update', userKey=userKey, body=body) def doRemoveUsersAliases(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: - user_aliases = callGAPI(cd.users(), u'get', userKey=user, fields=u'aliases,id,primaryEmail') - user_id = user_aliases[u'id'] - user_primary = user_aliases[u'primaryEmail'] - if u'aliases' in user_aliases: - print u'%s has %s aliases' % (user_primary, len(user_aliases[u'aliases'])) - for an_alias in user_aliases[u'aliases']: - print u' removing alias %s for %s...' % (an_alias, user_primary) - callGAPI(cd.users().aliases(), u'delete', userKey=user_id, alias=an_alias) + user_aliases = callGAPI(cd.users(), 'get', userKey=user, fields='aliases,id,primaryEmail') + user_id = user_aliases['id'] + user_primary = user_aliases['primaryEmail'] + if 'aliases' in user_aliases: + print('%s has %s aliases' % (user_primary, len(user_aliases['aliases']))) + for an_alias in user_aliases['aliases']: + print(' removing alias %s for %s...' % (an_alias, user_primary)) + callGAPI(cd.users().aliases(), 'delete', userKey=user_id, alias=an_alias) else: - print u'%s has no aliases' % user_primary + print('%s has no aliases' % user_primary) def deleteUserFromGroups(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: - user_groups = callGAPIpages(cd.groups(), u'list', u'groups', userKey=user, fields=u'groups(id,email)') + user_groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user, fields='groups(id,email)') num_groups = len(user_groups) - print u'%s is in %s groups' % (user, num_groups) + print('%s is in %s groups' % (user, num_groups)) j = 0 for user_group in user_groups: j += 1 - print u' removing %s from %s (%s/%s)' % (user, user_group[u'email'], j, num_groups) - callGAPI(cd.members(), u'delete', soft_errors=True, groupKey=user_group[u'id'], memberKey=user) - print u'' + print(' removing %s from %s (%s/%s)' % (user, user_group['email'], j, num_groups)) + callGAPI(cd.members(), 'delete', soft_errors=True, groupKey=user_group['id'], memberKey=user) + print('') def checkGroupExists(cd, group, i=0, count=0): group = normalizeEmailAddressOrUID(group) try: - return callGAPI(cd.groups(), u'get', + return callGAPI(cd.groups(), 'get', throw_reasons=GAPI_GROUP_GET_THROW_REASONS, retry_reasons=GAPI_GROUP_GET_RETRY_REASONS, - groupKey=group, fields=u'email')[u'email'] + groupKey=group, fields='email')['email'] except (GAPI_groupNotFound, GAPI_domainNotFound, GAPI_domainCannotUseApis, GAPI_forbidden, GAPI_badRequest): - entityUnknownWarning(u'Group', group, i, count) + entityUnknownWarning('Group', group, i, count) return None -UPDATE_GROUP_SUBCMDS = [u'add', u'clear', u'delete', u'remove', u'sync', u'update'] +UPDATE_GROUP_SUBCMDS = ['add', 'clear', 'delete', 'remove', 'sync', 'update'] GROUP_ROLES_MAP = { - u'owner': ROLE_OWNER, u'owners': ROLE_OWNER, - u'manager': ROLE_MANAGER, u'managers': ROLE_MANAGER, - u'member': ROLE_MEMBER, u'members': ROLE_MEMBER, + 'owner': ROLE_OWNER, 'owners': ROLE_OWNER, + 'manager': ROLE_MANAGER, 'managers': ROLE_MANAGER, + 'member': ROLE_MEMBER, 'members': ROLE_MEMBER, } MEMBER_DELIVERY_MAP = { - u'allmail': u'ALL_MAIL', u'digest': u'DIGEST', u'daily': u'DAILY', - u'abridged': u'DAILY', u'nomail': u'NONE', u'none': u'NONE' + 'allmail': 'ALL_MAIL', 'digest': 'DIGEST', 'daily': 'DAILY', + 'abridged': 'DAILY', 'nomail': 'NONE', 'none': 'NONE' } def doUpdateGroup(): # Convert foo@googlemail.com to foo@gmail.com; eliminate periods in name for foo.bar@gmail.com def _cleanConsumerAddress(emailAddress, mapCleanToOriginal): - atLoc = emailAddress.find(u'@') + atLoc = emailAddress.find('@') if atLoc > 0: - if emailAddress[atLoc+1:] in [u'gmail.com', u'googlemail.com']: - cleanEmailAddress = emailAddress[:atLoc].replace(u'.', u'')+u'@gmail.com' + if emailAddress[atLoc+1:] in ['gmail.com', 'googlemail.com']: + cleanEmailAddress = emailAddress[:atLoc].replace('.', '')+'@gmail.com' if cleanEmailAddress != emailAddress: mapCleanToOriginal[cleanEmailAddress] = emailAddress return cleanEmailAddress @@ -8845,10 +8855,10 @@ def doUpdateGroup(): if sys.argv[i].lower() in GROUP_ROLES_MAP: role = GROUP_ROLES_MAP[sys.argv[i].lower()] i += 1 - if sys.argv[i].lower() in [u'suspended', u'notsuspended']: - checkSuspended = sys.argv[i].lower() == u'suspended' + if sys.argv[i].lower() in ['suspended', 'notsuspended']: + checkSuspended = sys.argv[i].lower() == 'suspended' i += 1 - if sys.argv[i].lower().replace(u'_', u'') in MEMBER_DELIVERY_MAP: + if sys.argv[i].lower().replace('_', '') in MEMBER_DELIVERY_MAP: delivery = MEMBER_DELIVERY_MAP[sys.argv[i].lower()] i += 1 if sys.argv[i].lower() in usergroup_types: @@ -8858,20 +8868,20 @@ def doUpdateGroup(): return (role, users_email, delivery) gs_get_before_update = False - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') group = sys.argv[3] myarg = sys.argv[4].lower() items = [] if myarg in UPDATE_GROUP_SUBCMDS: group = normalizeEmailAddressOrUID(group) - if myarg == u'add': + if myarg == 'add': role, users_email, delivery = _getRoleAndUsers() if not role: role = ROLE_MEMBER if not checkGroupExists(cd, group): return if len(users_email) > 1: - sys.stderr.write(u'Group: {0}, Will add {1} {2}s.\n'.format(group, len(users_email), role)) + sys.stderr.write('Group: {0}, Will add {1} {2}s.\n'.format(group, len(users_email), role)) for user_email in users_email: item = ['gam', 'update', 'group', group, 'add', role] if delivery: @@ -8879,38 +8889,38 @@ def doUpdateGroup(): item.append(user_email) items.append(item) else: - body = {u'role': role, u'email' if users_email[0].find(u'@') != -1 else u'id': users_email[0]} - add_text = [u'as %s' % role] + body = {'role': role, 'email' if users_email[0].find('@') != -1 else 'id': users_email[0]} + add_text = ['as %s' % role] if delivery: - body[u'delivery_settings'] = delivery - add_text.append(u'delivery %s' % delivery) + body['delivery_settings'] = delivery + add_text.append('delivery %s' % delivery) for i in range(2): try: - callGAPI(cd.members(), u'insert', + callGAPI(cd.members(), 'insert', throw_reasons=[GAPI_DUPLICATE, GAPI_MEMBER_NOT_FOUND, GAPI_RESOURCE_NOT_FOUND, GAPI_INVALID_MEMBER, GAPI_CYCLIC_MEMBERSHIPS_NOT_ALLOWED], groupKey=group, body=body) - print u' Group: {0}, {1} Added {2}'.format(group, users_email[0], u' '.join(add_text)) + print(' Group: {0}, {1} Added {2}'.format(group, users_email[0], ' '.join(add_text))) break except GAPI_duplicate as e: # check if user is a full member, not pending try: - result = callGAPI(cd.members(), u'get', throw_reasons=[GAPI_MEMBER_NOT_FOUND], memberKey=users_email[0], groupKey=group, fields=u'role') - print u' Group: {0}, {1} Add {2} Failed: Duplicate, already a {3}'.format(group, users_email[0], u' '.join(add_text), result[u'role']) + result = callGAPI(cd.members(), 'get', throw_reasons=[GAPI_MEMBER_NOT_FOUND], memberKey=users_email[0], groupKey=group, fields='role') + print(' Group: {0}, {1} Add {2} Failed: Duplicate, already a {3}'.format(group, users_email[0], ' '.join(add_text), result['role'])) break # if get succeeds, user is a full member and we throw duplicate error except GAPI_memberNotFound: # insert fails on duplicate and get fails on not found, user is pending - print u' Group: {0}, {1} member is pending, deleting and re-adding to solve...'.format(group, users_email[0]) - callGAPI(cd.members(), u'delete', memberKey=users_email[0], groupKey=group) + print(' Group: {0}, {1} member is pending, deleting and re-adding to solve...'.format(group, users_email[0])) + callGAPI(cd.members(), 'delete', memberKey=users_email[0], groupKey=group) continue # 2nd insert should succeed now that pending is clear except (GAPI_memberNotFound, GAPI_resourceNotFound, GAPI_invalidMember, GAPI_cyclicMembershipsNotAllowed) as e: - print u' Group: {0}, {1} Add {2} Failed: {3}'.format(group, users_email[0], u' '.join(add_text), str(e)) + print(' Group: {0}, {1} Add {2} Failed: {3}'.format(group, users_email[0], ' '.join(add_text), str(e))) break - elif myarg == u'sync': + elif myarg == 'sync': syncMembersSet = set() syncMembersMap = {} role, users_email, delivery = _getRoleAndUsers() for user_email in users_email: - if user_email == u'*' or user_email == GC_Values[GC_CUSTOMER_ID]: + if user_email == '*' or user_email == GC_Values[GC_CUSTOMER_ID]: syncMembersSet.add(GC_Values[GC_CUSTOMER_ID]) else: syncMembersSet.add(_cleanConsumerAddress(user_email.lower(), syncMembersMap)) @@ -8918,7 +8928,7 @@ def doUpdateGroup(): if group: currentMembersSet = set() currentMembersMap = {} - for current_email in getUsersToModify(entity_type=u'group', entity=group, member_type=role, groupUserMembersOnly=False): + for current_email in getUsersToModify(entity_type='group', entity=group, member_type=role, groupUserMembersOnly=False): if current_email == GC_Values[GC_CUSTOMER_ID]: currentMembersSet.add(current_email) else: @@ -8926,9 +8936,9 @@ def doUpdateGroup(): # Compare incoming members and current memebers using the cleaned addresses; we actually add/remove with the original addresses to_add = [syncMembersMap.get(emailAddress, emailAddress) for emailAddress in syncMembersSet-currentMembersSet] to_remove = [currentMembersMap.get(emailAddress, emailAddress) for emailAddress in currentMembersSet-syncMembersSet] - sys.stderr.write(u'Group: {0}, Will add {1} and remove {2} {3}s.\n'.format(group, len(to_add), len(to_remove), role)) + sys.stderr.write('Group: {0}, Will add {1} and remove {2} {3}s.\n'.format(group, len(to_add), len(to_remove), role)) for user in to_add: - item = [u'gam', u'update', u'group', group, u'add'] + item = ['gam', 'update', 'group', group, 'add'] if role: item.append(role) if delivery: @@ -8936,31 +8946,31 @@ def doUpdateGroup(): item.append(user) items.append(item) for user in to_remove: - items.append([u'gam', u'update', u'group', group, u'remove', user]) - elif myarg in [u'delete', u'remove']: + items.append(['gam', 'update', 'group', group, 'remove', user]) + elif myarg in ['delete', 'remove']: _, users_email, _ = _getRoleAndUsers() if not checkGroupExists(cd, group): return if len(users_email) > 1: - sys.stderr.write(u'Group: {0}, Will remove {1} emails.\n'.format(group, len(users_email))) + sys.stderr.write('Group: {0}, Will remove {1} emails.\n'.format(group, len(users_email))) for user_email in users_email: items.append(['gam', 'update', 'group', group, 'remove', user_email]) else: try: - callGAPI(cd.members(), u'delete', + callGAPI(cd.members(), 'delete', throw_reasons=[GAPI_MEMBER_NOT_FOUND, GAPI_INVALID_MEMBER], groupKey=group, memberKey=users_email[0]) - print u' Group: {0}, {1} Removed'.format(group, users_email[0]) + print(' Group: {0}, {1} Removed'.format(group, users_email[0])) except (GAPI_memberNotFound, GAPI_invalidMember) as e: - print u' Group: {0}, {1} Remove Failed: {2}'.format(group, users_email[0], str(e)) - elif myarg == u'update': + print(' Group: {0}, {1} Remove Failed: {2}'.format(group, users_email[0], str(e))) + elif myarg == 'update': role, users_email, delivery = _getRoleAndUsers() group = checkGroupExists(cd, group) if group: if not role and not delivery: role = ROLE_MEMBER if len(users_email) > 1: - sys.stderr.write(u'Group: {0}, Will update {1} {2}s.\n'.format(group, len(users_email), role)) + sys.stderr.write('Group: {0}, Will update {1} {2}s.\n'.format(group, len(users_email), role)) for user_email in users_email: item = ['gam', 'update', 'group', group, 'update'] if role: @@ -8973,21 +8983,21 @@ def doUpdateGroup(): body = {} update_text = [] if role: - body[u'role'] = role - update_text.append(u'to %s' % role) + body['role'] = role + update_text.append('to %s' % role) if delivery: - body[u'delivery_settings'] = delivery - update_text.append(u'delivery %s' % delivery) + body['delivery_settings'] = delivery + update_text.append('delivery %s' % delivery) try: - callGAPI(cd.members(), u'update', + callGAPI(cd.members(), 'update', throw_reasons=[GAPI_MEMBER_NOT_FOUND, GAPI_INVALID_MEMBER], groupKey=group, memberKey=users_email[0], body=body) - print u' Group: {0}, {1} Updated {2}'.format(group, users_email[0], u' '.join(update_text)) + print(' Group: {0}, {1} Updated {2}'.format(group, users_email[0], ' '.join(update_text))) except (GAPI_memberNotFound, GAPI_invalidMember) as e: - print u' Group: {0}, {1} Update to {2} Failed: {3}'.format(group, users_email[0], role, str(e)) + print(' Group: {0}, {1} Update to {2} Failed: {3}'.format(group, users_email[0], role, str(e))) else: # clear checkSuspended = None - fields = [u'email', u'id'] + fields = ['email', 'id'] roles = [] i = 5 while i < len(sys.argv): @@ -8995,49 +9005,49 @@ def doUpdateGroup(): if myarg.upper() in [ROLE_OWNER, ROLE_MANAGER, ROLE_MEMBER]: roles.append(myarg.upper()) i += 1 - elif myarg in [u'suspended', u'notsuspended']: - checkSuspended = myarg == u'suspended' - fields.append(u'status') + elif myarg in ['suspended', 'notsuspended']: + checkSuspended = myarg == 'suspended' + fields.append('status') i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam update group clear"' % sys.argv[i]) if roles: - roles = u','.join(sorted(set(roles))) + roles = ','.join(sorted(set(roles))) else: roles = ROLE_MEMBER group = normalizeEmailAddressOrUID(group) - member_type_message = u'%ss' % roles.lower() - sys.stderr.write(u"Getting %s of %s (may take some time for large groups)...\n" % (member_type_message, group)) - page_message = u'Got %%%%total_items%%%% %s...' % member_type_message - validRoles, listRoles, listFields = _getRoleVerification(roles, u'nextPageToken,members({0})'.format(u','.join(fields))) + member_type_message = '%ss' % roles.lower() + sys.stderr.write("Getting %s of %s (may take some time for large groups)...\n" % (member_type_message, group)) + page_message = 'Got %%%%total_items%%%% %s...' % member_type_message + validRoles, listRoles, listFields = _getRoleVerification(roles, 'nextPageToken,members({0})'.format(','.join(fields))) try: - result = callGAPIpages(cd.members(), u'list', u'members', + result = callGAPIpages(cd.members(), 'list', 'members', page_message=page_message, throw_reasons=GAPI_MEMBERS_THROW_REASONS, groupKey=group, roles=listRoles, fields=listFields, maxResults=GC_Values[GC_MEMBER_MAX_RESULTS]) if not result: - print u'Group already has 0 members' + print('Group already has 0 members') return if checkSuspended is None: - users_email = [member.get(u'email', member[u'id']) for member in result if not validRoles or member.get(u'role', ROLE_MEMBER) in validRoles] + users_email = [member.get('email', member['id']) for member in result if not validRoles or member.get('role', ROLE_MEMBER) in validRoles] elif checkSuspended: - users_email = [member.get(u'email', member[u'id']) for member in result if (not validRoles or member.get(u'role', ROLE_MEMBER) in validRoles) and member[u'status'] == u'SUSPENDED'] + users_email = [member.get('email', member['id']) for member in result if (not validRoles or member.get('role', ROLE_MEMBER) in validRoles) and member['status'] == 'SUSPENDED'] else: # elif not checkSuspended - users_email = [member.get(u'email', member[u'id']) for member in result if (not validRoles or member.get(u'role', ROLE_MEMBER) in validRoles) and member[u'status'] != u'SUSPENDED'] + users_email = [member.get('email', member['id']) for member in result if (not validRoles or member.get('role', ROLE_MEMBER) in validRoles) and member['status'] != 'SUSPENDED'] if len(users_email) > 1: - sys.stderr.write(u'Group: {0}, Will remove {1} {2}{3}s.\n'.format(group, len(users_email), u'' if checkSuspended is None else [u'Non-suspended ', u'Suspended '][checkSuspended], roles)) + sys.stderr.write('Group: {0}, Will remove {1} {2}{3}s.\n'.format(group, len(users_email), '' if checkSuspended is None else ['Non-suspended ', 'Suspended '][checkSuspended], roles)) for user_email in users_email: items.append(['gam', 'update', 'group', group, 'remove', user_email]) else: try: - callGAPI(cd.members(), u'delete', + callGAPI(cd.members(), 'delete', throw_reasons=[GAPI_MEMBER_NOT_FOUND, GAPI_INVALID_MEMBER], groupKey=group, memberKey=users_email[0]) - print u' Group: {0}, {1} Removed'.format(group, users_email[0]) + print(' Group: {0}, {1} Removed'.format(group, users_email[0])) except (GAPI_memberNotFound, GAPI_invalidMember) as e: - print u' Group: {0}, {1} Remove Failed: {2}'.format(group, users_email[0], str(e)) + print(' Group: {0}, {1} Remove Failed: {2}'.format(group, users_email[0], str(e))) except (GAPI_groupNotFound, GAPI_domainNotFound, GAPI_invalid, GAPI_forbidden): - entityUnknownWarning(u'Group', group, 0, 0) + entityUnknownWarning('Group', group, 0, 0) if items: run_batch(items) else: @@ -9047,122 +9057,122 @@ def doUpdateGroup(): gs_body = {} cd_body = {} while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'email': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'email': use_cd_api = True - cd_body[u'email'] = normalizeEmailAddressOrUID(sys.argv[i+1]) + cd_body['email'] = normalizeEmailAddressOrUID(sys.argv[i+1]) i += 2 - elif myarg == u'admincreated': + elif myarg == 'admincreated': use_cd_api = True - cd_body[u'adminCreated'] = getBoolean(sys.argv[i+1], myarg) + cd_body['adminCreated'] = getBoolean(sys.argv[i+1], myarg) i += 2 - elif myarg == u'getbeforeupdate': + elif myarg == 'getbeforeupdate': gs_get_before_update = True i += 1 else: if not gs: - gs = buildGAPIObject(u'groupssettings') + gs = buildGAPIObject('groupssettings') gs_object = gs._rootDesc - getGroupAttrValue(myarg, sys.argv[i+1], gs_object, gs_body, u'update') + getGroupAttrValue(myarg, sys.argv[i+1], gs_object, gs_body, 'update') i += 2 group = normalizeEmailAddressOrUID(group) - if use_cd_api or (group.find(u'@') == -1): # group settings API won't take uid so we make sure cd API is used so that we can grab real email. - group = callGAPI(cd.groups(), u'update', groupKey=group, body=cd_body, fields=u'email')[u'email'] + if use_cd_api or (group.find('@') == -1): # group settings API won't take uid so we make sure cd API is used so that we can grab real email. + group = callGAPI(cd.groups(), 'update', groupKey=group, body=cd_body, fields='email')['email'] if gs: if not GroupIsAbuseOrPostmaster(group): if gs_get_before_update: - current_settings = callGAPI(gs.groups(), u'get', - retry_reasons=[u'serviceLimit'], - groupUniqueId=group, fields=u'*') + current_settings = callGAPI(gs.groups(), 'get', + retry_reasons=['serviceLimit'], + groupUniqueId=group, fields='*') if current_settings is not None: - gs_body = dict(current_settings.items() + gs_body.items()) + gs_body = dict(list(current_settings.items()) + list(gs_body.items())) if gs_body: - callGAPI(gs.groups(), u'update', retry_reasons=[u'serviceLimit'], groupUniqueId=group, body=gs_body) - print u'updated group %s' % group + callGAPI(gs.groups(), 'update', retry_reasons=['serviceLimit'], groupUniqueId=group, body=gs_body) + print('updated group %s' % group) def doUpdateAlias(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') alias = normalizeEmailAddressOrUID(sys.argv[3], noUid=True, noLower=True) target_type = sys.argv[4].lower() - if target_type not in [u'user', u'group', u'target']: + if target_type not in ['user', 'group', 'target']: systemErrorExit(2, 'target type must be one of user, group, target; got %s' % target_type) target_email = normalizeEmailAddressOrUID(sys.argv[5]) try: - callGAPI(cd.users().aliases(), u'delete', throw_reasons=[GAPI_INVALID], userKey=alias, alias=alias) + callGAPI(cd.users().aliases(), 'delete', throw_reasons=[GAPI_INVALID], userKey=alias, alias=alias) except GAPI_invalid: - callGAPI(cd.groups().aliases(), u'delete', groupKey=alias, alias=alias) - if target_type == u'user': - callGAPI(cd.users().aliases(), u'insert', userKey=target_email, body={u'alias': alias}) - elif target_type == u'group': - callGAPI(cd.groups().aliases(), u'insert', groupKey=target_email, body={u'alias': alias}) - elif target_type == u'target': + callGAPI(cd.groups().aliases(), 'delete', groupKey=alias, alias=alias) + if target_type == 'user': + callGAPI(cd.users().aliases(), 'insert', userKey=target_email, body={'alias': alias}) + elif target_type == 'group': + callGAPI(cd.groups().aliases(), 'insert', groupKey=target_email, body={'alias': alias}) + elif target_type == 'target': try: - callGAPI(cd.users().aliases(), u'insert', throw_reasons=[GAPI_INVALID], userKey=target_email, body={u'alias': alias}) + callGAPI(cd.users().aliases(), 'insert', throw_reasons=[GAPI_INVALID], userKey=target_email, body={'alias': alias}) except GAPI_invalid: - callGAPI(cd.groups().aliases(), u'insert', groupKey=target_email, body={u'alias': alias}) - print u'updated alias %s' % alias + callGAPI(cd.groups().aliases(), 'insert', groupKey=target_email, body={'alias': alias}) + print('updated alias %s' % alias) def getCrOSDeviceEntity(i, cd): myarg = sys.argv[i].lower() - if myarg == u'cros_sn': - return i+2, getUsersToModify(u'cros_sn', sys.argv[i+1]) - if myarg == u'query': - return i+2, getUsersToModify(u'crosquery', sys.argv[i+1]) - if myarg[:6] == u'query:': + if myarg == 'cros_sn': + return i+2, getUsersToModify('cros_sn', sys.argv[i+1]) + if myarg == 'query': + return i+2, getUsersToModify('crosquery', sys.argv[i+1]) + if myarg[:6] == 'query:': query = sys.argv[i][6:] - if query[:12].lower() == u'orgunitpath:': - kwargs = {u'orgUnitPath': query[12:]} + if query[:12].lower() == 'orgunitpath:': + kwargs = {'orgUnitPath': query[12:]} else: - kwargs = {u'query': query} - devices = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', + kwargs = {'query': query} + devices = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', customerId=GC_Values[GC_CUSTOMER_ID], - fields=u'nextPageToken,chromeosdevices(deviceId)', **kwargs) - return i+1, [device[u'deviceId'] for device in devices] - return i+1, sys.argv[i].replace(u',', u' ').split() + fields='nextPageToken,chromeosdevices(deviceId)', **kwargs) + return i+1, [device['deviceId'] for device in devices] + return i+1, sys.argv[i].replace(',', ' ').split() def doUpdateCros(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i, devices = getCrOSDeviceEntity(3, cd) update_body = {} action_body = {} orgUnitPath = None ack_wipe = False while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'user': - update_body[u'annotatedUser'] = sys.argv[i+1] + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'user': + update_body['annotatedUser'] = sys.argv[i+1] i += 2 - elif myarg == u'location': - update_body[u'annotatedLocation'] = sys.argv[i+1] + elif myarg == 'location': + update_body['annotatedLocation'] = sys.argv[i+1] i += 2 - elif myarg == u'notes': - update_body[u'notes'] = sys.argv[i+1].replace(u'\\n', u'\n') + elif myarg == 'notes': + update_body['notes'] = sys.argv[i+1].replace('\\n', '\n') i += 2 - elif myarg in [u'tag', u'asset', u'assetid']: - update_body[u'annotatedAssetId'] = sys.argv[i+1] + elif myarg in ['tag', 'asset', 'assetid']: + update_body['annotatedAssetId'] = sys.argv[i+1] i += 2 - elif myarg in [u'ou', u'org']: + elif myarg in ['ou', 'org']: orgUnitPath = getOrgUnitItem(sys.argv[i+1]) i += 2 - elif myarg == u'action': - action = sys.argv[i+1].lower().replace(u'_', u'').replace(u'-', u'') + elif myarg == 'action': + action = sys.argv[i+1].lower().replace('_', '').replace('-', '') deprovisionReason = None - if action in [u'deprovisionsamemodelreplace', u'deprovisionsamemodelreplacement']: - action = u'deprovision' - deprovisionReason = u'same_model_replacement' - elif action in [u'deprovisiondifferentmodelreplace', u'deprovisiondifferentmodelreplacement']: - action = u'deprovision' - deprovisionReason = u'different_model_replacement' - elif action in [u'deprovisionretiringdevice']: - action = u'deprovision' - deprovisionReason = u'retiring_device' - elif action not in [u'disable', u'reenable']: + if action in ['deprovisionsamemodelreplace', 'deprovisionsamemodelreplacement']: + action = 'deprovision' + deprovisionReason = 'same_model_replacement' + elif action in ['deprovisiondifferentmodelreplace', 'deprovisiondifferentmodelreplacement']: + action = 'deprovision' + deprovisionReason = 'different_model_replacement' + elif action in ['deprovisionretiringdevice']: + action = 'deprovision' + deprovisionReason = 'retiring_device' + elif action not in ['disable', 'reenable']: systemErrorExit(2, 'expected action of deprovision_same_model_replace, deprovision_different_model_replace, deprovision_retiring_device, disable or reenable, got %s' % action) - action_body = {u'action': action} + action_body = {'action': action} if deprovisionReason: - action_body[u'deprovisionReason'] = deprovisionReason + action_body['deprovisionReason'] = deprovisionReason i += 2 - elif myarg == u'acknowledgedevicetouchrequirement': + elif myarg == 'acknowledgedevicetouchrequirement': ack_wipe = True i += 1 else: @@ -9170,76 +9180,76 @@ def doUpdateCros(): i = 0 count = len(devices) if action_body: - if action_body[u'action'] == u'deprovision' and not ack_wipe: - print u'WARNING: Refusing to deprovision %s devices because acknowledge_device_touch_requirement not specified. Deprovisioning a device means the device will have to be physically wiped and re-enrolled to be managed by your domain again. This requires physical access to the device and is very time consuming to perform for each device. Please add "acknowledge_device_touch_requirement" to the GAM command if you understand this and wish to proceed with the deprovision. Please also be aware that deprovisioning can have an effect on your device license count. See https://support.google.com/chrome/a/answer/3523633 for full details.' % (count) + if action_body['action'] == 'deprovision' and not ack_wipe: + print('WARNING: Refusing to deprovision %s devices because acknowledge_device_touch_requirement not specified. Deprovisioning a device means the device will have to be physically wiped and re-enrolled to be managed by your domain again. This requires physical access to the device and is very time consuming to perform for each device. Please add "acknowledge_device_touch_requirement" to the GAM command if you understand this and wish to proceed with the deprovision. Please also be aware that deprovisioning can have an effect on your device license count. See https://support.google.com/chrome/a/answer/3523633 for full details.' % (count)) sys.exit(3) for deviceId in devices: i += 1 - print u' performing action %s for %s (%s of %s)' % (action, deviceId, i, count) - callGAPI(cd.chromeosdevices(), function=u'action', customerId=GC_Values[GC_CUSTOMER_ID], resourceId=deviceId, body=action_body) + print(' performing action %s for %s (%s of %s)' % (action, deviceId, i, count)) + callGAPI(cd.chromeosdevices(), function='action', customerId=GC_Values[GC_CUSTOMER_ID], resourceId=deviceId, body=action_body) else: if update_body: for deviceId in devices: i += 1 - print u' updating %s (%s of %s)' % (deviceId, i, count) - callGAPI(service=cd.chromeosdevices(), function=u'update', customerId=GC_Values[GC_CUSTOMER_ID], deviceId=deviceId, body=update_body) + print(' updating %s (%s of %s)' % (deviceId, i, count)) + callGAPI(service=cd.chromeosdevices(), function='update', customerId=GC_Values[GC_CUSTOMER_ID], deviceId=deviceId, body=update_body) if orgUnitPath: #move_body[u'deviceIds'] = devices # split moves into max 50 devices per batch for l in range(0, len(devices), 50): - move_body = {u'deviceIds': devices[l:l+50]} - print u' moving %s devices to %s' % (len(move_body[u'deviceIds']), orgUnitPath) - callGAPI(cd.chromeosdevices(), u'moveDevicesToOu', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnitPath, body=move_body) + move_body = {'deviceIds': devices[l:l+50]} + print(' moving %s devices to %s' % (len(move_body['deviceIds']), orgUnitPath)) + callGAPI(cd.chromeosdevices(), 'moveDevicesToOu', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnitPath, body=move_body) def doUpdateMobile(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') resourceId = sys.argv[3] i = 4 body = {} while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'action': - body[u'action'] = sys.argv[i+1].lower() - if body[u'action'] == u'wipe': - body[u'action'] = u'admin_remote_wipe' - elif body[u'action'].replace(u'_', u'') in [u'accountwipe', u'wipeaccount']: - body[u'action'] = u'admin_account_wipe' - if body[u'action'] not in [u'admin_remote_wipe', u'admin_account_wipe', u'approve', u'block', u'cancel_remote_wipe_then_activate', u'cancel_remote_wipe_then_block']: - systemErrorExit(2, 'action must be one of wipe, wipeaccount, approve, block, cancel_remote_wipe_then_activate, cancel_remote_wipe_then_block; got %s' % body[u'action']) + if myarg == 'action': + body['action'] = sys.argv[i+1].lower() + if body['action'] == 'wipe': + body['action'] = 'admin_remote_wipe' + elif body['action'].replace('_', '') in ['accountwipe', 'wipeaccount']: + body['action'] = 'admin_account_wipe' + if body['action'] not in ['admin_remote_wipe', 'admin_account_wipe', 'approve', 'block', 'cancel_remote_wipe_then_activate', 'cancel_remote_wipe_then_block']: + systemErrorExit(2, 'action must be one of wipe, wipeaccount, approve, block, cancel_remote_wipe_then_activate, cancel_remote_wipe_then_block; got %s' % body['action']) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam update mobile"' % sys.argv[i]) if body: - callGAPI(cd.mobiledevices(), u'action', resourceId=resourceId, body=body, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.mobiledevices(), 'action', resourceId=resourceId, body=body, customerId=GC_Values[GC_CUSTOMER_ID]) def doDeleteMobile(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') resourceId = sys.argv[3] - callGAPI(cd.mobiledevices(), u'delete', resourceId=resourceId, customerId=GC_Values[GC_CUSTOMER_ID]) + callGAPI(cd.mobiledevices(), 'delete', resourceId=resourceId, customerId=GC_Values[GC_CUSTOMER_ID]) def doUpdateOrg(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') orgUnitPath = getOrgUnitItem(sys.argv[3]) - if sys.argv[4].lower() in [u'move', u'add']: + if sys.argv[4].lower() in ['move', 'add']: entity_type = sys.argv[5].lower() if entity_type in usergroup_types: users = getUsersToModify(entity_type=entity_type, entity=sys.argv[6]) else: - entity_type = u'users' + entity_type = 'users' users = getUsersToModify(entity_type=entity_type, entity=sys.argv[5]) - if (entity_type.startswith(u'cros')) or ((entity_type == u'all') and (sys.argv[6].lower() == u'cros')): + if (entity_type.startswith('cros')) or ((entity_type == 'all') and (sys.argv[6].lower() == 'cros')): for l in range(0, len(users), 50): - move_body = {u'deviceIds': users[l:l+50]} - print u' moving %s devices to %s' % (len(move_body[u'deviceIds']), orgUnitPath) - callGAPI(cd.chromeosdevices(), u'moveDevicesToOu', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnitPath, body=move_body) + move_body = {'deviceIds': users[l:l+50]} + print(' moving %s devices to %s' % (len(move_body['deviceIds']), orgUnitPath)) + callGAPI(cd.chromeosdevices(), 'moveDevicesToOu', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=orgUnitPath, body=move_body) else: current_user = 0 user_count = len(users) for user in users: current_user += 1 - sys.stderr.write(u' moving %s to %s (%s/%s)\n' % (user, orgUnitPath, current_user, user_count)) + sys.stderr.write(' moving %s to %s (%s/%s)\n' % (user, orgUnitPath, current_user, user_count)) try: - callGAPI(cd.users(), u'update', throw_reasons=[GAPI_CONDITION_NOT_MET], userKey=user, body={u'orgUnitPath': orgUnitPath}) + callGAPI(cd.users(), 'update', throw_reasons=[GAPI_CONDITION_NOT_MET], userKey=user, body={'orgUnitPath': orgUnitPath}) except GAPI_conditionNotMet: pass else: @@ -9247,82 +9257,82 @@ def doUpdateOrg(): i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'name': - body[u'name'] = sys.argv[i+1] + if myarg == 'name': + body['name'] = sys.argv[i+1] i += 2 - elif myarg == u'description': - body[u'description'] = sys.argv[i+1].replace(u'\\n', u'\n') + elif myarg == 'description': + body['description'] = sys.argv[i+1].replace('\\n', '\n') i += 2 - elif myarg == u'parent': + elif myarg == 'parent': parent = getOrgUnitItem(sys.argv[i+1]) - if parent.startswith(u'id:'): - body[u'parentOrgUnitId'] = parent + if parent.startswith('id:'): + body['parentOrgUnitId'] = parent else: - body[u'parentOrgUnitPath'] = parent + body['parentOrgUnitPath'] = parent i += 2 - elif myarg == u'noinherit': - body[u'blockInheritance'] = True + elif myarg == 'noinherit': + body['blockInheritance'] = True i += 1 - elif myarg == u'inherit': - body[u'blockInheritance'] = False + elif myarg == 'inherit': + body['blockInheritance'] = False i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam update org"' % sys.argv[i]) - callGAPI(cd.orgunits(), u'update', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(orgUnitPath)), body=body) + callGAPI(cd.orgunits(), 'update', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(orgUnitPath)), body=body) def doWhatIs(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') email = normalizeEmailAddressOrUID(sys.argv[2]) try: - user_or_alias = callGAPI(cd.users(), u'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_BAD_REQUEST, GAPI_INVALID], userKey=email, fields=u'id,primaryEmail') - if (user_or_alias[u'primaryEmail'].lower() == email) or (user_or_alias[u'id'] == email): - sys.stderr.write(u'%s is a user\n\n' % email) + user_or_alias = callGAPI(cd.users(), 'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_BAD_REQUEST, GAPI_INVALID], userKey=email, fields='id,primaryEmail') + if (user_or_alias['primaryEmail'].lower() == email) or (user_or_alias['id'] == email): + sys.stderr.write('%s is a user\n\n' % email) doGetUserInfo(user_email=email) return else: - sys.stderr.write(u'%s is a user alias\n\n' % email) + sys.stderr.write('%s is a user alias\n\n' % email) doGetAliasInfo(alias_email=email) return except (GAPI_notFound, GAPI_badRequest, GAPI_invalid): - sys.stderr.write(u'%s is not a user...\n' % email) - sys.stderr.write(u'%s is not a user alias...\n' % email) + sys.stderr.write('%s is not a user...\n' % email) + sys.stderr.write('%s is not a user alias...\n' % email) try: - group = callGAPI(cd.groups(), u'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_BAD_REQUEST], groupKey=email, fields=u'id,email') + group = callGAPI(cd.groups(), 'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_BAD_REQUEST], groupKey=email, fields='id,email') except (GAPI_notFound, GAPI_badRequest): - systemErrorExit(1, u'%s is not a group either!\n\nDoesn\'t seem to exist!\n\n' % email) - if (group[u'email'].lower() == email) or (group[u'id'] == email): - sys.stderr.write(u'%s is a group\n\n' % email) + systemErrorExit(1, '%s is not a group either!\n\nDoesn\'t seem to exist!\n\n' % email) + if (group['email'].lower() == email) or (group['id'] == email): + sys.stderr.write('%s is a group\n\n' % email) doGetGroupInfo(group_name=email) else: - sys.stderr.write(u'%s is a group alias\n\n' % email) + sys.stderr.write('%s is a group alias\n\n' % email) doGetAliasInfo(alias_email=email) def convertSKU2ProductId(res, sku, customerId): - results = callGAPI(res.subscriptions(), u'list', customerId=customerId) - for subscription in results[u'subscriptions']: - if sku == subscription[u'skuId']: - return subscription[u'subscriptionId'] + results = callGAPI(res.subscriptions(), 'list', customerId=customerId) + for subscription in results['subscriptions']: + if sku == subscription['skuId']: + return subscription['subscriptionId'] systemErrorExit(3, 'could not find subscription for customer %s and SKU %s' % (customerId, sku)) def doDeleteResoldSubscription(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') customerId = sys.argv[3] sku = sys.argv[4] deletionType = sys.argv[5] subscriptionId = convertSKU2ProductId(res, sku, customerId) - callGAPI(res.subscriptions(), u'delete', customerId=customerId, subscriptionId=subscriptionId, deletionType=deletionType) - print u'Cancelled %s for %s' % (sku, customerId) + callGAPI(res.subscriptions(), 'delete', customerId=customerId, subscriptionId=subscriptionId, deletionType=deletionType) + print('Cancelled %s for %s' % (sku, customerId)) def doCreateResoldSubscription(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') customerId = sys.argv[3] customerAuthToken, body = _getResoldSubscriptionAttr(sys.argv[4:], customerId) - result = callGAPI(res.subscriptions(), u'insert', customerId=customerId, customerAuthToken=customerAuthToken, body=body, fields=u'customerId') - print u'Created subscription:' + result = callGAPI(res.subscriptions(), 'insert', customerId=customerId, customerAuthToken=customerAuthToken, body=body, fields='customerId') + print('Created subscription:') print_json(None, result) def doUpdateResoldSubscription(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') function = None customerId = sys.argv[3] sku = sys.argv[4] @@ -9331,92 +9341,92 @@ def doUpdateResoldSubscription(): i = 5 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'activate': - function = u'activate' + if myarg == 'activate': + function = 'activate' i += 1 - elif myarg == u'suspend': - function = u'suspend' + elif myarg == 'suspend': + function = 'suspend' i += 1 - elif myarg == u'startpaidservice': - function = u'startPaidService' + elif myarg == 'startpaidservice': + function = 'startPaidService' i += 1 - elif myarg in [u'renewal', u'renewaltype']: - function = u'changeRenewalSettings' - kwargs[u'body'] = {u'renewalType': sys.argv[i+1].upper()} + elif myarg in ['renewal', 'renewaltype']: + function = 'changeRenewalSettings' + kwargs['body'] = {'renewalType': sys.argv[i+1].upper()} i += 2 - elif myarg in [u'seats']: - function = u'changeSeats' - kwargs[u'body'] = {u'numberOfSeats': getInteger(sys.argv[i+1], u'numberOfSeats', minVal=0)} + elif myarg in ['seats']: + function = 'changeSeats' + kwargs['body'] = {'numberOfSeats': getInteger(sys.argv[i+1], 'numberOfSeats', minVal=0)} if len(sys.argv) > i + 2 and sys.argv[i+2].isdigit(): - kwargs[u'body'][u'maximumNumberOfSeats'] = getInteger(sys.argv[i+2], u'maximumNumberOfSeats', minVal=0) + kwargs['body']['maximumNumberOfSeats'] = getInteger(sys.argv[i+2], 'maximumNumberOfSeats', minVal=0) i += 3 else: i += 2 - elif myarg in [u'plan']: - function = u'changePlan' - kwargs[u'body'] = {u'planName': sys.argv[i+1].upper()} + elif myarg in ['plan']: + function = 'changePlan' + kwargs['body'] = {'planName': sys.argv[i+1].upper()} i += 2 while i < len(sys.argv): planarg = sys.argv[i].lower() - if planarg == u'seats': - kwargs[u'body'][u'seats'] = {u'numberOfSeats': getInteger(sys.argv[i+1], u'numberOfSeats', minVal=0)} + if planarg == 'seats': + kwargs['body']['seats'] = {'numberOfSeats': getInteger(sys.argv[i+1], 'numberOfSeats', minVal=0)} if len(sys.argv) > i + 2 and sys.argv[i+2].isdigit(): - kwargs[u'body'][u'seats'][u'maximumNumberOfSeats'] = getInteger(sys.argv[i+2], u'maximumNumberOfSeats', minVal=0) + kwargs['body']['seats']['maximumNumberOfSeats'] = getInteger(sys.argv[i+2], 'maximumNumberOfSeats', minVal=0) i += 3 else: i += 2 - elif planarg in [u'purchaseorderid', u'po']: - kwargs[u'body'][u'purchaseOrderId'] = sys.argv[i+1] + elif planarg in ['purchaseorderid', 'po']: + kwargs['body']['purchaseOrderId'] = sys.argv[i+1] i += 2 - elif planarg in [u'dealcode', u'deal']: - kwargs[u'body'][u'dealCode'] = sys.argv[i+1] + elif planarg in ['dealcode', 'deal']: + kwargs['body']['dealCode'] = sys.argv[i+1] i += 2 else: systemErrorExit(3, '%s is not a valid argument to "gam update resoldsubscription plan"' % planarg) else: systemErrorExit(3, '%s is not a valid argument to "gam update resoldsubscription"' % myarg) result = callGAPI(res.subscriptions(), function, customerId=customerId, subscriptionId=subscriptionId, **kwargs) - print u'Updated %s SKU %s subscription:' % (customerId, sku) + print('Updated %s SKU %s subscription:' % (customerId, sku)) if result: print_json(None, result) def doGetResoldSubscriptions(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') customerId = sys.argv[3] customerAuthToken = None i = 4 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg in [u'customerauthtoken', u'transfertoken']: + myarg = sys.argv[i].lower().replace('_', '') + if myarg in ['customerauthtoken', 'transfertoken']: customerAuthToken = sys.argv[i+1] i += 2 else: systemErrorExit(3, '%s is not a valid argument for "gam info resoldsubscriptions"' % myarg) - result = callGAPI(res.subscriptions(), u'list', customerId=customerId, customerAuthToken=customerAuthToken) + result = callGAPI(res.subscriptions(), 'list', customerId=customerId, customerAuthToken=customerAuthToken) print_json(None, result) def _getResoldSubscriptionAttr(arg, customerId): - body = {u'plan': {}, - u'seats': {}, - u'customerId': customerId} + body = {'plan': {}, + 'seats': {}, + 'customerId': customerId} customerAuthToken = None i = 0 while i < len(arg): - myarg = arg[i].lower().replace(u'_', u'') - if myarg in [u'deal', u'dealcode']: - body[u'dealCode'] = arg[i+1] - elif myarg in [u'plan', u'planname']: - body[u'plan'][u'planName'] = arg[i+1].upper() - elif myarg in [u'purchaseorderid', u'po']: - body[u'purchaseOrderId'] = arg[i+1] - elif myarg in [u'seats']: - body[u'seats'][u'numberOfSeats'] = getInteger(sys.argv[i+1], u'numberOfSeats', minVal=0) + myarg = arg[i].lower().replace('_', '') + if myarg in ['deal', 'dealcode']: + body['dealCode'] = arg[i+1] + elif myarg in ['plan', 'planname']: + body['plan']['planName'] = arg[i+1].upper() + elif myarg in ['purchaseorderid', 'po']: + body['purchaseOrderId'] = arg[i+1] + elif myarg in ['seats']: + body['seats']['numberOfSeats'] = getInteger(sys.argv[i+1], 'numberOfSeats', minVal=0) if len(arg) > i + 2 and arg[i+2].isdigit(): - body[u'seats'][u'maximumNumberOfSeats'] = getInteger(sys.argv[i+2], u'maximumNumberOfSeats', minVal=0) + body['seats']['maximumNumberOfSeats'] = getInteger(sys.argv[i+2], 'maximumNumberOfSeats', minVal=0) i += 1 - elif myarg in [u'sku', u'skuid']: - _, body[u'skuId'] = getProductAndSKU(arg[i+1]) - elif myarg in [u'customerauthtoken', u'transfertoken']: + elif myarg in ['sku', 'skuid']: + _, body['skuId'] = getProductAndSKU(arg[i+1]) + elif myarg in ['customerauthtoken', 'transfertoken']: customerAuthToken = arg[i+1] else: systemErrorExit(3, '%s is not a valid argument for "gam create resoldsubscription"' % myarg) @@ -9424,9 +9434,9 @@ def _getResoldSubscriptionAttr(arg, customerId): return customerAuthToken, body def doGetResoldCustomer(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') customerId = sys.argv[3] - result = callGAPI(res.customers(), u'get', customerId=customerId) + result = callGAPI(res.customers(), 'get', customerId=customerId) print_json(None, result) def _getResoldCustomerAttr(arg): @@ -9434,15 +9444,15 @@ def _getResoldCustomerAttr(arg): customerAuthToken = None i = 0 while i < len(arg): - myarg = arg[i].lower().replace(u'_', u'') + myarg = arg[i].lower().replace('_', '') if myarg in ADDRESS_FIELDS_ARGUMENT_MAP: - body.setdefault(u'postalAddress', {}) - body[u'postalAddress'][ADDRESS_FIELDS_ARGUMENT_MAP[myarg]] = arg[i+1] - elif myarg in [u'email', u'alternateemail']: - body[u'alternateEmail'] = arg[i+1] - elif myarg in [u'phone', u'phonenumber']: - body[u'phoneNumber'] = arg[i+1] - elif myarg in [u'customerauthtoken', u'transfertoken']: + body.setdefault('postalAddress', {}) + body['postalAddress'][ADDRESS_FIELDS_ARGUMENT_MAP[myarg]] = arg[i+1] + elif myarg in ['email', 'alternateemail']: + body['alternateEmail'] = arg[i+1] + elif myarg in ['phone', 'phonenumber']: + body['phoneNumber'] = arg[i+1] + elif myarg in ['customerauthtoken', 'transfertoken']: customerAuthToken = arg[i+1] else: systemErrorExit(3, '%s is not a valid argument for "gam %s resoldcustomer"' % (myarg, sys.argv[1])) @@ -9450,311 +9460,311 @@ def _getResoldCustomerAttr(arg): return customerAuthToken, body def doUpdateResoldCustomer(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') customerId = sys.argv[3] customerAuthToken, body = _getResoldCustomerAttr(sys.argv[4:]) - callGAPI(res.customers(), u'patch', customerId=customerId, body=body, customerAuthToken=customerAuthToken, fields=u'customerId') - print u'updated customer %s' % customerId + callGAPI(res.customers(), 'patch', customerId=customerId, body=body, customerAuthToken=customerAuthToken, fields='customerId') + print('updated customer %s' % customerId) def doCreateResoldCustomer(): - res = buildGAPIObject(u'reseller') + res = buildGAPIObject('reseller') customerAuthToken, body = _getResoldCustomerAttr(sys.argv[4:]) - body[u'customerDomain'] = sys.argv[3] - result = callGAPI(res.customers(), u'insert', body=body, customerAuthToken=customerAuthToken, fields=u'customerId,customerDomain') - print u'Created customer %s with id %s' % (result[u'customerDomain'], result[u'customerId']) + body['customerDomain'] = sys.argv[3] + result = callGAPI(res.customers(), 'insert', body=body, customerAuthToken=customerAuthToken, fields='customerId,customerDomain') + print('Created customer %s with id %s' % (result['customerDomain'], result['customerId'])) def _getValueFromOAuth(field, credentials=None): credentials = credentials if credentials is not None else getValidOauth2TxtCredentials() - return credentials.id_token.get(field, u'Unknown') + return credentials.id_token.get(field, 'Unknown') def doGetMemberInfo(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') memberKey = normalizeEmailAddressOrUID(sys.argv[3]) groupKey = normalizeEmailAddressOrUID(sys.argv[4]) - info = callGAPI(cd.members(), u'get', memberKey=memberKey, groupKey=groupKey) + info = callGAPI(cd.members(), 'get', memberKey=memberKey, groupKey=groupKey) print_json(None, info) def doGetUserInfo(user_email=None): def user_lic_result(request_id, response, exception): - if response and u'skuId' in response: - user_licenses.append(response[u'skuId']) + if response and 'skuId' in response: + user_licenses.append(response['skuId']) - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i = 3 if user_email is None: if len(sys.argv) > 3: user_email = normalizeEmailAddressOrUID(sys.argv[3]) i = 4 else: - user_email = _getValueFromOAuth(u'email') + user_email = _getValueFromOAuth('email') getSchemas = getAliases = getGroups = getLicenses = True - projection = u'full' + projection = 'full' customFieldMask = viewType = None skus = sorted(SKUS.keys()) while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'noaliases': + if myarg == 'noaliases': getAliases = False i += 1 - elif myarg == u'nogroups': + elif myarg == 'nogroups': getGroups = False i += 1 - elif myarg in [u'nolicenses', u'nolicences']: + elif myarg in ['nolicenses', 'nolicences']: getLicenses = False i += 1 - elif myarg in [u'sku', u'skus']: - skus = sys.argv[i+1].split(u',') + elif myarg in ['sku', 'skus']: + skus = sys.argv[i+1].split(',') i += 2 - elif myarg == u'noschemas': + elif myarg == 'noschemas': getSchemas = False - projection = u'basic' + projection = 'basic' i += 1 - elif myarg in [u'custom', u'schemas']: + elif myarg in ['custom', 'schemas']: getSchemas = True - projection = u'custom' + projection = 'custom' customFieldMask = sys.argv[i+1] i += 2 - elif myarg == u'userview': - viewType = u'domain_public' + elif myarg == 'userview': + viewType = 'domain_public' getGroups = getLicenses = False i += 1 - elif myarg in [u'nousers', u'groups']: + elif myarg in ['nousers', 'groups']: i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam info user"' % myarg) - user = callGAPI(cd.users(), u'get', userKey=user_email, projection=projection, customFieldMask=customFieldMask, viewType=viewType) - print u'User: %s' % user[u'primaryEmail'] - if u'name' in user and u'givenName' in user[u'name']: - print utils.convertUTF8(u'First Name: %s' % user[u'name'][u'givenName']) - if u'name' in user and u'familyName' in user[u'name']: - print utils.convertUTF8(u'Last Name: %s' % user[u'name'][u'familyName']) - if u'languages' in user: - up = u'languageCode' - languages = [row[up] for row in user[u'languages'] if up in row] + user = callGAPI(cd.users(), 'get', userKey=user_email, projection=projection, customFieldMask=customFieldMask, viewType=viewType) + print('User: %s' % user['primaryEmail']) + if 'name' in user and 'givenName' in user['name']: + print(utils.convertUTF8('First Name: %s' % user['name']['givenName'])) + if 'name' in user and 'familyName' in user['name']: + print(utils.convertUTF8('Last Name: %s' % user['name']['familyName'])) + if 'languages' in user: + up = 'languageCode' + languages = [row[up] for row in user['languages'] if up in row] if languages: - print u'Languages: %s' % u','.join(languages) - up = u'customLanguage' - languages = [row[up] for row in user[u'languages'] if up in row] + print('Languages: %s' % ','.join(languages)) + up = 'customLanguage' + languages = [row[up] for row in user['languages'] if up in row] if languages: - print u'Custom Languages: %s' % u','.join(languages) - if u'isAdmin' in user: - print u'Is a Super Admin: %s' % user[u'isAdmin'] - if u'isDelegatedAdmin' in user: - print u'Is Delegated Admin: %s' % user[u'isDelegatedAdmin'] - if u'isEnrolledIn2Sv' in user: - print u'2-step enrolled: %s' % user[u'isEnrolledIn2Sv'] - if u'isEnforcedIn2Sv' in user: - print u'2-step enforced: %s' % user[u'isEnforcedIn2Sv'] - if u'agreedToTerms' in user: - print u'Has Agreed to Terms: %s' % user[u'agreedToTerms'] - if u'ipWhitelisted' in user: - print u'IP Whitelisted: %s' % user[u'ipWhitelisted'] - if u'suspended' in user: - print u'Account Suspended: %s' % user[u'suspended'] - if u'suspensionReason' in user: - print u'Suspension Reason: %s' % user[u'suspensionReason'] - if u'changePasswordAtNextLogin' in user: - print u'Must Change Password: %s' % user[u'changePasswordAtNextLogin'] - if u'id' in user: - print u'Google Unique ID: %s' % user[u'id'] - if u'customerId' in user: - print u'Customer ID: %s' % user[u'customerId'] - if u'isMailboxSetup' in user: - print u'Mailbox is setup: %s' % user[u'isMailboxSetup'] - if u'includeInGlobalAddressList' in user: - print u'Included in GAL: %s' % user[u'includeInGlobalAddressList'] - if u'creationTime' in user: - print u'Creation Time: %s' % user[u'creationTime'] - if u'lastLoginTime' in user: - if user[u'lastLoginTime'] == NEVER_TIME: - print u'Last login time: Never' + print('Custom Languages: %s' % ','.join(languages)) + if 'isAdmin' in user: + print('Is a Super Admin: %s' % user['isAdmin']) + if 'isDelegatedAdmin' in user: + print('Is Delegated Admin: %s' % user['isDelegatedAdmin']) + if 'isEnrolledIn2Sv' in user: + print('2-step enrolled: %s' % user['isEnrolledIn2Sv']) + if 'isEnforcedIn2Sv' in user: + print('2-step enforced: %s' % user['isEnforcedIn2Sv']) + if 'agreedToTerms' in user: + print('Has Agreed to Terms: %s' % user['agreedToTerms']) + if 'ipWhitelisted' in user: + print('IP Whitelisted: %s' % user['ipWhitelisted']) + if 'suspended' in user: + print('Account Suspended: %s' % user['suspended']) + if 'suspensionReason' in user: + print('Suspension Reason: %s' % user['suspensionReason']) + if 'changePasswordAtNextLogin' in user: + print('Must Change Password: %s' % user['changePasswordAtNextLogin']) + if 'id' in user: + print('Google Unique ID: %s' % user['id']) + if 'customerId' in user: + print('Customer ID: %s' % user['customerId']) + if 'isMailboxSetup' in user: + print('Mailbox is setup: %s' % user['isMailboxSetup']) + if 'includeInGlobalAddressList' in user: + print('Included in GAL: %s' % user['includeInGlobalAddressList']) + if 'creationTime' in user: + print('Creation Time: %s' % user['creationTime']) + if 'lastLoginTime' in user: + if user['lastLoginTime'] == NEVER_TIME: + print('Last login time: Never') else: - print u'Last login time: %s' % user[u'lastLoginTime'] - if u'orgUnitPath' in user: - print u'Google Org Unit Path: %s\n' % user[u'orgUnitPath'] - if u'thumbnailPhotoUrl' in user: - print u'Photo URL: %s\n' % user[u'thumbnailPhotoUrl'] - if u'notes' in user: - print u'Notes:' - notes = user[u'notes'] + print('Last login time: %s' % user['lastLoginTime']) + if 'orgUnitPath' in user: + print('Google Org Unit Path: %s\n' % user['orgUnitPath']) + if 'thumbnailPhotoUrl' in user: + print('Photo URL: %s\n' % user['thumbnailPhotoUrl']) + if 'notes' in user: + print('Notes:') + notes = user['notes'] if isinstance(notes, dict): - contentType = notes.get(u'contentType', u'text_plain') - print u' %s: %s' % (u'contentType', contentType) - if contentType == u'text_html': - print utils.convertUTF8(utils.indentMultiLineText(u' value: {0}'.format(utils.dehtml(notes[u'value'])), n=2)) + contentType = notes.get('contentType', 'text_plain') + print(' %s: %s' % ('contentType', contentType)) + if contentType == 'text_html': + print(utils.convertUTF8(utils.indentMultiLineText(' value: {0}'.format(utils.dehtml(notes['value'])), n=2))) else: - print utils.convertUTF8(utils.indentMultiLineText(u' value: {0}'.format(notes[u'value']), n=2)) + print(utils.convertUTF8(utils.indentMultiLineText(' value: {0}'.format(notes['value']), n=2))) else: - print utils.convertUTF8(utils.indentMultiLineText(u' value: {0}'.format(notes), n=2)) - print u'' - if u'gender' in user: - print u'Gender' - gender = user[u'gender'] + print(utils.convertUTF8(utils.indentMultiLineText(' value: {0}'.format(notes), n=2))) + print('') + if 'gender' in user: + print('Gender') + gender = user['gender'] for key in gender: - if key == u'customGender' and not gender[key]: + if key == 'customGender' and not gender[key]: continue - print utils.convertUTF8(u' %s: %s' % (key, gender[key])) - print u'' - if u'keywords' in user: - print u'Keywords:' - for keyword in user[u'keywords']: + print(utils.convertUTF8(' %s: %s' % (key, gender[key]))) + print('') + if 'keywords' in user: + print('Keywords:') + for keyword in user['keywords']: for key in keyword: - if key == u'customType' and not keyword[key]: + if key == 'customType' and not keyword[key]: continue - print utils.convertUTF8(u' %s: %s' % (key, keyword[key])) - print u'' - if u'ims' in user: - print u'IMs:' - for im in user[u'ims']: + print(utils.convertUTF8(' %s: %s' % (key, keyword[key]))) + print('') + if 'ims' in user: + print('IMs:') + for im in user['ims']: for key in im: - print utils.convertUTF8(u' %s: %s' % (key, im[key])) - print u'' - if u'addresses' in user: - print u'Addresses:' - for address in user[u'addresses']: + print(utils.convertUTF8(' %s: %s' % (key, im[key]))) + print('') + if 'addresses' in user: + print('Addresses:') + for address in user['addresses']: for key in address: - if key != u'formatted': - print utils.convertUTF8(u' %s: %s' % (key, address[key])) + if key != 'formatted': + print(utils.convertUTF8(' %s: %s' % (key, address[key]))) else: - print utils.convertUTF8(u' %s: %s' % (key, address[key].replace(u'\n', u'\\n'))) - print u'' - if u'organizations' in user: - print u'Organizations:' - for org in user[u'organizations']: + print(utils.convertUTF8(' %s: %s' % (key, address[key].replace('\n', '\\n')))) + print('') + if 'organizations' in user: + print('Organizations:') + for org in user['organizations']: for key in org: - if key == u'customType' and not org[key]: + if key == 'customType' and not org[key]: continue - print utils.convertUTF8(u' %s: %s' % (key, org[key])) - print u'' - if u'locations' in user: - print u'Locations:' - for location in user[u'locations']: + print(utils.convertUTF8(' %s: %s' % (key, org[key]))) + print('') + if 'locations' in user: + print('Locations:') + for location in user['locations']: for key in location: - if key == u'customType' and not location[key]: + if key == 'customType' and not location[key]: continue - print utils.convertUTF8(u' %s: %s' % (key, location[key])) - print u'' - if u'sshPublicKeys' in user: - print u'SSH Public Keys:' - for sshkey in user[u'sshPublicKeys']: + print(utils.convertUTF8(' %s: %s' % (key, location[key]))) + print('') + if 'sshPublicKeys' in user: + print('SSH Public Keys:') + for sshkey in user['sshPublicKeys']: for key in sshkey: - print utils.convertUTF8(u' %s: %s' % (key, sshkey[key])) - print u'' - if u'posixAccounts' in user: - print u'Posix Accounts:' - for posix in user[u'posixAccounts']: + print(utils.convertUTF8(' %s: %s' % (key, sshkey[key]))) + print('') + if 'posixAccounts' in user: + print('Posix Accounts:') + for posix in user['posixAccounts']: for key in posix: - print utils.convertUTF8(u' %s: %s' % (key, posix[key])) - print u'' - if u'phones' in user: - print u'Phones:' - for phone in user[u'phones']: + print(utils.convertUTF8(' %s: %s' % (key, posix[key]))) + print('') + if 'phones' in user: + print('Phones:') + for phone in user['phones']: for key in phone: - print utils.convertUTF8(u' %s: %s' % (key, phone[key])) - print u'' - if u'emails' in user: - if len(user[u'emails']) > 1: - print u'Other Emails:' - for an_email in user[u'emails']: - if an_email[u'address'].lower() == user[u'primaryEmail'].lower(): + print(utils.convertUTF8(' %s: %s' % (key, phone[key]))) + print('') + if 'emails' in user: + if len(user['emails']) > 1: + print('Other Emails:') + for an_email in user['emails']: + if an_email['address'].lower() == user['primaryEmail'].lower(): continue for key in an_email: - if key == u'type' and an_email[key] == u'custom': + if key == 'type' and an_email[key] == 'custom': continue - if key == u'customType': - print utils.convertUTF8(u' type: %s' % an_email[key]) + if key == 'customType': + print(utils.convertUTF8(' type: %s' % an_email[key])) else: - print utils.convertUTF8(u' %s: %s' % (key, an_email[key])) - print u'' - if u'relations' in user: - print u'Relations:' - for relation in user[u'relations']: + print(utils.convertUTF8(' %s: %s' % (key, an_email[key]))) + print('') + if 'relations' in user: + print('Relations:') + for relation in user['relations']: for key in relation: - if key == u'type' and relation[key] == u'custom': + if key == 'type' and relation[key] == 'custom': continue - elif key == u'customType': - print utils.convertUTF8(u' %s: %s' % (u'type', relation[key])) + elif key == 'customType': + print(utils.convertUTF8(' %s: %s' % ('type', relation[key]))) else: - print utils.convertUTF8(u' %s: %s' % (key, relation[key])) - print u'' - if u'externalIds' in user: - print u'External IDs:' - for externalId in user[u'externalIds']: + print(utils.convertUTF8(' %s: %s' % (key, relation[key]))) + print('') + if 'externalIds' in user: + print('External IDs:') + for externalId in user['externalIds']: for key in externalId: - if key == u'type' and externalId[key] == u'custom': + if key == 'type' and externalId[key] == 'custom': continue - elif key == u'customType': - print utils.convertUTF8(u' %s: %s' % (u'type', externalId[key])) + elif key == 'customType': + print(utils.convertUTF8(' %s: %s' % ('type', externalId[key]))) else: - print utils.convertUTF8(u' %s: %s' % (key, externalId[key])) - print u'' - if u'websites' in user: - print u'Websites:' - for website in user[u'websites']: + print(utils.convertUTF8(' %s: %s' % (key, externalId[key]))) + print('') + if 'websites' in user: + print('Websites:') + for website in user['websites']: for key in website: - if key == u'type' and website[key] == u'custom': + if key == 'type' and website[key] == 'custom': continue - elif key == u'customType': - print utils.convertUTF8(u' %s: %s' % (u'type', website[key])) + elif key == 'customType': + print(utils.convertUTF8(' %s: %s' % ('type', website[key]))) else: - print utils.convertUTF8(u' %s: %s' % (key, website[key])) - print u'' + print(utils.convertUTF8(' %s: %s' % (key, website[key]))) + print('') if getSchemas: - if u'customSchemas' in user: - print u'Custom Schemas:' - for schema in user[u'customSchemas']: - print u' Schema: %s' % schema - for field in user[u'customSchemas'][schema]: - if isinstance(user[u'customSchemas'][schema][field], list): - print u' %s:' % field - for an_item in user[u'customSchemas'][schema][field]: - print utils.convertUTF8(u' type: %s' % (an_item[u'type'])) - if an_item[u'type'] == u'custom': - print utils.convertUTF8(u' customType: %s' % (an_item[u'customType'])) - print utils.convertUTF8(u' value: %s' % (an_item[u'value'])) + if 'customSchemas' in user: + print('Custom Schemas:') + for schema in user['customSchemas']: + print(' Schema: %s' % schema) + for field in user['customSchemas'][schema]: + if isinstance(user['customSchemas'][schema][field], list): + print(' %s:' % field) + for an_item in user['customSchemas'][schema][field]: + print(utils.convertUTF8(' type: %s' % (an_item['type']))) + if an_item['type'] == 'custom': + print(utils.convertUTF8(' customType: %s' % (an_item['customType']))) + print(utils.convertUTF8(' value: %s' % (an_item['value']))) else: - print utils.convertUTF8(u' %s: %s' % (field, user[u'customSchemas'][schema][field])) - print + print(utils.convertUTF8(' %s: %s' % (field, user['customSchemas'][schema][field]))) + print() if getAliases: - if u'aliases' in user: - print u'Email Aliases:' - for alias in user[u'aliases']: - print u' %s' % alias - if u'nonEditableAliases' in user: - print u'Non-Editable Aliases:' - for alias in user[u'nonEditableAliases']: - print u' %s' % alias + if 'aliases' in user: + print('Email Aliases:') + for alias in user['aliases']: + print(' %s' % alias) + if 'nonEditableAliases' in user: + print('Non-Editable Aliases:') + for alias in user['nonEditableAliases']: + print(' %s' % alias) if getGroups: - groups = callGAPIpages(cd.groups(), u'list', u'groups', userKey=user_email, fields=u'groups(name,email),nextPageToken') + groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user_email, fields='groups(name,email),nextPageToken') if len(groups) > 0: - print u'Groups: (%s)' % len(groups) + print('Groups: (%s)' % len(groups)) for group in groups: - print u' %s <%s>' % (group[u'name'], group[u'email']) + print(' %s <%s>' % (group['name'], group['email'])) if getLicenses: - print u'Licenses:' - lic = buildGAPIObject(u'licensing') + print('Licenses:') + lic = buildGAPIObject('licensing') lbatch = lic.new_batch_http_request(callback=user_lic_result) user_licenses = [] for sku in skus: productId, skuId = getProductAndSKU(sku) - lbatch.add(lic.licenseAssignments().get(userId=user_email, productId=productId, skuId=skuId, fields=u'skuId')) + lbatch.add(lic.licenseAssignments().get(userId=user_email, productId=productId, skuId=skuId, fields='skuId')) lbatch.execute() for user_license in user_licenses: - print ' %s' % (_formatSKUIdDisplayName(user_license)) + print(' %s' % (_formatSKUIdDisplayName(user_license))) def _skuIdToDisplayName(skuId): - return SKUS[skuId][u'displayName'] if skuId in SKUS else skuId + return SKUS[skuId]['displayName'] if skuId in SKUS else skuId def _formatSKUIdDisplayName(skuId): skuIdDisplay = _skuIdToDisplayName(skuId) if skuId == skuIdDisplay: return skuId - return u'{0} ({1})'.format(skuId, skuIdDisplay) + return '{0} ({1})'.format(skuId, skuIdDisplay) def doGetGroupInfo(group_name=None): - cd = buildGAPIObject(u'directory') - gs = buildGAPIObject(u'groupssettings') + cd = buildGAPIObject('directory') + gs = buildGAPIObject('groupssettings') getAliases = getUsers = True getGroups = False if group_name is None: @@ -9764,87 +9774,87 @@ def doGetGroupInfo(group_name=None): i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'nousers': + if myarg == 'nousers': getUsers = False i += 1 - elif myarg == u'noaliases': + elif myarg == 'noaliases': getAliases = False i += 1 - elif myarg == u'groups': + elif myarg == 'groups': getGroups = True i += 1 - elif myarg in [u'nogroups', u'nolicenses', u'nolicences', u'noschemas', u'schemas', u'userview']: + elif myarg in ['nogroups', 'nolicenses', 'nolicences', 'noschemas', 'schemas', 'userview']: i += 1 - if myarg == u'schemas': + if myarg == 'schemas': i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam info group"' % myarg) - basic_info = callGAPI(cd.groups(), u'get', groupKey=group_name) + basic_info = callGAPI(cd.groups(), 'get', groupKey=group_name) settings = {} - if not GroupIsAbuseOrPostmaster(basic_info[u'email']): + if not GroupIsAbuseOrPostmaster(basic_info['email']): try: - settings = callGAPI(gs.groups(), u'get', throw_reasons=[GAPI_AUTH_ERROR], retry_reasons=[u'serviceLimit'], - groupUniqueId=basic_info[u'email']) # Use email address retrieved from cd since GS API doesn't support uid + settings = callGAPI(gs.groups(), 'get', throw_reasons=[GAPI_AUTH_ERROR], retry_reasons=['serviceLimit'], + groupUniqueId=basic_info['email']) # Use email address retrieved from cd since GS API doesn't support uid if settings is None: settings = {} except GAPI_authError: pass - print u'' - print u'Group Settings:' - for key, value in basic_info.items(): - if (key in [u'kind', u'etag']) or ((key == u'aliases') and (not getAliases)): + print('') + print('Group Settings:') + for key, value in list(basic_info.items()): + if (key in ['kind', 'etag']) or ((key == 'aliases') and (not getAliases)): continue if isinstance(value, list): - print u' %s:' % key + print(' %s:' % key) for val in value: - print u' %s' % val + print(' %s' % val) else: - print utils.convertUTF8(u' %s: %s' % (key, value)) - for key, value in settings.items(): - if key in [u'kind', u'etag', u'description', u'email', u'name']: + print(utils.convertUTF8(' %s: %s' % (key, value))) + for key, value in list(settings.items()): + if key in ['kind', 'etag', 'description', 'email', 'name']: continue - print u' %s: %s' % (key, value) + print(' %s: %s' % (key, value)) if getGroups: - groups = callGAPIpages(cd.groups(), u'list', u'groups', - userKey=basic_info[u'email'], fields=u'nextPageToken,groups(name,email)') + groups = callGAPIpages(cd.groups(), 'list', 'groups', + userKey=basic_info['email'], fields='nextPageToken,groups(name,email)') if groups: - print u'Groups: ({0})'.format(len(groups)) + print('Groups: ({0})'.format(len(groups))) for groupm in groups: - print u' %s: %s' % (groupm[u'name'], groupm[u'email']) + print(' %s: %s' % (groupm['name'], groupm['email'])) if getUsers: - members = callGAPIpages(cd.members(), u'list', u'members', groupKey=group_name, fields=u'nextPageToken,members(email,id,role,type)', maxResults=GC_Values[GC_MEMBER_MAX_RESULTS]) - print u'Members:' + members = callGAPIpages(cd.members(), 'list', 'members', groupKey=group_name, fields='nextPageToken,members(email,id,role,type)', maxResults=GC_Values[GC_MEMBER_MAX_RESULTS]) + print('Members:') for member in members: - print u' %s: %s (%s)' % (member.get(u'role', ROLE_MEMBER).lower(), member.get(u'email', member[u'id']), member[u'type'].lower()) - print u'Total %s users in group' % len(members) + print(' %s: %s (%s)' % (member.get('role', ROLE_MEMBER).lower(), member.get('email', member['id']), member['type'].lower())) + print('Total %s users in group' % len(members)) def doGetAliasInfo(alias_email=None): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') if alias_email is None: alias_email = normalizeEmailAddressOrUID(sys.argv[3]) try: - result = callGAPI(cd.users(), u'get', throw_reasons=[GAPI_INVALID, GAPI_BAD_REQUEST], userKey=alias_email) + result = callGAPI(cd.users(), 'get', throw_reasons=[GAPI_INVALID, GAPI_BAD_REQUEST], userKey=alias_email) except (GAPI_invalid, GAPI_badRequest): - result = callGAPI(cd.groups(), u'get', groupKey=alias_email) - print u' Alias Email: %s' % alias_email + result = callGAPI(cd.groups(), 'get', groupKey=alias_email) + print(' Alias Email: %s' % alias_email) try: - if result[u'primaryEmail'].lower() == alias_email.lower(): + if result['primaryEmail'].lower() == alias_email.lower(): systemErrorExit(3, '%s is a primary user email address, not an alias.' % alias_email) - print u' User Email: %s' % result[u'primaryEmail'] + print(' User Email: %s' % result['primaryEmail']) except KeyError: - print u' Group Email: %s' % result[u'email'] - print u' Unique ID: %s' % result[u'id'] + print(' Group Email: %s' % result['email']) + print(' Unique ID: %s' % result['id']) def doGetResourceCalendarInfo(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') resId = sys.argv[3] - resource = callGAPI(cd.resources().calendars(), u'get', + resource = callGAPI(cd.resources().calendars(), 'get', customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId) - if u'featureInstances' in resource: - resource[u'features'] = u', '.join([a_feature[u'feature'][u'name'] for a_feature in resource.pop(u'featureInstances')]) - if u'buildingId' in resource: - resource[u'buildingName'] = _getBuildingNameById(cd, resource[u'buildingId']) - resource[u'buildingId'] = u'id:{0}'.format(resource[u'buildingId']) + if 'featureInstances' in resource: + resource['features'] = ', '.join([a_feature['feature']['name'] for a_feature in resource.pop('featureInstances')]) + if 'buildingId' in resource: + resource['buildingName'] = _getBuildingNameById(cd, resource['buildingId']) + resource['buildingId'] = 'id:{0}'.format(resource['buildingId']) print_json(None, resource) def _filterTimeRanges(activeTimeRanges, startDate, endDate): @@ -9852,7 +9862,7 @@ def _filterTimeRanges(activeTimeRanges, startDate, endDate): return activeTimeRanges filteredTimeRanges = [] for timeRange in activeTimeRanges: - activityDate = datetime.datetime.strptime(timeRange[u'date'], YYYYMMDD_FORMAT) + activityDate = datetime.datetime.strptime(timeRange['date'], YYYYMMDD_FORMAT) if ((startDate is None) or (activityDate >= startDate)) and ((endDate is None) or (activityDate <= endDate)): filteredTimeRanges.append(timeRange) return filteredTimeRanges @@ -9862,7 +9872,7 @@ def _filterCreateReportTime(items, timeField, startTime, endTime): return items filteredItems = [] for item in items: - timeValue = datetime.datetime.strptime(item[timeField], u'%Y-%m-%dT%H:%M:%S.%fZ') + timeValue = datetime.datetime.strptime(item[timeField], '%Y-%m-%dT%H:%M:%S.%fZ') if ((startTime is None) or (timeValue >= startTime)) and ((endTime is None) or (timeValue <= endTime)): filteredItems.append(item) return filteredItems @@ -9871,7 +9881,7 @@ def _getFilterDate(dateStr): return datetime.datetime.strptime(dateStr, YYYYMMDD_FORMAT) def doGetCrosInfo(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i, devices = getCrOSDeviceEntity(3, cd) downloadfile = None targetFolder = GC_Values[GC_DRIVE_DIR] @@ -9881,8 +9891,8 @@ def doGetCrosInfo(): startDate = endDate = None listLimit = 0 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'nolists': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'nolists': noLists = True i += 1 elif myarg in CROS_START_ARGUMENTS: @@ -9891,44 +9901,44 @@ def doGetCrosInfo(): elif myarg in CROS_END_ARGUMENTS: endDate = _getFilterDate(sys.argv[i+1]) i += 2 - elif myarg == u'listlimit': + elif myarg == 'listlimit': listLimit = getInteger(sys.argv[i+1], myarg, minVal=-1) i += 2 - elif myarg == u'allfields': - projection = u'FULL' + elif myarg == 'allfields': + projection = 'FULL' fieldsList = [] i += 1 elif myarg in PROJECTION_CHOICES_MAP: projection = PROJECTION_CHOICES_MAP[myarg] - if projection == u'FULL': + if projection == 'FULL': fieldsList = [] else: fieldsList = CROS_BASIC_FIELDS_LIST[:] i += 1 elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP: if not fieldsList: - fieldsList = [u'deviceId',] + fieldsList = ['deviceId',] fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg]) i += 1 - elif myarg == u'fields': + elif myarg == 'fields': if not fieldsList: - fieldsList = [u'deviceId',] + fieldsList = ['deviceId',] fieldNameList = sys.argv[i+1] - for field in fieldNameList.lower().replace(u',', u' ').split(): + for field in fieldNameList.lower().replace(',', ' ').split(): if field in CROS_ARGUMENT_TO_PROPERTY_MAP: fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field]) if field in CROS_ACTIVE_TIME_RANGES_ARGUMENTS+CROS_DEVICE_FILES_ARGUMENTS+CROS_RECENT_USERS_ARGUMENTS: - projection = u'FULL' + projection = 'FULL' noLists = False else: systemErrorExit(2, '%s is not a valid argument for "gam info cros fields"' % field) i += 2 - elif myarg == u'downloadfile': + elif myarg == 'downloadfile': downloadfile = sys.argv[i+1] - if downloadfile.lower() == u'latest': + if downloadfile.lower() == 'latest': downloadfile = downloadfile.lower() i += 2 - elif myarg == u'targetfolder': + elif myarg == 'targetfolder': targetFolder = os.path.expanduser(sys.argv[i+1]) if not os.path.isdir(targetFolder): os.makedirs(targetFolder) @@ -9936,268 +9946,268 @@ def doGetCrosInfo(): else: systemErrorExit(2, '%s is not a valid argument for "gam info cros"' % sys.argv[i]) if fieldsList: - fields = u','.join(set(fieldsList)).replace(u'.', u'/') + fields = ','.join(set(fieldsList)).replace('.', '/') else: fields = None i = 0 device_count = len(devices) for deviceId in devices: i += 1 - cros = callGAPI(cd.chromeosdevices(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], + cros = callGAPI(cd.chromeosdevices(), 'get', customerId=GC_Values[GC_CUSTOMER_ID], deviceId=deviceId, projection=projection, fields=fields) - print u'CrOS Device: {0} ({1} of {2})'.format(deviceId, i, device_count) - if u'notes' in cros: - cros[u'notes'] = cros[u'notes'].replace(u'\n', u'\\n') + print('CrOS Device: {0} ({1} of {2})'.format(deviceId, i, device_count)) + if 'notes' in cros: + cros['notes'] = cros['notes'].replace('\n', '\\n') cros = _checkTPMVulnerability(cros) for up in CROS_SCALAR_PROPERTY_PRINT_ORDER: if up in cros: - if isinstance(cros[up], basestring): - print u' {0}: {1}'.format(up, cros[up]) + if isinstance(cros[up], str): + print(' {0}: {1}'.format(up, cros[up])) else: - sys.stdout.write(u' %s:' % up) - print_json(None, cros[up], u' ') + sys.stdout.write(' %s:' % up) + print_json(None, cros[up], ' ') if not noLists: - activeTimeRanges = _filterTimeRanges(cros.get(u'activeTimeRanges', []), startDate, endDate) + activeTimeRanges = _filterTimeRanges(cros.get('activeTimeRanges', []), startDate, endDate) lenATR = len(activeTimeRanges) if lenATR: - print u' activeTimeRanges' + print(' activeTimeRanges') for activeTimeRange in activeTimeRanges[:min(lenATR, listLimit or lenATR)]: - print u' date: {0}'.format(activeTimeRange[u'date']) - print u' activeTime: {0}'.format(str(activeTimeRange[u'activeTime'])) - print u' duration: {0}'.format(utils.formatMilliSeconds(activeTimeRange[u'activeTime'])) - print u' minutes: {0}'.format(activeTimeRange[u'activeTime']/60000) - recentUsers = cros.get(u'recentUsers', []) + print(' date: {0}'.format(activeTimeRange['date'])) + print(' activeTime: {0}'.format(str(activeTimeRange['activeTime']))) + print(' duration: {0}'.format(utils.formatMilliSeconds(activeTimeRange['activeTime']))) + print(' minutes: {0}'.format(activeTimeRange['activeTime']/60000)) + recentUsers = cros.get('recentUsers', []) lenRU = len(recentUsers) if lenRU: - print u' recentUsers' + print(' recentUsers') for recentUser in recentUsers[:min(lenRU, listLimit or lenRU)]: - print u' type: {0}'.format(recentUser[u'type']) - print u' email: {0}'.format(recentUser.get(u'email', [u'Unknown', u'UnmanagedUser'][recentUser[u'type'] == u'USER_TYPE_UNMANAGED'])) - deviceFiles = _filterCreateReportTime(cros.get(u'deviceFiles', []), u'createTime', startDate, endDate) + print(' type: {0}'.format(recentUser['type'])) + print(' email: {0}'.format(recentUser.get('email', ['Unknown', 'UnmanagedUser'][recentUser['type'] == 'USER_TYPE_UNMANAGED']))) + deviceFiles = _filterCreateReportTime(cros.get('deviceFiles', []), 'createTime', startDate, endDate) lenDF = len(deviceFiles) if lenDF: - print u' deviceFiles' + print(' deviceFiles') for deviceFile in deviceFiles[:min(lenDF, listLimit or lenDF)]: - print u' {0}: {1}'.format(deviceFile[u'type'], deviceFile[u'createTime']) + print(' {0}: {1}'.format(deviceFile['type'], deviceFile['createTime'])) if downloadfile: - deviceFiles = cros.get(u'deviceFiles', []) + deviceFiles = cros.get('deviceFiles', []) lenDF = len(deviceFiles) if lenDF: - if downloadfile == u'latest': + if downloadfile == 'latest': deviceFile = deviceFiles[-1] else: for deviceFile in deviceFiles: - if deviceFile[u'createTime'] == downloadfile: + if deviceFile['createTime'] == downloadfile: break else: - print u'ERROR: file {0} not available to download.'.format(downloadfile) + print('ERROR: file {0} not available to download.'.format(downloadfile)) deviceFile = None if deviceFile: - downloadfilename = os.path.join(targetFolder, u'cros-logs-{0}-{1}.zip'.format(deviceId, deviceFile[u'createTime'])) - _, content = cd._http.request(deviceFile[u'downloadUrl']) + downloadfilename = os.path.join(targetFolder, 'cros-logs-{0}-{1}.zip'.format(deviceId, deviceFile['createTime'])) + _, content = cd._http.request(deviceFile['downloadUrl']) writeFile(downloadfilename, content, continueOnError=True) - print u'Downloaded: {0}'.format(downloadfilename) + print('Downloaded: {0}'.format(downloadfilename)) elif downloadfile: - print u'ERROR: no files to download.' - cpuStatusReports = _filterCreateReportTime(cros.get(u'cpuStatusReports', []), u'reportTime', startDate, endDate) + print('ERROR: no files to download.') + cpuStatusReports = _filterCreateReportTime(cros.get('cpuStatusReports', []), 'reportTime', startDate, endDate) lenCSR = len(cpuStatusReports) if lenCSR: - print u' cpuStatusReports' + print(' cpuStatusReports') for cpuStatusReport in cpuStatusReports[:min(lenCSR, listLimit or lenCSR)]: - print u' reportTime: {0}'.format(cpuStatusReport[u'reportTime']) - print u' cpuTemperatureInfo' - for tempInfo in cpuStatusReport.get(u'cpuTemperatureInfo', []): - print u' {0}: {1}'.format(tempInfo[u'label'].strip(), tempInfo[u'temperature']) - print u' cpuUtilizationPercentageInfo: {0}'.format(u','.join([str(x) for x in cpuStatusReport[u'cpuUtilizationPercentageInfo']])) - diskVolumeReports = cros.get(u'diskVolumeReports', []) + print(' reportTime: {0}'.format(cpuStatusReport['reportTime'])) + print(' cpuTemperatureInfo') + for tempInfo in cpuStatusReport.get('cpuTemperatureInfo', []): + print(' {0}: {1}'.format(tempInfo['label'].strip(), tempInfo['temperature'])) + print(' cpuUtilizationPercentageInfo: {0}'.format(','.join([str(x) for x in cpuStatusReport['cpuUtilizationPercentageInfo']]))) + diskVolumeReports = cros.get('diskVolumeReports', []) lenDVR = len(diskVolumeReports) if lenDVR: - print u' diskVolumeReports' - print u' volumeInfo' + print(' diskVolumeReports') + print(' volumeInfo') for diskVolumeReport in diskVolumeReports[:min(lenDVR, listLimit or lenDVR)]: - volumeInfo = diskVolumeReport[u'volumeInfo'] + volumeInfo = diskVolumeReport['volumeInfo'] for volume in volumeInfo: - print u' volumeId: {0}'.format(volume[u'volumeId']) - print u' storageFree: {0}'.format(volume[u'storageFree']) - print u' storageTotal: {0}'.format(volume[u'storageTotal']) - systemRamFreeReports = _filterCreateReportTime(cros.get(u'systemRamFreeReports', []), u'reportTime', startDate, endDate) + print(' volumeId: {0}'.format(volume['volumeId'])) + print(' storageFree: {0}'.format(volume['storageFree'])) + print(' storageTotal: {0}'.format(volume['storageTotal'])) + systemRamFreeReports = _filterCreateReportTime(cros.get('systemRamFreeReports', []), 'reportTime', startDate, endDate) lenSRFR = len(systemRamFreeReports) if lenSRFR: - print u' systemRamFreeReports' + print(' systemRamFreeReports') for systemRamFreeReport in systemRamFreeReports[:min(lenSRFR, listLimit or lenSRFR)]: - print u' reportTime: {0}'.format(systemRamFreeReport[u'reportTime']) - print u' systemRamFreeInfo: {0}'.format(u','.join(systemRamFreeReport[u'systemRamFreeInfo'])) + print(' reportTime: {0}'.format(systemRamFreeReport['reportTime'])) + print(' systemRamFreeInfo: {0}'.format(','.join(systemRamFreeReport['systemRamFreeInfo']))) def doGetMobileInfo(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') resourceId = sys.argv[3] - info = callGAPI(cd.mobiledevices(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], resourceId=resourceId) + info = callGAPI(cd.mobiledevices(), 'get', customerId=GC_Values[GC_CUSTOMER_ID], resourceId=resourceId) print_json(None, info) -def print_json(object_name, object_value, spacing=u''): - if object_name in [u'kind', u'etag', u'etags']: +def print_json(object_name, object_value, spacing=''): + if object_name in ['kind', 'etag', 'etags']: return if object_name is not None: - sys.stdout.write(u'%s%s: ' % (spacing, object_name)) + sys.stdout.write('%s%s: ' % (spacing, object_name)) if isinstance(object_value, list): - if len(object_value) == 1 and isinstance(object_value[0], (str, unicode, int, bool)): - sys.stdout.write(utils.convertUTF8(u'%s\n' % object_value[0])) + if len(object_value) == 1 and isinstance(object_value[0], (str, int, bool)): + sys.stdout.write(utils.convertUTF8('%s\n' % object_value[0])) return if object_name is not None: - sys.stdout.write(u'\n') + sys.stdout.write('\n') for a_value in object_value: - if isinstance(a_value, (str, unicode, int, bool)): - sys.stdout.write(utils.convertUTF8(u' %s%s\n' % (spacing, a_value))) + if isinstance(a_value, (str, int, bool)): + sys.stdout.write(utils.convertUTF8(' %s%s\n' % (spacing, a_value))) else: - print_json(None, a_value, u' %s' % spacing) + print_json(None, a_value, ' %s' % spacing) elif isinstance(object_value, dict): - print + print() if object_name is not None: - sys.stdout.write(u'\n') + sys.stdout.write('\n') for another_object in object_value: - print_json(another_object, object_value[another_object], u' %s' % spacing) + print_json(another_object, object_value[another_object], ' %s' % spacing) else: - sys.stdout.write(utils.convertUTF8(u'%s\n' % (object_value))) + sys.stdout.write(utils.convertUTF8('%s\n' % (object_value))) def doSiteVerifyShow(): - verif = buildGAPIObject(u'siteVerification') + verif = buildGAPIObject('siteVerification') a_domain = sys.argv[3] - txt_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'INET_DOMAIN', u'identifier':a_domain}, u'verificationMethod':u'DNS_TXT'}) - print u'TXT Record Name: %s' % a_domain - print u'TXT Record Value: %s' % txt_record[u'token'] - print - cname_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'INET_DOMAIN', u'identifier':a_domain}, u'verificationMethod':u'DNS_CNAME'}) - cname_token = cname_record[u'token'] - cname_list = cname_token.split(u' ') + txt_record = callGAPI(verif.webResource(), 'getToken', body={'site':{'type':'INET_DOMAIN', 'identifier':a_domain}, 'verificationMethod':'DNS_TXT'}) + print('TXT Record Name: %s' % a_domain) + print('TXT Record Value: %s' % txt_record['token']) + print() + cname_record = callGAPI(verif.webResource(), 'getToken', body={'site':{'type':'INET_DOMAIN', 'identifier':a_domain}, 'verificationMethod':'DNS_CNAME'}) + cname_token = cname_record['token'] + cname_list = cname_token.split(' ') cname_subdomain = cname_list[0] cname_value = cname_list[1] - print u'CNAME Record Name: %s.%s' % (cname_subdomain, a_domain) - print u'CNAME Record Value: %s' % cname_value - print u'' - webserver_file_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'FILE'}) - webserver_file_token = webserver_file_record[u'token'] - print u'Saving web server verification file to: %s' % webserver_file_token - writeFile(webserver_file_token, u'google-site-verification: {0}'.format(webserver_file_token), continueOnError=True) - print u'Verification File URL: http://%s/%s' % (a_domain, webserver_file_token) - print - webserver_meta_record = callGAPI(verif.webResource(), u'getToken', body={u'site':{u'type':u'SITE', u'identifier':u'http://%s/' % a_domain}, u'verificationMethod':u'META'}) - print u'Meta URL: http://%s/' % a_domain - print u'Meta HTML Header Data: %s' % webserver_meta_record[u'token'] - print + print('CNAME Record Name: %s.%s' % (cname_subdomain, a_domain)) + print('CNAME Record Value: %s' % cname_value) + print('') + webserver_file_record = callGAPI(verif.webResource(), 'getToken', body={'site':{'type':'SITE', 'identifier':'http://%s/' % a_domain}, 'verificationMethod':'FILE'}) + webserver_file_token = webserver_file_record['token'] + print('Saving web server verification file to: %s' % webserver_file_token) + writeFile(webserver_file_token, 'google-site-verification: {0}'.format(webserver_file_token), continueOnError=True) + print('Verification File URL: http://%s/%s' % (a_domain, webserver_file_token)) + print() + webserver_meta_record = callGAPI(verif.webResource(), 'getToken', body={'site':{'type':'SITE', 'identifier':'http://%s/' % a_domain}, 'verificationMethod':'META'}) + print('Meta URL: http://%s/' % a_domain) + print('Meta HTML Header Data: %s' % webserver_meta_record['token']) + print() def doGetSiteVerifications(): - verif = buildGAPIObject(u'siteVerification') - sites = callGAPIitems(verif.webResource(), u'list', u'items') + verif = buildGAPIObject('siteVerification') + sites = callGAPIitems(verif.webResource(), 'list', 'items') if len(sites) > 0: for site in sites: - print u'Site: %s' % site[u'site'][u'identifier'] - print u'Type: %s' % site[u'site'][u'type'] - print u'Owners:' - for owner in site[u'owners']: - print u' %s' % owner - print + print('Site: %s' % site['site']['identifier']) + print('Type: %s' % site['site']['type']) + print('Owners:') + for owner in site['owners']: + print(' %s' % owner) + print() else: - print u'No Sites Verified.' + print('No Sites Verified.') def doSiteVerifyAttempt(): - verif = buildGAPIObject(u'siteVerification') + verif = buildGAPIObject('siteVerification') a_domain = sys.argv[3] verificationMethod = sys.argv[4].upper() - if verificationMethod == u'CNAME': - verificationMethod = u'DNS_CNAME' - elif verificationMethod in [u'TXT', u'TEXT']: - verificationMethod = u'DNS_TXT' - if verificationMethod in [u'DNS_TXT', u'DNS_CNAME']: - verify_type = u'INET_DOMAIN' + if verificationMethod == 'CNAME': + verificationMethod = 'DNS_CNAME' + elif verificationMethod in ['TXT', 'TEXT']: + verificationMethod = 'DNS_TXT' + if verificationMethod in ['DNS_TXT', 'DNS_CNAME']: + verify_type = 'INET_DOMAIN' identifier = a_domain else: - verify_type = u'SITE' - identifier = u'http://%s/' % a_domain - body = {u'site':{u'type':verify_type, u'identifier':identifier}, u'verificationMethod':verificationMethod} + verify_type = 'SITE' + identifier = 'http://%s/' % a_domain + body = {'site':{'type':verify_type, 'identifier':identifier}, 'verificationMethod':verificationMethod} try: - verify_result = callGAPI(verif.webResource(), u'insert', throw_reasons=[GAPI_BAD_REQUEST], verificationMethod=verificationMethod, body=body) + verify_result = callGAPI(verif.webResource(), 'insert', throw_reasons=[GAPI_BAD_REQUEST], verificationMethod=verificationMethod, body=body) except GAPI_badRequest as e: - print u'ERROR: %s' % str(e) - verify_data = callGAPI(verif.webResource(), u'getToken', body=body) - print u'Method: %s' % verify_data[u'method'] - print u'Token: %s' % verify_data[u'token'] - if verify_data[u'method'] in [u'DNS_CNAME', u'DNS_TXT']: + print('ERROR: %s' % str(e)) + verify_data = callGAPI(verif.webResource(), 'getToken', body=body) + print('Method: %s' % verify_data['method']) + print('Token: %s' % verify_data['token']) + if verify_data['method'] in ['DNS_CNAME', 'DNS_TXT']: resolver = dns.resolver.Resolver() - resolver.nameservers = [u'8.8.8.8', u'8.8.4.4'] - if verify_data[u'method'] == u'DNS_CNAME': - cname_token = verify_data[u'token'] - cname_list = cname_token.split(u' ') + resolver.nameservers = ['8.8.8.8', '8.8.4.4'] + if verify_data['method'] == 'DNS_CNAME': + cname_token = verify_data['token'] + cname_list = cname_token.split(' ') cname_subdomain = cname_list[0] try: - answers = resolver.query(u'%s.%s' % (cname_subdomain, a_domain), u'A') + answers = resolver.query('%s.%s' % (cname_subdomain, a_domain), 'A') for answer in answers: - print u'DNS Record: %s' % answer + print('DNS Record: %s' % answer) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): - print u'ERROR: No such domain found in DNS!' + print('ERROR: No such domain found in DNS!') else: try: - answers = resolver.query(a_domain, u'TXT') + answers = resolver.query(a_domain, 'TXT') for answer in answers: - print u'DNS Record: %s' % str(answer).replace(u'"', u'') + print('DNS Record: %s' % str(answer).replace('"', '')) except dns.resolver.NXDOMAIN: - print u'ERROR: no such domain found in DNS!' + print('ERROR: no such domain found in DNS!') return - print u'SUCCESS!' - print u'Verified: %s' % verify_result[u'site'][u'identifier'] - print u'ID: %s' % verify_result[u'id'] - print u'Type: %s' % verify_result[u'site'][u'type'] - print u'All Owners:' + print('SUCCESS!') + print('Verified: %s' % verify_result['site']['identifier']) + print('ID: %s' % verify_result['id']) + print('Type: %s' % verify_result['site']['type']) + print('All Owners:') try: - for owner in verify_result[u'owners']: - print u' %s' % owner + for owner in verify_result['owners']: + print(' %s' % owner) except KeyError: pass - print - print u'You can now add %s or it\'s subdomains as secondary or domain aliases of the %s G Suite Account.' % (a_domain, GC_Values[GC_DOMAIN]) + print() + print('You can now add %s or it\'s subdomains as secondary or domain aliases of the %s G Suite Account.' % (a_domain, GC_Values[GC_DOMAIN])) def orgUnitPathQuery(path, checkSuspended): - query = u"orgUnitPath='{0}'".format(path.replace(u"'", u"\\'")) if path != u'/' else u'' + query = "orgUnitPath='{0}'".format(path.replace("'", "\\'")) if path != '/' else '' if checkSuspended is not None: - query += u' isSuspended={0}'.format(checkSuspended) + query += ' isSuspended={0}'.format(checkSuspended) return query def makeOrgUnitPathAbsolute(path): - if path == u'/': + if path == '/': return path - if path.startswith(u'/'): - return path.rstrip(u'/') - if path.startswith(u'id:'): + if path.startswith('/'): + return path.rstrip('/') + if path.startswith('id:'): return path - if path.startswith(u'uid:'): + if path.startswith('uid:'): return path[1:] - return u'/'+path.rstrip(u'/') + return '/'+path.rstrip('/') def makeOrgUnitPathRelative(path): - if path == u'/': + if path == '/': return path - if path.startswith(u'/'): - return path[1:].rstrip(u'/') - if path.startswith(u'id:'): + if path.startswith('/'): + return path[1:].rstrip('/') + if path.startswith('id:'): return path - if path.startswith(u'uid:'): + if path.startswith('uid:'): return path[1:] - return path.rstrip(u'/') + return path.rstrip('/') def encodeOrgUnitPath(path): - if path.find(u'+') == -1 and path.find(u'%') == -1: + if path.find('+') == -1 and path.find('%') == -1: return path - encpath = u'' + encpath = '' for c in path: - if c == u'+': - encpath += u'%2B' - elif c == u'%': - encpath += u'%25' + if c == '+': + encpath += '%2B' + elif c == '%': + encpath += '%25' else: encpath += c return encpath def getOrgUnitItem(orgUnit, pathOnly=False, absolutePath=True): - if pathOnly and (orgUnit.startswith(u'id:') or orgUnit.startswith(u'uid:')): + if pathOnly and (orgUnit.startswith('id:') or orgUnit.startswith('uid:')): systemErrorExit(2, '%s is not valid in this context' % orgUnit) if absolutePath: return makeOrgUnitPathAbsolute(orgUnit) @@ -10205,28 +10215,28 @@ def getOrgUnitItem(orgUnit, pathOnly=False, absolutePath=True): def getOrgUnitId(orgUnit, cd=None): if cd is None: - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') orgUnit = getOrgUnitItem(orgUnit) - if orgUnit[:3] == u'id:': + if orgUnit[:3] == 'id:': return (orgUnit, orgUnit) - result = callGAPI(cd.orgunits(), u'get', - customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(orgUnit)), fields=u'orgUnitId') - return (orgUnit, result[u'orgUnitId']) + result = callGAPI(cd.orgunits(), 'get', + customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(orgUnit)), fields='orgUnitId') + return (orgUnit, result['orgUnitId']) def getTopLevelOrgId(cd, orgUnitPath): try: # create a temp org so we can learn what the top level org ID is (sigh) - temp_org = callGAPI(cd.orgunits(), u'insert', customerId=GC_Values[GC_CUSTOMER_ID], - body={u'name': u'temp-delete-me', u'parentOrgUnitPath': orgUnitPath}, - fields=u'parentOrgUnitId,orgUnitId') - callGAPI(cd.orgunits(), u'delete', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=temp_org[u'orgUnitId']) - return temp_org[u'parentOrgUnitId'] + temp_org = callGAPI(cd.orgunits(), 'insert', customerId=GC_Values[GC_CUSTOMER_ID], + body={'name': 'temp-delete-me', 'parentOrgUnitPath': orgUnitPath}, + fields='parentOrgUnitId,orgUnitId') + callGAPI(cd.orgunits(), 'delete', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=temp_org['orgUnitId']) + return temp_org['parentOrgUnitId'] except: pass return None def doGetOrgInfo(name=None, return_attrib=None): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') checkSuspended = None if not name: name = getOrgUnitItem(sys.argv[3]) @@ -10235,141 +10245,141 @@ def doGetOrgInfo(name=None, return_attrib=None): i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'nousers': + if myarg == 'nousers': get_users = False i += 1 - elif myarg in [u'children', u'child']: + elif myarg in ['children', 'child']: show_children = True i += 1 - elif myarg in [u'suspended', u'notsuspended']: - checkSuspended = myarg == u'suspended' + elif myarg in ['suspended', 'notsuspended']: + checkSuspended = myarg == 'suspended' i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam info org"' % sys.argv[i]) - if name == u'/': - orgs = callGAPI(cd.orgunits(), u'list', - customerId=GC_Values[GC_CUSTOMER_ID], type=u'children', - fields=u'organizationUnits/parentOrgUnitId') - if u'organizationUnits' in orgs and orgs[u'organizationUnits']: - name = orgs[u'organizationUnits'][0][u'parentOrgUnitId'] + if name == '/': + orgs = callGAPI(cd.orgunits(), 'list', + customerId=GC_Values[GC_CUSTOMER_ID], type='children', + fields='organizationUnits/parentOrgUnitId') + if 'organizationUnits' in orgs and orgs['organizationUnits']: + name = orgs['organizationUnits'][0]['parentOrgUnitId'] else: - topLevelOrgId = getTopLevelOrgId(cd, u'/') + topLevelOrgId = getTopLevelOrgId(cd, '/') if topLevelOrgId: name = topLevelOrgId else: name = makeOrgUnitPathRelative(name) - result = callGAPI(cd.orgunits(), u'get', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(name)) + result = callGAPI(cd.orgunits(), 'get', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(name)) if return_attrib: return result[return_attrib] print_json(None, result) if get_users: - name = result[u'orgUnitPath'] - page_message = u'Got %%total_items%% Users: %%first_item%% - %%last_item%%\n' - users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, - message_attribute=u'primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=orgUnitPathQuery(name, checkSuspended), - fields=u'users(primaryEmail,orgUnitPath),nextPageToken', maxResults=GC_Values[GC_USER_MAX_RESULTS]) + name = result['orgUnitPath'] + page_message = 'Got %%total_items%% Users: %%first_item%% - %%last_item%%\n' + users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + message_attribute='primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=orgUnitPathQuery(name, checkSuspended), + fields='users(primaryEmail,orgUnitPath),nextPageToken', maxResults=GC_Values[GC_USER_MAX_RESULTS]) if checkSuspended is None: - print u'Users:' + print('Users:') elif not checkSuspended: - print u'Users (Not suspended):' + print('Users (Not suspended):') else: - print u'Users (Suspended):' + print('Users (Suspended):') for user in users: - if show_children or (name.lower() == user[u'orgUnitPath'].lower()): - sys.stdout.write(u' %s' % user[u'primaryEmail']) - if name.lower() != user[u'orgUnitPath'].lower(): - print u' (child)' + if show_children or (name.lower() == user['orgUnitPath'].lower()): + sys.stdout.write(' %s' % user['primaryEmail']) + if name.lower() != user['orgUnitPath'].lower(): + print(' (child)') else: - print u'' + print('') def doGetASPs(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: - asps = callGAPIitems(cd.asps(), u'list', u'items', userKey=user) + asps = callGAPIitems(cd.asps(), 'list', 'items', userKey=user) if len(asps) > 0: - print u'Application-Specific Passwords for %s' % user + print('Application-Specific Passwords for %s' % user) for asp in asps: - if asp[u'creationTime'] == u'0': - created_date = u'Unknown' + if asp['creationTime'] == '0': + created_date = 'Unknown' else: - created_date = datetime.datetime.fromtimestamp(int(asp[u'creationTime'])/1000).strftime(u'%Y-%m-%d %H:%M:%S') - if asp[u'lastTimeUsed'] == u'0': - used_date = u'Never' + created_date = datetime.datetime.fromtimestamp(int(asp['creationTime'])/1000).strftime('%Y-%m-%d %H:%M:%S') + if asp['lastTimeUsed'] == '0': + used_date = 'Never' else: - used_date = datetime.datetime.fromtimestamp(int(asp[u'lastTimeUsed'])/1000).strftime(u'%Y-%m-%d %H:%M:%S') - print u' ID: %s\n Name: %s\n Created: %s\n Last Used: %s\n' % (asp[u'codeId'], asp[u'name'], created_date, used_date) + used_date = datetime.datetime.fromtimestamp(int(asp['lastTimeUsed'])/1000).strftime('%Y-%m-%d %H:%M:%S') + print(' ID: %s\n Name: %s\n Created: %s\n Last Used: %s\n' % (asp['codeId'], asp['name'], created_date, used_date)) else: - print u' no ASPs for %s\n' % user + print(' no ASPs for %s\n' % user) def doDelASP(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') codeIdList = sys.argv[5].lower() - if codeIdList == u'all': + if codeIdList == 'all': allCodeIds = True else: allCodeIds = False - codeIds = codeIdList.replace(u',', u' ').split() + codeIds = codeIdList.replace(',', ' ').split() for user in users: if allCodeIds: - asps = callGAPIitems(cd.asps(), u'list', u'items', userKey=user, fields=u'items/codeId') - codeIds = [asp[u'codeId'] for asp in asps] + asps = callGAPIitems(cd.asps(), 'list', 'items', userKey=user, fields='items/codeId') + codeIds = [asp['codeId'] for asp in asps] for codeId in codeIds: - callGAPI(cd.asps(), u'delete', userKey=user, codeId=codeId) - print u'deleted ASP %s for %s' % (codeId, user) + callGAPI(cd.asps(), 'delete', userKey=user, codeId=codeId) + print('deleted ASP %s for %s' % (codeId, user)) def printBackupCodes(user, codes): jcount = len(codes) realcount = 0 for code in codes: - if u'verificationCode' in code and code[u'verificationCode']: + if 'verificationCode' in code and code['verificationCode']: realcount += 1 - print u'Backup verification codes for {0} ({1})'.format(user, realcount) - print u'' + print('Backup verification codes for {0} ({1})'.format(user, realcount)) + print('') if jcount > 0: j = 0 for code in codes: j += 1 - print u'{0}. {1}'.format(j, code[u'verificationCode']) - print u'' + print('{0}. {1}'.format(j, code['verificationCode'])) + print('') def doGetBackupCodes(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: try: - codes = callGAPIitems(cd.verificationCodes(), u'list', u'items', throw_reasons=[GAPI_INVALID_ARGUMENT, GAPI_INVALID], userKey=user) + codes = callGAPIitems(cd.verificationCodes(), 'list', 'items', throw_reasons=[GAPI_INVALID_ARGUMENT, GAPI_INVALID], userKey=user) except (GAPI_invalidArgument, GAPI_invalid): codes = [] printBackupCodes(user, codes) def doGenBackupCodes(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: - callGAPI(cd.verificationCodes(), u'generate', userKey=user) - codes = callGAPIitems(cd.verificationCodes(), u'list', u'items', userKey=user) + callGAPI(cd.verificationCodes(), 'generate', userKey=user) + codes = callGAPIitems(cd.verificationCodes(), 'list', 'items', userKey=user) printBackupCodes(user, codes) def doDelBackupCodes(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: try: - callGAPI(cd.verificationCodes(), u'invalidate', soft_errors=True, throw_reasons=[GAPI_INVALID], userKey=user) + callGAPI(cd.verificationCodes(), 'invalidate', soft_errors=True, throw_reasons=[GAPI_INVALID], userKey=user) except GAPI_invalid: - print u'No 2SV backup codes for %s' % user + print('No 2SV backup codes for %s' % user) continue - print u'2SV backup codes for %s invalidated' % user + print('2SV backup codes for %s invalidated' % user) def commonClientIds(clientId): - if clientId == u'gasmo': - return u'1095133494869.apps.googleusercontent.com' + if clientId == 'gasmo': + return '1095133494869.apps.googleusercontent.com' return clientId def doDelTokens(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') clientId = None i = 5 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', '') - if myarg == u'clientid': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'clientid': clientId = commonClientIds(sys.argv[i+1]) i += 2 else: @@ -10378,36 +10388,36 @@ def doDelTokens(users): systemErrorExit(3, 'you must specify a clientid for "gam delete token"') for user in users: try: - callGAPI(cd.tokens(), u'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_RESOURCE_NOT_FOUND], userKey=user, clientId=clientId) + callGAPI(cd.tokens(), 'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_RESOURCE_NOT_FOUND], userKey=user, clientId=clientId) except (GAPI_notFound, GAPI_resourceNotFound): - print u'User %s did not authorize %s' % (user, clientId) + print('User %s did not authorize %s' % (user, clientId)) continue - callGAPI(cd.tokens(), u'delete', userKey=user, clientId=clientId) - print u'Deleted token for %s' % user + callGAPI(cd.tokens(), 'delete', userKey=user, clientId=clientId) + print('Deleted token for %s' % user) def printShowTokens(i, entityType, users, csvFormat): def _showToken(token): - print u' Client ID: %s' % token[u'clientId'] + print(' Client ID: %s' % token['clientId']) for item in token: - if item not in [u'clientId', u'scopes']: - print utils.convertUTF8(u' %s: %s' % (item, token.get(item, u''))) - item = u'scopes' - print u' %s:' % item + if item not in ['clientId', 'scopes']: + print(utils.convertUTF8(' %s: %s' % (item, token.get(item, '')))) + item = 'scopes' + print(' %s:' % item) for it in token.get(item, []): - print u' %s' % it + print(' %s' % it) - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') if csvFormat: todrive = False - titles = [u'user', u'clientId', u'displayText', u'anonymous', u'nativeApp', u'userKey', u'scopes'] + titles = ['user', 'clientId', 'displayText', 'anonymous', 'nativeApp', 'userKey', 'scopes'] csvRows = [] clientId = None while i < len(sys.argv): myarg = sys.argv[i].lower() - if csvFormat and myarg == u'todrive': + if csvFormat and myarg == 'todrive': todrive = True i += 1 - elif myarg == u'clientid': + elif myarg == 'clientid': clientId = commonClientIds(sys.argv[i+1]) i += 2 elif not entityType: @@ -10415,28 +10425,28 @@ def printShowTokens(i, entityType, users, csvFormat): users = getUsersToModify(entity_type=entityType, entity=sys.argv[i+1], silent=False) i += 2 else: - systemErrorExit(2, '%s is not a valid argument for "gam %s tokens"' % (myarg, [u'show', u'print'][csvFormat])) + systemErrorExit(2, '%s is not a valid argument for "gam %s tokens"' % (myarg, ['show', 'print'][csvFormat])) if not entityType: - users = getUsersToModify(entity_type=u'all', entity=u'users', silent=False) - fields = u','.join([u'clientId', u'displayText', u'anonymous', u'nativeApp', u'userKey', u'scopes']) + users = getUsersToModify(entity_type='all', entity='users', silent=False) + fields = ','.join(['clientId', 'displayText', 'anonymous', 'nativeApp', 'userKey', 'scopes']) i = 0 count = len(users) for user in users: i += 1 try: if csvFormat: - sys.stderr.write(u'Getting Access Tokens for %s\n' % (user)) + sys.stderr.write('Getting Access Tokens for %s\n' % (user)) if clientId: - results = [callGAPI(cd.tokens(), u'get', + results = [callGAPI(cd.tokens(), 'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_USER_NOT_FOUND, GAPI_RESOURCE_NOT_FOUND], userKey=user, clientId=clientId, fields=fields)] else: - results = callGAPIitems(cd.tokens(), u'list', u'items', + results = callGAPIitems(cd.tokens(), 'list', 'items', throw_reasons=[GAPI_USER_NOT_FOUND], - userKey=user, fields=u'items({0})'.format(fields)) + userKey=user, fields='items({0})'.format(fields)) jcount = len(results) if not csvFormat: - print u'User: {0}, Access Tokens ({1}/{2})'.format(user, i, count) + print('User: {0}, Access Tokens ({1}/{2})'.format(user, i, count)) if jcount == 0: continue for token in results: @@ -10445,151 +10455,151 @@ def printShowTokens(i, entityType, users, csvFormat): if jcount == 0: continue for token in results: - row = {u'user': user, u'scopes': u' '.join(token.get(u'scopes', []))} + row = {'user': user, 'scopes': ' '.join(token.get('scopes', []))} for item in token: - if item not in [u'scopes']: - row[item] = token.get(item, u'') + if item not in ['scopes']: + row[item] = token.get(item, '') csvRows.append(row) except (GAPI_notFound, GAPI_userNotFound, GAPI_resourceNotFound): pass if csvFormat: - writeCSVfile(csvRows, titles, u'OAuth Tokens', todrive) + writeCSVfile(csvRows, titles, 'OAuth Tokens', todrive) def doDeprovUser(users): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') for user in users: - print u'Getting Application Specific Passwords for %s' % user - asps = callGAPIitems(cd.asps(), u'list', u'items', userKey=user, fields=u'items/codeId') + print('Getting Application Specific Passwords for %s' % user) + asps = callGAPIitems(cd.asps(), 'list', 'items', userKey=user, fields='items/codeId') jcount = len(asps) if jcount > 0: j = 0 for asp in asps: j += 1 - print u' deleting ASP %s of %s' % (j, jcount) - callGAPI(cd.asps(), u'delete', userKey=user, codeId=asp[u'codeId']) + print(' deleting ASP %s of %s' % (j, jcount)) + callGAPI(cd.asps(), 'delete', userKey=user, codeId=asp['codeId']) else: - print u'No ASPs' - print u'Invalidating 2SV Backup Codes for %s' % user + print('No ASPs') + print('Invalidating 2SV Backup Codes for %s' % user) try: - callGAPI(cd.verificationCodes(), u'invalidate', soft_errors=True, throw_reasons=[GAPI_INVALID], userKey=user) + callGAPI(cd.verificationCodes(), 'invalidate', soft_errors=True, throw_reasons=[GAPI_INVALID], userKey=user) except GAPI_invalid: - print u'No 2SV Backup Codes' - print u'Getting tokens for %s...' % user - tokens = callGAPIitems(cd.tokens(), u'list', u'items', userKey=user, fields=u'items/clientId') + print('No 2SV Backup Codes') + print('Getting tokens for %s...' % user) + tokens = callGAPIitems(cd.tokens(), 'list', 'items', userKey=user, fields='items/clientId') jcount = len(tokens) if jcount > 0: j = 0 for token in tokens: j += 1 - print u' deleting token %s of %s' % (j, jcount) - callGAPI(cd.tokens(), u'delete', userKey=user, clientId=token[u'clientId']) + print(' deleting token %s of %s' % (j, jcount)) + callGAPI(cd.tokens(), 'delete', userKey=user, clientId=token['clientId']) else: - print u'No Tokens' - print u'Done deprovisioning %s' % user + print('No Tokens') + print('Done deprovisioning %s' % user) def doDeleteUser(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') user_email = normalizeEmailAddressOrUID(sys.argv[3]) - print u"Deleting account for %s" % (user_email) - callGAPI(cd.users(), u'delete', userKey=user_email) + print("Deleting account for %s" % (user_email)) + callGAPI(cd.users(), 'delete', userKey=user_email) def doUndeleteUser(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') user = normalizeEmailAddressOrUID(sys.argv[3]) - orgUnit = u'/' + orgUnit = '/' i = 4 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg in [u'ou', u'org']: + if myarg in ['ou', 'org']: orgUnit = makeOrgUnitPathAbsolute(sys.argv[i+1]) i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam undelete user"' % sys.argv[i]) - if user.find(u'@') == -1: + if user.find('@') == -1: user_uid = user else: - print u'Looking up UID for %s...' % user - deleted_users = callGAPIpages(cd.users(), u'list', u'users', + print('Looking up UID for %s...' % user) + deleted_users = callGAPIpages(cd.users(), 'list', 'users', customer=GC_Values[GC_CUSTOMER_ID], showDeleted=True, maxResults=GC_Values[GC_USER_MAX_RESULTS]) matching_users = list() for deleted_user in deleted_users: - if str(deleted_user[u'primaryEmail']).lower() == user: + if str(deleted_user['primaryEmail']).lower() == user: matching_users.append(deleted_user) if len(matching_users) < 1: systemErrorExit(3, 'could not find deleted user with that address.') elif len(matching_users) > 1: - print u'ERROR: more than one matching deleted %s user. Please select the correct one to undelete and specify with "gam undelete user uid:"' % user - print u'' + print('ERROR: more than one matching deleted %s user. Please select the correct one to undelete and specify with "gam undelete user uid:"' % user) + print('') for matching_user in matching_users: - print u' uid:%s ' % matching_user[u'id'] - for attr_name in [u'creationTime', u'lastLoginTime', u'deletionTime']: + print(' uid:%s ' % matching_user['id']) + for attr_name in ['creationTime', 'lastLoginTime', 'deletionTime']: try: if matching_user[attr_name] == NEVER_TIME: - matching_user[attr_name] = u'Never' - print u' %s: %s ' % (attr_name, matching_user[attr_name]) + matching_user[attr_name] = 'Never' + print(' %s: %s ' % (attr_name, matching_user[attr_name])) except KeyError: pass - print + print() sys.exit(3) else: - user_uid = matching_users[0][u'id'] - print u"Undeleting account for %s" % user - callGAPI(cd.users(), u'undelete', userKey=user_uid, body={u'orgUnitPath': orgUnit}) + user_uid = matching_users[0]['id'] + print("Undeleting account for %s" % user) + callGAPI(cd.users(), 'undelete', userKey=user_uid, body={'orgUnitPath': orgUnit}) def doDeleteGroup(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') group = normalizeEmailAddressOrUID(sys.argv[3]) - print u"Deleting group %s" % group - callGAPI(cd.groups(), u'delete', groupKey=group) + print("Deleting group %s" % group) + callGAPI(cd.groups(), 'delete', groupKey=group) def doDeleteAlias(alias_email=None): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') is_user = is_group = False if alias_email is None: alias_email = sys.argv[3] - if alias_email.lower() == u'user': + if alias_email.lower() == 'user': is_user = True alias_email = sys.argv[4] - elif alias_email.lower() == u'group': + elif alias_email.lower() == 'group': is_group = True alias_email = sys.argv[4] alias_email = normalizeEmailAddressOrUID(alias_email, noUid=True, noLower=True) - print u"Deleting alias %s" % alias_email + print("Deleting alias %s" % alias_email) if is_user or (not is_user and not is_group): try: - callGAPI(cd.users().aliases(), u'delete', throw_reasons=[GAPI_INVALID, GAPI_BAD_REQUEST, GAPI_NOT_FOUND], userKey=alias_email, alias=alias_email) + callGAPI(cd.users().aliases(), 'delete', throw_reasons=[GAPI_INVALID, GAPI_BAD_REQUEST, GAPI_NOT_FOUND], userKey=alias_email, alias=alias_email) return except (GAPI_invalid, GAPI_badRequest): pass except GAPI_notFound: systemErrorExit(4, 'The alias %s does not exist' % alias_email) if not is_user or (not is_user and not is_group): - callGAPI(cd.groups().aliases(), u'delete', groupKey=alias_email, alias=alias_email) + callGAPI(cd.groups().aliases(), 'delete', groupKey=alias_email, alias=alias_email) def doDeleteResourceCalendar(): resId = sys.argv[3] - cd = buildGAPIObject(u'directory') - print u"Deleting resource calendar %s" % resId - callGAPI(cd.resources().calendars(), u'delete', + cd = buildGAPIObject('directory') + print("Deleting resource calendar %s" % resId) + callGAPI(cd.resources().calendars(), 'delete', customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId) def doDeleteOrg(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') name = getOrgUnitItem(sys.argv[3]) - print u"Deleting organization %s" % name - callGAPI(cd.orgunits(), u'delete', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(name))) + print("Deleting organization %s" % name) + callGAPI(cd.orgunits(), 'delete', customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(name))) # Send an email def send_email(msg_subj, msg_txt, msg_rcpt=None): - userId, gmail = buildGmailGAPIObject(_getValueFromOAuth(u'email')) + userId, gmail = buildGmailGAPIObject(_getValueFromOAuth('email')) if not msg_rcpt: msg_rcpt = userId msg = MIMEText(msg_txt) - msg[u'Subject'] = msg_subj - msg[u'From'] = userId - msg[u'To'] = msg_rcpt - callGAPI(gmail.users().messages(), u'send', - userId=userId, body={u'raw': base64.urlsafe_b64encode(msg.as_string())}) + msg['Subject'] = msg_subj + msg['From'] = userId + msg['To'] = msg_rcpt + callGAPI(gmail.users().messages(), 'send', + userId=userId, body={'raw': base64.urlsafe_b64encode(msg.as_string())}) def addFieldToFieldsList(fieldName, fieldsChoiceMap, fieldsList): fields = fieldsChoiceMap[fieldName.lower()] @@ -10657,317 +10667,317 @@ def sortCSVTitles(firstTitle, titles): def writeCSVfile(csvRows, titles, list_type, todrive): if GC_Values[GC_CSV_ROW_FILTER]: row_dict = json.loads(GC_Values[GC_CSV_ROW_FILTER]) - for match_column, filter_str in row_dict.items(): - if filter_str.lower()[:4] == u'date': + for match_column, filter_str in list(row_dict.items()): + if filter_str.lower()[:4] == 'date': new_csvRows = [] direction = filter_str[4] - if direction not in [u'<', u'>']: - systemErrorExit(3, u'%s is not a valid filter date direction' % direction) + if direction not in ['<', '>']: + systemErrorExit(3, '%s is not a valid filter date direction' % direction) try: date_str = filter_str[5:] filter_date = dateutil.parser.parse(date_str, ignoretz=True) except ValueError: - systemErrorExit(3, u'%s is not a date GAM understands' % filter_str) + systemErrorExit(3, '%s is not a date GAM understands' % filter_str) for row in csvRows: try: row_date = dateutil.parser.parse(row[match_column], ignoretz=True) except ValueError: - row_date = dateutil.parser.parse(u'1/1/1970') - if direction == u'<' and row_date < filter_date: + row_date = dateutil.parser.parse('1/1/1970') + if direction == '<' and row_date < filter_date: new_csvRows.append(row) - elif direction == u'>' and row_date > filter_date: + elif direction == '>' and row_date > filter_date: new_csvRows.append(row) csvRows = new_csvRows else: - if filter_str.lower()[:6] == u'regex': + if filter_str.lower()[:6] == 'regex': filter_str = filter_str[6:] if match_column not in titles: - sys.stderr.write(u'WARNING: Row filter %s is not in output columns\n' % match_column) + sys.stderr.write('WARNING: Row filter %s is not in output columns\n' % match_column) continue regex = re.compile(filter_str) - csvRows = [row for row in csvRows if regex.search(row.get(match_column, u''))] + csvRows = [row for row in csvRows if regex.search(row.get(match_column, ''))] if GC_Values[GC_CSV_HEADER_FILTER]: - titles_filter = GC_Values[GC_CSV_HEADER_FILTER].lower().split(u',') + titles_filter = GC_Values[GC_CSV_HEADER_FILTER].lower().split(',') titles = [t for t in titles if t.lower() in titles_filter] - csv.register_dialect(u'nixstdout', lineterminator=u'\n') + csv.register_dialect('nixstdout', lineterminator='\n') if todrive: - write_to = StringIO.StringIO() + write_to = io.StringIO() else: write_to = sys.stdout - writer = csv.DictWriter(write_to, fieldnames=titles, dialect=u'nixstdout', extrasaction=u'ignore', quoting=csv.QUOTE_MINIMAL) + writer = csv.DictWriter(write_to, fieldnames=titles, dialect='nixstdout', extrasaction='ignore', quoting=csv.QUOTE_MINIMAL) try: writer.writerow(dict((item, item) for item in writer.fieldnames)) writer.writerows(csvRows) except IOError as e: systemErrorExit(6, e) if todrive: - admin_email = _getValueFromOAuth(u'email') + admin_email = _getValueFromOAuth('email') _, drive = buildDrive3GAPIObject(admin_email) if not drive: - print u'''\nGAM is not authorized to create Drive files. Please run: + print('''\nGAM is not authorized to create Drive files. Please run: gam user %s check serviceaccount -and follow recommend steps to authorize GAM for Drive access.''' % (admin_email) +and follow recommend steps to authorize GAM for Drive access.''' % (admin_email)) sys.exit(5) - result = callGAPI(drive.about(), u'get', fields=u'maxImportSizes') + result = callGAPI(drive.about(), 'get', fields='maxImportSizes') columns = len(titles) rows = len(csvRows) cell_count = rows * columns data_size = string_file.len - max_sheet_bytes = int(result[u'maxImportSizes'][MIMETYPE_GA_SPREADSHEET]) + max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET]) if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes: - print u'{0}{1}'.format(WARNING_PREFIX, MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET) - mimeType = u'text/csv' + print('{0}{1}'.format(WARNING_PREFIX, MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET)) + mimeType = 'text/csv' else: mimeType = MIMETYPE_GA_SPREADSHEET - body = {u'description': u' '.join(sys.argv), - u'name': u'%s - %s' % (GC_Values[GC_DOMAIN], list_type), - u'mimeType': mimeType} - result = callGAPI(drive.files(), u'create', fields=u'webViewLink', + body = {'description': ' '.join(sys.argv), + 'name': '%s - %s' % (GC_Values[GC_DOMAIN], list_type), + 'mimeType': mimeType} + result = callGAPI(drive.files(), 'create', fields='webViewLink', body=body, media_body=googleapiclient.http.MediaInMemoryUpload(string_file.getvalue(), - mimetype=u'text/csv')) - file_url = result[u'webViewLink'] + mimetype='text/csv')) + file_url = result['webViewLink'] if GC_Values[GC_NO_BROWSER]: - msg_txt = u'Drive file uploaded to:\n %s' % file_url - msg_subj = u'%s - %s' % (GC_Values[GC_DOMAIN], list_type) + msg_txt = 'Drive file uploaded to:\n %s' % file_url + msg_subj = '%s - %s' % (GC_Values[GC_DOMAIN], list_type) send_email(msg_subj, msg_txt) - print msg_txt + print(msg_txt) else: webbrowser.open(file_url) -def flatten_json(structure, key=u'', path=u'', flattened=None, listLimit=None): +def flatten_json(structure, key='', path='', flattened=None, listLimit=None): if flattened is None: flattened = {} if not isinstance(structure, (dict, list)): - flattened[((path + u'.') if path else u'') + key] = structure + flattened[((path + '.') if path else '') + key] = structure elif isinstance(structure, list): for i, item in enumerate(structure): if listLimit and (i >= listLimit): break - flatten_json(item, u'{0}'.format(i), u'.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit) + flatten_json(item, '{0}'.format(i), '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit) else: - for new_key, value in structure.items(): - if new_key in [u'kind', u'etag', u'@type']: + for new_key, value in list(structure.items()): + if new_key in ['kind', 'etag', '@type']: continue if value == NEVER_TIME: - value = u'Never' - flatten_json(value, new_key, u'.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit) + value = 'Never' + flatten_json(value, new_key, '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit) return flattened USER_ARGUMENT_TO_PROPERTY_MAP = { - u'address': [u'addresses',], - u'addresses': [u'addresses',], - u'admin': [u'isAdmin', u'isDelegatedAdmin',], - u'agreed2terms': [u'agreedToTerms',], - u'agreedtoterms': [u'agreedToTerms',], - u'aliases': [u'aliases', u'nonEditableAliases',], - u'changepassword': [u'changePasswordAtNextLogin',], - u'changepasswordatnextlogin': [u'changePasswordAtNextLogin',], - u'creationtime': [u'creationTime',], - u'deletiontime': [u'deletionTime',], - u'email': [u'emails',], - u'emails': [u'emails',], - u'externalid': [u'externalIds',], - u'externalids': [u'externalIds',], - u'familyname': [u'name.familyName',], - u'firstname': [u'name.givenName',], - u'fullname': [u'name.fullName',], - u'gal': [u'includeInGlobalAddressList',], - u'gender': [u'gender.type', u'gender.customGender', u'gender.addressMeAs',], - u'givenname': [u'name.givenName',], - u'id': [u'id',], - u'im': [u'ims',], - u'ims': [u'ims',], - u'includeinglobaladdresslist': [u'includeInGlobalAddressList',], - u'ipwhitelisted': [u'ipWhitelisted',], - u'isadmin': [u'isAdmin', u'isDelegatedAdmin',], - u'isdelegatedadmin': [u'isAdmin', u'isDelegatedAdmin',], - u'isenforcedin2sv': [u'isEnforcedIn2Sv',], - u'isenrolledin2sv': [u'isEnrolledIn2Sv',], - u'is2svenforced': [u'isEnforcedIn2Sv',], - u'is2svenrolled': [u'isEnrolledIn2Sv',], - u'ismailboxsetup': [u'isMailboxSetup',], - u'keyword': [u'keywords',], - u'keywords': [u'keywords',], - u'language': [u'languages',], - u'languages': [u'languages',], - u'lastlogintime': [u'lastLoginTime',], - u'lastname': [u'name.familyName',], - u'location': [u'locations',], - u'locations': [u'locations',], - u'name': [u'name.givenName', u'name.familyName', u'name.fullName',], - u'nicknames': [u'aliases', u'nonEditableAliases',], - u'noneditablealiases': [u'aliases', u'nonEditableAliases',], - u'note': [u'notes',], - u'notes': [u'notes',], - u'org': [u'orgUnitPath',], - u'organization': [u'organizations',], - u'organizations': [u'organizations',], - u'orgunitpath': [u'orgUnitPath',], - u'otheremail': [u'emails',], - u'otheremails': [u'emails',], - u'ou': [u'orgUnitPath',], - u'phone': [u'phones',], - u'phones': [u'phones',], - u'photo': [u'thumbnailPhotoUrl',], - u'photourl': [u'thumbnailPhotoUrl',], - u'posix': [u'posixAccounts',], - u'posixaccounts': [u'posixAccounts',], - u'primaryemail': [u'primaryEmail',], - u'relation': [u'relations',], - u'relations': [u'relations',], - u'ssh': [u'sshPublicKeys',], - u'sshkeys': [u'sshPublicKeys',], - u'sshpublickeys': [u'sshPublicKeys',], - u'suspended': [u'suspended', u'suspensionReason',], - u'thumbnailphotourl': [u'thumbnailPhotoUrl',], - u'username': [u'primaryEmail',], - u'website': [u'websites',], - u'websites': [u'websites',], + 'address': ['addresses',], + 'addresses': ['addresses',], + 'admin': ['isAdmin', 'isDelegatedAdmin',], + 'agreed2terms': ['agreedToTerms',], + 'agreedtoterms': ['agreedToTerms',], + 'aliases': ['aliases', 'nonEditableAliases',], + 'changepassword': ['changePasswordAtNextLogin',], + 'changepasswordatnextlogin': ['changePasswordAtNextLogin',], + 'creationtime': ['creationTime',], + 'deletiontime': ['deletionTime',], + 'email': ['emails',], + 'emails': ['emails',], + 'externalid': ['externalIds',], + 'externalids': ['externalIds',], + 'familyname': ['name.familyName',], + 'firstname': ['name.givenName',], + 'fullname': ['name.fullName',], + 'gal': ['includeInGlobalAddressList',], + 'gender': ['gender.type', 'gender.customGender', 'gender.addressMeAs',], + 'givenname': ['name.givenName',], + 'id': ['id',], + 'im': ['ims',], + 'ims': ['ims',], + 'includeinglobaladdresslist': ['includeInGlobalAddressList',], + 'ipwhitelisted': ['ipWhitelisted',], + 'isadmin': ['isAdmin', 'isDelegatedAdmin',], + 'isdelegatedadmin': ['isAdmin', 'isDelegatedAdmin',], + 'isenforcedin2sv': ['isEnforcedIn2Sv',], + 'isenrolledin2sv': ['isEnrolledIn2Sv',], + 'is2svenforced': ['isEnforcedIn2Sv',], + 'is2svenrolled': ['isEnrolledIn2Sv',], + 'ismailboxsetup': ['isMailboxSetup',], + 'keyword': ['keywords',], + 'keywords': ['keywords',], + 'language': ['languages',], + 'languages': ['languages',], + 'lastlogintime': ['lastLoginTime',], + 'lastname': ['name.familyName',], + 'location': ['locations',], + 'locations': ['locations',], + 'name': ['name.givenName', 'name.familyName', 'name.fullName',], + 'nicknames': ['aliases', 'nonEditableAliases',], + 'noneditablealiases': ['aliases', 'nonEditableAliases',], + 'note': ['notes',], + 'notes': ['notes',], + 'org': ['orgUnitPath',], + 'organization': ['organizations',], + 'organizations': ['organizations',], + 'orgunitpath': ['orgUnitPath',], + 'otheremail': ['emails',], + 'otheremails': ['emails',], + 'ou': ['orgUnitPath',], + 'phone': ['phones',], + 'phones': ['phones',], + 'photo': ['thumbnailPhotoUrl',], + 'photourl': ['thumbnailPhotoUrl',], + 'posix': ['posixAccounts',], + 'posixaccounts': ['posixAccounts',], + 'primaryemail': ['primaryEmail',], + 'relation': ['relations',], + 'relations': ['relations',], + 'ssh': ['sshPublicKeys',], + 'sshkeys': ['sshPublicKeys',], + 'sshpublickeys': ['sshPublicKeys',], + 'suspended': ['suspended', 'suspensionReason',], + 'thumbnailphotourl': ['thumbnailPhotoUrl',], + 'username': ['primaryEmail',], + 'website': ['websites',], + 'websites': ['websites',], } def doPrintUsers(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False fieldsList = [] fieldsTitles = {} titles = [] csvRows = [] - addFieldToCSVfile(u'primaryemail', USER_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) + addFieldToCSVfile('primaryemail', USER_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) customer = GC_Values[GC_CUSTOMER_ID] domain = None queries = [None] - projection = u'basic' + projection = 'basic' customFieldMask = None sortHeaders = getGroupFeed = getLicenseFeed = email_parts = False viewType = deleted_only = orderBy = sortOrder = None - groupDelimiter = u' ' - licenseDelimiter = u',' + groupDelimiter = ' ' + licenseDelimiter = ',' i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') + myarg = sys.argv[i].lower().replace('_', '') if myarg in PROJECTION_CHOICES_MAP: projection = myarg sortHeaders = True fieldsList = [] i += 1 - elif myarg == u'allfields': - projection = u'basic' + elif myarg == 'allfields': + projection = 'basic' sortHeaders = True fieldsList = [] i += 1 - elif myarg == u'delimiter': + elif myarg == 'delimiter': groupDelimiter = licenseDelimiter = sys.argv[i+1] i += 2 - elif myarg == u'sortheaders': + elif myarg == 'sortheaders': sortHeaders = True i += 1 - elif myarg in [u'custom', u'schemas']: - fieldsList.append(u'customSchemas') - if sys.argv[i+1].lower() == u'all': - projection = u'full' + elif myarg in ['custom', 'schemas']: + fieldsList.append('customSchemas') + if sys.argv[i+1].lower() == 'all': + projection = 'full' else: - projection = u'custom' + projection = 'custom' customFieldMask = sys.argv[i+1] i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'deletedonly', u'onlydeleted']: + elif myarg in ['deletedonly', 'onlydeleted']: deleted_only = True i += 1 - elif myarg == u'orderby': + elif myarg == 'orderby': orderBy = sys.argv[i+1] - if orderBy.lower() not in [u'email', u'familyname', u'givenname', u'firstname', u'lastname']: + if orderBy.lower() not in ['email', 'familyname', 'givenname', 'firstname', 'lastname']: systemErrorExit(2, 'orderby must be one of email, familyName, givenName; got %s' % orderBy) - elif orderBy.lower() in [u'familyname', u'lastname']: - orderBy = u'familyName' - elif orderBy.lower() in [u'givenname', u'firstname']: - orderBy = u'givenName' + elif orderBy.lower() in ['familyname', 'lastname']: + orderBy = 'familyName' + elif orderBy.lower() in ['givenname', 'firstname']: + orderBy = 'givenName' i += 2 - elif myarg == u'userview': - viewType = u'domain_public' + elif myarg == 'userview': + viewType = 'domain_public' i += 1 elif myarg in SORTORDER_CHOICES_MAP: sortOrder = SORTORDER_CHOICES_MAP[myarg] i += 1 - elif myarg == u'domain': + elif myarg == 'domain': domain = sys.argv[i+1] customer = None i += 2 - elif myarg in [u'query', u'queries']: + elif myarg in ['query', 'queries']: queries = getQueries(myarg, sys.argv[i+1]) i += 2 elif myarg in USER_ARGUMENT_TO_PROPERTY_MAP: if not fieldsList: - fieldsList = [u'primaryEmail',] + fieldsList = ['primaryEmail',] addFieldToCSVfile(myarg, USER_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) i += 1 - elif myarg == u'fields': + elif myarg == 'fields': if not fieldsList: - fieldsList = [u'primaryEmail',] + fieldsList = ['primaryEmail',] fieldNameList = sys.argv[i+1] - for field in fieldNameList.lower().replace(u',', u' ').split(): + for field in fieldNameList.lower().replace(',', ' ').split(): if field in USER_ARGUMENT_TO_PROPERTY_MAP: addFieldToCSVfile(field, USER_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) else: systemErrorExit(2, '%s is not a valid argument for "gam print users fields"' % field) i += 2 - elif myarg == u'groups': + elif myarg == 'groups': getGroupFeed = True i += 1 - elif myarg in [u'license', u'licenses', u'licence', u'licences']: + elif myarg in ['license', 'licenses', 'licence', 'licences']: getLicenseFeed = True i += 1 - elif myarg in [u'emailpart', u'emailparts', u'username']: + elif myarg in ['emailpart', 'emailparts', 'username']: email_parts = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print users"' % sys.argv[i]) if fieldsList: - fields = u'nextPageToken,users(%s)' % u','.join(set(fieldsList)).replace(u'.', u'/') + fields = 'nextPageToken,users(%s)' % ','.join(set(fieldsList)).replace('.', '/') else: fields = None for query in queries: - printGettingAllItems(u'Users', query) - page_message = u'Got %%total_items%% Users: %%first_item%% - %%last_item%%\n' - all_users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, - message_attribute=u'primaryEmail', customer=customer, domain=domain, fields=fields, + printGettingAllItems('Users', query) + page_message = 'Got %%total_items%% Users: %%first_item%% - %%last_item%%\n' + all_users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + message_attribute='primaryEmail', customer=customer, domain=domain, fields=fields, showDeleted=deleted_only, orderBy=orderBy, sortOrder=sortOrder, viewType=viewType, query=query, projection=projection, customFieldMask=customFieldMask, maxResults=GC_Values[GC_USER_MAX_RESULTS]) for user in all_users: - if email_parts and (u'primaryEmail' in user): - user_email = user[u'primaryEmail'] - if user_email.find(u'@') != -1: - user[u'primaryEmailLocal'], user[u'primaryEmailDomain'] = splitEmailAddress(user_email) + if email_parts and ('primaryEmail' in user): + user_email = user['primaryEmail'] + if user_email.find('@') != -1: + user['primaryEmailLocal'], user['primaryEmailDomain'] = splitEmailAddress(user_email) addRowTitlesToCSVfile(flatten_json(user), csvRows, titles) if sortHeaders: - sortCSVTitles([u'primaryEmail',], titles) + sortCSVTitles(['primaryEmail',], titles) if getGroupFeed: total_users = len(csvRows) user_count = 1 - titles.append(u'Groups') + titles.append('Groups') for user in csvRows: - user_email = user[u'primaryEmail'] - sys.stderr.write(u"Getting Group Membership for %s (%s/%s)\r\n" % (user_email, user_count, total_users)) - groups = callGAPIpages(cd.groups(), u'list', u'groups', userKey=user_email) - user[u'Groups'] = groupDelimiter.join([groupname[u'email'] for groupname in groups]) + user_email = user['primaryEmail'] + sys.stderr.write("Getting Group Membership for %s (%s/%s)\r\n" % (user_email, user_count, total_users)) + groups = callGAPIpages(cd.groups(), 'list', 'groups', userKey=user_email) + user['Groups'] = groupDelimiter.join([groupname['email'] for groupname in groups]) user_count += 1 if getLicenseFeed: - titles.append(u'Licenses') - licenses = doPrintLicenses(returnFields=u'userId,skuId') + titles.append('Licenses') + licenses = doPrintLicenses(returnFields='userId,skuId') if licenses: for user in csvRows: - u_licenses = licenses.get(user[u'primaryEmail'].lower()) + u_licenses = licenses.get(user['primaryEmail'].lower()) if u_licenses: - user[u'Licenses'] = licenseDelimiter.join([_skuIdToDisplayName(skuId) for skuId in u_licenses]) - writeCSVfile(csvRows, titles, u'Users', todrive) + user['Licenses'] = licenseDelimiter.join([_skuIdToDisplayName(skuId) for skuId in u_licenses]) + writeCSVfile(csvRows, titles, 'Users', todrive) def doPrintShowAlerts(): - _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth(u'email')) - alerts = callGAPIpages(ac.alerts(), u'list', u'alerts') + _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth('email')) + alerts = callGAPIpages(ac.alerts(), 'list', 'alerts') titles = [] csv_rows = [] for alert in alerts: @@ -10976,129 +10986,129 @@ def doPrintShowAlerts(): if field not in titles: titles.append(field) csv_rows.append(aj) - writeCSVfile(csv_rows, titles, u'Alerts', False) + writeCSVfile(csv_rows, titles, 'Alerts', False) def doPrintShowAlertFeedback(): - _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth(u'email')) - feedback = callGAPIpages(ac.alerts().feedback(), u'list', u'feedback', alertId=u'-') + _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth('email')) + feedback = callGAPIpages(ac.alerts().feedback(), 'list', 'feedback', alertId='-') for feedbac in feedback: - print feedbac + print(feedbac) def _getValidAlertFeedbackTypes(ac): - return [aftype for aftype in ac._rootDesc[u'schemas'][u'AlertFeedback'][u'properties'][u'type'][u'enum'] if aftype != u'ALERT_FEEDBACK_TYPE_UNSPECIFIED'] + return [aftype for aftype in ac._rootDesc['schemas']['AlertFeedback']['properties']['type']['enum'] if aftype != 'ALERT_FEEDBACK_TYPE_UNSPECIFIED'] def doCreateAlertFeedback(): - _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth(u'email')) + _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth('email')) valid_types = _getValidAlertFeedbackTypes(ac) alertId = sys.argv[3] - body = {u'type': sys.argv[4].upper()} - if body[u'type'] not in valid_types: - systemErrorExit(2, '%s is not a valid feedback value, expected one of: %s' % (body[u'type'], u', '.join(valid_types))) - callGAPI(ac.alerts().feedback(), u'create', alertId=alertId, body=body) + body = {'type': sys.argv[4].upper()} + if body['type'] not in valid_types: + systemErrorExit(2, '%s is not a valid feedback value, expected one of: %s' % (body['type'], ', '.join(valid_types))) + callGAPI(ac.alerts().feedback(), 'create', alertId=alertId, body=body) def doDeleteOrUndeleteAlert(action): - _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth(u'email')) + _, ac = buildAlertCenterGAPIObject(_getValueFromOAuth('email')) alertId = sys.argv[3] kwargs = {} - if action == u'undelete': - kwargs[u'body'] = {} + if action == 'undelete': + kwargs['body'] = {} callGAPI(ac.alerts(), action, alertId=alertId, **kwargs) GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP = { - u'admincreated': [u'adminCreated', u'Admin_Created'], - u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'], - u'description': [u'description', u'Description'], - u'directmemberscount': [u'directMembersCount', u'DirectMembersCount'], - u'email': [u'email', u'Email'], - u'id': [u'id', u'ID'], - u'name': [u'name', u'Name'], + 'admincreated': ['adminCreated', 'Admin_Created'], + 'aliases': ['aliases', 'Aliases', 'nonEditableAliases', 'NonEditableAliases'], + 'description': ['description', 'Description'], + 'directmemberscount': ['directMembersCount', 'DirectMembersCount'], + 'email': ['email', 'Email'], + 'id': ['id', 'ID'], + 'name': ['name', 'Name'], } GROUP_ATTRIBUTES_ARGUMENT_TO_PROPERTY_MAP = { - u'allowexternalmembers': u'allowExternalMembers', - u'allowwebposting': u'allowWebPosting', - u'archiveonly': u'archiveOnly', - u'customfootertext': u'customFooterText', - u'customreplyto': u'customReplyTo', - u'defaultmessagedenynotificationtext': u'defaultMessageDenyNotificationText', - u'gal': u'includeInGlobalAddressList', - u'includecustomfooter': u'includeCustomFooter', - u'includeinglobaladdresslist': u'includeInGlobalAddressList', - u'isarchived': u'isArchived', - u'memberscanpostasthegroup': u'membersCanPostAsTheGroup', - u'messagemoderationlevel': u'messageModerationLevel', - u'primarylanguage': u'primaryLanguage', - u'replyto': u'replyTo', - u'sendmessagedenynotification': u'sendMessageDenyNotification', - u'showingroupdirectory': u'showInGroupDirectory', - u'spammoderationlevel': u'spamModerationLevel', - u'whocanadd': u'whoCanAdd', - u'whocanassigntopics': u'whoCanAssignTopics', - u'whocancontactowner': u'whoCanContactOwner', - u'whocanenterfreeformtags': u'whoCanEnterFreeFormTags', - u'whocaninvite': u'whoCanInvite', - u'whocanjoin': u'whoCanJoin', - u'whocanleavegroup': u'whoCanLeaveGroup', - u'whocanmarkduplicate': u'whoCanMarkDuplicate', - u'whocanmarkfavoritereplyonanytopic': u'whoCanMarkFavoriteReplyOnAnyTopic', - u'whocanmarknoresponseneeded': u'whoCanMarkNoResponseNeeded', - u'whocanmodifytagsandcategories': u'whoCanModifyTagsAndCategories', - u'whocanpostmessage': u'whoCanPostMessage', - u'whocantaketopics': u'whoCanTakeTopics', - u'whocanunassigntopic': u'whoCanUnassignTopic', - u'whocanunmarkfavoritereplyonanytopic': u'whoCanUnmarkFavoriteReplyOnAnyTopic', - u'whocanviewgroup': u'whoCanViewGroup', - u'whocanviewmembership': u'whoCanViewMembership', + 'allowexternalmembers': 'allowExternalMembers', + 'allowwebposting': 'allowWebPosting', + 'archiveonly': 'archiveOnly', + 'customfootertext': 'customFooterText', + 'customreplyto': 'customReplyTo', + 'defaultmessagedenynotificationtext': 'defaultMessageDenyNotificationText', + 'gal': 'includeInGlobalAddressList', + 'includecustomfooter': 'includeCustomFooter', + 'includeinglobaladdresslist': 'includeInGlobalAddressList', + 'isarchived': 'isArchived', + 'memberscanpostasthegroup': 'membersCanPostAsTheGroup', + 'messagemoderationlevel': 'messageModerationLevel', + 'primarylanguage': 'primaryLanguage', + 'replyto': 'replyTo', + 'sendmessagedenynotification': 'sendMessageDenyNotification', + 'showingroupdirectory': 'showInGroupDirectory', + 'spammoderationlevel': 'spamModerationLevel', + 'whocanadd': 'whoCanAdd', + 'whocanassigntopics': 'whoCanAssignTopics', + 'whocancontactowner': 'whoCanContactOwner', + 'whocanenterfreeformtags': 'whoCanEnterFreeFormTags', + 'whocaninvite': 'whoCanInvite', + 'whocanjoin': 'whoCanJoin', + 'whocanleavegroup': 'whoCanLeaveGroup', + 'whocanmarkduplicate': 'whoCanMarkDuplicate', + 'whocanmarkfavoritereplyonanytopic': 'whoCanMarkFavoriteReplyOnAnyTopic', + 'whocanmarknoresponseneeded': 'whoCanMarkNoResponseNeeded', + 'whocanmodifytagsandcategories': 'whoCanModifyTagsAndCategories', + 'whocanpostmessage': 'whoCanPostMessage', + 'whocantaketopics': 'whoCanTakeTopics', + 'whocanunassigntopic': 'whoCanUnassignTopic', + 'whocanunmarkfavoritereplyonanytopic': 'whoCanUnmarkFavoriteReplyOnAnyTopic', + 'whocanviewgroup': 'whoCanViewGroup', + 'whocanviewmembership': 'whoCanViewMembership', } def doPrintGroups(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') i = 3 members = membersCountOnly = managers = managersCountOnly = owners = ownersCountOnly = False customer = GC_Values[GC_CUSTOMER_ID] usedomain = usemember = usequery = None - aliasDelimiter = u' ' - memberDelimiter = u'\n' + aliasDelimiter = ' ' + memberDelimiter = '\n' todrive = False cdfieldsList = [] gsfieldsList = [] fieldsTitles = {} titles = [] csvRows = [] - addFieldTitleToCSVfile(u'email', GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP, cdfieldsList, fieldsTitles, titles) + addFieldTitleToCSVfile('email', GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP, cdfieldsList, fieldsTitles, titles) maxResults = None roles = [] getSettings = sortHeaders = False while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'domain': + elif myarg == 'domain': usedomain = sys.argv[i+1].lower() customer = None i += 2 - elif myarg == u'member': + elif myarg == 'member': usemember = normalizeEmailAddressOrUID(sys.argv[i+1]) customer = usequery = None i += 2 - elif myarg == u'query': + elif myarg == 'query': usequery = sys.argv[i+1] usemember = None i += 2 - elif myarg == u'maxresults': + elif myarg == 'maxresults': maxResults = getInteger(sys.argv[i+1], myarg, minVal=0) i += 2 - elif myarg == u'delimiter': + elif myarg == 'delimiter': aliasDelimiter = memberDelimiter = sys.argv[i+1] i += 2 elif myarg in GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP: addFieldTitleToCSVfile(myarg, GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP, cdfieldsList, fieldsTitles, titles) i += 1 - elif myarg == u'settings': + elif myarg == 'settings': getSettings = True i += 1 - elif myarg == u'allfields': + elif myarg == 'allfields': getSettings = sortHeaders = True cdfieldsList = [] gsfieldsList = [] @@ -11106,76 +11116,76 @@ def doPrintGroups(): for field in GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP: addFieldTitleToCSVfile(field, GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP, cdfieldsList, fieldsTitles, titles) i += 1 - elif myarg == u'sortheaders': + elif myarg == 'sortheaders': sortHeaders = True i += 1 - elif myarg == u'fields': + elif myarg == 'fields': fieldNameList = sys.argv[i+1] - for field in fieldNameList.lower().replace(u',', u' ').split(): + for field in fieldNameList.lower().replace(',', ' ').split(): if field in GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP: addFieldTitleToCSVfile(field, GROUP_ARGUMENT_TO_PROPERTY_TITLE_MAP, cdfieldsList, fieldsTitles, titles) elif field in GROUP_ATTRIBUTES_ARGUMENT_TO_PROPERTY_MAP: addFieldToCSVfile(field, {field: [GROUP_ATTRIBUTES_ARGUMENT_TO_PROPERTY_MAP[field]]}, gsfieldsList, fieldsTitles, titles) - elif field == u'collaborative': + elif field == 'collaborative': for attrName in COLLABORATIVE_INBOX_ATTRIBUTES: addFieldToCSVfile(attrName, {attrName: [attrName]}, gsfieldsList, fieldsTitles, titles) else: systemErrorExit(2, '%s is not a valid argument for "gam print groups fields"' % field) i += 2 - elif myarg in [u'members', u'memberscount']: + elif myarg in ['members', 'memberscount']: roles.append(ROLE_MEMBER) members = True - if myarg == u'memberscount': + if myarg == 'memberscount': membersCountOnly = True i += 1 - elif myarg in [u'owners', u'ownerscount']: + elif myarg in ['owners', 'ownerscount']: roles.append(ROLE_OWNER) owners = True - if myarg == u'ownerscount': + if myarg == 'ownerscount': ownersCountOnly = True i += 1 - elif myarg in [u'managers', u'managerscount']: + elif myarg in ['managers', 'managerscount']: roles.append(ROLE_MANAGER) managers = True - if myarg == u'managerscount': + if myarg == 'managerscount': managersCountOnly = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print groups"' % sys.argv[i]) - cdfields = u','.join(set(cdfieldsList)) + cdfields = ','.join(set(cdfieldsList)) if len(gsfieldsList) > 0: getSettings = True - gsfields = u','.join(set(gsfieldsList)) + gsfields = ','.join(set(gsfieldsList)) elif getSettings: gsfields = None if getSettings: - gs = buildGAPIObject(u'groupssettings') - roles = u','.join(sorted(set(roles))) + gs = buildGAPIObject('groupssettings') + roles = ','.join(sorted(set(roles))) if roles: if members: - addTitlesToCSVfile([u'MembersCount',], titles) + addTitlesToCSVfile(['MembersCount',], titles) if not membersCountOnly: - addTitlesToCSVfile([u'Members',], titles) + addTitlesToCSVfile(['Members',], titles) if managers: - addTitlesToCSVfile([u'ManagersCount',], titles) + addTitlesToCSVfile(['ManagersCount',], titles) if not managersCountOnly: - addTitlesToCSVfile([u'Managers',], titles) + addTitlesToCSVfile(['Managers',], titles) if owners: - addTitlesToCSVfile([u'OwnersCount',], titles) + addTitlesToCSVfile(['OwnersCount',], titles) if not ownersCountOnly: - addTitlesToCSVfile([u'Owners',], titles) - printGettingAllItems(u'Groups', None) - page_message = u'Got %%num_items%% Groups: %%first_item%% - %%last_item%%\n' - entityList = callGAPIpages(cd.groups(), u'list', u'groups', - page_message=page_message, message_attribute=u'email', + addTitlesToCSVfile(['Owners',], titles) + printGettingAllItems('Groups', None) + page_message = 'Got %%num_items%% Groups: %%first_item%% - %%last_item%%\n' + entityList = callGAPIpages(cd.groups(), 'list', 'groups', + page_message=page_message, message_attribute='email', customer=customer, domain=usedomain, userKey=usemember, query=usequery, - fields=u'nextPageToken,groups({0})'.format(cdfields), + fields='nextPageToken,groups({0})'.format(cdfields), maxResults=maxResults) i = 0 count = len(entityList) for groupEntity in entityList: i += 1 - groupEmail = groupEntity[u'email'] + groupEmail = groupEntity['email'] group = {} for field in cdfieldsList: if field in groupEntity: @@ -11184,11 +11194,11 @@ def doPrintGroups(): else: group[fieldsTitles[field]] = groupEntity[field] if roles: - sys.stderr.write(u' Getting %s for %s (%s/%s)\n' % (roles, groupEmail, i, count)) - page_message = u' Got %%num_items%% members: %%first_item%% - %%last_item%%\n' - validRoles, listRoles, listFields = _getRoleVerification(roles, u'nextPageToken,members(email,id,role)') - groupMembers = callGAPIpages(cd.members(), u'list', u'members', - page_message=page_message, message_attribute=u'email', + sys.stderr.write(' Getting %s for %s (%s/%s)\n' % (roles, groupEmail, i, count)) + page_message = ' Got %%num_items%% members: %%first_item%% - %%last_item%%\n' + validRoles, listRoles, listFields = _getRoleVerification(roles, 'nextPageToken,members(email,id,role)') + groupMembers = callGAPIpages(cd.members(), 'list', 'members', + page_message=page_message, message_attribute='email', soft_errors=True, groupKey=groupEmail, roles=listRoles, fields=listFields, maxResults=GC_Values[GC_MEMBER_MAX_RESULTS]) if members: @@ -11201,11 +11211,11 @@ def doPrintGroups(): ownersList = [] ownersCount = 0 for member in groupMembers: - member_email = member.get(u'email', member.get(u'id', None)) + member_email = member.get('email', member.get('id', None)) if not member_email: - sys.stderr.write(u' Not sure what to do with: %s' % member) + sys.stderr.write(' Not sure what to do with: %s' % member) continue - role = member.get(u'role', ROLE_MEMBER) + role = member.get('role', ROLE_MEMBER) if not validRoles or role in validRoles: if role == ROLE_MEMBER: if members: @@ -11227,104 +11237,104 @@ def doPrintGroups(): if not membersCountOnly: membersList.append(member_email) if members: - group[u'MembersCount'] = membersCount + group['MembersCount'] = membersCount if not membersCountOnly: - group[u'Members'] = memberDelimiter.join(membersList) + group['Members'] = memberDelimiter.join(membersList) if managers: - group[u'ManagersCount'] = managersCount + group['ManagersCount'] = managersCount if not managersCountOnly: - group[u'Managers'] = memberDelimiter.join(managersList) + group['Managers'] = memberDelimiter.join(managersList) if owners: - group[u'OwnersCount'] = ownersCount + group['OwnersCount'] = ownersCount if not ownersCountOnly: - group[u'Owners'] = memberDelimiter.join(ownersList) + group['Owners'] = memberDelimiter.join(ownersList) if getSettings and not GroupIsAbuseOrPostmaster(groupEmail): - sys.stderr.write(u" Retrieving Settings for group %s (%s/%s)...\r\n" % (groupEmail, i, count)) - settings = callGAPI(gs.groups(), u'get', + sys.stderr.write(" Retrieving Settings for group %s (%s/%s)...\r\n" % (groupEmail, i, count)) + settings = callGAPI(gs.groups(), 'get', soft_errors=True, - retry_reasons=[u'serviceLimit', u'invalid'], + retry_reasons=['serviceLimit', 'invalid'], groupUniqueId=groupEmail, fields=gsfields) if settings: for key in settings: - if key in [u'email', u'name', u'description', u'kind', u'etag']: + if key in ['email', 'name', 'description', 'kind', 'etag']: continue setting_value = settings[key] if setting_value is None: - setting_value = u'' + setting_value = '' if key not in titles: addTitleToCSVfile(key, titles) group[key] = setting_value else: - sys.stderr.write(u" Settings unavailable for group %s (%s/%s)...\r\n" % (groupEmail, i, count)) + sys.stderr.write(" Settings unavailable for group %s (%s/%s)...\r\n" % (groupEmail, i, count)) csvRows.append(group) if sortHeaders: - sortCSVTitles([u'Email',], titles) - writeCSVfile(csvRows, titles, u'Groups', todrive) + sortCSVTitles(['Email',], titles) + writeCSVfile(csvRows, titles, 'Groups', todrive) def doPrintOrgs(): - print_order = [u'orgUnitPath', u'orgUnitId', u'name', u'description', - u'parentOrgUnitPath', u'parentOrgUnitId', u'blockInheritance'] - cd = buildGAPIObject(u'directory') - listType = u'all' - orgUnitPath = u"/" + print_order = ['orgUnitPath', 'orgUnitId', 'name', 'description', + 'parentOrgUnitPath', 'parentOrgUnitId', 'blockInheritance'] + cd = buildGAPIObject('directory') + listType = 'all' + orgUnitPath = "/" todrive = False - fields = [u'orgUnitPath', u'name', u'orgUnitId', u'parentOrgUnitId'] + fields = ['orgUnitPath', 'name', 'orgUnitId', 'parentOrgUnitId'] titles = [] csvRows = [] parentOrgIds = [] retrievedOrgIds = [] i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'toplevelonly': - listType = u'children' + elif myarg == 'toplevelonly': + listType = 'children' i += 1 - elif myarg == u'fromparent': + elif myarg == 'fromparent': orgUnitPath = getOrgUnitItem(sys.argv[i+1]) i += 2 - elif myarg == u'allfields': + elif myarg == 'allfields': fields = None i += 1 - elif myarg == u'fields': - fields += sys.argv[i+1].split(u',') + elif myarg == 'fields': + fields += sys.argv[i+1].split(',') i += 2 else: systemErrorExit(2, '%s is not a valid argument for "gam print orgs"' % sys.argv[i]) - printGettingAllItems(u'Organizational Units', None) + printGettingAllItems('Organizational Units', None) if fields: - get_fields = u','.join(fields) - list_fields = u'organizationUnits(%s)' % get_fields + get_fields = ','.join(fields) + list_fields = 'organizationUnits(%s)' % get_fields else: list_fields = None get_fields = None - orgs = callGAPI(cd.orgunits(), u'list', + orgs = callGAPI(cd.orgunits(), 'list', customerId=GC_Values[GC_CUSTOMER_ID], type=listType, orgUnitPath=orgUnitPath, fields=list_fields) - if not u'organizationUnits' in orgs: + if not 'organizationUnits' in orgs: topLevelOrgId = getTopLevelOrgId(cd, orgUnitPath) if topLevelOrgId: parentOrgIds.append(topLevelOrgId) orgunits = [] else: - orgunits = orgs[u'organizationUnits'] + orgunits = orgs['organizationUnits'] for row in orgunits: - retrievedOrgIds.append(row[u'orgUnitId']) - if row[u'parentOrgUnitId'] not in parentOrgIds: - parentOrgIds.append(row[u'parentOrgUnitId']) + retrievedOrgIds.append(row['orgUnitId']) + if row['parentOrgUnitId'] not in parentOrgIds: + parentOrgIds.append(row['parentOrgUnitId']) missing_parents = set(parentOrgIds) - set(retrievedOrgIds) for missing_parent in missing_parents: try: - result = callGAPI(cd.orgunits(), u'get', throw_reasons=[u'required'], + result = callGAPI(cd.orgunits(), 'get', throw_reasons=['required'], customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=missing_parent, fields=get_fields) orgunits.append(result) except: pass for row in orgunits: orgEntity = {} - for key, value in row.items(): - if key in [u'kind', u'etag', u'etags']: + for key, value in list(row.items()): + if key in ['kind', 'etag', 'etags']: continue if key not in titles: titles.append(key) @@ -11335,36 +11345,36 @@ def doPrintOrgs(): print_order.append(title) titles = sorted(titles, key=print_order.index) # sort results similar to how they list in admin console - csvRows.sort(key=lambda x: x[u'orgUnitPath'].lower(), reverse=False) - writeCSVfile(csvRows, titles, u'Orgs', todrive) + csvRows.sort(key=lambda x: x['orgUnitPath'].lower(), reverse=False) + writeCSVfile(csvRows, titles, 'Orgs', todrive) def doPrintAliases(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False - titles = [u'Alias', u'Target', u'TargetType'] + titles = ['Alias', 'Target', 'TargetType'] csvRows = [] - userFields = [u'primaryEmail', u'aliases'] - groupFields = [u'email', u'aliases'] + userFields = ['primaryEmail', 'aliases'] + groupFields = ['email', 'aliases'] doGroups = doUsers = True queries = [None] i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'shownoneditable': - titles.insert(1, u'NonEditableAlias') - userFields.append(u'nonEditableAliases') - groupFields.append(u'nonEditableAliases') + elif myarg == 'shownoneditable': + titles.insert(1, 'NonEditableAlias') + userFields.append('nonEditableAliases') + groupFields.append('nonEditableAliases') i += 1 - elif myarg == u'nogroups': + elif myarg == 'nogroups': doGroups = False i += 1 - elif myarg == u'nousers': + elif myarg == 'nousers': doUsers = False i += 1 - elif myarg in [u'query', u'queries']: + elif myarg in ['query', 'queries']: queries = getQueries(myarg, sys.argv[i+1]) doGroups = False doUsers = True @@ -11373,150 +11383,150 @@ def doPrintAliases(): systemErrorExit(2, '%s is not a valid argument for "gam print aliases"' % sys.argv[i]) if doUsers: for query in queries: - printGettingAllItems(u'User Aliases', query) - page_message = u'Got %%num_items%% Users %%first_item%% - %%last_item%%\n' - all_users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, - message_attribute=u'primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=query, - fields=u'nextPageToken,users({0})'.format(u','.join(userFields)), maxResults=GC_Values[GC_USER_MAX_RESULTS]) + printGettingAllItems('User Aliases', query) + page_message = 'Got %%num_items%% Users %%first_item%% - %%last_item%%\n' + all_users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + message_attribute='primaryEmail', customer=GC_Values[GC_CUSTOMER_ID], query=query, + fields='nextPageToken,users({0})'.format(','.join(userFields)), maxResults=GC_Values[GC_USER_MAX_RESULTS]) for user in all_users: - for alias in user.get(u'aliases', []): - csvRows.append({u'Alias': alias, u'Target': user[u'primaryEmail'], u'TargetType': u'User'}) - for alias in user.get(u'nonEditableAliases', []): - csvRows.append({u'NonEditableAlias': alias, u'Target': user[u'primaryEmail'], u'TargetType': u'User'}) + for alias in user.get('aliases', []): + csvRows.append({'Alias': alias, 'Target': user['primaryEmail'], 'TargetType': 'User'}) + for alias in user.get('nonEditableAliases', []): + csvRows.append({'NonEditableAlias': alias, 'Target': user['primaryEmail'], 'TargetType': 'User'}) if doGroups: - printGettingAllItems(u'Group Aliases', None) - page_message = u'Got %%num_items%% Groups %%first_item%% - %%last_item%%\n' - all_groups = callGAPIpages(cd.groups(), u'list', u'groups', page_message=page_message, - message_attribute=u'email', customer=GC_Values[GC_CUSTOMER_ID], - fields=u'nextPageToken,groups({0})'.format(u','.join(groupFields))) + printGettingAllItems('Group Aliases', None) + page_message = 'Got %%num_items%% Groups %%first_item%% - %%last_item%%\n' + all_groups = callGAPIpages(cd.groups(), 'list', 'groups', page_message=page_message, + message_attribute='email', customer=GC_Values[GC_CUSTOMER_ID], + fields='nextPageToken,groups({0})'.format(','.join(groupFields))) for group in all_groups: - for alias in group.get(u'aliases', []): - csvRows.append({u'Alias': alias, u'Target': group[u'email'], u'TargetType': u'Group'}) - for alias in group.get(u'nonEditableAliases', []): - csvRows.append({u'NonEditableAlias': alias, u'Target': group[u'email'], u'TargetType': u'Group'}) - writeCSVfile(csvRows, titles, u'Aliases', todrive) + for alias in group.get('aliases', []): + csvRows.append({'Alias': alias, 'Target': group['email'], 'TargetType': 'Group'}) + for alias in group.get('nonEditableAliases', []): + csvRows.append({'NonEditableAlias': alias, 'Target': group['email'], 'TargetType': 'Group'}) + writeCSVfile(csvRows, titles, 'Aliases', todrive) def doPrintGroupMembers(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False membernames = False customer = GC_Values[GC_CUSTOMER_ID] checkSuspended = usedomain = usemember = usequery = None roles = [] - fields = u'nextPageToken,members(email,id,role,status,type)' - titles = [u'group'] + fields = 'nextPageToken,members(email,id,role,status,type)' + titles = ['group'] csvRows = [] groups_to_get = [] i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'domain': + elif myarg == 'domain': usedomain = sys.argv[i+1].lower() customer = None i += 2 - elif myarg == u'member': + elif myarg == 'member': usemember = normalizeEmailAddressOrUID(sys.argv[i+1]) customer = usequery = None i += 2 - elif myarg == u'query': + elif myarg == 'query': usequery = sys.argv[i+1] usemember = None i += 2 - elif myarg == u'fields': - memberFieldsList = sys.argv[i+1].replace(u',', u' ').lower().split() - fields = u'nextPageToken,members(%s)' % (','.join(memberFieldsList)) + elif myarg == 'fields': + memberFieldsList = sys.argv[i+1].replace(',', ' ').lower().split() + fields = 'nextPageToken,members(%s)' % (','.join(memberFieldsList)) i += 2 - elif myarg == u'membernames': + elif myarg == 'membernames': membernames = True - titles.append(u'name') + titles.append('name') i += 1 - elif myarg in [u'role', u'roles']: - for role in sys.argv[i+1].lower().replace(u',', u' ').split(): + elif myarg in ['role', 'roles']: + for role in sys.argv[i+1].lower().replace(',', ' ').split(): if role in GROUP_ROLES_MAP: roles.append(GROUP_ROLES_MAP[role]) else: systemErrorExit(2, '%s is not a valid role for "gam print group-members %s"' % (role, myarg)) i += 2 - elif myarg in [u'group', u'groupns', u'groupsusp']: + elif myarg in ['group', 'groupns', 'groupsusp']: group_email = normalizeEmailAddressOrUID(sys.argv[i+1]) - groups_to_get = [{u'email': group_email}] - if myarg == u'groupns': + groups_to_get = [{'email': group_email}] + if myarg == 'groupns': checkSuspended = False - elif myarg == u'groupsusp': + elif myarg == 'groupsusp': checkSuspended = True i += 2 - elif myarg in [u'suspended', u'notsuspended']: - checkSuspended = myarg == u'suspended' + elif myarg in ['suspended', 'notsuspended']: + checkSuspended = myarg == 'suspended' i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print group-members"' % sys.argv[i]) if not groups_to_get: - groups_to_get = callGAPIpages(cd.groups(), u'list', u'groups', message_attribute=u'email', + groups_to_get = callGAPIpages(cd.groups(), 'list', 'groups', message_attribute='email', customer=customer, domain=usedomain, userKey=usemember, query=usequery, - fields=u'nextPageToken,groups(email)') + fields='nextPageToken,groups(email)') i = 0 count = len(groups_to_get) for group in groups_to_get: i += 1 - group_email = group[u'email'] - sys.stderr.write(u'Getting members for %s (%s/%s)\n' % (group_email, i, count)) - validRoles, listRoles, listFields = _getRoleVerification(u','.join(roles), fields) - group_members = callGAPIpages(cd.members(), u'list', u'members', + group_email = group['email'] + sys.stderr.write('Getting members for %s (%s/%s)\n' % (group_email, i, count)) + validRoles, listRoles, listFields = _getRoleVerification(','.join(roles), fields) + group_members = callGAPIpages(cd.members(), 'list', 'members', soft_errors=True, groupKey=group_email, roles=listRoles, fields=listFields, maxResults=GC_Values[GC_MEMBER_MAX_RESULTS]) for member in group_members: - if ((validRoles and member.get(u'role', ROLE_MEMBER) not in validRoles) or - (checkSuspended is not None and ((not checkSuspended and member[u'status'] == u'SUSPENDED') or (checkSuspended and member[u'status'] != u'SUSPENDED')))): + if ((validRoles and member.get('role', ROLE_MEMBER) not in validRoles) or + (checkSuspended is not None and ((not checkSuspended and member['status'] == 'SUSPENDED') or (checkSuspended and member['status'] != 'SUSPENDED')))): continue for title in member: if title not in titles: titles.append(title) - member[u'group'] = group_email - if membernames and u'type' in member and u'id' in member: - if member[u'type'] == u'USER': + member['group'] = group_email + if membernames and 'type' in member and 'id' in member: + if member['type'] == 'USER': try: - mbinfo = callGAPI(cd.users(), u'get', + mbinfo = callGAPI(cd.users(), 'get', throw_reasons=[GAPI_USER_NOT_FOUND, GAPI_NOT_FOUND, GAPI_FORBIDDEN], - userKey=member[u'id'], fields=u'name') - memberName = mbinfo[u'name'][u'fullName'] + userKey=member['id'], fields='name') + memberName = mbinfo['name']['fullName'] except (GAPI_userNotFound, GAPI_notFound, GAPI_forbidden): - memberName = u'Unknown' - elif member[u'type'] == u'GROUP': + memberName = 'Unknown' + elif member['type'] == 'GROUP': try: - mbinfo = callGAPI(cd.groups(), u'get', + mbinfo = callGAPI(cd.groups(), 'get', throw_reasons=[GAPI_NOT_FOUND, GAPI_FORBIDDEN], - groupKey=member[u'id'], fields=u'name') - memberName = mbinfo[u'name'] + groupKey=member['id'], fields='name') + memberName = mbinfo['name'] except (GAPI_notFound, GAPI_forbidden): - memberName = u'Unknown' - elif member[u'type'] == u'CUSTOMER': + memberName = 'Unknown' + elif member['type'] == 'CUSTOMER': try: - mbinfo = callGAPI(cd.customers(), u'get', + mbinfo = callGAPI(cd.customers(), 'get', throw_reasons=[GAPI_BAD_REQUEST, GAPI_RESOURCE_NOT_FOUND, GAPI_FORBIDDEN], - customerKey=member[u'id'], fields=u'customerDomain') - memberName = mbinfo[u'customerDomain'] + customerKey=member['id'], fields='customerDomain') + memberName = mbinfo['customerDomain'] except (GAPI_badRequest, GAPI_resourceNotFound, GAPI_forbidden): - memberName = u'Unknown' + memberName = 'Unknown' else: - memberName = u'Unknown' - member[u'name'] = memberName + memberName = 'Unknown' + member['name'] = memberName csvRows.append(member) - writeCSVfile(csvRows, titles, u'Group Members', todrive) + writeCSVfile(csvRows, titles, 'Group Members', todrive) def doPrintVaultMatters(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') todrive = False csvRows = [] - initialTitles = [u'matterId', u'name', u'description', u'state'] + initialTitles = ['matterId', 'name', 'description', 'state'] titles = initialTitles[:] - view = u'FULL' + view = 'FULL' i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 elif myarg in PROJECTION_CHOICES_MAP: @@ -11524,129 +11534,129 @@ def doPrintVaultMatters(): i += 1 else: systemErrorExit(3, '%s is not a valid argument to "gam print matters"' % myarg) - printGettingAllItems(u'Vault Matters', None) - page_message = u'Got %%num_items%% Vault Matters...\n' - matters = callGAPIpages(v.matters(), u'list', u'matters', page_message=page_message, view=view) + printGettingAllItems('Vault Matters', None) + page_message = 'Got %%num_items%% Vault Matters...\n' + matters = callGAPIpages(v.matters(), 'list', 'matters', page_message=page_message, view=view) for matter in matters: addRowTitlesToCSVfile(flatten_json(matter), csvRows, titles) sortCSVTitles(initialTitles, titles) - writeCSVfile(csvRows, titles, u'Vault Matters', todrive) + writeCSVfile(csvRows, titles, 'Vault Matters', todrive) def doPrintVaultExports(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') todrive = False csvRows = [] - initialTitles = [u'matterId', u'id', u'name', u'createTime', u'status'] + initialTitles = ['matterId', 'id', 'name', 'createTime', 'status'] titles = initialTitles[:] matters = [] matterIds = [] i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'matter', u'matters']: - matters = sys.argv[i+1].split(u',') + elif myarg in ['matter', 'matters']: + matters = sys.argv[i+1].split(',') i += 2 else: systemErrorExit(3, '%s is not a valid a valid argument to "gam print exports"' % myarg) if not matters: - matters_results = callGAPIpages(v.matters(), u'list', u'matters', view=u'BASIC', fields=u'matters(matterId,state),nextPageToken') + matters_results = callGAPIpages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,state),nextPageToken') for matter in matters_results: - if matter[u'state'] != u'OPEN': - print u'ignoring matter %s in state %s' % (matter[u'matterId'], matter[u'state']) + if matter['state'] != 'OPEN': + print('ignoring matter %s in state %s' % (matter['matterId'], matter['state'])) continue - matterIds.append(matter[u'matterId']) + matterIds.append(matter['matterId']) else: for matter in matters: matterIds.append(getMatterItem(v, matter)) for matterId in matterIds: - sys.stderr.write(u'Retrieving exports for matter %s\n' % matterId) - exports = callGAPIpages(v.matters().exports(), u'list', u'exports', matterId=matterId) + sys.stderr.write('Retrieving exports for matter %s\n' % matterId) + exports = callGAPIpages(v.matters().exports(), 'list', 'exports', matterId=matterId) for export in exports: - addRowTitlesToCSVfile(flatten_json(export, flattened={u'matterId': matterId}), csvRows, titles) + addRowTitlesToCSVfile(flatten_json(export, flattened={'matterId': matterId}), csvRows, titles) sortCSVTitles(initialTitles, titles) - writeCSVfile(csvRows, titles, u'Vault Exports', todrive) + writeCSVfile(csvRows, titles, 'Vault Exports', todrive) def doPrintVaultHolds(): - v = buildGAPIObject(u'vault') + v = buildGAPIObject('vault') todrive = False csvRows = [] - initialTitles = [u'matterId', u'holdId', u'name', u'corpus', u'updateTime'] + initialTitles = ['matterId', 'holdId', 'name', 'corpus', 'updateTime'] titles = initialTitles[:] matters = [] matterIds = [] i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'matter', u'matters']: - matters = sys.argv[i+1].split(u',') + elif myarg in ['matter', 'matters']: + matters = sys.argv[i+1].split(',') i += 2 else: systemErrorExit(3, '%s is not a valid a valid argument to "gam print holds"' % myarg) if not matters: - matters_results = callGAPIpages(v.matters(), u'list', u'matters', view=u'BASIC', fields=u'matters(matterId,state),nextPageToken') + matters_results = callGAPIpages(v.matters(), 'list', 'matters', view='BASIC', fields='matters(matterId,state),nextPageToken') for matter in matters_results: - if matter[u'state'] != u'OPEN': - print u'ignoring matter %s in state %s' % (matter[u'matterId'], matter[u'state']) + if matter['state'] != 'OPEN': + print('ignoring matter %s in state %s' % (matter['matterId'], matter['state'])) continue - matterIds.append(matter[u'matterId']) + matterIds.append(matter['matterId']) else: for matter in matters: matterIds.append(getMatterItem(v, matter)) for matterId in matterIds: - sys.stderr.write(u'Retrieving holds for matter %s\n' % matterId) - holds = callGAPIpages(v.matters().holds(), u'list', u'holds', matterId=matterId) + sys.stderr.write('Retrieving holds for matter %s\n' % matterId) + holds = callGAPIpages(v.matters().holds(), 'list', 'holds', matterId=matterId) for hold in holds: - addRowTitlesToCSVfile(flatten_json(hold, flattened={u'matterId': matterId}), csvRows, titles) + addRowTitlesToCSVfile(flatten_json(hold, flattened={'matterId': matterId}), csvRows, titles) sortCSVTitles(initialTitles, titles) - writeCSVfile(csvRows, titles, u'Vault Holds', todrive) + writeCSVfile(csvRows, titles, 'Vault Holds', todrive) def doPrintMobileDevices(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False titles = [] csvRows = [] fields = None projection = orderBy = sortOrder = None queries = [None] - delimiter = u' ' + delimiter = ' ' listLimit = 1 appsLimit = -1 i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg == u'todrive': + myarg = sys.argv[i].lower().replace('_', '') + if myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'query', u'queries']: + elif myarg in ['query', 'queries']: queries = getQueries(myarg, sys.argv[i+1]) i += 2 - elif myarg == u'delimiter': + elif myarg == 'delimiter': delimiter = sys.argv[i+1] i += 2 - elif myarg == u'listlimit': + elif myarg == 'listlimit': listLimit = getInteger(sys.argv[i+1], myarg, minVal=-1) i += 2 - elif myarg == u'appslimit': + elif myarg == 'appslimit': appsLimit = getInteger(sys.argv[i+1], myarg, minVal=-1) i += 2 - elif myarg == u'fields': - fields = u'nextPageToken,mobiledevices(%s)' % sys.argv[i+1] + elif myarg == 'fields': + fields = 'nextPageToken,mobiledevices(%s)' % sys.argv[i+1] i += 2 - elif myarg == u'orderby': + elif myarg == 'orderby': orderBy = sys.argv[i+1].lower() - allowed_values = [u'deviceid', u'email', u'lastsync', u'model', u'name', u'os', u'status', u'type'] + allowed_values = ['deviceid', 'email', 'lastsync', 'model', 'name', 'os', 'status', 'type'] if orderBy.lower() not in allowed_values: - systemErrorExit(2, 'orderBy must be one of %s; got %s' % (u', '.join(allowed_values), orderBy)) - elif orderBy == u'lastsync': - orderBy = u'lastSync' - elif orderBy == u'deviceid': - orderBy = u'deviceId' + systemErrorExit(2, 'orderBy must be one of %s; got %s' % (', '.join(allowed_values), orderBy)) + elif orderBy == 'lastsync': + orderBy = 'lastSync' + elif orderBy == 'deviceid': + orderBy = 'deviceId' i += 2 elif myarg in SORTORDER_CHOICES_MAP: sortOrder = SORTORDER_CHOICES_MAP[myarg] @@ -11657,24 +11667,24 @@ def doPrintMobileDevices(): else: systemErrorExit(2, '%s is not a valid argument for "gam print mobile"' % sys.argv[i]) for query in queries: - printGettingAllItems(u'Mobile Devices', query) - page_message = u'Got %%num_items%% Mobile Devices...\n' - all_mobile = callGAPIpages(cd.mobiledevices(), u'list', u'mobiledevices', page_message=page_message, + printGettingAllItems('Mobile Devices', query) + page_message = 'Got %%num_items%% Mobile Devices...\n' + all_mobile = callGAPIpages(cd.mobiledevices(), 'list', 'mobiledevices', page_message=page_message, customerId=GC_Values[GC_CUSTOMER_ID], query=query, projection=projection, fields=fields, orderBy=orderBy, sortOrder=sortOrder, maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) for mobile in all_mobile: row = {} for attrib in mobile: - if attrib in [u'kind', u'etag']: + if attrib in ['kind', 'etag']: continue - if attrib in [u'name', u'email', u'otherAccountsInfo']: + if attrib in ['name', 'email', 'otherAccountsInfo']: if attrib not in titles: titles.append(attrib) if listLimit > 0: row[attrib] = delimiter.join(mobile[attrib][0:listLimit]) elif listLimit == 0: row[attrib] = delimiter.join(mobile[attrib]) - elif attrib == u'applications': + elif attrib == 'applications': if appsLimit >= 0: if attrib not in titles: titles.append(attrib) @@ -11685,46 +11695,46 @@ def doPrintMobileDevices(): if appsLimit and (j > appsLimit): break appDetails = [] - for field in [u'displayName', u'packageName', u'versionName']: - appDetails.append(app.get(field, u'')) - appDetails.append(unicode(app.get(u'versionCode', u''))) - permissions = app.get(u'permission', []) + for field in ['displayName', 'packageName', 'versionName']: + appDetails.append(app.get(field, '')) + appDetails.append(str(app.get('versionCode', ''))) + permissions = app.get('permission', []) if permissions: - appDetails.append(u'/'.join(permissions)) + appDetails.append('/'.join(permissions)) else: - appDetails.append(u'') - applications.append(u'-'.join(appDetails)) + appDetails.append('') + applications.append('-'.join(appDetails)) row[attrib] = delimiter.join(applications) else: if attrib not in titles: titles.append(attrib) row[attrib] = mobile[attrib] csvRows.append(row) - sortCSVTitles([u'resourceId', u'deviceId', u'serialNumber', u'name', u'email', u'status'], titles) - writeCSVfile(csvRows, titles, u'Mobile', todrive) + sortCSVTitles(['resourceId', 'deviceId', 'serialNumber', 'name', 'email', 'status'], titles) + writeCSVfile(csvRows, titles, 'Mobile', todrive) def doPrintCrosActivity(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False - titles = [u'deviceId', u'annotatedAssetId', u'annotatedLocation', u'serialNumber', u'orgUnitPath'] + titles = ['deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber', 'orgUnitPath'] csvRows = [] - fieldsList = [u'deviceId', u'annotatedAssetId', u'annotatedLocation', u'serialNumber', u'orgUnitPath'] + fieldsList = ['deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber', 'orgUnitPath'] startDate = endDate = None selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False listLimit = 0 - delimiter = u',' + delimiter = ',' orgUnitPath = None queries = [None] i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg in [u'query', u'queries']: + myarg = sys.argv[i].lower().replace('_', '') + if myarg in ['query', 'queries']: queries = getQueries(myarg, sys.argv[i+1]) i += 2 - elif myarg == u'limittoou': + elif myarg == 'limittoou': orgUnitPath = getOrgUnitItem(sys.argv[i+1]) i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 elif myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS: @@ -11736,10 +11746,10 @@ def doPrintCrosActivity(): elif myarg in CROS_RECENT_USERS_ARGUMENTS: selectRecentUsers = True i += 1 - elif myarg == u'both': + elif myarg == 'both': selectActiveTimeRanges = selectRecentUsers = True i += 1 - elif myarg == u'all': + elif myarg == 'all': selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = True i += 1 elif myarg in CROS_START_ARGUMENTS: @@ -11748,10 +11758,10 @@ def doPrintCrosActivity(): elif myarg in CROS_END_ARGUMENTS: endDate = _getFilterDate(sys.argv[i+1]) i += 2 - elif myarg == u'listlimit': + elif myarg == 'listlimit': listLimit = getInteger(sys.argv[i+1], myarg, minVal=0) i += 2 - elif myarg == u'delimiter': + elif myarg == 'delimiter': delimiter = sys.argv[i+1] i += 2 else: @@ -11759,82 +11769,82 @@ def doPrintCrosActivity(): if not selectActiveTimeRanges and not selectDeviceFiles and not selectRecentUsers: selectActiveTimeRanges = selectRecentUsers = True if selectRecentUsers: - fieldsList.append(u'recentUsers') - addTitlesToCSVfile([u'recentUsers.email',], titles) + fieldsList.append('recentUsers') + addTitlesToCSVfile(['recentUsers.email',], titles) if selectActiveTimeRanges: - fieldsList.append(u'activeTimeRanges') - addTitlesToCSVfile([u'activeTimeRanges.date', u'activeTimeRanges.duration', u'activeTimeRanges.minutes'], titles) + fieldsList.append('activeTimeRanges') + addTitlesToCSVfile(['activeTimeRanges.date', 'activeTimeRanges.duration', 'activeTimeRanges.minutes'], titles) if selectDeviceFiles: - fieldsList.append(u'deviceFiles') - addTitlesToCSVfile([u'deviceFiles.type', u'deviceFiles.createTime'], titles) - fields = u'chromeosdevices(%s),nextPageToken' % u','.join(fieldsList) + fieldsList.append('deviceFiles') + addTitlesToCSVfile(['deviceFiles.type', 'deviceFiles.createTime'], titles) + fields = 'chromeosdevices(%s),nextPageToken' % ','.join(fieldsList) for query in queries: - printGettingAllItems(u'CrOS Devices', query) - page_message = u'Got %%num_items%% CrOS Devices...\n' - all_cros = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', page_message=page_message, - query=query, customerId=GC_Values[GC_CUSTOMER_ID], projection=u'FULL', + printGettingAllItems('CrOS Devices', query) + page_message = 'Got %%num_items%% CrOS Devices...\n' + all_cros = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + query=query, customerId=GC_Values[GC_CUSTOMER_ID], projection='FULL', fields=fields, maxResults=GC_Values[GC_DEVICE_MAX_RESULTS], orgUnitPath=orgUnitPath) for cros in all_cros: row = {} for attrib in cros: - if attrib not in [u'recentUsers', u'activeTimeRanges', u'deviceFiles']: + if attrib not in ['recentUsers', 'activeTimeRanges', 'deviceFiles']: row[attrib] = cros[attrib] if selectActiveTimeRanges: - activeTimeRanges = _filterTimeRanges(cros.get(u'activeTimeRanges', []), startDate, endDate) + activeTimeRanges = _filterTimeRanges(cros.get('activeTimeRanges', []), startDate, endDate) lenATR = len(activeTimeRanges) for activeTimeRange in activeTimeRanges[:min(lenATR, listLimit or lenATR)]: new_row = row.copy() - new_row[u'activeTimeRanges.date'] = activeTimeRange[u'date'] - new_row[u'activeTimeRanges.duration'] = utils.formatMilliSeconds(activeTimeRange[u'activeTime']) - new_row[u'activeTimeRanges.minutes'] = activeTimeRange[u'activeTime']/60000 + new_row['activeTimeRanges.date'] = activeTimeRange['date'] + new_row['activeTimeRanges.duration'] = utils.formatMilliSeconds(activeTimeRange['activeTime']) + new_row['activeTimeRanges.minutes'] = activeTimeRange['activeTime']/60000 csvRows.append(new_row) if selectRecentUsers: - recentUsers = cros.get(u'recentUsers', []) + recentUsers = cros.get('recentUsers', []) lenRU = len(recentUsers) - row[u'recentUsers.email'] = delimiter.join([recent_user.get(u'email', [u'Unknown', u'UnmanagedUser'][recent_user[u'type'] == u'USER_TYPE_UNMANAGED']) for recent_user in recentUsers[:min(lenRU, listLimit or lenRU)]]) + row['recentUsers.email'] = delimiter.join([recent_user.get('email', ['Unknown', 'UnmanagedUser'][recent_user['type'] == 'USER_TYPE_UNMANAGED']) for recent_user in recentUsers[:min(lenRU, listLimit or lenRU)]]) csvRows.append(row) if selectDeviceFiles: - deviceFiles = _filterCreateReportTime(cros.get(u'deviceFiles', []), u'createTime', startDate, endDate) + deviceFiles = _filterCreateReportTime(cros.get('deviceFiles', []), 'createTime', startDate, endDate) lenDF = len(deviceFiles) for deviceFile in deviceFiles[:min(lenDF, listLimit or lenDF)]: new_row = row.copy() - new_row[u'deviceFiles.type'] = deviceFile[u'type'] - new_row[u'deviceFiles.createTime'] = deviceFile[u'createTime'] + new_row['deviceFiles.type'] = deviceFile['type'] + new_row['deviceFiles.createTime'] = deviceFile['createTime'] csvRows.append(new_row) - writeCSVfile(csvRows, titles, u'CrOS Activity', todrive) + writeCSVfile(csvRows, titles, 'CrOS Activity', todrive) def _checkTPMVulnerability(cros): - if u'tpmVersionInfo' in cros and u'firmwareVersion' in cros[u'tpmVersionInfo']: - if cros[u'tpmVersionInfo'][u'firmwareVersion'] in CROS_TPM_VULN_VERSIONS: - cros[u'tpmVersionInfo'][u'tpmVulnerability'] = u'VULNERABLE' - elif cros[u'tpmVersionInfo'][u'firmwareVersion'] in CROS_TPM_FIXED_VERSIONS: - cros[u'tpmVersionInfo'][u'tpmVulnerability'] = u'UPDATED' + if 'tpmVersionInfo' in cros and 'firmwareVersion' in cros['tpmVersionInfo']: + if cros['tpmVersionInfo']['firmwareVersion'] in CROS_TPM_VULN_VERSIONS: + cros['tpmVersionInfo']['tpmVulnerability'] = 'VULNERABLE' + elif cros['tpmVersionInfo']['firmwareVersion'] in CROS_TPM_FIXED_VERSIONS: + cros['tpmVersionInfo']['tpmVulnerability'] = 'UPDATED' else: - cros[u'tpmVersionInfo'][u'tpmVulnerability'] = u'NOT IMPACTED' + cros['tpmVersionInfo']['tpmVulnerability'] = 'NOT IMPACTED' return cros def doPrintCrosDevices(): def _getSelectedLists(myarg): if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS: - selectedLists[u'activeTimeRanges'] = True + selectedLists['activeTimeRanges'] = True elif myarg in CROS_RECENT_USERS_ARGUMENTS: - selectedLists[u'recentUsers'] = True + selectedLists['recentUsers'] = True elif myarg in CROS_DEVICE_FILES_ARGUMENTS: - selectedLists[u'deviceFiles'] = True + selectedLists['deviceFiles'] = True elif myarg in CROS_CPU_STATUS_REPORTS_ARGUMENTS: - selectedLists[u'cpuStatusReports'] = True + selectedLists['cpuStatusReports'] = True elif myarg in CROS_DISK_VOLUME_REPORTS_ARGUMENTS: - selectedLists[u'diskVolumeReports'] = True + selectedLists['diskVolumeReports'] = True elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS: - selectedLists[u'systemRamFreeReports'] = True + selectedLists['systemRamFreeReports'] = True - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False fieldsList = [] fieldsTitles = {} titles = [] csvRows = [] - addFieldToCSVfile(u'deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) + addFieldToCSVfile('deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) projection = orderBy = sortOrder = orgUnitPath = None queries = [None] noLists = sortHeaders = False @@ -11843,17 +11853,17 @@ def doPrintCrosDevices(): listLimit = 0 i = 3 while i < len(sys.argv): - myarg = sys.argv[i].lower().replace(u'_', u'') - if myarg in [u'query', u'queries']: + myarg = sys.argv[i].lower().replace('_', '') + if myarg in ['query', 'queries']: queries = getQueries(myarg, sys.argv[i+1]) i += 2 - elif myarg == u'limittoou': + elif myarg == 'limittoou': orgUnitPath = getOrgUnitItem(sys.argv[i+1]) i += 2 - elif myarg == u'todrive': + elif myarg == 'todrive': todrive = True i += 1 - elif myarg == u'nolists': + elif myarg == 'nolists': noLists = True selectedLists = {} i += 1 @@ -11863,24 +11873,24 @@ def doPrintCrosDevices(): elif myarg in CROS_END_ARGUMENTS: endDate = _getFilterDate(sys.argv[i+1]) i += 2 - elif myarg == u'listlimit': + elif myarg == 'listlimit': listLimit = getInteger(sys.argv[i+1], myarg, minVal=0) i += 2 - elif myarg == u'orderby': - orderBy = sys.argv[i+1].lower().replace(u'_', u'') - allowed_values = [u'location', u'user', u'lastsync', u'notes', u'serialnumber', u'status', u'supportenddate'] + elif myarg == 'orderby': + orderBy = sys.argv[i+1].lower().replace('_', '') + allowed_values = ['location', 'user', 'lastsync', 'notes', 'serialnumber', 'status', 'supportenddate'] if orderBy not in allowed_values: - systemErrorExit(2, 'orderBy must be one of %s; got %s' % (u', '.join(allowed_values), orderBy)) - elif orderBy == u'location': - orderBy = u'annotatedLocation' - elif orderBy == u'user': - orderBy = u'annotatedUser' - elif orderBy == u'lastsync': - orderBy = u'lastSync' - elif orderBy == u'serialnumber': - orderBy = u'serialNumber' - elif orderBy == u'supportenddate': - orderBy = u'supportEndDate' + systemErrorExit(2, 'orderBy must be one of %s; got %s' % (', '.join(allowed_values), orderBy)) + elif orderBy == 'location': + orderBy = 'annotatedLocation' + elif orderBy == 'user': + orderBy = 'annotatedUser' + elif orderBy == 'lastsync': + orderBy = 'lastSync' + elif orderBy == 'serialnumber': + orderBy = 'serialNumber' + elif orderBy == 'supportenddate': + orderBy = 'supportEndDate' i += 2 elif myarg in SORTORDER_CHOICES_MAP: sortOrder = SORTORDER_CHOICES_MAP[myarg] @@ -11888,17 +11898,17 @@ def doPrintCrosDevices(): elif myarg in PROJECTION_CHOICES_MAP: projection = PROJECTION_CHOICES_MAP[myarg] sortHeaders = True - if projection == u'FULL': + if projection == 'FULL': fieldsList = [] else: fieldsList = CROS_BASIC_FIELDS_LIST[:] i += 1 - elif myarg == u'allfields': - projection = u'FULL' + elif myarg == 'allfields': + projection = 'FULL' sortHeaders = True fieldsList = [] i += 1 - elif myarg == u'sortheaders': + elif myarg == 'sortheaders': sortHeaders = True i += 1 elif myarg in CROS_LISTS_ARGUMENTS: @@ -11907,9 +11917,9 @@ def doPrintCrosDevices(): elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP: addFieldToFieldsList(myarg, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList) i += 1 - elif myarg == u'fields': + elif myarg == 'fields': fieldNameList = sys.argv[i+1] - for field in fieldNameList.lower().replace(u',', u' ').split(): + for field in fieldNameList.lower().replace(',', ' ').split(): if field in CROS_LISTS_ARGUMENTS: _getSelectedLists(field) elif field in CROS_ARGUMENT_TO_PROPERTY_MAP: @@ -11921,45 +11931,45 @@ def doPrintCrosDevices(): systemErrorExit(2, '%s is not a valid argument for "gam print cros"' % sys.argv[i]) if selectedLists: noLists = False - projection = u'FULL' + projection = 'FULL' for selectList in selectedLists: addFieldToFieldsList(selectList, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList) if fieldsList: - fieldsList.append(u'deviceId') - fields = u'nextPageToken,chromeosdevices({0})'.format(u','.join(set(fieldsList))).replace(u'.', u'/') + fieldsList.append('deviceId') + fields = 'nextPageToken,chromeosdevices({0})'.format(','.join(set(fieldsList))).replace('.', '/') else: fields = None for query in queries: - printGettingAllItems(u'CrOS Devices', query) - page_message = u'Got %%num_items%% CrOS Devices...\n' - all_cros = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', page_message=page_message, + printGettingAllItems('CrOS Devices', query) + page_message = 'Got %%num_items%% CrOS Devices...\n' + all_cros = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, query=query, customerId=GC_Values[GC_CUSTOMER_ID], projection=projection, orgUnitPath=orgUnitPath, orderBy=orderBy, sortOrder=sortOrder, fields=fields, maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) for cros in all_cros: cros = _checkTPMVulnerability(cros) if not noLists and not selectedLists: for cros in all_cros: - if u'notes' in cros: - cros[u'notes'] = cros[u'notes'].replace(u'\n', u'\\n') - for cpuStatusReport in cros.get(u'cpuStatusReports', []): - for tempInfo in cpuStatusReport.get(u'cpuTemperatureInfo', []): - tempInfo[u'label'] = tempInfo[u'label'].strip() + if 'notes' in cros: + cros['notes'] = cros['notes'].replace('\n', '\\n') + for cpuStatusReport in cros.get('cpuStatusReports', []): + for tempInfo in cpuStatusReport.get('cpuTemperatureInfo', []): + tempInfo['label'] = tempInfo['label'].strip() addRowTitlesToCSVfile(flatten_json(cros, listLimit=listLimit), csvRows, titles) continue for cros in all_cros: - if u'notes' in cros: - cros[u'notes'] = cros[u'notes'].replace(u'\n', u'\\n') + if 'notes' in cros: + cros['notes'] = cros['notes'].replace('\n', '\\n') row = {} for attrib in cros: - if attrib not in set([u'kind', u'etag', u'tpmVersionInfo', u'recentUsers', u'activeTimeRanges', - u'deviceFiles', u'cpuStatusReports', u'diskVolumeReports', u'systemRamFreeReports']): + if attrib not in set(['kind', 'etag', 'tpmVersionInfo', 'recentUsers', 'activeTimeRanges', + 'deviceFiles', 'cpuStatusReports', 'diskVolumeReports', 'systemRamFreeReports']): row[attrib] = cros[attrib] - activeTimeRanges = _filterTimeRanges(cros.get(u'activeTimeRanges', []) if selectedLists.get(u'activeTimeRanges') else [], startDate, endDate) - recentUsers = cros.get(u'recentUsers', []) if selectedLists.get(u'recentUsers') else [] - deviceFiles = _filterCreateReportTime(cros.get(u'deviceFiles', []) if selectedLists.get(u'deviceFiles') else [], u'createTime', startDate, endDate) - cpuStatusReports = _filterCreateReportTime(cros.get(u'cpuStatusReports', []) if selectedLists.get(u'cpuStatusReports') else [], u'reportTime', startDate, endDate) - diskVolumeReports = cros.get(u'diskVolumeReports', []) if selectedLists.get(u'diskVolumeReports') else [] - systemRamFreeReports = _filterCreateReportTime(cros.get(u'systemRamFreeReports', []) if selectedLists.get(u'systemRamFreeReports') else [], u'reportTime', startDate, endDate) + activeTimeRanges = _filterTimeRanges(cros.get('activeTimeRanges', []) if selectedLists.get('activeTimeRanges') else [], startDate, endDate) + recentUsers = cros.get('recentUsers', []) if selectedLists.get('recentUsers') else [] + deviceFiles = _filterCreateReportTime(cros.get('deviceFiles', []) if selectedLists.get('deviceFiles') else [], 'createTime', startDate, endDate) + cpuStatusReports = _filterCreateReportTime(cros.get('cpuStatusReports', []) if selectedLists.get('cpuStatusReports') else [], 'reportTime', startDate, endDate) + diskVolumeReports = cros.get('diskVolumeReports', []) if selectedLists.get('diskVolumeReports') else [] + systemRamFreeReports = _filterCreateReportTime(cros.get('systemRamFreeReports', []) if selectedLists.get('systemRamFreeReports') else [], 'reportTime', startDate, endDate) if noLists or (not activeTimeRanges and not recentUsers and not deviceFiles and not cpuStatusReports and not diskVolumeReports and not systemRamFreeReports): addRowTitlesToCSVfile(row, csvRows, titles) @@ -11973,39 +11983,39 @@ def doPrintCrosDevices(): for i in range(min(max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR), listLimit or max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR))): new_row = row.copy() if i < lenATR: - new_row[u'activeTimeRanges.date'] = activeTimeRanges[i][u'date'] - new_row[u'activeTimeRanges.activeTime'] = str(activeTimeRanges[i][u'activeTime']) - new_row[u'activeTimeRanges.duration'] = utils.formatMilliSeconds(activeTimeRanges[i][u'activeTime']) - new_row[u'activeTimeRanges.minutes'] = activeTimeRanges[i][u'activeTime']/60000 + new_row['activeTimeRanges.date'] = activeTimeRanges[i]['date'] + new_row['activeTimeRanges.activeTime'] = str(activeTimeRanges[i]['activeTime']) + new_row['activeTimeRanges.duration'] = utils.formatMilliSeconds(activeTimeRanges[i]['activeTime']) + new_row['activeTimeRanges.minutes'] = activeTimeRanges[i]['activeTime']/60000 if i < lenRU: - new_row[u'recentUsers.email'] = recentUsers[i].get(u'email', [u'Unknown', u'UnmanagedUser'][recentUsers[i][u'type'] == u'USER_TYPE_UNMANAGED']) - new_row[u'recentUsers.type'] = recentUsers[i][u'type'] + new_row['recentUsers.email'] = recentUsers[i].get('email', ['Unknown', 'UnmanagedUser'][recentUsers[i]['type'] == 'USER_TYPE_UNMANAGED']) + new_row['recentUsers.type'] = recentUsers[i]['type'] if i < lenDF: - new_row[u'deviceFiles.type'] = deviceFiles[i][u'type'] - new_row[u'deviceFiles.createTime'] = deviceFiles[i][u'createTime'] + new_row['deviceFiles.type'] = deviceFiles[i]['type'] + new_row['deviceFiles.createTime'] = deviceFiles[i]['createTime'] if i < lenCSR: - new_row[u'cpuStatusReports.reportTime'] = cpuStatusReports[i][u'reportTime'] - for tempInfo in cpuStatusReports[i].get(u'cpuTemperatureInfo', []): - new_row[u'cpuStatusReports.cpuTemperatureInfo.{0}'.format(tempInfo[u'label'].strip())] = tempInfo[u'temperature'] - new_row[u'cpuStatusReports.cpuUtilizationPercentageInfo'] = u','.join([str(x) for x in cpuStatusReports[i][u'cpuUtilizationPercentageInfo']]) + new_row['cpuStatusReports.reportTime'] = cpuStatusReports[i]['reportTime'] + for tempInfo in cpuStatusReports[i].get('cpuTemperatureInfo', []): + new_row['cpuStatusReports.cpuTemperatureInfo.{0}'.format(tempInfo['label'].strip())] = tempInfo['temperature'] + new_row['cpuStatusReports.cpuUtilizationPercentageInfo'] = ','.join([str(x) for x in cpuStatusReports[i]['cpuUtilizationPercentageInfo']]) if i < lenDVR: - volumeInfo = diskVolumeReports[i][u'volumeInfo'] + volumeInfo = diskVolumeReports[i]['volumeInfo'] j = 0 for volume in volumeInfo: - new_row[u'diskVolumeReports.volumeInfo.{0}.volumeId'.format(j)] = volume[u'volumeId'] - new_row[u'diskVolumeReports.volumeInfo.{0}.storageFree'.format(j)] = volume[u'storageFree'] - new_row[u'diskVolumeReports.volumeInfo.{0}.storageTotal'.format(j)] = volume[u'storageTotal'] + new_row['diskVolumeReports.volumeInfo.{0}.volumeId'.format(j)] = volume['volumeId'] + new_row['diskVolumeReports.volumeInfo.{0}.storageFree'.format(j)] = volume['storageFree'] + new_row['diskVolumeReports.volumeInfo.{0}.storageTotal'.format(j)] = volume['storageTotal'] j += 1 if i < lenSRFR: - new_row[u'systemRamFreeReports.reportTime'] = systemRamFreeReports[i][u'reportTime'] - new_row[u'systenRamFreeReports.systemRamFreeInfo'] = u','.join([str(x) for x in systemRamFreeReports[i][u'systemRamFreeInfo']]) + new_row['systemRamFreeReports.reportTime'] = systemRamFreeReports[i]['reportTime'] + new_row['systenRamFreeReports.systemRamFreeInfo'] = ','.join([str(x) for x in systemRamFreeReports[i]['systemRamFreeInfo']]) addRowTitlesToCSVfile(new_row, csvRows, titles) if sortHeaders: - sortCSVTitles([u'deviceId',], titles) - writeCSVfile(csvRows, titles, u'CrOS', todrive) + sortCSVTitles(['deviceId',], titles) + writeCSVfile(csvRows, titles, 'CrOS', todrive) def doPrintLicenses(returnFields=None, skus=None, countsOnly=False, returnCounts=False): - lic = buildGAPIObject(u'licensing') + lic = buildGAPIObject('licensing') products = [] licenses = [] licenseCounts = [] @@ -12015,65 +12025,65 @@ def doPrintLicenses(returnFields=None, skus=None, countsOnly=False, returnCounts i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if not returnCounts and myarg == u'todrive': + if not returnCounts and myarg == 'todrive': todrive = True i += 1 - elif myarg in [u'products', u'product']: - products = sys.argv[i+1].split(u',') + elif myarg in ['products', 'product']: + products = sys.argv[i+1].split(',') i += 2 - elif myarg in [u'sku', u'skus']: - skus = sys.argv[i+1].split(u',') + elif myarg in ['sku', 'skus']: + skus = sys.argv[i+1].split(',') i += 2 - elif myarg == u'allskus': + elif myarg == 'allskus': skus = sorted(SKUS.keys()) products = [] i += 1 - elif myarg == u'gsuite': - skus = [skuId for skuId in SKUS if SKUS[skuId][u'product'] in [u'Google-Apps', u'101031']] + elif myarg == 'gsuite': + skus = [skuId for skuId in SKUS if SKUS[skuId]['product'] in ['Google-Apps', '101031']] products = [] i += 1 - elif myarg == u'countsonly': + elif myarg == 'countsonly': countsOnly = True i += 1 else: systemErrorExit(2, '%s is not a valid argument for "gam print licenses"' % sys.argv[i]) if not countsOnly: - fields = u'nextPageToken,items(productId,skuId,userId)' - titles = [u'userId', u'productId', u'skuId'] + fields = 'nextPageToken,items(productId,skuId,userId)' + titles = ['userId', 'productId', 'skuId'] else: - fields = u'nextPageToken,items(userId)' + fields = 'nextPageToken,items(userId)' if not returnCounts: if skus: - titles = [u'productId', u'skuId', u'licenses'] + titles = ['productId', 'skuId', 'licenses'] else: - titles = [u'productId', u'licenses'] + titles = ['productId', 'licenses'] else: - fields = u'nextPageToken,items({0})'.format(returnFields) + fields = 'nextPageToken,items({0})'.format(returnFields) if skus: for sku in skus: product, sku = getProductAndSKU(sku) - page_message = u'Got %%%%total_items%%%% Licenses for %s...\n' % sku + page_message = 'Got %%%%total_items%%%% Licenses for %s...\n' % sku try: - licenses += callGAPIpages(lic.licenseAssignments(), u'listForProductAndSku', u'items', throw_reasons=[GAPI_INVALID, GAPI_FORBIDDEN], page_message=page_message, + licenses += callGAPIpages(lic.licenseAssignments(), 'listForProductAndSku', 'items', throw_reasons=[GAPI_INVALID, GAPI_FORBIDDEN], page_message=page_message, customerId=GC_Values[GC_DOMAIN], productId=product, skuId=sku, fields=fields) if countsOnly: - licenseCounts.append([u'Product', product, u'SKU', sku, u'Licenses', len(licenses)]) + licenseCounts.append(['Product', product, 'SKU', sku, 'Licenses', len(licenses)]) licenses = [] except (GAPI_invalid, GAPI_forbidden): pass else: if not products: - for sku in SKUS.values(): - if sku[u'product'] not in products: - products.append(sku[u'product']) + for sku in list(SKUS.values()): + if sku['product'] not in products: + products.append(sku['product']) products.sort() for productId in products: - page_message = u'Got %%%%total_items%%%% Licenses for %s...\n' % productId + page_message = 'Got %%%%total_items%%%% Licenses for %s...\n' % productId try: - licenses += callGAPIpages(lic.licenseAssignments(), u'listForProduct', u'items', throw_reasons=[GAPI_INVALID, GAPI_FORBIDDEN], page_message=page_message, + licenses += callGAPIpages(lic.licenseAssignments(), 'listForProduct', 'items', throw_reasons=[GAPI_INVALID, GAPI_FORBIDDEN], page_message=page_message, customerId=GC_Values[GC_DOMAIN], productId=productId, fields=fields) if countsOnly: - licenseCounts.append([u'Product', productId, u'Licenses', len(licenses)]) + licenseCounts.append(['Product', productId, 'Licenses', len(licenses)]) licenses = [] except (GAPI_invalid, GAPI_forbidden): pass @@ -12082,164 +12092,164 @@ def doPrintLicenses(returnFields=None, skus=None, countsOnly=False, returnCounts return licenseCounts if skus: for u_license in licenseCounts: - csvRows.append({u'productId': u_license[1], u'skuId': u_license[3], u'licenses': u_license[5]}) + csvRows.append({'productId': u_license[1], 'skuId': u_license[3], 'licenses': u_license[5]}) else: for u_license in licenseCounts: - csvRows.append({u'productId': u_license[1], u'licenses': u_license[3]}) - writeCSVfile(csvRows, titles, u'Licenses', todrive) + csvRows.append({'productId': u_license[1], 'licenses': u_license[3]}) + writeCSVfile(csvRows, titles, 'Licenses', todrive) return if returnFields: - if returnFields == u'userId': + if returnFields == 'userId': userIds = [] for u_license in licenses: - userId = u_license.get(u'userId', u'').lower() + userId = u_license.get('userId', '').lower() if userId: userIds.append(userId) return userIds else: userSkuIds = {} for u_license in licenses: - userId = u_license.get(u'userId', u'').lower() - skuId = u_license.get(u'skuId') + userId = u_license.get('userId', '').lower() + skuId = u_license.get('skuId') if userId and skuId: userSkuIds.setdefault(userId, []) userSkuIds[userId].append(skuId) return userSkuIds for u_license in licenses: - userId = u_license.get(u'userId', u'').lower() - skuId = u_license.get(u'skuId', u'') - csvRows.append({u'userId': userId, u'productId': u_license.get(u'productId', u''), - u'skuId': _skuIdToDisplayName(skuId)}) - writeCSVfile(csvRows, titles, u'Licenses', todrive) + userId = u_license.get('userId', '').lower() + skuId = u_license.get('skuId', '') + csvRows.append({'userId': userId, 'productId': u_license.get('productId', ''), + 'skuId': _skuIdToDisplayName(skuId)}) + writeCSVfile(csvRows, titles, 'Licenses', todrive) def doShowLicenses(): licenseCounts = doPrintLicenses(countsOnly=True, returnCounts=True) for u_license in licenseCounts: - line = u'' - for i in xrange(0, len(u_license), 2): - line += u'{0}: {1}, '.format(u_license[i], u_license[i+1]) - print line[:-2] + line = '' + for i in range(0, len(u_license), 2): + line += '{0}: {1}, '.format(u_license[i], u_license[i+1]) + print(line[:-2]) -RESCAL_DFLTFIELDS = [u'id', u'name', u'email',] -RESCAL_ALLFIELDS = [u'id', u'name', u'email', u'description', u'type', u'buildingid', u'category', u'capacity', - u'features', u'floor', u'floorsection', u'generatedresourcename', u'uservisibledescription',] +RESCAL_DFLTFIELDS = ['id', 'name', 'email',] +RESCAL_ALLFIELDS = ['id', 'name', 'email', 'description', 'type', 'buildingid', 'category', 'capacity', + 'features', 'floor', 'floorsection', 'generatedresourcename', 'uservisibledescription',] RESCAL_ARGUMENT_TO_PROPERTY_MAP = { - u'description': [u'resourceDescription'], - u'building': [u'buildingId',], - u'buildingid': [u'buildingId',], - u'capacity': [u'capacity',], - u'category': [u'resourceCategory',], - u'email': [u'resourceEmail'], - u'feature': [u'featureInstances',], - u'features': [u'featureInstances',], - u'floor': [u'floorName',], - u'floorname': [u'floorName',], - u'floorsection': [u'floorSection',], - u'generatedresourcename': [u'generatedResourceName',], - u'id': [u'resourceId'], - u'name': [u'resourceName'], - u'type': [u'resourceType'], - u'userdescription': [u'userVisibleDescription',], - u'uservisibledescription': [u'userVisibleDescription',], + 'description': ['resourceDescription'], + 'building': ['buildingId',], + 'buildingid': ['buildingId',], + 'capacity': ['capacity',], + 'category': ['resourceCategory',], + 'email': ['resourceEmail'], + 'feature': ['featureInstances',], + 'features': ['featureInstances',], + 'floor': ['floorName',], + 'floorname': ['floorName',], + 'floorsection': ['floorSection',], + 'generatedresourcename': ['generatedResourceName',], + 'id': ['resourceId'], + 'name': ['resourceName'], + 'type': ['resourceType'], + 'userdescription': ['userVisibleDescription',], + 'uservisibledescription': ['userVisibleDescription',], } def doPrintFeatures(): to_drive = False - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') titles = [] csvRows = [] - fieldsList = [u'name'] - fields = u'nextPageToken,features(%s)' + fieldsList = ['name'] + fields = 'nextPageToken,features(%s)' possible_fields = {} - for pfield in cd._rootDesc[u'schemas'][u'Feature'][u'properties']: + for pfield in cd._rootDesc['schemas']['Feature']['properties']: possible_fields[pfield.lower()] = pfield i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': to_drive = True i += 1 - elif myarg == u'allfields': + elif myarg == 'allfields': fields = None i += 1 elif myarg in possible_fields: fieldsList.append(possible_fields[myarg]) i += 1 - elif u'feature'+myarg in possible_fields: - fieldsList.append(possible_fields[u'feature'+myarg]) + elif 'feature'+myarg in possible_fields: + fieldsList.append(possible_fields['feature'+myarg]) i += 1 else: systemErrorExit(3, '%s is not a valid argument to "gam print features"' % sys.argv[i]) if fields: - fields = fields % u','.join(fieldsList) - features = callGAPIpages(cd.resources().features(), u'list', u'features', + fields = fields % ','.join(fieldsList) + features = callGAPIpages(cd.resources().features(), 'list', 'features', customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for feature in features: - feature.pop(u'etags', None) - feature.pop(u'etag', None) - feature.pop(u'kind', None) + feature.pop('etags', None) + feature.pop('etag', None) + feature.pop('kind', None) feature = flatten_json(feature) for item in feature: if item not in titles: titles.append(item) csvRows.append(feature) - sortCSVTitles(u'name', titles) - writeCSVfile(csvRows, titles, u'Features', to_drive) + sortCSVTitles('name', titles) + writeCSVfile(csvRows, titles, 'Features', to_drive) def doPrintBuildings(): to_drive = False - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') titles = [] csvRows = [] - fieldsList = [u'buildingId'] + fieldsList = ['buildingId'] # buildings.list() currently doesn't support paging # but should soon, attempt to use it now so we # won't break when it's turned on. - fields = u'nextPageToken,buildings(%s)' + fields = 'nextPageToken,buildings(%s)' possible_fields = {} - for pfield in cd._rootDesc[u'schemas'][u'Building'][u'properties']: + for pfield in cd._rootDesc['schemas']['Building']['properties']: possible_fields[pfield.lower()] = pfield i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': to_drive = True i += 1 - elif myarg == u'allfields': + elif myarg == 'allfields': fields = None i += 1 elif myarg in possible_fields: fieldsList.append(possible_fields[myarg]) i += 1 # Allows shorter arguments like "name" instead of "buildingname" - elif u'building'+myarg in possible_fields: - fieldsList.append(possible_fields[u'building'+myarg]) + elif 'building'+myarg in possible_fields: + fieldsList.append(possible_fields['building'+myarg]) i += 1 else: systemErrorExit(3, '%s is not a valid argument to "gam print buildings"' % sys.argv[i]) if fields: - fields = fields % u','.join(fieldsList) - buildings = callGAPIpages(cd.resources().buildings(), u'list', u'buildings', + fields = fields % ','.join(fieldsList) + buildings = callGAPIpages(cd.resources().buildings(), 'list', 'buildings', customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for building in buildings: - building.pop(u'etags', None) - building.pop(u'etag', None) - building.pop(u'kind', None) - if u'buildingId' in building: - building[u'buildingId'] = u'id:{0}'.format(building[u'buildingId']) - if u'floorNames' in building: - building[u'floorNames'] = u','.join(building[u'floorNames']) + building.pop('etags', None) + building.pop('etag', None) + building.pop('kind', None) + if 'buildingId' in building: + building['buildingId'] = 'id:{0}'.format(building['buildingId']) + if 'floorNames' in building: + building['floorNames'] = ','.join(building['floorNames']) building = flatten_json(building) for item in building: if item not in titles: titles.append(item) csvRows.append(building) - sortCSVTitles(u'buildingId', titles) - writeCSVfile(csvRows, titles, u'Buildings', to_drive) + sortCSVTitles('buildingId', titles) + writeCSVfile(csvRows, titles, 'Buildings', to_drive) def doPrintResourceCalendars(): - cd = buildGAPIObject(u'directory') + cd = buildGAPIObject('directory') todrive = False fieldsList = [] fieldsTitles = {} @@ -12248,10 +12258,10 @@ def doPrintResourceCalendars(): i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower() - if myarg == u'todrive': + if myarg == 'todrive': todrive = True i += 1 - elif myarg == u'allfields': + elif myarg == 'allfields': fieldsList = [] fieldsTitles = {} titles = [] @@ -12266,42 +12276,42 @@ def doPrintResourceCalendars(): if not fieldsList: for field in RESCAL_DFLTFIELDS: addFieldToCSVfile(field, RESCAL_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles) - fields = u'nextPageToken,items({0})'.format(u','.join(set(fieldsList))) - if u'buildingId' in fieldsList: - addFieldToCSVfile(u'buildingName', {u'buildingName': [u'buildingName',]}, fieldsList, fieldsTitles, titles) - printGettingAllItems(u'Resource Calendars', None) - page_message = u'Got %%total_items%% Resource Calendars: %%first_item%% - %%last_item%%\n' - resources = callGAPIpages(cd.resources().calendars(), u'list', u'items', - page_message=page_message, message_attribute=u'resourceId', + fields = 'nextPageToken,items({0})'.format(','.join(set(fieldsList))) + if 'buildingId' in fieldsList: + addFieldToCSVfile('buildingName', {'buildingName': ['buildingName',]}, fieldsList, fieldsTitles, titles) + printGettingAllItems('Resource Calendars', None) + page_message = 'Got %%total_items%% Resource Calendars: %%first_item%% - %%last_item%%\n' + resources = callGAPIpages(cd.resources().calendars(), 'list', 'items', + page_message=page_message, message_attribute='resourceId', customer=GC_Values[GC_CUSTOMER_ID], fields=fields) for resource in resources: - if u'featureInstances' in resource: - resource[u'featureInstances'] = u','.join([a_feature[u'feature'][u'name'] for a_feature in resource.pop(u'featureInstances')]) - if u'buildingId' in resource: - resource[u'buildingName'] = _getBuildingNameById(cd, resource[u'buildingId']) - resource[u'buildingId'] = u'id:{0}'.format(resource[u'buildingId']) + if 'featureInstances' in resource: + resource['featureInstances'] = ','.join([a_feature['feature']['name'] for a_feature in resource.pop('featureInstances')]) + if 'buildingId' in resource: + resource['buildingName'] = _getBuildingNameById(cd, resource['buildingId']) + resource['buildingId'] = 'id:{0}'.format(resource['buildingId']) resUnit = {} for field in fieldsList: - resUnit[fieldsTitles[field]] = resource.get(field, u'') + resUnit[fieldsTitles[field]] = resource.get(field, '') csvRows.append(resUnit) - sortCSVTitles([u'resourceId', u'resourceName', u'resourceEmail'], titles) - writeCSVfile(csvRows, titles, u'Resources', todrive) + sortCSVTitles(['resourceId', 'resourceName', 'resourceEmail'], titles) + writeCSVfile(csvRows, titles, 'Resources', todrive) -def shlexSplitList(entity, dataDelimiter=u' ,'): +def shlexSplitList(entity, dataDelimiter=' ,'): lexer = shlex.shlex(entity, posix=True) lexer.whitespace = dataDelimiter lexer.whitespace_split = True return list(lexer) def getQueries(myarg, argstr): - if myarg == u'query': + if myarg == 'query': return [argstr] else: return shlexSplitList(argstr) def _getRoleVerification(memberRoles, fields): if memberRoles and memberRoles.find(ROLE_MEMBER) != -1: - return (set(memberRoles.split(u',')), None, fields if fields.find(u'role') != -1 else fields[:-1]+u',role)') + return (set(memberRoles.split(',')), None, fields if fields.find('role') != -1 else fields[:-1]+',role)') else: return (set(), memberRoles, fields) @@ -12311,65 +12321,65 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No entity_type = sys.argv[1].lower() if entity is None: entity = sys.argv[2] - cd = buildGAPIObject(u'directory') - if entity_type == u'user': + cd = buildGAPIObject('directory') + if entity_type == 'user': users = [entity,] - elif entity_type == u'users': - users = entity.replace(u',', u' ').split() - elif entity_type in [u'group', u'group_ns', u'group_susp']: - if entity_type == u'group_ns': + elif entity_type == 'users': + users = entity.replace(',', ' ').split() + elif entity_type in ['group', 'group_ns', 'group_susp']: + if entity_type == 'group_ns': checkSuspended = False - elif entity_type == u'group_susp': + elif entity_type == 'group_susp': checkSuspended = True got_uids = True group = entity if member_type is None: - member_type_message = u'all members' + member_type_message = 'all members' else: - member_type_message = u'%ss' % member_type.lower() + member_type_message = '%ss' % member_type.lower() group = normalizeEmailAddressOrUID(group) page_message = None if not silent: - sys.stderr.write(u"Getting %s of %s (may take some time for large groups)...\n" % (member_type_message, group)) - page_message = u'Got %%%%total_items%%%% %s...' % member_type_message - validRoles, listRoles, listFields = _getRoleVerification(member_type, u'nextPageToken,members(email,id,type,status)') - members = callGAPIpages(cd.members(), u'list', u'members', page_message=page_message, + sys.stderr.write("Getting %s of %s (may take some time for large groups)...\n" % (member_type_message, group)) + page_message = 'Got %%%%total_items%%%% %s...' % member_type_message + validRoles, listRoles, listFields = _getRoleVerification(member_type, 'nextPageToken,members(email,id,type,status)') + members = callGAPIpages(cd.members(), 'list', 'members', page_message=page_message, groupKey=group, roles=listRoles, fields=listFields, maxResults=GC_Values[GC_MEMBER_MAX_RESULTS]) users = [] for member in members: - if (((not groupUserMembersOnly) or (member[u'type'] == u'USER')) and - (not validRoles or member.get(u'role', ROLE_MEMBER) in validRoles) and - (checkSuspended is None or (not checkSuspended and member[u'status'] != u'SUSPENDED') or (checkSuspended and member[u'status'] == u'SUSPENDED'))): - users.append(member.get(u'email', member[u'id'])) - elif entity_type in [u'ou', u'org', u'ou_ns', u'org_ns', u'ou_susp', u'org_susp',]: - if entity_type in [u'ou_ns', u'org_ns']: + if (((not groupUserMembersOnly) or (member['type'] == 'USER')) and + (not validRoles or member.get('role', ROLE_MEMBER) in validRoles) and + (checkSuspended is None or (not checkSuspended and member['status'] != 'SUSPENDED') or (checkSuspended and member['status'] == 'SUSPENDED'))): + users.append(member.get('email', member['id'])) + elif entity_type in ['ou', 'org', 'ou_ns', 'org_ns', 'ou_susp', 'org_susp',]: + if entity_type in ['ou_ns', 'org_ns']: checkSuspended = False - elif entity_type in [u'ou_susp', u'org_susp']: + elif entity_type in ['ou_susp', 'org_susp']: checkSuspended = True got_uids = True ou = makeOrgUnitPathAbsolute(entity) users = [] - if ou.startswith(u'id:'): - ou = callGAPI(cd.orgunits(), u'get', - customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=ou, fields=u'orgUnitPath')[u'orgUnitPath'] + if ou.startswith('id:'): + ou = callGAPI(cd.orgunits(), 'get', + customerId=GC_Values[GC_CUSTOMER_ID], orgUnitPath=ou, fields='orgUnitPath')['orgUnitPath'] query = orgUnitPathQuery(ou, checkSuspended) page_message = None if not silent: - printGettingAllItems(u'Users', query) - page_message = u'Got %%total_items%% Users...' - members = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, - customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail,orgUnitPath)', + printGettingAllItems('Users', query) + page_message = 'Got %%total_items%% Users...' + members = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,users(primaryEmail,orgUnitPath)', query=query, maxResults=GC_Values[GC_USER_MAX_RESULTS]) ou = ou.lower() for member in members: - if ou == member.get(u'orgUnitPath', u'').lower(): - users.append(member[u'primaryEmail']) + if ou == member.get('orgUnitPath', '').lower(): + users.append(member['primaryEmail']) if not silent: - sys.stderr.write(u"%s Users are directly in the OU.\n" % len(users)) - elif entity_type in [u'ou_and_children', u'ou_and_child', u'ou_and_children_ns', u'ou_and_child_ns', u'ou_and_children_susp', u'ou_and_child_susp']: - if entity_type in [u'ou_and_children_ns', u'ou_and_child_ns']: + sys.stderr.write("%s Users are directly in the OU.\n" % len(users)) + elif entity_type in ['ou_and_children', 'ou_and_child', 'ou_and_children_ns', 'ou_and_child_ns', 'ou_and_children_susp', 'ou_and_child_susp']: + if entity_type in ['ou_and_children_ns', 'ou_and_child_ns']: checkSuspended = False - elif entity_type in [u'ou_and_children_susp', u'ou_and_child_susp']: + elif entity_type in ['ou_and_children_susp', 'ou_and_child_susp']: checkSuspended = True got_uids = True ou = makeOrgUnitPathAbsolute(entity) @@ -12377,17 +12387,17 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No query = orgUnitPathQuery(ou, checkSuspended) page_message = None if not silent: - printGettingAllItems(u'Users', query) - page_message = u'Got %%total_items%% Users...' - members = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, - customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail)', + printGettingAllItems('Users', query) + page_message = 'Got %%total_items%% Users...' + members = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,users(primaryEmail)', query=query, maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in members: - users.append(member[u'primaryEmail']) + users.append(member['primaryEmail']) if not silent: - sys.stderr.write(u"done.\r\n") - elif entity_type in [u'query', u'queries']: - if entity_type == u'query': + sys.stderr.write("done.\r\n") + elif entity_type in ['query', 'queries']: + if entity_type == 'query': queries = [entity] else: queries = shlexSplitList(entity) @@ -12396,21 +12406,21 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No usersSet = set() for query in queries: if not silent: - printGettingAllItems(u'Users', query) - page_message = u'Got %%total_items%% Users...' - members = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, - customer=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,users(primaryEmail,suspended)', + printGettingAllItems('Users', query) + page_message = 'Got %%total_items%% Users...' + members = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, + customer=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,users(primaryEmail,suspended)', query=query, maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in members: - email = member[u'primaryEmail'] - if (checkSuspended is None or checkSuspended == member[u'suspended']) and email not in usersSet: + email = member['primaryEmail'] + if (checkSuspended is None or checkSuspended == member['suspended']) and email not in usersSet: usersSet.add(email) users.append(email) if not silent: - sys.stderr.write(u"done.\r\n") - elif entity_type in [u'license', u'licenses', u'licence', u'licences']: - users = doPrintLicenses(returnFields=u'userId', skus=entity.split(u',')) - elif entity_type in [u'file', u'crosfile']: + sys.stderr.write("done.\r\n") + elif entity_type in ['license', 'licenses', 'licence', 'licences']: + users = doPrintLicenses(returnFields='userId', skus=entity.split(',')) + elif entity_type in ['file', 'crosfile']: users = [] f = openFile(entity) for row in f: @@ -12418,15 +12428,15 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if user: users.append(user) closeFile(f) - if entity_type == u'crosfile': - entity = u'cros' - elif entity_type in [u'csv', u'csvfile', u'croscsv', u'croscsvfile']: + if entity_type == 'crosfile': + entity = 'cros' + elif entity_type in ['csv', 'csvfile', 'croscsv', 'croscsvfile']: drive, filenameColumn = os.path.splitdrive(entity) - if filenameColumn.find(u':') == -1: - systemErrorExit(2, u'Expected {0} FileName:FieldName'.format(entity_type)) - (filename, column) = filenameColumn.split(u':') + if filenameColumn.find(':') == -1: + systemErrorExit(2, 'Expected {0} FileName:FieldName'.format(entity_type)) + (filename, column) = filenameColumn.split(':') f = openFile(drive+filename, mode='rbU') - input_file = csv.DictReader(f, restval=u'') + input_file = csv.DictReader(f, restval='') if column not in input_file.fieldnames: csvFieldErrorExit(column, input_file.fieldnames) users = [] @@ -12435,62 +12445,62 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No if user: users.append(user) closeFile(f) - if entity_type in [u'croscsv', u'croscsvfile']: - entity = u'cros' - elif entity_type in [u'courseparticipants', u'teachers', u'students']: - croom = buildGAPIObject(u'classroom') + if entity_type in ['croscsv', 'croscsvfile']: + entity = 'cros' + elif entity_type in ['courseparticipants', 'teachers', 'students']: + croom = buildGAPIObject('classroom') users = [] entity = addCourseIdScope(entity) - if entity_type in [u'courseparticipants', u'teachers']: - page_message = u'Got %%total_items%% Teachers...' - teachers = callGAPIpages(croom.courses().teachers(), u'list', u'teachers', page_message=page_message, courseId=entity) + if entity_type in ['courseparticipants', 'teachers']: + page_message = 'Got %%total_items%% Teachers...' + teachers = callGAPIpages(croom.courses().teachers(), 'list', 'teachers', page_message=page_message, courseId=entity) for teacher in teachers: - email = teacher[u'profile'].get(u'emailAddress', None) + email = teacher['profile'].get('emailAddress', None) if email: users.append(email) - if entity_type in [u'courseparticipants', u'students']: - page_message = u'Got %%total_items%% Students...' - students = callGAPIpages(croom.courses().students(), u'list', u'students', page_message=page_message, courseId=entity) + if entity_type in ['courseparticipants', 'students']: + page_message = 'Got %%total_items%% Students...' + students = callGAPIpages(croom.courses().students(), 'list', 'students', page_message=page_message, courseId=entity) for student in students: - email = student[u'profile'].get(u'emailAddress', None) + email = student['profile'].get('emailAddress', None) if email: users.append(email) - elif entity_type == u'all': + elif entity_type == 'all': got_uids = True users = [] entity = entity.lower() - if entity == u'users': - query = u'isSuspended=False' + if entity == 'users': + query = 'isSuspended=False' if not silent: - printGettingAllItems(u'Users', None) - page_message = u'Got %%total_items%% Users...' - all_users = callGAPIpages(cd.users(), u'list', u'users', page_message=page_message, + printGettingAllItems('Users', None) + page_message = 'Got %%total_items%% Users...' + all_users = callGAPIpages(cd.users(), 'list', 'users', page_message=page_message, customer=GC_Values[GC_CUSTOMER_ID], query=query, - fields=u'nextPageToken,users(primaryEmail)', maxResults=GC_Values[GC_USER_MAX_RESULTS]) + fields='nextPageToken,users(primaryEmail)', maxResults=GC_Values[GC_USER_MAX_RESULTS]) for member in all_users: - users.append(member[u'primaryEmail']) + users.append(member['primaryEmail']) if not silent: - sys.stderr.write(u"done getting %s Users.\r\n" % len(users)) - elif entity == u'cros': + sys.stderr.write("done getting %s Users.\r\n" % len(users)) + elif entity == 'cros': if not silent: - printGettingAllItems(u'CrOS Devices', None) - page_message = u'Got %%total_items%% CrOS Devices...' - all_cros = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', page_message=page_message, - customerId=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,chromeosdevices(deviceId)', + printGettingAllItems('CrOS Devices', None) + page_message = 'Got %%total_items%% CrOS Devices...' + all_cros = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + customerId=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,chromeosdevices(deviceId)', maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) for member in all_cros: - users.append(member[u'deviceId']) + users.append(member['deviceId']) if not silent: - sys.stderr.write(u"done getting %s CrOS Devices.\r\n" % len(users)) + sys.stderr.write("done getting %s CrOS Devices.\r\n" % len(users)) else: systemErrorExit(3, '%s is not a valid argument for "gam all"' % entity) - elif entity_type == u'cros': - users = entity.replace(u',', u' ').split() - entity = u'cros' - elif entity_type in [u'crosquery', u'crosqueries', u'cros_sn']: - if entity_type == u'cros_sn': - queries = [u'id:{0}'.format(sn) for sn in shlexSplitList(entity)] - elif entity_type == u'crosqueries': + elif entity_type == 'cros': + users = entity.replace(',', ' ').split() + entity = 'cros' + elif entity_type in ['crosquery', 'crosqueries', 'cros_sn']: + if entity_type == 'cros_sn': + queries = ['id:{0}'.format(sn) for sn in shlexSplitList(entity)] + elif entity_type == 'crosqueries': queries = shlexSplitList(entity) else: queries = [entity] @@ -12498,29 +12508,29 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No usersSet = set() for query in queries: if not silent: - printGettingAllItems(u'CrOS Devices', query) - page_message = u'Got %%total_items%% CrOS Devices...' - members = callGAPIpages(cd.chromeosdevices(), u'list', u'chromeosdevices', page_message=page_message, - customerId=GC_Values[GC_CUSTOMER_ID], fields=u'nextPageToken,chromeosdevices(deviceId)', + printGettingAllItems('CrOS Devices', query) + page_message = 'Got %%total_items%% CrOS Devices...' + members = callGAPIpages(cd.chromeosdevices(), 'list', 'chromeosdevices', page_message=page_message, + customerId=GC_Values[GC_CUSTOMER_ID], fields='nextPageToken,chromeosdevices(deviceId)', query=query, maxResults=GC_Values[GC_DEVICE_MAX_RESULTS]) for member in members: - deviceId = member[u'deviceId'] + deviceId = member['deviceId'] if deviceId not in usersSet: usersSet.add(deviceId) users.append(deviceId) if not silent: - sys.stderr.write(u"done.\r\n") - entity = u'cros' + sys.stderr.write("done.\r\n") + entity = 'cros' else: systemErrorExit(2, '%s is not a valid argument for "gam"' % entity_type) full_users = list() - if entity != u'cros' and not got_uids: + if entity != 'cros' and not got_uids: for user in users: cg = UID_PATTERN.match(user) if cg: full_users.append(cg.group(1)) - elif user != u'*' and user != GC_Values[GC_CUSTOMER_ID] and user.find(u'@') == -1: - full_users.append(u'%s@%s' % (user, GC_Values[GC_DOMAIN])) + elif user != '*' and user != GC_Values[GC_CUSTOMER_ID] and user.find('@') == -1: + full_users.append('%s@%s' % (user, GC_Values[GC_DOMAIN])) else: full_users.append(user) else: @@ -12535,18 +12545,18 @@ def OAuthInfo(): credentials = getValidOauth2TxtCredentials() credentials.user_agent = GAM_INFO access_token = credentials.access_token - print u"\nOAuth File: %s" % GC_Values[GC_OAUTH2_TXT] - oa2 = buildGAPIObject(u'oauth2') - token_info = callGAPI(oa2, u'tokeninfo', access_token=access_token) - print u"Client ID: %s" % token_info[u'issued_to'] + print("\nOAuth File: %s" % GC_Values[GC_OAUTH2_TXT]) + oa2 = buildGAPIObject('oauth2') + token_info = callGAPI(oa2, 'tokeninfo', access_token=access_token) + print("Client ID: %s" % token_info['issued_to']) if credentials is not None: - print u"Secret: %s" % credentials.client_secret - scopes = token_info[u'scope'].split(u' ') - print u'Scopes (%s):' % len(scopes) + print("Secret: %s" % credentials.client_secret) + scopes = token_info['scope'].split(' ') + print('Scopes (%s):' % len(scopes)) for scope in sorted(scopes): - print u' %s' % scope + print(' %s' % scope) if credentials is not None: - print u'G Suite Admin: %s' % _getValueFromOAuth(u'email', credentials=credentials) + print('G Suite Admin: %s' % _getValueFromOAuth('email', credentials=credentials)) def doDeleteOAuth(): storage, credentials = getOauth2TxtStorageCredentials() @@ -12556,14 +12566,14 @@ def doDeleteOAuth(): try: credentials.revoke_uri = oauth2client.GOOGLE_REVOKE_URI except AttributeError: - systemErrorExit(1, u'Authorization doesn\'t exist') - sys.stderr.write(u'This OAuth token will self-destruct in 3...') + systemErrorExit(1, 'Authorization doesn\'t exist') + sys.stderr.write('This OAuth token will self-destruct in 3...') time.sleep(1) - sys.stderr.write(u'2...') + sys.stderr.write('2...') time.sleep(1) - sys.stderr.write(u'1...') + sys.stderr.write('1...') time.sleep(1) - sys.stderr.write(u'boom!\n') + sys.stderr.write('boom!\n') try: credentials.revoke(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL])) except oauth2client.client.TokenRevokeError as e: @@ -12577,38 +12587,38 @@ def doRequestOAuth(login_hint=None): flags = cmd_flags(noLocalWebserver=GC_Values[GC_NO_BROWSER]) scopes = getScopesFromUser() if scopes is None: - systemErrorExit(0, u'') + systemErrorExit(0, '') client_id, client_secret = getOAuthClientIDAndSecret() login_hint = _getValidateLoginHint(login_hint) flow = oauth2client.client.OAuth2WebServerFlow(client_id=client_id, client_secret=client_secret, scope=scopes, redirect_uri=oauth2client.client.OOB_CALLBACK_URN, - user_agent=GAM_INFO, response_type=u'code', login_hint=login_hint) + user_agent=GAM_INFO, response_type='code', login_hint=login_hint) try: credentials = oauth2client.tools.run_flow(flow=flow, storage=storage, flags=flags, http=http) except httplib2.CertificateValidationUnsupported: noPythonSSLExit() else: - print u'It looks like you\'ve already authorized GAM. Refusing to overwrite existing file:\n\n%s' % GC_Values[GC_OAUTH2_TXT] + print('It looks like you\'ve already authorized GAM. Refusing to overwrite existing file:\n\n%s' % GC_Values[GC_OAUTH2_TXT]) def getOAuthClientIDAndSecret(): """Retrieves the OAuth client ID and client secret from JSON.""" - MISSING_CLIENT_SECRETS_MESSAGE = u'''To use GAM you need to create an API project. Please run: + MISSING_CLIENT_SECRETS_MESSAGE = '''To use GAM you need to create an API project. Please run: gam create project ''' filename = GC_Values[GC_CLIENT_SECRETS_JSON] - cs_data = readFile(filename, mode=u'rb', continueOnError=True, displayError=True, encoding=None) + cs_data = readFile(filename, mode='rb', continueOnError=True, displayError=True, encoding=None) if not cs_data: systemErrorExit(14, MISSING_CLIENT_SECRETS_MESSAGE) try: cs_json = json.loads(cs_data) - client_id = cs_json[u'installed'][u'client_id'] + client_id = cs_json['installed']['client_id'] # chop off .apps.googleusercontent.com suffix as it's not needed # and we need to keep things short for the Auth URL. - client_id = re.sub(r'\.apps\.googleusercontent\.com$', u'', client_id) - client_secret = cs_json[u'installed'][u'client_secret'] + client_id = re.sub(r'\.apps\.googleusercontent\.com$', '', client_id) + client_secret = cs_json['installed']['client_secret'] except (ValueError, IndexError, KeyError): - systemErrorExit(3, u'the format of your client secrets file:\n\n%s\n\n' + systemErrorExit(3, 'the format of your client secrets file:\n\n%s\n\n' 'is incorrect. Please recreate the file.' % filename) return (client_id, client_secret) @@ -12616,90 +12626,90 @@ class cmd_flags(object): def __init__(self, noLocalWebserver): self.short_url = True self.noauth_local_webserver = noLocalWebserver - self.logging_level = u'ERROR' - self.auth_host_name = u'localhost' + self.logging_level = 'ERROR' + self.auth_host_name = 'localhost' self.auth_host_port = [8080, 9090] OAUTH2_SCOPES = [ - {u'name': u'Classroom API - counts as 5 scopes', - u'subscopes': [], - u'scopes': [u'https://www.googleapis.com/auth/classroom.rosters', - u'https://www.googleapis.com/auth/classroom.courses', - u'https://www.googleapis.com/auth/classroom.profile.emails', - u'https://www.googleapis.com/auth/classroom.profile.photos', - u'https://www.googleapis.com/auth/classroom.guardianlinks.students']}, - {u'name': u'Cloud Print API', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/cloudprint'}, - {u'name': u'Data Transfer API', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.datatransfer'}, - {u'name': u'Directory API - Chrome OS Devices', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.device.chromeos'}, - {u'name': u'Directory API - Customers', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.customer'}, - {u'name': u'Directory API - Domains', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.domain'}, - {u'name': u'Directory API - Groups', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.group'}, - {u'name': u'Directory API - Mobile Devices', - u'subscopes': [u'readonly', u'action'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.device.mobile'}, - {u'name': u'Directory API - Organizational Units', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.orgunit'}, - {u'name': u'Directory API - Resource Calendars', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.resource.calendar'}, - {u'name': u'Directory API - Roles', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.rolemanagement'}, - {u'name': u'Directory API - User Schemas', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.userschema'}, - {u'name': u'Directory API - User Security', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.user.security'}, - {u'name': u'Directory API - Users', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/admin.directory.user'}, - {u'name': u'Group Settings API', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/apps.groups.settings'}, - {u'name': u'License Manager API', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/apps.licensing'}, - {u'name': u'Pub / Sub API', - u'subscopes': [], - u'offByDefault': True, - u'scopes': u'https://www.googleapis.com/auth/pubsub'}, - {u'name': u'Reports API - Audit Reports', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/admin.reports.audit.readonly'}, - {u'name': u'Reports API - Usage Reports', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/admin.reports.usage.readonly'}, - {u'name': u'Reseller API', - u'subscopes': [], - u'offByDefault': True, - u'scopes': u'https://www.googleapis.com/auth/apps.order'}, - {u'name': u'Site Verification API', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/siteverification'}, - {u'name': u'Vault Matters and Holds API', - u'subscopes': [u'readonly'], - u'scopes': u'https://www.googleapis.com/auth/ediscovery'}, - {u'name': u'Cloud Storage (Vault Export - read only)', - u'subscopes': [], - u'scopes': u'https://www.googleapis.com/auth/devstorage.read_only'}, - {u'name': u'User Profile (Email address - read only)', - u'subscopes': [], - u'scopes': u'email', - u'required': True}, + {'name': 'Classroom API - counts as 5 scopes', + 'subscopes': [], + 'scopes': ['https://www.googleapis.com/auth/classroom.rosters', + 'https://www.googleapis.com/auth/classroom.courses', + 'https://www.googleapis.com/auth/classroom.profile.emails', + 'https://www.googleapis.com/auth/classroom.profile.photos', + 'https://www.googleapis.com/auth/classroom.guardianlinks.students']}, + {'name': 'Cloud Print API', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/cloudprint'}, + {'name': 'Data Transfer API', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.datatransfer'}, + {'name': 'Directory API - Chrome OS Devices', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.device.chromeos'}, + {'name': 'Directory API - Customers', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.customer'}, + {'name': 'Directory API - Domains', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.domain'}, + {'name': 'Directory API - Groups', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.group'}, + {'name': 'Directory API - Mobile Devices', + 'subscopes': ['readonly', 'action'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.device.mobile'}, + {'name': 'Directory API - Organizational Units', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.orgunit'}, + {'name': 'Directory API - Resource Calendars', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.resource.calendar'}, + {'name': 'Directory API - Roles', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.rolemanagement'}, + {'name': 'Directory API - User Schemas', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.userschema'}, + {'name': 'Directory API - User Security', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.user.security'}, + {'name': 'Directory API - Users', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/admin.directory.user'}, + {'name': 'Group Settings API', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/apps.groups.settings'}, + {'name': 'License Manager API', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/apps.licensing'}, + {'name': 'Pub / Sub API', + 'subscopes': [], + 'offByDefault': True, + 'scopes': 'https://www.googleapis.com/auth/pubsub'}, + {'name': 'Reports API - Audit Reports', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/admin.reports.audit.readonly'}, + {'name': 'Reports API - Usage Reports', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/admin.reports.usage.readonly'}, + {'name': 'Reseller API', + 'subscopes': [], + 'offByDefault': True, + 'scopes': 'https://www.googleapis.com/auth/apps.order'}, + {'name': 'Site Verification API', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/siteverification'}, + {'name': 'Vault Matters and Holds API', + 'subscopes': ['readonly'], + 'scopes': 'https://www.googleapis.com/auth/ediscovery'}, + {'name': 'Cloud Storage (Vault Export - read only)', + 'subscopes': [], + 'scopes': 'https://www.googleapis.com/auth/devstorage.read_only'}, + {'name': 'User Profile (Email address - read only)', + 'subscopes': [], + 'scopes': 'email', + 'required': True}, ] def getScopesFromUser(menu_options=None): @@ -13028,7 +13038,7 @@ Append an 'r' to grant read-only access or an 'a' to grant action-only access. return ('Please enter 0-%d[%s] or %s: ' % (len(self._options)-1, # Keep the menu options 0-based '|'.join(restriction_choices), - '|'.join(ScopeSelectionMenu.MENU_CHOICE.values()))) + '|'.join(list(ScopeSelectionMenu.MENU_CHOICE.values())))) def run(self): """Displays the ScopeSelectionMenu to the user and prompts for input. @@ -13053,7 +13063,7 @@ Append an 'r' to grant read-only access or an 'a' to grant action-only access. sys.stdout.write(colored_error) error_message = None # Clear the pending error message - user_input = raw_input(self.get_prompt_text()) + user_input = input(self.get_prompt_text()) try: prompt_again = self._process_menu_input(user_input) if not prompt_again: @@ -13103,9 +13113,7 @@ Append an 'r' to grant read-only access or an 'a' to grant action-only access. # Find the restriction that the user intended to apply. if restriction_command != '': - matching_restrictions = filter( - lambda r: r.startswith(restriction_command), - selected_option.supported_restrictions) + matching_restrictions = [r for r in selected_option.supported_restrictions if r.startswith(restriction_command)] if len(matching_restrictions) < 1: raise ScopeSelectionMenu.MenuChoiceError( 'Scope "%s" does not support "%s" mode!' % ( @@ -13183,16 +13191,16 @@ def run_batch(items): return num_worker_threads = min(len(items), GC_Values[GC_NUM_THREADS]) pool = Pool(num_worker_threads, init_gam_worker) - sys.stderr.write(u'Using %s processes...\n' % num_worker_threads) + sys.stderr.write('Using %s processes...\n' % num_worker_threads) try: results = [] for item in items: - if item[0] == u'commit-batch': - sys.stderr.write(u'commit-batch - waiting for running processes to finish before proceeding\n') + if item[0] == 'commit-batch': + sys.stderr.write('commit-batch - waiting for running processes to finish before proceeding\n') pool.close() pool.join() pool = Pool(num_worker_threads, init_gam_worker) - sys.stderr.write(u'commit-batch - running processes finished, proceeding\n') + sys.stderr.write('commit-batch - running processes finished, proceeding\n') continue results.append(pool.apply_async(ProcessGAMCommandMulti, [item])) pool.close() @@ -13207,7 +13215,7 @@ def run_batch(items): break i += 1 if i == 20: - print u'Finished %s of %s processes.' % (num_done, num_total) + print('Finished %s of %s processes.' % (num_done, num_total)) i = 1 time.sleep(1) except KeyboardInterrupt: @@ -13248,7 +13256,7 @@ def getSubFields(i, fieldNames): csvFieldErrorExit(fieldName, fieldNames) pos = match.end() GAM_argv.append(myarg) - elif myarg[0] == u'~': + elif myarg[0] == '~': fieldName = myarg[1:] if fieldName in fieldNames: subFields[GAM_argvI] = [(fieldName, 0, len(myarg))] @@ -13263,9 +13271,9 @@ def getSubFields(i, fieldNames): # def processSubFields(GAM_argv, row, subFields): argv = GAM_argv[:] - for GAM_argvI, fields in subFields.iteritems(): + for GAM_argvI, fields in subFields.items(): oargv = argv[GAM_argvI][:] - argv[GAM_argvI] = u'' + argv[GAM_argvI] = '' pos = 0 for field in fields: argv[GAM_argvI] += oargv[pos:field[1]] @@ -13279,17 +13287,17 @@ def runCmdForUsers(cmd, users, default_to_batch=False, **kwargs): if default_to_batch and len(users) > 1: items = [] for user in users: - items.append([u'gam', u'user', user] + sys.argv[3:]) + items.append(['gam', 'user', user] + sys.argv[3:]) run_batch(items) sys.exit(0) else: cmd(users, **kwargs) def resetDefaultEncodingToUTF8(): - if sys.getdefaultencoding().upper() != u'UTF-8': + if sys.getdefaultencoding().upper() != 'UTF-8': reload(sys) - if hasattr(sys, u'setdefaultencoding'): - sys.setdefaultencoding(u'UTF-8') + if hasattr(sys, 'setdefaultencoding'): + sys.setdefaultencoding('UTF-8') def ProcessGAMCommandMulti(args): resetDefaultEncodingToUTF8() @@ -13303,747 +13311,747 @@ def ProcessGAMCommand(args): try: SetGlobalVariables() command = sys.argv[1].lower() - if command == u'batch': + if command == 'batch': i = 2 filename = sys.argv[i] i, encoding = getCharSet(i+1) f = openFile(filename) - batchFile = UTF8Recoder(f, encoding) if encoding != u'utf-8' else f + batchFile = UTF8Recoder(f, encoding) if encoding != 'utf-8' else f items = [] errors = 0 for line in batchFile: try: argv = shlex.split(line) except ValueError as e: - sys.stderr.write(utils.convertUTF8(u'Command: >>>{0}<<<\n'.format(line.strip()))) - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, str(e))) + sys.stderr.write(utils.convertUTF8('Command: >>>{0}<<<\n'.format(line.strip()))) + sys.stderr.write('{0}{1}\n'.format(ERROR_PREFIX, str(e))) errors += 1 continue if len(argv) > 0: cmd = argv[0].strip().lower() - if (not cmd) or cmd.startswith(u'#') or ((len(argv) == 1) and (cmd != u'commit-batch')): + if (not cmd) or cmd.startswith('#') or ((len(argv) == 1) and (cmd != 'commit-batch')): continue - if cmd == u'gam': + if cmd == 'gam': items.append(argv) - elif cmd == u'commit-batch': + elif cmd == 'commit-batch': items.append([cmd]) else: - sys.stderr.write(utils.convertUTF8(u'Command: >>>{0}<<<\n'.format(line.strip()))) - sys.stderr.write(u'{0}Invalid: Expected \n'.format(ERROR_PREFIX)) + sys.stderr.write(utils.convertUTF8('Command: >>>{0}<<<\n'.format(line.strip()))) + sys.stderr.write('{0}Invalid: Expected \n'.format(ERROR_PREFIX)) errors += 1 closeFile(f) if errors == 0: run_batch(items) sys.exit(0) else: - systemErrorExit(2, u'batch file: {0}, not processed, {1} error{2}'.format(filename, errors, [u'', u's'][errors != 1])) - elif command == u'csv': + systemErrorExit(2, 'batch file: {0}, not processed, {1} error{2}'.format(filename, errors, ['', 's'][errors != 1])) + elif command == 'csv': if httplib2.debuglevel > 0: - systemErrorExit(1, u'CSV commands are not compatible with debug. Delete debug.gam and try again.') + systemErrorExit(1, 'CSV commands are not compatible with debug. Delete debug.gam and try again.') i = 2 filename = sys.argv[i] i, encoding = getCharSet(i+1) - f = openFile(filename, mode=u'rbU') + f = openFile(filename, mode='rbU') csvFile = UnicodeDictReader(f, encoding=encoding) - if (i == len(sys.argv)) or (sys.argv[i].lower() != u'gam') or (i+1 == len(sys.argv)): + if (i == len(sys.argv)) or (sys.argv[i].lower() != 'gam') or (i+1 == len(sys.argv)): systemErrorExit(3, '"gam csv " must be followed by a full GAM command...') i += 1 GAM_argv, subFields = getSubFields(i, csvFile.fieldnames) items = [] for row in csvFile: - items.append([u'gam']+processSubFields(GAM_argv, row, subFields)) + items.append(['gam']+processSubFields(GAM_argv, row, subFields)) closeFile(f) run_batch(items) sys.exit(0) - elif command == u'version': + elif command == 'version': doGAMVersion() sys.exit(0) - elif command == u'create': + elif command == 'create': argument = sys.argv[2].lower() - if argument == u'user': + if argument == 'user': doCreateUser() - elif argument == u'group': + elif argument == 'group': doCreateGroup() - elif argument in [u'nickname', u'alias']: + elif argument in ['nickname', 'alias']: doCreateAlias() - elif argument in [u'org', u'ou']: + elif argument in ['org', 'ou']: doCreateOrg() - elif argument == u'resource': + elif argument == 'resource': doCreateResourceCalendar() - elif argument in [u'verify', u'verification']: + elif argument in ['verify', 'verification']: doSiteVerifyShow() - elif argument == u'schema': + elif argument == 'schema': doCreateOrUpdateUserSchema(False) - elif argument in [u'course', u'class']: + elif argument in ['course', 'class']: doCreateCourse() - elif argument in [u'transfer', u'datatransfer']: + elif argument in ['transfer', 'datatransfer']: doCreateDataTransfer() - elif argument == u'domain': + elif argument == 'domain': doCreateDomain() - elif argument in [u'domainalias', u'aliasdomain']: + elif argument in ['domainalias', 'aliasdomain']: doCreateDomainAlias() - elif argument == u'admin': + elif argument == 'admin': doCreateAdmin() - elif argument in [u'guardianinvite', u'inviteguardian', u'guardian']: + elif argument in ['guardianinvite', 'inviteguardian', 'guardian']: doInviteGuardian() - elif argument in [u'project', u'apiproject']: + elif argument in ['project', 'apiproject']: doCreateProject() - elif argument in [u'resoldcustomer', u'resellercustomer']: + elif argument in ['resoldcustomer', 'resellercustomer']: doCreateResoldCustomer() - elif argument in [u'resoldsubscription', u'resellersubscription']: + elif argument in ['resoldsubscription', 'resellersubscription']: doCreateResoldSubscription() - elif argument in [u'matter', u'vaultmatter']: + elif argument in ['matter', 'vaultmatter']: doCreateVaultMatter() - elif argument in [u'hold', u'vaulthold']: + elif argument in ['hold', 'vaulthold']: doCreateVaultHold() - elif argument in [u'export', u'vaultexport']: + elif argument in ['export', 'vaultexport']: doCreateVaultExport() - elif argument in [u'building']: + elif argument in ['building']: doCreateBuilding() - elif argument in [u'feature']: + elif argument in ['feature']: doCreateFeature() - elif argument in [u'alertfeedback']: + elif argument in ['alertfeedback']: doCreateAlertFeedback() else: systemErrorExit(2, '%s is not a valid argument for "gam create"' % argument) sys.exit(0) - elif command == u'use': + elif command == 'use': argument = sys.argv[2].lower() - if argument in [u'project', u'apiproject']: + if argument in ['project', 'apiproject']: doUseProject() else: systemErrorExit(2, '%s is not a valid argument for "gam use"' % argument) sys.exit(0) - elif command == u'update': + elif command == 'update': argument = sys.argv[2].lower() - if argument == u'user': + if argument == 'user': doUpdateUser(None, 4) - elif argument == u'group': + elif argument == 'group': doUpdateGroup() - elif argument in [u'nickname', u'alias']: + elif argument in ['nickname', 'alias']: doUpdateAlias() - elif argument in [u'ou', u'org']: + elif argument in ['ou', 'org']: doUpdateOrg() - elif argument == u'resource': + elif argument == 'resource': doUpdateResourceCalendar() - elif argument == u'cros': + elif argument == 'cros': doUpdateCros() - elif argument == u'mobile': + elif argument == 'mobile': doUpdateMobile() - elif argument in [u'verify', u'verification']: + elif argument in ['verify', 'verification']: doSiteVerifyAttempt() - elif argument in [u'schema', u'schemas']: + elif argument in ['schema', 'schemas']: doCreateOrUpdateUserSchema(True) - elif argument in [u'course', u'class']: + elif argument in ['course', 'class']: doUpdateCourse() - elif argument in [u'printer', u'print']: + elif argument in ['printer', 'print']: doUpdatePrinter() - elif argument == u'domain': + elif argument == 'domain': doUpdateDomain() - elif argument == u'customer': + elif argument == 'customer': doUpdateCustomer() - elif argument in [u'resoldcustomer', u'resellercustomer']: + elif argument in ['resoldcustomer', 'resellercustomer']: doUpdateResoldCustomer() - elif argument in [u'resoldsubscription', u'resellersubscription']: + elif argument in ['resoldsubscription', 'resellersubscription']: doUpdateResoldSubscription() - elif argument in [u'matter', u'vaultmatter']: + elif argument in ['matter', 'vaultmatter']: doUpdateVaultMatter() - elif argument in [u'hold', u'vaulthold']: + elif argument in ['hold', 'vaulthold']: doUpdateVaultHold() - elif argument in [u'project', u'projects', u'apiproject']: + elif argument in ['project', 'projects', 'apiproject']: doUpdateProjects() - elif argument in [u'building']: + elif argument in ['building']: doUpdateBuilding() - elif argument in [u'feature']: + elif argument in ['feature']: doUpdateFeature() else: systemErrorExit(2, '%s is not a valid argument for "gam update"' % argument) sys.exit(0) - elif command == u'info': + elif command == 'info': argument = sys.argv[2].lower() - if argument == u'user': + if argument == 'user': doGetUserInfo() - elif argument == u'group': + elif argument == 'group': doGetGroupInfo() - elif argument == u'member': + elif argument == 'member': doGetMemberInfo() - elif argument in [u'nickname', u'alias']: + elif argument in ['nickname', 'alias']: doGetAliasInfo() - elif argument == u'instance': + elif argument == 'instance': doGetCustomerInfo() - elif argument in [u'org', u'ou']: + elif argument in ['org', 'ou']: doGetOrgInfo() - elif argument == u'resource': + elif argument == 'resource': doGetResourceCalendarInfo() - elif argument == u'cros': + elif argument == 'cros': doGetCrosInfo() - elif argument == u'mobile': + elif argument == 'mobile': doGetMobileInfo() - elif argument in [u'verify', u'verification']: + elif argument in ['verify', 'verification']: doGetSiteVerifications() - elif argument in [u'schema', u'schemas']: + elif argument in ['schema', 'schemas']: doGetUserSchema() - elif argument in [u'course', u'class']: + elif argument in ['course', 'class']: doGetCourseInfo() - elif argument in [u'printer', u'print']: + elif argument in ['printer', 'print']: doGetPrinterInfo() - elif argument in [u'transfer', u'datatransfer']: + elif argument in ['transfer', 'datatransfer']: doGetDataTransferInfo() - elif argument == u'customer': + elif argument == 'customer': doGetCustomerInfo() - elif argument == u'domain': + elif argument == 'domain': doGetDomainInfo() - elif argument in [u'domainalias', u'aliasdomain']: + elif argument in ['domainalias', 'aliasdomain']: doGetDomainAliasInfo() - elif argument in [u'resoldcustomer', u'resellercustomer']: + elif argument in ['resoldcustomer', 'resellercustomer']: doGetResoldCustomer() - elif argument in [u'resoldsubscription', u'resoldsubscriptions', u'resellersubscription', u'resellersubscriptions']: + elif argument in ['resoldsubscription', 'resoldsubscriptions', 'resellersubscription', 'resellersubscriptions']: doGetResoldSubscriptions() - elif argument in [u'matter', u'vaultmatter']: + elif argument in ['matter', 'vaultmatter']: doGetVaultMatterInfo() - elif argument in [u'hold', u'vaulthold']: + elif argument in ['hold', 'vaulthold']: doGetVaultHoldInfo() - elif argument in [u'export', u'vaultexport']: + elif argument in ['export', 'vaultexport']: doGetVaultExportInfo() - elif argument in [u'building']: + elif argument in ['building']: doGetBuildingInfo() else: systemErrorExit(2, '%s is not a valid argument for "gam info"' % argument) sys.exit(0) - elif command == u'cancel': + elif command == 'cancel': argument = sys.argv[2].lower() - if argument in [u'guardianinvitation', u'guardianinvitations']: + if argument in ['guardianinvitation', 'guardianinvitations']: doCancelGuardianInvitation() else: systemErrorExit(2, '%s is not a valid argument for "gam cancel"' % argument) sys.exit(0) - elif command == u'delete': + elif command == 'delete': argument = sys.argv[2].lower() - if argument == u'user': + if argument == 'user': doDeleteUser() - elif argument == u'group': + elif argument == 'group': doDeleteGroup() - elif argument in [u'nickname', u'alias']: + elif argument in ['nickname', 'alias']: doDeleteAlias() - elif argument == u'org': + elif argument == 'org': doDeleteOrg() - elif argument == u'resource': + elif argument == 'resource': doDeleteResourceCalendar() - elif argument == u'mobile': + elif argument == 'mobile': doDeleteMobile() - elif argument in [u'schema', u'schemas']: + elif argument in ['schema', 'schemas']: doDelSchema() - elif argument in [u'course', u'class']: + elif argument in ['course', 'class']: doDelCourse() - elif argument in [u'printer', u'printers']: + elif argument in ['printer', 'printers']: doDelPrinter() - elif argument == u'domain': + elif argument == 'domain': doDelDomain() - elif argument in [u'domainalias', u'aliasdomain']: + elif argument in ['domainalias', 'aliasdomain']: doDelDomainAlias() - elif argument == u'admin': + elif argument == 'admin': doDelAdmin() - elif argument in [u'guardian', u'guardians']: + elif argument in ['guardian', 'guardians']: doDeleteGuardian() - elif argument in [u'project', u'projects']: + elif argument in ['project', 'projects']: doDelProjects() - elif argument in [u'resoldsubscription', u'resellersubscription']: + elif argument in ['resoldsubscription', 'resellersubscription']: doDeleteResoldSubscription() - elif argument in [u'matter', u'vaultmatter']: + elif argument in ['matter', 'vaultmatter']: doUpdateVaultMatter(action=command) - elif argument in [u'hold', u'vaulthold']: + elif argument in ['hold', 'vaulthold']: doDeleteVaultHold() - elif argument in [u'export', u'vaultexport']: + elif argument in ['export', 'vaultexport']: doDeleteVaultExport() - elif argument in [u'building']: + elif argument in ['building']: doDeleteBuilding() - elif argument in [u'feature']: + elif argument in ['feature']: doDeleteFeature() - elif argument in [u'alert']: - doDeleteOrUndeleteAlert(u'delete') + elif argument in ['alert']: + doDeleteOrUndeleteAlert('delete') else: systemErrorExit(2, '%s is not a valid argument for "gam delete"' % argument) sys.exit(0) - elif command == u'undelete': + elif command == 'undelete': argument = sys.argv[2].lower() - if argument == u'user': + if argument == 'user': doUndeleteUser() - elif argument in [u'matter', u'vaultmatter']: + elif argument in ['matter', 'vaultmatter']: doUpdateVaultMatter(action=command) - elif argument == u'alert': - doDeleteOrUndeleteAlert(u'undelete') + elif argument == 'alert': + doDeleteOrUndeleteAlert('undelete') else: systemErrorExit(2, '%s is not a valid argument for "gam undelete"' % argument) sys.exit(0) - elif command in [u'close', u'reopen']: + elif command in ['close', 'reopen']: # close and reopen will have to be split apart if either takes a new argument argument = sys.argv[2].lower() - if argument in [u'matter', u'vaultmatter']: + if argument in ['matter', 'vaultmatter']: doUpdateVaultMatter(action=command) else: systemErrorExit(2, '%s is not a valid argument for "gam %s"' % (argument, command)) sys.exit(0) - elif command == u'print': - argument = sys.argv[2].lower().replace(u'-', u'') - if argument == u'users': + elif command == 'print': + argument = sys.argv[2].lower().replace('-', '') + if argument == 'users': doPrintUsers() - elif argument in [u'nicknames', u'aliases']: + elif argument in ['nicknames', 'aliases']: doPrintAliases() - elif argument == u'groups': + elif argument == 'groups': doPrintGroups() - elif argument in [u'groupmembers', u'groupsmembers']: + elif argument in ['groupmembers', 'groupsmembers']: doPrintGroupMembers() - elif argument in [u'orgs', u'ous']: + elif argument in ['orgs', 'ous']: doPrintOrgs() - elif argument == u'resources': + elif argument == 'resources': doPrintResourceCalendars() - elif argument == u'cros': + elif argument == 'cros': doPrintCrosDevices() - elif argument == u'crosactivity': + elif argument == 'crosactivity': doPrintCrosActivity() - elif argument == u'mobile': + elif argument == 'mobile': doPrintMobileDevices() - elif argument in [u'license', u'licenses', u'licence', u'licences']: + elif argument in ['license', 'licenses', 'licence', 'licences']: doPrintLicenses() - elif argument in [u'token', u'tokens', u'oauth', u'3lo']: + elif argument in ['token', 'tokens', 'oauth', '3lo']: printShowTokens(3, None, None, True) - elif argument in [u'schema', u'schemas']: + elif argument in ['schema', 'schemas']: doPrintShowUserSchemas(True) - elif argument in [u'courses', u'classes']: + elif argument in ['courses', 'classes']: doPrintCourses() - elif argument in [u'courseparticipants', u'classparticipants']: + elif argument in ['courseparticipants', 'classparticipants']: doPrintCourseParticipants() - elif argument == u'printers': + elif argument == 'printers': doPrintPrinters() - elif argument == u'printjobs': + elif argument == 'printjobs': doPrintPrintJobs() - elif argument in [u'transfers', u'datatransfers']: + elif argument in ['transfers', 'datatransfers']: doPrintDataTransfers() - elif argument == u'transferapps': + elif argument == 'transferapps': doPrintTransferApps() - elif argument == u'domains': + elif argument == 'domains': doPrintDomains() - elif argument in [u'domainaliases', u'aliasdomains']: + elif argument in ['domainaliases', 'aliasdomains']: doPrintDomainAliases() - elif argument == u'admins': + elif argument == 'admins': doPrintAdmins() - elif argument in [u'roles', u'adminroles']: + elif argument in ['roles', 'adminroles']: doPrintAdminRoles() - elif argument in [u'guardian', u'guardians']: + elif argument in ['guardian', 'guardians']: doPrintShowGuardians(True) - elif argument in [u'matters', u'vaultmatters']: + elif argument in ['matters', 'vaultmatters']: doPrintVaultMatters() - elif argument in [u'holds', u'vaultholds']: + elif argument in ['holds', 'vaultholds']: doPrintVaultHolds() - elif argument in [u'exports', u'vaultexports']: + elif argument in ['exports', 'vaultexports']: doPrintVaultExports() - elif argument in [u'building', u'buildings']: + elif argument in ['building', 'buildings']: doPrintBuildings() - elif argument in [u'feature', u'features']: + elif argument in ['feature', 'features']: doPrintFeatures() - elif argument in [u'project', u'projects']: + elif argument in ['project', 'projects']: doPrintShowProjects(True) - elif argument in [u'alert', u'alerts']: + elif argument in ['alert', 'alerts']: doPrintShowAlerts() - elif argument in [u'alertfeedback', u'alertsfeedback']: + elif argument in ['alertfeedback', 'alertsfeedback']: doPrintShowAlertFeedback() else: systemErrorExit(2, '%s is not a valid argument for "gam print"' % argument) sys.exit(0) - elif command == u'show': + elif command == 'show': argument = sys.argv[2].lower() - if argument in [u'schema', u'schemas']: + if argument in ['schema', 'schemas']: doPrintShowUserSchemas(False) - elif argument in [u'guardian', u'guardians']: + elif argument in ['guardian', 'guardians']: doPrintShowGuardians(False) - elif argument in [u'license', u'licenses', u'licence', u'licences']: + elif argument in ['license', 'licenses', 'licence', 'licences']: doShowLicenses() - elif argument in [u'project', u'projects']: + elif argument in ['project', 'projects']: doPrintShowProjects(False) else: systemErrorExit(2, '%s is not a valid argument for "gam show"' % argument) sys.exit(0) - elif command in [u'oauth', u'oauth2']: + elif command in ['oauth', 'oauth2']: argument = sys.argv[2].lower() - if argument in [u'request', u'create']: + if argument in ['request', 'create']: try: login_hint = sys.argv[3].strip() except IndexError: login_hint = None doRequestOAuth(login_hint) - elif argument in [u'info', u'verify']: + elif argument in ['info', 'verify']: OAuthInfo() - elif argument in [u'delete', u'revoke']: + elif argument in ['delete', 'revoke']: doDeleteOAuth() else: systemErrorExit(2, '%s is not a valid argument for "gam oauth"' % argument) sys.exit(0) - elif command == u'calendar': + elif command == 'calendar': argument = sys.argv[3].lower() - if argument == u'showacl': + if argument == 'showacl': doCalendarShowACL() - elif argument == u'add': - doCalendarAddACL(u'Add') - elif argument in [u'del', u'delete']: + elif argument == 'add': + doCalendarAddACL('Add') + elif argument in ['del', 'delete']: doCalendarDelACL() - elif argument == u'update': - doCalendarAddACL(u'Update') - elif argument == u'wipe': + elif argument == 'update': + doCalendarAddACL('Update') + elif argument == 'wipe': doCalendarWipeData() - elif argument == u'addevent': + elif argument == 'addevent': doCalendarAddEvent() - elif argument == u'deleteevent': + elif argument == 'deleteevent': doCalendarDeleteEvent() - elif argument == u'modify': + elif argument == 'modify': doCalendarModifySettings() else: systemErrorExit(2, '%s is not a valid argument for "gam calendar"' % argument) sys.exit(0) - elif command == u'printer': - if sys.argv[2].lower() == u'register': + elif command == 'printer': + if sys.argv[2].lower() == 'register': doPrinterRegister() sys.exit(0) argument = sys.argv[3].lower() - if argument == u'showacl': + if argument == 'showacl': doPrinterShowACL() - elif argument == u'add': + elif argument == 'add': doPrinterAddACL() - elif argument in [u'del', u'delete', u'remove']: + elif argument in ['del', 'delete', 'remove']: doPrinterDelACL() else: systemErrorExit(2, '%s is not a valid argument for "gam printer..."' % argument) sys.exit(0) - elif command == u'printjob': + elif command == 'printjob': argument = sys.argv[3].lower() - if argument == u'delete': + if argument == 'delete': doDeletePrintJob() - elif argument == u'cancel': + elif argument == 'cancel': doCancelPrintJob() - elif argument == u'submit': + elif argument == 'submit': doPrintJobSubmit() - elif argument == u'fetch': + elif argument == 'fetch': doPrintJobFetch() - elif argument == u'resubmit': + elif argument == 'resubmit': doPrintJobResubmit() else: systemErrorExit(2, '%s is not a valid argument for "gam printjob"' % argument) sys.exit(0) - elif command == u'report': + elif command == 'report': showReport() sys.exit(0) - elif command == u'whatis': + elif command == 'whatis': doWhatIs() sys.exit(0) - elif command in [u'course', u'class']: + elif command in ['course', 'class']: argument = sys.argv[3].lower() - if argument in [u'add', u'create']: + if argument in ['add', 'create']: doAddCourseParticipant() - elif argument in [u'del', u'delete', u'remove']: + elif argument in ['del', 'delete', 'remove']: doDelCourseParticipant() - elif argument == u'sync': + elif argument == 'sync': doSyncCourseParticipants() else: systemErrorExit(2, '%s is not a valid argument for "gam course"' % argument) sys.exit(0) - elif command == u'download': + elif command == 'download': argument = sys.argv[2].lower() - if argument in [u'export', u'vaultexport']: + if argument in ['export', 'vaultexport']: doDownloadVaultExport() else: systemErrorExit(2, '%s is not a valid argument for "gam download"' % argument) sys.exit(0) users = getUsersToModify() command = sys.argv[3].lower() - if command == u'print' and len(sys.argv) == 4: + if command == 'print' and len(sys.argv) == 4: for user in users: - print user + print(user) sys.exit(0) if (GC_Values[GC_AUTO_BATCH_MIN] > 0) and (len(users) > GC_Values[GC_AUTO_BATCH_MIN]): runCmdForUsers(None, users, True) - if command == u'transfer': + if command == 'transfer': transferWhat = sys.argv[4].lower() - if transferWhat == u'drive': + if transferWhat == 'drive': transferDriveFiles(users) - elif transferWhat == u'seccals': + elif transferWhat == 'seccals': transferSecCals(users) else: systemErrorExit(2, '%s is not a valid argument for "gam transfer"' % transferWhat) - elif command == u'show': + elif command == 'show': showWhat = sys.argv[4].lower() - if showWhat in [u'labels', u'label']: + if showWhat in ['labels', 'label']: showLabels(users) - elif showWhat == u'profile': + elif showWhat == 'profile': showProfile(users) - elif showWhat == u'calendars': + elif showWhat == 'calendars': printShowCalendars(users, False) - elif showWhat == u'calsettings': + elif showWhat == 'calsettings': showCalSettings(users) - elif showWhat == u'drivesettings': + elif showWhat == 'drivesettings': printDriveSettings(users) - elif showWhat == u'teamdrivethemes': + elif showWhat == 'teamdrivethemes': getTeamDriveThemes(users) - elif showWhat == u'drivefileacl': + elif showWhat == 'drivefileacl': showDriveFileACL(users) - elif showWhat == u'filelist': + elif showWhat == 'filelist': printDriveFileList(users) - elif showWhat == u'filetree': + elif showWhat == 'filetree': showDriveFileTree(users) - elif showWhat == u'fileinfo': + elif showWhat == 'fileinfo': showDriveFileInfo(users) - elif showWhat == u'filerevisions': + elif showWhat == 'filerevisions': showDriveFileRevisions(users) - elif showWhat == u'sendas': + elif showWhat == 'sendas': printShowSendAs(users, False) - elif showWhat == u'smime': + elif showWhat == 'smime': printShowSmime(users, False) - elif showWhat == u'gmailprofile': + elif showWhat == 'gmailprofile': showGmailProfile(users) - elif showWhat in [u'sig', u'signature']: + elif showWhat in ['sig', 'signature']: getSignature(users) - elif showWhat == u'forward': + elif showWhat == 'forward': printShowForward(users, False) - elif showWhat in [u'pop', u'pop3']: + elif showWhat in ['pop', 'pop3']: getPop(users) - elif showWhat in [u'imap', u'imap4']: + elif showWhat in ['imap', 'imap4']: getImap(users) - elif showWhat == u'vacation': + elif showWhat == 'vacation': getVacation(users) - elif showWhat in [u'delegate', u'delegates']: + elif showWhat in ['delegate', 'delegates']: printShowDelegates(users, False) - elif showWhat in [u'backupcode', u'backupcodes', u'verificationcodes']: + elif showWhat in ['backupcode', 'backupcodes', 'verificationcodes']: doGetBackupCodes(users) - elif showWhat in [u'asp', u'asps', u'applicationspecificpasswords']: + elif showWhat in ['asp', 'asps', 'applicationspecificpasswords']: doGetASPs(users) - elif showWhat in [u'token', u'tokens', u'oauth', u'3lo']: - printShowTokens(5, u'users', users, False) - elif showWhat == u'driveactivity': + elif showWhat in ['token', 'tokens', 'oauth', '3lo']: + printShowTokens(5, 'users', users, False) + elif showWhat == 'driveactivity': printDriveActivity(users) - elif showWhat in [u'filter', u'filters']: + elif showWhat in ['filter', 'filters']: printShowFilters(users, False) - elif showWhat in [u'forwardingaddress', u'forwardingaddresses']: + elif showWhat in ['forwardingaddress', 'forwardingaddresses']: printShowForwardingAddresses(users, False) - elif showWhat in [u'teamdrive', u'teamdrives']: + elif showWhat in ['teamdrive', 'teamdrives']: printShowTeamDrives(users, False) - elif showWhat in [u'teamdriveinfo']: + elif showWhat in ['teamdriveinfo']: doGetTeamDriveInfo(users) else: systemErrorExit(2, '%s is not a valid argument for "gam show"' % showWhat) - elif command == u'print': + elif command == 'print': printWhat = sys.argv[4].lower() - if printWhat == u'calendars': + if printWhat == 'calendars': printShowCalendars(users, True) - elif printWhat in [u'delegate', u'delegates']: + elif printWhat in ['delegate', 'delegates']: printShowDelegates(users, True) - elif printWhat == u'driveactivity': + elif printWhat == 'driveactivity': printDriveActivity(users) - elif printWhat == u'drivesettings': + elif printWhat == 'drivesettings': printDriveSettings(users) - elif printWhat == u'filelist': + elif printWhat == 'filelist': printDriveFileList(users) - elif printWhat in [u'filter', u'filters']: + elif printWhat in ['filter', 'filters']: printShowFilters(users, True) - elif printWhat == u'forward': + elif printWhat == 'forward': printShowForward(users, True) - elif printWhat in [u'forwardingaddress', u'forwardingaddresses']: + elif printWhat in ['forwardingaddress', 'forwardingaddresses']: printShowForwardingAddresses(users, True) - elif printWhat == u'sendas': + elif printWhat == 'sendas': printShowSendAs(users, True) - elif printWhat == u'smime': + elif printWhat == 'smime': printShowSmime(users, True) - elif printWhat in [u'token', u'tokens', u'oauth', u'3lo']: - printShowTokens(5, u'users', users, True) - elif printWhat in [u'teamdrive', u'teamdrives']: + elif printWhat in ['token', 'tokens', 'oauth', '3lo']: + printShowTokens(5, 'users', users, True) + elif printWhat in ['teamdrive', 'teamdrives']: printShowTeamDrives(users, True) else: systemErrorExit(2, '%s is not a valid argument for "gam print"' % printWhat) - elif command == u'modify': + elif command == 'modify': modifyWhat = sys.argv[4].lower() - if modifyWhat in [u'message', u'messages']: - doProcessMessagesOrThreads(users, u'modify', u'messages') - elif modifyWhat in [u'thread', u'threads']: - doProcessMessagesOrThreads(users, u'modify', u'threads') + if modifyWhat in ['message', 'messages']: + doProcessMessagesOrThreads(users, 'modify', 'messages') + elif modifyWhat in ['thread', 'threads']: + doProcessMessagesOrThreads(users, 'modify', 'threads') else: systemErrorExit(2, '%s is not a valid argument for "gam modify"' % modifyWhat) - elif command == u'trash': + elif command == 'trash': trashWhat = sys.argv[4].lower() - if trashWhat in [u'message', u'messages']: - doProcessMessagesOrThreads(users, u'trash', u'messages') - elif trashWhat in [u'thread', u'threads']: - doProcessMessagesOrThreads(users, u'trash', u'threads') + if trashWhat in ['message', 'messages']: + doProcessMessagesOrThreads(users, 'trash', 'messages') + elif trashWhat in ['thread', 'threads']: + doProcessMessagesOrThreads(users, 'trash', 'threads') else: systemErrorExit(2, '%s is not a valid argument for "gam trash"' % trashWhat) - elif command == u'untrash': + elif command == 'untrash': untrashWhat = sys.argv[4].lower() - if untrashWhat in [u'message', u'messages']: - doProcessMessagesOrThreads(users, u'untrash', u'messages') - elif untrashWhat in [u'thread', u'threads']: - doProcessMessagesOrThreads(users, u'untrash', u'threads') + if untrashWhat in ['message', 'messages']: + doProcessMessagesOrThreads(users, 'untrash', 'messages') + elif untrashWhat in ['thread', 'threads']: + doProcessMessagesOrThreads(users, 'untrash', 'threads') else: systemErrorExit(2, '%s is not a valid argument for "gam untrash"' % untrashWhat) - elif command in [u'delete', u'del']: + elif command in ['delete', 'del']: delWhat = sys.argv[4].lower() - if delWhat == u'delegate': + if delWhat == 'delegate': deleteDelegate(users) - elif delWhat == u'calendar': + elif delWhat == 'calendar': deleteCalendar(users) - elif delWhat in [u'labels', u'label']: + elif delWhat in ['labels', 'label']: doDeleteLabel(users) - elif delWhat in [u'message', u'messages']: - runCmdForUsers(doProcessMessagesOrThreads, users, default_to_batch=True, function=u'delete', unit=u'messages') - elif delWhat in [u'thread', u'threads']: - runCmdForUsers(doProcessMessagesOrThreads, users, default_to_batch=True, function=u'delete', unit=u'threads') - elif delWhat == u'photo': + elif delWhat in ['message', 'messages']: + runCmdForUsers(doProcessMessagesOrThreads, users, default_to_batch=True, function='delete', unit='messages') + elif delWhat in ['thread', 'threads']: + runCmdForUsers(doProcessMessagesOrThreads, users, default_to_batch=True, function='delete', unit='threads') + elif delWhat == 'photo': deletePhoto(users) - elif delWhat in [u'license', u'licence']: - doLicense(users, u'delete') - elif delWhat in [u'backupcode', u'backupcodes', u'verificationcodes']: + elif delWhat in ['license', 'licence']: + doLicense(users, 'delete') + elif delWhat in ['backupcode', 'backupcodes', 'verificationcodes']: doDelBackupCodes(users) - elif delWhat in [u'asp', u'asps', u'applicationspecificpasswords']: + elif delWhat in ['asp', 'asps', 'applicationspecificpasswords']: doDelASP(users) - elif delWhat in [u'token', u'tokens', u'oauth', u'3lo']: + elif delWhat in ['token', 'tokens', 'oauth', '3lo']: doDelTokens(users) - elif delWhat in [u'group', u'groups']: + elif delWhat in ['group', 'groups']: deleteUserFromGroups(users) - elif delWhat in [u'alias', u'aliases']: + elif delWhat in ['alias', 'aliases']: doRemoveUsersAliases(users) - elif delWhat == u'emptydrivefolders': + elif delWhat == 'emptydrivefolders': deleteEmptyDriveFolders(users) - elif delWhat == u'drivefile': + elif delWhat == 'drivefile': deleteDriveFile(users) - elif delWhat in [u'drivefileacl', u'drivefileacls']: + elif delWhat in ['drivefileacl', 'drivefileacls']: delDriveFileACL(users) - elif delWhat in [u'filter', u'filters']: + elif delWhat in ['filter', 'filters']: deleteFilters(users) - elif delWhat in [u'forwardingaddress', u'forwardingaddresses']: + elif delWhat in ['forwardingaddress', 'forwardingaddresses']: deleteForwardingAddresses(users) - elif delWhat == u'sendas': + elif delWhat == 'sendas': deleteSendAs(users) - elif delWhat == u'smime': + elif delWhat == 'smime': deleteSmime(users) - elif delWhat == u'teamdrive': + elif delWhat == 'teamdrive': doDeleteTeamDrive(users) else: systemErrorExit(2, '%s is not a valid argument for "gam delete"' % delWhat) - elif command in [u'add', u'create']: + elif command in ['add', 'create']: addWhat = sys.argv[4].lower() - if addWhat == u'calendar': - if command == u'add': + if addWhat == 'calendar': + if command == 'add': addCalendar(users) else: systemErrorExit(2, '%s is not implemented for "gam %s"' % (addWhat, command)) - elif addWhat == u'drivefile': + elif addWhat == 'drivefile': createDriveFile(users) - elif addWhat in [u'license', u'licence']: - doLicense(users, u'insert') - elif addWhat in [u'drivefileacl', u'drivefileacls']: + elif addWhat in ['license', 'licence']: + doLicense(users, 'insert') + elif addWhat in ['drivefileacl', 'drivefileacls']: addDriveFileACL(users) - elif addWhat in [u'label', u'labels']: + elif addWhat in ['label', 'labels']: doLabel(users, 5) - elif addWhat in [u'delegate', u'delegates']: + elif addWhat in ['delegate', 'delegates']: addDelegates(users, 5) - elif addWhat in [u'filter', u'filters']: + elif addWhat in ['filter', 'filters']: addFilter(users, 5) - elif addWhat in [u'forwardingaddress', u'forwardingaddresses']: + elif addWhat in ['forwardingaddress', 'forwardingaddresses']: addForwardingAddresses(users) - elif addWhat == u'sendas': + elif addWhat == 'sendas': addUpdateSendAs(users, 5, True) - elif addWhat == u'smime': + elif addWhat == 'smime': addSmime(users) - elif addWhat == u'teamdrive': + elif addWhat == 'teamdrive': doCreateTeamDrive(users) else: systemErrorExit(2, '%s is not a valid argument for "gam %s"' % (addWhat, command)) - elif command == u'update': + elif command == 'update': updateWhat = sys.argv[4].lower() - if updateWhat == u'calendar': + if updateWhat == 'calendar': updateCalendar(users) - elif updateWhat == u'calattendees': + elif updateWhat == 'calattendees': changeCalendarAttendees(users) - elif updateWhat == u'photo': + elif updateWhat == 'photo': doPhoto(users) - elif updateWhat in [u'license', u'licence']: - doLicense(users, u'patch') - elif updateWhat == u'user': + elif updateWhat in ['license', 'licence']: + doLicense(users, 'patch') + elif updateWhat == 'user': doUpdateUser(users, 5) - elif updateWhat in [u'backupcode', u'backupcodes', u'verificationcodes']: + elif updateWhat in ['backupcode', 'backupcodes', 'verificationcodes']: doGenBackupCodes(users) - elif updateWhat == u'drivefile': + elif updateWhat == 'drivefile': doUpdateDriveFile(users) - elif updateWhat in [u'drivefileacls', u'drivefileacl']: + elif updateWhat in ['drivefileacls', 'drivefileacl']: updateDriveFileACL(users) - elif updateWhat in [u'label', u'labels']: + elif updateWhat in ['label', 'labels']: renameLabels(users) - elif updateWhat == u'labelsettings': + elif updateWhat == 'labelsettings': updateLabels(users) - elif updateWhat == u'sendas': + elif updateWhat == 'sendas': addUpdateSendAs(users, 5, False) - elif updateWhat == u'smime': + elif updateWhat == 'smime': updateSmime(users) - elif updateWhat == u'teamdrive': + elif updateWhat == 'teamdrive': doUpdateTeamDrive(users) else: systemErrorExit(2, '%s is not a valid argument for "gam update"' % updateWhat) - elif command in [u'deprov', u'deprovision']: + elif command in ['deprov', 'deprovision']: doDeprovUser(users) - elif command == u'get': + elif command == 'get': getWhat = sys.argv[4].lower() - if getWhat == u'photo': + if getWhat == 'photo': getPhoto(users) - elif getWhat == u'drivefile': + elif getWhat == 'drivefile': downloadDriveFile(users) else: systemErrorExit(2, '%s is not a valid argument for "gam get"' % getWhat) - elif command == u'empty': + elif command == 'empty': emptyWhat = sys.argv[4].lower() - if emptyWhat == u'drivetrash': + if emptyWhat == 'drivetrash': doEmptyDriveTrash(users) else: systemErrorExit(2, '%s is not a valid argument for "gam empty"' % emptyWhat) - elif command == u'info': + elif command == 'info': infoWhat = sys.argv[4].lower() - if infoWhat == u'calendar': + if infoWhat == 'calendar': infoCalendar(users) - elif infoWhat in [u'filter', u'filters']: + elif infoWhat in ['filter', 'filters']: infoFilters(users) - elif infoWhat in [u'forwardingaddress', u'forwardingaddresses']: + elif infoWhat in ['forwardingaddress', 'forwardingaddresses']: infoForwardingAddresses(users) - elif infoWhat == u'sendas': + elif infoWhat == 'sendas': infoSendAs(users) else: systemErrorExit(2, '%s is not a valid argument for "gam info"' % infoWhat) - elif command == u'check': - checkWhat = sys.argv[4].replace(u'_', '').lower() - if checkWhat == u'serviceaccount': + elif command == 'check': + checkWhat = sys.argv[4].replace('_', '').lower() + if checkWhat == 'serviceaccount': doCheckServiceAccount(users) else: systemErrorExit(2, '%s is not a valid argument for "gam check"' % checkWhat) - elif command == u'profile': + elif command == 'profile': doProfile(users) - elif command == u'imap': + elif command == 'imap': #doImap(users) runCmdForUsers(doImap, users, default_to_batch=True) - elif command in [u'pop', u'pop3']: + elif command in ['pop', 'pop3']: doPop(users) - elif command == u'sendas': + elif command == 'sendas': addUpdateSendAs(users, 4, True) - elif command == u'label': + elif command == 'label': doLabel(users, 4) - elif command == u'filter': + elif command == 'filter': addFilter(users, 4) - elif command == u'forward': + elif command == 'forward': doForward(users) - elif command in [u'sig', u'signature']: + elif command in ['sig', 'signature']: doSignature(users) - elif command == u'vacation': + elif command == 'vacation': doVacation(users) - elif command in [u'delegate', u'delegates']: + elif command in ['delegate', 'delegates']: addDelegates(users, 4) - elif command == u'watch': + elif command == 'watch': if len(sys.argv) > 4: watchWhat = sys.argv[4].lower() else: - watchWhat = u'gmail' - if watchWhat == u'gmail': + watchWhat = 'gmail' + if watchWhat == 'gmail': watchGmail(users) else: systemErrorExit(2, '%s is not a valid argument for "gam watch"' % watchWhat) @@ -14090,6 +14098,6 @@ if __name__ == "__main__": if sys.platform.startswith('win'): freeze_support() win32_unicode_argv() # cleanup sys.argv on Windows - if sys.version_info[:2] != (2, 7): - systemErrorExit(5, 'GAM requires Python 2.7. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.' % sys.version_info[:3]) + if sys.version_info[0] < 3 or sys.version_info[1] < 5: + systemErrorExit(5, 'GAM requires Python 3.5 or newer. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.' % sys.version_info[:3]) sys.exit(ProcessGAMCommand(sys.argv)) diff --git a/src/google/__init__.py b/src/google/__init__.py deleted file mode 100644 index a35569c3..00000000 --- a/src/google/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google namespace package.""" - -try: - import pkg_resources - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/src/google/auth/__init__.py b/src/google/auth/__init__.py deleted file mode 100644 index 65e13951..00000000 --- a/src/google/auth/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Auth Library for Python.""" - -import logging - -from google.auth._default import default - - -__all__ = [ - 'default', -] - - -# Set default logging handler to avoid "No handler found" warnings. -logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/src/google/auth/_cloud_sdk.py b/src/google/auth/_cloud_sdk.py deleted file mode 100644 index 0d4b222f..00000000 --- a/src/google/auth/_cloud_sdk.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers for reading the Google Cloud SDK's configuration.""" - -import json -import os -import subprocess - -from google.auth import environment_vars -import google.oauth2.credentials - - -# The ~/.config subdirectory containing gcloud credentials. -_CONFIG_DIRECTORY = 'gcloud' -# Windows systems store config at %APPDATA%\gcloud -_WINDOWS_CONFIG_ROOT_ENV_VAR = 'APPDATA' -# The name of the file in the Cloud SDK config that contains default -# credentials. -_CREDENTIALS_FILENAME = 'application_default_credentials.json' -# The name of the Cloud SDK shell script -_CLOUD_SDK_POSIX_COMMAND = 'gcloud' -_CLOUD_SDK_WINDOWS_COMMAND = 'gcloud.cmd' -# The command to get the Cloud SDK configuration -_CLOUD_SDK_CONFIG_COMMAND = ('config', 'config-helper', '--format', 'json') -# Cloud SDK's application-default client ID -CLOUD_SDK_CLIENT_ID = ( - '764086051850-6qr4p6gpi6hn506pt8ejuq83di341hur.apps.googleusercontent.com') - - -def get_config_path(): - """Returns the absolute path the the Cloud SDK's configuration directory. - - Returns: - str: The Cloud SDK config path. - """ - # If the path is explicitly set, return that. - try: - return os.environ[environment_vars.CLOUD_SDK_CONFIG_DIR] - except KeyError: - pass - - # Non-windows systems store this at ~/.config/gcloud - if os.name != 'nt': - return os.path.join( - os.path.expanduser('~'), '.config', _CONFIG_DIRECTORY) - # Windows systems store config at %APPDATA%\gcloud - else: - try: - return os.path.join( - os.environ[_WINDOWS_CONFIG_ROOT_ENV_VAR], - _CONFIG_DIRECTORY) - except KeyError: - # This should never happen unless someone is really - # messing with things, but we'll cover the case anyway. - drive = os.environ.get('SystemDrive', 'C:') - return os.path.join( - drive, '\\', _CONFIG_DIRECTORY) - - -def get_application_default_credentials_path(): - """Gets the path to the application default credentials file. - - The path may or may not exist. - - Returns: - str: The full path to application default credentials. - """ - config_path = get_config_path() - return os.path.join(config_path, _CREDENTIALS_FILENAME) - - -def load_authorized_user_credentials(info): - """Loads an authorized user credential. - - Args: - info (Mapping[str, str]): The loaded file's data. - - Returns: - google.oauth2.credentials.Credentials: The constructed credentials. - - Raises: - ValueError: if the info is in the wrong format or missing data. - """ - return google.oauth2.credentials.Credentials.from_authorized_user_info( - info) - - -def get_project_id(): - """Gets the project ID from the Cloud SDK. - - Returns: - Optional[str]: The project ID. - """ - if os.name == 'nt': - command = _CLOUD_SDK_WINDOWS_COMMAND - else: - command = _CLOUD_SDK_POSIX_COMMAND - - try: - output = subprocess.check_output( - (command,) + _CLOUD_SDK_CONFIG_COMMAND, - stderr=subprocess.STDOUT) - except (subprocess.CalledProcessError, OSError, IOError): - return None - - try: - configuration = json.loads(output.decode('utf-8')) - except ValueError: - return None - - try: - return configuration['configuration']['properties']['core']['project'] - except KeyError: - return None diff --git a/src/google/auth/_default.py b/src/google/auth/_default.py deleted file mode 100644 index c93b4896..00000000 --- a/src/google/auth/_default.py +++ /dev/null @@ -1,306 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Application default credentials. - -Implements application default credentials and project ID detection. -""" - -import io -import json -import logging -import os -import warnings - -import six - -from google.auth import environment_vars -from google.auth import exceptions -import google.auth.transport._http_client - -_LOGGER = logging.getLogger(__name__) - -# Valid types accepted for file-based credentials. -_AUTHORIZED_USER_TYPE = 'authorized_user' -_SERVICE_ACCOUNT_TYPE = 'service_account' -_VALID_TYPES = (_AUTHORIZED_USER_TYPE, _SERVICE_ACCOUNT_TYPE) - -# Help message when no credentials can be found. -_HELP_MESSAGE = """\ -Could not automatically determine credentials. Please set {env} or \ -explicitly create credentials and re-run the application. For more \ -information, please see \ -https://cloud.google.com/docs/authentication/getting-started -""".format(env=environment_vars.CREDENTIALS).strip() - -# Warning when using Cloud SDK user credentials -_CLOUD_SDK_CREDENTIALS_WARNING = """\ -Your application has authenticated using end user credentials from Google \ -Cloud SDK. We recommend that most server applications use service accounts \ -instead. If your application continues to use end user credentials from Cloud \ -SDK, you might receive a "quota exceeded" or "API not enabled" error. For \ -more information about service accounts, see \ -https://cloud.google.com/docs/authentication/""" - - -def _warn_about_problematic_credentials(credentials): - """Determines if the credentials are problematic. - - Credentials from the Cloud SDK that are associated with Cloud SDK's project - are problematic because they may not have APIs enabled and have limited - quota. If this is the case, warn about it. - """ - from google.auth import _cloud_sdk - if credentials.client_id == _cloud_sdk.CLOUD_SDK_CLIENT_ID: - warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING) - - -def _load_credentials_from_file(filename): - """Loads credentials from a file. - - The credentials file must be a service account key or stored authorized - user credentials. - - Args: - filename (str): The full path to the credentials file. - - Returns: - Tuple[google.auth.credentials.Credentials, Optional[str]]: Loaded - credentials and the project ID. Authorized user credentials do not - have the project ID information. - - Raises: - google.auth.exceptions.DefaultCredentialsError: if the file is in the - wrong format or is missing. - """ - if not os.path.exists(filename): - raise exceptions.DefaultCredentialsError( - 'File {} was not found.'.format(filename)) - - with io.open(filename, 'r') as file_obj: - try: - info = json.load(file_obj) - except ValueError as caught_exc: - new_exc = exceptions.DefaultCredentialsError( - 'File {} is not a valid json file.'.format(filename), - caught_exc) - six.raise_from(new_exc, caught_exc) - - # The type key should indicate that the file is either a service account - # credentials file or an authorized user credentials file. - credential_type = info.get('type') - - if credential_type == _AUTHORIZED_USER_TYPE: - from google.auth import _cloud_sdk - - try: - credentials = _cloud_sdk.load_authorized_user_credentials(info) - except ValueError as caught_exc: - msg = 'Failed to load authorized user credentials from {}'.format( - filename) - new_exc = exceptions.DefaultCredentialsError(msg, caught_exc) - six.raise_from(new_exc, caught_exc) - # Authorized user credentials do not contain the project ID. - _warn_about_problematic_credentials(credentials) - return credentials, None - - elif credential_type == _SERVICE_ACCOUNT_TYPE: - from google.oauth2 import service_account - - try: - credentials = ( - service_account.Credentials.from_service_account_info(info)) - except ValueError as caught_exc: - msg = 'Failed to load service account credentials from {}'.format( - filename) - new_exc = exceptions.DefaultCredentialsError(msg, caught_exc) - six.raise_from(new_exc, caught_exc) - return credentials, info.get('project_id') - - else: - raise exceptions.DefaultCredentialsError( - 'The file {file} does not have a valid type. ' - 'Type is {type}, expected one of {valid_types}.'.format( - file=filename, type=credential_type, valid_types=_VALID_TYPES)) - - -def _get_gcloud_sdk_credentials(): - """Gets the credentials and project ID from the Cloud SDK.""" - from google.auth import _cloud_sdk - - # Check if application default credentials exist. - credentials_filename = ( - _cloud_sdk.get_application_default_credentials_path()) - - if not os.path.isfile(credentials_filename): - return None, None - - credentials, project_id = _load_credentials_from_file( - credentials_filename) - - if not project_id: - project_id = _cloud_sdk.get_project_id() - - return credentials, project_id - - -def _get_explicit_environ_credentials(): - """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment - variable.""" - explicit_file = os.environ.get(environment_vars.CREDENTIALS) - - if explicit_file is not None: - credentials, project_id = _load_credentials_from_file( - os.environ[environment_vars.CREDENTIALS]) - - return credentials, project_id - - else: - return None, None - - -def _get_gae_credentials(): - """Gets Google App Engine App Identity credentials and project ID.""" - from google.auth import app_engine - - try: - credentials = app_engine.Credentials() - project_id = app_engine.get_project_id() - return credentials, project_id - except EnvironmentError: - return None, None - - -def _get_gce_credentials(request=None): - """Gets credentials and project ID from the GCE Metadata Service.""" - # Ping requires a transport, but we want application default credentials - # to require no arguments. So, we'll use the _http_client transport which - # uses http.client. This is only acceptable because the metadata server - # doesn't do SSL and never requires proxies. - from google.auth import compute_engine - from google.auth.compute_engine import _metadata - - if request is None: - request = google.auth.transport._http_client.Request() - - if _metadata.ping(request=request): - # Get the project ID. - try: - project_id = _metadata.get_project_id(request=request) - except exceptions.TransportError: - project_id = None - - return compute_engine.Credentials(), project_id - else: - return None, None - - -def default(scopes=None, request=None): - """Gets the default credentials for the current environment. - - `Application Default Credentials`_ provides an easy way to obtain - credentials to call Google APIs for server-to-server or local applications. - This function acquires credentials from the environment in the following - order: - - 1. If the environment variable ``GOOGLE_APPLICATION_CREDENTIALS`` is set - to the path of a valid service account JSON private key file, then it is - loaded and returned. The project ID returned is the project ID defined - in the service account file if available (some older files do not - contain project ID information). - 2. If the `Google Cloud SDK`_ is installed and has application default - credentials set they are loaded and returned. - - To enable application default credentials with the Cloud SDK run:: - - gcloud auth application-default login - - If the Cloud SDK has an active project, the project ID is returned. The - active project can be set using:: - - gcloud config set project - - 3. If the application is running in the `App Engine standard environment`_ - then the credentials and project ID from the `App Identity Service`_ - are used. - 4. If the application is running in `Compute Engine`_ or the - `App Engine flexible environment`_ then the credentials and project ID - are obtained from the `Metadata Service`_. - 5. If no credentials are found, - :class:`~google.auth.exceptions.DefaultCredentialsError` will be raised. - - .. _Application Default Credentials: https://developers.google.com\ - /identity/protocols/application-default-credentials - .. _Google Cloud SDK: https://cloud.google.com/sdk - .. _App Engine standard environment: https://cloud.google.com/appengine - .. _App Identity Service: https://cloud.google.com/appengine/docs/python\ - /appidentity/ - .. _Compute Engine: https://cloud.google.com/compute - .. _App Engine flexible environment: https://cloud.google.com\ - /appengine/flexible - .. _Metadata Service: https://cloud.google.com/compute/docs\ - /storing-retrieving-metadata - - Example:: - - import google.auth - - credentials, project_id = google.auth.default() - - Args: - scopes (Sequence[str]): The list of scopes for the credentials. If - specified, the credentials will automatically be scoped if - necessary. - request (google.auth.transport.Request): An object used to make - HTTP requests. This is used to detect whether the application - is running on Compute Engine. If not specified, then it will - use the standard library http client to make requests. - - Returns: - Tuple[~google.auth.credentials.Credentials, Optional[str]]: - the current environment's credentials and project ID. Project ID - may be None, which indicates that the Project ID could not be - ascertained from the environment. - - Raises: - ~google.auth.exceptions.DefaultCredentialsError: - If no credentials were found, or if the credentials found were - invalid. - """ - from google.auth.credentials import with_scopes_if_required - - explicit_project_id = os.environ.get( - environment_vars.PROJECT, - os.environ.get(environment_vars.LEGACY_PROJECT)) - - checkers = ( - _get_explicit_environ_credentials, - _get_gcloud_sdk_credentials, - _get_gae_credentials, - lambda: _get_gce_credentials(request)) - - for checker in checkers: - credentials, project_id = checker() - if credentials is not None: - credentials = with_scopes_if_required(credentials, scopes) - effective_project_id = explicit_project_id or project_id - if not effective_project_id: - _LOGGER.warning( - 'No project ID could be determined. Consider running ' - '`gcloud config set project` or setting the %s ' - 'environment variable', - environment_vars.PROJECT) - return credentials, effective_project_id - - raise exceptions.DefaultCredentialsError(_HELP_MESSAGE) diff --git a/src/google/auth/_helpers.py b/src/google/auth/_helpers.py deleted file mode 100644 index 860b8271..00000000 --- a/src/google/auth/_helpers.py +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper functions for commonly used utilities.""" - -import base64 -import calendar -import datetime - -import six -from six.moves import urllib - - -CLOCK_SKEW_SECS = 300 # 5 minutes in seconds -CLOCK_SKEW = datetime.timedelta(seconds=CLOCK_SKEW_SECS) - - -def copy_docstring(source_class): - """Decorator that copies a method's docstring from another class. - - Args: - source_class (type): The class that has the documented method. - - Returns: - Callable: A decorator that will copy the docstring of the same - named method in the source class to the decorated method. - """ - def decorator(method): - """Decorator implementation. - - Args: - method (Callable): The method to copy the docstring to. - - Returns: - Callable: the same method passed in with an updated docstring. - - Raises: - ValueError: if the method already has a docstring. - """ - if method.__doc__: - raise ValueError('Method already has a docstring.') - - source_method = getattr(source_class, method.__name__) - method.__doc__ = source_method.__doc__ - - return method - return decorator - - -def utcnow(): - """Returns the current UTC datetime. - - Returns: - datetime: The current time in UTC. - """ - return datetime.datetime.utcnow() - - -def datetime_to_secs(value): - """Convert a datetime object to the number of seconds since the UNIX epoch. - - Args: - value (datetime): The datetime to convert. - - Returns: - int: The number of seconds since the UNIX epoch. - """ - return calendar.timegm(value.utctimetuple()) - - -def to_bytes(value, encoding='utf-8'): - """Converts a string value to bytes, if necessary. - - Unfortunately, ``six.b`` is insufficient for this task since in - Python 2 because it does not modify ``unicode`` objects. - - Args: - value (Union[str, bytes]): The value to be converted. - encoding (str): The encoding to use to convert unicode to bytes. - Defaults to "utf-8". - - Returns: - bytes: The original value converted to bytes (if unicode) or as - passed in if it started out as bytes. - - Raises: - ValueError: If the value could not be converted to bytes. - """ - result = (value.encode(encoding) - if isinstance(value, six.text_type) else value) - if isinstance(result, six.binary_type): - return result - else: - raise ValueError('{0!r} could not be converted to bytes'.format(value)) - - -def from_bytes(value): - """Converts bytes to a string value, if necessary. - - Args: - value (Union[str, bytes]): The value to be converted. - - Returns: - str: The original value converted to unicode (if bytes) or as passed in - if it started out as unicode. - - Raises: - ValueError: If the value could not be converted to unicode. - """ - result = (value.decode('utf-8') - if isinstance(value, six.binary_type) else value) - if isinstance(result, six.text_type): - return result - else: - raise ValueError( - '{0!r} could not be converted to unicode'.format(value)) - - -def update_query(url, params, remove=None): - """Updates a URL's query parameters. - - Replaces any current values if they are already present in the URL. - - Args: - url (str): The URL to update. - params (Mapping[str, str]): A mapping of query parameter - keys to values. - remove (Sequence[str]): Parameters to remove from the query string. - - Returns: - str: The URL with updated query parameters. - - Examples: - - >>> url = 'http://example.com?a=1' - >>> update_query(url, {'a': '2'}) - http://example.com?a=2 - >>> update_query(url, {'b': '3'}) - http://example.com?a=1&b=3 - >> update_query(url, {'b': '3'}, remove=['a']) - http://example.com?b=3 - - """ - if remove is None: - remove = [] - - # Split the URL into parts. - parts = urllib.parse.urlparse(url) - # Parse the query string. - query_params = urllib.parse.parse_qs(parts.query) - # Update the query parameters with the new parameters. - query_params.update(params) - # Remove any values specified in remove. - query_params = { - key: value for key, value - in six.iteritems(query_params) - if key not in remove} - # Re-encoded the query string. - new_query = urllib.parse.urlencode(query_params, doseq=True) - # Unsplit the url. - new_parts = parts._replace(query=new_query) - return urllib.parse.urlunparse(new_parts) - - -def scopes_to_string(scopes): - """Converts scope value to a string suitable for sending to OAuth 2.0 - authorization servers. - - Args: - scopes (Sequence[str]): The sequence of scopes to convert. - - Returns: - str: The scopes formatted as a single string. - """ - return ' '.join(scopes) - - -def string_to_scopes(scopes): - """Converts stringifed scopes value to a list. - - Args: - scopes (Union[Sequence, str]): The string of space-separated scopes - to convert. - Returns: - Sequence(str): The separated scopes. - """ - if not scopes: - return [] - - return scopes.split(' ') - - -def padded_urlsafe_b64decode(value): - """Decodes base64 strings lacking padding characters. - - Google infrastructure tends to omit the base64 padding characters. - - Args: - value (Union[str, bytes]): The encoded value. - - Returns: - bytes: The decoded value - """ - b64string = to_bytes(value) - padded = b64string + b'=' * (-len(b64string) % 4) - return base64.urlsafe_b64decode(padded) diff --git a/src/google/auth/_oauth2client.py b/src/google/auth/_oauth2client.py deleted file mode 100644 index afe7dc45..00000000 --- a/src/google/auth/_oauth2client.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers for transitioning from oauth2client to google-auth. - -.. warning:: - This module is private as it is intended to assist first-party downstream - clients with the transition from oauth2client to google-auth. -""" - -from __future__ import absolute_import - -import six - -from google.auth import _helpers -import google.auth.app_engine -import google.auth.compute_engine -import google.oauth2.credentials -import google.oauth2.service_account - -try: - import oauth2client.client - import oauth2client.contrib.gce - import oauth2client.service_account -except ImportError as caught_exc: - six.raise_from( - ImportError('oauth2client is not installed.'), caught_exc) - -try: - import oauth2client.contrib.appengine # pytype: disable=import-error - _HAS_APPENGINE = True -except ImportError: - _HAS_APPENGINE = False - - -_CONVERT_ERROR_TMPL = ( - 'Unable to convert {} to a google-auth credentials class.') - - -def _convert_oauth2_credentials(credentials): - """Converts to :class:`google.oauth2.credentials.Credentials`. - - Args: - credentials (Union[oauth2client.client.OAuth2Credentials, - oauth2client.client.GoogleCredentials]): The credentials to - convert. - - Returns: - google.oauth2.credentials.Credentials: The converted credentials. - """ - new_credentials = google.oauth2.credentials.Credentials( - token=credentials.access_token, - refresh_token=credentials.refresh_token, - token_uri=credentials.token_uri, - client_id=credentials.client_id, - client_secret=credentials.client_secret, - scopes=credentials.scopes) - - new_credentials._expires = credentials.token_expiry - - return new_credentials - - -def _convert_service_account_credentials(credentials): - """Converts to :class:`google.oauth2.service_account.Credentials`. - - Args: - credentials (Union[ - oauth2client.service_account.ServiceAccountCredentials, - oauth2client.service_account._JWTAccessCredentials]): The - credentials to convert. - - Returns: - google.oauth2.service_account.Credentials: The converted credentials. - """ - info = credentials.serialization_data.copy() - info['token_uri'] = credentials.token_uri - return google.oauth2.service_account.Credentials.from_service_account_info( - info) - - -def _convert_gce_app_assertion_credentials(credentials): - """Converts to :class:`google.auth.compute_engine.Credentials`. - - Args: - credentials (oauth2client.contrib.gce.AppAssertionCredentials): The - credentials to convert. - - Returns: - google.oauth2.service_account.Credentials: The converted credentials. - """ - return google.auth.compute_engine.Credentials( - service_account_email=credentials.service_account_email) - - -def _convert_appengine_app_assertion_credentials(credentials): - """Converts to :class:`google.auth.app_engine.Credentials`. - - Args: - credentials (oauth2client.contrib.app_engine.AppAssertionCredentials): - The credentials to convert. - - Returns: - google.oauth2.service_account.Credentials: The converted credentials. - """ - # pylint: disable=invalid-name - return google.auth.app_engine.Credentials( - scopes=_helpers.string_to_scopes(credentials.scope), - service_account_id=credentials.service_account_id) - - -_CLASS_CONVERSION_MAP = { - oauth2client.client.OAuth2Credentials: _convert_oauth2_credentials, - oauth2client.client.GoogleCredentials: _convert_oauth2_credentials, - oauth2client.service_account.ServiceAccountCredentials: - _convert_service_account_credentials, - oauth2client.service_account._JWTAccessCredentials: - _convert_service_account_credentials, - oauth2client.contrib.gce.AppAssertionCredentials: - _convert_gce_app_assertion_credentials, -} - -if _HAS_APPENGINE: - _CLASS_CONVERSION_MAP[ - oauth2client.contrib.appengine.AppAssertionCredentials] = ( - _convert_appengine_app_assertion_credentials) - - -def convert(credentials): - """Convert oauth2client credentials to google-auth credentials. - - This class converts: - - - :class:`oauth2client.client.OAuth2Credentials` to - :class:`google.oauth2.credentials.Credentials`. - - :class:`oauth2client.client.GoogleCredentials` to - :class:`google.oauth2.credentials.Credentials`. - - :class:`oauth2client.service_account.ServiceAccountCredentials` to - :class:`google.oauth2.service_account.Credentials`. - - :class:`oauth2client.service_account._JWTAccessCredentials` to - :class:`google.oauth2.service_account.Credentials`. - - :class:`oauth2client.contrib.gce.AppAssertionCredentials` to - :class:`google.auth.compute_engine.Credentials`. - - :class:`oauth2client.contrib.appengine.AppAssertionCredentials` to - :class:`google.auth.app_engine.Credentials`. - - Returns: - google.auth.credentials.Credentials: The converted credentials. - - Raises: - ValueError: If the credentials could not be converted. - """ - - credentials_class = type(credentials) - - try: - return _CLASS_CONVERSION_MAP[credentials_class](credentials) - except KeyError as caught_exc: - new_exc = ValueError(_CONVERT_ERROR_TMPL.format(credentials_class)) - six.raise_from(new_exc, caught_exc) diff --git a/src/google/auth/_service_account_info.py b/src/google/auth/_service_account_info.py deleted file mode 100644 index dd39ea7b..00000000 --- a/src/google/auth/_service_account_info.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper functions for loading data from a Google service account file.""" - -import io -import json - -import six - -from google.auth import crypt - - -def from_dict(data, require=None): - """Validates a dictionary containing Google service account data. - - Creates and returns a :class:`google.auth.crypt.Signer` instance from the - private key specified in the data. - - Args: - data (Mapping[str, str]): The service account data - require (Sequence[str]): List of keys required to be present in the - info. - - Returns: - google.auth.crypt.Signer: A signer created from the private key in the - service account file. - - Raises: - ValueError: if the data was in the wrong format, or if one of the - required keys is missing. - """ - keys_needed = set(require if require is not None else []) - - missing = keys_needed.difference(six.iterkeys(data)) - - if missing: - raise ValueError( - 'Service account info was not in the expected format, missing ' - 'fields {}.'.format(', '.join(missing))) - - # Create a signer. - signer = crypt.RSASigner.from_service_account_info(data) - - return signer - - -def from_filename(filename, require=None): - """Reads a Google service account JSON file and returns its parsed info. - - Args: - filename (str): The path to the service account .json file. - require (Sequence[str]): List of keys required to be present in the - info. - - Returns: - Tuple[ Mapping[str, str], google.auth.crypt.Signer ]: The verified - info and a signer instance. - """ - with io.open(filename, 'r', encoding='utf-8') as json_file: - data = json.load(json_file) - return data, from_dict(data, require=require) diff --git a/src/google/auth/app_engine.py b/src/google/auth/app_engine.py deleted file mode 100644 index 91ba8427..00000000 --- a/src/google/auth/app_engine.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google App Engine standard environment support. - -This module provides authentication and signing for applications running on App -Engine in the standard environment using the `App Identity API`_. - - -.. _App Identity API: - https://cloud.google.com/appengine/docs/python/appidentity/ -""" - -import datetime - -from google.auth import _helpers -from google.auth import credentials -from google.auth import crypt - -# pytype: disable=import-error -try: - from google.appengine.api import app_identity -except ImportError: - app_identity = None -# pytype: enable=import-error - - -class Signer(crypt.Signer): - """Signs messages using the App Engine App Identity service. - - This can be used in place of :class:`google.auth.crypt.Signer` when - running in the App Engine standard environment. - """ - - @property - def key_id(self): - """Optional[str]: The key ID used to identify this private key. - - .. warning:: - This is always ``None``. The key ID used by App Engine can not - be reliably determined ahead of time. - """ - return None - - @_helpers.copy_docstring(crypt.Signer) - def sign(self, message): - message = _helpers.to_bytes(message) - _, signature = app_identity.sign_blob(message) - return signature - - -def get_project_id(): - """Gets the project ID for the current App Engine application. - - Returns: - str: The project ID - - Raises: - EnvironmentError: If the App Engine APIs are unavailable. - """ - # pylint: disable=missing-raises-doc - # Pylint rightfully thinks EnvironmentError is OSError, but doesn't - # realize it's a valid alias. - if app_identity is None: - raise EnvironmentError( - 'The App Engine APIs are not available.') - return app_identity.get_application_id() - - -class Credentials(credentials.Scoped, credentials.Signing, - credentials.Credentials): - """App Engine standard environment credentials. - - These credentials use the App Engine App Identity API to obtain access - tokens. - """ - - def __init__(self, scopes=None, service_account_id=None): - """ - Args: - scopes (Sequence[str]): Scopes to request from the App Identity - API. - service_account_id (str): The service account ID passed into - :func:`google.appengine.api.app_identity.get_access_token`. - If not specified, the default application service account - ID will be used. - - Raises: - EnvironmentError: If the App Engine APIs are unavailable. - """ - # pylint: disable=missing-raises-doc - # Pylint rightfully thinks EnvironmentError is OSError, but doesn't - # realize it's a valid alias. - if app_identity is None: - raise EnvironmentError( - 'The App Engine APIs are not available.') - - super(Credentials, self).__init__() - self._scopes = scopes - self._service_account_id = service_account_id - self._signer = Signer() - - @_helpers.copy_docstring(credentials.Credentials) - def refresh(self, request): - # pylint: disable=unused-argument - token, ttl = app_identity.get_access_token( - self._scopes, self._service_account_id) - expiry = datetime.datetime.utcfromtimestamp(ttl) - - self.token, self.expiry = token, expiry - - @property - def service_account_email(self): - """The service account email.""" - if self._service_account_id is None: - self._service_account_id = app_identity.get_service_account_name() - return self._service_account_id - - @property - def requires_scopes(self): - """Checks if the credentials requires scopes. - - Returns: - bool: True if there are no scopes set otherwise False. - """ - return not self._scopes - - @_helpers.copy_docstring(credentials.Scoped) - def with_scopes(self, scopes): - return self.__class__( - scopes=scopes, service_account_id=self._service_account_id) - - @_helpers.copy_docstring(credentials.Signing) - def sign_bytes(self, message): - return self._signer.sign(message) - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer_email(self): - return self.service_account_email - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer(self): - return self._signer diff --git a/src/google/auth/compute_engine/__init__.py b/src/google/auth/compute_engine/__init__.py deleted file mode 100644 index ca31b464..00000000 --- a/src/google/auth/compute_engine/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Compute Engine authentication.""" - -from google.auth.compute_engine.credentials import Credentials -from google.auth.compute_engine.credentials import IDTokenCredentials - - -__all__ = [ - 'Credentials', - 'IDTokenCredentials', -] diff --git a/src/google/auth/compute_engine/_metadata.py b/src/google/auth/compute_engine/_metadata.py deleted file mode 100644 index c47be3fa..00000000 --- a/src/google/auth/compute_engine/_metadata.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides helper methods for talking to the Compute Engine metadata server. - -See https://cloud.google.com/compute/docs/metadata for more details. -""" - -import datetime -import json -import logging -import os - -import six -from six.moves import http_client -from six.moves.urllib import parse as urlparse - -from google.auth import _helpers -from google.auth import environment_vars -from google.auth import exceptions - -_LOGGER = logging.getLogger(__name__) - -_METADATA_ROOT = 'http://{}/computeMetadata/v1/'.format( - os.getenv(environment_vars.GCE_METADATA_ROOT, 'metadata.google.internal')) - -# This is used to ping the metadata server, it avoids the cost of a DNS -# lookup. -_METADATA_IP_ROOT = 'http://{}'.format( - os.getenv(environment_vars.GCE_METADATA_IP, '169.254.169.254')) -_METADATA_FLAVOR_HEADER = 'metadata-flavor' -_METADATA_FLAVOR_VALUE = 'Google' -_METADATA_HEADERS = {_METADATA_FLAVOR_HEADER: _METADATA_FLAVOR_VALUE} - -# Timeout in seconds to wait for the GCE metadata server when detecting the -# GCE environment. -try: - _METADATA_DEFAULT_TIMEOUT = int(os.getenv('GCE_METADATA_TIMEOUT', 3)) -except ValueError: # pragma: NO COVER - _METADATA_DEFAULT_TIMEOUT = 3 - - -def ping(request, timeout=_METADATA_DEFAULT_TIMEOUT): - """Checks to see if the metadata server is available. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - timeout (int): How long to wait for the metadata server to respond. - - Returns: - bool: True if the metadata server is reachable, False otherwise. - """ - # NOTE: The explicit ``timeout`` is a workaround. The underlying - # issue is that resolving an unknown host on some networks will take - # 20-30 seconds; making this timeout short fixes the issue, but - # could lead to false negatives in the event that we are on GCE, but - # the metadata resolution was particularly slow. The latter case is - # "unlikely". - try: - response = request( - url=_METADATA_IP_ROOT, method='GET', headers=_METADATA_HEADERS, - timeout=timeout) - - metadata_flavor = response.headers.get(_METADATA_FLAVOR_HEADER) - return (response.status == http_client.OK and - metadata_flavor == _METADATA_FLAVOR_VALUE) - - except exceptions.TransportError: - _LOGGER.info('Compute Engine Metadata server unavailable.') - return False - - -def get(request, path, root=_METADATA_ROOT, recursive=False): - """Fetch a resource from the metadata server. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - path (str): The resource to retrieve. For example, - ``'instance/service-accounts/default'``. - root (str): The full path to the metadata server root. - recursive (bool): Whether to do a recursive query of metadata. See - https://cloud.google.com/compute/docs/metadata#aggcontents for more - details. - - Returns: - Union[Mapping, str]: If the metadata server returns JSON, a mapping of - the decoded JSON is return. Otherwise, the response content is - returned as a string. - - Raises: - google.auth.exceptions.TransportError: if an error occurred while - retrieving metadata. - """ - base_url = urlparse.urljoin(root, path) - query_params = {} - - if recursive: - query_params['recursive'] = 'true' - - url = _helpers.update_query(base_url, query_params) - - response = request(url=url, method='GET', headers=_METADATA_HEADERS) - - if response.status == http_client.OK: - content = _helpers.from_bytes(response.data) - if response.headers['content-type'] == 'application/json': - try: - return json.loads(content) - except ValueError as caught_exc: - new_exc = exceptions.TransportError( - 'Received invalid JSON from the Google Compute Engine' - 'metadata service: {:.20}'.format(content)) - six.raise_from(new_exc, caught_exc) - else: - return content - else: - raise exceptions.TransportError( - 'Failed to retrieve {} from the Google Compute Engine' - 'metadata service. Status: {} Response:\n{}'.format( - url, response.status, response.data), response) - - -def get_project_id(request): - """Get the Google Cloud Project ID from the metadata server. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - - Returns: - str: The project ID - - Raises: - google.auth.exceptions.TransportError: if an error occurred while - retrieving metadata. - """ - return get(request, 'project/project-id') - - -def get_service_account_info(request, service_account='default'): - """Get information about a service account from the metadata server. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - service_account (str): The string 'default' or a service account email - address. The determines which service account for which to acquire - information. - - Returns: - Mapping: The service account's information, for example:: - - { - 'email': '...', - 'scopes': ['scope', ...], - 'aliases': ['default', '...'] - } - - Raises: - google.auth.exceptions.TransportError: if an error occurred while - retrieving metadata. - """ - return get( - request, - 'instance/service-accounts/{0}/'.format(service_account), - recursive=True) - - -def get_service_account_token(request, service_account='default'): - """Get the OAuth 2.0 access token for a service account. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - service_account (str): The string 'default' or a service account email - address. The determines which service account for which to acquire - an access token. - - Returns: - Union[str, datetime]: The access token and its expiration. - - Raises: - google.auth.exceptions.TransportError: if an error occurred while - retrieving metadata. - """ - token_json = get( - request, - 'instance/service-accounts/{0}/token'.format(service_account)) - token_expiry = _helpers.utcnow() + datetime.timedelta( - seconds=token_json['expires_in']) - return token_json['access_token'], token_expiry diff --git a/src/google/auth/compute_engine/credentials.py b/src/google/auth/compute_engine/credentials.py deleted file mode 100644 index d9c6e26d..00000000 --- a/src/google/auth/compute_engine/credentials.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Compute Engine credentials. - -This module provides authentication for application running on Google Compute -Engine using the Compute Engine metadata server. - -""" - -import datetime - -import six - -from google.auth import _helpers -from google.auth import credentials -from google.auth import exceptions -from google.auth import iam -from google.auth import jwt -from google.auth.compute_engine import _metadata -from google.oauth2 import _client - - -class Credentials(credentials.ReadOnlyScoped, credentials.Credentials): - """Compute Engine Credentials. - - These credentials use the Google Compute Engine metadata server to obtain - OAuth 2.0 access tokens associated with the instance's service account. - - For more information about Compute Engine authentication, including how - to configure scopes, see the `Compute Engine authentication - documentation`_. - - .. note:: Compute Engine instances can be created with scopes and therefore - these credentials are considered to be 'scoped'. However, you can - not use :meth:`~google.auth.credentials.ScopedCredentials.with_scopes` - because it is not possible to change the scopes that the instance - has. Also note that - :meth:`~google.auth.credentials.ScopedCredentials.has_scopes` will not - work until the credentials have been refreshed. - - .. _Compute Engine authentication documentation: - https://cloud.google.com/compute/docs/authentication#using - """ - - def __init__(self, service_account_email='default'): - """ - Args: - service_account_email (str): The service account email to use, or - 'default'. A Compute Engine instance may have multiple service - accounts. - """ - super(Credentials, self).__init__() - self._service_account_email = service_account_email - - def _retrieve_info(self, request): - """Retrieve information about the service account. - - Updates the scopes and retrieves the full service account email. - - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - """ - info = _metadata.get_service_account_info( - request, - service_account=self._service_account_email) - - self._service_account_email = info['email'] - self._scopes = info['scopes'] - - def refresh(self, request): - """Refresh the access token and scopes. - - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - - Raises: - google.auth.exceptions.RefreshError: If the Compute Engine metadata - service can't be reached if if the instance has not - credentials. - """ - try: - self._retrieve_info(request) - self.token, self.expiry = _metadata.get_service_account_token( - request, - service_account=self._service_account_email) - except exceptions.TransportError as caught_exc: - new_exc = exceptions.RefreshError(caught_exc) - six.raise_from(new_exc, caught_exc) - - @property - def service_account_email(self): - """The service account email. - - .. note: This is not guaranteed to be set until :meth`refresh` has been - called. - """ - return self._service_account_email - - @property - def requires_scopes(self): - """False: Compute Engine credentials can not be scoped.""" - return False - - -_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds -_DEFAULT_TOKEN_URI = 'https://www.googleapis.com/oauth2/v4/token' - - -class IDTokenCredentials(credentials.Credentials, credentials.Signing): - """Open ID Connect ID Token-based service account credentials. - - These credentials relies on the default service account of a GCE instance. - - In order for this to work, the GCE instance must have been started with - a service account that has access to the IAM Cloud API. - """ - def __init__(self, request, target_audience, - token_uri=_DEFAULT_TOKEN_URI, - additional_claims=None, - service_account_email=None): - """ - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - target_audience (str): The intended audience for these credentials, - used when requesting the ID Token. The ID Token's ``aud`` claim - will be set to this string. - token_uri (str): The OAuth 2.0 Token URI. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT assertion used in the authorization grant. - service_account_email (str): Optional explicit service account to - use to sign JWT tokens. - By default, this is the default GCE service account. - """ - super(IDTokenCredentials, self).__init__() - - if service_account_email is None: - sa_info = _metadata.get_service_account_info(request) - service_account_email = sa_info['email'] - self._service_account_email = service_account_email - - self._signer = iam.Signer( - request=request, - credentials=Credentials(), - service_account_email=service_account_email) - - self._token_uri = token_uri - self._target_audience = target_audience - - if additional_claims is not None: - self._additional_claims = additional_claims - else: - self._additional_claims = {} - - def with_target_audience(self, target_audience): - """Create a copy of these credentials with the specified target - audience. - Args: - target_audience (str): The intended audience for these credentials, - used when requesting the ID Token. - Returns: - google.auth.service_account.IDTokenCredentials: A new credentials - instance. - """ - return self.__class__( - self._signer, - service_account_email=self._service_account_email, - token_uri=self._token_uri, - target_audience=target_audience, - additional_claims=self._additional_claims.copy()) - - def _make_authorization_grant_assertion(self): - """Create the OAuth 2.0 assertion. - This assertion is used during the OAuth 2.0 grant to acquire an - ID token. - Returns: - bytes: The authorization grant assertion. - """ - now = _helpers.utcnow() - lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS) - expiry = now + lifetime - - payload = { - 'iat': _helpers.datetime_to_secs(now), - 'exp': _helpers.datetime_to_secs(expiry), - # The issuer must be the service account email. - 'iss': self.service_account_email, - # The audience must be the auth token endpoint's URI - 'aud': self._token_uri, - # The target audience specifies which service the ID token is - # intended for. - 'target_audience': self._target_audience - } - - payload.update(self._additional_claims) - - token = jwt.encode(self._signer, payload) - - return token - - @_helpers.copy_docstring(credentials.Credentials) - def refresh(self, request): - assertion = self._make_authorization_grant_assertion() - access_token, expiry, _ = _client.id_token_jwt_grant( - request, self._token_uri, assertion) - self.token = access_token - self.expiry = expiry - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer(self): - return self._signer - - @_helpers.copy_docstring(credentials.Signing) - def sign_bytes(self, message): - return self._signer.sign(message) - - @property - def service_account_email(self): - """The service account email.""" - return self._service_account_email - - @property - def signer_email(self): - return self._service_account_email diff --git a/src/google/auth/credentials.py b/src/google/auth/credentials.py deleted file mode 100644 index 8ff1f025..00000000 --- a/src/google/auth/credentials.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Interfaces for credentials.""" - -import abc - -import six - -from google.auth import _helpers - - -@six.add_metaclass(abc.ABCMeta) -class Credentials(object): - """Base class for all credentials. - - All credentials have a :attr:`token` that is used for authentication and - may also optionally set an :attr:`expiry` to indicate when the token will - no longer be valid. - - Most credentials will be :attr:`invalid` until :meth:`refresh` is called. - Credentials can do this automatically before the first HTTP request in - :meth:`before_request`. - - Although the token and expiration will change as the credentials are - :meth:`refreshed ` and used, credentials should be considered - immutable. Various credentials will accept configuration such as private - keys, scopes, and other options. These options are not changeable after - construction. Some classes will provide mechanisms to copy the credentials - with modifications such as :meth:`ScopedCredentials.with_scopes`. - """ - def __init__(self): - self.token = None - """str: The bearer token that can be used in HTTP headers to make - authenticated requests.""" - self.expiry = None - """Optional[datetime]: When the token expires and is no longer valid. - If this is None, the token is assumed to never expire.""" - - @property - def expired(self): - """Checks if the credentials are expired. - - Note that credentials can be invalid but not expired because - Credentials with :attr:`expiry` set to None is considered to never - expire. - """ - if not self.expiry: - return False - - # Remove 5 minutes from expiry to err on the side of reporting - # expiration early so that we avoid the 401-refresh-retry loop. - skewed_expiry = self.expiry - _helpers.CLOCK_SKEW - return _helpers.utcnow() >= skewed_expiry - - @property - def valid(self): - """Checks the validity of the credentials. - - This is True if the credentials have a :attr:`token` and the token - is not :attr:`expired`. - """ - return self.token is not None and not self.expired - - @abc.abstractmethod - def refresh(self, request): - """Refreshes the access token. - - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - - Raises: - google.auth.exceptions.RefreshError: If the credentials could - not be refreshed. - """ - # pylint: disable=missing-raises-doc - # (pylint doesn't recognize that this is abstract) - raise NotImplementedError('Refresh must be implemented') - - def apply(self, headers, token=None): - """Apply the token to the authentication header. - - Args: - headers (Mapping): The HTTP request headers. - token (Optional[str]): If specified, overrides the current access - token. - """ - headers['authorization'] = 'Bearer {}'.format( - _helpers.from_bytes(token or self.token)) - - def before_request(self, request, method, url, headers): - """Performs credential-specific before request logic. - - Refreshes the credentials if necessary, then calls :meth:`apply` to - apply the token to the authentication header. - - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - method (str): The request's HTTP method or the RPC method being - invoked. - url (str): The request's URI or the RPC service's URI. - headers (Mapping): The request's headers. - """ - # pylint: disable=unused-argument - # (Subclasses may use these arguments to ascertain information about - # the http request.) - if not self.valid: - self.refresh(request) - self.apply(headers) - - -class AnonymousCredentials(Credentials): - """Credentials that do not provide any authentication information. - - These are useful in the case of services that support anonymous access or - local service emulators that do not use credentials. - """ - - @property - def expired(self): - """Returns `False`, anonymous credentials never expire.""" - return False - - @property - def valid(self): - """Returns `True`, anonymous credentials are always valid.""" - return True - - def refresh(self, request): - """Raises :class:`ValueError``, anonymous credentials cannot be - refreshed.""" - raise ValueError("Anonymous credentials cannot be refreshed.") - - def apply(self, headers, token=None): - """Anonymous credentials do nothing to the request. - - The optional ``token`` argument is not supported. - - Raises: - ValueError: If a token was specified. - """ - if token is not None: - raise ValueError("Anonymous credentials don't support tokens.") - - def before_request(self, request, method, url, headers): - """Anonymous credentials do nothing to the request.""" - - -@six.add_metaclass(abc.ABCMeta) -class ReadOnlyScoped(object): - """Interface for credentials whose scopes can be queried. - - OAuth 2.0-based credentials allow limiting access using scopes as described - in `RFC6749 Section 3.3`_. - If a credential class implements this interface then the credentials either - use scopes in their implementation. - - Some credentials require scopes in order to obtain a token. You can check - if scoping is necessary with :attr:`requires_scopes`:: - - if credentials.requires_scopes: - # Scoping is required. - credentials = credentials.with_scopes(scopes=['one', 'two']) - - Credentials that require scopes must either be constructed with scopes:: - - credentials = SomeScopedCredentials(scopes=['one', 'two']) - - Or must copy an existing instance using :meth:`with_scopes`:: - - scoped_credentials = credentials.with_scopes(scopes=['one', 'two']) - - Some credentials have scopes but do not allow or require scopes to be set, - these credentials can be used as-is. - - .. _RFC6749 Section 3.3: https://tools.ietf.org/html/rfc6749#section-3.3 - """ - def __init__(self): - super(ReadOnlyScoped, self).__init__() - self._scopes = None - - @property - def scopes(self): - """Sequence[str]: the credentials' current set of scopes.""" - return self._scopes - - @abc.abstractproperty - def requires_scopes(self): - """True if these credentials require scopes to obtain an access token. - """ - return False - - def has_scopes(self, scopes): - """Checks if the credentials have the given scopes. - - .. warning: This method is not guaranteed to be accurate if the - credentials are :attr:`~Credentials.invalid`. - - Args: - scopes (Sequence[str]): The list of scopes to check. - - Returns: - bool: True if the credentials have the given scopes. - """ - return set(scopes).issubset(set(self._scopes or [])) - - -class Scoped(ReadOnlyScoped): - """Interface for credentials whose scopes can be replaced while copying. - - OAuth 2.0-based credentials allow limiting access using scopes as described - in `RFC6749 Section 3.3`_. - If a credential class implements this interface then the credentials either - use scopes in their implementation. - - Some credentials require scopes in order to obtain a token. You can check - if scoping is necessary with :attr:`requires_scopes`:: - - if credentials.requires_scopes: - # Scoping is required. - credentials = credentials.create_scoped(['one', 'two']) - - Credentials that require scopes must either be constructed with scopes:: - - credentials = SomeScopedCredentials(scopes=['one', 'two']) - - Or must copy an existing instance using :meth:`with_scopes`:: - - scoped_credentials = credentials.with_scopes(scopes=['one', 'two']) - - Some credentials have scopes but do not allow or require scopes to be set, - these credentials can be used as-is. - - .. _RFC6749 Section 3.3: https://tools.ietf.org/html/rfc6749#section-3.3 - """ - @abc.abstractmethod - def with_scopes(self, scopes): - """Create a copy of these credentials with the specified scopes. - - Args: - scopes (Sequence[str]): The list of scopes to attach to the - current credentials. - - Raises: - NotImplementedError: If the credentials' scopes can not be changed. - This can be avoided by checking :attr:`requires_scopes` before - calling this method. - """ - raise NotImplementedError('This class does not require scoping.') - - -def with_scopes_if_required(credentials, scopes): - """Creates a copy of the credentials with scopes if scoping is required. - - This helper function is useful when you do not know (or care to know) the - specific type of credentials you are using (such as when you use - :func:`google.auth.default`). This function will call - :meth:`Scoped.with_scopes` if the credentials are scoped credentials and if - the credentials require scoping. Otherwise, it will return the credentials - as-is. - - Args: - credentials (google.auth.credentials.Credentials): The credentials to - scope if necessary. - scopes (Sequence[str]): The list of scopes to use. - - Returns: - google.auth.credentials.Credentials: Either a new set of scoped - credentials, or the passed in credentials instance if no scoping - was required. - """ - if isinstance(credentials, Scoped) and credentials.requires_scopes: - return credentials.with_scopes(scopes) - else: - return credentials - - -@six.add_metaclass(abc.ABCMeta) -class Signing(object): - """Interface for credentials that can cryptographically sign messages.""" - - @abc.abstractmethod - def sign_bytes(self, message): - """Signs the given message. - - Args: - message (bytes): The message to sign. - - Returns: - bytes: The message's cryptographic signature. - """ - # pylint: disable=missing-raises-doc,redundant-returns-doc - # (pylint doesn't recognize that this is abstract) - raise NotImplementedError('Sign bytes must be implemented.') - - @abc.abstractproperty - def signer_email(self): - """Optional[str]: An email address that identifies the signer.""" - # pylint: disable=missing-raises-doc - # (pylint doesn't recognize that this is abstract) - raise NotImplementedError('Signer email must be implemented.') - - @abc.abstractproperty - def signer(self): - """google.auth.crypt.Signer: The signer used to sign bytes.""" - # pylint: disable=missing-raises-doc - # (pylint doesn't recognize that this is abstract) - raise NotImplementedError('Signer must be implemented.') diff --git a/src/google/auth/crypt/__init__.py b/src/google/auth/crypt/__init__.py deleted file mode 100644 index 7baa206e..00000000 --- a/src/google/auth/crypt/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Cryptography helpers for verifying and signing messages. - -The simplest way to verify signatures is using :func:`verify_signature`:: - - cert = open('certs.pem').read() - valid = crypt.verify_signature(message, signature, cert) - -If you're going to verify many messages with the same certificate, you can use -:class:`RSAVerifier`:: - - cert = open('certs.pem').read() - verifier = crypt.RSAVerifier.from_string(cert) - valid = verifier.verify(message, signature) - -To sign messages use :class:`RSASigner` with a private key:: - - private_key = open('private_key.pem').read() - signer = crypt.RSASigner.from_string(private_key) - signature = signer.sign(message) -""" - -import six - -from google.auth.crypt import base -from google.auth.crypt import rsa - - -__all__ = [ - 'RSASigner', - 'RSAVerifier', - 'Signer', - 'Verifier', -] - -# Aliases to maintain the v1.0.0 interface, as the crypt module was split -# into submodules. -Signer = base.Signer -Verifier = base.Verifier -RSASigner = rsa.RSASigner -RSAVerifier = rsa.RSAVerifier - - -def verify_signature(message, signature, certs): - """Verify an RSA cryptographic signature. - - Checks that the provided ``signature`` was generated from ``bytes`` using - the private key associated with the ``cert``. - - Args: - message (Union[str, bytes]): The plaintext message. - signature (Union[str, bytes]): The cryptographic signature to check. - certs (Union[Sequence, str, bytes]): The certificate or certificates - to use to check the signature. - - Returns: - bool: True if the signature is valid, otherwise False. - """ - if isinstance(certs, (six.text_type, six.binary_type)): - certs = [certs] - - for cert in certs: - verifier = rsa.RSAVerifier.from_string(cert) - if verifier.verify(message, signature): - return True - return False diff --git a/src/google/auth/crypt/_cryptography_rsa.py b/src/google/auth/crypt/_cryptography_rsa.py deleted file mode 100644 index 87076b0a..00000000 --- a/src/google/auth/crypt/_cryptography_rsa.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""RSA verifier and signer that use the ``cryptography`` library. - -This is a much faster implementation than the default (in -``google.auth.crypt._python_rsa``), which depends on the pure-Python -``rsa`` library. -""" - -import cryptography.exceptions -from cryptography.hazmat import backends -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import padding -import cryptography.x509 -import pkg_resources - -from google.auth import _helpers -from google.auth.crypt import base - -_IMPORT_ERROR_MSG = ( - 'cryptography>=1.4.0 is required to use cryptography-based RSA ' - 'implementation.') - -try: # pragma: NO COVER - release = pkg_resources.get_distribution('cryptography').parsed_version - if release < pkg_resources.parse_version('1.4.0'): - raise ImportError(_IMPORT_ERROR_MSG) -except pkg_resources.DistributionNotFound: # pragma: NO COVER - raise ImportError(_IMPORT_ERROR_MSG) - - -_CERTIFICATE_MARKER = b'-----BEGIN CERTIFICATE-----' -_BACKEND = backends.default_backend() -_PADDING = padding.PKCS1v15() -_SHA256 = hashes.SHA256() - - -class RSAVerifier(base.Verifier): - """Verifies RSA cryptographic signatures using public keys. - - Args: - public_key ( - cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey): - The public key used to verify signatures. - """ - - def __init__(self, public_key): - self._pubkey = public_key - - @_helpers.copy_docstring(base.Verifier) - def verify(self, message, signature): - message = _helpers.to_bytes(message) - try: - self._pubkey.verify(signature, message, _PADDING, _SHA256) - return True - except (ValueError, cryptography.exceptions.InvalidSignature): - return False - - @classmethod - def from_string(cls, public_key): - """Construct an Verifier instance from a public key or public - certificate string. - - Args: - public_key (Union[str, bytes]): The public key in PEM format or the - x509 public key certificate. - - Returns: - Verifier: The constructed verifier. - - Raises: - ValueError: If the public key can't be parsed. - """ - public_key_data = _helpers.to_bytes(public_key) - - if _CERTIFICATE_MARKER in public_key_data: - cert = cryptography.x509.load_pem_x509_certificate( - public_key_data, _BACKEND) - pubkey = cert.public_key() - - else: - pubkey = serialization.load_pem_public_key( - public_key_data, _BACKEND) - - return cls(pubkey) - - -class RSASigner(base.Signer, base.FromServiceAccountMixin): - """Signs messages with an RSA private key. - - Args: - private_key ( - cryptography.hazmat.primitives.asymmetric.rsa.RSAPrivateKey): - The private key to sign with. - key_id (str): Optional key ID used to identify this private key. This - can be useful to associate the private key with its associated - public key or certificate. - """ - - def __init__(self, private_key, key_id=None): - self._key = private_key - self._key_id = key_id - - @property - @_helpers.copy_docstring(base.Signer) - def key_id(self): - return self._key_id - - @_helpers.copy_docstring(base.Signer) - def sign(self, message): - message = _helpers.to_bytes(message) - return self._key.sign( - message, _PADDING, _SHA256) - - @classmethod - def from_string(cls, key, key_id=None): - """Construct a RSASigner from a private key in PEM format. - - Args: - key (Union[bytes, str]): Private key in PEM format. - key_id (str): An optional key id used to identify the private key. - - Returns: - google.auth.crypt._cryptography_rsa.RSASigner: The - constructed signer. - - Raises: - ValueError: If ``key`` is not ``bytes`` or ``str`` (unicode). - UnicodeDecodeError: If ``key`` is ``bytes`` but cannot be decoded - into a UTF-8 ``str``. - ValueError: If ``cryptography`` "Could not deserialize key data." - """ - key = _helpers.to_bytes(key) - private_key = serialization.load_pem_private_key( - key, password=None, backend=_BACKEND) - return cls(private_key, key_id=key_id) diff --git a/src/google/auth/crypt/_helpers.py b/src/google/auth/crypt/_helpers.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/google/auth/crypt/_python_rsa.py b/src/google/auth/crypt/_python_rsa.py deleted file mode 100644 index 44aa7919..00000000 --- a/src/google/auth/crypt/_python_rsa.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pure-Python RSA cryptography implementation. - -Uses the ``rsa``, ``pyasn1`` and ``pyasn1_modules`` packages -to parse PEM files storing PKCS#1 or PKCS#8 keys as well as -certificates. There is no support for p12 files. -""" - -from __future__ import absolute_import - -from pyasn1.codec.der import decoder -from pyasn1_modules import pem -from pyasn1_modules.rfc2459 import Certificate -from pyasn1_modules.rfc5208 import PrivateKeyInfo -import rsa -import six - -from google.auth import _helpers -from google.auth.crypt import base - -_POW2 = (128, 64, 32, 16, 8, 4, 2, 1) -_CERTIFICATE_MARKER = b'-----BEGIN CERTIFICATE-----' -_PKCS1_MARKER = ('-----BEGIN RSA PRIVATE KEY-----', - '-----END RSA PRIVATE KEY-----') -_PKCS8_MARKER = ('-----BEGIN PRIVATE KEY-----', - '-----END PRIVATE KEY-----') -_PKCS8_SPEC = PrivateKeyInfo() - - -def _bit_list_to_bytes(bit_list): - """Converts an iterable of 1s and 0s to bytes. - - Combines the list 8 at a time, treating each group of 8 bits - as a single byte. - - Args: - bit_list (Sequence): Sequence of 1s and 0s. - - Returns: - bytes: The decoded bytes. - """ - num_bits = len(bit_list) - byte_vals = bytearray() - for start in six.moves.xrange(0, num_bits, 8): - curr_bits = bit_list[start:start + 8] - char_val = sum( - val * digit for val, digit in six.moves.zip(_POW2, curr_bits)) - byte_vals.append(char_val) - return bytes(byte_vals) - - -class RSAVerifier(base.Verifier): - """Verifies RSA cryptographic signatures using public keys. - - Args: - public_key (rsa.key.PublicKey): The public key used to verify - signatures. - """ - - def __init__(self, public_key): - self._pubkey = public_key - - @_helpers.copy_docstring(base.Verifier) - def verify(self, message, signature): - message = _helpers.to_bytes(message) - try: - return rsa.pkcs1.verify(message, signature, self._pubkey) - except (ValueError, rsa.pkcs1.VerificationError): - return False - - @classmethod - def from_string(cls, public_key): - """Construct an Verifier instance from a public key or public - certificate string. - - Args: - public_key (Union[str, bytes]): The public key in PEM format or the - x509 public key certificate. - - Returns: - Verifier: The constructed verifier. - - Raises: - ValueError: If the public_key can't be parsed. - """ - public_key = _helpers.to_bytes(public_key) - is_x509_cert = _CERTIFICATE_MARKER in public_key - - # If this is a certificate, extract the public key info. - if is_x509_cert: - der = rsa.pem.load_pem(public_key, 'CERTIFICATE') - asn1_cert, remaining = decoder.decode(der, asn1Spec=Certificate()) - if remaining != b'': - raise ValueError('Unused bytes', remaining) - - cert_info = asn1_cert['tbsCertificate']['subjectPublicKeyInfo'] - key_bytes = _bit_list_to_bytes(cert_info['subjectPublicKey']) - pubkey = rsa.PublicKey.load_pkcs1(key_bytes, 'DER') - else: - pubkey = rsa.PublicKey.load_pkcs1(public_key, 'PEM') - return cls(pubkey) - - -class RSASigner(base.Signer, base.FromServiceAccountMixin): - """Signs messages with an RSA private key. - - Args: - private_key (rsa.key.PrivateKey): The private key to sign with. - key_id (str): Optional key ID used to identify this private key. This - can be useful to associate the private key with its associated - public key or certificate. - """ - - def __init__(self, private_key, key_id=None): - self._key = private_key - self._key_id = key_id - - @property - @_helpers.copy_docstring(base.Signer) - def key_id(self): - return self._key_id - - @_helpers.copy_docstring(base.Signer) - def sign(self, message): - message = _helpers.to_bytes(message) - return rsa.pkcs1.sign(message, self._key, 'SHA-256') - - @classmethod - def from_string(cls, key, key_id=None): - """Construct an Signer instance from a private key in PEM format. - - Args: - key (str): Private key in PEM format. - key_id (str): An optional key id used to identify the private key. - - Returns: - google.auth.crypt.Signer: The constructed signer. - - Raises: - ValueError: If the key cannot be parsed as PKCS#1 or PKCS#8 in - PEM format. - """ - key = _helpers.from_bytes(key) # PEM expects str in Python 3 - marker_id, key_bytes = pem.readPemBlocksFromFile( - six.StringIO(key), _PKCS1_MARKER, _PKCS8_MARKER) - - # Key is in pkcs1 format. - if marker_id == 0: - private_key = rsa.key.PrivateKey.load_pkcs1( - key_bytes, format='DER') - # Key is in pkcs8. - elif marker_id == 1: - key_info, remaining = decoder.decode( - key_bytes, asn1Spec=_PKCS8_SPEC) - if remaining != b'': - raise ValueError('Unused bytes', remaining) - private_key_info = key_info.getComponentByName('privateKey') - private_key = rsa.key.PrivateKey.load_pkcs1( - private_key_info.asOctets(), format='DER') - else: - raise ValueError('No key could be detected.') - - return cls(private_key, key_id=key_id) diff --git a/src/google/auth/crypt/base.py b/src/google/auth/crypt/base.py deleted file mode 100644 index c6c04272..00000000 --- a/src/google/auth/crypt/base.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Base classes for cryptographic signers and verifiers.""" - -import abc -import io -import json - -import six - - -_JSON_FILE_PRIVATE_KEY = 'private_key' -_JSON_FILE_PRIVATE_KEY_ID = 'private_key_id' - - -@six.add_metaclass(abc.ABCMeta) -class Verifier(object): - """Abstract base class for crytographic signature verifiers.""" - - @abc.abstractmethod - def verify(self, message, signature): - """Verifies a message against a cryptographic signature. - - Args: - message (Union[str, bytes]): The message to verify. - signature (Union[str, bytes]): The cryptography signature to check. - - Returns: - bool: True if message was signed by the private key associated - with the public key that this object was constructed with. - """ - # pylint: disable=missing-raises-doc,redundant-returns-doc - # (pylint doesn't recognize that this is abstract) - raise NotImplementedError('Verify must be implemented') - - -@six.add_metaclass(abc.ABCMeta) -class Signer(object): - """Abstract base class for cryptographic signers.""" - - @abc.abstractproperty - def key_id(self): - """Optional[str]: The key ID used to identify this private key.""" - raise NotImplementedError('Key id must be implemented') - - @abc.abstractmethod - def sign(self, message): - """Signs a message. - - Args: - message (Union[str, bytes]): The message to be signed. - - Returns: - bytes: The signature of the message. - """ - # pylint: disable=missing-raises-doc,redundant-returns-doc - # (pylint doesn't recognize that this is abstract) - raise NotImplementedError('Sign must be implemented') - - -@six.add_metaclass(abc.ABCMeta) -class FromServiceAccountMixin(object): - """Mix-in to enable factory constructors for a Signer.""" - - @abc.abstractmethod - def from_string(cls, key, key_id=None): - """Construct an Signer instance from a private key string. - - Args: - key (str): Private key as a string. - key_id (str): An optional key id used to identify the private key. - - Returns: - google.auth.crypt.Signer: The constructed signer. - - Raises: - ValueError: If the key cannot be parsed. - """ - raise NotImplementedError('from_string must be implemented') - - @classmethod - def from_service_account_info(cls, info): - """Creates a Signer instance instance from a dictionary containing - service account info in Google format. - - Args: - info (Mapping[str, str]): The service account info in Google - format. - - Returns: - google.auth.crypt.Signer: The constructed signer. - - Raises: - ValueError: If the info is not in the expected format. - """ - if _JSON_FILE_PRIVATE_KEY not in info: - raise ValueError( - 'The private_key field was not found in the service account ' - 'info.') - - return cls.from_string( - info[_JSON_FILE_PRIVATE_KEY], - info.get(_JSON_FILE_PRIVATE_KEY_ID)) - - @classmethod - def from_service_account_file(cls, filename): - """Creates a Signer instance from a service account .json file - in Google format. - - Args: - filename (str): The path to the service account .json file. - - Returns: - google.auth.crypt.Signer: The constructed signer. - """ - with io.open(filename, 'r', encoding='utf-8') as json_file: - data = json.load(json_file) - - return cls.from_service_account_info(data) diff --git a/src/google/auth/crypt/rsa.py b/src/google/auth/crypt/rsa.py deleted file mode 100644 index 5da1ba60..00000000 --- a/src/google/auth/crypt/rsa.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""RSA cryptography signer and verifier.""" - - -try: - # Prefer cryptograph-based RSA implementation. - from google.auth.crypt import _cryptography_rsa - - RSASigner = _cryptography_rsa.RSASigner - RSAVerifier = _cryptography_rsa.RSAVerifier -except ImportError: # pragma: NO COVER - # Fallback to pure-python RSA implementation if cryptography is - # unavailable. - from google.auth.crypt import _python_rsa - - RSASigner = _python_rsa.RSASigner - RSAVerifier = _python_rsa.RSAVerifier diff --git a/src/google/auth/environment_vars.py b/src/google/auth/environment_vars.py deleted file mode 100644 index 0110e6a3..00000000 --- a/src/google/auth/environment_vars.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Environment variables used by :mod:`google.auth`.""" - - -PROJECT = 'GOOGLE_CLOUD_PROJECT' -"""Environment variable defining default project. - -This used by :func:`google.auth.default` to explicitly set a project ID. This -environment variable is also used by the Google Cloud Python Library. -""" - -LEGACY_PROJECT = 'GCLOUD_PROJECT' -"""Previously used environment variable defining the default project. - -This environment variable is used instead of the current one in some -situations (such as Google App Engine). -""" - -CREDENTIALS = 'GOOGLE_APPLICATION_CREDENTIALS' -"""Environment variable defining the location of Google application default -credentials.""" - -# The environment variable name which can replace ~/.config if set. -CLOUD_SDK_CONFIG_DIR = 'CLOUDSDK_CONFIG' -"""Environment variable defines the location of Google Cloud SDK's config -files.""" - -# These two variables allow for customization of the addresses used when -# contacting the GCE metadata service. -GCE_METADATA_ROOT = 'GCE_METADATA_ROOT' -"""Environment variable providing an alternate hostname or host:port to be -used for GCE metadata requests.""" - -GCE_METADATA_IP = 'GCE_METADATA_IP' -"""Environment variable providing an alternate ip:port to be used for ip-only -GCE metadata requests.""" diff --git a/src/google/auth/exceptions.py b/src/google/auth/exceptions.py deleted file mode 100644 index 2be9fd6d..00000000 --- a/src/google/auth/exceptions.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Exceptions used in the google.auth package.""" - - -class GoogleAuthError(Exception): - """Base class for all google.auth errors.""" - - -class TransportError(GoogleAuthError): - """Used to indicate an error occurred during an HTTP request.""" - - -class RefreshError(GoogleAuthError): - """Used to indicate that an refreshing the credentials' access token - failed.""" - - -class DefaultCredentialsError(GoogleAuthError): - """Used to indicate that acquiring default credentials failed.""" diff --git a/src/google/auth/iam.py b/src/google/auth/iam.py deleted file mode 100644 index e091e47f..00000000 --- a/src/google/auth/iam.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tools for using the Google `Cloud Identity and Access Management (IAM) -API`_'s auth-related functionality. - -.. _Cloud Identity and Access Management (IAM) API: - https://cloud.google.com/iam/docs/ -""" - -import base64 -import json - -from six.moves import http_client - -from google.auth import _helpers -from google.auth import crypt -from google.auth import exceptions - -_IAM_API_ROOT_URI = 'https://iam.googleapis.com/v1' -_SIGN_BLOB_URI = ( - _IAM_API_ROOT_URI + '/projects/-/serviceAccounts/{}:signBlob?alt=json') - - -class Signer(crypt.Signer): - """Signs messages using the IAM `signBlob API`_. - - This is useful when you need to sign bytes but do not have access to the - credential's private key file. - - .. _signBlob API: - https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts - /signBlob - """ - - def __init__(self, request, credentials, service_account_email): - """ - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - credentials (google.auth.credentials.Credentials): The credentials - that will be used to authenticate the request to the IAM API. - The credentials must have of one the following scopes: - - - https://www.googleapis.com/auth/iam - - https://www.googleapis.com/auth/cloud-platform - service_account_email (str): The service account email identifying - which service account to use to sign bytes. Often, this can - be the same as the service account email in the given - credentials. - """ - self._request = request - self._credentials = credentials - self._service_account_email = service_account_email - - def _make_signing_request(self, message): - """Makes a request to the API signBlob API.""" - message = _helpers.to_bytes(message) - - method = 'POST' - url = _SIGN_BLOB_URI.format(self._service_account_email) - headers = {} - body = json.dumps({ - 'bytesToSign': base64.b64encode(message).decode('utf-8'), - }) - - self._credentials.before_request(self._request, method, url, headers) - response = self._request( - url=url, method=method, body=body, headers=headers) - - if response.status != http_client.OK: - raise exceptions.TransportError( - 'Error calling the IAM signBytes API: {}'.format( - response.data)) - - return json.loads(response.data.decode('utf-8')) - - @property - def key_id(self): - """Optional[str]: The key ID used to identify this private key. - - .. warning:: - This is always ``None``. The key ID used by IAM can not - be reliably determined ahead of time. - """ - return None - - @_helpers.copy_docstring(crypt.Signer) - def sign(self, message): - response = self._make_signing_request(message) - return base64.b64decode(response['signature']) diff --git a/src/google/auth/impersonated_credentials.py b/src/google/auth/impersonated_credentials.py deleted file mode 100644 index 32dfe830..00000000 --- a/src/google/auth/impersonated_credentials.py +++ /dev/null @@ -1,231 +0,0 @@ -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Impersonated credentials. - -This module provides authentication for applications where local credentials -impersonates a remote service account using `IAM Credentials API`_. - -This class can be used to impersonate a service account as long as the original -Credential object has the "Service Account Token Creator" role on the target -service account. - - .. _IAM Credentials API: - https://cloud.google.com/iam/credentials/reference/rest/ -""" - -import copy -from datetime import datetime -import json - -import six -from six.moves import http_client - -from google.auth import _helpers -from google.auth import credentials -from google.auth import exceptions - -_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds - -_IAM_SCOPE = ['https://www.googleapis.com/auth/iam'] - -_IAM_ENDPOINT = ('https://iamcredentials.googleapis.com/v1/projects/-' + - '/serviceAccounts/{}:generateAccessToken') - -_REFRESH_ERROR = 'Unable to acquire impersonated credentials' - - -def _make_iam_token_request(request, principal, headers, body): - """Makes a request to the Google Cloud IAM service for an access token. - Args: - request (Request): The Request object to use. - principal (str): The principal to request an access token for. - headers (Mapping[str, str]): Map of headers to transmit. - body (Mapping[str, str]): JSON Payload body for the iamcredentials - API call. - - Raises: - TransportError: Raised if there is an underlying HTTP connection - Error - DefaultCredentialsError: Raised if the impersonated credentials - are not available. Common reasons are - `iamcredentials.googleapis.com` is not enabled or the - `Service Account Token Creator` is not assigned - """ - iam_endpoint = _IAM_ENDPOINT.format(principal) - - body = json.dumps(body) - - response = request( - url=iam_endpoint, - method='POST', - headers=headers, - body=body) - - response_body = response.data.decode('utf-8') - - if response.status != http_client.OK: - exceptions.RefreshError(_REFRESH_ERROR, response_body) - - try: - token_response = json.loads(response.data.decode('utf-8')) - token = token_response['accessToken'] - expiry = datetime.strptime( - token_response['expireTime'], '%Y-%m-%dT%H:%M:%SZ') - - return token, expiry - - except (KeyError, ValueError) as caught_exc: - new_exc = exceptions.RefreshError( - '{}: No access token or invalid expiration in response.'.format( - _REFRESH_ERROR), - response_body) - six.raise_from(new_exc, caught_exc) - - -class Credentials(credentials.Credentials): - """This module defines impersonated credentials which are essentially - impersonated identities. - - Impersonated Credentials allows credentials issued to a user or - service account to impersonate another. The target service account must - grant the originating credential principal the - `Service Account Token Creator`_ IAM role: - - For more information about Token Creator IAM role and - IAMCredentials API, see - `Creating Short-Lived Service Account Credentials`_. - - .. _Service Account Token Creator: - https://cloud.google.com/iam/docs/service-accounts#the_service_account_token_creator_role - - .. _Creating Short-Lived Service Account Credentials: - https://cloud.google.com/iam/docs/creating-short-lived-service-account-credentials - - Usage: - - First grant source_credentials the `Service Account Token Creator` - role on the target account to impersonate. In this example, the - service account represented by svc_account.json has the - token creator role on - `impersonated-account@_project_.iam.gserviceaccount.com`. - - Enable the IAMCredentials API on the source project: - `gcloud services enable iamcredentials.googleapis.com`. - - Initialize a source credential which does not have access to - list bucket:: - - from google.oauth2 import service_acccount - - target_scopes = [ - 'https://www.googleapis.com/auth/devstorage.read_only'] - - source_credentials = ( - service_account.Credentials.from_service_account_file( - '/path/to/svc_account.json', - scopes=target_scopes)) - - Now use the source credentials to acquire credentials to impersonate - another service account:: - - from google.auth import impersonated_credentials - - target_credentials = impersonated_credentials.Credentials( - source_credentials=source_credentials, - target_principal='impersonated-account@_project_.iam.gserviceaccount.com', - target_scopes = target_scopes, - lifetime=500) - - Resource access is granted:: - - client = storage.Client(credentials=target_credentials) - buckets = client.list_buckets(project='your_project') - for bucket in buckets: - print bucket.name - """ - - def __init__(self, source_credentials, target_principal, - target_scopes, delegates=None, - lifetime=_DEFAULT_TOKEN_LIFETIME_SECS): - """ - Args: - source_credentials (google.auth.Credentials): The source credential - used as to acquire the impersonated credentials. - target_principal (str): The service account to impersonate. - target_scopes (Sequence[str]): Scopes to request during the - authorization grant. - delegates (Sequence[str]): The chained list of delegates required - to grant the final access_token. If set, the sequence of - identities must have "Service Account Token Creator" capability - granted to the prceeding identity. For example, if set to - [serviceAccountB, serviceAccountC], the source_credential - must have the Token Creator role on serviceAccountB. - serviceAccountB must have the Token Creator on serviceAccountC. - Finally, C must have Token Creator on target_principal. - If left unset, source_credential must have that role on - target_principal. - lifetime (int): Number of seconds the delegated credential should - be valid for (upto 3600). - """ - - super(Credentials, self).__init__() - - self._source_credentials = copy.copy(source_credentials) - self._source_credentials._scopes = _IAM_SCOPE - self._target_principal = target_principal - self._target_scopes = target_scopes - self._delegates = delegates - self._lifetime = lifetime - self.token = None - self.expiry = _helpers.utcnow() - - @_helpers.copy_docstring(credentials.Credentials) - def refresh(self, request): - self._update_token(request) - - @property - def expired(self): - return _helpers.utcnow() >= self.expiry - - def _update_token(self, request): - """Updates credentials with a new access_token representing - the impersonated account. - - Args: - request (google.auth.transport.requests.Request): Request object - to use for refreshing credentials. - """ - - # Refresh our source credentials. - self._source_credentials.refresh(request) - - body = { - "delegates": self._delegates, - "scope": self._target_scopes, - "lifetime": str(self._lifetime) + "s" - } - - headers = { - 'Content-Type': 'application/json', - } - - # Apply the source credentials authentication info. - self._source_credentials.apply(headers) - - self.token, self.expiry = _make_iam_token_request( - request=request, - principal=self._target_principal, - headers=headers, - body=body) diff --git a/src/google/auth/jwt.py b/src/google/auth/jwt.py deleted file mode 100644 index 3805f371..00000000 --- a/src/google/auth/jwt.py +++ /dev/null @@ -1,757 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""JSON Web Tokens - -Provides support for creating (encoding) and verifying (decoding) JWTs, -especially JWTs generated and consumed by Google infrastructure. - -See `rfc7519`_ for more details on JWTs. - -To encode a JWT use :func:`encode`:: - - from google.auth import crypt - from google.auth import jwt - - signer = crypt.Signer(private_key) - payload = {'some': 'payload'} - encoded = jwt.encode(signer, payload) - -To decode a JWT and verify claims use :func:`decode`:: - - claims = jwt.decode(encoded, certs=public_certs) - -You can also skip verification:: - - claims = jwt.decode(encoded, verify=False) - -.. _rfc7519: https://tools.ietf.org/html/rfc7519 - -""" - -import base64 -import collections -import copy -import datetime -import json - -import cachetools -import six -from six.moves import urllib - -from google.auth import _helpers -from google.auth import _service_account_info -from google.auth import crypt -from google.auth import exceptions -import google.auth.credentials - -_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds -_DEFAULT_MAX_CACHE_SIZE = 10 - - -def encode(signer, payload, header=None, key_id=None): - """Make a signed JWT. - - Args: - signer (google.auth.crypt.Signer): The signer used to sign the JWT. - payload (Mapping[str, str]): The JWT payload. - header (Mapping[str, str]): Additional JWT header payload. - key_id (str): The key id to add to the JWT header. If the - signer has a key id it will be used as the default. If this is - specified it will override the signer's key id. - - Returns: - bytes: The encoded JWT. - """ - if header is None: - header = {} - - if key_id is None: - key_id = signer.key_id - - header.update({'typ': 'JWT', 'alg': 'RS256'}) - - if key_id is not None: - header['kid'] = key_id - - segments = [ - base64.urlsafe_b64encode(json.dumps(header).encode('utf-8')), - base64.urlsafe_b64encode(json.dumps(payload).encode('utf-8')), - ] - - signing_input = b'.'.join(segments) - signature = signer.sign(signing_input) - segments.append(base64.urlsafe_b64encode(signature)) - - return b'.'.join(segments) - - -def _decode_jwt_segment(encoded_section): - """Decodes a single JWT segment.""" - section_bytes = _helpers.padded_urlsafe_b64decode(encoded_section) - try: - return json.loads(section_bytes.decode('utf-8')) - except ValueError as caught_exc: - new_exc = ValueError('Can\'t parse segment: {0}'.format(section_bytes)) - six.raise_from(new_exc, caught_exc) - - -def _unverified_decode(token): - """Decodes a token and does no verification. - - Args: - token (Union[str, bytes]): The encoded JWT. - - Returns: - Tuple[str, str, str, str]: header, payload, signed_section, and - signature. - - Raises: - ValueError: if there are an incorrect amount of segments in the token. - """ - token = _helpers.to_bytes(token) - - if token.count(b'.') != 2: - raise ValueError( - 'Wrong number of segments in token: {0}'.format(token)) - - encoded_header, encoded_payload, signature = token.split(b'.') - signed_section = encoded_header + b'.' + encoded_payload - signature = _helpers.padded_urlsafe_b64decode(signature) - - # Parse segments - header = _decode_jwt_segment(encoded_header) - payload = _decode_jwt_segment(encoded_payload) - - return header, payload, signed_section, signature - - -def decode_header(token): - """Return the decoded header of a token. - - No verification is done. This is useful to extract the key id from - the header in order to acquire the appropriate certificate to verify - the token. - - Args: - token (Union[str, bytes]): the encoded JWT. - - Returns: - Mapping: The decoded JWT header. - """ - header, _, _, _ = _unverified_decode(token) - return header - - -def _verify_iat_and_exp(payload): - """Verifies the ``iat`` (Issued At) and ``exp`` (Expires) claims in a token - payload. - - Args: - payload (Mapping[str, str]): The JWT payload. - - Raises: - ValueError: if any checks failed. - """ - now = _helpers.datetime_to_secs(_helpers.utcnow()) - - # Make sure the iat and exp claims are present. - for key in ('iat', 'exp'): - if key not in payload: - raise ValueError( - 'Token does not contain required claim {}'.format(key)) - - # Make sure the token wasn't issued in the future. - iat = payload['iat'] - # Err on the side of accepting a token that is slightly early to account - # for clock skew. - earliest = iat - _helpers.CLOCK_SKEW_SECS - if now < earliest: - raise ValueError('Token used too early, {} < {}'.format(now, iat)) - - # Make sure the token wasn't issued in the past. - exp = payload['exp'] - # Err on the side of accepting a token that is slightly out of date - # to account for clow skew. - latest = exp + _helpers.CLOCK_SKEW_SECS - if latest < now: - raise ValueError('Token expired, {} < {}'.format(latest, now)) - - -def decode(token, certs=None, verify=True, audience=None): - """Decode and verify a JWT. - - Args: - token (str): The encoded JWT. - certs (Union[str, bytes, Mapping[str, Union[str, bytes]]]): The - certificate used to validate the JWT signatyre. If bytes or string, - it must the the public key certificate in PEM format. If a mapping, - it must be a mapping of key IDs to public key certificates in PEM - format. The mapping must contain the same key ID that's specified - in the token's header. - verify (bool): Whether to perform signature and claim validation. - Verification is done by default. - audience (str): The audience claim, 'aud', that this JWT should - contain. If None then the JWT's 'aud' parameter is not verified. - - Returns: - Mapping[str, str]: The deserialized JSON payload in the JWT. - - Raises: - ValueError: if any verification checks failed. - """ - header, payload, signed_section, signature = _unverified_decode(token) - - if not verify: - return payload - - # If certs is specified as a dictionary of key IDs to certificates, then - # use the certificate identified by the key ID in the token header. - if isinstance(certs, collections.Mapping): - key_id = header.get('kid') - if key_id: - if key_id not in certs: - raise ValueError( - 'Certificate for key id {} not found.'.format(key_id)) - certs_to_check = [certs[key_id]] - # If there's no key id in the header, check against all of the certs. - else: - certs_to_check = certs.values() - else: - certs_to_check = certs - - # Verify that the signature matches the message. - if not crypt.verify_signature(signed_section, signature, certs_to_check): - raise ValueError('Could not verify token signature.') - - # Verify the issued at and created times in the payload. - _verify_iat_and_exp(payload) - - # Check audience. - if audience is not None: - claim_audience = payload.get('aud') - if audience != claim_audience: - raise ValueError( - 'Token has wrong audience {}, expected {}'.format( - claim_audience, audience)) - - return payload - - -class Credentials(google.auth.credentials.Signing, - google.auth.credentials.Credentials): - """Credentials that use a JWT as the bearer token. - - These credentials require an "audience" claim. This claim identifies the - intended recipient of the bearer token. - - The constructor arguments determine the claims for the JWT that is - sent with requests. Usually, you'll construct these credentials with - one of the helper constructors as shown in the next section. - - To create JWT credentials using a Google service account private key - JSON file:: - - audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher' - credentials = jwt.Credentials.from_service_account_file( - 'service-account.json', - audience=audience) - - If you already have the service account file loaded and parsed:: - - service_account_info = json.load(open('service_account.json')) - credentials = jwt.Credentials.from_service_account_info( - service_account_info, - audience=audience) - - Both helper methods pass on arguments to the constructor, so you can - specify the JWT claims:: - - credentials = jwt.Credentials.from_service_account_file( - 'service-account.json', - audience=audience, - additional_claims={'meta': 'data'}) - - You can also construct the credentials directly if you have a - :class:`~google.auth.crypt.Signer` instance:: - - credentials = jwt.Credentials( - signer, - issuer='your-issuer', - subject='your-subject', - audience=audience) - - The claims are considered immutable. If you want to modify the claims, - you can easily create another instance using :meth:`with_claims`:: - - new_audience = ( - 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber') - new_credentials = credentials.with_claims(audience=new_audience) - """ - - def __init__(self, signer, issuer, subject, audience, - additional_claims=None, - token_lifetime=_DEFAULT_TOKEN_LIFETIME_SECS): - """ - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - issuer (str): The `iss` claim. - subject (str): The `sub` claim. - audience (str): the `aud` claim. The intended audience for the - credentials. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT payload. - token_lifetime (int): The amount of time in seconds for - which the token is valid. Defaults to 1 hour. - """ - super(Credentials, self).__init__() - self._signer = signer - self._issuer = issuer - self._subject = subject - self._audience = audience - self._token_lifetime = token_lifetime - - if additional_claims is None: - additional_claims = {} - - self._additional_claims = additional_claims - - @classmethod - def _from_signer_and_info(cls, signer, info, **kwargs): - """Creates a Credentials instance from a signer and service account - info. - - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - info (Mapping[str, str]): The service account info. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.Credentials: The constructed credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - kwargs.setdefault('subject', info['client_email']) - kwargs.setdefault('issuer', info['client_email']) - return cls(signer, **kwargs) - - @classmethod - def from_service_account_info(cls, info, **kwargs): - """Creates an Credentials instance from a dictionary. - - Args: - info (Mapping[str, str]): The service account info in Google - format. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.Credentials: The constructed credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - signer = _service_account_info.from_dict( - info, require=['client_email']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @classmethod - def from_service_account_file(cls, filename, **kwargs): - """Creates a Credentials instance from a service account .json file - in Google format. - - Args: - filename (str): The path to the service account .json file. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.Credentials: The constructed credentials. - """ - info, signer = _service_account_info.from_filename( - filename, require=['client_email']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @classmethod - def from_signing_credentials(cls, credentials, audience, **kwargs): - """Creates a new :class:`google.auth.jwt.Credentials` instance from an - existing :class:`google.auth.credentials.Signing` instance. - - The new instance will use the same signer as the existing instance and - will use the existing instance's signer email as the issuer and - subject by default. - - Example:: - - svc_creds = service_account.Credentials.from_service_account_file( - 'service_account.json') - audience = ( - 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher') - jwt_creds = jwt.Credentials.from_signing_credentials( - svc_creds, audience=audience) - - Args: - credentials (google.auth.credentials.Signing): The credentials to - use to construct the new credentials. - audience (str): the `aud` claim. The intended audience for the - credentials. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.Credentials: A new Credentials instance. - """ - kwargs.setdefault('issuer', credentials.signer_email) - kwargs.setdefault('subject', credentials.signer_email) - return cls( - credentials.signer, - audience=audience, - **kwargs) - - def with_claims(self, issuer=None, subject=None, audience=None, - additional_claims=None): - """Returns a copy of these credentials with modified claims. - - Args: - issuer (str): The `iss` claim. If unspecified the current issuer - claim will be used. - subject (str): The `sub` claim. If unspecified the current subject - claim will be used. - audience (str): the `aud` claim. If unspecified the current - audience claim will be used. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT payload. This will be merged with the current - additional claims. - - Returns: - google.auth.jwt.Credentials: A new credentials instance. - """ - new_additional_claims = copy.deepcopy(self._additional_claims) - new_additional_claims.update(additional_claims or {}) - - return self.__class__( - self._signer, - issuer=issuer if issuer is not None else self._issuer, - subject=subject if subject is not None else self._subject, - audience=audience if audience is not None else self._audience, - additional_claims=new_additional_claims) - - def _make_jwt(self): - """Make a signed JWT. - - Returns: - Tuple[bytes, datetime]: The encoded JWT and the expiration. - """ - now = _helpers.utcnow() - lifetime = datetime.timedelta(seconds=self._token_lifetime) - expiry = now + lifetime - - payload = { - 'iss': self._issuer, - 'sub': self._subject, - 'iat': _helpers.datetime_to_secs(now), - 'exp': _helpers.datetime_to_secs(expiry), - 'aud': self._audience, - } - - payload.update(self._additional_claims) - - jwt = encode(self._signer, payload) - - return jwt, expiry - - def refresh(self, request): - """Refreshes the access token. - - Args: - request (Any): Unused. - """ - # pylint: disable=unused-argument - # (pylint doesn't correctly recognize overridden methods.) - self.token, self.expiry = self._make_jwt() - - @_helpers.copy_docstring(google.auth.credentials.Signing) - def sign_bytes(self, message): - return self._signer.sign(message) - - @property - @_helpers.copy_docstring(google.auth.credentials.Signing) - def signer_email(self): - return self._issuer - - @property - @_helpers.copy_docstring(google.auth.credentials.Signing) - def signer(self): - return self._signer - - -class OnDemandCredentials( - google.auth.credentials.Signing, - google.auth.credentials.Credentials): - """On-demand JWT credentials. - - Like :class:`Credentials`, this class uses a JWT as the bearer token for - authentication. However, this class does not require the audience at - construction time. Instead, it will generate a new token on-demand for - each request using the request URI as the audience. It caches tokens - so that multiple requests to the same URI do not incur the overhead - of generating a new token every time. - - This behavior is especially useful for `gRPC`_ clients. A gRPC service may - have multiple audience and gRPC clients may not know all of the audiences - required for accessing a particular service. With these credentials, - no knowledge of the audiences is required ahead of time. - - .. _grpc: http://www.grpc.io/ - """ - - def __init__(self, signer, issuer, subject, - additional_claims=None, - token_lifetime=_DEFAULT_TOKEN_LIFETIME_SECS, - max_cache_size=_DEFAULT_MAX_CACHE_SIZE): - """ - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - issuer (str): The `iss` claim. - subject (str): The `sub` claim. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT payload. - token_lifetime (int): The amount of time in seconds for - which the token is valid. Defaults to 1 hour. - max_cache_size (int): The maximum number of JWT tokens to keep in - cache. Tokens are cached using :class:`cachetools.LRUCache`. - """ - super(OnDemandCredentials, self).__init__() - self._signer = signer - self._issuer = issuer - self._subject = subject - self._token_lifetime = token_lifetime - - if additional_claims is None: - additional_claims = {} - - self._additional_claims = additional_claims - self._cache = cachetools.LRUCache(maxsize=max_cache_size) - - @classmethod - def _from_signer_and_info(cls, signer, info, **kwargs): - """Creates an OnDemandCredentials instance from a signer and service - account info. - - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - info (Mapping[str, str]): The service account info. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.OnDemandCredentials: The constructed credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - kwargs.setdefault('subject', info['client_email']) - kwargs.setdefault('issuer', info['client_email']) - return cls(signer, **kwargs) - - @classmethod - def from_service_account_info(cls, info, **kwargs): - """Creates an OnDemandCredentials instance from a dictionary. - - Args: - info (Mapping[str, str]): The service account info in Google - format. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.OnDemandCredentials: The constructed credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - signer = _service_account_info.from_dict( - info, require=['client_email']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @classmethod - def from_service_account_file(cls, filename, **kwargs): - """Creates an OnDemandCredentials instance from a service account .json - file in Google format. - - Args: - filename (str): The path to the service account .json file. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.OnDemandCredentials: The constructed credentials. - """ - info, signer = _service_account_info.from_filename( - filename, require=['client_email']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @classmethod - def from_signing_credentials(cls, credentials, **kwargs): - """Creates a new :class:`google.auth.jwt.OnDemandCredentials` instance - from an existing :class:`google.auth.credentials.Signing` instance. - - The new instance will use the same signer as the existing instance and - will use the existing instance's signer email as the issuer and - subject by default. - - Example:: - - svc_creds = service_account.Credentials.from_service_account_file( - 'service_account.json') - jwt_creds = jwt.OnDemandCredentials.from_signing_credentials( - svc_creds) - - Args: - credentials (google.auth.credentials.Signing): The credentials to - use to construct the new credentials. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.Credentials: A new Credentials instance. - """ - kwargs.setdefault('issuer', credentials.signer_email) - kwargs.setdefault('subject', credentials.signer_email) - return cls(credentials.signer, **kwargs) - - def with_claims(self, issuer=None, subject=None, additional_claims=None): - """Returns a copy of these credentials with modified claims. - - Args: - issuer (str): The `iss` claim. If unspecified the current issuer - claim will be used. - subject (str): The `sub` claim. If unspecified the current subject - claim will be used. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT payload. This will be merged with the current - additional claims. - - Returns: - google.auth.jwt.OnDemandCredentials: A new credentials instance. - """ - new_additional_claims = copy.deepcopy(self._additional_claims) - new_additional_claims.update(additional_claims or {}) - - return self.__class__( - self._signer, - issuer=issuer if issuer is not None else self._issuer, - subject=subject if subject is not None else self._subject, - additional_claims=new_additional_claims, - max_cache_size=self._cache.maxsize) - - @property - def valid(self): - """Checks the validity of the credentials. - - These credentials are always valid because it generates tokens on - demand. - """ - return True - - def _make_jwt_for_audience(self, audience): - """Make a new JWT for the given audience. - - Args: - audience (str): The intended audience. - - Returns: - Tuple[bytes, datetime]: The encoded JWT and the expiration. - """ - now = _helpers.utcnow() - lifetime = datetime.timedelta(seconds=self._token_lifetime) - expiry = now + lifetime - - payload = { - 'iss': self._issuer, - 'sub': self._subject, - 'iat': _helpers.datetime_to_secs(now), - 'exp': _helpers.datetime_to_secs(expiry), - 'aud': audience, - } - - payload.update(self._additional_claims) - - jwt = encode(self._signer, payload) - - return jwt, expiry - - def _get_jwt_for_audience(self, audience): - """Get a JWT For a given audience. - - If there is already an existing, non-expired token in the cache for - the audience, that token is used. Otherwise, a new token will be - created. - - Args: - audience (str): The intended audience. - - Returns: - bytes: The encoded JWT. - """ - token, expiry = self._cache.get(audience, (None, None)) - - if token is None or expiry < _helpers.utcnow(): - token, expiry = self._make_jwt_for_audience(audience) - self._cache[audience] = token, expiry - - return token - - def refresh(self, request): - """Raises an exception, these credentials can not be directly - refreshed. - - Args: - request (Any): Unused. - - Raises: - google.auth.RefreshError - """ - # pylint: disable=unused-argument - # (pylint doesn't correctly recognize overridden methods.) - raise exceptions.RefreshError( - 'OnDemandCredentials can not be directly refreshed.') - - def before_request(self, request, method, url, headers): - """Performs credential-specific before request logic. - - Args: - request (Any): Unused. JWT credentials do not need to make an - HTTP request to refresh. - method (str): The request's HTTP method. - url (str): The request's URI. This is used as the audience claim - when generating the JWT. - headers (Mapping): The request's headers. - """ - # pylint: disable=unused-argument - # (pylint doesn't correctly recognize overridden methods.) - parts = urllib.parse.urlsplit(url) - # Strip query string and fragment - audience = urllib.parse.urlunsplit( - (parts.scheme, parts.netloc, parts.path, "", "")) - token = self._get_jwt_for_audience(audience) - self.apply(headers, token=token) - - @_helpers.copy_docstring(google.auth.credentials.Signing) - def sign_bytes(self, message): - return self._signer.sign(message) - - @property - @_helpers.copy_docstring(google.auth.credentials.Signing) - def signer_email(self): - return self._issuer - - @property - @_helpers.copy_docstring(google.auth.credentials.Signing) - def signer(self): - return self._signer diff --git a/src/google/auth/transport/__init__.py b/src/google/auth/transport/__init__.py deleted file mode 100644 index d73c63cd..00000000 --- a/src/google/auth/transport/__init__.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Transport - HTTP client library support. - -:mod:`google.auth` is designed to work with various HTTP client libraries such -as urllib3 and requests. In order to work across these libraries with different -interfaces some abstraction is needed. - -This module provides two interfaces that are implemented by transport adapters -to support HTTP libraries. :class:`Request` defines the interface expected by -:mod:`google.auth` to make requests. :class:`Response` defines the interface -for the return value of :class:`Request`. -""" - -import abc - -import six -from six.moves import http_client - -DEFAULT_REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,) -"""Sequence[int]: Which HTTP status code indicate that credentials should be -refreshed and a request should be retried. -""" - -DEFAULT_MAX_REFRESH_ATTEMPTS = 2 -"""int: How many times to refresh the credentials and retry a request.""" - - -@six.add_metaclass(abc.ABCMeta) -class Response(object): - """HTTP Response data.""" - - @abc.abstractproperty - def status(self): - """int: The HTTP status code.""" - raise NotImplementedError('status must be implemented.') - - @abc.abstractproperty - def headers(self): - """Mapping[str, str]: The HTTP response headers.""" - raise NotImplementedError('headers must be implemented.') - - @abc.abstractproperty - def data(self): - """bytes: The response body.""" - raise NotImplementedError('data must be implemented.') - - -@six.add_metaclass(abc.ABCMeta) -class Request(object): - """Interface for a callable that makes HTTP requests. - - Specific transport implementations should provide an implementation of - this that adapts their specific request / response API. - - .. automethod:: __call__ - """ - - @abc.abstractmethod - def __call__(self, url, method='GET', body=None, headers=None, - timeout=None, **kwargs): - """Make an HTTP request. - - Args: - url (str): The URI to be requested. - method (str): The HTTP method to use for the request. Defaults - to 'GET'. - body (bytes): The payload / body in HTTP request. - headers (Mapping[str, str]): Request headers. - timeout (Optional[int]): The number of seconds to wait for a - response from the server. If not specified or if None, the - transport-specific default timeout will be used. - kwargs: Additionally arguments passed on to the transport's - request method. - - Returns: - Response: The HTTP response. - - Raises: - google.auth.exceptions.TransportError: If any exception occurred. - """ - # pylint: disable=redundant-returns-doc, missing-raises-doc - # (pylint doesn't play well with abstract docstrings.) - raise NotImplementedError('__call__ must be implemented.') diff --git a/src/google/auth/transport/_http_client.py b/src/google/auth/transport/_http_client.py deleted file mode 100644 index 08b1ab6c..00000000 --- a/src/google/auth/transport/_http_client.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Transport adapter for http.client, for internal use only.""" - -import logging -import socket - -import six -from six.moves import http_client -from six.moves import urllib - -from google.auth import exceptions -from google.auth import transport - -_LOGGER = logging.getLogger(__name__) - - -class Response(transport.Response): - """http.client transport response adapter. - - Args: - response (http.client.HTTPResponse): The raw http client response. - """ - def __init__(self, response): - self._status = response.status - self._headers = { - key.lower(): value for key, value in response.getheaders()} - self._data = response.read() - - @property - def status(self): - return self._status - - @property - def headers(self): - return self._headers - - @property - def data(self): - return self._data - - -class Request(transport.Request): - """http.client transport request adapter.""" - - def __call__(self, url, method='GET', body=None, headers=None, - timeout=None, **kwargs): - """Make an HTTP request using http.client. - - Args: - url (str): The URI to be requested. - method (str): The HTTP method to use for the request. Defaults - to 'GET'. - body (bytes): The payload / body in HTTP request. - headers (Mapping): Request headers. - timeout (Optional(int)): The number of seconds to wait for a - response from the server. If not specified or if None, the - socket global default timeout will be used. - kwargs: Additional arguments passed throught to the underlying - :meth:`~http.client.HTTPConnection.request` method. - - Returns: - Response: The HTTP response. - - Raises: - google.auth.exceptions.TransportError: If any exception occurred. - """ - # socket._GLOBAL_DEFAULT_TIMEOUT is the default in http.client. - if timeout is None: - timeout = socket._GLOBAL_DEFAULT_TIMEOUT - - # http.client doesn't allow None as the headers argument. - if headers is None: - headers = {} - - # http.client needs the host and path parts specified separately. - parts = urllib.parse.urlsplit(url) - path = urllib.parse.urlunsplit( - ('', '', parts.path, parts.query, parts.fragment)) - - if parts.scheme != 'http': - raise exceptions.TransportError( - 'http.client transport only supports the http scheme, {}' - 'was specified'.format(parts.scheme)) - - connection = http_client.HTTPConnection(parts.netloc, timeout=timeout) - - try: - _LOGGER.debug('Making request: %s %s', method, url) - - connection.request( - method, path, body=body, headers=headers, **kwargs) - response = connection.getresponse() - return Response(response) - - except (http_client.HTTPException, socket.error) as caught_exc: - new_exc = exceptions.TransportError(caught_exc) - six.raise_from(new_exc, caught_exc) - - finally: - connection.close() diff --git a/src/google/auth/transport/grpc.py b/src/google/auth/transport/grpc.py deleted file mode 100644 index 0d44f645..00000000 --- a/src/google/auth/transport/grpc.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Authorization support for gRPC.""" - -from __future__ import absolute_import - -import six -try: - import grpc -except ImportError as caught_exc: # pragma: NO COVER - six.raise_from( - ImportError( - 'gRPC is not installed, please install the grpcio package ' - 'to use the gRPC transport.' - ), - caught_exc, - ) - - -class AuthMetadataPlugin(grpc.AuthMetadataPlugin): - """A `gRPC AuthMetadataPlugin`_ that inserts the credentials into each - request. - - .. _gRPC AuthMetadataPlugin: - http://www.grpc.io/grpc/python/grpc.html#grpc.AuthMetadataPlugin - - Args: - credentials (google.auth.credentials.Credentials): The credentials to - add to requests. - request (google.auth.transport.Request): A HTTP transport request - object used to refresh credentials as needed. - """ - def __init__(self, credentials, request): - # pylint: disable=no-value-for-parameter - # pylint doesn't realize that the super method takes no arguments - # because this class is the same name as the superclass. - super(AuthMetadataPlugin, self).__init__() - self._credentials = credentials - self._request = request - - def _get_authorization_headers(self, context): - """Gets the authorization headers for a request. - - Returns: - Sequence[Tuple[str, str]]: A list of request headers (key, value) - to add to the request. - """ - headers = {} - self._credentials.before_request( - self._request, - context.method_name, - context.service_url, - headers) - - return list(six.iteritems(headers)) - - def __call__(self, context, callback): - """Passes authorization metadata into the given callback. - - Args: - context (grpc.AuthMetadataContext): The RPC context. - callback (grpc.AuthMetadataPluginCallback): The callback that will - be invoked to pass in the authorization metadata. - """ - callback(self._get_authorization_headers(context), None) - - -def secure_authorized_channel( - credentials, request, target, ssl_credentials=None, **kwargs): - """Creates a secure authorized gRPC channel. - - This creates a channel with SSL and :class:`AuthMetadataPlugin`. This - channel can be used to create a stub that can make authorized requests. - - Example:: - - import google.auth - import google.auth.transport.grpc - import google.auth.transport.requests - from google.cloud.speech.v1 import cloud_speech_pb2 - - # Get credentials. - credentials, _ = google.auth.default() - - # Get an HTTP request function to refresh credentials. - request = google.auth.transport.requests.Request() - - # Create a channel. - channel = google.auth.transport.grpc.secure_authorized_channel( - credentials, 'speech.googleapis.com:443', request) - - # Use the channel to create a stub. - cloud_speech.create_Speech_stub(channel) - - Args: - credentials (google.auth.credentials.Credentials): The credentials to - add to requests. - request (google.auth.transport.Request): A HTTP transport request - object used to refresh credentials as needed. Even though gRPC - is a separate transport, there's no way to refresh the credentials - without using a standard http transport. - target (str): The host and port of the service. - ssl_credentials (grpc.ChannelCredentials): Optional SSL channel - credentials. This can be used to specify different certificates. - kwargs: Additional arguments to pass to :func:`grpc.secure_channel`. - - Returns: - grpc.Channel: The created gRPC channel. - """ - # Create the metadata plugin for inserting the authorization header. - metadata_plugin = AuthMetadataPlugin(credentials, request) - - # Create a set of grpc.CallCredentials using the metadata plugin. - google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin) - - if ssl_credentials is None: - ssl_credentials = grpc.ssl_channel_credentials() - - # Combine the ssl credentials and the authorization credentials. - composite_credentials = grpc.composite_channel_credentials( - ssl_credentials, google_auth_credentials) - - return grpc.secure_channel(target, composite_credentials, **kwargs) diff --git a/src/google/auth/transport/requests.py b/src/google/auth/transport/requests.py deleted file mode 100644 index 2268243a..00000000 --- a/src/google/auth/transport/requests.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Transport adapter for Requests.""" - -from __future__ import absolute_import - -import functools -import logging - -try: - import requests -except ImportError as caught_exc: # pragma: NO COVER - import six - six.raise_from( - ImportError( - 'The requests library is not installed, please install the ' - 'requests package to use the requests transport.' - ), - caught_exc, - ) -import requests.adapters # pylint: disable=ungrouped-imports -import requests.exceptions # pylint: disable=ungrouped-imports -import six # pylint: disable=ungrouped-imports - -from google.auth import exceptions -from google.auth import transport - -_LOGGER = logging.getLogger(__name__) - - -class _Response(transport.Response): - """Requests transport response adapter. - - Args: - response (requests.Response): The raw Requests response. - """ - def __init__(self, response): - self._response = response - - @property - def status(self): - return self._response.status_code - - @property - def headers(self): - return self._response.headers - - @property - def data(self): - return self._response.content - - -class Request(transport.Request): - """Requests request adapter. - - This class is used internally for making requests using various transports - in a consistent way. If you use :class:`AuthorizedSession` you do not need - to construct or use this class directly. - - This class can be useful if you want to manually refresh a - :class:`~google.auth.credentials.Credentials` instance:: - - import google.auth.transport.requests - import requests - - request = google.auth.transport.requests.Request() - - credentials.refresh(request) - - Args: - session (requests.Session): An instance :class:`requests.Session` used - to make HTTP requests. If not specified, a session will be created. - - .. automethod:: __call__ - """ - def __init__(self, session=None): - if not session: - session = requests.Session() - - self.session = session - - def __call__(self, url, method='GET', body=None, headers=None, - timeout=None, **kwargs): - """Make an HTTP request using requests. - - Args: - url (str): The URI to be requested. - method (str): The HTTP method to use for the request. Defaults - to 'GET'. - body (bytes): The payload / body in HTTP request. - headers (Mapping[str, str]): Request headers. - timeout (Optional[int]): The number of seconds to wait for a - response from the server. If not specified or if None, the - requests default timeout will be used. - kwargs: Additional arguments passed through to the underlying - requests :meth:`~requests.Session.request` method. - - Returns: - google.auth.transport.Response: The HTTP response. - - Raises: - google.auth.exceptions.TransportError: If any exception occurred. - """ - try: - _LOGGER.debug('Making request: %s %s', method, url) - response = self.session.request( - method, url, data=body, headers=headers, timeout=timeout, - **kwargs) - return _Response(response) - except requests.exceptions.RequestException as caught_exc: - new_exc = exceptions.TransportError(caught_exc) - six.raise_from(new_exc, caught_exc) - - -class AuthorizedSession(requests.Session): - """A Requests Session class with credentials. - - This class is used to perform requests to API endpoints that require - authorization:: - - from google.auth.transport.requests import AuthorizedSession - - authed_session = AuthorizedSession(credentials) - - response = authed_session.request( - 'GET', 'https://www.googleapis.com/storage/v1/b') - - The underlying :meth:`request` implementation handles adding the - credentials' headers to the request and refreshing credentials as needed. - - Args: - credentials (google.auth.credentials.Credentials): The credentials to - add to the request. - refresh_status_codes (Sequence[int]): Which HTTP status codes indicate - that credentials should be refreshed and the request should be - retried. - max_refresh_attempts (int): The maximum number of times to attempt to - refresh the credentials and retry the request. - refresh_timeout (Optional[int]): The timeout value in seconds for - credential refresh HTTP requests. - kwargs: Additional arguments passed to the :class:`requests.Session` - constructor. - """ - def __init__(self, credentials, - refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES, - max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS, - refresh_timeout=None, - **kwargs): - super(AuthorizedSession, self).__init__(**kwargs) - self.credentials = credentials - self._refresh_status_codes = refresh_status_codes - self._max_refresh_attempts = max_refresh_attempts - self._refresh_timeout = refresh_timeout - - auth_request_session = requests.Session() - - # Using an adapter to make HTTP requests robust to network errors. - # This adapter retrys HTTP requests when network errors occur - # and the requests seems safely retryable. - retry_adapter = requests.adapters.HTTPAdapter(max_retries=3) - auth_request_session.mount("https://", retry_adapter) - - # Request instance used by internal methods (for example, - # credentials.refresh). - # Do not pass `self` as the session here, as it can lead to infinite - # recursion. - self._auth_request = Request(auth_request_session) - - def request(self, method, url, data=None, headers=None, **kwargs): - """Implementation of Requests' request.""" - # pylint: disable=arguments-differ - # Requests has a ton of arguments to request, but only two - # (method, url) are required. We pass through all of the other - # arguments to super, so no need to exhaustively list them here. - - # Use a kwarg for this instead of an attribute to maintain - # thread-safety. - _credential_refresh_attempt = kwargs.pop( - '_credential_refresh_attempt', 0) - - # Make a copy of the headers. They will be modified by the credentials - # and we want to pass the original headers if we recurse. - request_headers = headers.copy() if headers is not None else {} - - self.credentials.before_request( - self._auth_request, method, url, request_headers) - - response = super(AuthorizedSession, self).request( - method, url, data=data, headers=request_headers, **kwargs) - - # If the response indicated that the credentials needed to be - # refreshed, then refresh the credentials and re-attempt the - # request. - # A stored token may expire between the time it is retrieved and - # the time the request is made, so we may need to try twice. - if (response.status_code in self._refresh_status_codes - and _credential_refresh_attempt < self._max_refresh_attempts): - - _LOGGER.info( - 'Refreshing credentials due to a %s response. Attempt %s/%s.', - response.status_code, _credential_refresh_attempt + 1, - self._max_refresh_attempts) - - auth_request_with_timeout = functools.partial( - self._auth_request, timeout=self._refresh_timeout) - self.credentials.refresh(auth_request_with_timeout) - - # Recurse. Pass in the original headers, not our modified set. - return self.request( - method, url, data=data, headers=headers, - _credential_refresh_attempt=_credential_refresh_attempt + 1, - **kwargs) - - return response diff --git a/src/google/auth/transport/urllib3.py b/src/google/auth/transport/urllib3.py deleted file mode 100644 index 37eb3175..00000000 --- a/src/google/auth/transport/urllib3.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Transport adapter for urllib3.""" - -from __future__ import absolute_import - -import logging - - -# Certifi is Mozilla's certificate bundle. Urllib3 needs a certificate bundle -# to verify HTTPS requests, and certifi is the recommended and most reliable -# way to get a root certificate bundle. See -# http://urllib3.readthedocs.io/en/latest/user-guide.html\ -# #certificate-verification -# For more details. -try: - import certifi -except ImportError: # pragma: NO COVER - certifi = None - -try: - import urllib3 -except ImportError as caught_exc: # pragma: NO COVER - import six - six.raise_from( - ImportError( - 'The urllib3 library is not installed, please install the ' - 'urllib3 package to use the urllib3 transport.' - ), - caught_exc, - ) -import six -import urllib3.exceptions # pylint: disable=ungrouped-imports - -from google.auth import exceptions -from google.auth import transport - -_LOGGER = logging.getLogger(__name__) - - -class _Response(transport.Response): - """urllib3 transport response adapter. - - Args: - response (urllib3.response.HTTPResponse): The raw urllib3 response. - """ - def __init__(self, response): - self._response = response - - @property - def status(self): - return self._response.status - - @property - def headers(self): - return self._response.headers - - @property - def data(self): - return self._response.data - - -class Request(transport.Request): - """urllib3 request adapter. - - This class is used internally for making requests using various transports - in a consistent way. If you use :class:`AuthorizedHttp` you do not need - to construct or use this class directly. - - This class can be useful if you want to manually refresh a - :class:`~google.auth.credentials.Credentials` instance:: - - import google.auth.transport.urllib3 - import urllib3 - - http = urllib3.PoolManager() - request = google.auth.transport.urllib3.Request(http) - - credentials.refresh(request) - - Args: - http (urllib3.request.RequestMethods): An instance of any urllib3 - class that implements :class:`~urllib3.request.RequestMethods`, - usually :class:`urllib3.PoolManager`. - - .. automethod:: __call__ - """ - def __init__(self, http): - self.http = http - - def __call__(self, url, method='GET', body=None, headers=None, - timeout=None, **kwargs): - """Make an HTTP request using urllib3. - - Args: - url (str): The URI to be requested. - method (str): The HTTP method to use for the request. Defaults - to 'GET'. - body (bytes): The payload / body in HTTP request. - headers (Mapping[str, str]): Request headers. - timeout (Optional[int]): The number of seconds to wait for a - response from the server. If not specified or if None, the - urllib3 default timeout will be used. - kwargs: Additional arguments passed throught to the underlying - urllib3 :meth:`urlopen` method. - - Returns: - google.auth.transport.Response: The HTTP response. - - Raises: - google.auth.exceptions.TransportError: If any exception occurred. - """ - # urllib3 uses a sentinel default value for timeout, so only set it if - # specified. - if timeout is not None: - kwargs['timeout'] = timeout - - try: - _LOGGER.debug('Making request: %s %s', method, url) - response = self.http.request( - method, url, body=body, headers=headers, **kwargs) - return _Response(response) - except urllib3.exceptions.HTTPError as caught_exc: - new_exc = exceptions.TransportError(caught_exc) - six.raise_from(new_exc, caught_exc) - - -def _make_default_http(): - if certifi is not None: - return urllib3.PoolManager( - cert_reqs='CERT_REQUIRED', - ca_certs=certifi.where()) - else: - return urllib3.PoolManager() - - -class AuthorizedHttp(urllib3.request.RequestMethods): - """A urllib3 HTTP class with credentials. - - This class is used to perform requests to API endpoints that require - authorization:: - - from google.auth.transport.urllib3 import AuthorizedHttp - - authed_http = AuthorizedHttp(credentials) - - response = authed_http.request( - 'GET', 'https://www.googleapis.com/storage/v1/b') - - This class implements :class:`urllib3.request.RequestMethods` and can be - used just like any other :class:`urllib3.PoolManager`. - - The underlying :meth:`urlopen` implementation handles adding the - credentials' headers to the request and refreshing credentials as needed. - - Args: - credentials (google.auth.credentials.Credentials): The credentials to - add to the request. - http (urllib3.PoolManager): The underlying HTTP object to - use to make requests. If not specified, a - :class:`urllib3.PoolManager` instance will be constructed with - sane defaults. - refresh_status_codes (Sequence[int]): Which HTTP status codes indicate - that credentials should be refreshed and the request should be - retried. - max_refresh_attempts (int): The maximum number of times to attempt to - refresh the credentials and retry the request. - """ - def __init__(self, credentials, http=None, - refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES, - max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS): - - if http is None: - http = _make_default_http() - - self.credentials = credentials - self.http = http - self._refresh_status_codes = refresh_status_codes - self._max_refresh_attempts = max_refresh_attempts - # Request instance used by internal methods (for example, - # credentials.refresh). - self._request = Request(self.http) - - super(AuthorizedHttp, self).__init__() - - def urlopen(self, method, url, body=None, headers=None, **kwargs): - """Implementation of urllib3's urlopen.""" - # pylint: disable=arguments-differ - # We use kwargs to collect additional args that we don't need to - # introspect here. However, we do explicitly collect the two - # positional arguments. - - # Use a kwarg for this instead of an attribute to maintain - # thread-safety. - _credential_refresh_attempt = kwargs.pop( - '_credential_refresh_attempt', 0) - - if headers is None: - headers = self.headers - - # Make a copy of the headers. They will be modified by the credentials - # and we want to pass the original headers if we recurse. - request_headers = headers.copy() - - self.credentials.before_request( - self._request, method, url, request_headers) - - response = self.http.urlopen( - method, url, body=body, headers=request_headers, **kwargs) - - # If the response indicated that the credentials needed to be - # refreshed, then refresh the credentials and re-attempt the - # request. - # A stored token may expire between the time it is retrieved and - # the time the request is made, so we may need to try twice. - # The reason urllib3's retries aren't used is because they - # don't allow you to modify the request headers. :/ - if (response.status in self._refresh_status_codes - and _credential_refresh_attempt < self._max_refresh_attempts): - - _LOGGER.info( - 'Refreshing credentials due to a %s response. Attempt %s/%s.', - response.status, _credential_refresh_attempt + 1, - self._max_refresh_attempts) - - self.credentials.refresh(self._request) - - # Recurse. Pass in the original headers, not our modified set. - return self.urlopen( - method, url, body=body, headers=headers, - _credential_refresh_attempt=_credential_refresh_attempt + 1, - **kwargs) - - return response - - # Proxy methods for compliance with the urllib3.PoolManager interface - - def __enter__(self): - """Proxy to ``self.http``.""" - return self.http.__enter__() - - def __exit__(self, exc_type, exc_val, exc_tb): - """Proxy to ``self.http``.""" - return self.http.__exit__(exc_type, exc_val, exc_tb) - - @property - def headers(self): - """Proxy to ``self.http``.""" - return self.http.headers - - @headers.setter - def headers(self, value): - """Proxy to ``self.http``.""" - self.http.headers = value diff --git a/src/google/oauth2/__init__.py b/src/google/oauth2/__init__.py deleted file mode 100644 index 6d3ee7f9..00000000 --- a/src/google/oauth2/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google OAuth 2.0 Library for Python.""" diff --git a/src/google/oauth2/_client.py b/src/google/oauth2/_client.py deleted file mode 100644 index dc35be27..00000000 --- a/src/google/oauth2/_client.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""OAuth 2.0 client. - -This is a client for interacting with an OAuth 2.0 authorization server's -token endpoint. - -For more information about the token endpoint, see -`Section 3.1 of rfc6749`_ - -.. _Section 3.1 of rfc6749: https://tools.ietf.org/html/rfc6749#section-3.2 -""" - -import datetime -import json - -import six -from six.moves import http_client -from six.moves import urllib - -from google.auth import _helpers -from google.auth import exceptions -from google.auth import jwt - -_URLENCODED_CONTENT_TYPE = 'application/x-www-form-urlencoded' -_JWT_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer' -_REFRESH_GRANT_TYPE = 'refresh_token' - - -def _handle_error_response(response_body): - """"Translates an error response into an exception. - - Args: - response_body (str): The decoded response data. - - Raises: - google.auth.exceptions.RefreshError - """ - try: - error_data = json.loads(response_body) - error_details = '{}: {}'.format( - error_data['error'], - error_data.get('error_description')) - # If no details could be extracted, use the response data. - except (KeyError, ValueError): - error_details = response_body - - raise exceptions.RefreshError( - error_details, response_body) - - -def _parse_expiry(response_data): - """Parses the expiry field from a response into a datetime. - - Args: - response_data (Mapping): The JSON-parsed response data. - - Returns: - Optional[datetime]: The expiration or ``None`` if no expiration was - specified. - """ - expires_in = response_data.get('expires_in', None) - - if expires_in is not None: - return _helpers.utcnow() + datetime.timedelta( - seconds=expires_in) - else: - return None - - -def _token_endpoint_request(request, token_uri, body): - """Makes a request to the OAuth 2.0 authorization server's token endpoint. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - token_uri (str): The OAuth 2.0 authorizations server's token endpoint - URI. - body (Mapping[str, str]): The parameters to send in the request body. - - Returns: - Mapping[str, str]: The JSON-decoded response data. - - Raises: - google.auth.exceptions.RefreshError: If the token endpoint returned - an error. - """ - body = urllib.parse.urlencode(body) - headers = { - 'content-type': _URLENCODED_CONTENT_TYPE, - } - - response = request( - method='POST', url=token_uri, headers=headers, body=body) - - response_body = response.data.decode('utf-8') - - if response.status != http_client.OK: - _handle_error_response(response_body) - - response_data = json.loads(response_body) - - return response_data - - -def jwt_grant(request, token_uri, assertion): - """Implements the JWT Profile for OAuth 2.0 Authorization Grants. - - For more details, see `rfc7523 section 4`_. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - token_uri (str): The OAuth 2.0 authorizations server's token endpoint - URI. - assertion (str): The OAuth 2.0 assertion. - - Returns: - Tuple[str, Optional[datetime], Mapping[str, str]]: The access token, - expiration, and additional data returned by the token endpoint. - - Raises: - google.auth.exceptions.RefreshError: If the token endpoint returned - an error. - - .. _rfc7523 section 4: https://tools.ietf.org/html/rfc7523#section-4 - """ - body = { - 'assertion': assertion, - 'grant_type': _JWT_GRANT_TYPE, - } - - response_data = _token_endpoint_request(request, token_uri, body) - - try: - access_token = response_data['access_token'] - except KeyError as caught_exc: - new_exc = exceptions.RefreshError( - 'No access token in response.', response_data) - six.raise_from(new_exc, caught_exc) - - expiry = _parse_expiry(response_data) - - return access_token, expiry, response_data - - -def id_token_jwt_grant(request, token_uri, assertion): - """Implements the JWT Profile for OAuth 2.0 Authorization Grants, but - requests an OpenID Connect ID Token instead of an access token. - - This is a variant on the standard JWT Profile that is currently unique - to Google. This was added for the benefit of authenticating to services - that require ID Tokens instead of access tokens or JWT bearer tokens. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - token_uri (str): The OAuth 2.0 authorization server's token endpoint - URI. - assertion (str): JWT token signed by a service account. The token's - payload must include a ``target_audience`` claim. - - Returns: - Tuple[str, Optional[datetime], Mapping[str, str]]: - The (encoded) Open ID Connect ID Token, expiration, and additional - data returned by the endpoint. - - Raises: - google.auth.exceptions.RefreshError: If the token endpoint returned - an error. - """ - body = { - 'assertion': assertion, - 'grant_type': _JWT_GRANT_TYPE, - } - - response_data = _token_endpoint_request(request, token_uri, body) - - try: - id_token = response_data['id_token'] - except KeyError as caught_exc: - new_exc = exceptions.RefreshError( - 'No ID token in response.', response_data) - six.raise_from(new_exc, caught_exc) - - payload = jwt.decode(id_token, verify=False) - expiry = datetime.datetime.utcfromtimestamp(payload['exp']) - - return id_token, expiry, response_data - - -def refresh_grant(request, token_uri, refresh_token, client_id, client_secret): - """Implements the OAuth 2.0 refresh token grant. - - For more details, see `rfc678 section 6`_. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - token_uri (str): The OAuth 2.0 authorizations server's token endpoint - URI. - refresh_token (str): The refresh token to use to get a new access - token. - client_id (str): The OAuth 2.0 application's client ID. - client_secret (str): The Oauth 2.0 appliaction's client secret. - - Returns: - Tuple[str, Optional[str], Optional[datetime], Mapping[str, str]]: The - access token, new refresh token, expiration, and additional data - returned by the token endpoint. - - Raises: - google.auth.exceptions.RefreshError: If the token endpoint returned - an error. - - .. _rfc6748 section 6: https://tools.ietf.org/html/rfc6749#section-6 - """ - body = { - 'grant_type': _REFRESH_GRANT_TYPE, - 'client_id': client_id, - 'client_secret': client_secret, - 'refresh_token': refresh_token, - } - - response_data = _token_endpoint_request(request, token_uri, body) - - try: - access_token = response_data['access_token'] - except KeyError as caught_exc: - new_exc = exceptions.RefreshError( - 'No access token in response.', response_data) - six.raise_from(new_exc, caught_exc) - - refresh_token = response_data.get('refresh_token', refresh_token) - expiry = _parse_expiry(response_data) - - return access_token, refresh_token, expiry, response_data diff --git a/src/google/oauth2/credentials.py b/src/google/oauth2/credentials.py deleted file mode 100644 index 4cb909cb..00000000 --- a/src/google/oauth2/credentials.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""OAuth 2.0 Credentials. - -This module provides credentials based on OAuth 2.0 access and refresh tokens. -These credentials usually access resources on behalf of a user (resource -owner). - -Specifically, this is intended to use access tokens acquired using the -`Authorization Code grant`_ and can refresh those tokens using a -optional `refresh token`_. - -Obtaining the initial access and refresh token is outside of the scope of this -module. Consult `rfc6749 section 4.1`_ for complete details on the -Authorization Code grant flow. - -.. _Authorization Code grant: https://tools.ietf.org/html/rfc6749#section-1.3.1 -.. _refresh token: https://tools.ietf.org/html/rfc6749#section-6 -.. _rfc6749 section 4.1: https://tools.ietf.org/html/rfc6749#section-4.1 -""" - -import io -import json - -import six - -from google.auth import _helpers -from google.auth import credentials -from google.auth import exceptions -from google.oauth2 import _client - - -# The Google OAuth 2.0 token endpoint. Used for authorized user credentials. -_GOOGLE_OAUTH2_TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token' - - -class Credentials(credentials.ReadOnlyScoped, credentials.Credentials): - """Credentials using OAuth 2.0 access and refresh tokens.""" - - def __init__(self, token, refresh_token=None, id_token=None, - token_uri=None, client_id=None, client_secret=None, - scopes=None): - """ - Args: - token (Optional(str)): The OAuth 2.0 access token. Can be None - if refresh information is provided. - refresh_token (str): The OAuth 2.0 refresh token. If specified, - credentials can be refreshed. - id_token (str): The Open ID Connect ID Token. - token_uri (str): The OAuth 2.0 authorization server's token - endpoint URI. Must be specified for refresh, can be left as - None if the token can not be refreshed. - client_id (str): The OAuth 2.0 client ID. Must be specified for - refresh, can be left as None if the token can not be refreshed. - client_secret(str): The OAuth 2.0 client secret. Must be specified - for refresh, can be left as None if the token can not be - refreshed. - scopes (Sequence[str]): The scopes that were originally used - to obtain authorization. This is a purely informative parameter - that can be used by :meth:`has_scopes`. OAuth 2.0 credentials - can not request additional scopes after authorization. - """ - super(Credentials, self).__init__() - self.token = token - self._refresh_token = refresh_token - self._id_token = id_token - self._scopes = scopes - self._token_uri = token_uri - self._client_id = client_id - self._client_secret = client_secret - - @property - def refresh_token(self): - """Optional[str]: The OAuth 2.0 refresh token.""" - return self._refresh_token - - @property - def token_uri(self): - """Optional[str]: The OAuth 2.0 authorization server's token endpoint - URI.""" - return self._token_uri - - @property - def id_token(self): - """Optional[str]: The Open ID Connect ID Token. - - Depending on the authorization server and the scopes requested, this - may be populated when credentials are obtained and updated when - :meth:`refresh` is called. This token is a JWT. It can be verified - and decoded using :func:`google.oauth2.id_token.verify_oauth2_token`. - """ - return self._id_token - - @property - def client_id(self): - """Optional[str]: The OAuth 2.0 client ID.""" - return self._client_id - - @property - def client_secret(self): - """Optional[str]: The OAuth 2.0 client secret.""" - return self._client_secret - - @property - def requires_scopes(self): - """False: OAuth 2.0 credentials have their scopes set when - the initial token is requested and can not be changed.""" - return False - - @_helpers.copy_docstring(credentials.Credentials) - def refresh(self, request): - if (self._refresh_token is None or - self._token_uri is None or - self._client_id is None or - self._client_secret is None): - raise exceptions.RefreshError( - 'The credentials do not contain the necessary fields need to ' - 'refresh the access token. You must specify refresh_token, ' - 'token_uri, client_id, and client_secret.') - - access_token, refresh_token, expiry, grant_response = ( - _client.refresh_grant( - request, self._token_uri, self._refresh_token, self._client_id, - self._client_secret)) - - self.token = access_token - self.expiry = expiry - self._refresh_token = refresh_token - self._id_token = grant_response.get('id_token') - - @classmethod - def from_authorized_user_info(cls, info, scopes=None): - """Creates a Credentials instance from parsed authorized user info. - - Args: - info (Mapping[str, str]): The authorized user info in Google - format. - scopes (Sequence[str]): Optional list of scopes to include in the - credentials. - - Returns: - google.oauth2.credentials.Credentials: The constructed - credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - keys_needed = set(('refresh_token', 'client_id', 'client_secret')) - missing = keys_needed.difference(six.iterkeys(info)) - - if missing: - raise ValueError( - 'Authorized user info was not in the expected format, missing ' - 'fields {}.'.format(', '.join(missing))) - - return Credentials( - None, # No access token, must be refreshed. - refresh_token=info['refresh_token'], - token_uri=_GOOGLE_OAUTH2_TOKEN_ENDPOINT, - scopes=scopes, - client_id=info['client_id'], - client_secret=info['client_secret']) - - @classmethod - def from_authorized_user_file(cls, filename, scopes=None): - """Creates a Credentials instance from an authorized user json file. - - Args: - filename (str): The path to the authorized user json file. - scopes (Sequence[str]): Optional list of scopes to include in the - credentials. - - Returns: - google.oauth2.credentials.Credentials: The constructed - credentials. - - Raises: - ValueError: If the file is not in the expected format. - """ - with io.open(filename, 'r', encoding='utf-8') as json_file: - data = json.load(json_file) - return cls.from_authorized_user_info(data, scopes) diff --git a/src/google/oauth2/id_token.py b/src/google/oauth2/id_token.py deleted file mode 100644 index 208ab622..00000000 --- a/src/google/oauth2/id_token.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google ID Token helpers. - -Provides support for verifying `OpenID Connect ID Tokens`_, especially ones -generated by Google infrastructure. - -To parse and verify an ID Token issued by Google's OAuth 2.0 authorization -server use :func:`verify_oauth2_token`. To verify an ID Token issued by -Firebase, use :func:`verify_firebase_token`. - -A general purpose ID Token verifier is available as :func:`verify_token`. - -Example:: - - from google.oauth2 import id_token - from google.auth.transport import requests - - request = requests.Request() - - id_info = id_token.verify_oauth2_token( - token, request, 'my-client-id.example.com') - - if id_info['iss'] != 'https://accounts.google.com': - raise ValueError('Wrong issuer.') - - userid = id_info['sub'] - -By default, this will re-fetch certificates for each verification. Because -Google's public keys are only changed infrequently (on the order of once per -day), you may wish to take advantage of caching to reduce latency and the -potential for network errors. This can be accomplished using an external -library like `CacheControl`_ to create a cache-aware -:class:`google.auth.transport.Request`:: - - import cachecontrol - import google.auth.transport.requests - import requests - - session = requests.session() - cached_session = cachecontrol.CacheControl(session) - request = google.auth.transport.requests.Request(session=cached_session) - -.. _OpenID Connect ID Token: - http://openid.net/specs/openid-connect-core-1_0.html#IDToken -.. _CacheControl: https://cachecontrol.readthedocs.io -""" - -import json - -from six.moves import http_client - -from google.auth import exceptions -from google.auth import jwt - -# The URL that provides public certificates for verifying ID tokens issued -# by Google's OAuth 2.0 authorization server. -_GOOGLE_OAUTH2_CERTS_URL = 'https://www.googleapis.com/oauth2/v1/certs' - -# The URL that provides public certificates for verifying ID tokens issued -# by Firebase and the Google APIs infrastructure -_GOOGLE_APIS_CERTS_URL = ( - 'https://www.googleapis.com/robot/v1/metadata/x509' - '/securetoken@system.gserviceaccount.com') - - -def _fetch_certs(request, certs_url): - """Fetches certificates. - - Google-style cerificate endpoints return JSON in the format of - ``{'key id': 'x509 certificate'}``. - - Args: - request (google.auth.transport.Request): The object used to make - HTTP requests. - certs_url (str): The certificate endpoint URL. - - Returns: - Mapping[str, str]: A mapping of public key ID to x.509 certificate - data. - """ - response = request(certs_url, method='GET') - - if response.status != http_client.OK: - raise exceptions.TransportError( - 'Could not fetch certificates at {}'.format(certs_url)) - - return json.loads(response.data.decode('utf-8')) - - -def verify_token(id_token, request, audience=None, - certs_url=_GOOGLE_OAUTH2_CERTS_URL): - """Verifies an ID token and returns the decoded token. - - Args: - id_token (Union[str, bytes]): The encoded token. - request (google.auth.transport.Request): The object used to make - HTTP requests. - audience (str): The audience that this token is intended for. If None - then the audience is not verified. - certs_url (str): The URL that specifies the certificates to use to - verify the token. This URL should return JSON in the format of - ``{'key id': 'x509 certificate'}``. - - Returns: - Mapping[str, Any]: The decoded token. - """ - certs = _fetch_certs(request, certs_url) - - return jwt.decode(id_token, certs=certs, audience=audience) - - -def verify_oauth2_token(id_token, request, audience=None): - """Verifies an ID Token issued by Google's OAuth 2.0 authorization server. - - Args: - id_token (Union[str, bytes]): The encoded token. - request (google.auth.transport.Request): The object used to make - HTTP requests. - audience (str): The audience that this token is intended for. This is - typically your application's OAuth 2.0 client ID. If None then the - audience is not verified. - - Returns: - Mapping[str, Any]: The decoded token. - """ - return verify_token( - id_token, request, audience=audience, - certs_url=_GOOGLE_OAUTH2_CERTS_URL) - - -def verify_firebase_token(id_token, request, audience=None): - """Verifies an ID Token issued by Firebase Authentication. - - Args: - id_token (Union[str, bytes]): The encoded token. - request (google.auth.transport.Request): The object used to make - HTTP requests. - audience (str): The audience that this token is intended for. This is - typically your Firebase application ID. If None then the audience - is not verified. - - Returns: - Mapping[str, Any]: The decoded token. - """ - return verify_token( - id_token, request, audience=audience, certs_url=_GOOGLE_APIS_CERTS_URL) diff --git a/src/google/oauth2/service_account.py b/src/google/oauth2/service_account.py deleted file mode 100644 index c60c5654..00000000 --- a/src/google/oauth2/service_account.py +++ /dev/null @@ -1,542 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Service Accounts: JSON Web Token (JWT) Profile for OAuth 2.0 - -This module implements the JWT Profile for OAuth 2.0 Authorization Grants -as defined by `RFC 7523`_ with particular support for how this RFC is -implemented in Google's infrastructure. Google refers to these credentials -as *Service Accounts*. - -Service accounts are used for server-to-server communication, such as -interactions between a web application server and a Google service. The -service account belongs to your application instead of to an individual end -user. In contrast to other OAuth 2.0 profiles, no users are involved and your -application "acts" as the service account. - -Typically an application uses a service account when the application uses -Google APIs to work with its own data rather than a user's data. For example, -an application that uses Google Cloud Datastore for data persistence would use -a service account to authenticate its calls to the Google Cloud Datastore API. -However, an application that needs to access a user's Drive documents would -use the normal OAuth 2.0 profile. - -Additionally, Google Apps domain administrators can grant service accounts -`domain-wide delegation`_ authority to access user data on behalf of users in -the domain. - -This profile uses a JWT to acquire an OAuth 2.0 access token. The JWT is used -in place of the usual authorization token returned during the standard -OAuth 2.0 Authorization Code grant. The JWT is only used for this purpose, as -the acquired access token is used as the bearer token when making requests -using these credentials. - -This profile differs from normal OAuth 2.0 profile because no user consent -step is required. The use of the private key allows this profile to assert -identity directly. - -This profile also differs from the :mod:`google.auth.jwt` authentication -because the JWT credentials use the JWT directly as the bearer token. This -profile instead only uses the JWT to obtain an OAuth 2.0 access token. The -obtained OAuth 2.0 access token is used as the bearer token. - -Domain-wide delegation ----------------------- - -Domain-wide delegation allows a service account to access user data on -behalf of any user in a Google Apps domain without consent from the user. -For example, an application that uses the Google Calendar API to add events to -the calendars of all users in a Google Apps domain would use a service account -to access the Google Calendar API on behalf of users. - -The Google Apps administrator must explicitly authorize the service account to -do this. This authorization step is referred to as "delegating domain-wide -authority" to a service account. - -You can use domain-wise delegation by creating a set of credentials with a -specific subject using :meth:`~Credentials.with_subject`. - -.. _RFC 7523: https://tools.ietf.org/html/rfc7523 -""" - -import copy -import datetime - -from google.auth import _helpers -from google.auth import _service_account_info -from google.auth import credentials -from google.auth import jwt -from google.oauth2 import _client - -_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds - - -class Credentials(credentials.Signing, - credentials.Scoped, - credentials.Credentials): - """Service account credentials - - Usually, you'll create these credentials with one of the helper - constructors. To create credentials using a Google service account - private key JSON file:: - - credentials = service_account.Credentials.from_service_account_file( - 'service-account.json') - - Or if you already have the service account file loaded:: - - service_account_info = json.load(open('service_account.json')) - credentials = service_account.Credentials.from_service_account_info( - service_account_info) - - Both helper methods pass on arguments to the constructor, so you can - specify additional scopes and a subject if necessary:: - - credentials = service_account.Credentials.from_service_account_file( - 'service-account.json', - scopes=['email'], - subject='user@example.com') - - The credentials are considered immutable. If you want to modify the scopes - or the subject used for delegation, use :meth:`with_scopes` or - :meth:`with_subject`:: - - scoped_credentials = credentials.with_scopes(['email']) - delegated_credentials = credentials.with_subject(subject) - """ - - def __init__(self, signer, service_account_email, token_uri, scopes=None, - subject=None, project_id=None, additional_claims=None): - """ - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - service_account_email (str): The service account's email. - scopes (Sequence[str]): Scopes to request during the authorization - grant. - token_uri (str): The OAuth 2.0 Token URI. - subject (str): For domain-wide delegation, the email address of the - user to for which to request delegated access. - project_id (str): Project ID associated with the service account - credential. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT assertion used in the authorization grant. - - .. note:: Typically one of the helper constructors - :meth:`from_service_account_file` or - :meth:`from_service_account_info` are used instead of calling the - constructor directly. - """ - super(Credentials, self).__init__() - - self._scopes = scopes - self._signer = signer - self._service_account_email = service_account_email - self._subject = subject - self._project_id = project_id - self._token_uri = token_uri - - if additional_claims is not None: - self._additional_claims = additional_claims - else: - self._additional_claims = {} - - @classmethod - def _from_signer_and_info(cls, signer, info, **kwargs): - """Creates a Credentials instance from a signer and service account - info. - - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - info (Mapping[str, str]): The service account info. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.Credentials: The constructed credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - return cls( - signer, - service_account_email=info['client_email'], - token_uri=info['token_uri'], - project_id=info.get('project_id'), **kwargs) - - @classmethod - def from_service_account_info(cls, info, **kwargs): - """Creates a Credentials instance from parsed service account info. - - Args: - info (Mapping[str, str]): The service account info in Google - format. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.service_account.Credentials: The constructed - credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - signer = _service_account_info.from_dict( - info, require=['client_email', 'token_uri']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @classmethod - def from_service_account_file(cls, filename, **kwargs): - """Creates a Credentials instance from a service account json file. - - Args: - filename (str): The path to the service account json file. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.service_account.Credentials: The constructed - credentials. - """ - info, signer = _service_account_info.from_filename( - filename, require=['client_email', 'token_uri']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @property - def service_account_email(self): - """The service account email.""" - return self._service_account_email - - @property - def project_id(self): - """Project ID associated with this credential.""" - return self._project_id - - @property - def requires_scopes(self): - """Checks if the credentials requires scopes. - - Returns: - bool: True if there are no scopes set otherwise False. - """ - return True if not self._scopes else False - - @_helpers.copy_docstring(credentials.Scoped) - def with_scopes(self, scopes): - return self.__class__( - self._signer, - service_account_email=self._service_account_email, - scopes=scopes, - token_uri=self._token_uri, - subject=self._subject, - project_id=self._project_id, - additional_claims=self._additional_claims.copy()) - - def with_subject(self, subject): - """Create a copy of these credentials with the specified subject. - - Args: - subject (str): The subject claim. - - Returns: - google.auth.service_account.Credentials: A new credentials - instance. - """ - return self.__class__( - self._signer, - service_account_email=self._service_account_email, - scopes=self._scopes, - token_uri=self._token_uri, - subject=subject, - project_id=self._project_id, - additional_claims=self._additional_claims.copy()) - - def with_claims(self, additional_claims): - """Returns a copy of these credentials with modified claims. - - Args: - additional_claims (Mapping[str, str]): Any additional claims for - the JWT payload. This will be merged with the current - additional claims. - - Returns: - google.auth.service_account.Credentials: A new credentials - instance. - """ - new_additional_claims = copy.deepcopy(self._additional_claims) - new_additional_claims.update(additional_claims or {}) - - return self.__class__( - self._signer, - service_account_email=self._service_account_email, - scopes=self._scopes, - token_uri=self._token_uri, - subject=self._subject, - project_id=self._project_id, - additional_claims=new_additional_claims) - - def _make_authorization_grant_assertion(self): - """Create the OAuth 2.0 assertion. - - This assertion is used during the OAuth 2.0 grant to acquire an - access token. - - Returns: - bytes: The authorization grant assertion. - """ - now = _helpers.utcnow() - lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS) - expiry = now + lifetime - - payload = { - 'iat': _helpers.datetime_to_secs(now), - 'exp': _helpers.datetime_to_secs(expiry), - # The issuer must be the service account email. - 'iss': self._service_account_email, - # The audience must be the auth token endpoint's URI - 'aud': self._token_uri, - 'scope': _helpers.scopes_to_string(self._scopes or ()) - } - - payload.update(self._additional_claims) - - # The subject can be a user email for domain-wide delegation. - if self._subject: - payload.setdefault('sub', self._subject) - - token = jwt.encode(self._signer, payload) - - return token - - @_helpers.copy_docstring(credentials.Credentials) - def refresh(self, request): - assertion = self._make_authorization_grant_assertion() - access_token, expiry, _ = _client.jwt_grant( - request, self._token_uri, assertion) - self.token = access_token - self.expiry = expiry - - @_helpers.copy_docstring(credentials.Signing) - def sign_bytes(self, message): - return self._signer.sign(message) - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer(self): - return self._signer - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer_email(self): - return self._service_account_email - - -class IDTokenCredentials(credentials.Signing, credentials.Credentials): - """Open ID Connect ID Token-based service account credentials. - - These credentials are largely similar to :class:`.Credentials`, but instead - of using an OAuth 2.0 Access Token as the bearer token, they use an Open - ID Connect ID Token as the bearer token. These credentials are useful when - communicating to services that require ID Tokens and can not accept access - tokens. - - Usually, you'll create these credentials with one of the helper - constructors. To create credentials using a Google service account - private key JSON file:: - - credentials = ( - service_account.IDTokenCredentials.from_service_account_file( - 'service-account.json')) - - Or if you already have the service account file loaded:: - - service_account_info = json.load(open('service_account.json')) - credentials = ( - service_account.IDTokenCredentials.from_service_account_info( - service_account_info)) - - Both helper methods pass on arguments to the constructor, so you can - specify additional scopes and a subject if necessary:: - - credentials = ( - service_account.IDTokenCredentials.from_service_account_file( - 'service-account.json', - scopes=['email'], - subject='user@example.com')) -` - The credentials are considered immutable. If you want to modify the scopes - or the subject used for delegation, use :meth:`with_scopes` or - :meth:`with_subject`:: - - scoped_credentials = credentials.with_scopes(['email']) - delegated_credentials = credentials.with_subject(subject) - - """ - def __init__(self, signer, service_account_email, token_uri, - target_audience, additional_claims=None): - """ - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - service_account_email (str): The service account's email. - token_uri (str): The OAuth 2.0 Token URI. - target_audience (str): The intended audience for these credentials, - used when requesting the ID Token. The ID Token's ``aud`` claim - will be set to this string. - additional_claims (Mapping[str, str]): Any additional claims for - the JWT assertion used in the authorization grant. - - .. note:: Typically one of the helper constructors - :meth:`from_service_account_file` or - :meth:`from_service_account_info` are used instead of calling the - constructor directly. - """ - super(IDTokenCredentials, self).__init__() - self._signer = signer - self._service_account_email = service_account_email - self._token_uri = token_uri - self._target_audience = target_audience - - if additional_claims is not None: - self._additional_claims = additional_claims - else: - self._additional_claims = {} - - @classmethod - def _from_signer_and_info(cls, signer, info, **kwargs): - """Creates a credentials instance from a signer and service account - info. - - Args: - signer (google.auth.crypt.Signer): The signer used to sign JWTs. - info (Mapping[str, str]): The service account info. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.jwt.IDTokenCredentials: The constructed credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - kwargs.setdefault('service_account_email', info['client_email']) - kwargs.setdefault('token_uri', info['token_uri']) - return cls(signer, **kwargs) - - @classmethod - def from_service_account_info(cls, info, **kwargs): - """Creates a credentials instance from parsed service account info. - - Args: - info (Mapping[str, str]): The service account info in Google - format. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.service_account.IDTokenCredentials: The constructed - credentials. - - Raises: - ValueError: If the info is not in the expected format. - """ - signer = _service_account_info.from_dict( - info, require=['client_email', 'token_uri']) - return cls._from_signer_and_info(signer, info, **kwargs) - - @classmethod - def from_service_account_file(cls, filename, **kwargs): - """Creates a credentials instance from a service account json file. - - Args: - filename (str): The path to the service account json file. - kwargs: Additional arguments to pass to the constructor. - - Returns: - google.auth.service_account.IDTokenCredentials: The constructed - credentials. - """ - info, signer = _service_account_info.from_filename( - filename, require=['client_email', 'token_uri']) - return cls._from_signer_and_info(signer, info, **kwargs) - - def with_target_audience(self, target_audience): - """Create a copy of these credentials with the specified target - audience. - - Args: - target_audience (str): The intended audience for these credentials, - used when requesting the ID Token. - - Returns: - google.auth.service_account.IDTokenCredentials: A new credentials - instance. - """ - return self.__class__( - self._signer, - service_account_email=self._service_account_email, - token_uri=self._token_uri, - target_audience=target_audience, - additional_claims=self._additional_claims.copy()) - - def _make_authorization_grant_assertion(self): - """Create the OAuth 2.0 assertion. - - This assertion is used during the OAuth 2.0 grant to acquire an - ID token. - - Returns: - bytes: The authorization grant assertion. - """ - now = _helpers.utcnow() - lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS) - expiry = now + lifetime - - payload = { - 'iat': _helpers.datetime_to_secs(now), - 'exp': _helpers.datetime_to_secs(expiry), - # The issuer must be the service account email. - 'iss': self.service_account_email, - # The audience must be the auth token endpoint's URI - 'aud': self._token_uri, - # The target audience specifies which service the ID token is - # intended for. - 'target_audience': self._target_audience - } - - payload.update(self._additional_claims) - - token = jwt.encode(self._signer, payload) - - return token - - @_helpers.copy_docstring(credentials.Credentials) - def refresh(self, request): - assertion = self._make_authorization_grant_assertion() - access_token, expiry, _ = _client.id_token_jwt_grant( - request, self._token_uri, assertion) - self.token = access_token - self.expiry = expiry - - @property - def service_account_email(self): - """The service account email.""" - return self._service_account_email - - @_helpers.copy_docstring(credentials.Signing) - def sign_bytes(self, message): - return self._signer.sign(message) - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer(self): - return self._signer - - @property - @_helpers.copy_docstring(credentials.Signing) - def signer_email(self): - return self._service_account_email diff --git a/src/google_auth_httplib2/__init__.py b/src/google_auth_httplib2/__init__.py deleted file mode 100644 index 51c406eb..00000000 --- a/src/google_auth_httplib2/__init__.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Transport adapter for httplib2.""" - -from __future__ import absolute_import - -import logging - -from google.auth import exceptions -from google.auth import transport -import httplib2 -from six.moves import http_client - - -_LOGGER = logging.getLogger(__name__) -# Properties present in file-like streams / buffers. -_STREAM_PROPERTIES = ('read', 'seek', 'tell') - - -class _Response(transport.Response): - """httplib2 transport response adapter. - - Args: - response (httplib2.Response): The raw httplib2 response. - data (bytes): The response body. - """ - def __init__(self, response, data): - self._response = response - self._data = data - - @property - def status(self): - """int: The HTTP status code.""" - return self._response.status - - @property - def headers(self): - """Mapping[str, str]: The HTTP response headers.""" - return dict(self._response) - - @property - def data(self): - """bytes: The response body.""" - return self._data - - -class Request(transport.Request): - """httplib2 request adapter. - - This class is used internally for making requests using various transports - in a consistent way. If you use :class:`AuthorizedHttp` you do not need - to construct or use this class directly. - - This class can be useful if you want to manually refresh a - :class:`~google.auth.credentials.Credentials` instance:: - - import google_auth_httplib2 - import httplib2 - - http = httplib2.Http() - request = google_auth_httplib2.Request(http) - - credentials.refresh(request) - - Args: - http (httplib2.Http): The underlying http object to use to make - requests. - - .. automethod:: __call__ - """ - def __init__(self, http): - self.http = http - - def __call__(self, url, method='GET', body=None, headers=None, - timeout=None, **kwargs): - """Make an HTTP request using httplib2. - - Args: - url (str): The URI to be requested. - method (str): The HTTP method to use for the request. Defaults - to 'GET'. - body (bytes): The payload / body in HTTP request. - headers (Mapping[str, str]): Request headers. - timeout (Optional[int]): The number of seconds to wait for a - response from the server. This is ignored by httplib2 and will - issue a warning. - kwargs: Additional arguments passed throught to the underlying - :meth:`httplib2.Http.request` method. - - Returns: - google.auth.transport.Response: The HTTP response. - - Raises: - google.auth.exceptions.TransportError: If any exception occurred. - """ - if timeout is not None: - _LOGGER.warning( - 'httplib2 transport does not support per-request timeout. ' - 'Set the timeout when constructing the httplib2.Http instance.' - ) - - try: - _LOGGER.debug('Making request: %s %s', method, url) - response, data = self.http.request( - url, method=method, body=body, headers=headers, **kwargs) - return _Response(response, data) - # httplib2 should catch the lower http error, this is a bug and - # needs to be fixed there. Catch the error for the meanwhile. - except (httplib2.HttpLib2Error, http_client.HTTPException) as exc: - raise exceptions.TransportError(exc) - - -def _make_default_http(): - """Returns a default httplib2.Http instance.""" - return httplib2.Http() - - -class AuthorizedHttp(object): - """A httplib2 HTTP class with credentials. - - This class is used to perform requests to API endpoints that require - authorization:: - - from google.auth.transport._httplib2 import AuthorizedHttp - - authed_http = AuthorizedHttp(credentials) - - response = authed_http.request( - 'https://www.googleapis.com/storage/v1/b') - - This class implements :meth:`request` in the same way as - :class:`httplib2.Http` and can usually be used just like any other - instance of :class:``httplib2.Http`. - - The underlying :meth:`request` implementation handles adding the - credentials' headers to the request and refreshing credentials as needed. - """ - def __init__(self, credentials, http=None, - refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES, - max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS): - """ - Args: - credentials (google.auth.credentials.Credentials): The credentials - to add to the request. - http (httplib2.Http): The underlying HTTP object to - use to make requests. If not specified, a - :class:`httplib2.Http` instance will be constructed. - refresh_status_codes (Sequence[int]): Which HTTP status codes - indicate that credentials should be refreshed and the request - should be retried. - max_refresh_attempts (int): The maximum number of times to attempt - to refresh the credentials and retry the request. - """ - - if http is None: - http = _make_default_http() - - self.http = http - self.credentials = credentials - self._refresh_status_codes = refresh_status_codes - self._max_refresh_attempts = max_refresh_attempts - # Request instance used by internal methods (for example, - # credentials.refresh). - self._request = Request(self.http) - - def request(self, uri, method='GET', body=None, headers=None, - **kwargs): - """Implementation of httplib2's Http.request.""" - - _credential_refresh_attempt = kwargs.pop( - '_credential_refresh_attempt', 0) - - # Make a copy of the headers. They will be modified by the credentials - # and we want to pass the original headers if we recurse. - request_headers = headers.copy() if headers is not None else {} - - self.credentials.before_request( - self._request, method, uri, request_headers) - - # Check if the body is a file-like stream, and if so, save the body - # stream position so that it can be restored in case of refresh. - body_stream_position = None - if all(getattr(body, stream_prop, None) for stream_prop in - _STREAM_PROPERTIES): - body_stream_position = body.tell() - - # Make the request. - response, content = self.http.request( - uri, method, body=body, headers=request_headers, **kwargs) - - # If the response indicated that the credentials needed to be - # refreshed, then refresh the credentials and re-attempt the - # request. - # A stored token may expire between the time it is retrieved and - # the time the request is made, so we may need to try twice. - if (response.status in self._refresh_status_codes - and _credential_refresh_attempt < self._max_refresh_attempts): - - _LOGGER.info( - 'Refreshing credentials due to a %s response. Attempt %s/%s.', - response.status, _credential_refresh_attempt + 1, - self._max_refresh_attempts) - - self.credentials.refresh(self._request) - - # Restore the body's stream position if needed. - if body_stream_position is not None: - body.seek(body_stream_position) - - # Recurse. Pass in the original headers, not our modified set. - return self.request( - uri, method, body=body, headers=headers, - _credential_refresh_attempt=_credential_refresh_attempt + 1, - **kwargs) - - return response, content - - @property - def connections(self): - """Proxy to httplib2.Http.connections.""" - return self.http.connections - - @connections.setter - def connections(self, value): - """Proxy to httplib2.Http.connections.""" - self.http.connections = value diff --git a/src/googleapiclient/__init__.py b/src/googleapiclient/__init__.py deleted file mode 100644 index ee986d29..00000000 --- a/src/googleapiclient/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version__ = "1.7.8" - -# Set default logging handler to avoid "No handler found" warnings. -import logging - -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass - -logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/src/googleapiclient/_auth.py b/src/googleapiclient/_auth.py deleted file mode 100644 index 9d6d363f..00000000 --- a/src/googleapiclient/_auth.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers for authentication using oauth2client or google-auth.""" - -import httplib2 - -try: - import google.auth - import google.auth.credentials - HAS_GOOGLE_AUTH = True -except ImportError: # pragma: NO COVER - HAS_GOOGLE_AUTH = False - -try: - import google_auth_httplib2 -except ImportError: # pragma: NO COVER - google_auth_httplib2 = None - -try: - import oauth2client - import oauth2client.client - HAS_OAUTH2CLIENT = True -except ImportError: # pragma: NO COVER - HAS_OAUTH2CLIENT = False - - -def default_credentials(): - """Returns Application Default Credentials.""" - if HAS_GOOGLE_AUTH: - credentials, _ = google.auth.default() - return credentials - elif HAS_OAUTH2CLIENT: - return oauth2client.client.GoogleCredentials.get_application_default() - else: - raise EnvironmentError( - 'No authentication library is available. Please install either ' - 'google-auth or oauth2client.') - - -def with_scopes(credentials, scopes): - """Scopes the credentials if necessary. - - Args: - credentials (Union[ - google.auth.credentials.Credentials, - oauth2client.client.Credentials]): The credentials to scope. - scopes (Sequence[str]): The list of scopes. - - Returns: - Union[google.auth.credentials.Credentials, - oauth2client.client.Credentials]: The scoped credentials. - """ - if HAS_GOOGLE_AUTH and isinstance( - credentials, google.auth.credentials.Credentials): - return google.auth.credentials.with_scopes_if_required( - credentials, scopes) - else: - try: - if credentials.create_scoped_required(): - return credentials.create_scoped(scopes) - else: - return credentials - except AttributeError: - return credentials - - -def authorized_http(credentials): - """Returns an http client that is authorized with the given credentials. - - Args: - credentials (Union[ - google.auth.credentials.Credentials, - oauth2client.client.Credentials]): The credentials to use. - - Returns: - Union[httplib2.Http, google_auth_httplib2.AuthorizedHttp]: An - authorized http client. - """ - from googleapiclient.http import build_http - - if HAS_GOOGLE_AUTH and isinstance( - credentials, google.auth.credentials.Credentials): - if google_auth_httplib2 is None: - raise ValueError( - 'Credentials from google.auth specified, but ' - 'google-api-python-client is unable to use these credentials ' - 'unless google-auth-httplib2 is installed. Please install ' - 'google-auth-httplib2.') - return google_auth_httplib2.AuthorizedHttp(credentials, - http=build_http()) - else: - return credentials.authorize(build_http()) - - -def refresh_credentials(credentials): - # Refresh must use a new http instance, as the one associated with the - # credentials could be a AuthorizedHttp or an oauth2client-decorated - # Http instance which would cause a weird recursive loop of refreshing - # and likely tear a hole in spacetime. - refresh_http = httplib2.Http() - if HAS_GOOGLE_AUTH and isinstance( - credentials, google.auth.credentials.Credentials): - request = google_auth_httplib2.Request(refresh_http) - return credentials.refresh(request) - else: - return credentials.refresh(refresh_http) - - -def apply_credentials(credentials, headers): - # oauth2client and google-auth have the same interface for this. - if not is_valid(credentials): - refresh_credentials(credentials) - return credentials.apply(headers) - - -def is_valid(credentials): - if HAS_GOOGLE_AUTH and isinstance( - credentials, google.auth.credentials.Credentials): - return credentials.valid - else: - return ( - credentials.access_token is not None and - not credentials.access_token_expired) - - -def get_credentials_from_http(http): - if http is None: - return None - elif hasattr(http.request, 'credentials'): - return http.request.credentials - elif (hasattr(http, 'credentials') - and not isinstance(http.credentials, httplib2.Credentials)): - return http.credentials - else: - return None diff --git a/src/googleapiclient/_helpers.py b/src/googleapiclient/_helpers.py deleted file mode 100644 index 5e8184ba..00000000 --- a/src/googleapiclient/_helpers.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper functions for commonly used utilities.""" - -import functools -import inspect -import logging -import warnings - -import six -from six.moves import urllib - - -logger = logging.getLogger(__name__) - -POSITIONAL_WARNING = 'WARNING' -POSITIONAL_EXCEPTION = 'EXCEPTION' -POSITIONAL_IGNORE = 'IGNORE' -POSITIONAL_SET = frozenset([POSITIONAL_WARNING, POSITIONAL_EXCEPTION, - POSITIONAL_IGNORE]) - -positional_parameters_enforcement = POSITIONAL_WARNING - -_SYM_LINK_MESSAGE = 'File: {0}: Is a symbolic link.' -_IS_DIR_MESSAGE = '{0}: Is a directory' -_MISSING_FILE_MESSAGE = 'Cannot access {0}: No such file or directory' - - -def positional(max_positional_args): - """A decorator to declare that only the first N arguments my be positional. - - This decorator makes it easy to support Python 3 style keyword-only - parameters. For example, in Python 3 it is possible to write:: - - def fn(pos1, *, kwonly1=None, kwonly1=None): - ... - - All named parameters after ``*`` must be a keyword:: - - fn(10, 'kw1', 'kw2') # Raises exception. - fn(10, kwonly1='kw1') # Ok. - - Example - ^^^^^^^ - - To define a function like above, do:: - - @positional(1) - def fn(pos1, kwonly1=None, kwonly2=None): - ... - - If no default value is provided to a keyword argument, it becomes a - required keyword argument:: - - @positional(0) - def fn(required_kw): - ... - - This must be called with the keyword parameter:: - - fn() # Raises exception. - fn(10) # Raises exception. - fn(required_kw=10) # Ok. - - When defining instance or class methods always remember to account for - ``self`` and ``cls``:: - - class MyClass(object): - - @positional(2) - def my_method(self, pos1, kwonly1=None): - ... - - @classmethod - @positional(2) - def my_method(cls, pos1, kwonly1=None): - ... - - The positional decorator behavior is controlled by - ``_helpers.positional_parameters_enforcement``, which may be set to - ``POSITIONAL_EXCEPTION``, ``POSITIONAL_WARNING`` or - ``POSITIONAL_IGNORE`` to raise an exception, log a warning, or do - nothing, respectively, if a declaration is violated. - - Args: - max_positional_arguments: Maximum number of positional arguments. All - parameters after the this index must be - keyword only. - - Returns: - A decorator that prevents using arguments after max_positional_args - from being used as positional parameters. - - Raises: - TypeError: if a key-word only argument is provided as a positional - parameter, but only if - _helpers.positional_parameters_enforcement is set to - POSITIONAL_EXCEPTION. - """ - - def positional_decorator(wrapped): - @functools.wraps(wrapped) - def positional_wrapper(*args, **kwargs): - if len(args) > max_positional_args: - plural_s = '' - if max_positional_args != 1: - plural_s = 's' - message = ('{function}() takes at most {args_max} positional ' - 'argument{plural} ({args_given} given)'.format( - function=wrapped.__name__, - args_max=max_positional_args, - args_given=len(args), - plural=plural_s)) - if positional_parameters_enforcement == POSITIONAL_EXCEPTION: - raise TypeError(message) - elif positional_parameters_enforcement == POSITIONAL_WARNING: - logger.warning(message) - return wrapped(*args, **kwargs) - return positional_wrapper - - if isinstance(max_positional_args, six.integer_types): - return positional_decorator - else: - args, _, _, defaults = inspect.getargspec(max_positional_args) - return positional(len(args) - len(defaults))(max_positional_args) - - -def parse_unique_urlencoded(content): - """Parses unique key-value parameters from urlencoded content. - - Args: - content: string, URL-encoded key-value pairs. - - Returns: - dict, The key-value pairs from ``content``. - - Raises: - ValueError: if one of the keys is repeated. - """ - urlencoded_params = urllib.parse.parse_qs(content) - params = {} - for key, value in six.iteritems(urlencoded_params): - if len(value) != 1: - msg = ('URL-encoded content contains a repeated value:' - '%s -> %s' % (key, ', '.join(value))) - raise ValueError(msg) - params[key] = value[0] - return params - - -def update_query_params(uri, params): - """Updates a URI with new query parameters. - - If a given key from ``params`` is repeated in the ``uri``, then - the URI will be considered invalid and an error will occur. - - If the URI is valid, then each value from ``params`` will - replace the corresponding value in the query parameters (if - it exists). - - Args: - uri: string, A valid URI, with potential existing query parameters. - params: dict, A dictionary of query parameters. - - Returns: - The same URI but with the new query parameters added. - """ - parts = urllib.parse.urlparse(uri) - query_params = parse_unique_urlencoded(parts.query) - query_params.update(params) - new_query = urllib.parse.urlencode(query_params) - new_parts = parts._replace(query=new_query) - return urllib.parse.urlunparse(new_parts) - - -def _add_query_parameter(url, name, value): - """Adds a query parameter to a url. - - Replaces the current value if it already exists in the URL. - - Args: - url: string, url to add the query parameter to. - name: string, query parameter name. - value: string, query parameter value. - - Returns: - Updated query parameter. Does not update the url if value is None. - """ - if value is None: - return url - else: - return update_query_params(url, {name: value}) diff --git a/src/googleapiclient/channel.py b/src/googleapiclient/channel.py deleted file mode 100644 index 3caee13a..00000000 --- a/src/googleapiclient/channel.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Channel notifications support. - -Classes and functions to support channel subscriptions and notifications -on those channels. - -Notes: - - This code is based on experimental APIs and is subject to change. - - Notification does not do deduplication of notification ids, that's up to - the receiver. - - Storing the Channel between calls is up to the caller. - - -Example setting up a channel: - - # Create a new channel that gets notifications via webhook. - channel = new_webhook_channel("https://example.com/my_web_hook") - - # Store the channel, keyed by 'channel.id'. Store it before calling the - # watch method because notifications may start arriving before the watch - # method returns. - ... - - resp = service.objects().watchAll( - bucket="some_bucket_id", body=channel.body()).execute() - channel.update(resp) - - # Store the channel, keyed by 'channel.id'. Store it after being updated - # since the resource_id value will now be correct, and that's needed to - # stop a subscription. - ... - - -An example Webhook implementation using webapp2. Note that webapp2 puts -headers in a case insensitive dictionary, as headers aren't guaranteed to -always be upper case. - - id = self.request.headers[X_GOOG_CHANNEL_ID] - - # Retrieve the channel by id. - channel = ... - - # Parse notification from the headers, including validating the id. - n = notification_from_headers(channel, self.request.headers) - - # Do app specific stuff with the notification here. - if n.resource_state == 'sync': - # Code to handle sync state. - elif n.resource_state == 'exists': - # Code to handle the exists state. - elif n.resource_state == 'not_exists': - # Code to handle the not exists state. - - -Example of unsubscribing. - - service.channels().stop(channel.body()).execute() -""" -from __future__ import absolute_import - -import datetime -import uuid - -from googleapiclient import errors -from googleapiclient import _helpers as util -import six - - -# The unix time epoch starts at midnight 1970. -EPOCH = datetime.datetime.utcfromtimestamp(0) - -# Map the names of the parameters in the JSON channel description to -# the parameter names we use in the Channel class. -CHANNEL_PARAMS = { - 'address': 'address', - 'id': 'id', - 'expiration': 'expiration', - 'params': 'params', - 'resourceId': 'resource_id', - 'resourceUri': 'resource_uri', - 'type': 'type', - 'token': 'token', - } - -X_GOOG_CHANNEL_ID = 'X-GOOG-CHANNEL-ID' -X_GOOG_MESSAGE_NUMBER = 'X-GOOG-MESSAGE-NUMBER' -X_GOOG_RESOURCE_STATE = 'X-GOOG-RESOURCE-STATE' -X_GOOG_RESOURCE_URI = 'X-GOOG-RESOURCE-URI' -X_GOOG_RESOURCE_ID = 'X-GOOG-RESOURCE-ID' - - -def _upper_header_keys(headers): - new_headers = {} - for k, v in six.iteritems(headers): - new_headers[k.upper()] = v - return new_headers - - -class Notification(object): - """A Notification from a Channel. - - Notifications are not usually constructed directly, but are returned - from functions like notification_from_headers(). - - Attributes: - message_number: int, The unique id number of this notification. - state: str, The state of the resource being monitored. - uri: str, The address of the resource being monitored. - resource_id: str, The unique identifier of the version of the resource at - this event. - """ - @util.positional(5) - def __init__(self, message_number, state, resource_uri, resource_id): - """Notification constructor. - - Args: - message_number: int, The unique id number of this notification. - state: str, The state of the resource being monitored. Can be one - of "exists", "not_exists", or "sync". - resource_uri: str, The address of the resource being monitored. - resource_id: str, The identifier of the watched resource. - """ - self.message_number = message_number - self.state = state - self.resource_uri = resource_uri - self.resource_id = resource_id - - -class Channel(object): - """A Channel for notifications. - - Usually not constructed directly, instead it is returned from helper - functions like new_webhook_channel(). - - Attributes: - type: str, The type of delivery mechanism used by this channel. For - example, 'web_hook'. - id: str, A UUID for the channel. - token: str, An arbitrary string associated with the channel that - is delivered to the target address with each event delivered - over this channel. - address: str, The address of the receiving entity where events are - delivered. Specific to the channel type. - expiration: int, The time, in milliseconds from the epoch, when this - channel will expire. - params: dict, A dictionary of string to string, with additional parameters - controlling delivery channel behavior. - resource_id: str, An opaque id that identifies the resource that is - being watched. Stable across different API versions. - resource_uri: str, The canonicalized ID of the watched resource. - """ - - @util.positional(5) - def __init__(self, type, id, token, address, expiration=None, - params=None, resource_id="", resource_uri=""): - """Create a new Channel. - - In user code, this Channel constructor will not typically be called - manually since there are functions for creating channels for each specific - type with a more customized set of arguments to pass. - - Args: - type: str, The type of delivery mechanism used by this channel. For - example, 'web_hook'. - id: str, A UUID for the channel. - token: str, An arbitrary string associated with the channel that - is delivered to the target address with each event delivered - over this channel. - address: str, The address of the receiving entity where events are - delivered. Specific to the channel type. - expiration: int, The time, in milliseconds from the epoch, when this - channel will expire. - params: dict, A dictionary of string to string, with additional parameters - controlling delivery channel behavior. - resource_id: str, An opaque id that identifies the resource that is - being watched. Stable across different API versions. - resource_uri: str, The canonicalized ID of the watched resource. - """ - self.type = type - self.id = id - self.token = token - self.address = address - self.expiration = expiration - self.params = params - self.resource_id = resource_id - self.resource_uri = resource_uri - - def body(self): - """Build a body from the Channel. - - Constructs a dictionary that's appropriate for passing into watch() - methods as the value of body argument. - - Returns: - A dictionary representation of the channel. - """ - result = { - 'id': self.id, - 'token': self.token, - 'type': self.type, - 'address': self.address - } - if self.params: - result['params'] = self.params - if self.resource_id: - result['resourceId'] = self.resource_id - if self.resource_uri: - result['resourceUri'] = self.resource_uri - if self.expiration: - result['expiration'] = self.expiration - - return result - - def update(self, resp): - """Update a channel with information from the response of watch(). - - When a request is sent to watch() a resource, the response returned - from the watch() request is a dictionary with updated channel information, - such as the resource_id, which is needed when stopping a subscription. - - Args: - resp: dict, The response from a watch() method. - """ - for json_name, param_name in six.iteritems(CHANNEL_PARAMS): - value = resp.get(json_name) - if value is not None: - setattr(self, param_name, value) - - -def notification_from_headers(channel, headers): - """Parse a notification from the webhook request headers, validate - the notification, and return a Notification object. - - Args: - channel: Channel, The channel that the notification is associated with. - headers: dict, A dictionary like object that contains the request headers - from the webhook HTTP request. - - Returns: - A Notification object. - - Raises: - errors.InvalidNotificationError if the notification is invalid. - ValueError if the X-GOOG-MESSAGE-NUMBER can't be converted to an int. - """ - headers = _upper_header_keys(headers) - channel_id = headers[X_GOOG_CHANNEL_ID] - if channel.id != channel_id: - raise errors.InvalidNotificationError( - 'Channel id mismatch: %s != %s' % (channel.id, channel_id)) - else: - message_number = int(headers[X_GOOG_MESSAGE_NUMBER]) - state = headers[X_GOOG_RESOURCE_STATE] - resource_uri = headers[X_GOOG_RESOURCE_URI] - resource_id = headers[X_GOOG_RESOURCE_ID] - return Notification(message_number, state, resource_uri, resource_id) - - -@util.positional(2) -def new_webhook_channel(url, token=None, expiration=None, params=None): - """Create a new webhook Channel. - - Args: - url: str, URL to post notifications to. - token: str, An arbitrary string associated with the channel that - is delivered to the target address with each notification delivered - over this channel. - expiration: datetime.datetime, A time in the future when the channel - should expire. Can also be None if the subscription should use the - default expiration. Note that different services may have different - limits on how long a subscription lasts. Check the response from the - watch() method to see the value the service has set for an expiration - time. - params: dict, Extra parameters to pass on channel creation. Currently - not used for webhook channels. - """ - expiration_ms = 0 - if expiration: - delta = expiration - EPOCH - expiration_ms = delta.microseconds/1000 + ( - delta.seconds + delta.days*24*3600)*1000 - if expiration_ms < 0: - expiration_ms = 0 - - return Channel('web_hook', str(uuid.uuid4()), - token, url, expiration=expiration_ms, - params=params) - diff --git a/src/googleapiclient/discovery.py b/src/googleapiclient/discovery.py deleted file mode 100644 index 7d895bbe..00000000 --- a/src/googleapiclient/discovery.py +++ /dev/null @@ -1,1191 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Client for discovery based APIs. - -A client library for Google's discovery based APIs. -""" -from __future__ import absolute_import -import six -from six.moves import zip - -__author__ = 'jcgregorio@google.com (Joe Gregorio)' -__all__ = [ - 'build', - 'build_from_document', - 'fix_method_name', - 'key2param', - ] - -from six import BytesIO -from six.moves import http_client -from six.moves.urllib.parse import urlencode, urlparse, urljoin, \ - urlunparse, parse_qsl - -# Standard library imports -import copy -try: - from email.generator import BytesGenerator -except ImportError: - from email.generator import Generator as BytesGenerator -from email.mime.multipart import MIMEMultipart -from email.mime.nonmultipart import MIMENonMultipart -import json -import keyword -import logging -import mimetypes -import os -import re - -# Third-party imports -import httplib2 -import uritemplate - -# Local imports -from googleapiclient import _auth -from googleapiclient import mimeparse -from googleapiclient.errors import HttpError -from googleapiclient.errors import InvalidJsonError -from googleapiclient.errors import MediaUploadSizeError -from googleapiclient.errors import UnacceptableMimeTypeError -from googleapiclient.errors import UnknownApiNameOrVersion -from googleapiclient.errors import UnknownFileType -from googleapiclient.http import build_http -from googleapiclient.http import BatchHttpRequest -from googleapiclient.http import HttpMock -from googleapiclient.http import HttpMockSequence -from googleapiclient.http import HttpRequest -from googleapiclient.http import MediaFileUpload -from googleapiclient.http import MediaUpload -from googleapiclient.model import JsonModel -from googleapiclient.model import MediaModel -from googleapiclient.model import RawModel -from googleapiclient.schema import Schemas - -from googleapiclient._helpers import _add_query_parameter -from googleapiclient._helpers import positional - - -# The client library requires a version of httplib2 that supports RETRIES. -httplib2.RETRIES = 1 - -logger = logging.getLogger(__name__) - -URITEMPLATE = re.compile('{[^}]*}') -VARNAME = re.compile('[a-zA-Z0-9_-]+') -DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/' - '{api}/{apiVersion}/rest') -V1_DISCOVERY_URI = DISCOVERY_URI -V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?' - 'version={apiVersion}') -DEFAULT_METHOD_DOC = 'A description of how to use this function' -HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH']) - -_MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} -BODY_PARAMETER_DEFAULT_VALUE = { - 'description': 'The request body.', - 'type': 'object', - 'required': True, -} -MEDIA_BODY_PARAMETER_DEFAULT_VALUE = { - 'description': ('The filename of the media request body, or an instance ' - 'of a MediaUpload object.'), - 'type': 'string', - 'required': False, -} -MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = { - 'description': ('The MIME type of the media request body, or an instance ' - 'of a MediaUpload object.'), - 'type': 'string', - 'required': False, -} -_PAGE_TOKEN_NAMES = ('pageToken', 'nextPageToken') - -# Parameters accepted by the stack, but not visible via discovery. -# TODO(dhermes): Remove 'userip' in 'v2'. -STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict']) -STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'} - -# Library-specific reserved words beyond Python keywords. -RESERVED_WORDS = frozenset(['body']) - -# patch _write_lines to avoid munging '\r' into '\n' -# ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 ) -class _BytesGenerator(BytesGenerator): - _write_lines = BytesGenerator.write - -def fix_method_name(name): - """Fix method names to avoid '$' characters and reserved word conflicts. - - Args: - name: string, method name. - - Returns: - The name with '_' appended if the name is a reserved word and '$' - replaced with '_'. - """ - name = name.replace('$', '_') - if keyword.iskeyword(name) or name in RESERVED_WORDS: - return name + '_' - else: - return name - - -def key2param(key): - """Converts key names into parameter names. - - For example, converting "max-results" -> "max_results" - - Args: - key: string, the method key name. - - Returns: - A safe method name based on the key name. - """ - result = [] - key = list(key) - if not key[0].isalpha(): - result.append('x') - for c in key: - if c.isalnum(): - result.append(c) - else: - result.append('_') - - return ''.join(result) - - -@positional(2) -def build(serviceName, - version, - http=None, - discoveryServiceUrl=DISCOVERY_URI, - developerKey=None, - model=None, - requestBuilder=HttpRequest, - credentials=None, - cache_discovery=True, - cache=None): - """Construct a Resource for interacting with an API. - - Construct a Resource object for interacting with an API. The serviceName and - version are the names from the Discovery service. - - Args: - serviceName: string, name of the service. - version: string, the version of the service. - http: httplib2.Http, An instance of httplib2.Http or something that acts - like it that HTTP requests will be made through. - discoveryServiceUrl: string, a URI Template that points to the location of - the discovery service. It should have two parameters {api} and - {apiVersion} that when filled in produce an absolute URI to the discovery - document for that service. - developerKey: string, key obtained from - https://code.google.com/apis/console. - model: googleapiclient.Model, converts to and from the wire format. - requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP - request. - credentials: oauth2client.Credentials or - google.auth.credentials.Credentials, credentials to be used for - authentication. - cache_discovery: Boolean, whether or not to cache the discovery doc. - cache: googleapiclient.discovery_cache.base.CacheBase, an optional - cache object for the discovery documents. - - Returns: - A Resource object with methods for interacting with the service. - """ - params = { - 'api': serviceName, - 'apiVersion': version - } - - if http is None: - discovery_http = build_http() - else: - discovery_http = http - - for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,): - requested_url = uritemplate.expand(discovery_url, params) - - try: - content = _retrieve_discovery_doc( - requested_url, discovery_http, cache_discovery, cache, developerKey) - return build_from_document(content, base=discovery_url, http=http, - developerKey=developerKey, model=model, requestBuilder=requestBuilder, - credentials=credentials) - except HttpError as e: - if e.resp.status == http_client.NOT_FOUND: - continue - else: - raise e - - raise UnknownApiNameOrVersion( - "name: %s version: %s" % (serviceName, version)) - - -def _retrieve_discovery_doc(url, http, cache_discovery, cache=None, - developerKey=None): - """Retrieves the discovery_doc from cache or the internet. - - Args: - url: string, the URL of the discovery document. - http: httplib2.Http, An instance of httplib2.Http or something that acts - like it through which HTTP requests will be made. - cache_discovery: Boolean, whether or not to cache the discovery doc. - cache: googleapiclient.discovery_cache.base.Cache, an optional cache - object for the discovery documents. - - Returns: - A unicode string representation of the discovery document. - """ - if cache_discovery: - from . import discovery_cache - from .discovery_cache import base - if cache is None: - cache = discovery_cache.autodetect() - if cache: - content = cache.get(url) - if content: - return content - - actual_url = url - # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment - # variable that contains the network address of the client sending the - # request. If it exists then add that to the request for the discovery - # document to avoid exceeding the quota on discovery requests. - if 'REMOTE_ADDR' in os.environ: - actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR']) - if developerKey: - actual_url = _add_query_parameter(url, 'key', developerKey) - logger.info('URL being requested: GET %s', actual_url) - - resp, content = http.request(actual_url) - - if resp.status >= 400: - raise HttpError(resp, content, uri=actual_url) - - try: - content = content.decode('utf-8') - except AttributeError: - pass - - try: - service = json.loads(content) - except ValueError as e: - logger.error('Failed to parse as JSON: ' + content) - raise InvalidJsonError() - if cache_discovery and cache: - cache.set(url, content) - return content - - -@positional(1) -def build_from_document( - service, - base=None, - future=None, - http=None, - developerKey=None, - model=None, - requestBuilder=HttpRequest, - credentials=None): - """Create a Resource for interacting with an API. - - Same as `build()`, but constructs the Resource object from a discovery - document that is it given, as opposed to retrieving one over HTTP. - - Args: - service: string or object, the JSON discovery document describing the API. - The value passed in may either be the JSON string or the deserialized - JSON. - base: string, base URI for all HTTP requests, usually the discovery URI. - This parameter is no longer used as rootUrl and servicePath are included - within the discovery document. (deprecated) - future: string, discovery document with future capabilities (deprecated). - http: httplib2.Http, An instance of httplib2.Http or something that acts - like it that HTTP requests will be made through. - developerKey: string, Key for controlling API usage, generated - from the API Console. - model: Model class instance that serializes and de-serializes requests and - responses. - requestBuilder: Takes an http request and packages it up to be executed. - credentials: oauth2client.Credentials or - google.auth.credentials.Credentials, credentials to be used for - authentication. - - Returns: - A Resource object with methods for interacting with the service. - """ - - if http is not None and credentials is not None: - raise ValueError('Arguments http and credentials are mutually exclusive.') - - if isinstance(service, six.string_types): - service = json.loads(service) - - if 'rootUrl' not in service and (isinstance(http, (HttpMock, - HttpMockSequence))): - logger.error("You are using HttpMock or HttpMockSequence without" + - "having the service discovery doc in cache. Try calling " + - "build() without mocking once first to populate the " + - "cache.") - raise InvalidJsonError() - - base = urljoin(service['rootUrl'], service['servicePath']) - schema = Schemas(service) - - # If the http client is not specified, then we must construct an http client - # to make requests. If the service has scopes, then we also need to setup - # authentication. - if http is None: - # Does the service require scopes? - scopes = list( - service.get('auth', {}).get('oauth2', {}).get('scopes', {}).keys()) - - # If so, then the we need to setup authentication if no developerKey is - # specified. - if scopes and not developerKey: - # If the user didn't pass in credentials, attempt to acquire application - # default credentials. - if credentials is None: - credentials = _auth.default_credentials() - - # The credentials need to be scoped. - credentials = _auth.with_scopes(credentials, scopes) - - # If credentials are provided, create an authorized http instance; - # otherwise, skip authentication. - if credentials: - http = _auth.authorized_http(credentials) - - # If the service doesn't require scopes then there is no need for - # authentication. - else: - http = build_http() - - if model is None: - features = service.get('features', []) - model = JsonModel('dataWrapper' in features) - - return Resource(http=http, baseUrl=base, model=model, - developerKey=developerKey, requestBuilder=requestBuilder, - resourceDesc=service, rootDesc=service, schema=schema) - - -def _cast(value, schema_type): - """Convert value to a string based on JSON Schema type. - - See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on - JSON Schema. - - Args: - value: any, the value to convert - schema_type: string, the type that value should be interpreted as - - Returns: - A string representation of 'value' based on the schema_type. - """ - if schema_type == 'string': - if type(value) == type('') or type(value) == type(u''): - return value - else: - return str(value) - elif schema_type == 'integer': - return str(int(value)) - elif schema_type == 'number': - return str(float(value)) - elif schema_type == 'boolean': - return str(bool(value)).lower() - else: - if type(value) == type('') or type(value) == type(u''): - return value - else: - return str(value) - - -def _media_size_to_long(maxSize): - """Convert a string media size, such as 10GB or 3TB into an integer. - - Args: - maxSize: string, size as a string, such as 2MB or 7GB. - - Returns: - The size as an integer value. - """ - if len(maxSize) < 2: - return 0 - units = maxSize[-2:].upper() - bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units) - if bit_shift is not None: - return int(maxSize[:-2]) << bit_shift - else: - return int(maxSize) - - -def _media_path_url_from_info(root_desc, path_url): - """Creates an absolute media path URL. - - Constructed using the API root URI and service path from the discovery - document and the relative path for the API method. - - Args: - root_desc: Dictionary; the entire original deserialized discovery document. - path_url: String; the relative URL for the API method. Relative to the API - root, which is specified in the discovery document. - - Returns: - String; the absolute URI for media upload for the API method. - """ - return '%(root)supload/%(service_path)s%(path)s' % { - 'root': root_desc['rootUrl'], - 'service_path': root_desc['servicePath'], - 'path': path_url, - } - - -def _fix_up_parameters(method_desc, root_desc, http_method, schema): - """Updates parameters of an API method with values specific to this library. - - Specifically, adds whatever global parameters are specified by the API to the - parameters for the individual method. Also adds parameters which don't - appear in the discovery document, but are available to all discovery based - APIs (these are listed in STACK_QUERY_PARAMETERS). - - SIDE EFFECTS: This updates the parameters dictionary object in the method - description. - - Args: - method_desc: Dictionary with metadata describing an API method. Value comes - from the dictionary of methods stored in the 'methods' key in the - deserialized discovery document. - root_desc: Dictionary; the entire original deserialized discovery document. - http_method: String; the HTTP method used to call the API method described - in method_desc. - schema: Object, mapping of schema names to schema descriptions. - - Returns: - The updated Dictionary stored in the 'parameters' key of the method - description dictionary. - """ - parameters = method_desc.setdefault('parameters', {}) - - # Add in the parameters common to all methods. - for name, description in six.iteritems(root_desc.get('parameters', {})): - parameters[name] = description - - # Add in undocumented query parameters. - for name in STACK_QUERY_PARAMETERS: - parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy() - - # Add 'body' (our own reserved word) to parameters if the method supports - # a request payload. - if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc: - body = BODY_PARAMETER_DEFAULT_VALUE.copy() - body.update(method_desc['request']) - # Make body optional for requests with no parameters. - if not _methodProperties(method_desc, schema, 'request'): - body['required'] = False - parameters['body'] = body - - return parameters - - -def _fix_up_media_upload(method_desc, root_desc, path_url, parameters): - """Adds 'media_body' and 'media_mime_type' parameters if supported by method. - - SIDE EFFECTS: If the method supports media upload and has a required body, - sets body to be optional (required=False) instead. Also, if there is a - 'mediaUpload' in the method description, adds 'media_upload' key to - parameters. - - Args: - method_desc: Dictionary with metadata describing an API method. Value comes - from the dictionary of methods stored in the 'methods' key in the - deserialized discovery document. - root_desc: Dictionary; the entire original deserialized discovery document. - path_url: String; the relative URL for the API method. Relative to the API - root, which is specified in the discovery document. - parameters: A dictionary describing method parameters for method described - in method_desc. - - Returns: - Triple (accept, max_size, media_path_url) where: - - accept is a list of strings representing what content types are - accepted for media upload. Defaults to empty list if not in the - discovery document. - - max_size is a long representing the max size in bytes allowed for a - media upload. Defaults to 0L if not in the discovery document. - - media_path_url is a String; the absolute URI for media upload for the - API method. Constructed using the API root URI and service path from - the discovery document and the relative path for the API method. If - media upload is not supported, this is None. - """ - media_upload = method_desc.get('mediaUpload', {}) - accept = media_upload.get('accept', []) - max_size = _media_size_to_long(media_upload.get('maxSize', '')) - media_path_url = None - - if media_upload: - media_path_url = _media_path_url_from_info(root_desc, path_url) - parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy() - parameters['media_mime_type'] = MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE.copy() - if 'body' in parameters: - parameters['body']['required'] = False - - return accept, max_size, media_path_url - - -def _fix_up_method_description(method_desc, root_desc, schema): - """Updates a method description in a discovery document. - - SIDE EFFECTS: Changes the parameters dictionary in the method description with - extra parameters which are used locally. - - Args: - method_desc: Dictionary with metadata describing an API method. Value comes - from the dictionary of methods stored in the 'methods' key in the - deserialized discovery document. - root_desc: Dictionary; the entire original deserialized discovery document. - schema: Object, mapping of schema names to schema descriptions. - - Returns: - Tuple (path_url, http_method, method_id, accept, max_size, media_path_url) - where: - - path_url is a String; the relative URL for the API method. Relative to - the API root, which is specified in the discovery document. - - http_method is a String; the HTTP method used to call the API method - described in the method description. - - method_id is a String; the name of the RPC method associated with the - API method, and is in the method description in the 'id' key. - - accept is a list of strings representing what content types are - accepted for media upload. Defaults to empty list if not in the - discovery document. - - max_size is a long representing the max size in bytes allowed for a - media upload. Defaults to 0L if not in the discovery document. - - media_path_url is a String; the absolute URI for media upload for the - API method. Constructed using the API root URI and service path from - the discovery document and the relative path for the API method. If - media upload is not supported, this is None. - """ - path_url = method_desc['path'] - http_method = method_desc['httpMethod'] - method_id = method_desc['id'] - - parameters = _fix_up_parameters(method_desc, root_desc, http_method, schema) - # Order is important. `_fix_up_media_upload` needs `method_desc` to have a - # 'parameters' key and needs to know if there is a 'body' parameter because it - # also sets a 'media_body' parameter. - accept, max_size, media_path_url = _fix_up_media_upload( - method_desc, root_desc, path_url, parameters) - - return path_url, http_method, method_id, accept, max_size, media_path_url - - -def _urljoin(base, url): - """Custom urljoin replacement supporting : before / in url.""" - # In general, it's unsafe to simply join base and url. However, for - # the case of discovery documents, we know: - # * base will never contain params, query, or fragment - # * url will never contain a scheme or net_loc. - # In general, this means we can safely join on /; we just need to - # ensure we end up with precisely one / joining base and url. The - # exception here is the case of media uploads, where url will be an - # absolute url. - if url.startswith('http://') or url.startswith('https://'): - return urljoin(base, url) - new_base = base if base.endswith('/') else base + '/' - new_url = url[1:] if url.startswith('/') else url - return new_base + new_url - - -# TODO(dhermes): Convert this class to ResourceMethod and make it callable -class ResourceMethodParameters(object): - """Represents the parameters associated with a method. - - Attributes: - argmap: Map from method parameter name (string) to query parameter name - (string). - required_params: List of required parameters (represented by parameter - name as string). - repeated_params: List of repeated parameters (represented by parameter - name as string). - pattern_params: Map from method parameter name (string) to regular - expression (as a string). If the pattern is set for a parameter, the - value for that parameter must match the regular expression. - query_params: List of parameters (represented by parameter name as string) - that will be used in the query string. - path_params: Set of parameters (represented by parameter name as string) - that will be used in the base URL path. - param_types: Map from method parameter name (string) to parameter type. Type - can be any valid JSON schema type; valid values are 'any', 'array', - 'boolean', 'integer', 'number', 'object', or 'string'. Reference: - http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1 - enum_params: Map from method parameter name (string) to list of strings, - where each list of strings is the list of acceptable enum values. - """ - - def __init__(self, method_desc): - """Constructor for ResourceMethodParameters. - - Sets default values and defers to set_parameters to populate. - - Args: - method_desc: Dictionary with metadata describing an API method. Value - comes from the dictionary of methods stored in the 'methods' key in - the deserialized discovery document. - """ - self.argmap = {} - self.required_params = [] - self.repeated_params = [] - self.pattern_params = {} - self.query_params = [] - # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE - # parsing is gotten rid of. - self.path_params = set() - self.param_types = {} - self.enum_params = {} - - self.set_parameters(method_desc) - - def set_parameters(self, method_desc): - """Populates maps and lists based on method description. - - Iterates through each parameter for the method and parses the values from - the parameter dictionary. - - Args: - method_desc: Dictionary with metadata describing an API method. Value - comes from the dictionary of methods stored in the 'methods' key in - the deserialized discovery document. - """ - for arg, desc in six.iteritems(method_desc.get('parameters', {})): - param = key2param(arg) - self.argmap[param] = arg - - if desc.get('pattern'): - self.pattern_params[param] = desc['pattern'] - if desc.get('enum'): - self.enum_params[param] = desc['enum'] - if desc.get('required'): - self.required_params.append(param) - if desc.get('repeated'): - self.repeated_params.append(param) - if desc.get('location') == 'query': - self.query_params.append(param) - if desc.get('location') == 'path': - self.path_params.add(param) - self.param_types[param] = desc.get('type', 'string') - - # TODO(dhermes): Determine if this is still necessary. Discovery based APIs - # should have all path parameters already marked with - # 'location: path'. - for match in URITEMPLATE.finditer(method_desc['path']): - for namematch in VARNAME.finditer(match.group(0)): - name = key2param(namematch.group(0)) - self.path_params.add(name) - if name in self.query_params: - self.query_params.remove(name) - - -def createMethod(methodName, methodDesc, rootDesc, schema): - """Creates a method for attaching to a Resource. - - Args: - methodName: string, name of the method to use. - methodDesc: object, fragment of deserialized discovery document that - describes the method. - rootDesc: object, the entire deserialized discovery document. - schema: object, mapping of schema names to schema descriptions. - """ - methodName = fix_method_name(methodName) - (pathUrl, httpMethod, methodId, accept, - maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc, schema) - - parameters = ResourceMethodParameters(methodDesc) - - def method(self, **kwargs): - # Don't bother with doc string, it will be over-written by createMethod. - - for name in six.iterkeys(kwargs): - if name not in parameters.argmap: - raise TypeError('Got an unexpected keyword argument "%s"' % name) - - # Remove args that have a value of None. - keys = list(kwargs.keys()) - for name in keys: - if kwargs[name] is None: - del kwargs[name] - - for name in parameters.required_params: - if name not in kwargs: - # temporary workaround for non-paging methods incorrectly requiring - # page token parameter (cf. drive.changes.watch vs. drive.changes.list) - if name not in _PAGE_TOKEN_NAMES or _findPageTokenName( - _methodProperties(methodDesc, schema, 'response')): - raise TypeError('Missing required parameter "%s"' % name) - - for name, regex in six.iteritems(parameters.pattern_params): - if name in kwargs: - if isinstance(kwargs[name], six.string_types): - pvalues = [kwargs[name]] - else: - pvalues = kwargs[name] - for pvalue in pvalues: - if re.match(regex, pvalue) is None: - raise TypeError( - 'Parameter "%s" value "%s" does not match the pattern "%s"' % - (name, pvalue, regex)) - - for name, enums in six.iteritems(parameters.enum_params): - if name in kwargs: - # We need to handle the case of a repeated enum - # name differently, since we want to handle both - # arg='value' and arg=['value1', 'value2'] - if (name in parameters.repeated_params and - not isinstance(kwargs[name], six.string_types)): - values = kwargs[name] - else: - values = [kwargs[name]] - for value in values: - if value not in enums: - raise TypeError( - 'Parameter "%s" value "%s" is not an allowed value in "%s"' % - (name, value, str(enums))) - - actual_query_params = {} - actual_path_params = {} - for key, value in six.iteritems(kwargs): - to_type = parameters.param_types.get(key, 'string') - # For repeated parameters we cast each member of the list. - if key in parameters.repeated_params and type(value) == type([]): - cast_value = [_cast(x, to_type) for x in value] - else: - cast_value = _cast(value, to_type) - if key in parameters.query_params: - actual_query_params[parameters.argmap[key]] = cast_value - if key in parameters.path_params: - actual_path_params[parameters.argmap[key]] = cast_value - body_value = kwargs.get('body', None) - media_filename = kwargs.get('media_body', None) - media_mime_type = kwargs.get('media_mime_type', None) - - if self._developerKey: - actual_query_params['key'] = self._developerKey - - model = self._model - if methodName.endswith('_media'): - model = MediaModel() - elif 'response' not in methodDesc: - model = RawModel() - - headers = {} - headers, params, query, body = model.request(headers, - actual_path_params, actual_query_params, body_value) - - expanded_url = uritemplate.expand(pathUrl, params) - url = _urljoin(self._baseUrl, expanded_url + query) - - resumable = None - multipart_boundary = '' - - if media_filename: - # Ensure we end up with a valid MediaUpload object. - if isinstance(media_filename, six.string_types): - if media_mime_type is None: - logger.warning( - 'media_mime_type argument not specified: trying to auto-detect for %s', - media_filename) - media_mime_type, _ = mimetypes.guess_type(media_filename) - if media_mime_type is None: - raise UnknownFileType(media_filename) - if not mimeparse.best_match([media_mime_type], ','.join(accept)): - raise UnacceptableMimeTypeError(media_mime_type) - media_upload = MediaFileUpload(media_filename, - mimetype=media_mime_type) - elif isinstance(media_filename, MediaUpload): - media_upload = media_filename - else: - raise TypeError('media_filename must be str or MediaUpload.') - - # Check the maxSize - if media_upload.size() is not None and media_upload.size() > maxSize > 0: - raise MediaUploadSizeError("Media larger than: %s" % maxSize) - - # Use the media path uri for media uploads - expanded_url = uritemplate.expand(mediaPathUrl, params) - url = _urljoin(self._baseUrl, expanded_url + query) - if media_upload.resumable(): - url = _add_query_parameter(url, 'uploadType', 'resumable') - - if media_upload.resumable(): - # This is all we need to do for resumable, if the body exists it gets - # sent in the first request, otherwise an empty body is sent. - resumable = media_upload - else: - # A non-resumable upload - if body is None: - # This is a simple media upload - headers['content-type'] = media_upload.mimetype() - body = media_upload.getbytes(0, media_upload.size()) - url = _add_query_parameter(url, 'uploadType', 'media') - else: - # This is a multipart/related upload. - msgRoot = MIMEMultipart('related') - # msgRoot should not write out it's own headers - setattr(msgRoot, '_write_headers', lambda self: None) - - # attach the body as one part - msg = MIMENonMultipart(*headers['content-type'].split('/')) - msg.set_payload(body) - msgRoot.attach(msg) - - # attach the media as the second part - msg = MIMENonMultipart(*media_upload.mimetype().split('/')) - msg['Content-Transfer-Encoding'] = 'binary' - - payload = media_upload.getbytes(0, media_upload.size()) - msg.set_payload(payload) - msgRoot.attach(msg) - # encode the body: note that we can't use `as_string`, because - # it plays games with `From ` lines. - fp = BytesIO() - g = _BytesGenerator(fp, mangle_from_=False) - g.flatten(msgRoot, unixfrom=False) - body = fp.getvalue() - - multipart_boundary = msgRoot.get_boundary() - headers['content-type'] = ('multipart/related; ' - 'boundary="%s"') % multipart_boundary - url = _add_query_parameter(url, 'uploadType', 'multipart') - - logger.info('URL being requested: %s %s' % (httpMethod,url)) - return self._requestBuilder(self._http, - model.response, - url, - method=httpMethod, - body=body, - headers=headers, - methodId=methodId, - resumable=resumable) - - docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n'] - if len(parameters.argmap) > 0: - docs.append('Args:\n') - - # Skip undocumented params and params common to all methods. - skip_parameters = list(rootDesc.get('parameters', {}).keys()) - skip_parameters.extend(STACK_QUERY_PARAMETERS) - - all_args = list(parameters.argmap.keys()) - args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])] - - # Move body to the front of the line. - if 'body' in all_args: - args_ordered.append('body') - - for name in all_args: - if name not in args_ordered: - args_ordered.append(name) - - for arg in args_ordered: - if arg in skip_parameters: - continue - - repeated = '' - if arg in parameters.repeated_params: - repeated = ' (repeated)' - required = '' - if arg in parameters.required_params: - required = ' (required)' - paramdesc = methodDesc['parameters'][parameters.argmap[arg]] - paramdoc = paramdesc.get('description', 'A parameter') - if '$ref' in paramdesc: - docs.append( - (' %s: object, %s%s%s\n The object takes the' - ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated, - schema.prettyPrintByName(paramdesc['$ref']))) - else: - paramtype = paramdesc.get('type', 'string') - docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required, - repeated)) - enum = paramdesc.get('enum', []) - enumDesc = paramdesc.get('enumDescriptions', []) - if enum and enumDesc: - docs.append(' Allowed values\n') - for (name, desc) in zip(enum, enumDesc): - docs.append(' %s - %s\n' % (name, desc)) - if 'response' in methodDesc: - if methodName.endswith('_media'): - docs.append('\nReturns:\n The media object as a string.\n\n ') - else: - docs.append('\nReturns:\n An object of the form:\n\n ') - docs.append(schema.prettyPrintSchema(methodDesc['response'])) - - setattr(method, '__doc__', ''.join(docs)) - return (methodName, method) - - -def createNextMethod(methodName, - pageTokenName='pageToken', - nextPageTokenName='nextPageToken', - isPageTokenParameter=True): - """Creates any _next methods for attaching to a Resource. - - The _next methods allow for easy iteration through list() responses. - - Args: - methodName: string, name of the method to use. - pageTokenName: string, name of request page token field. - nextPageTokenName: string, name of response page token field. - isPageTokenParameter: Boolean, True if request page token is a query - parameter, False if request page token is a field of the request body. - """ - methodName = fix_method_name(methodName) - - def methodNext(self, previous_request, previous_response): - """Retrieves the next page of results. - -Args: - previous_request: The request for the previous page. (required) - previous_response: The response from the request for the previous page. (required) - -Returns: - A request object that you can call 'execute()' on to request the next - page. Returns None if there are no more items in the collection. - """ - # Retrieve nextPageToken from previous_response - # Use as pageToken in previous_request to create new request. - - nextPageToken = previous_response.get(nextPageTokenName, None) - if not nextPageToken: - return None - - request = copy.copy(previous_request) - - if isPageTokenParameter: - # Replace pageToken value in URI - request.uri = _add_query_parameter( - request.uri, pageTokenName, nextPageToken) - logger.info('Next page request URL: %s %s' % (methodName, request.uri)) - else: - # Replace pageToken value in request body - model = self._model - body = model.deserialize(request.body) - body[pageTokenName] = nextPageToken - request.body = model.serialize(body) - logger.info('Next page request body: %s %s' % (methodName, body)) - - return request - - return (methodName, methodNext) - - -class Resource(object): - """A class for interacting with a resource.""" - - def __init__(self, http, baseUrl, model, requestBuilder, developerKey, - resourceDesc, rootDesc, schema): - """Build a Resource from the API description. - - Args: - http: httplib2.Http, Object to make http requests with. - baseUrl: string, base URL for the API. All requests are relative to this - URI. - model: googleapiclient.Model, converts to and from the wire format. - requestBuilder: class or callable that instantiates an - googleapiclient.HttpRequest object. - developerKey: string, key obtained from - https://code.google.com/apis/console - resourceDesc: object, section of deserialized discovery document that - describes a resource. Note that the top level discovery document - is considered a resource. - rootDesc: object, the entire deserialized discovery document. - schema: object, mapping of schema names to schema descriptions. - """ - self._dynamic_attrs = [] - - self._http = http - self._baseUrl = baseUrl - self._model = model - self._developerKey = developerKey - self._requestBuilder = requestBuilder - self._resourceDesc = resourceDesc - self._rootDesc = rootDesc - self._schema = schema - - self._set_service_methods() - - def _set_dynamic_attr(self, attr_name, value): - """Sets an instance attribute and tracks it in a list of dynamic attributes. - - Args: - attr_name: string; The name of the attribute to be set - value: The value being set on the object and tracked in the dynamic cache. - """ - self._dynamic_attrs.append(attr_name) - self.__dict__[attr_name] = value - - def __getstate__(self): - """Trim the state down to something that can be pickled. - - Uses the fact that the instance variable _dynamic_attrs holds attrs that - will be wiped and restored on pickle serialization. - """ - state_dict = copy.copy(self.__dict__) - for dynamic_attr in self._dynamic_attrs: - del state_dict[dynamic_attr] - del state_dict['_dynamic_attrs'] - return state_dict - - def __setstate__(self, state): - """Reconstitute the state of the object from being pickled. - - Uses the fact that the instance variable _dynamic_attrs holds attrs that - will be wiped and restored on pickle serialization. - """ - self.__dict__.update(state) - self._dynamic_attrs = [] - self._set_service_methods() - - def _set_service_methods(self): - self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema) - self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema) - self._add_next_methods(self._resourceDesc, self._schema) - - def _add_basic_methods(self, resourceDesc, rootDesc, schema): - # If this is the root Resource, add a new_batch_http_request() method. - if resourceDesc == rootDesc: - batch_uri = '%s%s' % ( - rootDesc['rootUrl'], rootDesc.get('batchPath', 'batch')) - def new_batch_http_request(callback=None): - """Create a BatchHttpRequest object based on the discovery document. - - Args: - callback: callable, A callback to be called for each response, of the - form callback(id, response, exception). The first parameter is the - request id, and the second is the deserialized response object. The - third is an apiclient.errors.HttpError exception object if an HTTP - error occurred while processing the request, or None if no error - occurred. - - Returns: - A BatchHttpRequest object based on the discovery document. - """ - return BatchHttpRequest(callback=callback, batch_uri=batch_uri) - self._set_dynamic_attr('new_batch_http_request', new_batch_http_request) - - # Add basic methods to Resource - if 'methods' in resourceDesc: - for methodName, methodDesc in six.iteritems(resourceDesc['methods']): - fixedMethodName, method = createMethod( - methodName, methodDesc, rootDesc, schema) - self._set_dynamic_attr(fixedMethodName, - method.__get__(self, self.__class__)) - # Add in _media methods. The functionality of the attached method will - # change when it sees that the method name ends in _media. - if methodDesc.get('supportsMediaDownload', False): - fixedMethodName, method = createMethod( - methodName + '_media', methodDesc, rootDesc, schema) - self._set_dynamic_attr(fixedMethodName, - method.__get__(self, self.__class__)) - - def _add_nested_resources(self, resourceDesc, rootDesc, schema): - # Add in nested resources - if 'resources' in resourceDesc: - - def createResourceMethod(methodName, methodDesc): - """Create a method on the Resource to access a nested Resource. - - Args: - methodName: string, name of the method to use. - methodDesc: object, fragment of deserialized discovery document that - describes the method. - """ - methodName = fix_method_name(methodName) - - def methodResource(self): - return Resource(http=self._http, baseUrl=self._baseUrl, - model=self._model, developerKey=self._developerKey, - requestBuilder=self._requestBuilder, - resourceDesc=methodDesc, rootDesc=rootDesc, - schema=schema) - - setattr(methodResource, '__doc__', 'A collection resource.') - setattr(methodResource, '__is_resource__', True) - - return (methodName, methodResource) - - for methodName, methodDesc in six.iteritems(resourceDesc['resources']): - fixedMethodName, method = createResourceMethod(methodName, methodDesc) - self._set_dynamic_attr(fixedMethodName, - method.__get__(self, self.__class__)) - - def _add_next_methods(self, resourceDesc, schema): - # Add _next() methods if and only if one of the names 'pageToken' or - # 'nextPageToken' occurs among the fields of both the method's response - # type either the method's request (query parameters) or request body. - if 'methods' not in resourceDesc: - return - for methodName, methodDesc in six.iteritems(resourceDesc['methods']): - nextPageTokenName = _findPageTokenName( - _methodProperties(methodDesc, schema, 'response')) - if not nextPageTokenName: - continue - isPageTokenParameter = True - pageTokenName = _findPageTokenName(methodDesc.get('parameters', {})) - if not pageTokenName: - isPageTokenParameter = False - pageTokenName = _findPageTokenName( - _methodProperties(methodDesc, schema, 'request')) - if not pageTokenName: - continue - fixedMethodName, method = createNextMethod( - methodName + '_next', pageTokenName, nextPageTokenName, - isPageTokenParameter) - self._set_dynamic_attr(fixedMethodName, - method.__get__(self, self.__class__)) - - -def _findPageTokenName(fields): - """Search field names for one like a page token. - - Args: - fields: container of string, names of fields. - - Returns: - First name that is either 'pageToken' or 'nextPageToken' if one exists, - otherwise None. - """ - return next((tokenName for tokenName in _PAGE_TOKEN_NAMES - if tokenName in fields), None) - -def _methodProperties(methodDesc, schema, name): - """Get properties of a field in a method description. - - Args: - methodDesc: object, fragment of deserialized discovery document that - describes the method. - schema: object, mapping of schema names to schema descriptions. - name: string, name of top-level field in method description. - - Returns: - Object representing fragment of deserialized discovery document - corresponding to 'properties' field of object corresponding to named field - in method description, if it exists, otherwise empty dict. - """ - desc = methodDesc.get(name, {}) - if '$ref' in desc: - desc = schema.get(desc['$ref'], {}) - return desc.get('properties', {}) diff --git a/src/googleapiclient/discovery_cache/__init__.py b/src/googleapiclient/discovery_cache/__init__.py deleted file mode 100644 index f86a06de..00000000 --- a/src/googleapiclient/discovery_cache/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Caching utility for the discovery document.""" - -from __future__ import absolute_import - -import logging -import datetime - - -LOGGER = logging.getLogger(__name__) - -DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24 # 1 day - - -def autodetect(): - """Detects an appropriate cache module and returns it. - - Returns: - googleapiclient.discovery_cache.base.Cache, a cache object which - is auto detected, or None if no cache object is available. - """ - try: - from google.appengine.api import memcache - from . import appengine_memcache - return appengine_memcache.cache - except Exception: - try: - from . import file_cache - return file_cache.cache - except Exception as e: - LOGGER.warning(e, exc_info=True) - return None diff --git a/src/googleapiclient/discovery_cache/appengine_memcache.py b/src/googleapiclient/discovery_cache/appengine_memcache.py deleted file mode 100644 index 7e43e66c..00000000 --- a/src/googleapiclient/discovery_cache/appengine_memcache.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""App Engine memcache based cache for the discovery document.""" - -import logging - -# This is only an optional dependency because we only import this -# module when google.appengine.api.memcache is available. -from google.appengine.api import memcache - -from . import base -from ..discovery_cache import DISCOVERY_DOC_MAX_AGE - - -LOGGER = logging.getLogger(__name__) - -NAMESPACE = 'google-api-client' - - -class Cache(base.Cache): - """A cache with app engine memcache API.""" - - def __init__(self, max_age): - """Constructor. - - Args: - max_age: Cache expiration in seconds. - """ - self._max_age = max_age - - def get(self, url): - try: - return memcache.get(url, namespace=NAMESPACE) - except Exception as e: - LOGGER.warning(e, exc_info=True) - - def set(self, url, content): - try: - memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE) - except Exception as e: - LOGGER.warning(e, exc_info=True) - -cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE) diff --git a/src/googleapiclient/discovery_cache/base.py b/src/googleapiclient/discovery_cache/base.py deleted file mode 100644 index 00e466d1..00000000 --- a/src/googleapiclient/discovery_cache/base.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""An abstract class for caching the discovery document.""" - -import abc - - -class Cache(object): - """A base abstract cache class.""" - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def get(self, url): - """Gets the content from the memcache with a given key. - - Args: - url: string, the key for the cache. - - Returns: - object, the value in the cache for the given key, or None if the key is - not in the cache. - """ - raise NotImplementedError() - - @abc.abstractmethod - def set(self, url, content): - """Sets the given key and content in the cache. - - Args: - url: string, the key for the cache. - content: string, the discovery document. - """ - raise NotImplementedError() diff --git a/src/googleapiclient/discovery_cache/file_cache.py b/src/googleapiclient/discovery_cache/file_cache.py deleted file mode 100644 index 48bddea1..00000000 --- a/src/googleapiclient/discovery_cache/file_cache.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""File based cache for the discovery document. - -The cache is stored in a single file so that multiple processes can -share the same cache. It locks the file whenever accesing to the -file. When the cache content is corrupted, it will be initialized with -an empty cache. -""" - -from __future__ import division - -import datetime -import json -import logging -import os -import tempfile -import threading - -try: - from oauth2client.contrib.locked_file import LockedFile -except ImportError: - # oauth2client < 2.0.0 - try: - from oauth2client.locked_file import LockedFile - except ImportError: - # oauth2client > 4.0.0 or google-auth - raise ImportError( - 'file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth') - -from . import base -from ..discovery_cache import DISCOVERY_DOC_MAX_AGE - -LOGGER = logging.getLogger(__name__) - -FILENAME = 'google-api-python-client-discovery-doc.cache' -EPOCH = datetime.datetime.utcfromtimestamp(0) - - -def _to_timestamp(date): - try: - return (date - EPOCH).total_seconds() - except AttributeError: - # The following is the equivalent of total_seconds() in Python2.6. - # See also: https://docs.python.org/2/library/datetime.html - delta = date - EPOCH - return ((delta.microseconds + (delta.seconds + delta.days * 24 * 3600) - * 10**6) / 10**6) - - -def _read_or_initialize_cache(f): - f.file_handle().seek(0) - try: - cache = json.load(f.file_handle()) - except Exception: - # This means it opens the file for the first time, or the cache is - # corrupted, so initializing the file with an empty dict. - cache = {} - f.file_handle().truncate(0) - f.file_handle().seek(0) - json.dump(cache, f.file_handle()) - return cache - - -class Cache(base.Cache): - """A file based cache for the discovery documents.""" - - def __init__(self, max_age): - """Constructor. - - Args: - max_age: Cache expiration in seconds. - """ - self._max_age = max_age - self._file = os.path.join(tempfile.gettempdir(), FILENAME) - f = LockedFile(self._file, 'a+', 'r') - try: - f.open_and_lock() - if f.is_locked(): - _read_or_initialize_cache(f) - # If we can not obtain the lock, other process or thread must - # have initialized the file. - except Exception as e: - LOGGER.warning(e, exc_info=True) - finally: - f.unlock_and_close() - - def get(self, url): - f = LockedFile(self._file, 'r+', 'r') - try: - f.open_and_lock() - if f.is_locked(): - cache = _read_or_initialize_cache(f) - if url in cache: - content, t = cache.get(url, (None, 0)) - if _to_timestamp(datetime.datetime.now()) < t + self._max_age: - return content - return None - else: - LOGGER.debug('Could not obtain a lock for the cache file.') - return None - except Exception as e: - LOGGER.warning(e, exc_info=True) - finally: - f.unlock_and_close() - - def set(self, url, content): - f = LockedFile(self._file, 'r+', 'r') - try: - f.open_and_lock() - if f.is_locked(): - cache = _read_or_initialize_cache(f) - cache[url] = (content, _to_timestamp(datetime.datetime.now())) - # Remove stale cache. - for k, (_, timestamp) in list(cache.items()): - if _to_timestamp(datetime.datetime.now()) >= timestamp + self._max_age: - del cache[k] - f.file_handle().truncate(0) - f.file_handle().seek(0) - json.dump(cache, f.file_handle()) - else: - LOGGER.debug('Could not obtain a lock for the cache file.') - except Exception as e: - LOGGER.warning(e, exc_info=True) - finally: - f.unlock_and_close() - - -cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE) diff --git a/src/googleapiclient/errors.py b/src/googleapiclient/errors.py deleted file mode 100644 index 8c4795c4..00000000 --- a/src/googleapiclient/errors.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Errors for the library. - -All exceptions defined by the library -should be defined in this file. -""" -from __future__ import absolute_import - -__author__ = 'jcgregorio@google.com (Joe Gregorio)' - -import json - -from googleapiclient import _helpers as util - - -class Error(Exception): - """Base error for this module.""" - pass - - -class HttpError(Error): - """HTTP data was invalid or unexpected.""" - - @util.positional(3) - def __init__(self, resp, content, uri=None): - self.resp = resp - if not isinstance(content, bytes): - raise TypeError("HTTP content should be bytes") - self.content = content - self.uri = uri - self.error_details = '' - - def _get_reason(self): - """Calculate the reason for the error from the response content.""" - reason = self.resp.reason - try: - data = json.loads(self.content.decode('utf-8')) - if isinstance(data, dict): - reason = data['error']['message'] - if 'details' in data['error']: - self.error_details = data['error']['details'] - elif isinstance(data, list) and len(data) > 0: - first_error = data[0] - reason = first_error['error']['message'] - if 'details' in first_error['error']: - self.error_details = first_error['error']['details'] - except (ValueError, KeyError, TypeError): - pass - if reason is None: - reason = '' - return reason - - def __repr__(self): - reason = self._get_reason() - if self.error_details: - return '' % \ - (self.resp.status, self.uri, reason.strip(), self.error_details) - elif self.uri: - return '' % ( - self.resp.status, self.uri, self._get_reason().strip()) - else: - return '' % (self.resp.status, self._get_reason()) - - __str__ = __repr__ - - -class InvalidJsonError(Error): - """The JSON returned could not be parsed.""" - pass - - -class UnknownFileType(Error): - """File type unknown or unexpected.""" - pass - - -class UnknownLinkType(Error): - """Link type unknown or unexpected.""" - pass - - -class UnknownApiNameOrVersion(Error): - """No API with that name and version exists.""" - pass - - -class UnacceptableMimeTypeError(Error): - """That is an unacceptable mimetype for this operation.""" - pass - - -class MediaUploadSizeError(Error): - """Media is larger than the method can accept.""" - pass - - -class ResumableUploadError(HttpError): - """Error occured during resumable upload.""" - pass - - -class InvalidChunkSizeError(Error): - """The given chunksize is not valid.""" - pass - -class InvalidNotificationError(Error): - """The channel Notification is invalid.""" - pass - -class BatchError(HttpError): - """Error occured during batch operations.""" - - @util.positional(2) - def __init__(self, reason, resp=None, content=None): - self.resp = resp - self.content = content - self.reason = reason - - def __repr__(self): - if getattr(self.resp, 'status', None) is None: - return '' % (self.reason) - else: - return '' % (self.resp.status, self.reason) - - __str__ = __repr__ - - -class UnexpectedMethodError(Error): - """Exception raised by RequestMockBuilder on unexpected calls.""" - - @util.positional(1) - def __init__(self, methodId=None): - """Constructor for an UnexpectedMethodError.""" - super(UnexpectedMethodError, self).__init__( - 'Received unexpected call %s' % methodId) - - -class UnexpectedBodyError(Error): - """Exception raised by RequestMockBuilder on unexpected bodies.""" - - def __init__(self, expected, provided): - """Constructor for an UnexpectedMethodError.""" - super(UnexpectedBodyError, self).__init__( - 'Expected: [%s] - Provided: [%s]' % (expected, provided)) diff --git a/src/googleapiclient/http.py b/src/googleapiclient/http.py deleted file mode 100644 index 4949d0cf..00000000 --- a/src/googleapiclient/http.py +++ /dev/null @@ -1,1787 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Classes to encapsulate a single HTTP request. - -The classes implement a command pattern, with every -object supporting an execute() method that does the -actual HTTP request. -""" -from __future__ import absolute_import -import six -from six.moves import http_client -from six.moves import range - -__author__ = 'jcgregorio@google.com (Joe Gregorio)' - -from six import BytesIO, StringIO -from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote - -import base64 -import copy -import gzip -import httplib2 -import json -import logging -import mimetypes -import os -import random -import socket -import sys -import time -import uuid - -# TODO(issue 221): Remove this conditional import jibbajabba. -try: - import ssl -except ImportError: - _ssl_SSLError = object() -else: - _ssl_SSLError = ssl.SSLError - -from email.generator import Generator -from email.mime.multipart import MIMEMultipart -from email.mime.nonmultipart import MIMENonMultipart -from email.parser import FeedParser - -from googleapiclient import _helpers as util - -from googleapiclient import _auth -from googleapiclient.errors import BatchError -from googleapiclient.errors import HttpError -from googleapiclient.errors import InvalidChunkSizeError -from googleapiclient.errors import ResumableUploadError -from googleapiclient.errors import UnexpectedBodyError -from googleapiclient.errors import UnexpectedMethodError -from googleapiclient.model import JsonModel - - -LOGGER = logging.getLogger(__name__) - -DEFAULT_CHUNK_SIZE = 100*1024*1024 - -MAX_URI_LENGTH = 2048 - -MAX_BATCH_LIMIT = 1000 - -_TOO_MANY_REQUESTS = 429 - -DEFAULT_HTTP_TIMEOUT_SEC = 60 - -_LEGACY_BATCH_URI = 'https://www.googleapis.com/batch' - - -def _should_retry_response(resp_status, content): - """Determines whether a response should be retried. - - Args: - resp_status: The response status received. - content: The response content body. - - Returns: - True if the response should be retried, otherwise False. - """ - # Retry on 5xx errors. - if resp_status >= 500: - return True - - # Retry on 429 errors. - if resp_status == _TOO_MANY_REQUESTS: - return True - - # For 403 errors, we have to check for the `reason` in the response to - # determine if we should retry. - if resp_status == six.moves.http_client.FORBIDDEN: - # If there's no details about the 403 type, don't retry. - if not content: - return False - - # Content is in JSON format. - try: - data = json.loads(content.decode('utf-8')) - if isinstance(data, dict): - reason = data['error']['errors'][0]['reason'] - else: - reason = data[0]['error']['errors']['reason'] - except (UnicodeDecodeError, ValueError, KeyError): - LOGGER.warning('Invalid JSON content from response: %s', content) - return False - - LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) - - # Only retry on rate limit related failures. - if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): - return True - - # Everything else is a success or non-retriable so break. - return False - - -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, - **kwargs): - """Retries an HTTP request multiple times while handling errors. - - If after all retries the request still fails, last error is either returned as - return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). - - Args: - http: Http object to be used to execute request. - num_retries: Maximum number of retries. - req_type: Type of the request (used for logging retries). - sleep, rand: Functions to sleep for random time between retries. - uri: URI to be requested. - method: HTTP method to be used. - args, kwargs: Additional arguments passed to http.request. - - Returns: - resp, content - Response from the http request (may be HTTP 5xx). - """ - resp = None - content = None - for retry_num in range(num_retries + 1): - if retry_num > 0: - # Sleep before retrying. - sleep_time = rand() * 2 ** retry_num - LOGGER.warning( - 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', - sleep_time, retry_num, num_retries, req_type, method, uri, - resp.status if resp else exception) - sleep(sleep_time) - - try: - exception = None - resp, content = http.request(uri, method, *args, **kwargs) - # Retry on SSL errors and socket timeout errors. - except _ssl_SSLError as ssl_error: - exception = ssl_error - except socket.timeout as socket_timeout: - # It's important that this be before socket.error as it's a subclass - # socket.timeout has no errorcode - exception = socket_timeout - except socket.error as socket_error: - # errno's contents differ by platform, so we have to match by name. - if socket.errno.errorcode.get(socket_error.errno) not in { - 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED'}: - raise - exception = socket_error - except httplib2.ServerNotFoundError as server_not_found_error: - exception = server_not_found_error - - if exception: - if retry_num == num_retries: - raise exception - else: - continue - - if not _should_retry_response(resp.status, content): - break - - return resp, content - - -class MediaUploadProgress(object): - """Status of a resumable upload.""" - - def __init__(self, resumable_progress, total_size): - """Constructor. - - Args: - resumable_progress: int, bytes sent so far. - total_size: int, total bytes in complete upload, or None if the total - upload size isn't known ahead of time. - """ - self.resumable_progress = resumable_progress - self.total_size = total_size - - def progress(self): - """Percent of upload completed, as a float. - - Returns: - the percentage complete as a float, returning 0.0 if the total size of - the upload is unknown. - """ - if self.total_size is not None and self.total_size != 0: - return float(self.resumable_progress) / float(self.total_size) - else: - return 0.0 - - -class MediaDownloadProgress(object): - """Status of a resumable download.""" - - def __init__(self, resumable_progress, total_size): - """Constructor. - - Args: - resumable_progress: int, bytes received so far. - total_size: int, total bytes in complete download. - """ - self.resumable_progress = resumable_progress - self.total_size = total_size - - def progress(self): - """Percent of download completed, as a float. - - Returns: - the percentage complete as a float, returning 0.0 if the total size of - the download is unknown. - """ - if self.total_size is not None and self.total_size != 0: - return float(self.resumable_progress) / float(self.total_size) - else: - return 0.0 - - -class MediaUpload(object): - """Describes a media object to upload. - - Base class that defines the interface of MediaUpload subclasses. - - Note that subclasses of MediaUpload may allow you to control the chunksize - when uploading a media object. It is important to keep the size of the chunk - as large as possible to keep the upload efficient. Other factors may influence - the size of the chunk you use, particularly if you are working in an - environment where individual HTTP requests may have a hardcoded time limit, - such as under certain classes of requests under Google App Engine. - - Streams are io.Base compatible objects that support seek(). Some MediaUpload - subclasses support using streams directly to upload data. Support for - streaming may be indicated by a MediaUpload sub-class and if appropriate for a - platform that stream will be used for uploading the media object. The support - for streaming is indicated by has_stream() returning True. The stream() method - should return an io.Base object that supports seek(). On platforms where the - underlying httplib module supports streaming, for example Python 2.6 and - later, the stream will be passed into the http library which will result in - less memory being used and possibly faster uploads. - - If you need to upload media that can't be uploaded using any of the existing - MediaUpload sub-class then you can sub-class MediaUpload for your particular - needs. - """ - - def chunksize(self): - """Chunk size for resumable uploads. - - Returns: - Chunk size in bytes. - """ - raise NotImplementedError() - - def mimetype(self): - """Mime type of the body. - - Returns: - Mime type. - """ - return 'application/octet-stream' - - def size(self): - """Size of upload. - - Returns: - Size of the body, or None of the size is unknown. - """ - return None - - def resumable(self): - """Whether this upload is resumable. - - Returns: - True if resumable upload or False. - """ - return False - - def getbytes(self, begin, end): - """Get bytes from the media. - - Args: - begin: int, offset from beginning of file. - length: int, number of bytes to read, starting at begin. - - Returns: - A string of bytes read. May be shorter than length if EOF was reached - first. - """ - raise NotImplementedError() - - def has_stream(self): - """Does the underlying upload support a streaming interface. - - Streaming means it is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - - Returns: - True if the call to stream() will return an instance of a seekable io.Base - subclass. - """ - return False - - def stream(self): - """A stream interface to the data being uploaded. - - Returns: - The returned value is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - """ - raise NotImplementedError() - - @util.positional(1) - def _to_json(self, strip=None): - """Utility function for creating a JSON representation of a MediaUpload. - - Args: - strip: array, An array of names of members to not include in the JSON. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - t = type(self) - d = copy.copy(self.__dict__) - if strip is not None: - for member in strip: - del d[member] - d['_class'] = t.__name__ - d['_module'] = t.__module__ - return json.dumps(d) - - def to_json(self): - """Create a JSON representation of an instance of MediaUpload. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - return self._to_json() - - @classmethod - def new_from_json(cls, s): - """Utility class method to instantiate a MediaUpload subclass from a JSON - representation produced by to_json(). - - Args: - s: string, JSON from to_json(). - - Returns: - An instance of the subclass of MediaUpload that was serialized with - to_json(). - """ - data = json.loads(s) - # Find and call the right classmethod from_json() to restore the object. - module = data['_module'] - m = __import__(module, fromlist=module.split('.')[:-1]) - kls = getattr(m, data['_class']) - from_json = getattr(kls, 'from_json') - return from_json(s) - - -class MediaIoBaseUpload(MediaUpload): - """A MediaUpload for a io.Base objects. - - Note that the Python file object is compatible with io.Base and can be used - with this class also. - - fh = BytesIO('...Some data to upload...') - media = MediaIoBaseUpload(fh, mimetype='image/png', - chunksize=1024*1024, resumable=True) - farm.animals().insert( - id='cow', - name='cow.png', - media_body=media).execute() - - Depending on the platform you are working on, you may pass -1 as the - chunksize, which indicates that the entire file should be uploaded in a single - request. If the underlying platform supports streams, such as Python 2.6 or - later, then this can be very efficient as it avoids multiple connections, and - also avoids loading the entire file into memory before sending it. Note that - Google App Engine has a 5MB limit on request size, so you should never set - your chunksize larger than 5MB, or to -1. - """ - - @util.positional(3) - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, - resumable=False): - """Constructor. - - Args: - fd: io.Base or file object, The source of the bytes to upload. MUST be - opened in blocking mode, do not use streams opened in non-blocking mode. - The given stream must be seekable, that is, it must be able to call - seek() on fd. - mimetype: string, Mime-type of the file. - chunksize: int, File will be uploaded in chunks of this many bytes. Only - used if resumable=True. Pass in a value of -1 if the file is to be - uploaded as a single chunk. Note that Google App Engine has a 5MB limit - on request size, so you should never set your chunksize larger than 5MB, - or to -1. - resumable: bool, True if this is a resumable upload. False means upload - in a single request. - """ - super(MediaIoBaseUpload, self).__init__() - self._fd = fd - self._mimetype = mimetype - if not (chunksize == -1 or chunksize > 0): - raise InvalidChunkSizeError() - self._chunksize = chunksize - self._resumable = resumable - - self._fd.seek(0, os.SEEK_END) - self._size = self._fd.tell() - - def chunksize(self): - """Chunk size for resumable uploads. - - Returns: - Chunk size in bytes. - """ - return self._chunksize - - def mimetype(self): - """Mime type of the body. - - Returns: - Mime type. - """ - return self._mimetype - - def size(self): - """Size of upload. - - Returns: - Size of the body, or None of the size is unknown. - """ - return self._size - - def resumable(self): - """Whether this upload is resumable. - - Returns: - True if resumable upload or False. - """ - return self._resumable - - def getbytes(self, begin, length): - """Get bytes from the media. - - Args: - begin: int, offset from beginning of file. - length: int, number of bytes to read, starting at begin. - - Returns: - A string of bytes read. May be shorted than length if EOF was reached - first. - """ - self._fd.seek(begin) - return self._fd.read(length) - - def has_stream(self): - """Does the underlying upload support a streaming interface. - - Streaming means it is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - - Returns: - True if the call to stream() will return an instance of a seekable io.Base - subclass. - """ - return True - - def stream(self): - """A stream interface to the data being uploaded. - - Returns: - The returned value is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - """ - return self._fd - - def to_json(self): - """This upload type is not serializable.""" - raise NotImplementedError('MediaIoBaseUpload is not serializable.') - - -class MediaFileUpload(MediaIoBaseUpload): - """A MediaUpload for a file. - - Construct a MediaFileUpload and pass as the media_body parameter of the - method. For example, if we had a service that allowed uploading images: - - media = MediaFileUpload('cow.png', mimetype='image/png', - chunksize=1024*1024, resumable=True) - farm.animals().insert( - id='cow', - name='cow.png', - media_body=media).execute() - - Depending on the platform you are working on, you may pass -1 as the - chunksize, which indicates that the entire file should be uploaded in a single - request. If the underlying platform supports streams, such as Python 2.6 or - later, then this can be very efficient as it avoids multiple connections, and - also avoids loading the entire file into memory before sending it. Note that - Google App Engine has a 5MB limit on request size, so you should never set - your chunksize larger than 5MB, or to -1. - """ - - @util.positional(2) - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, - resumable=False): - """Constructor. - - Args: - filename: string, Name of the file. - mimetype: string, Mime-type of the file. If None then a mime-type will be - guessed from the file extension. - chunksize: int, File will be uploaded in chunks of this many bytes. Only - used if resumable=True. Pass in a value of -1 if the file is to be - uploaded in a single chunk. Note that Google App Engine has a 5MB limit - on request size, so you should never set your chunksize larger than 5MB, - or to -1. - resumable: bool, True if this is a resumable upload. False means upload - in a single request. - """ - self._filename = filename - fd = open(self._filename, 'rb') - if mimetype is None: - # No mimetype provided, make a guess. - mimetype, _ = mimetypes.guess_type(filename) - if mimetype is None: - # Guess failed, use octet-stream. - mimetype = 'application/octet-stream' - super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize, - resumable=resumable) - - def to_json(self): - """Creating a JSON representation of an instance of MediaFileUpload. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - return self._to_json(strip=['_fd']) - - @staticmethod - def from_json(s): - d = json.loads(s) - return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'], - chunksize=d['_chunksize'], resumable=d['_resumable']) - - -class MediaInMemoryUpload(MediaIoBaseUpload): - """MediaUpload for a chunk of bytes. - - DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for - the stream. - """ - - @util.positional(2) - def __init__(self, body, mimetype='application/octet-stream', - chunksize=DEFAULT_CHUNK_SIZE, resumable=False): - """Create a new MediaInMemoryUpload. - - DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for - the stream. - - Args: - body: string, Bytes of body content. - mimetype: string, Mime-type of the file or default of - 'application/octet-stream'. - chunksize: int, File will be uploaded in chunks of this many bytes. Only - used if resumable=True. - resumable: bool, True if this is a resumable upload. False means upload - in a single request. - """ - fd = BytesIO(body) - super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize, - resumable=resumable) - - -class MediaIoBaseDownload(object): - """"Download media resources. - - Note that the Python file object is compatible with io.Base and can be used - with this class also. - - - Example: - request = farms.animals().get_media(id='cow') - fh = io.FileIO('cow.png', mode='wb') - downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) - - done = False - while done is False: - status, done = downloader.next_chunk() - if status: - print "Download %d%%." % int(status.progress() * 100) - print "Download Complete!" - """ - - @util.positional(3) - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE): - """Constructor. - - Args: - fd: io.Base or file object, The stream in which to write the downloaded - bytes. - request: googleapiclient.http.HttpRequest, the media request to perform in - chunks. - chunksize: int, File will be downloaded in chunks of this many bytes. - """ - self._fd = fd - self._request = request - self._uri = request.uri - self._chunksize = chunksize - self._progress = 0 - self._total_size = None - self._done = False - - # Stubs for testing. - self._sleep = time.sleep - self._rand = random.random - - self._headers = {} - for k, v in six.iteritems(request.headers): - # allow users to supply custom headers by setting them on the request - # but strip out the ones that are set by default on requests generated by - # API methods like Drive's files().get(fileId=...) - if not k.lower() in ('accept', 'accept-encoding', 'user-agent'): - self._headers[k] = v - - @util.positional(1) - def next_chunk(self, num_retries=0): - """Get the next chunk of the download. - - Args: - num_retries: Integer, number of times to retry with randomized - exponential backoff. If all retries fail, the raised HttpError - represents the last request. If zero (default), we attempt the - request only once. - - Returns: - (status, done): (MediaDownloadProgress, boolean) - The value of 'done' will be True when the media has been fully - downloaded or the total size of the media is unknown. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx. - httplib2.HttpLib2Error if a transport error has occured. - """ - headers = self._headers.copy() - headers['range'] = 'bytes=%d-%d' % ( - self._progress, self._progress + self._chunksize) - http = self._request.http - - resp, content = _retry_request( - http, num_retries, 'media download', self._sleep, self._rand, self._uri, - 'GET', headers=headers) - - if resp.status in [200, 206]: - if 'content-location' in resp and resp['content-location'] != self._uri: - self._uri = resp['content-location'] - self._progress += len(content) - self._fd.write(content) - - if 'content-range' in resp: - content_range = resp['content-range'] - length = content_range.rsplit('/', 1)[1] - self._total_size = int(length) - elif 'content-length' in resp: - self._total_size = int(resp['content-length']) - - if self._total_size is None or self._progress == self._total_size: - self._done = True - return MediaDownloadProgress(self._progress, self._total_size), self._done - else: - raise HttpError(resp, content, uri=self._uri) - - -class _StreamSlice(object): - """Truncated stream. - - Takes a stream and presents a stream that is a slice of the original stream. - This is used when uploading media in chunks. In later versions of Python a - stream can be passed to httplib in place of the string of data to send. The - problem is that httplib just blindly reads to the end of the stream. This - wrapper presents a virtual stream that only reads to the end of the chunk. - """ - - def __init__(self, stream, begin, chunksize): - """Constructor. - - Args: - stream: (io.Base, file object), the stream to wrap. - begin: int, the seek position the chunk begins at. - chunksize: int, the size of the chunk. - """ - self._stream = stream - self._begin = begin - self._chunksize = chunksize - self._stream.seek(begin) - - def read(self, n=-1): - """Read n bytes. - - Args: - n, int, the number of bytes to read. - - Returns: - A string of length 'n', or less if EOF is reached. - """ - # The data left available to read sits in [cur, end) - cur = self._stream.tell() - end = self._begin + self._chunksize - if n == -1 or cur + n > end: - n = end - cur - return self._stream.read(n) - - -class HttpRequest(object): - """Encapsulates a single HTTP request.""" - - @util.positional(4) - def __init__(self, http, postproc, uri, - method='GET', - body=None, - headers=None, - methodId=None, - resumable=None): - """Constructor for an HttpRequest. - - Args: - http: httplib2.Http, the transport object to use to make a request - postproc: callable, called on the HTTP response and content to transform - it into a data object before returning, or raising an exception - on an error. - uri: string, the absolute URI to send the request to - method: string, the HTTP method to use - body: string, the request body of the HTTP request, - headers: dict, the HTTP request headers - methodId: string, a unique identifier for the API method being called. - resumable: MediaUpload, None if this is not a resumbale request. - """ - self.uri = uri - self.method = method - self.body = body - self.headers = headers or {} - self.methodId = methodId - self.http = http - self.postproc = postproc - self.resumable = resumable - self.response_callbacks = [] - self._in_error_state = False - - # The size of the non-media part of the request. - self.body_size = len(self.body or '') - - # The resumable URI to send chunks to. - self.resumable_uri = None - - # The bytes that have been uploaded. - self.resumable_progress = 0 - - # Stubs for testing. - self._rand = random.random - self._sleep = time.sleep - - @util.positional(1) - def execute(self, http=None, num_retries=0): - """Execute the request. - - Args: - http: httplib2.Http, an http object to be used in place of the - one the HttpRequest request object was constructed with. - num_retries: Integer, number of times to retry with randomized - exponential backoff. If all retries fail, the raised HttpError - represents the last request. If zero (default), we attempt the - request only once. - - Returns: - A deserialized object model of the response body as determined - by the postproc. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx. - httplib2.HttpLib2Error if a transport error has occured. - """ - if http is None: - http = self.http - - if self.resumable: - body = None - while body is None: - _, body = self.next_chunk(http=http, num_retries=num_retries) - return body - - # Non-resumable case. - - if 'content-length' not in self.headers: - self.headers['content-length'] = str(self.body_size) - # If the request URI is too long then turn it into a POST request. - # Assume that a GET request never contains a request body. - if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': - self.method = 'POST' - self.headers['x-http-method-override'] = 'GET' - self.headers['content-type'] = 'application/x-www-form-urlencoded' - parsed = urlparse(self.uri) - self.uri = urlunparse( - (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, - None) - ) - self.body = parsed.query - self.headers['content-length'] = str(len(self.body)) - - # Handle retries for server-side errors. - resp, content = _retry_request( - http, num_retries, 'request', self._sleep, self._rand, str(self.uri), - method=str(self.method), body=self.body, headers=self.headers) - - for callback in self.response_callbacks: - callback(resp) - if resp.status >= 300: - raise HttpError(resp, content, uri=self.uri) - return self.postproc(resp, content) - - @util.positional(2) - def add_response_callback(self, cb): - """add_response_headers_callback - - Args: - cb: Callback to be called on receiving the response headers, of signature: - - def cb(resp): - # Where resp is an instance of httplib2.Response - """ - self.response_callbacks.append(cb) - - @util.positional(1) - def next_chunk(self, http=None, num_retries=0): - """Execute the next step of a resumable upload. - - Can only be used if the method being executed supports media uploads and - the MediaUpload object passed in was flagged as using resumable upload. - - Example: - - media = MediaFileUpload('cow.png', mimetype='image/png', - chunksize=1000, resumable=True) - request = farm.animals().insert( - id='cow', - name='cow.png', - media_body=media) - - response = None - while response is None: - status, response = request.next_chunk() - if status: - print "Upload %d%% complete." % int(status.progress() * 100) - - - Args: - http: httplib2.Http, an http object to be used in place of the - one the HttpRequest request object was constructed with. - num_retries: Integer, number of times to retry with randomized - exponential backoff. If all retries fail, the raised HttpError - represents the last request. If zero (default), we attempt the - request only once. - - Returns: - (status, body): (ResumableMediaStatus, object) - The body will be None until the resumable media is fully uploaded. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx. - httplib2.HttpLib2Error if a transport error has occured. - """ - if http is None: - http = self.http - - if self.resumable.size() is None: - size = '*' - else: - size = str(self.resumable.size()) - - if self.resumable_uri is None: - start_headers = copy.copy(self.headers) - start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() - if size != '*': - start_headers['X-Upload-Content-Length'] = size - start_headers['content-length'] = str(self.body_size) - - resp, content = _retry_request( - http, num_retries, 'resumable URI request', self._sleep, self._rand, - self.uri, method=self.method, body=self.body, headers=start_headers) - - if resp.status == 200 and 'location' in resp: - self.resumable_uri = resp['location'] - else: - raise ResumableUploadError(resp, content) - elif self._in_error_state: - # If we are in an error state then query the server for current state of - # the upload by sending an empty PUT and reading the 'range' header in - # the response. - headers = { - 'Content-Range': 'bytes */%s' % size, - 'content-length': '0' - } - resp, content = http.request(self.resumable_uri, 'PUT', - headers=headers) - status, body = self._process_response(resp, content) - if body: - # The upload was complete. - return (status, body) - - if self.resumable.has_stream(): - data = self.resumable.stream() - if self.resumable.chunksize() == -1: - data.seek(self.resumable_progress) - chunk_end = self.resumable.size() - self.resumable_progress - 1 - else: - # Doing chunking with a stream, so wrap a slice of the stream. - data = _StreamSlice(data, self.resumable_progress, - self.resumable.chunksize()) - chunk_end = min( - self.resumable_progress + self.resumable.chunksize() - 1, - self.resumable.size() - 1) - else: - data = self.resumable.getbytes( - self.resumable_progress, self.resumable.chunksize()) - - # A short read implies that we are at EOF, so finish the upload. - if len(data) < self.resumable.chunksize(): - size = str(self.resumable_progress + len(data)) - - chunk_end = self.resumable_progress + len(data) - 1 - - headers = { - 'Content-Range': 'bytes %d-%d/%s' % ( - self.resumable_progress, chunk_end, size), - # Must set the content-length header here because httplib can't - # calculate the size when working with _StreamSlice. - 'Content-Length': str(chunk_end - self.resumable_progress + 1) - } - - for retry_num in range(num_retries + 1): - if retry_num > 0: - self._sleep(self._rand() * 2**retry_num) - LOGGER.warning( - 'Retry #%d for media upload: %s %s, following status: %d' - % (retry_num, self.method, self.uri, resp.status)) - - try: - resp, content = http.request(self.resumable_uri, method='PUT', - body=data, - headers=headers) - except: - self._in_error_state = True - raise - if not _should_retry_response(resp.status, content): - break - - return self._process_response(resp, content) - - def _process_response(self, resp, content): - """Process the response from a single chunk upload. - - Args: - resp: httplib2.Response, the response object. - content: string, the content of the response. - - Returns: - (status, body): (ResumableMediaStatus, object) - The body will be None until the resumable media is fully uploaded. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx or a 308. - """ - if resp.status in [200, 201]: - self._in_error_state = False - return None, self.postproc(resp, content) - elif resp.status == 308: - self._in_error_state = False - # A "308 Resume Incomplete" indicates we are not done. - try: - self.resumable_progress = int(resp['range'].split('-')[1]) + 1 - except KeyError: - # If resp doesn't contain range header, resumable progress is 0 - self.resumable_progress = 0 - if 'location' in resp: - self.resumable_uri = resp['location'] - else: - self._in_error_state = True - raise HttpError(resp, content, uri=self.uri) - - return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), - None) - - def to_json(self): - """Returns a JSON representation of the HttpRequest.""" - d = copy.copy(self.__dict__) - if d['resumable'] is not None: - d['resumable'] = self.resumable.to_json() - del d['http'] - del d['postproc'] - del d['_sleep'] - del d['_rand'] - - return json.dumps(d) - - @staticmethod - def from_json(s, http, postproc): - """Returns an HttpRequest populated with info from a JSON object.""" - d = json.loads(s) - if d['resumable'] is not None: - d['resumable'] = MediaUpload.new_from_json(d['resumable']) - return HttpRequest( - http, - postproc, - uri=d['uri'], - method=d['method'], - body=d['body'], - headers=d['headers'], - methodId=d['methodId'], - resumable=d['resumable']) - - -class BatchHttpRequest(object): - """Batches multiple HttpRequest objects into a single HTTP request. - - Example: - from googleapiclient.http import BatchHttpRequest - - def list_animals(request_id, response, exception): - \"\"\"Do something with the animals list response.\"\"\" - if exception is not None: - # Do something with the exception. - pass - else: - # Do something with the response. - pass - - def list_farmers(request_id, response, exception): - \"\"\"Do something with the farmers list response.\"\"\" - if exception is not None: - # Do something with the exception. - pass - else: - # Do something with the response. - pass - - service = build('farm', 'v2') - - batch = BatchHttpRequest() - - batch.add(service.animals().list(), list_animals) - batch.add(service.farmers().list(), list_farmers) - batch.execute(http=http) - """ - - @util.positional(1) - def __init__(self, callback=None, batch_uri=None): - """Constructor for a BatchHttpRequest. - - Args: - callback: callable, A callback to be called for each response, of the - form callback(id, response, exception). The first parameter is the - request id, and the second is the deserialized response object. The - third is an googleapiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no error occurred. - batch_uri: string, URI to send batch requests to. - """ - if batch_uri is None: - batch_uri = _LEGACY_BATCH_URI - - if batch_uri == _LEGACY_BATCH_URI: - LOGGER.warn( - "You have constructed a BatchHttpRequest using the legacy batch " - "endpoint %s. This endpoint will be turned down on March 25, 2019. " - "Please provide the API-specific endpoint or use " - "service.new_batch_http_request(). For more details see " - "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" - "and https://developers.google.com/api-client-library/python/guide/batch.", - _LEGACY_BATCH_URI) - self._batch_uri = batch_uri - - # Global callback to be called for each individual response in the batch. - self._callback = callback - - # A map from id to request. - self._requests = {} - - # A map from id to callback. - self._callbacks = {} - - # List of request ids, in the order in which they were added. - self._order = [] - - # The last auto generated id. - self._last_auto_id = 0 - - # Unique ID on which to base the Content-ID headers. - self._base_id = None - - # A map from request id to (httplib2.Response, content) response pairs - self._responses = {} - - # A map of id(Credentials) that have been refreshed. - self._refreshed_credentials = {} - - def _refresh_and_apply_credentials(self, request, http): - """Refresh the credentials and apply to the request. - - Args: - request: HttpRequest, the request. - http: httplib2.Http, the global http object for the batch. - """ - # For the credentials to refresh, but only once per refresh_token - # If there is no http per the request then refresh the http passed in - # via execute() - creds = None - request_credentials = False - - if request.http is not None: - creds = _auth.get_credentials_from_http(request.http) - request_credentials = True - - if creds is None and http is not None: - creds = _auth.get_credentials_from_http(http) - - if creds is not None: - if id(creds) not in self._refreshed_credentials: - _auth.refresh_credentials(creds) - self._refreshed_credentials[id(creds)] = 1 - - # Only apply the credentials if we are using the http object passed in, - # otherwise apply() will get called during _serialize_request(). - if request.http is None or not request_credentials: - _auth.apply_credentials(creds, request.headers) - - - def _id_to_header(self, id_): - """Convert an id to a Content-ID header value. - - Args: - id_: string, identifier of individual request. - - Returns: - A Content-ID header with the id_ encoded into it. A UUID is prepended to - the value because Content-ID headers are supposed to be universally - unique. - """ - if self._base_id is None: - self._base_id = uuid.uuid4() - - # NB: we intentionally leave whitespace between base/id and '+', so RFC2822 - # line folding works properly on Python 3; see - # https://github.com/google/google-api-python-client/issues/164 - return '<%s + %s>' % (self._base_id, quote(id_)) - - def _header_to_id(self, header): - """Convert a Content-ID header value to an id. - - Presumes the Content-ID header conforms to the format that _id_to_header() - returns. - - Args: - header: string, Content-ID header value. - - Returns: - The extracted id value. - - Raises: - BatchError if the header is not in the expected format. - """ - if header[0] != '<' or header[-1] != '>': - raise BatchError("Invalid value for Content-ID: %s" % header) - if '+' not in header: - raise BatchError("Invalid value for Content-ID: %s" % header) - base, id_ = header[1:-1].split(' + ', 1) - - return unquote(id_) - - def _serialize_request(self, request): - """Convert an HttpRequest object into a string. - - Args: - request: HttpRequest, the request to serialize. - - Returns: - The request as a string in application/http format. - """ - # Construct status line - parsed = urlparse(request.uri) - request_line = urlunparse( - ('', '', parsed.path, parsed.params, parsed.query, '') - ) - status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' - major, minor = request.headers.get('content-type', 'application/json').split('/') - msg = MIMENonMultipart(major, minor) - headers = request.headers.copy() - - if request.http is not None: - credentials = _auth.get_credentials_from_http(request.http) - if credentials is not None: - _auth.apply_credentials(credentials, headers) - - # MIMENonMultipart adds its own Content-Type header. - if 'content-type' in headers: - del headers['content-type'] - - for key, value in six.iteritems(headers): - msg[key] = value - msg['Host'] = parsed.netloc - msg.set_unixfrom(None) - - if request.body is not None: - msg.set_payload(request.body) - msg['content-length'] = str(len(request.body)) - - # Serialize the mime message. - fp = StringIO() - # maxheaderlen=0 means don't line wrap headers. - g = Generator(fp, maxheaderlen=0) - g.flatten(msg, unixfrom=False) - body = fp.getvalue() - - return status_line + body - - def _deserialize_response(self, payload): - """Convert string into httplib2 response and content. - - Args: - payload: string, headers and body as a string. - - Returns: - A pair (resp, content), such as would be returned from httplib2.request. - """ - # Strip off the status line - status_line, payload = payload.split('\n', 1) - protocol, status, reason = status_line.split(' ', 2) - - # Parse the rest of the response - parser = FeedParser() - parser.feed(payload) - msg = parser.close() - msg['status'] = status - - # Create httplib2.Response from the parsed headers. - resp = httplib2.Response(msg) - resp.reason = reason - resp.version = int(protocol.split('/', 1)[1].replace('.', '')) - - content = payload.split('\r\n\r\n', 1)[1] - - return resp, content - - def _new_id(self): - """Create a new id. - - Auto incrementing number that avoids conflicts with ids already used. - - Returns: - string, a new unique id. - """ - self._last_auto_id += 1 - while str(self._last_auto_id) in self._requests: - self._last_auto_id += 1 - return str(self._last_auto_id) - - @util.positional(2) - def add(self, request, callback=None, request_id=None): - """Add a new request. - - Every callback added will be paired with a unique id, the request_id. That - unique id will be passed back to the callback when the response comes back - from the server. The default behavior is to have the library generate it's - own unique id. If the caller passes in a request_id then they must ensure - uniqueness for each request_id, and if they are not an exception is - raised. Callers should either supply all request_ids or never supply a - request id, to avoid such an error. - - Args: - request: HttpRequest, Request to add to the batch. - callback: callable, A callback to be called for this response, of the - form callback(id, response, exception). The first parameter is the - request id, and the second is the deserialized response object. The - third is an googleapiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no errors occurred. - request_id: string, A unique id for the request. The id will be passed - to the callback with the response. - - Returns: - None - - Raises: - BatchError if a media request is added to a batch. - KeyError is the request_id is not unique. - """ - - if len(self._order) >= MAX_BATCH_LIMIT: - raise BatchError("Exceeded the maximum calls(%d) in a single batch request." - % MAX_BATCH_LIMIT) - if request_id is None: - request_id = self._new_id() - if request.resumable is not None: - raise BatchError("Media requests cannot be used in a batch request.") - if request_id in self._requests: - raise KeyError("A request with this ID already exists: %s" % request_id) - self._requests[request_id] = request - self._callbacks[request_id] = callback - self._order.append(request_id) - - def _execute(self, http, order, requests): - """Serialize batch request, send to server, process response. - - Args: - http: httplib2.Http, an http object to be used to make the request with. - order: list, list of request ids in the order they were added to the - batch. - request: list, list of request objects to send. - - Raises: - httplib2.HttpLib2Error if a transport error has occured. - googleapiclient.errors.BatchError if the response is the wrong format. - """ - message = MIMEMultipart('mixed') - # Message should not write out it's own headers. - setattr(message, '_write_headers', lambda self: None) - - # Add all the individual requests. - for request_id in order: - request = requests[request_id] - - msg = MIMENonMultipart('application', 'http') - msg['Content-Transfer-Encoding'] = 'binary' - msg['Content-ID'] = self._id_to_header(request_id) - - body = self._serialize_request(request) - msg.set_payload(body) - message.attach(msg) - - # encode the body: note that we can't use `as_string`, because - # it plays games with `From ` lines. - fp = StringIO() - g = Generator(fp, mangle_from_=False) - g.flatten(message, unixfrom=False) - body = fp.getvalue() - - headers = {} - headers['content-type'] = ('multipart/mixed; ' - 'boundary="%s"') % message.get_boundary() - - resp, content = http.request(self._batch_uri, method='POST', body=body, - headers=headers) - - if resp.status >= 300: - raise HttpError(resp, content, uri=self._batch_uri) - - # Prepend with a content-type header so FeedParser can handle it. - header = 'content-type: %s\r\n\r\n' % resp['content-type'] - # PY3's FeedParser only accepts unicode. So we should decode content - # here, and encode each payload again. - if six.PY3: - content = content.decode('utf-8') - for_parser = header + content - - parser = FeedParser() - parser.feed(for_parser) - mime_response = parser.close() - - if not mime_response.is_multipart(): - raise BatchError("Response not in multipart/mixed format.", resp=resp, - content=content) - - for part in mime_response.get_payload(): - request_id = self._header_to_id(part['Content-ID']) - response, content = self._deserialize_response(part.get_payload()) - # We encode content here to emulate normal http response. - if isinstance(content, six.text_type): - content = content.encode('utf-8') - self._responses[request_id] = (response, content) - - @util.positional(1) - def execute(self, http=None): - """Execute all the requests as a single batched HTTP request. - - Args: - http: httplib2.Http, an http object to be used in place of the one the - HttpRequest request object was constructed with. If one isn't supplied - then use a http object from the requests in this batch. - - Returns: - None - - Raises: - httplib2.HttpLib2Error if a transport error has occured. - googleapiclient.errors.BatchError if the response is the wrong format. - """ - # If we have no requests return - if len(self._order) == 0: - return None - - # If http is not supplied use the first valid one given in the requests. - if http is None: - for request_id in self._order: - request = self._requests[request_id] - if request is not None: - http = request.http - break - - if http is None: - raise ValueError("Missing a valid http object.") - - # Special case for OAuth2Credentials-style objects which have not yet been - # refreshed with an initial access_token. - creds = _auth.get_credentials_from_http(http) - if creds is not None: - if not _auth.is_valid(creds): - LOGGER.info('Attempting refresh to obtain initial access_token') - _auth.refresh_credentials(creds) - - self._execute(http, self._order, self._requests) - - # Loop over all the requests and check for 401s. For each 401 request the - # credentials should be refreshed and then sent again in a separate batch. - redo_requests = {} - redo_order = [] - - for request_id in self._order: - resp, content = self._responses[request_id] - if resp['status'] == '401': - redo_order.append(request_id) - request = self._requests[request_id] - self._refresh_and_apply_credentials(request, http) - redo_requests[request_id] = request - - if redo_requests: - self._execute(http, redo_order, redo_requests) - - # Now process all callbacks that are erroring, and raise an exception for - # ones that return a non-2xx response? Or add extra parameter to callback - # that contains an HttpError? - - for request_id in self._order: - resp, content = self._responses[request_id] - - request = self._requests[request_id] - callback = self._callbacks[request_id] - - response = None - exception = None - try: - if resp.status >= 300: - raise HttpError(resp, content, uri=request.uri) - response = request.postproc(resp, content) - except HttpError as e: - exception = e - - if callback is not None: - callback(request_id, response, exception) - if self._callback is not None: - self._callback(request_id, response, exception) - - -class HttpRequestMock(object): - """Mock of HttpRequest. - - Do not construct directly, instead use RequestMockBuilder. - """ - - def __init__(self, resp, content, postproc): - """Constructor for HttpRequestMock - - Args: - resp: httplib2.Response, the response to emulate coming from the request - content: string, the response body - postproc: callable, the post processing function usually supplied by - the model class. See model.JsonModel.response() as an example. - """ - self.resp = resp - self.content = content - self.postproc = postproc - if resp is None: - self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) - if 'reason' in self.resp: - self.resp.reason = self.resp['reason'] - - def execute(self, http=None): - """Execute the request. - - Same behavior as HttpRequest.execute(), but the response is - mocked and not really from an HTTP request/response. - """ - return self.postproc(self.resp, self.content) - - -class RequestMockBuilder(object): - """A simple mock of HttpRequest - - Pass in a dictionary to the constructor that maps request methodIds to - tuples of (httplib2.Response, content, opt_expected_body) that should be - returned when that method is called. None may also be passed in for the - httplib2.Response, in which case a 200 OK response will be generated. - If an opt_expected_body (str or dict) is provided, it will be compared to - the body and UnexpectedBodyError will be raised on inequality. - - Example: - response = '{"data": {"id": "tag:google.c...' - requestBuilder = RequestMockBuilder( - { - 'plus.activities.get': (None, response), - } - ) - googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) - - Methods that you do not supply a response for will return a - 200 OK with an empty string as the response content or raise an excpetion - if check_unexpected is set to True. The methodId is taken from the rpcName - in the discovery document. - - For more details see the project wiki. - """ - - def __init__(self, responses, check_unexpected=False): - """Constructor for RequestMockBuilder - - The constructed object should be a callable object - that can replace the class HttpResponse. - - responses - A dictionary that maps methodIds into tuples - of (httplib2.Response, content). The methodId - comes from the 'rpcName' field in the discovery - document. - check_unexpected - A boolean setting whether or not UnexpectedMethodError - should be raised on unsupplied method. - """ - self.responses = responses - self.check_unexpected = check_unexpected - - def __call__(self, http, postproc, uri, method='GET', body=None, - headers=None, methodId=None, resumable=None): - """Implements the callable interface that discovery.build() expects - of requestBuilder, which is to build an object compatible with - HttpRequest.execute(). See that method for the description of the - parameters and the expected response. - """ - if methodId in self.responses: - response = self.responses[methodId] - resp, content = response[:2] - if len(response) > 2: - # Test the body against the supplied expected_body. - expected_body = response[2] - if bool(expected_body) != bool(body): - # Not expecting a body and provided one - # or expecting a body and not provided one. - raise UnexpectedBodyError(expected_body, body) - if isinstance(expected_body, str): - expected_body = json.loads(expected_body) - body = json.loads(body) - if body != expected_body: - raise UnexpectedBodyError(expected_body, body) - return HttpRequestMock(resp, content, postproc) - elif self.check_unexpected: - raise UnexpectedMethodError(methodId=methodId) - else: - model = JsonModel(False) - return HttpRequestMock(None, '{}', model.response) - - -class HttpMock(object): - """Mock of httplib2.Http""" - - def __init__(self, filename=None, headers=None): - """ - Args: - filename: string, absolute filename to read response from - headers: dict, header to return with response - """ - if headers is None: - headers = {'status': '200'} - if filename: - f = open(filename, 'rb') - self.data = f.read() - f.close() - else: - self.data = None - self.response_headers = headers - self.headers = None - self.uri = None - self.method = None - self.body = None - self.headers = None - - - def request(self, uri, - method='GET', - body=None, - headers=None, - redirections=1, - connection_type=None): - self.uri = uri - self.method = method - self.body = body - self.headers = headers - return httplib2.Response(self.response_headers), self.data - - -class HttpMockSequence(object): - """Mock of httplib2.Http - - Mocks a sequence of calls to request returning different responses for each - call. Create an instance initialized with the desired response headers - and content and then use as if an httplib2.Http instance. - - http = HttpMockSequence([ - ({'status': '401'}, ''), - ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), - ({'status': '200'}, 'echo_request_headers'), - ]) - resp, content = http.request("http://examples.com") - - There are special values you can pass in for content to trigger - behavours that are helpful in testing. - - 'echo_request_headers' means return the request headers in the response body - 'echo_request_headers_as_json' means return the request headers in - the response body - 'echo_request_body' means return the request body in the response body - 'echo_request_uri' means return the request uri in the response body - """ - - def __init__(self, iterable): - """ - Args: - iterable: iterable, a sequence of pairs of (headers, body) - """ - self._iterable = iterable - self.follow_redirects = True - - def request(self, uri, - method='GET', - body=None, - headers=None, - redirections=1, - connection_type=None): - resp, content = self._iterable.pop(0) - if content == 'echo_request_headers': - content = headers - elif content == 'echo_request_headers_as_json': - content = json.dumps(headers) - elif content == 'echo_request_body': - if hasattr(body, 'read'): - content = body.read() - else: - content = body - elif content == 'echo_request_uri': - content = uri - if isinstance(content, six.text_type): - content = content.encode('utf-8') - return httplib2.Response(resp), content - - -def set_user_agent(http, user_agent): - """Set the user-agent on every request. - - Args: - http - An instance of httplib2.Http - or something that acts like it. - user_agent: string, the value for the user-agent header. - - Returns: - A modified instance of http that was passed in. - - Example: - - h = httplib2.Http() - h = set_user_agent(h, "my-app-name/6.0") - - Most of the time the user-agent will be set doing auth, this is for the rare - cases where you are accessing an unauthenticated endpoint. - """ - request_orig = http.request - - # The closure that will replace 'httplib2.Http.request'. - def new_request(uri, method='GET', body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None): - """Modify the request headers to add the user-agent.""" - if headers is None: - headers = {} - if 'user-agent' in headers: - headers['user-agent'] = user_agent + ' ' + headers['user-agent'] - else: - headers['user-agent'] = user_agent - resp, content = request_orig(uri, method, body, headers, - redirections, connection_type) - return resp, content - - http.request = new_request - return http - - -def tunnel_patch(http): - """Tunnel PATCH requests over POST. - Args: - http - An instance of httplib2.Http - or something that acts like it. - - Returns: - A modified instance of http that was passed in. - - Example: - - h = httplib2.Http() - h = tunnel_patch(h, "my-app-name/6.0") - - Useful if you are running on a platform that doesn't support PATCH. - Apply this last if you are using OAuth 1.0, as changing the method - will result in a different signature. - """ - request_orig = http.request - - # The closure that will replace 'httplib2.Http.request'. - def new_request(uri, method='GET', body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None): - """Modify the request headers to add the user-agent.""" - if headers is None: - headers = {} - if method == 'PATCH': - if 'oauth_token' in headers.get('authorization', ''): - LOGGER.warning( - 'OAuth 1.0 request made with Credentials after tunnel_patch.') - headers['x-http-method-override'] = "PATCH" - method = 'POST' - resp, content = request_orig(uri, method, body, headers, - redirections, connection_type) - return resp, content - - http.request = new_request - return http - - -def build_http(): - """Builds httplib2.Http object - - Returns: - A httplib2.Http object, which is used to make http requests, and which has timeout set by default. - To override default timeout call - - socket.setdefaulttimeout(timeout_in_sec) - - before interacting with this method. - """ - if socket.getdefaulttimeout() is not None: - http_timeout = socket.getdefaulttimeout() - else: - http_timeout = DEFAULT_HTTP_TIMEOUT_SEC - return httplib2.Http(timeout=http_timeout) diff --git a/src/googleapiclient/mimeparse.py b/src/googleapiclient/mimeparse.py deleted file mode 100644 index bc9ad094..00000000 --- a/src/googleapiclient/mimeparse.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2014 Joe Gregorio -# -# Licensed under the MIT License - -"""MIME-Type Parser - -This module provides basic functions for handling mime-types. It can handle -matching mime-types against a list of media-ranges. See section 14.1 of the -HTTP specification [RFC 2616] for a complete explanation. - - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 - -Contents: - - parse_mime_type(): Parses a mime-type into its component parts. - - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q' - quality parameter. - - quality(): Determines the quality ('q') of a mime-type when - compared against a list of media-ranges. - - quality_parsed(): Just like quality() except the second parameter must be - pre-parsed. - - best_match(): Choose the mime-type with the highest quality ('q') - from a list of candidates. -""" -from __future__ import absolute_import -from functools import reduce -import six - -__version__ = '0.1.3' -__author__ = 'Joe Gregorio' -__email__ = 'joe@bitworking.org' -__license__ = 'MIT License' -__credits__ = '' - - -def parse_mime_type(mime_type): - """Parses a mime-type into its component parts. - - Carves up a mime-type and returns a tuple of the (type, subtype, params) - where 'params' is a dictionary of all the parameters for the media range. - For example, the media range 'application/xhtml;q=0.5' would get parsed - into: - - ('application', 'xhtml', {'q', '0.5'}) - """ - parts = mime_type.split(';') - params = dict([tuple([s.strip() for s in param.split('=', 1)])\ - for param in parts[1:] - ]) - full_type = parts[0].strip() - # Java URLConnection class sends an Accept header that includes a - # single '*'. Turn it into a legal wildcard. - if full_type == '*': - full_type = '*/*' - (type, subtype) = full_type.split('/') - - return (type.strip(), subtype.strip(), params) - - -def parse_media_range(range): - """Parse a media-range into its component parts. - - Carves up a media range and returns a tuple of the (type, subtype, - params) where 'params' is a dictionary of all the parameters for the media - range. For example, the media range 'application/*;q=0.5' would get parsed - into: - - ('application', '*', {'q', '0.5'}) - - In addition this function also guarantees that there is a value for 'q' - in the params dictionary, filling it in with a proper default if - necessary. - """ - (type, subtype, params) = parse_mime_type(range) - if 'q' not in params or not params['q'] or \ - not float(params['q']) or float(params['q']) > 1\ - or float(params['q']) < 0: - params['q'] = '1' - - return (type, subtype, params) - - -def fitness_and_quality_parsed(mime_type, parsed_ranges): - """Find the best match for a mime-type amongst parsed media-ranges. - - Find the best match for a given mime-type against a list of media_ranges - that have already been parsed by parse_media_range(). Returns a tuple of - the fitness value and the value of the 'q' quality parameter of the best - match, or (-1, 0) if no match was found. Just as for quality_parsed(), - 'parsed_ranges' must be a list of parsed media ranges. - """ - best_fitness = -1 - best_fit_q = 0 - (target_type, target_subtype, target_params) =\ - parse_media_range(mime_type) - for (type, subtype, params) in parsed_ranges: - type_match = (type == target_type or\ - type == '*' or\ - target_type == '*') - subtype_match = (subtype == target_subtype or\ - subtype == '*' or\ - target_subtype == '*') - if type_match and subtype_match: - param_matches = reduce(lambda x, y: x + y, [1 for (key, value) in \ - six.iteritems(target_params) if key != 'q' and \ - key in params and value == params[key]], 0) - fitness = (type == target_type) and 100 or 0 - fitness += (subtype == target_subtype) and 10 or 0 - fitness += param_matches - if fitness > best_fitness: - best_fitness = fitness - best_fit_q = params['q'] - - return best_fitness, float(best_fit_q) - - -def quality_parsed(mime_type, parsed_ranges): - """Find the best match for a mime-type amongst parsed media-ranges. - - Find the best match for a given mime-type against a list of media_ranges - that have already been parsed by parse_media_range(). Returns the 'q' - quality parameter of the best match, 0 if no match was found. This function - bahaves the same as quality() except that 'parsed_ranges' must be a list of - parsed media ranges. - """ - - return fitness_and_quality_parsed(mime_type, parsed_ranges)[1] - - -def quality(mime_type, ranges): - """Return the quality ('q') of a mime-type against a list of media-ranges. - - Returns the quality 'q' of a mime-type when compared against the - media-ranges in ranges. For example: - - >>> quality('text/html','text/*;q=0.3, text/html;q=0.7, - text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5') - 0.7 - - """ - parsed_ranges = [parse_media_range(r) for r in ranges.split(',')] - - return quality_parsed(mime_type, parsed_ranges) - - -def best_match(supported, header): - """Return mime-type with the highest quality ('q') from list of candidates. - - Takes a list of supported mime-types and finds the best match for all the - media-ranges listed in header. The value of header must be a string that - conforms to the format of the HTTP Accept: header. The value of 'supported' - is a list of mime-types. The list of supported mime-types should be sorted - in order of increasing desirability, in case of a situation where there is - a tie. - - >>> best_match(['application/xbel+xml', 'text/xml'], - 'text/*;q=0.5,*/*; q=0.1') - 'text/xml' - """ - split_header = _filter_blank(header.split(',')) - parsed_header = [parse_media_range(r) for r in split_header] - weighted_matches = [] - pos = 0 - for mime_type in supported: - weighted_matches.append((fitness_and_quality_parsed(mime_type, - parsed_header), pos, mime_type)) - pos += 1 - weighted_matches.sort() - - return weighted_matches[-1][0][1] and weighted_matches[-1][2] or '' - - -def _filter_blank(i): - for s in i: - if s.strip(): - yield s diff --git a/src/googleapiclient/model.py b/src/googleapiclient/model.py deleted file mode 100644 index dded04ea..00000000 --- a/src/googleapiclient/model.py +++ /dev/null @@ -1,389 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Model objects for requests and responses. - -Each API may support one or more serializations, such -as JSON, Atom, etc. The model classes are responsible -for converting between the wire format and the Python -object representation. -""" -from __future__ import absolute_import -import six - -__author__ = 'jcgregorio@google.com (Joe Gregorio)' - -import json -import logging - -from six.moves.urllib.parse import urlencode - -from googleapiclient import __version__ -from googleapiclient.errors import HttpError - - -LOGGER = logging.getLogger(__name__) - -dump_request_response = False - - -def _abstract(): - raise NotImplementedError('You need to override this function') - - -class Model(object): - """Model base class. - - All Model classes should implement this interface. - The Model serializes and de-serializes between a wire - format such as JSON and a Python object representation. - """ - - def request(self, headers, path_params, query_params, body_value): - """Updates outgoing requests with a serialized body. - - Args: - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query_params: dict, parameters that appear in the query - body_value: object, the request body as a Python object, which must be - serializable. - Returns: - A tuple of (headers, path_params, query, body) - - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query: string, query part of the request URI - body: string, the body serialized in the desired wire format. - """ - _abstract() - - def response(self, resp, content): - """Convert the response wire format into a Python object. - - Args: - resp: httplib2.Response, the HTTP response headers and status - content: string, the body of the HTTP response - - Returns: - The body de-serialized as a Python object. - - Raises: - googleapiclient.errors.HttpError if a non 2xx response is received. - """ - _abstract() - - -class BaseModel(Model): - """Base model class. - - Subclasses should provide implementations for the "serialize" and - "deserialize" methods, as well as values for the following class attributes. - - Attributes: - accept: The value to use for the HTTP Accept header. - content_type: The value to use for the HTTP Content-type header. - no_content_response: The value to return when deserializing a 204 "No - Content" response. - alt_param: The value to supply as the "alt" query parameter for requests. - """ - - accept = None - content_type = None - no_content_response = None - alt_param = None - - def _log_request(self, headers, path_params, query, body): - """Logs debugging information about the request if requested.""" - if dump_request_response: - LOGGER.info('--request-start--') - LOGGER.info('-headers-start-') - for h, v in six.iteritems(headers): - LOGGER.info('%s: %s', h, v) - LOGGER.info('-headers-end-') - LOGGER.info('-path-parameters-start-') - for h, v in six.iteritems(path_params): - LOGGER.info('%s: %s', h, v) - LOGGER.info('-path-parameters-end-') - LOGGER.info('body: %s', body) - LOGGER.info('query: %s', query) - LOGGER.info('--request-end--') - - def request(self, headers, path_params, query_params, body_value): - """Updates outgoing requests with a serialized body. - - Args: - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query_params: dict, parameters that appear in the query - body_value: object, the request body as a Python object, which must be - serializable by json. - Returns: - A tuple of (headers, path_params, query, body) - - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query: string, query part of the request URI - body: string, the body serialized as JSON - """ - query = self._build_query(query_params) - headers['accept'] = self.accept - headers['accept-encoding'] = 'gzip, deflate' - if 'user-agent' in headers: - headers['user-agent'] += ' ' - else: - headers['user-agent'] = '' - headers['user-agent'] += 'google-api-python-client/%s (gzip)' % __version__ - - if body_value is not None: - headers['content-type'] = self.content_type - body_value = self.serialize(body_value) - self._log_request(headers, path_params, query, body_value) - return (headers, path_params, query, body_value) - - def _build_query(self, params): - """Builds a query string. - - Args: - params: dict, the query parameters - - Returns: - The query parameters properly encoded into an HTTP URI query string. - """ - if self.alt_param is not None: - params.update({'alt': self.alt_param}) - astuples = [] - for key, value in six.iteritems(params): - if type(value) == type([]): - for x in value: - x = x.encode('utf-8') - astuples.append((key, x)) - else: - if isinstance(value, six.text_type) and callable(value.encode): - value = value.encode('utf-8') - astuples.append((key, value)) - return '?' + urlencode(astuples) - - def _log_response(self, resp, content): - """Logs debugging information about the response if requested.""" - if dump_request_response: - LOGGER.info('--response-start--') - for h, v in six.iteritems(resp): - LOGGER.info('%s: %s', h, v) - if content: - LOGGER.info(content) - LOGGER.info('--response-end--') - - def response(self, resp, content): - """Convert the response wire format into a Python object. - - Args: - resp: httplib2.Response, the HTTP response headers and status - content: string, the body of the HTTP response - - Returns: - The body de-serialized as a Python object. - - Raises: - googleapiclient.errors.HttpError if a non 2xx response is received. - """ - self._log_response(resp, content) - # Error handling is TBD, for example, do we retry - # for some operation/error combinations? - if resp.status < 300: - if resp.status == 204: - # A 204: No Content response should be treated differently - # to all the other success states - return self.no_content_response - return self.deserialize(content) - else: - LOGGER.debug('Content from bad request was: %s' % content) - raise HttpError(resp, content) - - def serialize(self, body_value): - """Perform the actual Python object serialization. - - Args: - body_value: object, the request body as a Python object. - - Returns: - string, the body in serialized form. - """ - _abstract() - - def deserialize(self, content): - """Perform the actual deserialization from response string to Python - object. - - Args: - content: string, the body of the HTTP response - - Returns: - The body de-serialized as a Python object. - """ - _abstract() - - -class JsonModel(BaseModel): - """Model class for JSON. - - Serializes and de-serializes between JSON and the Python - object representation of HTTP request and response bodies. - """ - accept = 'application/json' - content_type = 'application/json' - alt_param = 'json' - - def __init__(self, data_wrapper=False): - """Construct a JsonModel. - - Args: - data_wrapper: boolean, wrap requests and responses in a data wrapper - """ - self._data_wrapper = data_wrapper - - def serialize(self, body_value): - if (isinstance(body_value, dict) and 'data' not in body_value and - self._data_wrapper): - body_value = {'data': body_value} - return json.dumps(body_value) - - def deserialize(self, content): - try: - content = content.decode('utf-8') - except AttributeError: - pass - body = json.loads(content) - if self._data_wrapper and isinstance(body, dict) and 'data' in body: - body = body['data'] - return body - - @property - def no_content_response(self): - return {} - - -class RawModel(JsonModel): - """Model class for requests that don't return JSON. - - Serializes and de-serializes between JSON and the Python - object representation of HTTP request, and returns the raw bytes - of the response body. - """ - accept = '*/*' - content_type = 'application/json' - alt_param = None - - def deserialize(self, content): - return content - - @property - def no_content_response(self): - return '' - - -class MediaModel(JsonModel): - """Model class for requests that return Media. - - Serializes and de-serializes between JSON and the Python - object representation of HTTP request, and returns the raw bytes - of the response body. - """ - accept = '*/*' - content_type = 'application/json' - alt_param = 'media' - - def deserialize(self, content): - return content - - @property - def no_content_response(self): - return '' - - -class ProtocolBufferModel(BaseModel): - """Model class for protocol buffers. - - Serializes and de-serializes the binary protocol buffer sent in the HTTP - request and response bodies. - """ - accept = 'application/x-protobuf' - content_type = 'application/x-protobuf' - alt_param = 'proto' - - def __init__(self, protocol_buffer): - """Constructs a ProtocolBufferModel. - - The serialzed protocol buffer returned in an HTTP response will be - de-serialized using the given protocol buffer class. - - Args: - protocol_buffer: The protocol buffer class used to de-serialize a - response from the API. - """ - self._protocol_buffer = protocol_buffer - - def serialize(self, body_value): - return body_value.SerializeToString() - - def deserialize(self, content): - return self._protocol_buffer.FromString(content) - - @property - def no_content_response(self): - return self._protocol_buffer() - - -def makepatch(original, modified): - """Create a patch object. - - Some methods support PATCH, an efficient way to send updates to a resource. - This method allows the easy construction of patch bodies by looking at the - differences between a resource before and after it was modified. - - Args: - original: object, the original deserialized resource - modified: object, the modified deserialized resource - Returns: - An object that contains only the changes from original to modified, in a - form suitable to pass to a PATCH method. - - Example usage: - item = service.activities().get(postid=postid, userid=userid).execute() - original = copy.deepcopy(item) - item['object']['content'] = 'This is updated.' - service.activities.patch(postid=postid, userid=userid, - body=makepatch(original, item)).execute() - """ - patch = {} - for key, original_value in six.iteritems(original): - modified_value = modified.get(key, None) - if modified_value is None: - # Use None to signal that the element is deleted - patch[key] = None - elif original_value != modified_value: - if type(original_value) == type({}): - # Recursively descend objects - patch[key] = makepatch(original_value, modified_value) - else: - # In the case of simple types or arrays we just replace - patch[key] = modified_value - else: - # Don't add anything to patch if there's no change - pass - for key in modified: - if key not in original: - patch[key] = modified[key] - - return patch diff --git a/src/googleapiclient/sample_tools.py b/src/googleapiclient/sample_tools.py deleted file mode 100644 index 5cb7a06e..00000000 --- a/src/googleapiclient/sample_tools.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for making samples. - -Consolidates a lot of code commonly repeated in sample applications. -""" -from __future__ import absolute_import - -__author__ = 'jcgregorio@google.com (Joe Gregorio)' -__all__ = ['init'] - - -import argparse -import os - -from googleapiclient import discovery -from googleapiclient.http import build_http - -def init(argv, name, version, doc, filename, scope=None, parents=[], discovery_filename=None): - """A common initialization routine for samples. - - Many of the sample applications do the same initialization, which has now - been consolidated into this function. This function uses common idioms found - in almost all the samples, i.e. for an API with name 'apiname', the - credentials are stored in a file named apiname.dat, and the - client_secrets.json file is stored in the same directory as the application - main file. - - Args: - argv: list of string, the command-line parameters of the application. - name: string, name of the API. - version: string, version of the API. - doc: string, description of the application. Usually set to __doc__. - file: string, filename of the application. Usually set to __file__. - parents: list of argparse.ArgumentParser, additional command-line flags. - scope: string, The OAuth scope used. - discovery_filename: string, name of local discovery file (JSON). Use when discovery doc not available via URL. - - Returns: - A tuple of (service, flags), where service is the service object and flags - is the parsed command-line flags. - """ - try: - from oauth2client import client - from oauth2client import file - from oauth2client import tools - except ImportError: - raise ImportError('googleapiclient.sample_tools requires oauth2client. Please install oauth2client and try again.') - - if scope is None: - scope = 'https://www.googleapis.com/auth/' + name - - # Parser command-line arguments. - parent_parsers = [tools.argparser] - parent_parsers.extend(parents) - parser = argparse.ArgumentParser( - description=doc, - formatter_class=argparse.RawDescriptionHelpFormatter, - parents=parent_parsers) - flags = parser.parse_args(argv[1:]) - - # Name of a file containing the OAuth 2.0 information for this - # application, including client_id and client_secret, which are found - # on the API Access tab on the Google APIs - # Console . - client_secrets = os.path.join(os.path.dirname(filename), - 'client_secrets.json') - - # Set up a Flow object to be used if we need to authenticate. - flow = client.flow_from_clientsecrets(client_secrets, - scope=scope, - message=tools.message_if_missing(client_secrets)) - - # Prepare credentials, and authorize HTTP object with them. - # If the credentials don't exist or are invalid run through the native client - # flow. The Storage object will ensure that if successful the good - # credentials will get written back to a file. - storage = file.Storage(name + '.dat') - credentials = storage.get() - if credentials is None or credentials.invalid: - credentials = tools.run_flow(flow, storage, flags) - http = credentials.authorize(http=build_http()) - - if discovery_filename is None: - # Construct a service object via the discovery service. - service = discovery.build(name, version, http=http) - else: - # Construct a service object using a local discovery document file. - with open(discovery_filename) as discovery_file: - service = discovery.build_from_document( - discovery_file.read(), - base='https://www.googleapis.com/', - http=http) - return (service, flags) diff --git a/src/googleapiclient/schema.py b/src/googleapiclient/schema.py deleted file mode 100644 index 10d4a1b5..00000000 --- a/src/googleapiclient/schema.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Schema processing for discovery based APIs - -Schemas holds an APIs discovery schemas. It can return those schema as -deserialized JSON objects, or pretty print them as prototype objects that -conform to the schema. - -For example, given the schema: - - schema = \"\"\"{ - "Foo": { - "type": "object", - "properties": { - "etag": { - "type": "string", - "description": "ETag of the collection." - }, - "kind": { - "type": "string", - "description": "Type of the collection ('calendar#acl').", - "default": "calendar#acl" - }, - "nextPageToken": { - "type": "string", - "description": "Token used to access the next - page of this result. Omitted if no further results are available." - } - } - } - }\"\"\" - - s = Schemas(schema) - print s.prettyPrintByName('Foo') - - Produces the following output: - - { - "nextPageToken": "A String", # Token used to access the - # next page of this result. Omitted if no further results are available. - "kind": "A String", # Type of the collection ('calendar#acl'). - "etag": "A String", # ETag of the collection. - }, - -The constructor takes a discovery document in which to look up named schema. -""" -from __future__ import absolute_import -import six - -# TODO(jcgregorio) support format, enum, minimum, maximum - -__author__ = 'jcgregorio@google.com (Joe Gregorio)' - -import copy - -from googleapiclient import _helpers as util - - -class Schemas(object): - """Schemas for an API.""" - - def __init__(self, discovery): - """Constructor. - - Args: - discovery: object, Deserialized discovery document from which we pull - out the named schema. - """ - self.schemas = discovery.get('schemas', {}) - - # Cache of pretty printed schemas. - self.pretty = {} - - @util.positional(2) - def _prettyPrintByName(self, name, seen=None, dent=0): - """Get pretty printed object prototype from the schema name. - - Args: - name: string, Name of schema in the discovery document. - seen: list of string, Names of schema already seen. Used to handle - recursive definitions. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - if seen is None: - seen = [] - - if name in seen: - # Do not fall into an infinite loop over recursive definitions. - return '# Object with schema name: %s' % name - seen.append(name) - - if name not in self.pretty: - self.pretty[name] = _SchemaToStruct(self.schemas[name], - seen, dent=dent).to_str(self._prettyPrintByName) - - seen.pop() - - return self.pretty[name] - - def prettyPrintByName(self, name): - """Get pretty printed object prototype from the schema name. - - Args: - name: string, Name of schema in the discovery document. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - # Return with trailing comma and newline removed. - return self._prettyPrintByName(name, seen=[], dent=1)[:-2] - - @util.positional(2) - def _prettyPrintSchema(self, schema, seen=None, dent=0): - """Get pretty printed object prototype of schema. - - Args: - schema: object, Parsed JSON schema. - seen: list of string, Names of schema already seen. Used to handle - recursive definitions. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - if seen is None: - seen = [] - - return _SchemaToStruct(schema, seen, dent=dent).to_str(self._prettyPrintByName) - - def prettyPrintSchema(self, schema): - """Get pretty printed object prototype of schema. - - Args: - schema: object, Parsed JSON schema. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - # Return with trailing comma and newline removed. - return self._prettyPrintSchema(schema, dent=1)[:-2] - - def get(self, name, default=None): - """Get deserialized JSON schema from the schema name. - - Args: - name: string, Schema name. - default: object, return value if name not found. - """ - return self.schemas.get(name, default) - - -class _SchemaToStruct(object): - """Convert schema to a prototype object.""" - - @util.positional(3) - def __init__(self, schema, seen, dent=0): - """Constructor. - - Args: - schema: object, Parsed JSON schema. - seen: list, List of names of schema already seen while parsing. Used to - handle recursive definitions. - dent: int, Initial indentation depth. - """ - # The result of this parsing kept as list of strings. - self.value = [] - - # The final value of the parsing. - self.string = None - - # The parsed JSON schema. - self.schema = schema - - # Indentation level. - self.dent = dent - - # Method that when called returns a prototype object for the schema with - # the given name. - self.from_cache = None - - # List of names of schema already seen while parsing. - self.seen = seen - - def emit(self, text): - """Add text as a line to the output. - - Args: - text: string, Text to output. - """ - self.value.extend([" " * self.dent, text, '\n']) - - def emitBegin(self, text): - """Add text to the output, but with no line terminator. - - Args: - text: string, Text to output. - """ - self.value.extend([" " * self.dent, text]) - - def emitEnd(self, text, comment): - """Add text and comment to the output with line terminator. - - Args: - text: string, Text to output. - comment: string, Python comment. - """ - if comment: - divider = '\n' + ' ' * (self.dent + 2) + '# ' - lines = comment.splitlines() - lines = [x.rstrip() for x in lines] - comment = divider.join(lines) - self.value.extend([text, ' # ', comment, '\n']) - else: - self.value.extend([text, '\n']) - - def indent(self): - """Increase indentation level.""" - self.dent += 1 - - def undent(self): - """Decrease indentation level.""" - self.dent -= 1 - - def _to_str_impl(self, schema): - """Prototype object based on the schema, in Python code with comments. - - Args: - schema: object, Parsed JSON schema file. - - Returns: - Prototype object based on the schema, in Python code with comments. - """ - stype = schema.get('type') - if stype == 'object': - self.emitEnd('{', schema.get('description', '')) - self.indent() - if 'properties' in schema: - for pname, pschema in six.iteritems(schema.get('properties', {})): - self.emitBegin('"%s": ' % pname) - self._to_str_impl(pschema) - elif 'additionalProperties' in schema: - self.emitBegin('"a_key": ') - self._to_str_impl(schema['additionalProperties']) - self.undent() - self.emit('},') - elif '$ref' in schema: - schemaName = schema['$ref'] - description = schema.get('description', '') - s = self.from_cache(schemaName, seen=self.seen) - parts = s.splitlines() - self.emitEnd(parts[0], description) - for line in parts[1:]: - self.emit(line.rstrip()) - elif stype == 'boolean': - value = schema.get('default', 'True or False') - self.emitEnd('%s,' % str(value), schema.get('description', '')) - elif stype == 'string': - value = schema.get('default', 'A String') - self.emitEnd('"%s",' % str(value), schema.get('description', '')) - elif stype == 'integer': - value = schema.get('default', '42') - self.emitEnd('%s,' % str(value), schema.get('description', '')) - elif stype == 'number': - value = schema.get('default', '3.14') - self.emitEnd('%s,' % str(value), schema.get('description', '')) - elif stype == 'null': - self.emitEnd('None,', schema.get('description', '')) - elif stype == 'any': - self.emitEnd('"",', schema.get('description', '')) - elif stype == 'array': - self.emitEnd('[', schema.get('description')) - self.indent() - self.emitBegin('') - self._to_str_impl(schema['items']) - self.undent() - self.emit('],') - else: - self.emit('Unknown type! %s' % stype) - self.emitEnd('', '') - - self.string = ''.join(self.value) - return self.string - - def to_str(self, from_cache): - """Prototype object based on the schema, in Python code with comments. - - Args: - from_cache: callable(name, seen), Callable that retrieves an object - prototype for a schema with the given name. Seen is a list of schema - names already seen as we recursively descend the schema definition. - - Returns: - Prototype object based on the schema, in Python code with comments. - The lines of the code will all be properly indented. - """ - self.from_cache = from_cache - return self._to_str_impl(self.schema) diff --git a/src/httplib2/__init__.py b/src/httplib2/__init__.py deleted file mode 100644 index fee091d7..00000000 --- a/src/httplib2/__init__.py +++ /dev/null @@ -1,2231 +0,0 @@ -"""Small, fast HTTP client library for Python. - -Features persistent connections, cache, and Google App Engine Standard -Environment support. -""" - -from __future__ import print_function - -__author__ = "Joe Gregorio (joe@bitworking.org)" -__copyright__ = "Copyright 2006, Joe Gregorio" -__contributors__ = [ - "Thomas Broyer (t.broyer@ltgt.net)", - "James Antill", - "Xavier Verges Farrero", - "Jonathan Feinberg", - "Blair Zajac", - "Sam Ruby", - "Louis Nyffenegger", - "Alex Yu", -] -__license__ = "MIT" -__version__ = '0.12.1' - -import base64 -import calendar -import copy -import email -import email.FeedParser -import email.Message -import email.Utils -import errno -import gzip -import httplib -import os -import random -import re -import StringIO -import sys -import time -import urllib -import urlparse -import zlib - -try: - from hashlib import sha1 as _sha, md5 as _md5 -except ImportError: - # prior to Python 2.5, these were separate modules - import sha - import md5 - - _sha = sha.new - _md5 = md5.new -import hmac -from gettext import gettext as _ -import socket - -try: - from httplib2 import socks -except ImportError: - try: - import socks - except (ImportError, AttributeError): - socks = None - -# Build the appropriate socket wrapper for ssl -ssl = None -ssl_SSLError = None -ssl_CertificateError = None -try: - import ssl # python 2.6 -except ImportError: - pass -if ssl is not None: - ssl_SSLError = getattr(ssl, "SSLError", None) - ssl_CertificateError = getattr(ssl, "CertificateError", None) - - -def _ssl_wrap_socket( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname -): - if disable_validation: - cert_reqs = ssl.CERT_NONE - else: - cert_reqs = ssl.CERT_REQUIRED - if ssl_version is None: - ssl_version = ssl.PROTOCOL_SSLv23 - - if hasattr(ssl, "SSLContext"): # Python 2.7.9 - context = ssl.SSLContext(ssl_version) - context.verify_mode = cert_reqs - context.check_hostname = cert_reqs != ssl.CERT_NONE - if cert_file: - context.load_cert_chain(cert_file, key_file) - if ca_certs: - context.load_verify_locations(ca_certs) - return context.wrap_socket(sock, server_hostname=hostname) - else: - return ssl.wrap_socket( - sock, - keyfile=key_file, - certfile=cert_file, - cert_reqs=cert_reqs, - ca_certs=ca_certs, - ssl_version=ssl_version, - ) - - -def _ssl_wrap_socket_unsupported( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname -): - if not disable_validation: - raise CertificateValidationUnsupported( - "SSL certificate validation is not supported without " - "the ssl module installed. To avoid this error, install " - "the ssl module, or explicity disable validation." - ) - ssl_sock = socket.ssl(sock, key_file, cert_file) - return httplib.FakeSocket(sock, ssl_sock) - - -if ssl is None: - _ssl_wrap_socket = _ssl_wrap_socket_unsupported - -if sys.version_info >= (2, 3): - from iri2uri import iri2uri -else: - - def iri2uri(uri): - return uri - - -def has_timeout(timeout): # python 2.6 - if hasattr(socket, "_GLOBAL_DEFAULT_TIMEOUT"): - return timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT - return timeout is not None - - -__all__ = [ - "Http", - "Response", - "ProxyInfo", - "HttpLib2Error", - "RedirectMissingLocation", - "RedirectLimit", - "FailedToDecompressContent", - "UnimplementedDigestAuthOptionError", - "UnimplementedHmacDigestAuthOptionError", - "debuglevel", - "ProxiesUnavailableError", -] - -# The httplib debug level, set to a non-zero value to get debug output -debuglevel = 0 - -# A request will be tried 'RETRIES' times if it fails at the socket/connection level. -RETRIES = 2 - -# Python 2.3 support -if sys.version_info < (2, 4): - - def sorted(seq): - seq.sort() - return seq - - -# Python 2.3 support -def HTTPResponse__getheaders(self): - """Return list of (header, value) tuples.""" - if self.msg is None: - raise httplib.ResponseNotReady() - return self.msg.items() - - -if not hasattr(httplib.HTTPResponse, "getheaders"): - httplib.HTTPResponse.getheaders = HTTPResponse__getheaders - - -# All exceptions raised here derive from HttpLib2Error -class HttpLib2Error(Exception): - pass - - -# Some exceptions can be caught and optionally -# be turned back into responses. -class HttpLib2ErrorWithResponse(HttpLib2Error): - def __init__(self, desc, response, content): - self.response = response - self.content = content - HttpLib2Error.__init__(self, desc) - - -class RedirectMissingLocation(HttpLib2ErrorWithResponse): - pass - - -class RedirectLimit(HttpLib2ErrorWithResponse): - pass - - -class FailedToDecompressContent(HttpLib2ErrorWithResponse): - pass - - -class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): - pass - - -class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): - pass - - -class MalformedHeader(HttpLib2Error): - pass - - -class RelativeURIError(HttpLib2Error): - pass - - -class ServerNotFoundError(HttpLib2Error): - pass - - -class ProxiesUnavailableError(HttpLib2Error): - pass - - -class CertificateValidationUnsupported(HttpLib2Error): - pass - - -class SSLHandshakeError(HttpLib2Error): - pass - - -class NotSupportedOnThisPlatform(HttpLib2Error): - pass - - -class CertificateHostnameMismatch(SSLHandshakeError): - def __init__(self, desc, host, cert): - HttpLib2Error.__init__(self, desc) - self.host = host - self.cert = cert - - -class NotRunningAppEngineEnvironment(HttpLib2Error): - pass - - -# Open Items: -# ----------- -# Proxy support - -# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) - -# Pluggable cache storage (supports storing the cache in -# flat files by default. We need a plug-in architecture -# that can support Berkeley DB and Squid) - -# == Known Issues == -# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. -# Does not handle Cache-Control: max-stale -# Does not use Age: headers when calculating cache freshness. - -# The number of redirections to follow before giving up. -# Note that only GET redirects are automatically followed. -# Will also honor 301 requests by saving that info and never -# requesting that URI again. -DEFAULT_MAX_REDIRECTS = 5 - -from httplib2 import certs -CA_CERTS = certs.where() - -# Which headers are hop-by-hop headers by default -HOP_BY_HOP = [ - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", -] - - -def _get_end2end_headers(response): - hopbyhop = list(HOP_BY_HOP) - hopbyhop.extend([x.strip() for x in response.get("connection", "").split(",")]) - return [header for header in response.keys() if header not in hopbyhop] - - -URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") - - -def parse_uri(uri): - """Parses a URI using the regex given in Appendix B of RFC 3986. - - (scheme, authority, path, query, fragment) = parse_uri(uri) - """ - groups = URI.match(uri).groups() - return (groups[1], groups[3], groups[4], groups[6], groups[8]) - - -def urlnorm(uri): - (scheme, authority, path, query, fragment) = parse_uri(uri) - if not scheme or not authority: - raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) - authority = authority.lower() - scheme = scheme.lower() - if not path: - path = "/" - # Could do syntax based normalization of the URI before - # computing the digest. See Section 6.2.2 of Std 66. - request_uri = query and "?".join([path, query]) or path - scheme = scheme.lower() - defrag_uri = scheme + "://" + authority + request_uri - return scheme, authority, request_uri, defrag_uri - - -# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) -re_url_scheme = re.compile(r"^\w+://") -re_unsafe = re.compile(r"[^\w\-_.()=!]+") - - -def safename(filename): - """Return a filename suitable for the cache. - Strips dangerous and common characters to create a filename we - can use to store the cache in. - """ - if isinstance(filename, str): - filename_bytes = filename - filename = filename.decode("utf-8") - else: - filename_bytes = filename.encode("utf-8") - filemd5 = _md5(filename_bytes).hexdigest() - filename = re_url_scheme.sub("", filename) - filename = re_unsafe.sub("", filename) - - # limit length of filename (vital for Windows) - # https://github.com/httplib2/httplib2/pull/74 - # C:\Users\ \AppData\Local\Temp\ , - # 9 chars + max 104 chars + 20 chars + x + 1 + 32 = max 259 chars - # Thus max safe filename x = 93 chars. Let it be 90 to make a round sum: - filename = filename[:90] - - return ",".join((filename, filemd5)) - - -NORMALIZE_SPACE = re.compile(r"(?:\r\n)?[ \t]+") - - -def _normalize_headers(headers): - return dict( - [ - (key.lower(), NORMALIZE_SPACE.sub(value, " ").strip()) - for (key, value) in headers.iteritems() - ] - ) - - -def _parse_cache_control(headers): - retval = {} - if "cache-control" in headers: - parts = headers["cache-control"].split(",") - parts_with_args = [ - tuple([x.strip().lower() for x in part.split("=", 1)]) - for part in parts - if -1 != part.find("=") - ] - parts_wo_args = [ - (name.strip().lower(), 1) for name in parts if -1 == name.find("=") - ] - retval = dict(parts_with_args + parts_wo_args) - return retval - - -# Whether to use a strict mode to parse WWW-Authenticate headers -# Might lead to bad results in case of ill-formed header value, -# so disabled by default, falling back to relaxed parsing. -# Set to true to turn on, usefull for testing servers. -USE_WWW_AUTH_STRICT_PARSING = 0 - -# In regex below: -# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP -# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space -# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: -# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? -WWW_AUTH_STRICT = re.compile( - r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$" -) -WWW_AUTH_RELAXED = re.compile( - r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? current_age: - retval = "FRESH" - return retval - - -def _decompressContent(response, new_content): - content = new_content - try: - encoding = response.get("content-encoding", None) - if encoding in ["gzip", "deflate"]: - if encoding == "gzip": - content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() - if encoding == "deflate": - content = zlib.decompress(content, -zlib.MAX_WBITS) - response["content-length"] = str(len(content)) - # Record the historical presence of the encoding in a way the won't interfere. - response["-content-encoding"] = response["content-encoding"] - del response["content-encoding"] - except (IOError, zlib.error): - content = "" - raise FailedToDecompressContent( - _("Content purported to be compressed with %s but failed to decompress.") - % response.get("content-encoding"), - response, - content, - ) - return content - - -def _updateCache(request_headers, response_headers, content, cache, cachekey): - if cachekey: - cc = _parse_cache_control(request_headers) - cc_response = _parse_cache_control(response_headers) - if "no-store" in cc or "no-store" in cc_response: - cache.delete(cachekey) - else: - info = email.Message.Message() - for key, value in response_headers.iteritems(): - if key not in ["status", "content-encoding", "transfer-encoding"]: - info[key] = value - - # Add annotations to the cache to indicate what headers - # are variant for this request. - vary = response_headers.get("vary", None) - if vary: - vary_headers = vary.lower().replace(" ", "").split(",") - for header in vary_headers: - key = "-varied-%s" % header - try: - info[key] = request_headers[header] - except KeyError: - pass - - status = response_headers.status - if status == 304: - status = 200 - - status_header = "status: %d\r\n" % status - - header_str = info.as_string() - - header_str = re.sub("\r(?!\n)|(? 0: - service = "cl" - # No point in guessing Base or Spreadsheet - # elif request_uri.find("spreadsheets") > 0: - # service = "wise" - - auth = dict( - Email=credentials[0], - Passwd=credentials[1], - service=service, - source=headers["user-agent"], - ) - resp, content = self.http.request( - "https://www.google.com/accounts/ClientLogin", - method="POST", - body=urlencode(auth), - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - lines = content.split("\n") - d = dict([tuple(line.split("=", 1)) for line in lines if line]) - if resp.status == 403: - self.Auth = "" - else: - self.Auth = d["Auth"] - - def request(self, method, request_uri, headers, content): - """Modify the request headers to add the appropriate - Authorization header.""" - headers["authorization"] = "GoogleLogin Auth=" + self.Auth - - -AUTH_SCHEME_CLASSES = { - "basic": BasicAuthentication, - "wsse": WsseAuthentication, - "digest": DigestAuthentication, - "hmacdigest": HmacDigestAuthentication, - "googlelogin": GoogleLoginAuthentication, -} - -AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] - - -class FileCache(object): - """Uses a local directory as a store for cached files. - Not really safe to use if multiple threads or processes are going to - be running on the same cache. - """ - - def __init__( - self, cache, safe=safename - ): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior - self.cache = cache - self.safe = safe - if not os.path.exists(cache): - os.makedirs(self.cache) - - def get(self, key): - retval = None - cacheFullPath = os.path.join(self.cache, self.safe(key)) - try: - f = file(cacheFullPath, "rb") - retval = f.read() - f.close() - except IOError: - pass - return retval - - def set(self, key, value): - cacheFullPath = os.path.join(self.cache, self.safe(key)) - f = file(cacheFullPath, "wb") - f.write(value) - f.close() - - def delete(self, key): - cacheFullPath = os.path.join(self.cache, self.safe(key)) - if os.path.exists(cacheFullPath): - os.remove(cacheFullPath) - - -class Credentials(object): - def __init__(self): - self.credentials = [] - - def add(self, name, password, domain=""): - self.credentials.append((domain.lower(), name, password)) - - def clear(self): - self.credentials = [] - - def iter(self, domain): - for (cdomain, name, password) in self.credentials: - if cdomain == "" or domain == cdomain: - yield (name, password) - - -class KeyCerts(Credentials): - """Identical to Credentials except that - name/password are mapped to key/cert.""" - - pass - - -class AllHosts(object): - pass - - -class ProxyInfo(object): - """Collect information required to use a proxy.""" - - bypass_hosts = () - - def __init__( - self, - proxy_type, - proxy_host, - proxy_port, - proxy_rdns=True, - proxy_user=None, - proxy_pass=None, - proxy_headers=None, - ): - """Args: - - proxy_type: The type of proxy server. This must be set to one of - socks.PROXY_TYPE_XXX constants. For example: p = - ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', - proxy_port=8000) - proxy_host: The hostname or IP address of the proxy server. - proxy_port: The port that the proxy server is running on. - proxy_rdns: If True (default), DNS queries will not be performed - locally, and instead, handed to the proxy to resolve. This is useful - if the network does not allow resolution of non-local names. In - httplib2 0.9 and earlier, this defaulted to False. - proxy_user: The username used to authenticate with the proxy server. - proxy_pass: The password used to authenticate with the proxy server. - proxy_headers: Additional or modified headers for the proxy connect - request. - """ - self.proxy_type = proxy_type - self.proxy_host = proxy_host - self.proxy_port = proxy_port - self.proxy_rdns = proxy_rdns - self.proxy_user = proxy_user - self.proxy_pass = proxy_pass - self.proxy_headers = proxy_headers - - def astuple(self): - return ( - self.proxy_type, - self.proxy_host, - self.proxy_port, - self.proxy_rdns, - self.proxy_user, - self.proxy_pass, - self.proxy_headers, - ) - - def isgood(self): - return (self.proxy_host != None) and (self.proxy_port != None) - - def applies_to(self, hostname): - return not self.bypass_host(hostname) - - def bypass_host(self, hostname): - """Has this host been excluded from the proxy config""" - if self.bypass_hosts is AllHosts: - return True - - hostname = "." + hostname.lstrip(".") - for skip_name in self.bypass_hosts: - # *.suffix - if skip_name.startswith(".") and hostname.endswith(skip_name): - return True - # exact match - if hostname == "." + skip_name: - return True - return False - - def __repr__(self): - return ( - "" - ).format(p=self) - - -def proxy_info_from_environment(method="http"): - """Read proxy info from the environment variables. - """ - if method not in ["http", "https"]: - return - - env_var = method + "_proxy" - url = os.environ.get(env_var, os.environ.get(env_var.upper())) - if not url: - return - return proxy_info_from_url(url, method, None) - - -def proxy_info_from_url(url, method="http", noproxy=None): - """Construct a ProxyInfo from a URL (such as http_proxy env var) - """ - url = urlparse.urlparse(url) - username = None - password = None - port = None - if "@" in url[1]: - ident, host_port = url[1].split("@", 1) - if ":" in ident: - username, password = ident.split(":", 1) - else: - password = ident - else: - host_port = url[1] - if ":" in host_port: - host, port = host_port.split(":", 1) - else: - host = host_port - - if port: - port = int(port) - else: - port = dict(https=443, http=80)[method] - - proxy_type = 3 # socks.PROXY_TYPE_HTTP - pi = ProxyInfo( - proxy_type=proxy_type, - proxy_host=host, - proxy_port=port, - proxy_user=username or None, - proxy_pass=password or None, - proxy_headers=None, - ) - - bypass_hosts = [] - # If not given an explicit noproxy value, respect values in env vars. - if noproxy is None: - noproxy = os.environ.get("no_proxy", os.environ.get("NO_PROXY", "")) - # Special case: A single '*' character means all hosts should be bypassed. - if noproxy == "*": - bypass_hosts = AllHosts - elif noproxy.strip(): - bypass_hosts = noproxy.split(",") - bypass_hosts = filter(bool, bypass_hosts) # To exclude empty string. - - pi.bypass_hosts = bypass_hosts - return pi - - -class HTTPConnectionWithTimeout(httplib.HTTPConnection): - """HTTPConnection subclass that supports timeouts - - All timeouts are in seconds. If None is passed for timeout then - Python's default timeout for sockets will be used. See for example - the docs of socket.setdefaulttimeout(): - http://docs.python.org/library/socket.html#socket.setdefaulttimeout - """ - - def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): - httplib.HTTPConnection.__init__(self, host, port, strict) - self.timeout = timeout - self.proxy_info = proxy_info - - def connect(self): - """Connect to the host and port specified in __init__.""" - # Mostly verbatim from httplib.py. - if self.proxy_info and socks is None: - raise ProxiesUnavailableError( - "Proxy support missing but proxy use was requested!" - ) - msg = "getaddrinfo returns an empty list" - if self.proxy_info and self.proxy_info.isgood(): - use_proxy = True - proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( - self.proxy_info.astuple() - ) - - host = proxy_host - port = proxy_port - else: - use_proxy = False - - host = self.host - port = self.port - - for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - try: - if use_proxy: - self.sock = socks.socksocket(af, socktype, proto) - self.sock.setproxy( - proxy_type, - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - else: - self.sock = socket.socket(af, socktype, proto) - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - # Different from httplib: support timeouts. - if has_timeout(self.timeout): - self.sock.settimeout(self.timeout) - # End of difference from httplib. - if self.debuglevel > 0: - print("connect: (%s, %s) ************" % (self.host, self.port)) - if use_proxy: - print( - "proxy: %s ************" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) - ) - if use_proxy: - self.sock.connect((self.host, self.port) + sa[2:]) - else: - self.sock.connect(sa) - except socket.error as msg: - if self.debuglevel > 0: - print("connect fail: (%s, %s)" % (self.host, self.port)) - if use_proxy: - print( - "proxy: %s" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) - ) - if self.sock: - self.sock.close() - self.sock = None - continue - break - if not self.sock: - raise socket.error(msg) - - -class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): - """This class allows communication via SSL. - - All timeouts are in seconds. If None is passed for timeout then - Python's default timeout for sockets will be used. See for example - the docs of socket.setdefaulttimeout(): - http://docs.python.org/library/socket.html#socket.setdefaulttimeout - """ - - def __init__( - self, - host, - port=None, - key_file=None, - cert_file=None, - strict=None, - timeout=None, - proxy_info=None, - ca_certs=None, - disable_ssl_certificate_validation=False, - ssl_version=None, - ): - httplib.HTTPSConnection.__init__( - self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict - ) - self.timeout = timeout - self.proxy_info = proxy_info - if ca_certs is None: - ca_certs = CA_CERTS - self.ca_certs = ca_certs - self.disable_ssl_certificate_validation = disable_ssl_certificate_validation - self.ssl_version = ssl_version - - # The following two methods were adapted from https_wrapper.py, released - # with the Google Appengine SDK at - # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py - # under the following license: - # - # Copyright 2007 Google Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - # - - def _GetValidHostsForCert(self, cert): - """Returns a list of valid host globs for an SSL certificate. - - Args: - cert: A dictionary representing an SSL certificate. - Returns: - list: A list of valid host globs. - """ - if "subjectAltName" in cert: - return [x[1] for x in cert["subjectAltName"] if x[0].lower() == "dns"] - else: - return [x[0][1] for x in cert["subject"] if x[0][0].lower() == "commonname"] - - def _ValidateCertificateHostname(self, cert, hostname): - """Validates that a given hostname is valid for an SSL certificate. - - Args: - cert: A dictionary representing an SSL certificate. - hostname: The hostname to test. - Returns: - bool: Whether or not the hostname is valid for this certificate. - """ - hosts = self._GetValidHostsForCert(cert) - for host in hosts: - host_re = host.replace(".", "\.").replace("*", "[^.]*") - if re.search("^%s$" % (host_re,), hostname, re.I): - return True - return False - - def connect(self): - "Connect to a host on a given (SSL) port." - - msg = "getaddrinfo returns an empty list" - if self.proxy_info and self.proxy_info.isgood(): - use_proxy = True - proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( - self.proxy_info.astuple() - ) - - host = proxy_host - port = proxy_port - else: - use_proxy = False - - host = self.host - port = self.port - - address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) - for family, socktype, proto, canonname, sockaddr in address_info: - try: - if use_proxy: - sock = socks.socksocket(family, socktype, proto) - - sock.setproxy( - proxy_type, - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - else: - sock = socket.socket(family, socktype, proto) - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if has_timeout(self.timeout): - sock.settimeout(self.timeout) - - if use_proxy: - sock.connect((self.host, self.port) + sockaddr[:2]) - else: - sock.connect(sockaddr) - self.sock = _ssl_wrap_socket( - sock, - self.key_file, - self.cert_file, - self.disable_ssl_certificate_validation, - self.ca_certs, - self.ssl_version, - self.host, - ) - if self.debuglevel > 0: - print("connect: (%s, %s)" % (self.host, self.port)) - if use_proxy: - print( - "proxy: %s" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) - ) - if not self.disable_ssl_certificate_validation: - cert = self.sock.getpeercert() - hostname = self.host.split(":", 0)[0] - if not self._ValidateCertificateHostname(cert, hostname): - raise CertificateHostnameMismatch( - "Server presented certificate that does not match " - "host %s: %s" % (hostname, cert), - hostname, - cert, - ) - except ( - ssl_SSLError, - ssl_CertificateError, - CertificateHostnameMismatch, - ) as e: - if sock: - sock.close() - if self.sock: - self.sock.close() - self.sock = None - # Unfortunately the ssl module doesn't seem to provide any way - # to get at more detailed error information, in particular - # whether the error is due to certificate validation or - # something else (such as SSL protocol mismatch). - if getattr(e, "errno", None) == ssl.SSL_ERROR_SSL: - raise SSLHandshakeError(e) - else: - raise - except (socket.timeout, socket.gaierror): - raise - except socket.error as msg: - if self.debuglevel > 0: - print("connect fail: (%s, %s)" % (self.host, self.port)) - if use_proxy: - print( - "proxy: %s" - % str( - ( - proxy_host, - proxy_port, - proxy_rdns, - proxy_user, - proxy_pass, - proxy_headers, - ) - ) - ) - if self.sock: - self.sock.close() - self.sock = None - continue - break - if not self.sock: - raise socket.error(msg) - - -SCHEME_TO_CONNECTION = { - "http": HTTPConnectionWithTimeout, - "https": HTTPSConnectionWithTimeout, -} - - -def _new_fixed_fetch(validate_certificate): - - def fixed_fetch( - url, - payload=None, - method="GET", - headers={}, - allow_truncated=False, - follow_redirects=True, - deadline=None, - ): - return fetch( - url, - payload=payload, - method=method, - headers=headers, - allow_truncated=allow_truncated, - follow_redirects=follow_redirects, - deadline=deadline, - validate_certificate=validate_certificate, - ) - - return fixed_fetch - - -class AppEngineHttpConnection(httplib.HTTPConnection): - """Use httplib on App Engine, but compensate for its weirdness. - - The parameters key_file, cert_file, proxy_info, ca_certs, - disable_ssl_certificate_validation, and ssl_version are all dropped on - the ground. - """ - - def __init__( - self, - host, - port=None, - key_file=None, - cert_file=None, - strict=None, - timeout=None, - proxy_info=None, - ca_certs=None, - disable_ssl_certificate_validation=False, - ssl_version=None, - ): - httplib.HTTPConnection.__init__( - self, host, port=port, strict=strict, timeout=timeout - ) - - -class AppEngineHttpsConnection(httplib.HTTPSConnection): - """Same as AppEngineHttpConnection, but for HTTPS URIs. - - The parameters proxy_info, ca_certs, disable_ssl_certificate_validation, - and ssl_version are all dropped on the ground. - """ - - def __init__( - self, - host, - port=None, - key_file=None, - cert_file=None, - strict=None, - timeout=None, - proxy_info=None, - ca_certs=None, - disable_ssl_certificate_validation=False, - ssl_version=None, - ): - httplib.HTTPSConnection.__init__( - self, - host, - port=port, - key_file=key_file, - cert_file=cert_file, - strict=strict, - timeout=timeout, - ) - self._fetch = _new_fixed_fetch(not disable_ssl_certificate_validation) - - -# Use a different connection object for Google App Engine Standard Environment. -def is_gae_instance(): - server_software = os.environ.get('SERVER_SOFTWARE', '') - if (server_software.startswith('Google App Engine/') or - server_software.startswith('Development/') or - server_software.startswith('testutil/')): - return True - return False - - -try: - if not is_gae_instance(): - raise NotRunningAppEngineEnvironment() - - from google.appengine.api import apiproxy_stub_map - if apiproxy_stub_map.apiproxy.GetStub("urlfetch") is None: - raise ImportError - - from google.appengine.api.urlfetch import fetch - - # Update the connection classes to use the Googel App Engine specific ones. - SCHEME_TO_CONNECTION = { - "http": AppEngineHttpConnection, - "https": AppEngineHttpsConnection, - } -except (ImportError, NotRunningAppEngineEnvironment): - pass - - -class Http(object): - """An HTTP client that handles: - - - all methods - - caching - - ETags - - compression, - - HTTPS - - Basic - - Digest - - WSSE - - and more. - """ - - def __init__( - self, - cache=None, - timeout=None, - proxy_info=proxy_info_from_environment, - ca_certs=None, - disable_ssl_certificate_validation=False, - ssl_version=None, - ): - """If 'cache' is a string then it is used as a directory name for - a disk cache. Otherwise it must be an object that supports the - same interface as FileCache. - - All timeouts are in seconds. If None is passed for timeout - then Python's default timeout for sockets will be used. See - for example the docs of socket.setdefaulttimeout(): - http://docs.python.org/library/socket.html#socket.setdefaulttimeout - - `proxy_info` may be: - - a callable that takes the http scheme ('http' or 'https') and - returns a ProxyInfo instance per request. By default, uses - proxy_nfo_from_environment. - - a ProxyInfo instance (static proxy config). - - None (proxy disabled). - - ca_certs is the path of a file containing root CA certificates for SSL - server certificate validation. By default, a CA cert file bundled with - httplib2 is used. - - If disable_ssl_certificate_validation is true, SSL cert validation will - not be performed. - - By default, ssl.PROTOCOL_SSLv23 will be used for the ssl version. - """ - self.proxy_info = proxy_info - self.ca_certs = ca_certs - self.disable_ssl_certificate_validation = disable_ssl_certificate_validation - self.ssl_version = ssl_version - - # Map domain name to an httplib connection - self.connections = {} - # The location of the cache, for now a directory - # where cached responses are held. - if cache and isinstance(cache, basestring): - self.cache = FileCache(cache) - else: - self.cache = cache - - # Name/password - self.credentials = Credentials() - - # Key/cert - self.certificates = KeyCerts() - - # authorization objects - self.authorizations = [] - - # If set to False then no redirects are followed, even safe ones. - self.follow_redirects = True - - # Which HTTP methods do we apply optimistic concurrency to, i.e. - # which methods get an "if-match:" etag header added to them. - self.optimistic_concurrency_methods = ["PUT", "PATCH"] - - # If 'follow_redirects' is True, and this is set to True then - # all redirecs are followed, including unsafe ones. - self.follow_all_redirects = False - - self.ignore_etag = False - - self.force_exception_to_status_code = False - - self.timeout = timeout - - # Keep Authorization: headers on a redirect. - self.forward_authorization_headers = False - - def __getstate__(self): - state_dict = copy.copy(self.__dict__) - # In case request is augmented by some foreign object such as - # credentials which handle auth - if "request" in state_dict: - del state_dict["request"] - if "connections" in state_dict: - del state_dict["connections"] - return state_dict - - def __setstate__(self, state): - self.__dict__.update(state) - self.connections = {} - - def _auth_from_challenge(self, host, request_uri, headers, response, content): - """A generator that creates Authorization objects - that can be applied to requests. - """ - challenges = _parse_www_authenticate(response, "www-authenticate") - for cred in self.credentials.iter(host): - for scheme in AUTH_SCHEME_ORDER: - if scheme in challenges: - yield AUTH_SCHEME_CLASSES[scheme]( - cred, host, request_uri, headers, response, content, self - ) - - def add_credentials(self, name, password, domain=""): - """Add a name and password that will be used - any time a request requires authentication.""" - self.credentials.add(name, password, domain) - - def add_certificate(self, key, cert, domain): - """Add a key and cert that will be used - any time a request requires authentication.""" - self.certificates.add(key, cert, domain) - - def clear_credentials(self): - """Remove all the names and passwords - that are used for authentication""" - self.credentials.clear() - self.authorizations = [] - - def _conn_request(self, conn, request_uri, method, body, headers): - i = 0 - seen_bad_status_line = False - while i < RETRIES: - i += 1 - try: - if hasattr(conn, "sock") and conn.sock is None: - conn.connect() - conn.request(method, request_uri, body, headers) - except socket.timeout: - raise - except socket.gaierror: - conn.close() - raise ServerNotFoundError("Unable to find the server at %s" % conn.host) - except ssl_SSLError: - conn.close() - raise - except socket.error as e: - err = 0 - if hasattr(e, "args"): - err = getattr(e, "args")[0] - else: - err = e.errno - if err == errno.ECONNREFUSED: # Connection refused - raise - if err in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES: - continue # retry on potentially transient socket errors - except httplib.HTTPException: - # Just because the server closed the connection doesn't apparently mean - # that the server didn't send a response. - if hasattr(conn, "sock") and conn.sock is None: - if i < RETRIES - 1: - conn.close() - conn.connect() - continue - else: - conn.close() - raise - if i < RETRIES - 1: - conn.close() - conn.connect() - continue - try: - response = conn.getresponse() - except httplib.BadStatusLine: - # If we get a BadStatusLine on the first try then that means - # the connection just went stale, so retry regardless of the - # number of RETRIES set. - if not seen_bad_status_line and i == 1: - i = 0 - seen_bad_status_line = True - conn.close() - conn.connect() - continue - else: - conn.close() - raise - except (socket.error, httplib.HTTPException): - if i < RETRIES - 1: - conn.close() - conn.connect() - continue - else: - conn.close() - raise - else: - content = "" - if method == "HEAD": - conn.close() - else: - content = response.read() - response = Response(response) - if method != "HEAD": - content = _decompressContent(response, content) - break - return (response, content) - - def _request( - self, - conn, - host, - absolute_uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, - ): - """Do the actual request using the connection object - and also follow one level of redirects if necessary""" - - auths = [ - (auth.depth(request_uri), auth) - for auth in self.authorizations - if auth.inscope(host, request_uri) - ] - auth = auths and sorted(auths)[0][1] or None - if auth: - auth.request(method, request_uri, headers, body) - - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) - - if auth: - if auth.response(response, body): - auth.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) - response._stale_digest = 1 - - if response.status == 401: - for authorization in self._auth_from_challenge( - host, request_uri, headers, response, content - ): - authorization.request(method, request_uri, headers, body) - (response, content) = self._conn_request( - conn, request_uri, method, body, headers - ) - if response.status != 401: - self.authorizations.append(authorization) - authorization.response(response, body) - break - - if ( - self.follow_all_redirects - or (method in ["GET", "HEAD"]) - or response.status == 303 - ): - if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: - # Pick out the location header and basically start from the beginning - # remembering first to strip the ETag header and decrement our 'depth' - if redirections: - if "location" not in response and response.status != 300: - raise RedirectMissingLocation( - _( - "Redirected but the response is missing a Location: header." - ), - response, - content, - ) - # Fix-up relative redirects (which violate an RFC 2616 MUST) - if "location" in response: - location = response["location"] - (scheme, authority, path, query, fragment) = parse_uri(location) - if authority == None: - response["location"] = urlparse.urljoin( - absolute_uri, location - ) - if response.status == 301 and method in ["GET", "HEAD"]: - response["-x-permanent-redirect-url"] = response["location"] - if "content-location" not in response: - response["content-location"] = absolute_uri - _updateCache(headers, response, content, self.cache, cachekey) - if "if-none-match" in headers: - del headers["if-none-match"] - if "if-modified-since" in headers: - del headers["if-modified-since"] - if ( - "authorization" in headers - and not self.forward_authorization_headers - ): - del headers["authorization"] - if "location" in response: - location = response["location"] - old_response = copy.deepcopy(response) - if "content-location" not in old_response: - old_response["content-location"] = absolute_uri - redirect_method = method - if response.status in [302, 303]: - redirect_method = "GET" - body = None - (response, content) = self.request( - location, - method=redirect_method, - body=body, - headers=headers, - redirections=redirections - 1, - ) - response.previous = old_response - else: - raise RedirectLimit( - "Redirected more times than rediection_limit allows.", - response, - content, - ) - elif response.status in [200, 203] and method in ["GET", "HEAD"]: - # Don't cache 206's since we aren't going to handle byte range requests - if "content-location" not in response: - response["content-location"] = absolute_uri - _updateCache(headers, response, content, self.cache, cachekey) - - return (response, content) - - def _normalize_headers(self, headers): - return _normalize_headers(headers) - - # Need to catch and rebrand some exceptions - # Then need to optionally turn all exceptions into status codes - # including all socket.* and httplib.* exceptions. - - def request( - self, - uri, - method="GET", - body=None, - headers=None, - redirections=DEFAULT_MAX_REDIRECTS, - connection_type=None, - ): - """ Performs a single HTTP request. - - The 'uri' is the URI of the HTTP resource and can begin with either - 'http' or 'https'. The value of 'uri' must be an absolute URI. - - The 'method' is the HTTP method to perform, such as GET, POST, DELETE, - etc. There is no restriction on the methods allowed. - - The 'body' is the entity body to be sent with the request. It is a - string object. - - Any extra headers that are to be sent with the request should be - provided in the 'headers' dictionary. - - The maximum number of redirect to follow before raising an - exception is 'redirections. The default is 5. - - The return value is a tuple of (response, content), the first - being and instance of the 'Response' class, the second being - a string that contains the response entity body. - """ - conn_key = '' - - try: - if headers is None: - headers = {} - else: - headers = self._normalize_headers(headers) - - if "user-agent" not in headers: - headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__ - - uri = iri2uri(uri) - - (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) - - proxy_info = self._get_proxy_info(scheme, authority) - - conn_key = scheme + ":" + authority - conn = self.connections.get(conn_key) - if conn is None: - if not connection_type: - connection_type = SCHEME_TO_CONNECTION[scheme] - certs = list(self.certificates.iter(authority)) - if scheme == "https": - if certs: - conn = self.connections[conn_key] = connection_type( - authority, - key_file=certs[0][0], - cert_file=certs[0][1], - timeout=self.timeout, - proxy_info=proxy_info, - ca_certs=self.ca_certs, - disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, - ssl_version=self.ssl_version, - ) - else: - conn = self.connections[conn_key] = connection_type( - authority, - timeout=self.timeout, - proxy_info=proxy_info, - ca_certs=self.ca_certs, - disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, - ssl_version=self.ssl_version, - ) - else: - conn = self.connections[conn_key] = connection_type( - authority, timeout=self.timeout, proxy_info=proxy_info - ) - conn.set_debuglevel(debuglevel) - - if "range" not in headers and "accept-encoding" not in headers: - headers["accept-encoding"] = "gzip, deflate" - - info = email.Message.Message() - cached_value = None - if self.cache: - cachekey = defrag_uri.encode("utf-8") - cached_value = self.cache.get(cachekey) - if cached_value: - # info = email.message_from_string(cached_value) - # - # Need to replace the line above with the kludge below - # to fix the non-existent bug not fixed in this - # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html - try: - info, content = cached_value.split("\r\n\r\n", 1) - feedparser = email.FeedParser.FeedParser() - feedparser.feed(info) - info = feedparser.close() - feedparser._parse = None - except (IndexError, ValueError): - self.cache.delete(cachekey) - cachekey = None - cached_value = None - else: - cachekey = None - - if ( - method in self.optimistic_concurrency_methods - and self.cache - and "etag" in info - and not self.ignore_etag - and "if-match" not in headers - ): - # http://www.w3.org/1999/04/Editing/ - headers["if-match"] = info["etag"] - - if method not in ["GET", "HEAD"] and self.cache and cachekey: - # RFC 2616 Section 13.10 - self.cache.delete(cachekey) - - # Check the vary header in the cache to see if this request - # matches what varies in the cache. - if method in ["GET", "HEAD"] and "vary" in info: - vary = info["vary"] - vary_headers = vary.lower().replace(" ", "").split(",") - for header in vary_headers: - key = "-varied-%s" % header - value = info[key] - if headers.get(header, None) != value: - cached_value = None - break - - if ( - cached_value - and method in ["GET", "HEAD"] - and self.cache - and "range" not in headers - ): - if "-x-permanent-redirect-url" in info: - # Should cached permanent redirects be counted in our redirection count? For now, yes. - if redirections <= 0: - raise RedirectLimit( - "Redirected more times than rediection_limit allows.", - {}, - "", - ) - (response, new_content) = self.request( - info["-x-permanent-redirect-url"], - method="GET", - headers=headers, - redirections=redirections - 1, - ) - response.previous = Response(info) - response.previous.fromcache = True - else: - # Determine our course of action: - # Is the cached entry fresh or stale? - # Has the client requested a non-cached response? - # - # There seems to be three possible answers: - # 1. [FRESH] Return the cache entry w/o doing a GET - # 2. [STALE] Do the GET (but add in cache validators if available) - # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request - entry_disposition = _entry_disposition(info, headers) - - if entry_disposition == "FRESH": - if not cached_value: - info["status"] = "504" - content = "" - response = Response(info) - if cached_value: - response.fromcache = True - return (response, content) - - if entry_disposition == "STALE": - if ( - "etag" in info - and not self.ignore_etag - and not "if-none-match" in headers - ): - headers["if-none-match"] = info["etag"] - if "last-modified" in info and not "last-modified" in headers: - headers["if-modified-since"] = info["last-modified"] - elif entry_disposition == "TRANSPARENT": - pass - - (response, new_content) = self._request( - conn, - authority, - uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, - ) - - if response.status == 304 and method == "GET": - # Rewrite the cache entry with the new end-to-end headers - # Take all headers that are in response - # and overwrite their values in info. - # unless they are hop-by-hop, or are listed in the connection header. - - for key in _get_end2end_headers(response): - info[key] = response[key] - merged_response = Response(info) - if hasattr(response, "_stale_digest"): - merged_response._stale_digest = response._stale_digest - _updateCache( - headers, merged_response, content, self.cache, cachekey - ) - response = merged_response - response.status = 200 - response.fromcache = True - - elif response.status == 200: - content = new_content - else: - self.cache.delete(cachekey) - content = new_content - else: - cc = _parse_cache_control(headers) - if "only-if-cached" in cc: - info["status"] = "504" - response = Response(info) - content = "" - else: - (response, content) = self._request( - conn, - authority, - uri, - request_uri, - method, - body, - headers, - redirections, - cachekey, - ) - except Exception as e: - is_timeout = isinstance(e, socket.timeout) - if is_timeout: - conn = self.connections.pop(conn_key, None) - if conn: - conn.close() - - if self.force_exception_to_status_code: - if isinstance(e, HttpLib2ErrorWithResponse): - response = e.response - content = e.content - response.status = 500 - response.reason = str(e) - elif is_timeout: - content = "Request Timeout" - response = Response( - { - "content-type": "text/plain", - "status": "408", - "content-length": len(content), - } - ) - response.reason = "Request Timeout" - else: - content = str(e) - response = Response( - { - "content-type": "text/plain", - "status": "400", - "content-length": len(content), - } - ) - response.reason = "Bad Request" - else: - raise - - return (response, content) - - def _get_proxy_info(self, scheme, authority): - """Return a ProxyInfo instance (or None) based on the scheme - and authority. - """ - hostname, port = urllib.splitport(authority) - proxy_info = self.proxy_info - if callable(proxy_info): - proxy_info = proxy_info(scheme) - - if hasattr(proxy_info, "applies_to") and not proxy_info.applies_to(hostname): - proxy_info = None - return proxy_info - - -class Response(dict): - """An object more like email.Message than httplib.HTTPResponse.""" - - """Is this response from our local cache""" - fromcache = False - """HTTP protocol version used by server. - - 10 for HTTP/1.0, 11 for HTTP/1.1. - """ - version = 11 - - "Status code returned by server. " - status = 200 - """Reason phrase returned by server.""" - reason = "Ok" - - previous = None - - def __init__(self, info): - # info is either an email.Message or - # an httplib.HTTPResponse object. - if isinstance(info, httplib.HTTPResponse): - for key, value in info.getheaders(): - self[key.lower()] = value - self.status = info.status - self["status"] = str(self.status) - self.reason = info.reason - self.version = info.version - elif isinstance(info, email.Message.Message): - for key, value in info.items(): - self[key.lower()] = value - self.status = int(self["status"]) - else: - for key, value in info.iteritems(): - self[key.lower()] = value - self.status = int(self.get("status", self.status)) - self.reason = self.get("reason", self.reason) - - def __getattr__(self, name): - if name == "dict": - return self - else: - raise AttributeError(name) diff --git a/src/httplib2/certs.py b/src/httplib2/certs.py deleted file mode 100644 index 59d1ffc7..00000000 --- a/src/httplib2/certs.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Utilities for certificate management.""" - -import os - -certifi_available = False -certifi_where = None -try: - from certifi import where as certifi_where - certifi_available = True -except ImportError: - pass - -custom_ca_locater_available = False -custom_ca_locater_where = None -try: - from ca_certs_locater import get as custom_ca_locater_where - custom_ca_locater_available = True -except ImportError: - pass - - -BUILTIN_CA_CERTS = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "cacerts.txt" -) - - -def where(): - env = os.environ.get("HTTPLIB2_CA_CERTS") - if env is not None: - if os.path.isfile(env): - return env - else: - raise RuntimeError("Environment variable HTTPLIB2_CA_CERTS not a valid file") - if custom_ca_locater_available: - return custom_ca_locater_where() - if certifi_available: - return certifi_where() - return BUILTIN_CA_CERTS - - -if __name__ == "__main__": - print(where()) diff --git a/src/httplib2/iri2uri.py b/src/httplib2/iri2uri.py deleted file mode 100644 index 0a978a78..00000000 --- a/src/httplib2/iri2uri.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Converts an IRI to a URI.""" - -__author__ = "Joe Gregorio (joe@bitworking.org)" -__copyright__ = "Copyright 2006, Joe Gregorio" -__contributors__ = [] -__version__ = "1.0.0" -__license__ = "MIT" - -import urlparse - -# Convert an IRI to a URI following the rules in RFC 3987 -# -# The characters we need to enocde and escape are defined in the spec: -# -# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD -# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF -# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD -# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD -# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD -# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD -# / %xD0000-DFFFD / %xE1000-EFFFD - -escape_range = [ - (0xA0, 0xD7FF), - (0xE000, 0xF8FF), - (0xF900, 0xFDCF), - (0xFDF0, 0xFFEF), - (0x10000, 0x1FFFD), - (0x20000, 0x2FFFD), - (0x30000, 0x3FFFD), - (0x40000, 0x4FFFD), - (0x50000, 0x5FFFD), - (0x60000, 0x6FFFD), - (0x70000, 0x7FFFD), - (0x80000, 0x8FFFD), - (0x90000, 0x9FFFD), - (0xA0000, 0xAFFFD), - (0xB0000, 0xBFFFD), - (0xC0000, 0xCFFFD), - (0xD0000, 0xDFFFD), - (0xE1000, 0xEFFFD), - (0xF0000, 0xFFFFD), - (0x100000, 0x10FFFD), -] - - -def encode(c): - retval = c - i = ord(c) - for low, high in escape_range: - if i < low: - break - if i >= low and i <= high: - retval = "".join(["%%%2X" % ord(o) for o in c.encode("utf-8")]) - break - return retval - - -def iri2uri(uri): - """Convert an IRI to a URI. Note that IRIs must be - passed in a unicode strings. That is, do not utf-8 encode - the IRI before passing it into the function.""" - if isinstance(uri, unicode): - (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) - authority = authority.encode("idna") - # For each character in 'ucschar' or 'iprivate' - # 1. encode as utf-8 - # 2. then %-encode each octet of that utf-8 - uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) - uri = "".join([encode(c) for c in uri]) - return uri - - -if __name__ == "__main__": - import unittest - - class Test(unittest.TestCase): - def test_uris(self): - """Test that URIs are invariant under the transformation.""" - invariant = [ - u"ftp://ftp.is.co.za/rfc/rfc1808.txt", - u"http://www.ietf.org/rfc/rfc2396.txt", - u"ldap://[2001:db8::7]/c=GB?objectClass?one", - u"mailto:John.Doe@example.com", - u"news:comp.infosystems.www.servers.unix", - u"tel:+1-816-555-1212", - u"telnet://192.0.2.16:80/", - u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2", - ] - for uri in invariant: - self.assertEqual(uri, iri2uri(uri)) - - def test_iri(self): - """Test that the right type of escaping is done for each part of the URI.""" - self.assertEqual( - "http://xn--o3h.com/%E2%98%84", - iri2uri(u"http://\N{COMET}.com/\N{COMET}"), - ) - self.assertEqual( - "http://bitworking.org/?fred=%E2%98%84", - iri2uri(u"http://bitworking.org/?fred=\N{COMET}"), - ) - self.assertEqual( - "http://bitworking.org/#%E2%98%84", - iri2uri(u"http://bitworking.org/#\N{COMET}"), - ) - self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}")) - self.assertEqual( - "/fred?bar=%E2%98%9A#%E2%98%84", - iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"), - ) - self.assertEqual( - "/fred?bar=%E2%98%9A#%E2%98%84", - iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")), - ) - self.assertNotEqual( - "/fred?bar=%E2%98%9A#%E2%98%84", - iri2uri( - u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode("utf-8") - ), - ) - - unittest.main() diff --git a/src/httplib2/socks.py b/src/httplib2/socks.py deleted file mode 100644 index 5cef7760..00000000 --- a/src/httplib2/socks.py +++ /dev/null @@ -1,510 +0,0 @@ -"""SocksiPy - Python SOCKS module. - -Version 1.00 - -Copyright 2006 Dan-Haim. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. -3. Neither the name of Dan Haim nor the names of his contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED -WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. - -This module provides a standard socket-like interface for Python -for tunneling connections through SOCKS proxies. - -Minor modifications made by Christopher Gilbert (http://motomastyle.com/) for -use in PyLoris (http://pyloris.sourceforge.net/). - -Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) -mainly to merge bug fixes found in Sourceforge. -""" - -import base64 -import socket -import struct -import sys - -if getattr(socket, "socket", None) is None: - raise ImportError("socket.socket missing, proxy support unusable") - -PROXY_TYPE_SOCKS4 = 1 -PROXY_TYPE_SOCKS5 = 2 -PROXY_TYPE_HTTP = 3 -PROXY_TYPE_HTTP_NO_TUNNEL = 4 - -_defaultproxy = None -_orgsocket = socket.socket - - -class ProxyError(Exception): - pass - - -class GeneralProxyError(ProxyError): - pass - - -class Socks5AuthError(ProxyError): - pass - - -class Socks5Error(ProxyError): - pass - - -class Socks4Error(ProxyError): - pass - - -class HTTPError(ProxyError): - pass - - -_generalerrors = ( - "success", - "invalid data", - "not connected", - "not available", - "bad proxy type", - "bad input", -) - -_socks5errors = ( - "succeeded", - "general SOCKS server failure", - "connection not allowed by ruleset", - "Network unreachable", - "Host unreachable", - "Connection refused", - "TTL expired", - "Command not supported", - "Address type not supported", - "Unknown error", -) - -_socks5autherrors = ( - "succeeded", - "authentication is required", - "all offered authentication methods were rejected", - "unknown username or invalid password", - "unknown error", -) - -_socks4errors = ( - "request granted", - "request rejected or failed", - "request rejected because SOCKS server cannot connect to identd on the client", - "request rejected because the client program and identd report different " - "user-ids", - "unknown error", -) - - -def setdefaultproxy( - proxytype=None, addr=None, port=None, rdns=True, username=None, password=None -): - """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) - Sets a default proxy which all further socksocket objects will use, - unless explicitly changed. - """ - global _defaultproxy - _defaultproxy = (proxytype, addr, port, rdns, username, password) - - -def wrapmodule(module): - """wrapmodule(module) - - Attempts to replace a module's socket library with a SOCKS socket. Must set - a default proxy using setdefaultproxy(...) first. - This will only work on modules that import socket directly into the - namespace; - most of the Python Standard Library falls into this category. - """ - if _defaultproxy != None: - module.socket.socket = socksocket - else: - raise GeneralProxyError((4, "no proxy specified")) - - -class socksocket(socket.socket): - """socksocket([family[, type[, proto]]]) -> socket object - Open a SOCKS enabled socket. The parameters are the same as - those of the standard socket init. In order for SOCKS to work, - you must specify family=AF_INET, type=SOCK_STREAM and proto=0. - """ - - def __init__( - self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None - ): - _orgsocket.__init__(self, family, type, proto, _sock) - if _defaultproxy != None: - self.__proxy = _defaultproxy - else: - self.__proxy = (None, None, None, None, None, None) - self.__proxysockname = None - self.__proxypeername = None - self.__httptunnel = True - - def __recvall(self, count): - """__recvall(count) -> data - Receive EXACTLY the number of bytes requested from the socket. - Blocks until the required number of bytes have been received. - """ - data = self.recv(count) - while len(data) < count: - d = self.recv(count - len(data)) - if not d: - raise GeneralProxyError((0, "connection closed unexpectedly")) - data = data + d - return data - - def sendall(self, content, *args): - """ override socket.socket.sendall method to rewrite the header - for non-tunneling proxies if needed - """ - if not self.__httptunnel: - content = self.__rewriteproxy(content) - return super(socksocket, self).sendall(content, *args) - - def __rewriteproxy(self, header): - """ rewrite HTTP request headers to support non-tunneling proxies - (i.e. those which do not support the CONNECT method). - This only works for HTTP (not HTTPS) since HTTPS requires tunneling. - """ - host, endpt = None, None - hdrs = header.split("\r\n") - for hdr in hdrs: - if hdr.lower().startswith("host:"): - host = hdr - elif hdr.lower().startswith("get") or hdr.lower().startswith("post"): - endpt = hdr - if host and endpt: - hdrs.remove(host) - hdrs.remove(endpt) - host = host.split(" ")[1] - endpt = endpt.split(" ") - if self.__proxy[4] != None and self.__proxy[5] != None: - hdrs.insert(0, self.__getauthheader()) - hdrs.insert(0, "Host: %s" % host) - hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2])) - return "\r\n".join(hdrs) - - def __getauthheader(self): - auth = self.__proxy[4] + ":" + self.__proxy[5] - return "Proxy-Authorization: Basic " + base64.b64encode(auth) - - def setproxy( - self, - proxytype=None, - addr=None, - port=None, - rdns=True, - username=None, - password=None, - headers=None, - ): - """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) - - Sets the proxy to be used. - proxytype - The type of the proxy to be used. Three types - are supported: PROXY_TYPE_SOCKS4 (including socks4a), - PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP - addr - The address of the server (IP or DNS). - port - The port of the server. Defaults to 1080 for SOCKS - servers and 8080 for HTTP proxy servers. - rdns - Should DNS queries be preformed on the remote side - (rather than the local side). The default is True. - Note: This has no effect with SOCKS4 servers. - username - Username to authenticate with to the server. - The default is no authentication. - password - Password to authenticate with to the server. - Only relevant when username is also provided. - headers - Additional or modified headers for the proxy connect - request. - """ - self.__proxy = (proxytype, addr, port, rdns, username, password, headers) - - def __negotiatesocks5(self, destaddr, destport): - """__negotiatesocks5(self,destaddr,destport) - Negotiates a connection through a SOCKS5 server. - """ - # First we'll send the authentication packages we support. - if (self.__proxy[4] != None) and (self.__proxy[5] != None): - # The username/password details were supplied to the - # setproxy method so we support the USERNAME/PASSWORD - # authentication (in addition to the standard none). - self.sendall(struct.pack("BBBB", 0x05, 0x02, 0x00, 0x02)) - else: - # No username/password were entered, therefore we - # only support connections with no authentication. - self.sendall(struct.pack("BBB", 0x05, 0x01, 0x00)) - # We'll receive the server's response to determine which - # method was selected - chosenauth = self.__recvall(2) - if chosenauth[0:1] != chr(0x05).encode(): - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - # Check the chosen authentication method - if chosenauth[1:2] == chr(0x00).encode(): - # No authentication is required - pass - elif chosenauth[1:2] == chr(0x02).encode(): - # Okay, we need to perform a basic username/password - # authentication. - self.sendall( - chr(0x01).encode() - + chr(len(self.__proxy[4])) - + self.__proxy[4] - + chr(len(self.__proxy[5])) - + self.__proxy[5] - ) - authstat = self.__recvall(2) - if authstat[0:1] != chr(0x01).encode(): - # Bad response - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - if authstat[1:2] != chr(0x00).encode(): - # Authentication failed - self.close() - raise Socks5AuthError((3, _socks5autherrors[3])) - # Authentication succeeded - else: - # Reaching here is always bad - self.close() - if chosenauth[1] == chr(0xFF).encode(): - raise Socks5AuthError((2, _socks5autherrors[2])) - else: - raise GeneralProxyError((1, _generalerrors[1])) - # Now we can request the actual connection - req = struct.pack("BBB", 0x05, 0x01, 0x00) - # If the given destination address is an IP address, we'll - # use the IPv4 address request even if remote resolving was specified. - try: - ipaddr = socket.inet_aton(destaddr) - req = req + chr(0x01).encode() + ipaddr - except socket.error: - # Well it's not an IP number, so it's probably a DNS name. - if self.__proxy[3]: - # Resolve remotely - ipaddr = None - req = ( - req - + chr(0x03).encode() - + chr(len(destaddr)).encode() - + destaddr.encode() - ) - else: - # Resolve locally - ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) - req = req + chr(0x01).encode() + ipaddr - req = req + struct.pack(">H", destport) - self.sendall(req) - # Get the response - resp = self.__recvall(4) - if resp[0:1] != chr(0x05).encode(): - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - elif resp[1:2] != chr(0x00).encode(): - # Connection failed - self.close() - if ord(resp[1:2]) <= 8: - raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) - else: - raise Socks5Error((9, _socks5errors[9])) - # Get the bound address/port - elif resp[3:4] == chr(0x01).encode(): - boundaddr = self.__recvall(4) - elif resp[3:4] == chr(0x03).encode(): - resp = resp + self.recv(1) - boundaddr = self.__recvall(ord(resp[4:5])) - else: - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - boundport = struct.unpack(">H", self.__recvall(2))[0] - self.__proxysockname = (boundaddr, boundport) - if ipaddr != None: - self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) - else: - self.__proxypeername = (destaddr, destport) - - def getproxysockname(self): - """getsockname() -> address info - Returns the bound IP address and port number at the proxy. - """ - return self.__proxysockname - - def getproxypeername(self): - """getproxypeername() -> address info - Returns the IP and port number of the proxy. - """ - return _orgsocket.getpeername(self) - - def getpeername(self): - """getpeername() -> address info - Returns the IP address and port number of the destination - machine (note: getproxypeername returns the proxy) - """ - return self.__proxypeername - - def __negotiatesocks4(self, destaddr, destport): - """__negotiatesocks4(self,destaddr,destport) - Negotiates a connection through a SOCKS4 server. - """ - # Check if the destination address provided is an IP address - rmtrslv = False - try: - ipaddr = socket.inet_aton(destaddr) - except socket.error: - # It's a DNS name. Check where it should be resolved. - if self.__proxy[3]: - ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01) - rmtrslv = True - else: - ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) - # Construct the request packet - req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr - # The username parameter is considered userid for SOCKS4 - if self.__proxy[4] != None: - req = req + self.__proxy[4] - req = req + chr(0x00).encode() - # DNS name if remote resolving is required - # NOTE: This is actually an extension to the SOCKS4 protocol - # called SOCKS4A and may not be supported in all cases. - if rmtrslv: - req = req + destaddr + chr(0x00).encode() - self.sendall(req) - # Get the response from the server - resp = self.__recvall(8) - if resp[0:1] != chr(0x00).encode(): - # Bad data - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - if resp[1:2] != chr(0x5A).encode(): - # Server returned an error - self.close() - if ord(resp[1:2]) in (91, 92, 93): - self.close() - raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90])) - else: - raise Socks4Error((94, _socks4errors[4])) - # Get the bound address/port - self.__proxysockname = ( - socket.inet_ntoa(resp[4:]), - struct.unpack(">H", resp[2:4])[0], - ) - if rmtrslv != None: - self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) - else: - self.__proxypeername = (destaddr, destport) - - def __negotiatehttp(self, destaddr, destport): - """__negotiatehttp(self,destaddr,destport) - Negotiates a connection through an HTTP server. - """ - # If we need to resolve locally, we do this now - if not self.__proxy[3]: - addr = socket.gethostbyname(destaddr) - else: - addr = destaddr - headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"] - wrote_host_header = False - wrote_auth_header = False - if self.__proxy[6] != None: - for key, val in self.__proxy[6].iteritems(): - headers += [key, ": ", val, "\r\n"] - wrote_host_header = key.lower() == "host" - wrote_auth_header = key.lower() == "proxy-authorization" - if not wrote_host_header: - headers += ["Host: ", destaddr, "\r\n"] - if not wrote_auth_header: - if self.__proxy[4] != None and self.__proxy[5] != None: - headers += [self.__getauthheader(), "\r\n"] - headers.append("\r\n") - self.sendall("".join(headers).encode()) - # We read the response until we get the string "\r\n\r\n" - resp = self.recv(1) - while resp.find("\r\n\r\n".encode()) == -1: - resp = resp + self.recv(1) - # We just need the first line to check if the connection - # was successful - statusline = resp.splitlines()[0].split(" ".encode(), 2) - if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()): - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - try: - statuscode = int(statusline[1]) - except ValueError: - self.close() - raise GeneralProxyError((1, _generalerrors[1])) - if statuscode != 200: - self.close() - raise HTTPError((statuscode, statusline[2])) - self.__proxysockname = ("0.0.0.0", 0) - self.__proxypeername = (addr, destport) - - def connect(self, destpair): - """connect(self, despair) - Connects to the specified destination through a proxy. - destpar - A tuple of the IP/DNS address and the port number. - (identical to socket's connect). - To select the proxy server use setproxy(). - """ - # Do a minimal input check first - if ( - (not type(destpair) in (list, tuple)) - or (len(destpair) < 2) - or (not isinstance(destpair[0], basestring)) - or (type(destpair[1]) != int) - ): - raise GeneralProxyError((5, _generalerrors[5])) - if self.__proxy[0] == PROXY_TYPE_SOCKS5: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 1080 - _orgsocket.connect(self, (self.__proxy[1], portnum)) - self.__negotiatesocks5(destpair[0], destpair[1]) - elif self.__proxy[0] == PROXY_TYPE_SOCKS4: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 1080 - _orgsocket.connect(self, (self.__proxy[1], portnum)) - self.__negotiatesocks4(destpair[0], destpair[1]) - elif self.__proxy[0] == PROXY_TYPE_HTTP: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 8080 - _orgsocket.connect(self, (self.__proxy[1], portnum)) - self.__negotiatehttp(destpair[0], destpair[1]) - elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 8080 - _orgsocket.connect(self, (self.__proxy[1], portnum)) - if destpair[1] == 443: - self.__negotiatehttp(destpair[0], destpair[1]) - else: - self.__httptunnel = False - elif self.__proxy[0] == None: - _orgsocket.connect(self, (destpair[0], destpair[1])) - else: - raise GeneralProxyError((4, _generalerrors[4])) diff --git a/src/httplib2/test/__init__.py b/src/httplib2/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/httplib2/test/brokensocket/socket.py b/src/httplib2/test/brokensocket/socket.py deleted file mode 100644 index ff7c0b74..00000000 --- a/src/httplib2/test/brokensocket/socket.py +++ /dev/null @@ -1 +0,0 @@ -from realsocket import gaierror, error, getaddrinfo, SOCK_STREAM diff --git a/src/httplib2/test/functional/test_proxies.py b/src/httplib2/test/functional/test_proxies.py deleted file mode 100644 index 939140d4..00000000 --- a/src/httplib2/test/functional/test_proxies.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import print_function -import unittest -import errno -import os -import signal -import subprocess -import tempfile - -import nose - -import httplib2 -from httplib2 import socks -from httplib2.test import miniserver - -tinyproxy_cfg = """ -User "%(user)s" -Port %(port)s -Listen 127.0.0.1 -PidFile "%(pidfile)s" -LogFile "%(logfile)s" -MaxClients 2 -StartServers 1 -LogLevel Info -""" - - -class FunctionalProxyHttpTest(unittest.TestCase): - def setUp(self): - if not socks: - raise nose.SkipTest("socks module unavailable") - if not subprocess: - raise nose.SkipTest("subprocess module unavailable") - - # start a short-lived miniserver so we can get a likely port - # for the proxy - self.httpd, self.proxyport = miniserver.start_server(miniserver.ThisDirHandler) - self.httpd.shutdown() - self.httpd, self.port = miniserver.start_server(miniserver.ThisDirHandler) - - self.pidfile = tempfile.mktemp() - self.logfile = tempfile.mktemp() - fd, self.conffile = tempfile.mkstemp() - f = os.fdopen(fd, "w") - our_cfg = tinyproxy_cfg % { - "user": os.getlogin(), - "pidfile": self.pidfile, - "port": self.proxyport, - "logfile": self.logfile, - } - f.write(our_cfg) - f.close() - try: - # TODO use subprocess.check_call when 2.4 is dropped - ret = subprocess.call(["tinyproxy", "-c", self.conffile]) - self.assertEqual(0, ret) - except OSError as e: - if e.errno == errno.ENOENT: - raise nose.SkipTest("tinyproxy not available") - raise - - def tearDown(self): - self.httpd.shutdown() - try: - pid = int(open(self.pidfile).read()) - os.kill(pid, signal.SIGTERM) - except OSError as e: - if e.errno == errno.ESRCH: - print("\n\n\nTinyProxy Failed to start, log follows:") - print(open(self.logfile).read()) - print("end tinyproxy log\n\n\n") - raise - map(os.unlink, (self.pidfile, self.logfile, self.conffile)) - - def testSimpleProxy(self): - proxy_info = httplib2.ProxyInfo( - socks.PROXY_TYPE_HTTP, "localhost", self.proxyport - ) - client = httplib2.Http(proxy_info=proxy_info) - src = "miniserver.py" - response, body = client.request("http://localhost:%d/%s" % (self.port, src)) - self.assertEqual(response.status, 200) - self.assertEqual(body, open(os.path.join(miniserver.HERE, src)).read()) - lf = open(self.logfile).read() - expect = 'Established connection to host "127.0.0.1" ' "using file descriptor" - self.assertTrue( - expect in lf, "tinyproxy did not proxy a request for miniserver" - ) diff --git a/src/httplib2/test/miniserver.py b/src/httplib2/test/miniserver.py deleted file mode 100644 index 47c3ee5b..00000000 --- a/src/httplib2/test/miniserver.py +++ /dev/null @@ -1,114 +0,0 @@ -import logging -import os -import select -import SimpleHTTPServer -import socket -import SocketServer -import threading - -HERE = os.path.dirname(__file__) -logger = logging.getLogger(__name__) - - -class ThisDirHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - def translate_path(self, path): - path = path.split("?", 1)[0].split("#", 1)[0] - return os.path.join(HERE, *filter(None, path.split("/"))) - - def log_message(self, s, *args): - # output via logging so nose can catch it - logger.info(s, *args) - - -class ShutdownServer(SocketServer.TCPServer): - """Mixin that allows serve_forever to be shut down. - - The methods in this mixin are backported from SocketServer.py in the Python - 2.6.4 standard library. The mixin is unnecessary in 2.6 and later, when - BaseServer supports the shutdown method directly. - """ - - def __init__(self, use_tls, *args, **kwargs): - self.__use_tls = use_tls - SocketServer.TCPServer.__init__(self, *args, **kwargs) - self.__is_shut_down = threading.Event() - self.__serving = False - - def server_bind(self): - SocketServer.TCPServer.server_bind(self) - if self.__use_tls: - import ssl - - self.socket = ssl.wrap_socket( - self.socket, - os.path.join(os.path.dirname(__file__), "server.key"), - os.path.join(os.path.dirname(__file__), "server.pem"), - True, - ) - - def serve_forever(self, poll_interval=0.1): - """Handle one request at a time until shutdown. - - Polls for shutdown every poll_interval seconds. Ignores - self.timeout. If you need to do periodic tasks, do them in - another thread. - """ - self.__serving = True - self.__is_shut_down.clear() - while self.__serving: - r, w, e = select.select([self.socket], [], [], poll_interval) - if r: - self._handle_request_noblock() - self.__is_shut_down.set() - - def shutdown(self): - """Stops the serve_forever loop. - - Blocks until the loop has finished. This must be called while - serve_forever() is running in another thread, or it will deadlock. - """ - self.__serving = False - self.__is_shut_down.wait() - - def handle_request(self): - """Handle one request, possibly blocking. - - Respects self.timeout. - """ - # Support people who used socket.settimeout() to escape - # handle_request before self.timeout was available. - timeout = self.socket.gettimeout() - if timeout is None: - timeout = self.timeout - elif self.timeout is not None: - timeout = min(timeout, self.timeout) - fd_sets = select.select([self], [], [], timeout) - if not fd_sets[0]: - self.handle_timeout() - return - self._handle_request_noblock() - - def _handle_request_noblock(self): - """Handle one request, without blocking. - - I assume that select.select has returned that the socket is - readable before this function was called, so there should be - no risk of blocking in get_request(). - """ - try: - request, client_address = self.get_request() - except socket.error: - return - if self.verify_request(request, client_address): - try: - self.process_request(request, client_address) - except: - self.handle_error(request, client_address) - self.close_request(request) - - -def start_server(handler, use_tls=False): - httpd = ShutdownServer(use_tls, ("", 0), handler) - threading.Thread(target=httpd.serve_forever).start() - _, port = httpd.socket.getsockname() - return httpd, port diff --git a/src/httplib2/test/other_cacerts.txt b/src/httplib2/test/other_cacerts.txt deleted file mode 100644 index b2c24884..00000000 --- a/src/httplib2/test/other_cacerts.txt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBzCCAe+gAwIBAgIJAIw94zvO7fk1MA0GCSqGSIb3DQEBBQUAMBoxGDAWBgNV -BAMMD3d3dy5leGFtcGxlLmNvbTAeFw0xNjA2MDQwMjMxMTRaFw0yNjA2MDIwMjMx -MTRaMBoxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK3YNcDIwK/wlTa0/iBARvDFOncQ6Jkk+Ymql1HXny7v -mWPFWeLXEW+Zw1NrQEx/SIUGvxpRA+QyhTOhu2Gcwvtqilix/dHgaKgqWEcRYu8m -L70uVDPVgB/kfNI8bpXM1Mz8Crjo0tHw5oUSD3wny8SyT6CYlXVmF923L8c2zdN9 -n9blFgYwxBq2+q+mqOiDErMFbwHES8FNBSWGBXdE1xjBdITtlfeHezmJhj/ylPW1 -7v8HInsv/WqU9DcJYlFxSnK0SZCLFBM/31Ez8O1gCfMlDUFvJoo59GyFqukUjuO1 -uB85wpu27gtcLm/J9X1Md71IxbDupV7a0dDoTvbhO4kCAwEAAaNQME4wHQYDVR0O -BBYEFIHgAmwppZSKLz2peyFSO2kwVobNMB8GA1UdIwQYMBaAFIHgAmwppZSKLz2p -eyFSO2kwVobNMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAJxz+AU/ -Iq8fMEStJ0BgPP1N86W9Jpb7aPMFCYTEZ+nd8hFPhPs4//55J0yIve+1I43MNFFz -yflwwCzrIIhZdkvbsyea6CmlTo4jBc4+ihaDGobYnoNzFhavC47n5kYqJ8Ikyb2W -OMrmNRiaTeSBl0wQmftnnQCbonenjmE1LDuJtE6bCwfFjfLbMxwdWtp/ymOlXsb5 -80XcWwcqc12UHWexYwHFzEJmDfncak/8tjHBsLWMJg5p2sVTY9kVt7TYgSIl+mFb -4WVGrqZd2uTlJkRQQ4pCl+D+PKwadHuV6YI7oxkeajjcHCgbK/ANwW28MXYho6t6 -aWVIN4bWHrZ38kE= ------END CERTIFICATE----- diff --git a/src/httplib2/test/server.key b/src/httplib2/test/server.key deleted file mode 100644 index 4c21facf..00000000 --- a/src/httplib2/test/server.key +++ /dev/null @@ -1,146 +0,0 @@ -Private TLS server key file used for HTTPS-related unit tests -------------------------------------------------------------- - -Public Key Info: - Public Key Algorithm: RSA - Key Security Level: Medium (2048 bits) - -modulus: - 00:cc:fb:f1:c5:de:29:29:40:3f:c4:9f:af:da:f6:be - 27:f4:6a:00:ae:5a:f2:99:c3:5f:7a:e6:9b:cf:d9:08 - 34:01:9b:ea:fb:da:b5:d0:b5:b2:4e:60:b4:0d:8d:05 - 57:e4:2e:04:d4:57:1a:58:3c:0b:3a:ed:67:a3:13:31 - 58:0a:c2:eb:fd:d6:27:ee:07:95:30:35:b5:98:91:c7 - a5:9b:be:a9:7e:ae:fd:73:c3:6b:21:bc:52:f8:ef:71 - db:d3:b1:cd:51:df:b3:37:b3:fd:7d:ae:7e:02:38:be - 8e:6f:45:55:e5:6d:8a:02:cb:36:c4:17:7a:ea:24:9a - 72:8d:1e:75:03:3a:6f:c4:cb:a0:3a:50:56:32:bb:4c - e2:ea:74:f0:96:31:74:b2:c1:03:e8:c3:d4:a3:59:fc - 7a:cc:68:35:c4:97:eb:aa:46:fa:64:c3:f9:55:59:22 - b5:2b:3c:96:84:c6:d2:7d:b4:9f:b9:9c:af:d1:20:30 - 7c:e8:60:4e:ee:0a:60:a0:9d:4e:8a:d8:34:74:bd:f2 - 40:bc:d7:c2:b3:1a:b2:bb:d7:a5:4a:4c:65:94:43:82 - 16:9a:8f:76:2a:05:b0:9e:3d:a7:fb:e2:c7:78:25:f7 - df:ca:08:ee:ec:4f:cd:1a:3c:03:41:ec:91:c5:50:70 - 4b: - -public exponent: - 01:00:01: - -private exponent: - 00:ad:01:83:b8:7d:dd:fd:ab:f5:66:2d:64:ce:08:ec - cb:6a:15:41:87:e6:c8:d5:10:39:78:d0:43:f7:73:f4 - e1:77:ee:31:b0:e9:92:04:9a:25:e8:d2:e3:84:80:5e - 5f:24:fd:d6:23:a5:74:5d:be:27:b8:4f:80:e5:f9:1f - ef:6f:fd:be:12:1a:7a:cf:02:65:5f:30:25:99:a4:88 - 7d:74:ea:c1:c1:63:4e:15:33:7d:2b:16:f8:6c:94:23 - 63:e6:d3:2d:38:89:f6:87:f0:08:e5:d7:ad:10:90:f5 - fb:df:5c:04:b8:43:f0:74:95:31:1e:e5:b6:5f:02:0f - bb:55:cb:e1:b5:48:9f:1f:d3:1b:55:a7:bc:39:2b:8e - 6d:14:64:3b:bf:e8:ca:6b:af:a9:f3:13:9a:c6:df:15 - ef:6d:17:4e:8e:67:6c:41:20:dc:6b:08:0d:b9:14:cd - 83:10:62:15:e6:b0:89:5d:37:fb:f6:fd:f0:bf:3b:9c - 0b:e9:fd:b8:de:e4:64:90:bf:81:d5:59:2c:30:43:07 - b9:60:8c:d0:ac:4f:95:87:aa:38:62:bd:c7:06:a7:c4 - 2d:08:c1:3c:86:10:c7:8e:1e:df:58:bf:95:ad:39:84 - a0:2b:13:e2:18:e6:4a:80:f0:bc:04:50:bd:7d:cf:23 - a1: - -prime1: - 00:f0:8f:ad:2f:c9:64:f3:0d:2c:aa:06:17:05:8f:2f - d5:cb:92:22:90:05:66:3c:78:75:9d:7b:4c:6a:af:a9 - 1e:d6:28:4f:13:0e:3a:e7:31:49:3d:87:ef:2c:17:70 - be:69:b3:42:82:6d:9c:b4:13:0a:e4:bc:8c:0f:1a:bd - 04:b6:a0:be:ba:12:15:bf:04:db:91:1c:26:91:d6:d7 - f2:ff:2f:0e:5f:96:a1:7c:4b:90:a8:2f:07:2a:cb:dc - 40:a0:0b:1d:2a:1d:48:98:bd:4a:6b:9d:5c:69:b0:2b - 6e:9b:2c:b2:a9:cb:28:fe:fa:7f:93:eb:20:c8:59:d0 - 11: - -prime2: - 00:da:23:c0:3e:82:4c:88:7c:d4:fb:de:24:45:eb:9c - ae:2c:80:2d:52:a6:95:05:33:b9:d8:c1:7b:52:01:62 - 11:e6:b6:c6:0d:56:a3:68:39:26:9a:90:08:95:12:a9 - 1c:59:f6:0b:1d:af:6d:c0:c6:9b:2e:7a:62:98:21:36 - e1:15:4c:e6:6d:a4:08:ac:90:af:57:86:71:78:2e:0e - cf:59:0f:35:79:cb:6a:a2:e2:30:2a:a8:f2:84:68:bc - 8a:f2:48:3b:07:d5:a5:34:f3:d3:ec:25:61:38:f1:0a - 07:f7:7e:29:61:e4:15:01:80:e3:7b:bd:63:9c:2e:16 - 9b: - -coefficient: - 00:cb:b4:d2:9f:b4:04:db:8c:54:e6:ae:a9:28:a0:c9 - 70:ad:7a:94:72:5e:86:33:91:d9:43:61:2b:4d:55:e8 - b7:25:d2:cd:db:1e:c4:56:95:68:85:e2:9b:4f:31:24 - 3a:40:06:41:1c:aa:7a:31:13:fa:07:e0:a6:59:c3:d1 - d2:c5:2c:6a:82:98:bb:a1:59:c0:6f:ad:d7:2e:ed:5a - 64:5f:e6:ea:4a:ee:45:29:d9:0f:96:b3:39:f7:ab:57 - 97:aa:c9:f7:b6:9c:c0:51:5d:9f:01:2c:ec:58:8d:06 - 6a:19:d0:33:74:11:6a:25:7c:8f:b7:31:d2:97:05:02 - 6f: - -exp1: - 00:87:60:43:95:1d:e0:0a:8b:82:74:18:43:42:64:a7 - 05:c8:ae:ef:76:5f:23:7e:aa:47:7e:1d:52:0e:c3:d6 - 07:bd:7b:27:ac:d0:98:43:5c:d0:1b:a9:70:e6:3e:36 - bb:61:5e:78:f2:4f:5f:1d:53:8e:10:d5:2e:78:9d:92 - 7b:a1:8e:ea:66:6a:21:04:c3:66:10:ce:67:c2:30:c6 - 8c:40:21:2a:14:8e:ff:47:a4:7a:be:ba:e0:6c:ac:16 - c1:e3:8e:fd:95:a2:af:25:0d:79:61:00:48:6e:4d:ae - d3:6a:ce:07:a9:57:e4:35:41:a1:24:0b:f1:01:ee:d1 - 11: - -exp2: - 00:ca:ca:bd:a7:de:fe:43:4c:b9:bb:c4:d2:37:e6:47 - ec:6c:16:65:0c:17:2d:26:7e:e5:e1:2a:4d:f8:f8:ac - 31:34:28:ea:89:ef:e7:4d:b7:03:ba:60:f8:79:8d:b5 - 85:53:e4:b6:84:cc:57:de:05:44:b2:ba:b7:f9:f1:b6 - d1:1d:3a:36:65:eb:3e:dd:1e:4c:c3:b3:8a:bd:4d:24 - 1b:83:11:ee:86:e1:a2:aa:f6:58:0c:f0:af:34:85:21 - f2:92:36:b0:1a:22:75:c9:7a:7b:a3:67:44:b0:e8:f4 - 88:5f:7e:fb:fd:b3:4a:0b:f1:c4:89:7e:91:a1:d9:fe - cd: - - -Public Key ID: 92:D5:B4:2A:B6:A8:64:67:2C:2A:08:DB:51:B8:97:86:5E:44:CD:6C -Public key's random art: -+--[ RSA 2048]----+ -| + . | -| . E o . | -| o . . o | -| . o o . | -| = .= S | -|. +.=o + | -|o++== . | -|++o= | -|o . | -+-----------------+ - - ------BEGIN RSA PRIVATE KEY----- -MIIEpgIBAAKCAQEAzPvxxd4pKUA/xJ+v2va+J/RqAK5a8pnDX3rmm8/ZCDQBm+r7 -2rXQtbJOYLQNjQVX5C4E1FcaWDwLOu1noxMxWArC6/3WJ+4HlTA1tZiRx6Wbvql+ -rv1zw2shvFL473Hb07HNUd+zN7P9fa5+Aji+jm9FVeVtigLLNsQXeuokmnKNHnUD -Om/Ey6A6UFYyu0zi6nTwljF0ssED6MPUo1n8esxoNcSX66pG+mTD+VVZIrUrPJaE -xtJ9tJ+5nK/RIDB86GBO7gpgoJ1Oitg0dL3yQLzXwrMasrvXpUpMZZRDghaaj3Yq -BbCePaf74sd4Jfffygju7E/NGjwDQeyRxVBwSwIDAQABAoIBAQCtAYO4fd39q/Vm -LWTOCOzLahVBh+bI1RA5eNBD93P04XfuMbDpkgSaJejS44SAXl8k/dYjpXRdvie4 -T4Dl+R/vb/2+Ehp6zwJlXzAlmaSIfXTqwcFjThUzfSsW+GyUI2Pm0y04ifaH8Ajl -160QkPX731wEuEPwdJUxHuW2XwIPu1XL4bVInx/TG1WnvDkrjm0UZDu/6Mprr6nz -E5rG3xXvbRdOjmdsQSDcawgNuRTNgxBiFeawiV03+/b98L87nAvp/bje5GSQv4HV -WSwwQwe5YIzQrE+Vh6o4Yr3HBqfELQjBPIYQx44e31i/la05hKArE+IY5kqA8LwE -UL19zyOhAoGBAPCPrS/JZPMNLKoGFwWPL9XLkiKQBWY8eHWde0xqr6ke1ihPEw46 -5zFJPYfvLBdwvmmzQoJtnLQTCuS8jA8avQS2oL66EhW/BNuRHCaR1tfy/y8OX5ah -fEuQqC8HKsvcQKALHSodSJi9SmudXGmwK26bLLKpyyj++n+T6yDIWdARAoGBANoj -wD6CTIh81PveJEXrnK4sgC1SppUFM7nYwXtSAWIR5rbGDVajaDkmmpAIlRKpHFn2 -Cx2vbcDGmy56YpghNuEVTOZtpAiskK9XhnF4Lg7PWQ81ectqouIwKqjyhGi8ivJI -OwfVpTTz0+wlYTjxCgf3filh5BUBgON7vWOcLhabAoGBAIdgQ5Ud4AqLgnQYQ0Jk -pwXIru92XyN+qkd+HVIOw9YHvXsnrNCYQ1zQG6lw5j42u2FeePJPXx1TjhDVLnid -knuhjupmaiEEw2YQzmfCMMaMQCEqFI7/R6R6vrrgbKwWweOO/ZWiryUNeWEASG5N -rtNqzgepV+Q1QaEkC/EB7tERAoGBAMrKvafe/kNMubvE0jfmR+xsFmUMFy0mfuXh -Kk34+KwxNCjqie/nTbcDumD4eY21hVPktoTMV94FRLK6t/nxttEdOjZl6z7dHkzD -s4q9TSQbgxHuhuGiqvZYDPCvNIUh8pI2sBoidcl6e6NnRLDo9Ihffvv9s0oL8cSJ -fpGh2f7NAoGBAMu00p+0BNuMVOauqSigyXCtepRyXoYzkdlDYStNVei3JdLN2x7E -VpVoheKbTzEkOkAGQRyqejET+gfgplnD0dLFLGqCmLuhWcBvrdcu7VpkX+bqSu5F -KdkPlrM596tXl6rJ97acwFFdnwEs7FiNBmoZ0DN0EWolfI+3MdKXBQJv ------END RSA PRIVATE KEY----- diff --git a/src/httplib2/test/server.pem b/src/httplib2/test/server.pem deleted file mode 100644 index 49b6efc8..00000000 --- a/src/httplib2/test/server.pem +++ /dev/null @@ -1,50 +0,0 @@ -Public, self-signed TLS server key file used for HTTPS-related unit tests -------------------------------------------------------------------------- - -Public Key Information: - Public Key Algorithm: RSA - Algorithm Security Level: Medium (2048 bits) - Modulus (bits 2048): - 00:cc:fb:f1:c5:de:29:29:40:3f:c4:9f:af:da:f6:be - 27:f4:6a:00:ae:5a:f2:99:c3:5f:7a:e6:9b:cf:d9:08 - 34:01:9b:ea:fb:da:b5:d0:b5:b2:4e:60:b4:0d:8d:05 - 57:e4:2e:04:d4:57:1a:58:3c:0b:3a:ed:67:a3:13:31 - 58:0a:c2:eb:fd:d6:27:ee:07:95:30:35:b5:98:91:c7 - a5:9b:be:a9:7e:ae:fd:73:c3:6b:21:bc:52:f8:ef:71 - db:d3:b1:cd:51:df:b3:37:b3:fd:7d:ae:7e:02:38:be - 8e:6f:45:55:e5:6d:8a:02:cb:36:c4:17:7a:ea:24:9a - 72:8d:1e:75:03:3a:6f:c4:cb:a0:3a:50:56:32:bb:4c - e2:ea:74:f0:96:31:74:b2:c1:03:e8:c3:d4:a3:59:fc - 7a:cc:68:35:c4:97:eb:aa:46:fa:64:c3:f9:55:59:22 - b5:2b:3c:96:84:c6:d2:7d:b4:9f:b9:9c:af:d1:20:30 - 7c:e8:60:4e:ee:0a:60:a0:9d:4e:8a:d8:34:74:bd:f2 - 40:bc:d7:c2:b3:1a:b2:bb:d7:a5:4a:4c:65:94:43:82 - 16:9a:8f:76:2a:05:b0:9e:3d:a7:fb:e2:c7:78:25:f7 - df:ca:08:ee:ec:4f:cd:1a:3c:03:41:ec:91:c5:50:70 - 4b - Exponent (bits 24): - 01:00:01 - -Public Key Usage: - -Public Key ID: 92d5b42ab6a864672c2a08db51b897865e44cd6c - - ------BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIJAISbkoXpX75CMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAeFw0xNjA2MTcxNDA1NTlaFw0yNjA2MTUxNDA1NTlaMBQx -EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAMz78cXeKSlAP8Sfr9r2vif0agCuWvKZw1965pvP2Qg0AZvq+9q10LWyTmC0 -DY0FV+QuBNRXGlg8CzrtZ6MTMVgKwuv91ifuB5UwNbWYkcelm76pfq79c8NrIbxS -+O9x29OxzVHfszez/X2ufgI4vo5vRVXlbYoCyzbEF3rqJJpyjR51AzpvxMugOlBW -MrtM4up08JYxdLLBA+jD1KNZ/HrMaDXEl+uqRvpkw/lVWSK1KzyWhMbSfbSfuZyv -0SAwfOhgTu4KYKCdTorYNHS98kC818KzGrK716VKTGWUQ4IWmo92KgWwnj2n++LH -eCX338oI7uxPzRo8A0HskcVQcEsCAwEAAaNQME4wHQYDVR0OBBYEFFdXD8Z8k0et -ZNyM4e4WypNnGlcCMB8GA1UdIwQYMBaAFFdXD8Z8k0etZNyM4e4WypNnGlcCMAwG -A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAC4FsnK1Ph/JpdoqSRTCJiVM -MPFaaavKEEyYdAKPk/Acmb9vf07sqsT+OZg/0obsZG9LxJb7x0iAnhfM3aS+CmO9 -Ym2lXeFDaJ2bHooB9MsG2C3+n8lJUMwxm7Cqpff/lpCK6Z+6MGPx3GRs6HUEl34k -BB5pue2vqhtFQ03UdHMpAK0M7n3TloAWbFb1a/JmqzTbsQ0oaMHGoECQEAbaBl+a -/up6vA3iZHq+ZPYS1KIx+xuT/SapLcyUtjfhmq1bROVZP4+6EHMsBMnhJBYKxxHy -0qKvqJL9X3NQLMgMKKUKzX+BuG2u5aRRyVIqewT/ORjaUr9Y8lU7WlXPf7Ljm6s= ------END CERTIFICATE----- diff --git a/src/httplib2/test/smoke_test.py b/src/httplib2/test/smoke_test.py deleted file mode 100644 index 25e9cf2e..00000000 --- a/src/httplib2/test/smoke_test.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import unittest - -import httplib2 - -from httplib2.test import miniserver - - -class HttpSmokeTest(unittest.TestCase): - def setUp(self): - self.httpd, self.port = miniserver.start_server(miniserver.ThisDirHandler) - - def tearDown(self): - self.httpd.shutdown() - - def testGetFile(self): - client = httplib2.Http() - src = "miniserver.py" - response, body = client.request("http://localhost:%d/%s" % (self.port, src)) - self.assertEqual(response.status, 200) - self.assertEqual(body, open(os.path.join(miniserver.HERE, src)).read()) diff --git a/src/httplib2/test/test_no_socket.py b/src/httplib2/test/test_no_socket.py deleted file mode 100644 index d251cbc3..00000000 --- a/src/httplib2/test/test_no_socket.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Tests for httplib2 when the socket module is missing. - -This helps ensure compatibility with environments such as AppEngine. -""" -import os -import sys -import unittest - -import httplib2 - - -class MissingSocketTest(unittest.TestCase): - def setUp(self): - self._oldsocks = httplib2.socks - httplib2.socks = None - - def tearDown(self): - httplib2.socks = self._oldsocks - - def testProxyDisabled(self): - proxy_info = httplib2.ProxyInfo("blah", "localhost", 0) - client = httplib2.Http(proxy_info=proxy_info) - self.assertRaises( - httplib2.ProxiesUnavailableError, client.request, "http://localhost:-1/" - ) diff --git a/src/httplib2/test/test_ssl_context.py b/src/httplib2/test/test_ssl_context.py deleted file mode 100644 index 43504dc7..00000000 --- a/src/httplib2/test/test_ssl_context.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python2 -from __future__ import print_function -import BaseHTTPServer -import logging -import os.path -import ssl -import sys -import unittest - -import httplib2 -from httplib2.test import miniserver - -logger = logging.getLogger(__name__) - - -class KeepAliveHandler(BaseHTTPServer.BaseHTTPRequestHandler): - """Request handler that keeps the HTTP connection open, so that the test can inspect the resulting SSL connection object - - """ - - def do_GET(self): - self.send_response(200) - self.send_header("Content-Length", "0") - self.send_header("Connection", "keep-alive") - self.end_headers() - - self.close_connection = 0 - - def log_message(self, s, *args): - # output via logging so nose can catch it - logger.info(s, *args) - - -class HttpsContextTest(unittest.TestCase): - def setUp(self): - if sys.version_info < (2, 7, 9): - if hasattr(self, "skipTest"): - self.skipTest("SSLContext requires Python 2.7.9") - else: - return - - self.ca_certs_path = os.path.join(os.path.dirname(__file__), "server.pem") - self.httpd, self.port = miniserver.start_server(KeepAliveHandler, True) - - def tearDown(self): - self.httpd.shutdown() - - def testHttpsContext(self): - client = httplib2.Http(ca_certs=self.ca_certs_path) - - # Establish connection to local server - client.request("https://localhost:%d/" % (self.port)) - - # Verify that connection uses a TLS context with the correct hostname - conn = client.connections["https:localhost:%d" % self.port] - - self.assertIsInstance(conn.sock, ssl.SSLSocket) - self.assertTrue(hasattr(conn.sock, "context")) - self.assertIsInstance(conn.sock.context, ssl.SSLContext) - self.assertTrue(conn.sock.context.check_hostname) - self.assertEqual(conn.sock.server_hostname, "localhost") - self.assertEqual(conn.sock.context.verify_mode, ssl.CERT_REQUIRED) - self.assertEqual(conn.sock.context.protocol, ssl.PROTOCOL_SSLv23) - - def test_ssl_hostname_mismatch_repeat(self): - # https://github.com/httplib2/httplib2/issues/5 - - # FIXME(temoto): as of 2017-01-05 this is only a reference code, not useful test. - # Because it doesn't provoke described error on my machine. - # Instead `SSLContext.wrap_socket` raises `ssl.CertificateError` - # which was also added to original patch. - - # url host is intentionally different, we provoke ssl hostname mismatch error - url = "https://127.0.0.1:%d/" % (self.port,) - http = httplib2.Http(ca_certs=self.ca_certs_path, proxy_info=None) - - def once(): - try: - http.request(url) - assert False, "expected certificate hostname mismatch error" - except Exception as e: - print("%s errno=%s" % (repr(e), getattr(e, "errno", None))) - - once() - once() diff --git a/src/linux-gam.spec b/src/linux-gam.spec index 2b9252ce..135e7e70 100644 --- a/src/linux-gam.spec +++ b/src/linux-gam.spec @@ -8,7 +8,7 @@ for d in a.datas: if 'pyconfig' in d[0]: a.datas.remove(d) break -a.datas += [('httplib2/cacerts.txt', 'httplib2/cacerts.txt', 'DATA')] +a.datas += [('httplib2/cacerts.txt', 'cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, diff --git a/src/macos-gam.spec b/src/macos-gam.spec index 2b9252ce..135e7e70 100644 --- a/src/macos-gam.spec +++ b/src/macos-gam.spec @@ -8,7 +8,7 @@ for d in a.datas: if 'pyconfig' in d[0]: a.datas.remove(d) break -a.datas += [('httplib2/cacerts.txt', 'httplib2/cacerts.txt', 'DATA')] +a.datas += [('httplib2/cacerts.txt', 'cacerts.txt', 'DATA')] a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')] pyz = PYZ(a.pure) exe = EXE(pyz, diff --git a/src/oauth2client/__init__.py b/src/oauth2client/__init__.py deleted file mode 100644 index ff2790f8..00000000 --- a/src/oauth2client/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Client library for using OAuth2, especially with Google APIs.""" - -__version__ = '4.1.3' - -GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/v2/auth' -GOOGLE_DEVICE_URI = 'https://oauth2.googleapis.com/device/code' -GOOGLE_REVOKE_URI = 'https://accounts.google.com/o/oauth2/revoke' -GOOGLE_TOKEN_URI = 'https://oauth2.googleapis.com/token' -GOOGLE_TOKEN_INFO_URI = 'https://oauth2.googleapis.com/tokeninfo' - diff --git a/src/oauth2client/_helpers.py b/src/oauth2client/_helpers.py deleted file mode 100644 index e9123971..00000000 --- a/src/oauth2client/_helpers.py +++ /dev/null @@ -1,341 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper functions for commonly used utilities.""" - -import base64 -import functools -import inspect -import json -import logging -import os -import warnings - -import six -from six.moves import urllib - - -logger = logging.getLogger(__name__) - -POSITIONAL_WARNING = 'WARNING' -POSITIONAL_EXCEPTION = 'EXCEPTION' -POSITIONAL_IGNORE = 'IGNORE' -POSITIONAL_SET = frozenset([POSITIONAL_WARNING, POSITIONAL_EXCEPTION, - POSITIONAL_IGNORE]) - -positional_parameters_enforcement = POSITIONAL_WARNING - -_SYM_LINK_MESSAGE = 'File: {0}: Is a symbolic link.' -_IS_DIR_MESSAGE = '{0}: Is a directory' -_MISSING_FILE_MESSAGE = 'Cannot access {0}: No such file or directory' - - -def positional(max_positional_args): - """A decorator to declare that only the first N arguments my be positional. - - This decorator makes it easy to support Python 3 style keyword-only - parameters. For example, in Python 3 it is possible to write:: - - def fn(pos1, *, kwonly1=None, kwonly1=None): - ... - - All named parameters after ``*`` must be a keyword:: - - fn(10, 'kw1', 'kw2') # Raises exception. - fn(10, kwonly1='kw1') # Ok. - - Example - ^^^^^^^ - - To define a function like above, do:: - - @positional(1) - def fn(pos1, kwonly1=None, kwonly2=None): - ... - - If no default value is provided to a keyword argument, it becomes a - required keyword argument:: - - @positional(0) - def fn(required_kw): - ... - - This must be called with the keyword parameter:: - - fn() # Raises exception. - fn(10) # Raises exception. - fn(required_kw=10) # Ok. - - When defining instance or class methods always remember to account for - ``self`` and ``cls``:: - - class MyClass(object): - - @positional(2) - def my_method(self, pos1, kwonly1=None): - ... - - @classmethod - @positional(2) - def my_method(cls, pos1, kwonly1=None): - ... - - The positional decorator behavior is controlled by - ``_helpers.positional_parameters_enforcement``, which may be set to - ``POSITIONAL_EXCEPTION``, ``POSITIONAL_WARNING`` or - ``POSITIONAL_IGNORE`` to raise an exception, log a warning, or do - nothing, respectively, if a declaration is violated. - - Args: - max_positional_arguments: Maximum number of positional arguments. All - parameters after the this index must be - keyword only. - - Returns: - A decorator that prevents using arguments after max_positional_args - from being used as positional parameters. - - Raises: - TypeError: if a key-word only argument is provided as a positional - parameter, but only if - _helpers.positional_parameters_enforcement is set to - POSITIONAL_EXCEPTION. - """ - - def positional_decorator(wrapped): - @functools.wraps(wrapped) - def positional_wrapper(*args, **kwargs): - if len(args) > max_positional_args: - plural_s = '' - if max_positional_args != 1: - plural_s = 's' - message = ('{function}() takes at most {args_max} positional ' - 'argument{plural} ({args_given} given)'.format( - function=wrapped.__name__, - args_max=max_positional_args, - args_given=len(args), - plural=plural_s)) - if positional_parameters_enforcement == POSITIONAL_EXCEPTION: - raise TypeError(message) - elif positional_parameters_enforcement == POSITIONAL_WARNING: - logger.warning(message) - return wrapped(*args, **kwargs) - return positional_wrapper - - if isinstance(max_positional_args, six.integer_types): - return positional_decorator - else: - args, _, _, defaults = inspect.getargspec(max_positional_args) - return positional(len(args) - len(defaults))(max_positional_args) - - -def scopes_to_string(scopes): - """Converts scope value to a string. - - If scopes is a string then it is simply passed through. If scopes is an - iterable then a string is returned that is all the individual scopes - concatenated with spaces. - - Args: - scopes: string or iterable of strings, the scopes. - - Returns: - The scopes formatted as a single string. - """ - if isinstance(scopes, six.string_types): - return scopes - else: - return ' '.join(scopes) - - -def string_to_scopes(scopes): - """Converts stringifed scope value to a list. - - If scopes is a list then it is simply passed through. If scopes is an - string then a list of each individual scope is returned. - - Args: - scopes: a string or iterable of strings, the scopes. - - Returns: - The scopes in a list. - """ - if not scopes: - return [] - elif isinstance(scopes, six.string_types): - return scopes.split(' ') - else: - return scopes - - -def parse_unique_urlencoded(content): - """Parses unique key-value parameters from urlencoded content. - - Args: - content: string, URL-encoded key-value pairs. - - Returns: - dict, The key-value pairs from ``content``. - - Raises: - ValueError: if one of the keys is repeated. - """ - urlencoded_params = urllib.parse.parse_qs(content) - params = {} - for key, value in six.iteritems(urlencoded_params): - if len(value) != 1: - msg = ('URL-encoded content contains a repeated value:' - '%s -> %s' % (key, ', '.join(value))) - raise ValueError(msg) - params[key] = value[0] - return params - - -def update_query_params(uri, params): - """Updates a URI with new query parameters. - - If a given key from ``params`` is repeated in the ``uri``, then - the URI will be considered invalid and an error will occur. - - If the URI is valid, then each value from ``params`` will - replace the corresponding value in the query parameters (if - it exists). - - Args: - uri: string, A valid URI, with potential existing query parameters. - params: dict, A dictionary of query parameters. - - Returns: - The same URI but with the new query parameters added. - """ - parts = urllib.parse.urlparse(uri) - query_params = parse_unique_urlencoded(parts.query) - query_params.update(params) - new_query = urllib.parse.urlencode(query_params) - new_parts = parts._replace(query=new_query) - return urllib.parse.urlunparse(new_parts) - - -def _add_query_parameter(url, name, value): - """Adds a query parameter to a url. - - Replaces the current value if it already exists in the URL. - - Args: - url: string, url to add the query parameter to. - name: string, query parameter name. - value: string, query parameter value. - - Returns: - Updated query parameter. Does not update the url if value is None. - """ - if value is None: - return url - else: - return update_query_params(url, {name: value}) - - -def validate_file(filename): - if os.path.islink(filename): - raise IOError(_SYM_LINK_MESSAGE.format(filename)) - elif os.path.isdir(filename): - raise IOError(_IS_DIR_MESSAGE.format(filename)) - elif not os.path.isfile(filename): - warnings.warn(_MISSING_FILE_MESSAGE.format(filename)) - - -def _parse_pem_key(raw_key_input): - """Identify and extract PEM keys. - - Determines whether the given key is in the format of PEM key, and extracts - the relevant part of the key if it is. - - Args: - raw_key_input: The contents of a private key file (either PEM or - PKCS12). - - Returns: - string, The actual key if the contents are from a PEM file, or - else None. - """ - offset = raw_key_input.find(b'-----BEGIN ') - if offset != -1: - return raw_key_input[offset:] - - -def _json_encode(data): - return json.dumps(data, separators=(',', ':')) - - -def _to_bytes(value, encoding='ascii'): - """Converts a string value to bytes, if necessary. - - Unfortunately, ``six.b`` is insufficient for this task since in - Python2 it does not modify ``unicode`` objects. - - Args: - value: The string/bytes value to be converted. - encoding: The encoding to use to convert unicode to bytes. Defaults - to "ascii", which will not allow any characters from ordinals - larger than 127. Other useful values are "latin-1", which - which will only allows byte ordinals (up to 255) and "utf-8", - which will encode any unicode that needs to be. - - Returns: - The original value converted to bytes (if unicode) or as passed in - if it started out as bytes. - - Raises: - ValueError if the value could not be converted to bytes. - """ - result = (value.encode(encoding) - if isinstance(value, six.text_type) else value) - if isinstance(result, six.binary_type): - return result - else: - raise ValueError('{0!r} could not be converted to bytes'.format(value)) - - -def _from_bytes(value): - """Converts bytes to a string value, if necessary. - - Args: - value: The string/bytes value to be converted. - - Returns: - The original value converted to unicode (if bytes) or as passed in - if it started out as unicode. - - Raises: - ValueError if the value could not be converted to unicode. - """ - result = (value.decode('utf-8') - if isinstance(value, six.binary_type) else value) - if isinstance(result, six.text_type): - return result - else: - raise ValueError( - '{0!r} could not be converted to unicode'.format(value)) - - -def _urlsafe_b64encode(raw_bytes): - raw_bytes = _to_bytes(raw_bytes, encoding='utf-8') - return base64.urlsafe_b64encode(raw_bytes).rstrip(b'=') - - -def _urlsafe_b64decode(b64string): - # Guard against unicode strings, which base64 can't handle. - b64string = _to_bytes(b64string) - padded = b64string + b'=' * (4 - len(b64string) % 4) - return base64.urlsafe_b64decode(padded) diff --git a/src/oauth2client/_openssl_crypt.py b/src/oauth2client/_openssl_crypt.py deleted file mode 100644 index 77fac743..00000000 --- a/src/oauth2client/_openssl_crypt.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""OpenSSL Crypto-related routines for oauth2client.""" - -from OpenSSL import crypto - -from oauth2client import _helpers - - -class OpenSSLVerifier(object): - """Verifies the signature on a message.""" - - def __init__(self, pubkey): - """Constructor. - - Args: - pubkey: OpenSSL.crypto.PKey, The public key to verify with. - """ - self._pubkey = pubkey - - def verify(self, message, signature): - """Verifies a message against a signature. - - Args: - message: string or bytes, The message to verify. If string, will be - encoded to bytes as utf-8. - signature: string or bytes, The signature on the message. If string, - will be encoded to bytes as utf-8. - - Returns: - True if message was signed by the private key associated with the - public key that this object was constructed with. - """ - message = _helpers._to_bytes(message, encoding='utf-8') - signature = _helpers._to_bytes(signature, encoding='utf-8') - try: - crypto.verify(self._pubkey, signature, message, 'sha256') - return True - except crypto.Error: - return False - - @staticmethod - def from_string(key_pem, is_x509_cert): - """Construct a Verified instance from a string. - - Args: - key_pem: string, public key in PEM format. - is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it - is expected to be an RSA key in PEM format. - - Returns: - Verifier instance. - - Raises: - OpenSSL.crypto.Error: if the key_pem can't be parsed. - """ - key_pem = _helpers._to_bytes(key_pem) - if is_x509_cert: - pubkey = crypto.load_certificate(crypto.FILETYPE_PEM, key_pem) - else: - pubkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key_pem) - return OpenSSLVerifier(pubkey) - - -class OpenSSLSigner(object): - """Signs messages with a private key.""" - - def __init__(self, pkey): - """Constructor. - - Args: - pkey: OpenSSL.crypto.PKey (or equiv), The private key to sign with. - """ - self._key = pkey - - def sign(self, message): - """Signs a message. - - Args: - message: bytes, Message to be signed. - - Returns: - string, The signature of the message for the given key. - """ - message = _helpers._to_bytes(message, encoding='utf-8') - return crypto.sign(self._key, message, 'sha256') - - @staticmethod - def from_string(key, password=b'notasecret'): - """Construct a Signer instance from a string. - - Args: - key: string, private key in PKCS12 or PEM format. - password: string, password for the private key file. - - Returns: - Signer instance. - - Raises: - OpenSSL.crypto.Error if the key can't be parsed. - """ - key = _helpers._to_bytes(key) - parsed_pem_key = _helpers._parse_pem_key(key) - if parsed_pem_key: - pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, parsed_pem_key) - else: - password = _helpers._to_bytes(password, encoding='utf-8') - pkey = crypto.load_pkcs12(key, password).get_privatekey() - return OpenSSLSigner(pkey) - - -def pkcs12_key_as_pem(private_key_bytes, private_key_password): - """Convert the contents of a PKCS#12 key to PEM using pyOpenSSL. - - Args: - private_key_bytes: Bytes. PKCS#12 key in DER format. - private_key_password: String. Password for PKCS#12 key. - - Returns: - String. PEM contents of ``private_key_bytes``. - """ - private_key_password = _helpers._to_bytes(private_key_password) - pkcs12 = crypto.load_pkcs12(private_key_bytes, private_key_password) - return crypto.dump_privatekey(crypto.FILETYPE_PEM, - pkcs12.get_privatekey()) diff --git a/src/oauth2client/_pkce.py b/src/oauth2client/_pkce.py deleted file mode 100644 index e4952d8c..00000000 --- a/src/oauth2client/_pkce.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Utility functions for implementing Proof Key for Code Exchange (PKCE) by OAuth -Public Clients - -See RFC7636. -""" - -import base64 -import hashlib -import os - - -def code_verifier(n_bytes=64): - """ - Generates a 'code_verifier' as described in section 4.1 of RFC 7636. - - This is a 'high-entropy cryptographic random string' that will be - impractical for an attacker to guess. - - Args: - n_bytes: integer between 31 and 96, inclusive. default: 64 - number of bytes of entropy to include in verifier. - - Returns: - Bytestring, representing urlsafe base64-encoded random data. - """ - verifier = base64.urlsafe_b64encode(os.urandom(n_bytes)).rstrip(b'=') - # https://tools.ietf.org/html/rfc7636#section-4.1 - # minimum length of 43 characters and a maximum length of 128 characters. - if len(verifier) < 43: - raise ValueError("Verifier too short. n_bytes must be > 30.") - elif len(verifier) > 128: - raise ValueError("Verifier too long. n_bytes must be < 97.") - else: - return verifier - - -def code_challenge(verifier): - """ - Creates a 'code_challenge' as described in section 4.2 of RFC 7636 - by taking the sha256 hash of the verifier and then urlsafe - base64-encoding it. - - Args: - verifier: bytestring, representing a code_verifier as generated by - code_verifier(). - - Returns: - Bytestring, representing a urlsafe base64-encoded sha256 hash digest, - without '=' padding. - """ - digest = hashlib.sha256(verifier).digest() - return base64.urlsafe_b64encode(digest).rstrip(b'=') diff --git a/src/oauth2client/_pure_python_crypt.py b/src/oauth2client/_pure_python_crypt.py deleted file mode 100644 index 2c5d43aa..00000000 --- a/src/oauth2client/_pure_python_crypt.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pure Python crypto-related routines for oauth2client. - -Uses the ``rsa``, ``pyasn1`` and ``pyasn1_modules`` packages -to parse PEM files storing PKCS#1 or PKCS#8 keys as well as -certificates. -""" - -from pyasn1.codec.der import decoder -from pyasn1_modules import pem -from pyasn1_modules.rfc2459 import Certificate -from pyasn1_modules.rfc5208 import PrivateKeyInfo -import rsa -import six - -from oauth2client import _helpers - - -_PKCS12_ERROR = r"""\ -PKCS12 format is not supported by the RSA library. -Either install PyOpenSSL, or please convert .p12 format -to .pem format: - $ cat key.p12 | \ - > openssl pkcs12 -nodes -nocerts -passin pass:notasecret | \ - > openssl rsa > key.pem -""" - -_POW2 = (128, 64, 32, 16, 8, 4, 2, 1) -_PKCS1_MARKER = ('-----BEGIN RSA PRIVATE KEY-----', - '-----END RSA PRIVATE KEY-----') -_PKCS8_MARKER = ('-----BEGIN PRIVATE KEY-----', - '-----END PRIVATE KEY-----') -_PKCS8_SPEC = PrivateKeyInfo() - - -def _bit_list_to_bytes(bit_list): - """Converts an iterable of 1's and 0's to bytes. - - Combines the list 8 at a time, treating each group of 8 bits - as a single byte. - """ - num_bits = len(bit_list) - byte_vals = bytearray() - for start in six.moves.xrange(0, num_bits, 8): - curr_bits = bit_list[start:start + 8] - char_val = sum(val * digit - for val, digit in zip(_POW2, curr_bits)) - byte_vals.append(char_val) - return bytes(byte_vals) - - -class RsaVerifier(object): - """Verifies the signature on a message. - - Args: - pubkey: rsa.key.PublicKey (or equiv), The public key to verify with. - """ - - def __init__(self, pubkey): - self._pubkey = pubkey - - def verify(self, message, signature): - """Verifies a message against a signature. - - Args: - message: string or bytes, The message to verify. If string, will be - encoded to bytes as utf-8. - signature: string or bytes, The signature on the message. If - string, will be encoded to bytes as utf-8. - - Returns: - True if message was signed by the private key associated with the - public key that this object was constructed with. - """ - message = _helpers._to_bytes(message, encoding='utf-8') - try: - return rsa.pkcs1.verify(message, signature, self._pubkey) - except (ValueError, rsa.pkcs1.VerificationError): - return False - - @classmethod - def from_string(cls, key_pem, is_x509_cert): - """Construct an RsaVerifier instance from a string. - - Args: - key_pem: string, public key in PEM format. - is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it - is expected to be an RSA key in PEM format. - - Returns: - RsaVerifier instance. - - Raises: - ValueError: if the key_pem can't be parsed. In either case, error - will begin with 'No PEM start marker'. If - ``is_x509_cert`` is True, will fail to find the - "-----BEGIN CERTIFICATE-----" error, otherwise fails - to find "-----BEGIN RSA PUBLIC KEY-----". - """ - key_pem = _helpers._to_bytes(key_pem) - if is_x509_cert: - der = rsa.pem.load_pem(key_pem, 'CERTIFICATE') - asn1_cert, remaining = decoder.decode(der, asn1Spec=Certificate()) - if remaining != b'': - raise ValueError('Unused bytes', remaining) - - cert_info = asn1_cert['tbsCertificate']['subjectPublicKeyInfo'] - key_bytes = _bit_list_to_bytes(cert_info['subjectPublicKey']) - pubkey = rsa.PublicKey.load_pkcs1(key_bytes, 'DER') - else: - pubkey = rsa.PublicKey.load_pkcs1(key_pem, 'PEM') - return cls(pubkey) - - -class RsaSigner(object): - """Signs messages with a private key. - - Args: - pkey: rsa.key.PrivateKey (or equiv), The private key to sign with. - """ - - def __init__(self, pkey): - self._key = pkey - - def sign(self, message): - """Signs a message. - - Args: - message: bytes, Message to be signed. - - Returns: - string, The signature of the message for the given key. - """ - message = _helpers._to_bytes(message, encoding='utf-8') - return rsa.pkcs1.sign(message, self._key, 'SHA-256') - - @classmethod - def from_string(cls, key, password='notasecret'): - """Construct an RsaSigner instance from a string. - - Args: - key: string, private key in PEM format. - password: string, password for private key file. Unused for PEM - files. - - Returns: - RsaSigner instance. - - Raises: - ValueError if the key cannot be parsed as PKCS#1 or PKCS#8 in - PEM format. - """ - key = _helpers._from_bytes(key) # pem expects str in Py3 - marker_id, key_bytes = pem.readPemBlocksFromFile( - six.StringIO(key), _PKCS1_MARKER, _PKCS8_MARKER) - - if marker_id == 0: - pkey = rsa.key.PrivateKey.load_pkcs1(key_bytes, - format='DER') - elif marker_id == 1: - key_info, remaining = decoder.decode( - key_bytes, asn1Spec=_PKCS8_SPEC) - if remaining != b'': - raise ValueError('Unused bytes', remaining) - pkey_info = key_info.getComponentByName('privateKey') - pkey = rsa.key.PrivateKey.load_pkcs1(pkey_info.asOctets(), - format='DER') - else: - raise ValueError('No key could be detected.') - - return cls(pkey) diff --git a/src/oauth2client/_pycrypto_crypt.py b/src/oauth2client/_pycrypto_crypt.py deleted file mode 100644 index fd2ce0cd..00000000 --- a/src/oauth2client/_pycrypto_crypt.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""pyCrypto Crypto-related routines for oauth2client.""" - -from Crypto.Hash import SHA256 -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Util.asn1 import DerSequence - -from oauth2client import _helpers - - -class PyCryptoVerifier(object): - """Verifies the signature on a message.""" - - def __init__(self, pubkey): - """Constructor. - - Args: - pubkey: OpenSSL.crypto.PKey (or equiv), The public key to verify - with. - """ - self._pubkey = pubkey - - def verify(self, message, signature): - """Verifies a message against a signature. - - Args: - message: string or bytes, The message to verify. If string, will be - encoded to bytes as utf-8. - signature: string or bytes, The signature on the message. - - Returns: - True if message was signed by the private key associated with the - public key that this object was constructed with. - """ - message = _helpers._to_bytes(message, encoding='utf-8') - return PKCS1_v1_5.new(self._pubkey).verify( - SHA256.new(message), signature) - - @staticmethod - def from_string(key_pem, is_x509_cert): - """Construct a Verified instance from a string. - - Args: - key_pem: string, public key in PEM format. - is_x509_cert: bool, True if key_pem is an X509 cert, otherwise it - is expected to be an RSA key in PEM format. - - Returns: - Verifier instance. - """ - if is_x509_cert: - key_pem = _helpers._to_bytes(key_pem) - pemLines = key_pem.replace(b' ', b'').split() - certDer = _helpers._urlsafe_b64decode(b''.join(pemLines[1:-1])) - certSeq = DerSequence() - certSeq.decode(certDer) - tbsSeq = DerSequence() - tbsSeq.decode(certSeq[0]) - pubkey = RSA.importKey(tbsSeq[6]) - else: - pubkey = RSA.importKey(key_pem) - return PyCryptoVerifier(pubkey) - - -class PyCryptoSigner(object): - """Signs messages with a private key.""" - - def __init__(self, pkey): - """Constructor. - - Args: - pkey, OpenSSL.crypto.PKey (or equiv), The private key to sign with. - """ - self._key = pkey - - def sign(self, message): - """Signs a message. - - Args: - message: string, Message to be signed. - - Returns: - string, The signature of the message for the given key. - """ - message = _helpers._to_bytes(message, encoding='utf-8') - return PKCS1_v1_5.new(self._key).sign(SHA256.new(message)) - - @staticmethod - def from_string(key, password='notasecret'): - """Construct a Signer instance from a string. - - Args: - key: string, private key in PEM format. - password: string, password for private key file. Unused for PEM - files. - - Returns: - Signer instance. - - Raises: - NotImplementedError if the key isn't in PEM format. - """ - parsed_pem_key = _helpers._parse_pem_key(_helpers._to_bytes(key)) - if parsed_pem_key: - pkey = RSA.importKey(parsed_pem_key) - else: - raise NotImplementedError( - 'No key in PEM format was detected. This implementation ' - 'can only use the PyCrypto library for keys in PEM ' - 'format.') - return PyCryptoSigner(pkey) diff --git a/src/oauth2client/client.py b/src/oauth2client/client.py deleted file mode 100644 index 7618960e..00000000 --- a/src/oauth2client/client.py +++ /dev/null @@ -1,2170 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""An OAuth 2.0 client. - -Tools for interacting with OAuth 2.0 protected resources. -""" - -import collections -import copy -import datetime -import json -import logging -import os -import shutil -import socket -import sys -import tempfile - -import six -from six.moves import http_client -from six.moves import urllib - -import oauth2client -from oauth2client import _helpers -from oauth2client import _pkce -from oauth2client import clientsecrets -from oauth2client import transport - - -HAS_OPENSSL = False -HAS_CRYPTO = False -try: - from oauth2client import crypt - HAS_CRYPTO = True - HAS_OPENSSL = crypt.OpenSSLVerifier is not None -except ImportError: # pragma: NO COVER - pass - - -logger = logging.getLogger(__name__) - -# Expiry is stored in RFC3339 UTC format -EXPIRY_FORMAT = '%Y-%m-%dT%H:%M:%SZ' - -# Which certs to use to validate id_tokens received. -ID_TOKEN_VERIFICATION_CERTS = 'https://www.googleapis.com/oauth2/v1/certs' -# This symbol previously had a typo in the name; we keep the old name -# around for now, but will remove it in the future. -ID_TOKEN_VERIFICATON_CERTS = ID_TOKEN_VERIFICATION_CERTS - -# Constant to use for the out of band OAuth 2.0 flow. -OOB_CALLBACK_URN = 'urn:ietf:wg:oauth:2.0:oob' - -# The value representing user credentials. -AUTHORIZED_USER = 'authorized_user' - -# The value representing service account credentials. -SERVICE_ACCOUNT = 'service_account' - -# The environment variable pointing the file with local -# Application Default Credentials. -GOOGLE_APPLICATION_CREDENTIALS = 'GOOGLE_APPLICATION_CREDENTIALS' -# The ~/.config subdirectory containing gcloud credentials. Intended -# to be swapped out in tests. -_CLOUDSDK_CONFIG_DIRECTORY = 'gcloud' -# The environment variable name which can replace ~/.config if set. -_CLOUDSDK_CONFIG_ENV_VAR = 'CLOUDSDK_CONFIG' - -# The error message we show users when we can't find the Application -# Default Credentials. -ADC_HELP_MSG = ( - 'The Application Default Credentials are not available. They are ' - 'available if running in Google Compute Engine. Otherwise, the ' - 'environment variable ' + - GOOGLE_APPLICATION_CREDENTIALS + - ' must be defined pointing to a file defining the credentials. See ' - 'https://developers.google.com/accounts/docs/' - 'application-default-credentials for more information.') - -_WELL_KNOWN_CREDENTIALS_FILE = 'application_default_credentials.json' - -# The access token along with the seconds in which it expires. -AccessTokenInfo = collections.namedtuple( - 'AccessTokenInfo', ['access_token', 'expires_in']) - -DEFAULT_ENV_NAME = 'UNKNOWN' - -# If set to True _get_environment avoid GCE check (_detect_gce_environment) -NO_GCE_CHECK = os.getenv('NO_GCE_CHECK', 'False') - -# Timeout in seconds to wait for the GCE metadata server when detecting the -# GCE environment. -try: - GCE_METADATA_TIMEOUT = int(os.getenv('GCE_METADATA_TIMEOUT', 3)) -except ValueError: # pragma: NO COVER - GCE_METADATA_TIMEOUT = 3 - -_SERVER_SOFTWARE = 'SERVER_SOFTWARE' -_GCE_METADATA_URI = 'http://' + os.getenv('GCE_METADATA_IP', '169.254.169.254') -_METADATA_FLAVOR_HEADER = 'metadata-flavor' # lowercase header -_DESIRED_METADATA_FLAVOR = 'Google' -_GCE_HEADERS = {_METADATA_FLAVOR_HEADER: _DESIRED_METADATA_FLAVOR} - -# Expose utcnow() at module level to allow for -# easier testing (by replacing with a stub). -_UTCNOW = datetime.datetime.utcnow - -# NOTE: These names were previously defined in this module but have been -# moved into `oauth2client.transport`, -clean_headers = transport.clean_headers -MemoryCache = transport.MemoryCache -REFRESH_STATUS_CODES = transport.REFRESH_STATUS_CODES - - -class SETTINGS(object): - """Settings namespace for globally defined values.""" - env_name = None - - -class Error(Exception): - """Base error for this module.""" - - -class FlowExchangeError(Error): - """Error trying to exchange an authorization grant for an access token.""" - - -class AccessTokenRefreshError(Error): - """Error trying to refresh an expired access token.""" - - -class HttpAccessTokenRefreshError(AccessTokenRefreshError): - """Error (with HTTP status) trying to refresh an expired access token.""" - def __init__(self, *args, **kwargs): - super(HttpAccessTokenRefreshError, self).__init__(*args) - self.status = kwargs.get('status') - - -class TokenRevokeError(Error): - """Error trying to revoke a token.""" - - -class UnknownClientSecretsFlowError(Error): - """The client secrets file called for an unknown type of OAuth 2.0 flow.""" - - -class AccessTokenCredentialsError(Error): - """Having only the access_token means no refresh is possible.""" - - -class VerifyJwtTokenError(Error): - """Could not retrieve certificates for validation.""" - - -class NonAsciiHeaderError(Error): - """Header names and values must be ASCII strings.""" - - -class ApplicationDefaultCredentialsError(Error): - """Error retrieving the Application Default Credentials.""" - - -class OAuth2DeviceCodeError(Error): - """Error trying to retrieve a device code.""" - - -class CryptoUnavailableError(Error, NotImplementedError): - """Raised when a crypto library is required, but none is available.""" - - -def _parse_expiry(expiry): - if expiry and isinstance(expiry, datetime.datetime): - return expiry.strftime(EXPIRY_FORMAT) - else: - return None - - -class Credentials(object): - """Base class for all Credentials objects. - - Subclasses must define an authorize() method that applies the credentials - to an HTTP transport. - - Subclasses must also specify a classmethod named 'from_json' that takes a - JSON string as input and returns an instantiated Credentials object. - """ - - NON_SERIALIZED_MEMBERS = frozenset(['store']) - - def authorize(self, http): - """Take an httplib2.Http instance (or equivalent) and authorizes it. - - Authorizes it for the set of credentials, usually by replacing - http.request() with a method that adds in the appropriate headers and - then delegates to the original Http.request() method. - - Args: - http: httplib2.Http, an http object to be used to make the refresh - request. - """ - raise NotImplementedError - - def refresh(self, http): - """Forces a refresh of the access_token. - - Args: - http: httplib2.Http, an http object to be used to make the refresh - request. - """ - raise NotImplementedError - - def revoke(self, http): - """Revokes a refresh_token and makes the credentials void. - - Args: - http: httplib2.Http, an http object to be used to make the revoke - request. - """ - raise NotImplementedError - - def apply(self, headers): - """Add the authorization to the headers. - - Args: - headers: dict, the headers to add the Authorization header to. - """ - raise NotImplementedError - - def _to_json(self, strip, to_serialize=None): - """Utility function that creates JSON repr. of a Credentials object. - - Args: - strip: array, An array of names of members to exclude from the - JSON. - to_serialize: dict, (Optional) The properties for this object - that will be serialized. This allows callers to - modify before serializing. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - curr_type = self.__class__ - if to_serialize is None: - to_serialize = copy.copy(self.__dict__) - else: - # Assumes it is a str->str dictionary, so we don't deep copy. - to_serialize = copy.copy(to_serialize) - for member in strip: - if member in to_serialize: - del to_serialize[member] - to_serialize['token_expiry'] = _parse_expiry( - to_serialize.get('token_expiry')) - # Add in information we will need later to reconstitute this instance. - to_serialize['_class'] = curr_type.__name__ - to_serialize['_module'] = curr_type.__module__ - for key, val in to_serialize.items(): - if isinstance(val, bytes): - to_serialize[key] = val.decode('utf-8') - if isinstance(val, set): - to_serialize[key] = list(val) - return json.dumps(to_serialize) - - def to_json(self): - """Creating a JSON representation of an instance of Credentials. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - return self._to_json(self.NON_SERIALIZED_MEMBERS) - - @classmethod - def new_from_json(cls, json_data): - """Utility class method to instantiate a Credentials subclass from JSON. - - Expects the JSON string to have been produced by to_json(). - - Args: - json_data: string or bytes, JSON from to_json(). - - Returns: - An instance of the subclass of Credentials that was serialized with - to_json(). - """ - json_data_as_unicode = _helpers._from_bytes(json_data) - data = json.loads(json_data_as_unicode) - # Find and call the right classmethod from_json() to restore - # the object. - module_name = data['_module'] - try: - module_obj = __import__(module_name) - except ImportError: - # In case there's an object from the old package structure, - # update it - module_name = module_name.replace('.googleapiclient', '') - module_obj = __import__(module_name) - - module_obj = __import__(module_name, - fromlist=module_name.split('.')[:-1]) - kls = getattr(module_obj, data['_class']) - return kls.from_json(json_data_as_unicode) - - @classmethod - def from_json(cls, unused_data): - """Instantiate a Credentials object from a JSON description of it. - - The JSON should have been produced by calling .to_json() on the object. - - Args: - unused_data: dict, A deserialized JSON object. - - Returns: - An instance of a Credentials subclass. - """ - return Credentials() - - -class Flow(object): - """Base class for all Flow objects.""" - pass - - -class Storage(object): - """Base class for all Storage objects. - - Store and retrieve a single credential. This class supports locking - such that multiple processes and threads can operate on a single - store. - """ - def __init__(self, lock=None): - """Create a Storage instance. - - Args: - lock: An optional threading.Lock-like object. Must implement at - least acquire() and release(). Does not need to be - re-entrant. - """ - self._lock = lock - - def acquire_lock(self): - """Acquires any lock necessary to access this Storage. - - This lock is not reentrant. - """ - if self._lock is not None: - self._lock.acquire() - - def release_lock(self): - """Release the Storage lock. - - Trying to release a lock that isn't held will result in a - RuntimeError in the case of a threading.Lock or multiprocessing.Lock. - """ - if self._lock is not None: - self._lock.release() - - def locked_get(self): - """Retrieve credential. - - The Storage lock must be held when this is called. - - Returns: - oauth2client.client.Credentials - """ - raise NotImplementedError - - def locked_put(self, credentials): - """Write a credential. - - The Storage lock must be held when this is called. - - Args: - credentials: Credentials, the credentials to store. - """ - raise NotImplementedError - - def locked_delete(self): - """Delete a credential. - - The Storage lock must be held when this is called. - """ - raise NotImplementedError - - def get(self): - """Retrieve credential. - - The Storage lock must *not* be held when this is called. - - Returns: - oauth2client.client.Credentials - """ - self.acquire_lock() - try: - return self.locked_get() - finally: - self.release_lock() - - def put(self, credentials): - """Write a credential. - - The Storage lock must be held when this is called. - - Args: - credentials: Credentials, the credentials to store. - """ - self.acquire_lock() - try: - self.locked_put(credentials) - finally: - self.release_lock() - - def delete(self): - """Delete credential. - - Frees any resources associated with storing the credential. - The Storage lock must *not* be held when this is called. - - Returns: - None - """ - self.acquire_lock() - try: - return self.locked_delete() - finally: - self.release_lock() - - -class OAuth2Credentials(Credentials): - """Credentials object for OAuth 2.0. - - Credentials can be applied to an httplib2.Http object using the authorize() - method, which then adds the OAuth 2.0 access token to each request. - - OAuth2Credentials objects may be safely pickled and unpickled. - """ - - @_helpers.positional(8) - def __init__(self, access_token, client_id, client_secret, refresh_token, - token_expiry, token_uri, user_agent, revoke_uri=None, - id_token=None, token_response=None, scopes=None, - token_info_uri=None, id_token_jwt=None): - """Create an instance of OAuth2Credentials. - - This constructor is not usually called by the user, instead - OAuth2Credentials objects are instantiated by the OAuth2WebServerFlow. - - Args: - access_token: string, access token. - client_id: string, client identifier. - client_secret: string, client secret. - refresh_token: string, refresh token. - token_expiry: datetime, when the access_token expires. - token_uri: string, URI of token endpoint. - user_agent: string, The HTTP User-Agent to provide for this - application. - revoke_uri: string, URI for revoke endpoint. Defaults to None; a - token can't be revoked if this is None. - id_token: object, The identity of the resource owner. - token_response: dict, the decoded response to the token request. - None if a token hasn't been requested yet. Stored - because some providers (e.g. wordpress.com) include - extra fields that clients may want. - scopes: list, authorized scopes for these credentials. - token_info_uri: string, the URI for the token info endpoint. - Defaults to None; scopes can not be refreshed if - this is None. - id_token_jwt: string, the encoded and signed identity JWT. The - decoded version of this is stored in id_token. - - Notes: - store: callable, A callable that when passed a Credential - will store the credential back to where it came from. - This is needed to store the latest access_token if it - has expired and been refreshed. - """ - self.access_token = access_token - self.client_id = client_id - self.client_secret = client_secret - self.refresh_token = refresh_token - self.store = None - self.token_expiry = token_expiry - self.token_uri = token_uri - self.user_agent = user_agent - self.revoke_uri = revoke_uri - self.id_token = id_token - self.id_token_jwt = id_token_jwt - self.token_response = token_response - self.scopes = set(_helpers.string_to_scopes(scopes or [])) - self.token_info_uri = token_info_uri - - # True if the credentials have been revoked or expired and can't be - # refreshed. - self.invalid = False - - def authorize(self, http): - """Authorize an httplib2.Http instance with these credentials. - - The modified http.request method will add authentication headers to - each request and will refresh access_tokens when a 401 is received on a - request. In addition the http.request method has a credentials - property, http.request.credentials, which is the Credentials object - that authorized it. - - Args: - http: An instance of ``httplib2.Http`` or something that acts - like it. - - Returns: - A modified instance of http that was passed in. - - Example:: - - h = httplib2.Http() - h = credentials.authorize(h) - - You can't create a new OAuth subclass of httplib2.Authentication - because it never gets passed the absolute URI, which is needed for - signing. So instead we have to overload 'request' with a closure - that adds in the Authorization header and then calls the original - version of 'request()'. - """ - transport.wrap_http_for_auth(self, http) - return http - - def refresh(self, http): - """Forces a refresh of the access_token. - - Args: - http: httplib2.Http, an http object to be used to make the refresh - request. - """ - self._refresh(http) - - def revoke(self, http): - """Revokes a refresh_token and makes the credentials void. - - Args: - http: httplib2.Http, an http object to be used to make the revoke - request. - """ - self._revoke(http) - - def apply(self, headers): - """Add the authorization to the headers. - - Args: - headers: dict, the headers to add the Authorization header to. - """ - headers['Authorization'] = 'Bearer ' + self.access_token - - def has_scopes(self, scopes): - """Verify that the credentials are authorized for the given scopes. - - Returns True if the credentials authorized scopes contain all of the - scopes given. - - Args: - scopes: list or string, the scopes to check. - - Notes: - There are cases where the credentials are unaware of which scopes - are authorized. Notably, credentials obtained and stored before - this code was added will not have scopes, AccessTokenCredentials do - not have scopes. In both cases, you can use refresh_scopes() to - obtain the canonical set of scopes. - """ - scopes = _helpers.string_to_scopes(scopes) - return set(scopes).issubset(self.scopes) - - def retrieve_scopes(self, http): - """Retrieves the canonical list of scopes for this access token. - - Gets the scopes from the OAuth2 provider. - - Args: - http: httplib2.Http, an http object to be used to make the refresh - request. - - Returns: - A set of strings containing the canonical list of scopes. - """ - self._retrieve_scopes(http) - return self.scopes - - @classmethod - def from_json(cls, json_data): - """Instantiate a Credentials object from a JSON description of it. - - The JSON should have been produced by calling .to_json() on the object. - - Args: - json_data: string or bytes, JSON to deserialize. - - Returns: - An instance of a Credentials subclass. - """ - data = json.loads(_helpers._from_bytes(json_data)) - if (data.get('token_expiry') and - not isinstance(data['token_expiry'], datetime.datetime)): - try: - data['token_expiry'] = datetime.datetime.strptime( - data['token_expiry'], EXPIRY_FORMAT) - except ValueError: - data['token_expiry'] = None - retval = cls( - data['access_token'], - data['client_id'], - data['client_secret'], - data['refresh_token'], - data['token_expiry'], - data['token_uri'], - data['user_agent'], - revoke_uri=data.get('revoke_uri', None), - id_token=data.get('id_token', None), - id_token_jwt=data.get('id_token_jwt', None), - token_response=data.get('token_response', None), - scopes=data.get('scopes', None), - token_info_uri=data.get('token_info_uri', None)) - retval.invalid = data['invalid'] - return retval - - @property - def access_token_expired(self): - """True if the credential is expired or invalid. - - If the token_expiry isn't set, we assume the token doesn't expire. - """ - if self.invalid: - return True - - if not self.token_expiry: - return False - - now = _UTCNOW() - if now >= self.token_expiry: - logger.info('access_token is expired. Now: %s, token_expiry: %s', - now, self.token_expiry) - return True - return False - - def get_access_token(self, http=None): - """Return the access token and its expiration information. - - If the token does not exist, get one. - If the token expired, refresh it. - """ - if not self.access_token or self.access_token_expired: - if not http: - http = transport.get_http_object() - self.refresh(http) - return AccessTokenInfo(access_token=self.access_token, - expires_in=self._expires_in()) - - def set_store(self, store): - """Set the Storage for the credential. - - Args: - store: Storage, an implementation of Storage object. - This is needed to store the latest access_token if it - has expired and been refreshed. This implementation uses - locking to check for updates before updating the - access_token. - """ - self.store = store - - def _expires_in(self): - """Return the number of seconds until this token expires. - - If token_expiry is in the past, this method will return 0, meaning the - token has already expired. - - If token_expiry is None, this method will return None. Note that - returning 0 in such a case would not be fair: the token may still be - valid; we just don't know anything about it. - """ - if self.token_expiry: - now = _UTCNOW() - if self.token_expiry > now: - time_delta = self.token_expiry - now - # TODO(orestica): return time_delta.total_seconds() - # once dropping support for Python 2.6 - return time_delta.days * 86400 + time_delta.seconds - else: - return 0 - - def _updateFromCredential(self, other): - """Update this Credential from another instance.""" - self.__dict__.update(other.__getstate__()) - - def __getstate__(self): - """Trim the state down to something that can be pickled.""" - d = copy.copy(self.__dict__) - del d['store'] - return d - - def __setstate__(self, state): - """Reconstitute the state of the object from being pickled.""" - self.__dict__.update(state) - self.store = None - - def _generate_refresh_request_body(self): - """Generate the body that will be used in the refresh request.""" - body = urllib.parse.urlencode({ - 'grant_type': 'refresh_token', - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'refresh_token': self.refresh_token, - }) - return body - - def _generate_refresh_request_headers(self): - """Generate the headers that will be used in the refresh request.""" - headers = { - 'content-type': 'application/x-www-form-urlencoded', - } - - if self.user_agent is not None: - headers['user-agent'] = self.user_agent - - return headers - - def _refresh(self, http): - """Refreshes the access_token. - - This method first checks by reading the Storage object if available. - If a refresh is still needed, it holds the Storage lock until the - refresh is completed. - - Args: - http: an object to be used to make HTTP requests. - - Raises: - HttpAccessTokenRefreshError: When the refresh fails. - """ - if not self.store: - self._do_refresh_request(http) - else: - self.store.acquire_lock() - try: - new_cred = self.store.locked_get() - - if (new_cred and not new_cred.invalid and - new_cred.access_token != self.access_token and - not new_cred.access_token_expired): - logger.info('Updated access_token read from Storage') - self._updateFromCredential(new_cred) - else: - self._do_refresh_request(http) - finally: - self.store.release_lock() - - def _do_refresh_request(self, http): - """Refresh the access_token using the refresh_token. - - Args: - http: an object to be used to make HTTP requests. - - Raises: - HttpAccessTokenRefreshError: When the refresh fails. - """ - body = self._generate_refresh_request_body() - headers = self._generate_refresh_request_headers() - - logger.info('Refreshing access_token') - resp, content = transport.request( - http, self.token_uri, method='POST', - body=body, headers=headers) - content = _helpers._from_bytes(content) - if resp.status == http_client.OK: - d = json.loads(content) - self.token_response = d - self.access_token = d['access_token'] - self.refresh_token = d.get('refresh_token', self.refresh_token) - if 'expires_in' in d: - delta = datetime.timedelta(seconds=int(d['expires_in'])) - self.token_expiry = delta + _UTCNOW() - else: - self.token_expiry = None - if 'id_token' in d: - self.id_token = _extract_id_token(d['id_token']) - self.id_token_jwt = d['id_token'] - else: - self.id_token = None - self.id_token_jwt = None - # On temporary refresh errors, the user does not actually have to - # re-authorize, so we unflag here. - self.invalid = False - if self.store: - self.store.locked_put(self) - else: - # An {'error':...} response body means the token is expired or - # revoked, so we flag the credentials as such. - logger.info('Failed to retrieve access token: %s', content) - error_msg = 'Invalid response {0}.'.format(resp.status) - try: - d = json.loads(content) - if 'error' in d: - error_msg = d['error'] - if 'error_description' in d: - error_msg += ': ' + d['error_description'] - self.invalid = True - if self.store is not None: - self.store.locked_put(self) - except (TypeError, ValueError): - pass - raise HttpAccessTokenRefreshError(error_msg, status=resp.status) - - def _revoke(self, http): - """Revokes this credential and deletes the stored copy (if it exists). - - Args: - http: an object to be used to make HTTP requests. - """ - self._do_revoke(http, self.refresh_token or self.access_token) - - def _do_revoke(self, http, token): - """Revokes this credential and deletes the stored copy (if it exists). - - Args: - http: an object to be used to make HTTP requests. - token: A string used as the token to be revoked. Can be either an - access_token or refresh_token. - - Raises: - TokenRevokeError: If the revoke request does not return with a - 200 OK. - """ - logger.info('Revoking token') - query_params = {'token': token} - token_revoke_uri = _helpers.update_query_params( - self.revoke_uri, query_params) - resp, content = transport.request(http, token_revoke_uri) - if resp.status == http_client.METHOD_NOT_ALLOWED: - body = urllib.parse.urlencode(query_params) - resp, content = transport.request(http, token_revoke_uri, - method='POST', body=body) - if resp.status == http_client.OK: - self.invalid = True - else: - error_msg = 'Invalid response {0}.'.format(resp.status) - try: - d = json.loads(_helpers._from_bytes(content)) - if 'error' in d: - error_msg = d['error'] - except (TypeError, ValueError): - pass - raise TokenRevokeError(error_msg) - - if self.store: - self.store.delete() - - def _retrieve_scopes(self, http): - """Retrieves the list of authorized scopes from the OAuth2 provider. - - Args: - http: an object to be used to make HTTP requests. - """ - self._do_retrieve_scopes(http, self.access_token) - - def _do_retrieve_scopes(self, http, token): - """Retrieves the list of authorized scopes from the OAuth2 provider. - - Args: - http: an object to be used to make HTTP requests. - token: A string used as the token to identify the credentials to - the provider. - - Raises: - Error: When refresh fails, indicating the the access token is - invalid. - """ - logger.info('Refreshing scopes') - query_params = {'access_token': token, 'fields': 'scope'} - token_info_uri = _helpers.update_query_params( - self.token_info_uri, query_params) - resp, content = transport.request(http, token_info_uri) - content = _helpers._from_bytes(content) - if resp.status == http_client.OK: - d = json.loads(content) - self.scopes = set(_helpers.string_to_scopes(d.get('scope', ''))) - else: - error_msg = 'Invalid response {0}.'.format(resp.status) - try: - d = json.loads(content) - if 'error_description' in d: - error_msg = d['error_description'] - except (TypeError, ValueError): - pass - raise Error(error_msg) - - -class AccessTokenCredentials(OAuth2Credentials): - """Credentials object for OAuth 2.0. - - Credentials can be applied to an httplib2.Http object using the - authorize() method, which then signs each request from that object - with the OAuth 2.0 access token. This set of credentials is for the - use case where you have acquired an OAuth 2.0 access_token from - another place such as a JavaScript client or another web - application, and wish to use it from Python. Because only the - access_token is present it can not be refreshed and will in time - expire. - - AccessTokenCredentials objects may be safely pickled and unpickled. - - Usage:: - - credentials = AccessTokenCredentials('', - 'my-user-agent/1.0') - http = httplib2.Http() - http = credentials.authorize(http) - - Raises: - AccessTokenCredentialsExpired: raised when the access_token expires or - is revoked. - """ - - def __init__(self, access_token, user_agent, revoke_uri=None): - """Create an instance of OAuth2Credentials - - This is one of the few types if Credentials that you should contrust, - Credentials objects are usually instantiated by a Flow. - - Args: - access_token: string, access token. - user_agent: string, The HTTP User-Agent to provide for this - application. - revoke_uri: string, URI for revoke endpoint. Defaults to None; a - token can't be revoked if this is None. - """ - super(AccessTokenCredentials, self).__init__( - access_token, - None, - None, - None, - None, - None, - user_agent, - revoke_uri=revoke_uri) - - @classmethod - def from_json(cls, json_data): - data = json.loads(_helpers._from_bytes(json_data)) - retval = AccessTokenCredentials( - data['access_token'], - data['user_agent']) - return retval - - def _refresh(self, http): - """Refreshes the access token. - - Args: - http: unused HTTP object. - - Raises: - AccessTokenCredentialsError: always - """ - raise AccessTokenCredentialsError( - 'The access_token is expired or invalid and can\'t be refreshed.') - - def _revoke(self, http): - """Revokes the access_token and deletes the store if available. - - Args: - http: an object to be used to make HTTP requests. - """ - self._do_revoke(http, self.access_token) - - -def _detect_gce_environment(): - """Determine if the current environment is Compute Engine. - - Returns: - Boolean indicating whether or not the current environment is Google - Compute Engine. - """ - # NOTE: The explicit ``timeout`` is a workaround. The underlying - # issue is that resolving an unknown host on some networks will take - # 20-30 seconds; making this timeout short fixes the issue, but - # could lead to false negatives in the event that we are on GCE, but - # the metadata resolution was particularly slow. The latter case is - # "unlikely". - http = transport.get_http_object(timeout=GCE_METADATA_TIMEOUT) - try: - response, _ = transport.request( - http, _GCE_METADATA_URI, headers=_GCE_HEADERS) - return ( - response.status == http_client.OK and - response.get(_METADATA_FLAVOR_HEADER) == _DESIRED_METADATA_FLAVOR) - except socket.error: # socket.timeout or socket.error(64, 'Host is down') - logger.info('Timeout attempting to reach GCE metadata service.') - return False - - -def _in_gae_environment(): - """Detects if the code is running in the App Engine environment. - - Returns: - True if running in the GAE environment, False otherwise. - """ - if SETTINGS.env_name is not None: - return SETTINGS.env_name in ('GAE_PRODUCTION', 'GAE_LOCAL') - - try: - import google.appengine # noqa: unused import - except ImportError: - pass - else: - server_software = os.environ.get(_SERVER_SOFTWARE, '') - if server_software.startswith('Google App Engine/'): - SETTINGS.env_name = 'GAE_PRODUCTION' - return True - elif server_software.startswith('Development/'): - SETTINGS.env_name = 'GAE_LOCAL' - return True - - return False - - -def _in_gce_environment(): - """Detect if the code is running in the Compute Engine environment. - - Returns: - True if running in the GCE environment, False otherwise. - """ - if SETTINGS.env_name is not None: - return SETTINGS.env_name == 'GCE_PRODUCTION' - - if NO_GCE_CHECK != 'True' and _detect_gce_environment(): - SETTINGS.env_name = 'GCE_PRODUCTION' - return True - return False - - -class GoogleCredentials(OAuth2Credentials): - """Application Default Credentials for use in calling Google APIs. - - The Application Default Credentials are being constructed as a function of - the environment where the code is being run. - More details can be found on this page: - https://developers.google.com/accounts/docs/application-default-credentials - - Here is an example of how to use the Application Default Credentials for a - service that requires authentication:: - - from googleapiclient.discovery import build - from oauth2client.client import GoogleCredentials - - credentials = GoogleCredentials.get_application_default() - service = build('compute', 'v1', credentials=credentials) - - PROJECT = 'bamboo-machine-422' - ZONE = 'us-central1-a' - request = service.instances().list(project=PROJECT, zone=ZONE) - response = request.execute() - - print(response) - """ - - NON_SERIALIZED_MEMBERS = ( - frozenset(['_private_key']) | - OAuth2Credentials.NON_SERIALIZED_MEMBERS) - """Members that aren't serialized when object is converted to JSON.""" - - def __init__(self, access_token, client_id, client_secret, refresh_token, - token_expiry, token_uri, user_agent, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI): - """Create an instance of GoogleCredentials. - - This constructor is not usually called by the user, instead - GoogleCredentials objects are instantiated by - GoogleCredentials.from_stream() or - GoogleCredentials.get_application_default(). - - Args: - access_token: string, access token. - client_id: string, client identifier. - client_secret: string, client secret. - refresh_token: string, refresh token. - token_expiry: datetime, when the access_token expires. - token_uri: string, URI of token endpoint. - user_agent: string, The HTTP User-Agent to provide for this - application. - revoke_uri: string, URI for revoke endpoint. Defaults to - oauth2client.GOOGLE_REVOKE_URI; a token can't be - revoked if this is None. - """ - super(GoogleCredentials, self).__init__( - access_token, client_id, client_secret, refresh_token, - token_expiry, token_uri, user_agent, revoke_uri=revoke_uri) - - def create_scoped_required(self): - """Whether this Credentials object is scopeless. - - create_scoped(scopes) method needs to be called in order to create - a Credentials object for API calls. - """ - return False - - def create_scoped(self, scopes): - """Create a Credentials object for the given scopes. - - The Credentials type is preserved. - """ - return self - - @classmethod - def from_json(cls, json_data): - # TODO(issue 388): eliminate the circularity that is the reason for - # this non-top-level import. - from oauth2client import service_account - data = json.loads(_helpers._from_bytes(json_data)) - - # We handle service_account.ServiceAccountCredentials since it is a - # possible return type of GoogleCredentials.get_application_default() - if (data['_module'] == 'oauth2client.service_account' and - data['_class'] == 'ServiceAccountCredentials'): - return service_account.ServiceAccountCredentials.from_json(data) - elif (data['_module'] == 'oauth2client.service_account' and - data['_class'] == '_JWTAccessCredentials'): - return service_account._JWTAccessCredentials.from_json(data) - - token_expiry = _parse_expiry(data.get('token_expiry')) - google_credentials = cls( - data['access_token'], - data['client_id'], - data['client_secret'], - data['refresh_token'], - token_expiry, - data['token_uri'], - data['user_agent'], - revoke_uri=data.get('revoke_uri', None)) - google_credentials.invalid = data['invalid'] - return google_credentials - - @property - def serialization_data(self): - """Get the fields and values identifying the current credentials.""" - return { - 'type': 'authorized_user', - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'refresh_token': self.refresh_token - } - - @staticmethod - def _implicit_credentials_from_gae(): - """Attempts to get implicit credentials in Google App Engine env. - - If the current environment is not detected as App Engine, returns None, - indicating no Google App Engine credentials can be detected from the - current environment. - - Returns: - None, if not in GAE, else an appengine.AppAssertionCredentials - object. - """ - if not _in_gae_environment(): - return None - - return _get_application_default_credential_GAE() - - @staticmethod - def _implicit_credentials_from_gce(): - """Attempts to get implicit credentials in Google Compute Engine env. - - If the current environment is not detected as Compute Engine, returns - None, indicating no Google Compute Engine credentials can be detected - from the current environment. - - Returns: - None, if not in GCE, else a gce.AppAssertionCredentials object. - """ - if not _in_gce_environment(): - return None - - return _get_application_default_credential_GCE() - - @staticmethod - def _implicit_credentials_from_files(): - """Attempts to get implicit credentials from local credential files. - - First checks if the environment variable GOOGLE_APPLICATION_CREDENTIALS - is set with a filename and then falls back to a configuration file (the - "well known" file) associated with the 'gcloud' command line tool. - - Returns: - Credentials object associated with the - GOOGLE_APPLICATION_CREDENTIALS file or the "well known" file if - either exist. If neither file is define, returns None, indicating - no credentials from a file can detected from the current - environment. - """ - credentials_filename = _get_environment_variable_file() - if not credentials_filename: - credentials_filename = _get_well_known_file() - if os.path.isfile(credentials_filename): - extra_help = (' (produced automatically when running' - ' "gcloud auth login" command)') - else: - credentials_filename = None - else: - extra_help = (' (pointed to by ' + GOOGLE_APPLICATION_CREDENTIALS + - ' environment variable)') - - if not credentials_filename: - return - - # If we can read the credentials from a file, we don't need to know - # what environment we are in. - SETTINGS.env_name = DEFAULT_ENV_NAME - - try: - return _get_application_default_credential_from_file( - credentials_filename) - except (ApplicationDefaultCredentialsError, ValueError) as error: - _raise_exception_for_reading_json(credentials_filename, - extra_help, error) - - @classmethod - def _get_implicit_credentials(cls): - """Gets credentials implicitly from the environment. - - Checks environment in order of precedence: - - Environment variable GOOGLE_APPLICATION_CREDENTIALS pointing to - a file with stored credentials information. - - Stored "well known" file associated with `gcloud` command line tool. - - Google App Engine (production and testing) - - Google Compute Engine production environment. - - Raises: - ApplicationDefaultCredentialsError: raised when the credentials - fail to be retrieved. - """ - # Environ checks (in order). - environ_checkers = [ - cls._implicit_credentials_from_files, - cls._implicit_credentials_from_gae, - cls._implicit_credentials_from_gce, - ] - - for checker in environ_checkers: - credentials = checker() - if credentials is not None: - return credentials - - # If no credentials, fail. - raise ApplicationDefaultCredentialsError(ADC_HELP_MSG) - - @staticmethod - def get_application_default(): - """Get the Application Default Credentials for the current environment. - - Raises: - ApplicationDefaultCredentialsError: raised when the credentials - fail to be retrieved. - """ - return GoogleCredentials._get_implicit_credentials() - - @staticmethod - def from_stream(credential_filename): - """Create a Credentials object by reading information from a file. - - It returns an object of type GoogleCredentials. - - Args: - credential_filename: the path to the file from where the - credentials are to be read - - Raises: - ApplicationDefaultCredentialsError: raised when the credentials - fail to be retrieved. - """ - if credential_filename and os.path.isfile(credential_filename): - try: - return _get_application_default_credential_from_file( - credential_filename) - except (ApplicationDefaultCredentialsError, ValueError) as error: - extra_help = (' (provided as parameter to the ' - 'from_stream() method)') - _raise_exception_for_reading_json(credential_filename, - extra_help, - error) - else: - raise ApplicationDefaultCredentialsError( - 'The parameter passed to the from_stream() ' - 'method should point to a file.') - - -def _save_private_file(filename, json_contents): - """Saves a file with read-write permissions on for the owner. - - Args: - filename: String. Absolute path to file. - json_contents: JSON serializable object to be saved. - """ - temp_filename = tempfile.mktemp() - file_desc = os.open(temp_filename, os.O_WRONLY | os.O_CREAT, 0o600) - with os.fdopen(file_desc, 'w') as file_handle: - json.dump(json_contents, file_handle, sort_keys=True, - indent=2, separators=(',', ': ')) - shutil.move(temp_filename, filename) - - -def save_to_well_known_file(credentials, well_known_file=None): - """Save the provided GoogleCredentials to the well known file. - - Args: - credentials: the credentials to be saved to the well known file; - it should be an instance of GoogleCredentials - well_known_file: the name of the file where the credentials are to be - saved; this parameter is supposed to be used for - testing only - """ - # TODO(orestica): move this method to tools.py - # once the argparse import gets fixed (it is not present in Python 2.6) - - if well_known_file is None: - well_known_file = _get_well_known_file() - - config_dir = os.path.dirname(well_known_file) - if not os.path.isdir(config_dir): - raise OSError( - 'Config directory does not exist: {0}'.format(config_dir)) - - credentials_data = credentials.serialization_data - _save_private_file(well_known_file, credentials_data) - - -def _get_environment_variable_file(): - application_default_credential_filename = ( - os.environ.get(GOOGLE_APPLICATION_CREDENTIALS, None)) - - if application_default_credential_filename: - if os.path.isfile(application_default_credential_filename): - return application_default_credential_filename - else: - raise ApplicationDefaultCredentialsError( - 'File ' + application_default_credential_filename + - ' (pointed by ' + - GOOGLE_APPLICATION_CREDENTIALS + - ' environment variable) does not exist!') - - -def _get_well_known_file(): - """Get the well known file produced by command 'gcloud auth login'.""" - # TODO(orestica): Revisit this method once gcloud provides a better way - # of pinpointing the exact location of the file. - default_config_dir = os.getenv(_CLOUDSDK_CONFIG_ENV_VAR) - if default_config_dir is None: - if os.name == 'nt': - try: - default_config_dir = os.path.join(os.environ['APPDATA'], - _CLOUDSDK_CONFIG_DIRECTORY) - except KeyError: - # This should never happen unless someone is really - # messing with things. - drive = os.environ.get('SystemDrive', 'C:') - default_config_dir = os.path.join(drive, '\\', - _CLOUDSDK_CONFIG_DIRECTORY) - else: - default_config_dir = os.path.join(os.path.expanduser('~'), - '.config', - _CLOUDSDK_CONFIG_DIRECTORY) - - return os.path.join(default_config_dir, _WELL_KNOWN_CREDENTIALS_FILE) - - -def _get_application_default_credential_from_file(filename): - """Build the Application Default Credentials from file.""" - # read the credentials from the file - with open(filename) as file_obj: - client_credentials = json.load(file_obj) - - credentials_type = client_credentials.get('type') - if credentials_type == AUTHORIZED_USER: - required_fields = set(['client_id', 'client_secret', 'refresh_token']) - elif credentials_type == SERVICE_ACCOUNT: - required_fields = set(['client_id', 'client_email', 'private_key_id', - 'private_key']) - else: - raise ApplicationDefaultCredentialsError( - "'type' field should be defined (and have one of the '" + - AUTHORIZED_USER + "' or '" + SERVICE_ACCOUNT + "' values)") - - missing_fields = required_fields.difference(client_credentials.keys()) - - if missing_fields: - _raise_exception_for_missing_fields(missing_fields) - - if client_credentials['type'] == AUTHORIZED_USER: - return GoogleCredentials( - access_token=None, - client_id=client_credentials['client_id'], - client_secret=client_credentials['client_secret'], - refresh_token=client_credentials['refresh_token'], - token_expiry=None, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - user_agent='Python client library') - else: # client_credentials['type'] == SERVICE_ACCOUNT - from oauth2client import service_account - return service_account._JWTAccessCredentials.from_json_keyfile_dict( - client_credentials) - - -def _raise_exception_for_missing_fields(missing_fields): - raise ApplicationDefaultCredentialsError( - 'The following field(s) must be defined: ' + ', '.join(missing_fields)) - - -def _raise_exception_for_reading_json(credential_file, - extra_help, - error): - raise ApplicationDefaultCredentialsError( - 'An error was encountered while reading json file: ' + - credential_file + extra_help + ': ' + str(error)) - - -def _get_application_default_credential_GAE(): - from oauth2client.contrib.appengine import AppAssertionCredentials - - return AppAssertionCredentials([]) - - -def _get_application_default_credential_GCE(): - from oauth2client.contrib.gce import AppAssertionCredentials - - return AppAssertionCredentials() - - -class AssertionCredentials(GoogleCredentials): - """Abstract Credentials object used for OAuth 2.0 assertion grants. - - This credential does not require a flow to instantiate because it - represents a two legged flow, and therefore has all of the required - information to generate and refresh its own access tokens. It must - be subclassed to generate the appropriate assertion string. - - AssertionCredentials objects may be safely pickled and unpickled. - """ - - @_helpers.positional(2) - def __init__(self, assertion_type, user_agent=None, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI, - **unused_kwargs): - """Constructor for AssertionFlowCredentials. - - Args: - assertion_type: string, assertion type that will be declared to the - auth server - user_agent: string, The HTTP User-Agent to provide for this - application. - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - revoke_uri: string, URI for revoke endpoint. - """ - super(AssertionCredentials, self).__init__( - None, - None, - None, - None, - None, - token_uri, - user_agent, - revoke_uri=revoke_uri) - self.assertion_type = assertion_type - - def _generate_refresh_request_body(self): - assertion = self._generate_assertion() - - body = urllib.parse.urlencode({ - 'assertion': assertion, - 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', - }) - - return body - - def _generate_assertion(self): - """Generate assertion string to be used in the access token request.""" - raise NotImplementedError - - def _revoke(self, http): - """Revokes the access_token and deletes the store if available. - - Args: - http: an object to be used to make HTTP requests. - """ - self._do_revoke(http, self.access_token) - - def sign_blob(self, blob): - """Cryptographically sign a blob (of bytes). - - Args: - blob: bytes, Message to be signed. - - Returns: - tuple, A pair of the private key ID used to sign the blob and - the signed contents. - """ - raise NotImplementedError('This method is abstract.') - - -def _require_crypto_or_die(): - """Ensure we have a crypto library, or throw CryptoUnavailableError. - - The oauth2client.crypt module requires either PyCrypto or PyOpenSSL - to be available in order to function, but these are optional - dependencies. - """ - if not HAS_CRYPTO: - raise CryptoUnavailableError('No crypto library available') - - -@_helpers.positional(2) -def verify_id_token(id_token, audience, http=None, - cert_uri=ID_TOKEN_VERIFICATION_CERTS): - """Verifies a signed JWT id_token. - - This function requires PyOpenSSL and because of that it does not work on - App Engine. - - Args: - id_token: string, A Signed JWT. - audience: string, The audience 'aud' that the token should be for. - http: httplib2.Http, instance to use to make the HTTP request. Callers - should supply an instance that has caching enabled. - cert_uri: string, URI of the certificates in JSON format to - verify the JWT against. - - Returns: - The deserialized JSON in the JWT. - - Raises: - oauth2client.crypt.AppIdentityError: if the JWT fails to verify. - CryptoUnavailableError: if no crypto library is available. - """ - _require_crypto_or_die() - if http is None: - http = transport.get_cached_http() - - resp, content = transport.request(http, cert_uri) - if resp.status == http_client.OK: - certs = json.loads(_helpers._from_bytes(content)) - return crypt.verify_signed_jwt_with_certs(id_token, certs, audience) - else: - raise VerifyJwtTokenError('Status code: {0}'.format(resp.status)) - - -def _extract_id_token(id_token): - """Extract the JSON payload from a JWT. - - Does the extraction w/o checking the signature. - - Args: - id_token: string or bytestring, OAuth 2.0 id_token. - - Returns: - object, The deserialized JSON payload. - """ - if type(id_token) == bytes: - segments = id_token.split(b'.') - else: - segments = id_token.split(u'.') - - if len(segments) != 3: - raise VerifyJwtTokenError( - 'Wrong number of segments in token: {0}'.format(id_token)) - - return json.loads( - _helpers._from_bytes(_helpers._urlsafe_b64decode(segments[1]))) - - -def _parse_exchange_token_response(content): - """Parses response of an exchange token request. - - Most providers return JSON but some (e.g. Facebook) return a - url-encoded string. - - Args: - content: The body of a response - - Returns: - Content as a dictionary object. Note that the dict could be empty, - i.e. {}. That basically indicates a failure. - """ - resp = {} - content = _helpers._from_bytes(content) - try: - resp = json.loads(content) - except Exception: - # different JSON libs raise different exceptions, - # so we just do a catch-all here - resp = _helpers.parse_unique_urlencoded(content) - - # some providers respond with 'expires', others with 'expires_in' - if resp and 'expires' in resp: - resp['expires_in'] = resp.pop('expires') - - return resp - - -@_helpers.positional(4) -def credentials_from_code(client_id, client_secret, scope, code, - redirect_uri='postmessage', http=None, - user_agent=None, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - auth_uri=oauth2client.GOOGLE_AUTH_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI, - device_uri=oauth2client.GOOGLE_DEVICE_URI, - token_info_uri=oauth2client.GOOGLE_TOKEN_INFO_URI, - pkce=False, - code_verifier=None): - """Exchanges an authorization code for an OAuth2Credentials object. - - Args: - client_id: string, client identifier. - client_secret: string, client secret. - scope: string or iterable of strings, scope(s) to request. - code: string, An authorization code, most likely passed down from - the client - redirect_uri: string, this is generally set to 'postmessage' to match - the redirect_uri that the client specified - http: httplib2.Http, optional http instance to use to do the fetch - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - auth_uri: string, URI for authorization endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 provider - can be used. - revoke_uri: string, URI for revoke endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 provider - can be used. - device_uri: string, URI for device authorization endpoint. For - convenience defaults to Google's endpoints but any OAuth - 2.0 provider can be used. - pkce: boolean, default: False, Generate and include a "Proof Key - for Code Exchange" (PKCE) with your authorization and token - requests. This adds security for installed applications that - cannot protect a client_secret. See RFC 7636 for details. - code_verifier: bytestring or None, default: None, parameter passed - as part of the code exchange when pkce=True. If - None, a code_verifier will automatically be - generated as part of step1_get_authorize_url(). See - RFC 7636 for details. - - Returns: - An OAuth2Credentials object. - - Raises: - FlowExchangeError if the authorization code cannot be exchanged for an - access token - """ - flow = OAuth2WebServerFlow(client_id, client_secret, scope, - redirect_uri=redirect_uri, - user_agent=user_agent, - auth_uri=auth_uri, - token_uri=token_uri, - revoke_uri=revoke_uri, - device_uri=device_uri, - token_info_uri=token_info_uri, - pkce=pkce, - code_verifier=code_verifier) - - credentials = flow.step2_exchange(code, http=http) - return credentials - - -@_helpers.positional(3) -def credentials_from_clientsecrets_and_code(filename, scope, code, - message=None, - redirect_uri='postmessage', - http=None, - cache=None, - device_uri=None): - """Returns OAuth2Credentials from a clientsecrets file and an auth code. - - Will create the right kind of Flow based on the contents of the - clientsecrets file or will raise InvalidClientSecretsError for unknown - types of Flows. - - Args: - filename: string, File name of clientsecrets. - scope: string or iterable of strings, scope(s) to request. - code: string, An authorization code, most likely passed down from - the client - message: string, A friendly string to display to the user if the - clientsecrets file is missing or invalid. If message is - provided then sys.exit will be called in the case of an error. - If message in not provided then - clientsecrets.InvalidClientSecretsError will be raised. - redirect_uri: string, this is generally set to 'postmessage' to match - the redirect_uri that the client specified - http: httplib2.Http, optional http instance to use to do the fetch - cache: An optional cache service client that implements get() and set() - methods. See clientsecrets.loadfile() for details. - device_uri: string, OAuth 2.0 device authorization endpoint - pkce: boolean, default: False, Generate and include a "Proof Key - for Code Exchange" (PKCE) with your authorization and token - requests. This adds security for installed applications that - cannot protect a client_secret. See RFC 7636 for details. - code_verifier: bytestring or None, default: None, parameter passed - as part of the code exchange when pkce=True. If - None, a code_verifier will automatically be - generated as part of step1_get_authorize_url(). See - RFC 7636 for details. - - Returns: - An OAuth2Credentials object. - - Raises: - FlowExchangeError: if the authorization code cannot be exchanged for an - access token - UnknownClientSecretsFlowError: if the file describes an unknown kind - of Flow. - clientsecrets.InvalidClientSecretsError: if the clientsecrets file is - invalid. - """ - flow = flow_from_clientsecrets(filename, scope, message=message, - cache=cache, redirect_uri=redirect_uri, - device_uri=device_uri) - credentials = flow.step2_exchange(code, http=http) - return credentials - - -class DeviceFlowInfo(collections.namedtuple('DeviceFlowInfo', ( - 'device_code', 'user_code', 'interval', 'verification_url', - 'user_code_expiry'))): - """Intermediate information the OAuth2 for devices flow.""" - - @classmethod - def FromResponse(cls, response): - """Create a DeviceFlowInfo from a server response. - - The response should be a dict containing entries as described here: - - http://tools.ietf.org/html/draft-ietf-oauth-v2-05#section-3.7.1 - """ - # device_code, user_code, and verification_url are required. - kwargs = { - 'device_code': response['device_code'], - 'user_code': response['user_code'], - } - # The response may list the verification address as either - # verification_url or verification_uri, so we check for both. - verification_url = response.get( - 'verification_url', response.get('verification_uri')) - if verification_url is None: - raise OAuth2DeviceCodeError( - 'No verification_url provided in server response') - kwargs['verification_url'] = verification_url - # expires_in and interval are optional. - kwargs.update({ - 'interval': response.get('interval'), - 'user_code_expiry': None, - }) - if 'expires_in' in response: - kwargs['user_code_expiry'] = ( - _UTCNOW() + - datetime.timedelta(seconds=int(response['expires_in']))) - return cls(**kwargs) - - -def _oauth2_web_server_flow_params(kwargs): - """Configures redirect URI parameters for OAuth2WebServerFlow.""" - params = { - 'access_type': 'offline', - 'response_type': 'code', - } - - params.update(kwargs) - - # Check for the presence of the deprecated approval_prompt param and - # warn appropriately. - approval_prompt = params.get('approval_prompt') - if approval_prompt is not None: - logger.warning( - 'The approval_prompt parameter for OAuth2WebServerFlow is ' - 'deprecated. Please use the prompt parameter instead.') - - if approval_prompt == 'force': - logger.warning( - 'approval_prompt="force" has been adjusted to ' - 'prompt="consent"') - params['prompt'] = 'consent' - del params['approval_prompt'] - - return params - - -class OAuth2WebServerFlow(Flow): - """Does the Web Server Flow for OAuth 2.0. - - OAuth2WebServerFlow objects may be safely pickled and unpickled. - """ - - @_helpers.positional(4) - def __init__(self, client_id, - client_secret=None, - scope=None, - redirect_uri=None, - user_agent=None, - auth_uri=oauth2client.GOOGLE_AUTH_URI, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI, - login_hint=None, - device_uri=oauth2client.GOOGLE_DEVICE_URI, - token_info_uri=oauth2client.GOOGLE_TOKEN_INFO_URI, - authorization_header=None, - pkce=False, - code_verifier=None, - **kwargs): - """Constructor for OAuth2WebServerFlow. - - The kwargs argument is used to set extra query parameters on the - auth_uri. For example, the access_type and prompt - query parameters can be set via kwargs. - - Args: - client_id: string, client identifier. - client_secret: string client secret. - scope: string or iterable of strings, scope(s) of the credentials - being requested. - redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' - for a non-web-based application, or a URI that - handles the callback from the authorization server. - user_agent: string, HTTP User-Agent to provide for this - application. - auth_uri: string, URI for authorization endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 provider - can be used. - token_uri: string, URI for token endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 - provider can be used. - revoke_uri: string, URI for revoke endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 - provider can be used. - login_hint: string, Either an email address or domain. Passing this - hint will either pre-fill the email box on the sign-in - form or select the proper multi-login session, thereby - simplifying the login flow. - device_uri: string, URI for device authorization endpoint. For - convenience defaults to Google's endpoints but any - OAuth 2.0 provider can be used. - authorization_header: string, For use with OAuth 2.0 providers that - require a client to authenticate using a - header value instead of passing client_secret - in the POST body. - pkce: boolean, default: False, Generate and include a "Proof Key - for Code Exchange" (PKCE) with your authorization and token - requests. This adds security for installed applications that - cannot protect a client_secret. See RFC 7636 for details. - code_verifier: bytestring or None, default: None, parameter passed - as part of the code exchange when pkce=True. If - None, a code_verifier will automatically be - generated as part of step1_get_authorize_url(). See - RFC 7636 for details. - **kwargs: dict, The keyword arguments are all optional and required - parameters for the OAuth calls. - """ - # scope is a required argument, but to preserve backwards-compatibility - # we don't want to rearrange the positional arguments - if scope is None: - raise TypeError("The value of scope must not be None") - self.client_id = client_id - self.client_secret = client_secret - self.scope = _helpers.scopes_to_string(scope) - self.redirect_uri = redirect_uri - self.login_hint = login_hint - self.user_agent = user_agent - self.auth_uri = auth_uri - self.token_uri = token_uri - self.revoke_uri = revoke_uri - self.device_uri = device_uri - self.token_info_uri = token_info_uri - self.authorization_header = authorization_header - self._pkce = pkce - self.code_verifier = code_verifier - self.params = _oauth2_web_server_flow_params(kwargs) - - @_helpers.positional(1) - def step1_get_authorize_url(self, redirect_uri=None, state=None): - """Returns a URI to redirect to the provider. - - Args: - redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' - for a non-web-based application, or a URI that - handles the callback from the authorization server. - This parameter is deprecated, please move to passing - the redirect_uri in via the constructor. - state: string, Opaque state string which is passed through the - OAuth2 flow and returned to the client as a query parameter - in the callback. - - Returns: - A URI as a string to redirect the user to begin the authorization - flow. - """ - if redirect_uri is not None: - logger.warning(( - 'The redirect_uri parameter for ' - 'OAuth2WebServerFlow.step1_get_authorize_url is deprecated. ' - 'Please move to passing the redirect_uri in via the ' - 'constructor.')) - self.redirect_uri = redirect_uri - - if self.redirect_uri is None: - raise ValueError('The value of redirect_uri must not be None.') - - query_params = { - 'client_id': self.client_id, - 'redirect_uri': self.redirect_uri, - 'scope': self.scope, - } - if state is not None: - query_params['state'] = state - if self.login_hint is not None: - query_params['login_hint'] = self.login_hint - if self._pkce: - if not self.code_verifier: - self.code_verifier = _pkce.code_verifier() - challenge = _pkce.code_challenge(self.code_verifier) - query_params['code_challenge'] = challenge - query_params['code_challenge_method'] = 'S256' - - query_params.update(self.params) - return _helpers.update_query_params(self.auth_uri, query_params) - - @_helpers.positional(1) - def step1_get_device_and_user_codes(self, http=None): - """Returns a user code and the verification URL where to enter it - - Returns: - A user code as a string for the user to authorize the application - An URL as a string where the user has to enter the code - """ - if self.device_uri is None: - raise ValueError('The value of device_uri must not be None.') - - body = urllib.parse.urlencode({ - 'client_id': self.client_id, - 'scope': self.scope, - }) - headers = { - 'content-type': 'application/x-www-form-urlencoded', - } - - if self.user_agent is not None: - headers['user-agent'] = self.user_agent - - if http is None: - http = transport.get_http_object() - - resp, content = transport.request( - http, self.device_uri, method='POST', body=body, headers=headers) - content = _helpers._from_bytes(content) - if resp.status == http_client.OK: - try: - flow_info = json.loads(content) - except ValueError as exc: - raise OAuth2DeviceCodeError( - 'Could not parse server response as JSON: "{0}", ' - 'error: "{1}"'.format(content, exc)) - return DeviceFlowInfo.FromResponse(flow_info) - else: - error_msg = 'Invalid response {0}.'.format(resp.status) - try: - error_dict = json.loads(content) - if 'error' in error_dict: - error_msg += ' Error: {0}'.format(error_dict['error']) - except ValueError: - # Couldn't decode a JSON response, stick with the - # default message. - pass - raise OAuth2DeviceCodeError(error_msg) - - @_helpers.positional(2) - def step2_exchange(self, code=None, http=None, device_flow_info=None): - """Exchanges a code for OAuth2Credentials. - - Args: - code: string, a dict-like object, or None. For a non-device - flow, this is either the response code as a string, or a - dictionary of query parameters to the redirect_uri. For a - device flow, this should be None. - http: httplib2.Http, optional http instance to use when fetching - credentials. - device_flow_info: DeviceFlowInfo, return value from step1 in the - case of a device flow. - - Returns: - An OAuth2Credentials object that can be used to authorize requests. - - Raises: - FlowExchangeError: if a problem occurred exchanging the code for a - refresh_token. - ValueError: if code and device_flow_info are both provided or both - missing. - """ - if code is None and device_flow_info is None: - raise ValueError('No code or device_flow_info provided.') - if code is not None and device_flow_info is not None: - raise ValueError('Cannot provide both code and device_flow_info.') - - if code is None: - code = device_flow_info.device_code - elif not isinstance(code, (six.string_types, six.binary_type)): - if 'code' not in code: - raise FlowExchangeError(code.get( - 'error', 'No code was supplied in the query parameters.')) - code = code['code'] - - post_data = { - 'client_id': self.client_id, - 'code': code, - 'scope': self.scope, - } - if self.client_secret is not None: - post_data['client_secret'] = self.client_secret - if self._pkce: - post_data['code_verifier'] = self.code_verifier - if device_flow_info is not None: - post_data['grant_type'] = 'http://oauth.net/grant_type/device/1.0' - else: - post_data['grant_type'] = 'authorization_code' - post_data['redirect_uri'] = self.redirect_uri - body = urllib.parse.urlencode(post_data) - headers = { - 'content-type': 'application/x-www-form-urlencoded', - } - if self.authorization_header is not None: - headers['Authorization'] = self.authorization_header - if self.user_agent is not None: - headers['user-agent'] = self.user_agent - - if http is None: - http = transport.get_http_object() - - resp, content = transport.request( - http, self.token_uri, method='POST', body=body, headers=headers) - d = _parse_exchange_token_response(content) - if resp.status == http_client.OK and 'access_token' in d: - access_token = d['access_token'] - refresh_token = d.get('refresh_token', None) - if not refresh_token: - logger.info( - 'Received token response with no refresh_token. Consider ' - "reauthenticating with prompt='consent'.") - token_expiry = None - if 'expires_in' in d: - delta = datetime.timedelta(seconds=int(d['expires_in'])) - token_expiry = delta + _UTCNOW() - - extracted_id_token = None - id_token_jwt = None - if 'id_token' in d: - extracted_id_token = _extract_id_token(d['id_token']) - id_token_jwt = d['id_token'] - - logger.info('Successfully retrieved access token') - return OAuth2Credentials( - access_token, self.client_id, self.client_secret, - refresh_token, token_expiry, self.token_uri, self.user_agent, - revoke_uri=self.revoke_uri, id_token=extracted_id_token, - id_token_jwt=id_token_jwt, token_response=d, scopes=self.scope, - token_info_uri=self.token_info_uri) - else: - logger.info('Failed to retrieve access token: %s', content) - if 'error' in d: - # you never know what those providers got to say - error_msg = (str(d['error']) + - str(d.get('error_description', ''))) - else: - error_msg = 'Invalid response: {0}.'.format(str(resp.status)) - raise FlowExchangeError(error_msg) - - -@_helpers.positional(2) -def flow_from_clientsecrets(filename, scope, redirect_uri=None, - message=None, cache=None, login_hint=None, - device_uri=None, pkce=None, code_verifier=None, - prompt=None): - """Create a Flow from a clientsecrets file. - - Will create the right kind of Flow based on the contents of the - clientsecrets file or will raise InvalidClientSecretsError for unknown - types of Flows. - - Args: - filename: string, File name of client secrets. - scope: string or iterable of strings, scope(s) to request. - redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' for - a non-web-based application, or a URI that handles the - callback from the authorization server. - message: string, A friendly string to display to the user if the - clientsecrets file is missing or invalid. If message is - provided then sys.exit will be called in the case of an error. - If message in not provided then - clientsecrets.InvalidClientSecretsError will be raised. - cache: An optional cache service client that implements get() and set() - methods. See clientsecrets.loadfile() for details. - login_hint: string, Either an email address or domain. Passing this - hint will either pre-fill the email box on the sign-in form - or select the proper multi-login session, thereby - simplifying the login flow. - device_uri: string, URI for device authorization endpoint. For - convenience defaults to Google's endpoints but any - OAuth 2.0 provider can be used. - - Returns: - A Flow object. - - Raises: - UnknownClientSecretsFlowError: if the file describes an unknown kind of - Flow. - clientsecrets.InvalidClientSecretsError: if the clientsecrets file is - invalid. - """ - try: - client_type, client_info = clientsecrets.loadfile(filename, - cache=cache) - if client_type in (clientsecrets.TYPE_WEB, - clientsecrets.TYPE_INSTALLED): - constructor_kwargs = { - 'redirect_uri': redirect_uri, - 'auth_uri': client_info['auth_uri'], - 'token_uri': client_info['token_uri'], - 'login_hint': login_hint, - } - revoke_uri = client_info.get('revoke_uri') - optional = ( - 'revoke_uri', - 'device_uri', - 'pkce', - 'code_verifier', - 'prompt' - ) - for param in optional: - if locals()[param] is not None: - constructor_kwargs[param] = locals()[param] - - return OAuth2WebServerFlow( - client_info['client_id'], client_info['client_secret'], - scope, **constructor_kwargs) - - except clientsecrets.InvalidClientSecretsError as e: - if message is not None: - if e.args: - message = ('The client secrets were invalid: ' - '\n{0}\n{1}'.format(e, message)) - sys.exit(message) - else: - raise - else: - raise UnknownClientSecretsFlowError( - 'This OAuth 2.0 flow is unsupported: {0!r}'.format(client_type)) diff --git a/src/oauth2client/clientsecrets.py b/src/oauth2client/clientsecrets.py deleted file mode 100644 index 1598142e..00000000 --- a/src/oauth2client/clientsecrets.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for reading OAuth 2.0 client secret files. - -A client_secrets.json file contains all the information needed to interact with -an OAuth 2.0 protected service. -""" - -import json - -import six - - -# Properties that make a client_secrets.json file valid. -TYPE_WEB = 'web' -TYPE_INSTALLED = 'installed' - -VALID_CLIENT = { - TYPE_WEB: { - 'required': [ - 'client_id', - 'client_secret', - 'redirect_uris', - 'auth_uri', - 'token_uri', - ], - 'string': [ - 'client_id', - 'client_secret', - ], - }, - TYPE_INSTALLED: { - 'required': [ - 'client_id', - 'client_secret', - 'redirect_uris', - 'auth_uri', - 'token_uri', - ], - 'string': [ - 'client_id', - 'client_secret', - ], - }, -} - - -class Error(Exception): - """Base error for this module.""" - - -class InvalidClientSecretsError(Error): - """Format of ClientSecrets file is invalid.""" - - -def _validate_clientsecrets(clientsecrets_dict): - """Validate parsed client secrets from a file. - - Args: - clientsecrets_dict: dict, a dictionary holding the client secrets. - - Returns: - tuple, a string of the client type and the information parsed - from the file. - """ - _INVALID_FILE_FORMAT_MSG = ( - 'Invalid file format. See ' - 'https://developers.google.com/api-client-library/' - 'python/guide/aaa_client_secrets') - - if clientsecrets_dict is None: - raise InvalidClientSecretsError(_INVALID_FILE_FORMAT_MSG) - try: - (client_type, client_info), = clientsecrets_dict.items() - except (ValueError, AttributeError): - raise InvalidClientSecretsError( - _INVALID_FILE_FORMAT_MSG + ' ' - 'Expected a JSON object with a single property for a "web" or ' - '"installed" application') - - if client_type not in VALID_CLIENT: - raise InvalidClientSecretsError( - 'Unknown client type: {0}.'.format(client_type)) - - for prop_name in VALID_CLIENT[client_type]['required']: - if prop_name not in client_info: - raise InvalidClientSecretsError( - 'Missing property "{0}" in a client type of "{1}".'.format( - prop_name, client_type)) - for prop_name in VALID_CLIENT[client_type]['string']: - if client_info[prop_name].startswith('[['): - raise InvalidClientSecretsError( - 'Property "{0}" is not configured.'.format(prop_name)) - return client_type, client_info - - -def load(fp): - obj = json.load(fp) - return _validate_clientsecrets(obj) - - -def loads(s): - obj = json.loads(s) - return _validate_clientsecrets(obj) - - -def _loadfile(filename): - try: - with open(filename, 'r') as fp: - obj = json.load(fp) - except IOError as exc: - raise InvalidClientSecretsError('Error opening file', exc.filename, - exc.strerror, exc.errno) - return _validate_clientsecrets(obj) - - -def loadfile(filename, cache=None): - """Loading of client_secrets JSON file, optionally backed by a cache. - - Typical cache storage would be App Engine memcache service, - but you can pass in any other cache client that implements - these methods: - - * ``get(key, namespace=ns)`` - * ``set(key, value, namespace=ns)`` - - Usage:: - - # without caching - client_type, client_info = loadfile('secrets.json') - # using App Engine memcache service - from google.appengine.api import memcache - client_type, client_info = loadfile('secrets.json', cache=memcache) - - Args: - filename: string, Path to a client_secrets.json file on a filesystem. - cache: An optional cache service client that implements get() and set() - methods. If not specified, the file is always being loaded from - a filesystem. - - Raises: - InvalidClientSecretsError: In case of a validation error or some - I/O failure. Can happen only on cache miss. - - Returns: - (client_type, client_info) tuple, as _loadfile() normally would. - JSON contents is validated only during first load. Cache hits are not - validated. - """ - _SECRET_NAMESPACE = 'oauth2client:secrets#ns' - - if not cache: - return _loadfile(filename) - - obj = cache.get(filename, namespace=_SECRET_NAMESPACE) - if obj is None: - client_type, client_info = _loadfile(filename) - obj = {client_type: client_info} - cache.set(filename, obj, namespace=_SECRET_NAMESPACE) - - return next(six.iteritems(obj)) diff --git a/src/oauth2client/contrib/__init__.py b/src/oauth2client/contrib/__init__.py deleted file mode 100644 index ecfd06c9..00000000 --- a/src/oauth2client/contrib/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Contributed modules. - -Contrib contains modules that are not considered part of the core oauth2client -library but provide additional functionality. These modules are intended to -make it easier to use oauth2client. -""" diff --git a/src/oauth2client/contrib/_appengine_ndb.py b/src/oauth2client/contrib/_appengine_ndb.py deleted file mode 100644 index c863e8f4..00000000 --- a/src/oauth2client/contrib/_appengine_ndb.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google App Engine utilities helper. - -Classes that directly require App Engine's ndb library. Provided -as a separate module in case of failure to import ndb while -other App Engine libraries are present. -""" - -import logging - -from google.appengine.ext import ndb - -from oauth2client import client - - -NDB_KEY = ndb.Key -"""Key constant used by :mod:`oauth2client.contrib.appengine`.""" - -NDB_MODEL = ndb.Model -"""Model constant used by :mod:`oauth2client.contrib.appengine`.""" - -_LOGGER = logging.getLogger(__name__) - - -class SiteXsrfSecretKeyNDB(ndb.Model): - """NDB Model for storage for the sites XSRF secret key. - - Since this model uses the same kind as SiteXsrfSecretKey, it can be - used interchangeably. This simply provides an NDB model for interacting - with the same data the DB model interacts with. - - There should only be one instance stored of this model, the one used - for the site. - """ - secret = ndb.StringProperty() - - @classmethod - def _get_kind(cls): - """Return the kind name for this class.""" - return 'SiteXsrfSecretKey' - - -class FlowNDBProperty(ndb.PickleProperty): - """App Engine NDB datastore Property for Flow. - - Serves the same purpose as the DB FlowProperty, but for NDB models. - Since PickleProperty inherits from BlobProperty, the underlying - representation of the data in the datastore will be the same as in the - DB case. - - Utility property that allows easy storage and retrieval of an - oauth2client.Flow - """ - - def _validate(self, value): - """Validates a value as a proper Flow object. - - Args: - value: A value to be set on the property. - - Raises: - TypeError if the value is not an instance of Flow. - """ - _LOGGER.info('validate: Got type %s', type(value)) - if value is not None and not isinstance(value, client.Flow): - raise TypeError( - 'Property {0} must be convertible to a flow ' - 'instance; received: {1}.'.format(self._name, value)) - - -class CredentialsNDBProperty(ndb.BlobProperty): - """App Engine NDB datastore Property for Credentials. - - Serves the same purpose as the DB CredentialsProperty, but for NDB - models. Since CredentialsProperty stores data as a blob and this - inherits from BlobProperty, the data in the datastore will be the same - as in the DB case. - - Utility property that allows easy storage and retrieval of Credentials - and subclasses. - """ - - def _validate(self, value): - """Validates a value as a proper credentials object. - - Args: - value: A value to be set on the property. - - Raises: - TypeError if the value is not an instance of Credentials. - """ - _LOGGER.info('validate: Got type %s', type(value)) - if value is not None and not isinstance(value, client.Credentials): - raise TypeError( - 'Property {0} must be convertible to a credentials ' - 'instance; received: {1}.'.format(self._name, value)) - - def _to_base_type(self, value): - """Converts our validated value to a JSON serialized string. - - Args: - value: A value to be set in the datastore. - - Returns: - A JSON serialized version of the credential, else '' if value - is None. - """ - if value is None: - return '' - else: - return value.to_json() - - def _from_base_type(self, value): - """Converts our stored JSON string back to the desired type. - - Args: - value: A value from the datastore to be converted to the - desired type. - - Returns: - A deserialized Credentials (or subclass) object, else None if - the value can't be parsed. - """ - if not value: - return None - try: - # Uses the from_json method of the implied class of value - credentials = client.Credentials.new_from_json(value) - except ValueError: - credentials = None - return credentials - - -class CredentialsNDBModel(ndb.Model): - """NDB Model for storage of OAuth 2.0 Credentials - - Since this model uses the same kind as CredentialsModel and has a - property which can serialize and deserialize Credentials correctly, it - can be used interchangeably with a CredentialsModel to access, insert - and delete the same entities. This simply provides an NDB model for - interacting with the same data the DB model interacts with. - - Storage of the model is keyed by the user.user_id(). - """ - credentials = CredentialsNDBProperty() - - @classmethod - def _get_kind(cls): - """Return the kind name for this class.""" - return 'CredentialsModel' diff --git a/src/oauth2client/contrib/_metadata.py b/src/oauth2client/contrib/_metadata.py deleted file mode 100644 index 564cd398..00000000 --- a/src/oauth2client/contrib/_metadata.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides helper methods for talking to the Compute Engine metadata server. - -See https://cloud.google.com/compute/docs/metadata -""" - -import datetime -import json -import os - -from six.moves import http_client -from six.moves.urllib import parse as urlparse - -from oauth2client import _helpers -from oauth2client import client -from oauth2client import transport - - -METADATA_ROOT = 'http://{}/computeMetadata/v1/'.format( - os.getenv('GCE_METADATA_ROOT', 'metadata.google.internal')) -METADATA_HEADERS = {'Metadata-Flavor': 'Google'} - - -def get(http, path, root=METADATA_ROOT, recursive=None): - """Fetch a resource from the metadata server. - - Args: - http: an object to be used to make HTTP requests. - path: A string indicating the resource to retrieve. For example, - 'instance/service-accounts/default' - root: A string indicating the full path to the metadata server root. - recursive: A boolean indicating whether to do a recursive query of - metadata. See - https://cloud.google.com/compute/docs/metadata#aggcontents - - Returns: - A dictionary if the metadata server returns JSON, otherwise a string. - - Raises: - http_client.HTTPException if an error corrured while - retrieving metadata. - """ - url = urlparse.urljoin(root, path) - url = _helpers._add_query_parameter(url, 'recursive', recursive) - - response, content = transport.request( - http, url, headers=METADATA_HEADERS) - - if response.status == http_client.OK: - decoded = _helpers._from_bytes(content) - if response['content-type'] == 'application/json': - return json.loads(decoded) - else: - return decoded - else: - raise http_client.HTTPException( - 'Failed to retrieve {0} from the Google Compute Engine' - 'metadata service. Response:\n{1}'.format(url, response)) - - -def get_service_account_info(http, service_account='default'): - """Get information about a service account from the metadata server. - - Args: - http: an object to be used to make HTTP requests. - service_account: An email specifying the service account for which to - look up information. Default will be information for the "default" - service account of the current compute engine instance. - - Returns: - A dictionary with information about the specified service account, - for example: - - { - 'email': '...', - 'scopes': ['scope', ...], - 'aliases': ['default', '...'] - } - """ - return get( - http, - 'instance/service-accounts/{0}/'.format(service_account), - recursive=True) - - -def get_token(http, service_account='default'): - """Fetch an oauth token for the - - Args: - http: an object to be used to make HTTP requests. - service_account: An email specifying the service account this token - should represent. Default will be a token for the "default" service - account of the current compute engine instance. - - Returns: - A tuple of (access token, token expiration), where access token is the - access token as a string and token expiration is a datetime object - that indicates when the access token will expire. - """ - token_json = get( - http, - 'instance/service-accounts/{0}/token'.format(service_account)) - token_expiry = client._UTCNOW() + datetime.timedelta( - seconds=token_json['expires_in']) - return token_json['access_token'], token_expiry diff --git a/src/oauth2client/contrib/appengine.py b/src/oauth2client/contrib/appengine.py deleted file mode 100644 index c1326eeb..00000000 --- a/src/oauth2client/contrib/appengine.py +++ /dev/null @@ -1,910 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for Google App Engine - -Utilities for making it easier to use OAuth 2.0 on Google App Engine. -""" - -import cgi -import json -import logging -import os -import pickle -import threading - -from google.appengine.api import app_identity -from google.appengine.api import memcache -from google.appengine.api import users -from google.appengine.ext import db -from google.appengine.ext.webapp.util import login_required -import webapp2 as webapp - -import oauth2client -from oauth2client import _helpers -from oauth2client import client -from oauth2client import clientsecrets -from oauth2client import transport -from oauth2client.contrib import xsrfutil - -# This is a temporary fix for a Google internal issue. -try: - from oauth2client.contrib import _appengine_ndb -except ImportError: # pragma: NO COVER - _appengine_ndb = None - - -logger = logging.getLogger(__name__) - -OAUTH2CLIENT_NAMESPACE = 'oauth2client#ns' - -XSRF_MEMCACHE_ID = 'xsrf_secret_key' - -if _appengine_ndb is None: # pragma: NO COVER - CredentialsNDBModel = None - CredentialsNDBProperty = None - FlowNDBProperty = None - _NDB_KEY = None - _NDB_MODEL = None - SiteXsrfSecretKeyNDB = None -else: - CredentialsNDBModel = _appengine_ndb.CredentialsNDBModel - CredentialsNDBProperty = _appengine_ndb.CredentialsNDBProperty - FlowNDBProperty = _appengine_ndb.FlowNDBProperty - _NDB_KEY = _appengine_ndb.NDB_KEY - _NDB_MODEL = _appengine_ndb.NDB_MODEL - SiteXsrfSecretKeyNDB = _appengine_ndb.SiteXsrfSecretKeyNDB - - -def _safe_html(s): - """Escape text to make it safe to display. - - Args: - s: string, The text to escape. - - Returns: - The escaped text as a string. - """ - return cgi.escape(s, quote=1).replace("'", ''') - - -class SiteXsrfSecretKey(db.Model): - """Storage for the sites XSRF secret key. - - There will only be one instance stored of this model, the one used for the - site. - """ - secret = db.StringProperty() - - -def _generate_new_xsrf_secret_key(): - """Returns a random XSRF secret key.""" - return os.urandom(16).encode("hex") - - -def xsrf_secret_key(): - """Return the secret key for use for XSRF protection. - - If the Site entity does not have a secret key, this method will also create - one and persist it. - - Returns: - The secret key. - """ - secret = memcache.get(XSRF_MEMCACHE_ID, namespace=OAUTH2CLIENT_NAMESPACE) - if not secret: - # Load the one and only instance of SiteXsrfSecretKey. - model = SiteXsrfSecretKey.get_or_insert(key_name='site') - if not model.secret: - model.secret = _generate_new_xsrf_secret_key() - model.put() - secret = model.secret - memcache.add(XSRF_MEMCACHE_ID, secret, - namespace=OAUTH2CLIENT_NAMESPACE) - - return str(secret) - - -class AppAssertionCredentials(client.AssertionCredentials): - """Credentials object for App Engine Assertion Grants - - This object will allow an App Engine application to identify itself to - Google and other OAuth 2.0 servers that can verify assertions. It can be - used for the purpose of accessing data stored under an account assigned to - the App Engine application itself. - - This credential does not require a flow to instantiate because it - represents a two legged flow, and therefore has all of the required - information to generate and refresh its own access tokens. - """ - - @_helpers.positional(2) - def __init__(self, scope, **kwargs): - """Constructor for AppAssertionCredentials - - Args: - scope: string or iterable of strings, scope(s) of the credentials - being requested. - **kwargs: optional keyword args, including: - service_account_id: service account id of the application. If None - or unspecified, the default service account for - the app is used. - """ - self.scope = _helpers.scopes_to_string(scope) - self._kwargs = kwargs - self.service_account_id = kwargs.get('service_account_id', None) - self._service_account_email = None - - # Assertion type is no longer used, but still in the - # parent class signature. - super(AppAssertionCredentials, self).__init__(None) - - @classmethod - def from_json(cls, json_data): - data = json.loads(json_data) - return AppAssertionCredentials(data['scope']) - - def _refresh(self, http): - """Refreshes the access token. - - Since the underlying App Engine app_identity implementation does its - own caching we can skip all the storage hoops and just to a refresh - using the API. - - Args: - http: unused HTTP object - - Raises: - AccessTokenRefreshError: When the refresh fails. - """ - try: - scopes = self.scope.split() - (token, _) = app_identity.get_access_token( - scopes, service_account_id=self.service_account_id) - except app_identity.Error as e: - raise client.AccessTokenRefreshError(str(e)) - self.access_token = token - - @property - def serialization_data(self): - raise NotImplementedError('Cannot serialize credentials ' - 'for Google App Engine.') - - def create_scoped_required(self): - return not self.scope - - def create_scoped(self, scopes): - return AppAssertionCredentials(scopes, **self._kwargs) - - def sign_blob(self, blob): - """Cryptographically sign a blob (of bytes). - - Implements abstract method - :meth:`oauth2client.client.AssertionCredentials.sign_blob`. - - Args: - blob: bytes, Message to be signed. - - Returns: - tuple, A pair of the private key ID used to sign the blob and - the signed contents. - """ - return app_identity.sign_blob(blob) - - @property - def service_account_email(self): - """Get the email for the current service account. - - Returns: - string, The email associated with the Google App Engine - service account. - """ - if self._service_account_email is None: - self._service_account_email = ( - app_identity.get_service_account_name()) - return self._service_account_email - - -class FlowProperty(db.Property): - """App Engine datastore Property for Flow. - - Utility property that allows easy storage and retrieval of an - oauth2client.Flow - """ - - # Tell what the user type is. - data_type = client.Flow - - # For writing to datastore. - def get_value_for_datastore(self, model_instance): - flow = super(FlowProperty, self).get_value_for_datastore( - model_instance) - return db.Blob(pickle.dumps(flow)) - - # For reading from datastore. - def make_value_from_datastore(self, value): - if value is None: - return None - return pickle.loads(value) - - def validate(self, value): - if value is not None and not isinstance(value, client.Flow): - raise db.BadValueError( - 'Property {0} must be convertible ' - 'to a FlowThreeLegged instance ({1})'.format(self.name, value)) - return super(FlowProperty, self).validate(value) - - def empty(self, value): - return not value - - -class CredentialsProperty(db.Property): - """App Engine datastore Property for Credentials. - - Utility property that allows easy storage and retrieval of - oauth2client.Credentials - """ - - # Tell what the user type is. - data_type = client.Credentials - - # For writing to datastore. - def get_value_for_datastore(self, model_instance): - logger.info("get: Got type " + str(type(model_instance))) - cred = super(CredentialsProperty, self).get_value_for_datastore( - model_instance) - if cred is None: - cred = '' - else: - cred = cred.to_json() - return db.Blob(cred) - - # For reading from datastore. - def make_value_from_datastore(self, value): - logger.info("make: Got type " + str(type(value))) - if value is None: - return None - if len(value) == 0: - return None - try: - credentials = client.Credentials.new_from_json(value) - except ValueError: - credentials = None - return credentials - - def validate(self, value): - value = super(CredentialsProperty, self).validate(value) - logger.info("validate: Got type " + str(type(value))) - if value is not None and not isinstance(value, client.Credentials): - raise db.BadValueError( - 'Property {0} must be convertible ' - 'to a Credentials instance ({1})'.format(self.name, value)) - return value - - -class StorageByKeyName(client.Storage): - """Store and retrieve a credential to and from the App Engine datastore. - - This Storage helper presumes the Credentials have been stored as a - CredentialsProperty or CredentialsNDBProperty on a datastore model class, - and that entities are stored by key_name. - """ - - @_helpers.positional(4) - def __init__(self, model, key_name, property_name, cache=None, user=None): - """Constructor for Storage. - - Args: - model: db.Model or ndb.Model, model class - key_name: string, key name for the entity that has the credentials - property_name: string, name of the property that is a - CredentialsProperty or CredentialsNDBProperty. - cache: memcache, a write-through cache to put in front of the - datastore. If the model you are using is an NDB model, using - a cache will be redundant since the model uses an instance - cache and memcache for you. - user: users.User object, optional. Can be used to grab user ID as a - key_name if no key name is specified. - """ - super(StorageByKeyName, self).__init__() - - if key_name is None: - if user is None: - raise ValueError('StorageByKeyName called with no ' - 'key name or user.') - key_name = user.user_id() - - self._model = model - self._key_name = key_name - self._property_name = property_name - self._cache = cache - - def _is_ndb(self): - """Determine whether the model of the instance is an NDB model. - - Returns: - Boolean indicating whether or not the model is an NDB or DB model. - """ - # issubclass will fail if one of the arguments is not a class, only - # need worry about new-style classes since ndb and db models are - # new-style - if isinstance(self._model, type): - if _NDB_MODEL is not None and issubclass(self._model, _NDB_MODEL): - return True - elif issubclass(self._model, db.Model): - return False - - raise TypeError( - 'Model class not an NDB or DB model: {0}.'.format(self._model)) - - def _get_entity(self): - """Retrieve entity from datastore. - - Uses a different model method for db or ndb models. - - Returns: - Instance of the model corresponding to the current storage object - and stored using the key name of the storage object. - """ - if self._is_ndb(): - return self._model.get_by_id(self._key_name) - else: - return self._model.get_by_key_name(self._key_name) - - def _delete_entity(self): - """Delete entity from datastore. - - Attempts to delete using the key_name stored on the object, whether or - not the given key is in the datastore. - """ - if self._is_ndb(): - _NDB_KEY(self._model, self._key_name).delete() - else: - entity_key = db.Key.from_path(self._model.kind(), self._key_name) - db.delete(entity_key) - - @db.non_transactional(allow_existing=True) - def locked_get(self): - """Retrieve Credential from datastore. - - Returns: - oauth2client.Credentials - """ - credentials = None - if self._cache: - json = self._cache.get(self._key_name) - if json: - credentials = client.Credentials.new_from_json(json) - if credentials is None: - entity = self._get_entity() - if entity is not None: - credentials = getattr(entity, self._property_name) - if self._cache: - self._cache.set(self._key_name, credentials.to_json()) - - if credentials and hasattr(credentials, 'set_store'): - credentials.set_store(self) - return credentials - - @db.non_transactional(allow_existing=True) - def locked_put(self, credentials): - """Write a Credentials to the datastore. - - Args: - credentials: Credentials, the credentials to store. - """ - entity = self._model.get_or_insert(self._key_name) - setattr(entity, self._property_name, credentials) - entity.put() - if self._cache: - self._cache.set(self._key_name, credentials.to_json()) - - @db.non_transactional(allow_existing=True) - def locked_delete(self): - """Delete Credential from datastore.""" - - if self._cache: - self._cache.delete(self._key_name) - - self._delete_entity() - - -class CredentialsModel(db.Model): - """Storage for OAuth 2.0 Credentials - - Storage of the model is keyed by the user.user_id(). - """ - credentials = CredentialsProperty() - - -def _build_state_value(request_handler, user): - """Composes the value for the 'state' parameter. - - Packs the current request URI and an XSRF token into an opaque string that - can be passed to the authentication server via the 'state' parameter. - - Args: - request_handler: webapp.RequestHandler, The request. - user: google.appengine.api.users.User, The current user. - - Returns: - The state value as a string. - """ - uri = request_handler.request.url - token = xsrfutil.generate_token(xsrf_secret_key(), user.user_id(), - action_id=str(uri)) - return uri + ':' + token - - -def _parse_state_value(state, user): - """Parse the value of the 'state' parameter. - - Parses the value and validates the XSRF token in the state parameter. - - Args: - state: string, The value of the state parameter. - user: google.appengine.api.users.User, The current user. - - Returns: - The redirect URI, or None if XSRF token is not valid. - """ - uri, token = state.rsplit(':', 1) - if xsrfutil.validate_token(xsrf_secret_key(), token, user.user_id(), - action_id=uri): - return uri - else: - return None - - -class OAuth2Decorator(object): - """Utility for making OAuth 2.0 easier. - - Instantiate and then use with oauth_required or oauth_aware - as decorators on webapp.RequestHandler methods. - - :: - - decorator = OAuth2Decorator( - client_id='837...ent.com', - client_secret='Qh...wwI', - scope='https://www.googleapis.com/auth/plus') - - class MainHandler(webapp.RequestHandler): - @decorator.oauth_required - def get(self): - http = decorator.http() - # http is authorized with the user's Credentials and can be - # used in API calls - - """ - - def set_credentials(self, credentials): - self._tls.credentials = credentials - - def get_credentials(self): - """A thread local Credentials object. - - Returns: - A client.Credentials object, or None if credentials hasn't been set - in this thread yet, which may happen when calling has_credentials - inside oauth_aware. - """ - return getattr(self._tls, 'credentials', None) - - credentials = property(get_credentials, set_credentials) - - def set_flow(self, flow): - self._tls.flow = flow - - def get_flow(self): - """A thread local Flow object. - - Returns: - A credentials.Flow object, or None if the flow hasn't been set in - this thread yet, which happens in _create_flow() since Flows are - created lazily. - """ - return getattr(self._tls, 'flow', None) - - flow = property(get_flow, set_flow) - - @_helpers.positional(4) - def __init__(self, client_id, client_secret, scope, - auth_uri=oauth2client.GOOGLE_AUTH_URI, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI, - user_agent=None, - message=None, - callback_path='/oauth2callback', - token_response_param=None, - _storage_class=StorageByKeyName, - _credentials_class=CredentialsModel, - _credentials_property_name='credentials', - **kwargs): - """Constructor for OAuth2Decorator - - Args: - client_id: string, client identifier. - client_secret: string client secret. - scope: string or iterable of strings, scope(s) of the credentials - being requested. - auth_uri: string, URI for authorization endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 provider - can be used. - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - revoke_uri: string, URI for revoke endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 - provider can be used. - user_agent: string, User agent of your application, default to - None. - message: Message to display if there are problems with the - OAuth 2.0 configuration. The message may contain HTML and - will be presented on the web interface for any method that - uses the decorator. - callback_path: string, The absolute path to use as the callback - URI. Note that this must match up with the URI given - when registering the application in the APIs - Console. - token_response_param: string. If provided, the full JSON response - to the access token request will be encoded - and included in this query parameter in the - callback URI. This is useful with providers - (e.g. wordpress.com) that include extra - fields that the client may want. - _storage_class: "Protected" keyword argument not typically provided - to this constructor. A storage class to aid in - storing a Credentials object for a user in the - datastore. Defaults to StorageByKeyName. - _credentials_class: "Protected" keyword argument not typically - provided to this constructor. A db or ndb Model - class to hold credentials. Defaults to - CredentialsModel. - _credentials_property_name: "Protected" keyword argument not - typically provided to this constructor. - A string indicating the name of the - field on the _credentials_class where a - Credentials object will be stored. - Defaults to 'credentials'. - **kwargs: dict, Keyword arguments are passed along as kwargs to - the OAuth2WebServerFlow constructor. - """ - self._tls = threading.local() - self.flow = None - self.credentials = None - self._client_id = client_id - self._client_secret = client_secret - self._scope = _helpers.scopes_to_string(scope) - self._auth_uri = auth_uri - self._token_uri = token_uri - self._revoke_uri = revoke_uri - self._user_agent = user_agent - self._kwargs = kwargs - self._message = message - self._in_error = False - self._callback_path = callback_path - self._token_response_param = token_response_param - self._storage_class = _storage_class - self._credentials_class = _credentials_class - self._credentials_property_name = _credentials_property_name - - def _display_error_message(self, request_handler): - request_handler.response.out.write('') - request_handler.response.out.write(_safe_html(self._message)) - request_handler.response.out.write('') - - def oauth_required(self, method): - """Decorator that starts the OAuth 2.0 dance. - - Starts the OAuth dance for the logged in user if they haven't already - granted access for this application. - - Args: - method: callable, to be decorated method of a webapp.RequestHandler - instance. - """ - - def check_oauth(request_handler, *args, **kwargs): - if self._in_error: - self._display_error_message(request_handler) - return - - user = users.get_current_user() - # Don't use @login_decorator as this could be used in a - # POST request. - if not user: - request_handler.redirect(users.create_login_url( - request_handler.request.uri)) - return - - self._create_flow(request_handler) - - # Store the request URI in 'state' so we can use it later - self.flow.params['state'] = _build_state_value( - request_handler, user) - self.credentials = self._storage_class( - self._credentials_class, None, - self._credentials_property_name, user=user).get() - - if not self.has_credentials(): - return request_handler.redirect(self.authorize_url()) - try: - resp = method(request_handler, *args, **kwargs) - except client.AccessTokenRefreshError: - return request_handler.redirect(self.authorize_url()) - finally: - self.credentials = None - return resp - - return check_oauth - - def _create_flow(self, request_handler): - """Create the Flow object. - - The Flow is calculated lazily since we don't know where this app is - running until it receives a request, at which point redirect_uri can be - calculated and then the Flow object can be constructed. - - Args: - request_handler: webapp.RequestHandler, the request handler. - """ - if self.flow is None: - redirect_uri = request_handler.request.relative_url( - self._callback_path) # Usually /oauth2callback - self.flow = client.OAuth2WebServerFlow( - self._client_id, self._client_secret, self._scope, - redirect_uri=redirect_uri, user_agent=self._user_agent, - auth_uri=self._auth_uri, token_uri=self._token_uri, - revoke_uri=self._revoke_uri, **self._kwargs) - - def oauth_aware(self, method): - """Decorator that sets up for OAuth 2.0 dance, but doesn't do it. - - Does all the setup for the OAuth dance, but doesn't initiate it. - This decorator is useful if you want to create a page that knows - whether or not the user has granted access to this application. - From within a method decorated with @oauth_aware the has_credentials() - and authorize_url() methods can be called. - - Args: - method: callable, to be decorated method of a webapp.RequestHandler - instance. - """ - - def setup_oauth(request_handler, *args, **kwargs): - if self._in_error: - self._display_error_message(request_handler) - return - - user = users.get_current_user() - # Don't use @login_decorator as this could be used in a - # POST request. - if not user: - request_handler.redirect(users.create_login_url( - request_handler.request.uri)) - return - - self._create_flow(request_handler) - - self.flow.params['state'] = _build_state_value(request_handler, - user) - self.credentials = self._storage_class( - self._credentials_class, None, - self._credentials_property_name, user=user).get() - try: - resp = method(request_handler, *args, **kwargs) - finally: - self.credentials = None - return resp - return setup_oauth - - def has_credentials(self): - """True if for the logged in user there are valid access Credentials. - - Must only be called from with a webapp.RequestHandler subclassed method - that had been decorated with either @oauth_required or @oauth_aware. - """ - return self.credentials is not None and not self.credentials.invalid - - def authorize_url(self): - """Returns the URL to start the OAuth dance. - - Must only be called from with a webapp.RequestHandler subclassed method - that had been decorated with either @oauth_required or @oauth_aware. - """ - url = self.flow.step1_get_authorize_url() - return str(url) - - def http(self, *args, **kwargs): - """Returns an authorized http instance. - - Must only be called from within an @oauth_required decorated method, or - from within an @oauth_aware decorated method where has_credentials() - returns True. - - Args: - *args: Positional arguments passed to httplib2.Http constructor. - **kwargs: Positional arguments passed to httplib2.Http constructor. - """ - return self.credentials.authorize( - transport.get_http_object(*args, **kwargs)) - - @property - def callback_path(self): - """The absolute path where the callback will occur. - - Note this is the absolute path, not the absolute URI, that will be - calculated by the decorator at runtime. See callback_handler() for how - this should be used. - - Returns: - The callback path as a string. - """ - return self._callback_path - - def callback_handler(self): - """RequestHandler for the OAuth 2.0 redirect callback. - - Usage:: - - app = webapp.WSGIApplication([ - ('/index', MyIndexHandler), - ..., - (decorator.callback_path, decorator.callback_handler()) - ]) - - Returns: - A webapp.RequestHandler that handles the redirect back from the - server during the OAuth 2.0 dance. - """ - decorator = self - - class OAuth2Handler(webapp.RequestHandler): - """Handler for the redirect_uri of the OAuth 2.0 dance.""" - - @login_required - def get(self): - error = self.request.get('error') - if error: - errormsg = self.request.get('error_description', error) - self.response.out.write( - 'The authorization request failed: {0}'.format( - _safe_html(errormsg))) - else: - user = users.get_current_user() - decorator._create_flow(self) - credentials = decorator.flow.step2_exchange( - self.request.params) - decorator._storage_class( - decorator._credentials_class, None, - decorator._credentials_property_name, - user=user).put(credentials) - redirect_uri = _parse_state_value( - str(self.request.get('state')), user) - if redirect_uri is None: - self.response.out.write( - 'The authorization request failed') - return - - if (decorator._token_response_param and - credentials.token_response): - resp_json = json.dumps(credentials.token_response) - redirect_uri = _helpers._add_query_parameter( - redirect_uri, decorator._token_response_param, - resp_json) - - self.redirect(redirect_uri) - - return OAuth2Handler - - def callback_application(self): - """WSGI application for handling the OAuth 2.0 redirect callback. - - If you need finer grained control use `callback_handler` which returns - just the webapp.RequestHandler. - - Returns: - A webapp.WSGIApplication that handles the redirect back from the - server during the OAuth 2.0 dance. - """ - return webapp.WSGIApplication([ - (self.callback_path, self.callback_handler()) - ]) - - -class OAuth2DecoratorFromClientSecrets(OAuth2Decorator): - """An OAuth2Decorator that builds from a clientsecrets file. - - Uses a clientsecrets file as the source for all the information when - constructing an OAuth2Decorator. - - :: - - decorator = OAuth2DecoratorFromClientSecrets( - os.path.join(os.path.dirname(__file__), 'client_secrets.json') - scope='https://www.googleapis.com/auth/plus') - - class MainHandler(webapp.RequestHandler): - @decorator.oauth_required - def get(self): - http = decorator.http() - # http is authorized with the user's Credentials and can be - # used in API calls - - """ - - @_helpers.positional(3) - def __init__(self, filename, scope, message=None, cache=None, **kwargs): - """Constructor - - Args: - filename: string, File name of client secrets. - scope: string or iterable of strings, scope(s) of the credentials - being requested. - message: string, A friendly string to display to the user if the - clientsecrets file is missing or invalid. The message may - contain HTML and will be presented on the web interface - for any method that uses the decorator. - cache: An optional cache service client that implements get() and - set() - methods. See clientsecrets.loadfile() for details. - **kwargs: dict, Keyword arguments are passed along as kwargs to - the OAuth2WebServerFlow constructor. - """ - client_type, client_info = clientsecrets.loadfile(filename, - cache=cache) - if client_type not in (clientsecrets.TYPE_WEB, - clientsecrets.TYPE_INSTALLED): - raise clientsecrets.InvalidClientSecretsError( - "OAuth2Decorator doesn't support this OAuth 2.0 flow.") - - constructor_kwargs = dict(kwargs) - constructor_kwargs.update({ - 'auth_uri': client_info['auth_uri'], - 'token_uri': client_info['token_uri'], - 'message': message, - }) - revoke_uri = client_info.get('revoke_uri') - if revoke_uri is not None: - constructor_kwargs['revoke_uri'] = revoke_uri - super(OAuth2DecoratorFromClientSecrets, self).__init__( - client_info['client_id'], client_info['client_secret'], - scope, **constructor_kwargs) - if message is not None: - self._message = message - else: - self._message = 'Please configure your application for OAuth 2.0.' - - -@_helpers.positional(2) -def oauth2decorator_from_clientsecrets(filename, scope, - message=None, cache=None): - """Creates an OAuth2Decorator populated from a clientsecrets file. - - Args: - filename: string, File name of client secrets. - scope: string or list of strings, scope(s) of the credentials being - requested. - message: string, A friendly string to display to the user if the - clientsecrets file is missing or invalid. The message may - contain HTML and will be presented on the web interface for - any method that uses the decorator. - cache: An optional cache service client that implements get() and set() - methods. See clientsecrets.loadfile() for details. - - Returns: An OAuth2Decorator - """ - return OAuth2DecoratorFromClientSecrets(filename, scope, - message=message, cache=cache) diff --git a/src/oauth2client/contrib/devshell.py b/src/oauth2client/contrib/devshell.py deleted file mode 100644 index 691765f0..00000000 --- a/src/oauth2client/contrib/devshell.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2015 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""OAuth 2.0 utitilies for Google Developer Shell environment.""" - -import datetime -import json -import os -import socket - -from oauth2client import _helpers -from oauth2client import client - -DEVSHELL_ENV = 'DEVSHELL_CLIENT_PORT' - - -class Error(Exception): - """Errors for this module.""" - pass - - -class CommunicationError(Error): - """Errors for communication with the Developer Shell server.""" - - -class NoDevshellServer(Error): - """Error when no Developer Shell server can be contacted.""" - - -# The request for credential information to the Developer Shell client socket -# is always an empty PBLite-formatted JSON object, so just define it as a -# constant. -CREDENTIAL_INFO_REQUEST_JSON = '[]' - - -class CredentialInfoResponse(object): - """Credential information response from Developer Shell server. - - The credential information response from Developer Shell socket is a - PBLite-formatted JSON array with fields encoded by their index in the - array: - - * Index 0 - user email - * Index 1 - default project ID. None if the project context is not known. - * Index 2 - OAuth2 access token. None if there is no valid auth context. - * Index 3 - Seconds until the access token expires. None if not present. - """ - - def __init__(self, json_string): - """Initialize the response data from JSON PBLite array.""" - pbl = json.loads(json_string) - if not isinstance(pbl, list): - raise ValueError('Not a list: ' + str(pbl)) - pbl_len = len(pbl) - self.user_email = pbl[0] if pbl_len > 0 else None - self.project_id = pbl[1] if pbl_len > 1 else None - self.access_token = pbl[2] if pbl_len > 2 else None - self.expires_in = pbl[3] if pbl_len > 3 else None - - -def _SendRecv(): - """Communicate with the Developer Shell server socket.""" - - port = int(os.getenv(DEVSHELL_ENV, 0)) - if port == 0: - raise NoDevshellServer() - - sock = socket.socket() - sock.connect(('localhost', port)) - - data = CREDENTIAL_INFO_REQUEST_JSON - msg = '{0}\n{1}'.format(len(data), data) - sock.sendall(_helpers._to_bytes(msg, encoding='utf-8')) - - header = sock.recv(6).decode() - if '\n' not in header: - raise CommunicationError('saw no newline in the first 6 bytes') - len_str, json_str = header.split('\n', 1) - to_read = int(len_str) - len(json_str) - if to_read > 0: - json_str += sock.recv(to_read, socket.MSG_WAITALL).decode() - - return CredentialInfoResponse(json_str) - - -class DevshellCredentials(client.GoogleCredentials): - """Credentials object for Google Developer Shell environment. - - This object will allow a Google Developer Shell session to identify its - user to Google and other OAuth 2.0 servers that can verify assertions. It - can be used for the purpose of accessing data stored under the user - account. - - This credential does not require a flow to instantiate because it - represents a two legged flow, and therefore has all of the required - information to generate and refresh its own access tokens. - """ - - def __init__(self, user_agent=None): - super(DevshellCredentials, self).__init__( - None, # access_token, initialized below - None, # client_id - None, # client_secret - None, # refresh_token - None, # token_expiry - None, # token_uri - user_agent) - self._refresh(None) - - def _refresh(self, http): - """Refreshes the access token. - - Args: - http: unused HTTP object - """ - self.devshell_response = _SendRecv() - self.access_token = self.devshell_response.access_token - expires_in = self.devshell_response.expires_in - if expires_in is not None: - delta = datetime.timedelta(seconds=expires_in) - self.token_expiry = client._UTCNOW() + delta - else: - self.token_expiry = None - - @property - def user_email(self): - return self.devshell_response.user_email - - @property - def project_id(self): - return self.devshell_response.project_id - - @classmethod - def from_json(cls, json_data): - raise NotImplementedError( - 'Cannot load Developer Shell credentials from JSON.') - - @property - def serialization_data(self): - raise NotImplementedError( - 'Cannot serialize Developer Shell credentials.') diff --git a/src/oauth2client/contrib/dictionary_storage.py b/src/oauth2client/contrib/dictionary_storage.py deleted file mode 100644 index 6ee333fa..00000000 --- a/src/oauth2client/contrib/dictionary_storage.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Dictionary storage for OAuth2 Credentials.""" - -from oauth2client import client - - -class DictionaryStorage(client.Storage): - """Store and retrieve credentials to and from a dictionary-like object. - - Args: - dictionary: A dictionary or dictionary-like object. - key: A string or other hashable. The credentials will be stored in - ``dictionary[key]``. - lock: An optional threading.Lock-like object. The lock will be - acquired before anything is written or read from the - dictionary. - """ - - def __init__(self, dictionary, key, lock=None): - """Construct a DictionaryStorage instance.""" - super(DictionaryStorage, self).__init__(lock=lock) - self._dictionary = dictionary - self._key = key - - def locked_get(self): - """Retrieve the credentials from the dictionary, if they exist. - - Returns: A :class:`oauth2client.client.OAuth2Credentials` instance. - """ - serialized = self._dictionary.get(self._key) - - if serialized is None: - return None - - credentials = client.OAuth2Credentials.from_json(serialized) - credentials.set_store(self) - - return credentials - - def locked_put(self, credentials): - """Save the credentials to the dictionary. - - Args: - credentials: A :class:`oauth2client.client.OAuth2Credentials` - instance. - """ - serialized = credentials.to_json() - self._dictionary[self._key] = serialized - - def locked_delete(self): - """Remove the credentials from the dictionary, if they exist.""" - self._dictionary.pop(self._key, None) diff --git a/src/oauth2client/contrib/django_util/__init__.py b/src/oauth2client/contrib/django_util/__init__.py deleted file mode 100644 index 644a8f9f..00000000 --- a/src/oauth2client/contrib/django_util/__init__.py +++ /dev/null @@ -1,489 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for the Django web framework. - -Provides Django views and helpers the make using the OAuth2 web server -flow easier. It includes an ``oauth_required`` decorator to automatically -ensure that user credentials are available, and an ``oauth_enabled`` decorator -to check if the user has authorized, and helper shortcuts to create the -authorization URL otherwise. - -There are two basic use cases supported. The first is using Google OAuth as the -primary form of authentication, which is the simpler approach recommended -for applications without their own user system. - -The second use case is adding Google OAuth credentials to an -existing Django model containing a Django user field. Most of the -configuration is the same, except for `GOOGLE_OAUTH_MODEL_STORAGE` in -settings.py. See "Adding Credentials To An Existing Django User System" for -usage differences. - -Only Django versions 1.8+ are supported. - -Configuration -=============== - -To configure, you'll need a set of OAuth2 web application credentials from -`Google Developer's Console `. - -Add the helper to your INSTALLED_APPS: - -.. code-block:: python - :caption: settings.py - :name: installed_apps - - INSTALLED_APPS = ( - # other apps - "django.contrib.sessions.middleware" - "oauth2client.contrib.django_util" - ) - -This helper also requires the Django Session Middleware, so -``django.contrib.sessions.middleware`` should be in INSTALLED_APPS as well. -MIDDLEWARE or MIDDLEWARE_CLASSES (in Django versions <1.10) should also -contain the string 'django.contrib.sessions.middleware.SessionMiddleware'. - - -Add the client secrets created earlier to the settings. You can either -specify the path to the credentials file in JSON format - -.. code-block:: python - :caption: settings.py - :name: secrets_file - - GOOGLE_OAUTH2_CLIENT_SECRETS_JSON=/path/to/client-secret.json - -Or, directly configure the client Id and client secret. - - -.. code-block:: python - :caption: settings.py - :name: secrets_config - - GOOGLE_OAUTH2_CLIENT_ID=client-id-field - GOOGLE_OAUTH2_CLIENT_SECRET=client-secret-field - -By default, the default scopes for the required decorator only contains the -``email`` scopes. You can change that default in the settings. - -.. code-block:: python - :caption: settings.py - :name: scopes - - GOOGLE_OAUTH2_SCOPES = ('email', 'https://www.googleapis.com/auth/calendar',) - -By default, the decorators will add an `oauth` object to the Django request -object, and include all of its state and helpers inside that object. If the -`oauth` name conflicts with another usage, it can be changed - -.. code-block:: python - :caption: settings.py - :name: request_prefix - - # changes request.oauth to request.google_oauth - GOOGLE_OAUTH2_REQUEST_ATTRIBUTE = 'google_oauth' - -Add the oauth2 routes to your application's urls.py urlpatterns. - -.. code-block:: python - :caption: urls.py - :name: urls - - from oauth2client.contrib.django_util.site import urls as oauth2_urls - - urlpatterns += [url(r'^oauth2/', include(oauth2_urls))] - -To require OAuth2 credentials for a view, use the `oauth2_required` decorator. -This creates a credentials object with an id_token, and allows you to create -an `http` object to build service clients with. These are all attached to the -request.oauth - -.. code-block:: python - :caption: views.py - :name: views_required - - from oauth2client.contrib.django_util.decorators import oauth_required - - @oauth_required - def requires_default_scopes(request): - email = request.oauth.credentials.id_token['email'] - service = build(serviceName='calendar', version='v3', - http=request.oauth.http, - developerKey=API_KEY) - events = service.events().list(calendarId='primary').execute()['items'] - return HttpResponse("email: {0} , calendar: {1}".format( - email,str(events))) - return HttpResponse( - "email: {0} , calendar: {1}".format(email, str(events))) - -To make OAuth2 optional and provide an authorization link in your own views. - -.. code-block:: python - :caption: views.py - :name: views_enabled2 - - from oauth2client.contrib.django_util.decorators import oauth_enabled - - @oauth_enabled - def optional_oauth2(request): - if request.oauth.has_credentials(): - # this could be passed into a view - # request.oauth.http is also initialized - return HttpResponse("User email: {0}".format( - request.oauth.credentials.id_token['email'])) - else: - return HttpResponse( - 'Here is an OAuth Authorize link: Authorize' - ''.format(request.oauth.get_authorize_redirect())) - -If a view needs a scope not included in the default scopes specified in -the settings, you can use [incremental auth](https://developers.google.com/identity/sign-in/web/incremental-auth) -and specify additional scopes in the decorator arguments. - -.. code-block:: python - :caption: views.py - :name: views_required_additional_scopes - - @oauth_enabled(scopes=['https://www.googleapis.com/auth/drive']) - def drive_required(request): - if request.oauth.has_credentials(): - service = build(serviceName='drive', version='v2', - http=request.oauth.http, - developerKey=API_KEY) - events = service.files().list().execute()['items'] - return HttpResponse(str(events)) - else: - return HttpResponse( - 'Here is an OAuth Authorize link: Authorize' - ''.format(request.oauth.get_authorize_redirect())) - - -To provide a callback on authorization being completed, use the -oauth2_authorized signal: - -.. code-block:: python - :caption: views.py - :name: signals - - from oauth2client.contrib.django_util.signals import oauth2_authorized - - def test_callback(sender, request, credentials, **kwargs): - print("Authorization Signal Received {0}".format( - credentials.id_token['email'])) - - oauth2_authorized.connect(test_callback) - -Adding Credentials To An Existing Django User System -===================================================== - -As an alternative to storing the credentials in the session, the helper -can be configured to store the fields on a Django model. This might be useful -if you need to use the credentials outside the context of a user request. It -also prevents the need for a logged in user to repeat the OAuth flow when -starting a new session. - -To use, change ``settings.py`` - -.. code-block:: python - :caption: settings.py - :name: storage_model_config - - GOOGLE_OAUTH2_STORAGE_MODEL = { - 'model': 'path.to.model.MyModel', - 'user_property': 'user_id', - 'credentials_property': 'credential' - } - -Where ``path.to.model`` class is the fully qualified name of a -``django.db.model`` class containing a ``django.contrib.auth.models.User`` -field with the name specified by `user_property` and a -:class:`oauth2client.contrib.django_util.models.CredentialsField` with the name -specified by `credentials_property`. For the sample configuration given, -our model would look like - -.. code-block:: python - :caption: models.py - :name: storage_model_model - - from django.contrib.auth.models import User - from oauth2client.contrib.django_util.models import CredentialsField - - class MyModel(models.Model): - # ... other fields here ... - user = models.OneToOneField(User) - credential = CredentialsField() -""" - -import importlib - -import django.conf -from django.core import exceptions -from django.core import urlresolvers -from six.moves.urllib import parse - -from oauth2client import clientsecrets -from oauth2client import transport -from oauth2client.contrib import dictionary_storage -from oauth2client.contrib.django_util import storage - -GOOGLE_OAUTH2_DEFAULT_SCOPES = ('email',) -GOOGLE_OAUTH2_REQUEST_ATTRIBUTE = 'oauth' - - -def _load_client_secrets(filename): - """Loads client secrets from the given filename. - - Args: - filename: The name of the file containing the JSON secret key. - - Returns: - A 2-tuple, the first item containing the client id, and the second - item containing a client secret. - """ - client_type, client_info = clientsecrets.loadfile(filename) - - if client_type != clientsecrets.TYPE_WEB: - raise ValueError( - 'The flow specified in {} is not supported, only the WEB flow ' - 'type is supported.'.format(client_type)) - return client_info['client_id'], client_info['client_secret'] - - -def _get_oauth2_client_id_and_secret(settings_instance): - """Initializes client id and client secret based on the settings. - - Args: - settings_instance: An instance of ``django.conf.settings``. - - Returns: - A 2-tuple, the first item is the client id and the second - item is the client secret. - """ - secret_json = getattr(settings_instance, - 'GOOGLE_OAUTH2_CLIENT_SECRETS_JSON', None) - if secret_json is not None: - return _load_client_secrets(secret_json) - else: - client_id = getattr(settings_instance, "GOOGLE_OAUTH2_CLIENT_ID", - None) - client_secret = getattr(settings_instance, - "GOOGLE_OAUTH2_CLIENT_SECRET", None) - if client_id is not None and client_secret is not None: - return client_id, client_secret - else: - raise exceptions.ImproperlyConfigured( - "Must specify either GOOGLE_OAUTH2_CLIENT_SECRETS_JSON, or " - "both GOOGLE_OAUTH2_CLIENT_ID and " - "GOOGLE_OAUTH2_CLIENT_SECRET in settings.py") - - -def _get_storage_model(): - """This configures whether the credentials will be stored in the session - or the Django ORM based on the settings. By default, the credentials - will be stored in the session, unless `GOOGLE_OAUTH2_STORAGE_MODEL` - is found in the settings. Usually, the ORM storage is used to integrate - credentials into an existing Django user system. - - Returns: - A tuple containing three strings, or None. If - ``GOOGLE_OAUTH2_STORAGE_MODEL`` is configured, the tuple - will contain the fully qualifed path of the `django.db.model`, - the name of the ``django.contrib.auth.models.User`` field on the - model, and the name of the - :class:`oauth2client.contrib.django_util.models.CredentialsField` - field on the model. If Django ORM storage is not configured, - this function returns None. - """ - storage_model_settings = getattr(django.conf.settings, - 'GOOGLE_OAUTH2_STORAGE_MODEL', None) - if storage_model_settings is not None: - return (storage_model_settings['model'], - storage_model_settings['user_property'], - storage_model_settings['credentials_property']) - else: - return None, None, None - - -class OAuth2Settings(object): - """Initializes Django OAuth2 Helper Settings - - This class loads the OAuth2 Settings from the Django settings, and then - provides those settings as attributes to the rest of the views and - decorators in the module. - - Attributes: - scopes: A list of OAuth2 scopes that the decorators and views will use - as defaults. - request_prefix: The name of the attribute that the decorators use to - attach the UserOAuth2 object to the Django request object. - client_id: The OAuth2 Client ID. - client_secret: The OAuth2 Client Secret. - """ - - def __init__(self, settings_instance): - self.scopes = getattr(settings_instance, 'GOOGLE_OAUTH2_SCOPES', - GOOGLE_OAUTH2_DEFAULT_SCOPES) - self.request_prefix = getattr(settings_instance, - 'GOOGLE_OAUTH2_REQUEST_ATTRIBUTE', - GOOGLE_OAUTH2_REQUEST_ATTRIBUTE) - info = _get_oauth2_client_id_and_secret(settings_instance) - self.client_id, self.client_secret = info - - # Django 1.10 deprecated MIDDLEWARE_CLASSES in favor of MIDDLEWARE - middleware_settings = getattr(settings_instance, 'MIDDLEWARE', None) - if middleware_settings is None: - middleware_settings = getattr( - settings_instance, 'MIDDLEWARE_CLASSES', None) - if middleware_settings is None: - raise exceptions.ImproperlyConfigured( - 'Django settings has neither MIDDLEWARE nor MIDDLEWARE_CLASSES' - 'configured') - - if ('django.contrib.sessions.middleware.SessionMiddleware' not in - middleware_settings): - raise exceptions.ImproperlyConfigured( - 'The Google OAuth2 Helper requires session middleware to ' - 'be installed. Edit your MIDDLEWARE_CLASSES or MIDDLEWARE ' - 'setting to include \'django.contrib.sessions.middleware.' - 'SessionMiddleware\'.') - (self.storage_model, self.storage_model_user_property, - self.storage_model_credentials_property) = _get_storage_model() - - -oauth2_settings = OAuth2Settings(django.conf.settings) - -_CREDENTIALS_KEY = 'google_oauth2_credentials' - - -def get_storage(request): - """ Gets a Credentials storage object provided by the Django OAuth2 Helper - object. - - Args: - request: Reference to the current request object. - - Returns: - An :class:`oauth2.client.Storage` object. - """ - storage_model = oauth2_settings.storage_model - user_property = oauth2_settings.storage_model_user_property - credentials_property = oauth2_settings.storage_model_credentials_property - - if storage_model: - module_name, class_name = storage_model.rsplit('.', 1) - module = importlib.import_module(module_name) - storage_model_class = getattr(module, class_name) - return storage.DjangoORMStorage(storage_model_class, - user_property, - request.user, - credentials_property) - else: - # use session - return dictionary_storage.DictionaryStorage( - request.session, key=_CREDENTIALS_KEY) - - -def _redirect_with_params(url_name, *args, **kwargs): - """Helper method to create a redirect response with URL params. - - This builds a redirect string that converts kwargs into a - query string. - - Args: - url_name: The name of the url to redirect to. - kwargs: the query string param and their values to build. - - Returns: - A properly formatted redirect string. - """ - url = urlresolvers.reverse(url_name, args=args) - params = parse.urlencode(kwargs, True) - return "{0}?{1}".format(url, params) - - -def _credentials_from_request(request): - """Gets the authorized credentials for this flow, if they exist.""" - # ORM storage requires a logged in user - if (oauth2_settings.storage_model is None or - request.user.is_authenticated()): - return get_storage(request).get() - else: - return None - - -class UserOAuth2(object): - """Class to create oauth2 objects on Django request objects containing - credentials and helper methods. - """ - - def __init__(self, request, scopes=None, return_url=None): - """Initialize the Oauth2 Object. - - Args: - request: Django request object. - scopes: Scopes desired for this OAuth2 flow. - return_url: The url to return to after the OAuth flow is complete, - defaults to the request's current URL path. - """ - self.request = request - self.return_url = return_url or request.get_full_path() - if scopes: - self._scopes = set(oauth2_settings.scopes) | set(scopes) - else: - self._scopes = set(oauth2_settings.scopes) - - def get_authorize_redirect(self): - """Creates a URl to start the OAuth2 authorization flow.""" - get_params = { - 'return_url': self.return_url, - 'scopes': self._get_scopes() - } - - return _redirect_with_params('google_oauth:authorize', **get_params) - - def has_credentials(self): - """Returns True if there are valid credentials for the current user - and required scopes.""" - credentials = _credentials_from_request(self.request) - return (credentials and not credentials.invalid and - credentials.has_scopes(self._get_scopes())) - - def _get_scopes(self): - """Returns the scopes associated with this object, kept up to - date for incremental auth.""" - if _credentials_from_request(self.request): - return (self._scopes | - _credentials_from_request(self.request).scopes) - else: - return self._scopes - - @property - def scopes(self): - """Returns the scopes associated with this OAuth2 object.""" - # make sure previously requested custom scopes are maintained - # in future authorizations - return self._get_scopes() - - @property - def credentials(self): - """Gets the authorized credentials for this flow, if they exist.""" - return _credentials_from_request(self.request) - - @property - def http(self): - """Helper: create HTTP client authorized with OAuth2 credentials.""" - if self.has_credentials(): - return self.credentials.authorize(transport.get_http_object()) - return None diff --git a/src/oauth2client/contrib/django_util/apps.py b/src/oauth2client/contrib/django_util/apps.py deleted file mode 100644 index 86676b91..00000000 --- a/src/oauth2client/contrib/django_util/apps.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Application Config For Django OAuth2 Helper. - -Django 1.7+ provides an -[applications](https://docs.djangoproject.com/en/1.8/ref/applications/) -API so that Django projects can introspect on installed applications using a -stable API. This module exists to follow that convention. -""" - -import sys - -# Django 1.7+ only supports Python 2.7+ -if sys.hexversion >= 0x02070000: # pragma: NO COVER - from django.apps import AppConfig - - class GoogleOAuth2HelperConfig(AppConfig): - """ App Config for Django Helper""" - name = 'oauth2client.django_util' - verbose_name = "Google OAuth2 Django Helper" diff --git a/src/oauth2client/contrib/django_util/decorators.py b/src/oauth2client/contrib/django_util/decorators.py deleted file mode 100644 index e62e1710..00000000 --- a/src/oauth2client/contrib/django_util/decorators.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Decorators for Django OAuth2 Flow. - -Contains two decorators, ``oauth_required`` and ``oauth_enabled``. - -``oauth_required`` will ensure that a user has an oauth object containing -credentials associated with the request, and if not, redirect to the -authorization flow. - -``oauth_enabled`` will attach the oauth2 object containing credentials if it -exists. If it doesn't, the view will still render, but helper methods will be -attached to start the oauth2 flow. -""" - -from django import shortcuts -import django.conf -from six import wraps -from six.moves.urllib import parse - -from oauth2client.contrib import django_util - - -def oauth_required(decorated_function=None, scopes=None, **decorator_kwargs): - """ Decorator to require OAuth2 credentials for a view. - - - .. code-block:: python - :caption: views.py - :name: views_required_2 - - - from oauth2client.django_util.decorators import oauth_required - - @oauth_required - def requires_default_scopes(request): - email = request.credentials.id_token['email'] - service = build(serviceName='calendar', version='v3', - http=request.oauth.http, - developerKey=API_KEY) - events = service.events().list( - calendarId='primary').execute()['items'] - return HttpResponse( - "email: {0}, calendar: {1}".format(email, str(events))) - - Args: - decorated_function: View function to decorate, must have the Django - request object as the first argument. - scopes: Scopes to require, will default. - decorator_kwargs: Can include ``return_url`` to specify the URL to - return to after OAuth2 authorization is complete. - - Returns: - An OAuth2 Authorize view if credentials are not found or if the - credentials are missing the required scopes. Otherwise, - the decorated view. - """ - def curry_wrapper(wrapped_function): - @wraps(wrapped_function) - def required_wrapper(request, *args, **kwargs): - if not (django_util.oauth2_settings.storage_model is None or - request.user.is_authenticated()): - redirect_str = '{0}?next={1}'.format( - django.conf.settings.LOGIN_URL, - parse.quote(request.path)) - return shortcuts.redirect(redirect_str) - - return_url = decorator_kwargs.pop('return_url', - request.get_full_path()) - user_oauth = django_util.UserOAuth2(request, scopes, return_url) - if not user_oauth.has_credentials(): - return shortcuts.redirect(user_oauth.get_authorize_redirect()) - setattr(request, django_util.oauth2_settings.request_prefix, - user_oauth) - return wrapped_function(request, *args, **kwargs) - - return required_wrapper - - if decorated_function: - return curry_wrapper(decorated_function) - else: - return curry_wrapper - - -def oauth_enabled(decorated_function=None, scopes=None, **decorator_kwargs): - """ Decorator to enable OAuth Credentials if authorized, and setup - the oauth object on the request object to provide helper functions - to start the flow otherwise. - - .. code-block:: python - :caption: views.py - :name: views_enabled3 - - from oauth2client.django_util.decorators import oauth_enabled - - @oauth_enabled - def optional_oauth2(request): - if request.oauth.has_credentials(): - # this could be passed into a view - # request.oauth.http is also initialized - return HttpResponse("User email: {0}".format( - request.oauth.credentials.id_token['email']) - else: - return HttpResponse('Here is an OAuth Authorize link: - Authorize'.format( - request.oauth.get_authorize_redirect())) - - - Args: - decorated_function: View function to decorate. - scopes: Scopes to require, will default. - decorator_kwargs: Can include ``return_url`` to specify the URL to - return to after OAuth2 authorization is complete. - - Returns: - The decorated view function. - """ - def curry_wrapper(wrapped_function): - @wraps(wrapped_function) - def enabled_wrapper(request, *args, **kwargs): - return_url = decorator_kwargs.pop('return_url', - request.get_full_path()) - user_oauth = django_util.UserOAuth2(request, scopes, return_url) - setattr(request, django_util.oauth2_settings.request_prefix, - user_oauth) - return wrapped_function(request, *args, **kwargs) - - return enabled_wrapper - - if decorated_function: - return curry_wrapper(decorated_function) - else: - return curry_wrapper diff --git a/src/oauth2client/contrib/django_util/models.py b/src/oauth2client/contrib/django_util/models.py deleted file mode 100644 index 37cc6970..00000000 --- a/src/oauth2client/contrib/django_util/models.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains classes used for the Django ORM storage.""" - -import base64 -import pickle - -from django.db import models -from django.utils import encoding -import jsonpickle - -import oauth2client - - -class CredentialsField(models.Field): - """Django ORM field for storing OAuth2 Credentials.""" - - def __init__(self, *args, **kwargs): - if 'null' not in kwargs: - kwargs['null'] = True - super(CredentialsField, self).__init__(*args, **kwargs) - - def get_internal_type(self): - return 'BinaryField' - - def from_db_value(self, value, expression, connection, context): - """Overrides ``models.Field`` method. This converts the value - returned from the database to an instance of this class. - """ - return self.to_python(value) - - def to_python(self, value): - """Overrides ``models.Field`` method. This is used to convert - bytes (from serialization etc) to an instance of this class""" - if value is None: - return None - elif isinstance(value, oauth2client.client.Credentials): - return value - else: - try: - return jsonpickle.decode( - base64.b64decode(encoding.smart_bytes(value)).decode()) - except ValueError: - return pickle.loads( - base64.b64decode(encoding.smart_bytes(value))) - - def get_prep_value(self, value): - """Overrides ``models.Field`` method. This is used to convert - the value from an instances of this class to bytes that can be - inserted into the database. - """ - if value is None: - return None - else: - return encoding.smart_text( - base64.b64encode(jsonpickle.encode(value).encode())) - - def value_to_string(self, obj): - """Convert the field value from the provided model to a string. - - Used during model serialization. - - Args: - obj: db.Model, model object - - Returns: - string, the serialized field value - """ - value = self._get_val_from_obj(obj) - return self.get_prep_value(value) diff --git a/src/oauth2client/contrib/django_util/signals.py b/src/oauth2client/contrib/django_util/signals.py deleted file mode 100644 index e9356b4d..00000000 --- a/src/oauth2client/contrib/django_util/signals.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Signals for Google OAuth2 Helper. - -This module contains signals for Google OAuth2 Helper. Currently it only -contains one, which fires when an OAuth2 authorization flow has completed. -""" - -import django.dispatch - -"""Signal that fires when OAuth2 Flow has completed. -It passes the Django request object and the OAuth2 credentials object to the - receiver. -""" -oauth2_authorized = django.dispatch.Signal( - providing_args=["request", "credentials"]) diff --git a/src/oauth2client/contrib/django_util/site.py b/src/oauth2client/contrib/django_util/site.py deleted file mode 100644 index 631f79be..00000000 --- a/src/oauth2client/contrib/django_util/site.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains Django URL patterns used for OAuth2 flow.""" - -from django.conf import urls - -from oauth2client.contrib.django_util import views - -urlpatterns = [ - urls.url(r'oauth2callback/', views.oauth2_callback, name="callback"), - urls.url(r'oauth2authorize/', views.oauth2_authorize, name="authorize") -] - -urls = (urlpatterns, "google_oauth", "google_oauth") diff --git a/src/oauth2client/contrib/django_util/storage.py b/src/oauth2client/contrib/django_util/storage.py deleted file mode 100644 index 5682919b..00000000 --- a/src/oauth2client/contrib/django_util/storage.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains a storage module that stores credentials using the Django ORM.""" - -from oauth2client import client - - -class DjangoORMStorage(client.Storage): - """Store and retrieve a single credential to and from the Django datastore. - - This Storage helper presumes the Credentials - have been stored as a CredentialsField - on a db model class. - """ - - def __init__(self, model_class, key_name, key_value, property_name): - """Constructor for Storage. - - Args: - model: string, fully qualified name of db.Model model class. - key_name: string, key name for the entity that has the credentials - key_value: string, key value for the entity that has the - credentials. - property_name: string, name of the property that is an - CredentialsProperty. - """ - super(DjangoORMStorage, self).__init__() - self.model_class = model_class - self.key_name = key_name - self.key_value = key_value - self.property_name = property_name - - def locked_get(self): - """Retrieve stored credential from the Django ORM. - - Returns: - oauth2client.Credentials retrieved from the Django ORM, associated - with the ``model``, ``key_value``->``key_name`` pair used to query - for the model, and ``property_name`` identifying the - ``CredentialsProperty`` field, all of which are defined in the - constructor for this Storage object. - - """ - query = {self.key_name: self.key_value} - entities = self.model_class.objects.filter(**query) - if len(entities) > 0: - credential = getattr(entities[0], self.property_name) - if getattr(credential, 'set_store', None) is not None: - credential.set_store(self) - return credential - else: - return None - - def locked_put(self, credentials): - """Write a Credentials to the Django datastore. - - Args: - credentials: Credentials, the credentials to store. - """ - entity, _ = self.model_class.objects.get_or_create( - **{self.key_name: self.key_value}) - - setattr(entity, self.property_name, credentials) - entity.save() - - def locked_delete(self): - """Delete Credentials from the datastore.""" - query = {self.key_name: self.key_value} - self.model_class.objects.filter(**query).delete() diff --git a/src/oauth2client/contrib/django_util/views.py b/src/oauth2client/contrib/django_util/views.py deleted file mode 100644 index 1835208a..00000000 --- a/src/oauth2client/contrib/django_util/views.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains the views used by the OAuth2 flows. - -Their are two views used by the OAuth2 flow, the authorize and the callback -view. The authorize view kicks off the three-legged OAuth flow, and the -callback view validates the flow and if successful stores the credentials -in the configured storage.""" - -import hashlib -import json -import os - -from django import http -from django import shortcuts -from django.conf import settings -from django.core import urlresolvers -from django.shortcuts import redirect -from django.utils import html -import jsonpickle -from six.moves.urllib import parse - -from oauth2client import client -from oauth2client.contrib import django_util -from oauth2client.contrib.django_util import get_storage -from oauth2client.contrib.django_util import signals - -_CSRF_KEY = 'google_oauth2_csrf_token' -_FLOW_KEY = 'google_oauth2_flow_{0}' - - -def _make_flow(request, scopes, return_url=None): - """Creates a Web Server Flow - - Args: - request: A Django request object. - scopes: the request oauth2 scopes. - return_url: The URL to return to after the flow is complete. Defaults - to the path of the current request. - - Returns: - An OAuth2 flow object that has been stored in the session. - """ - # Generate a CSRF token to prevent malicious requests. - csrf_token = hashlib.sha256(os.urandom(1024)).hexdigest() - - request.session[_CSRF_KEY] = csrf_token - - state = json.dumps({ - 'csrf_token': csrf_token, - 'return_url': return_url, - }) - - flow = client.OAuth2WebServerFlow( - client_id=django_util.oauth2_settings.client_id, - client_secret=django_util.oauth2_settings.client_secret, - scope=scopes, - state=state, - redirect_uri=request.build_absolute_uri( - urlresolvers.reverse("google_oauth:callback"))) - - flow_key = _FLOW_KEY.format(csrf_token) - request.session[flow_key] = jsonpickle.encode(flow) - return flow - - -def _get_flow_for_token(csrf_token, request): - """ Looks up the flow in session to recover information about requested - scopes. - - Args: - csrf_token: The token passed in the callback request that should - match the one previously generated and stored in the request on the - initial authorization view. - - Returns: - The OAuth2 Flow object associated with this flow based on the - CSRF token. - """ - flow_pickle = request.session.get(_FLOW_KEY.format(csrf_token), None) - return None if flow_pickle is None else jsonpickle.decode(flow_pickle) - - -def oauth2_callback(request): - """ View that handles the user's return from OAuth2 provider. - - This view verifies the CSRF state and OAuth authorization code, and on - success stores the credentials obtained in the storage provider, - and redirects to the return_url specified in the authorize view and - stored in the session. - - Args: - request: Django request. - - Returns: - A redirect response back to the return_url. - """ - if 'error' in request.GET: - reason = request.GET.get( - 'error_description', request.GET.get('error', '')) - reason = html.escape(reason) - return http.HttpResponseBadRequest( - 'Authorization failed {0}'.format(reason)) - - try: - encoded_state = request.GET['state'] - code = request.GET['code'] - except KeyError: - return http.HttpResponseBadRequest( - 'Request missing state or authorization code') - - try: - server_csrf = request.session[_CSRF_KEY] - except KeyError: - return http.HttpResponseBadRequest( - 'No existing session for this flow.') - - try: - state = json.loads(encoded_state) - client_csrf = state['csrf_token'] - return_url = state['return_url'] - except (ValueError, KeyError): - return http.HttpResponseBadRequest('Invalid state parameter.') - - if client_csrf != server_csrf: - return http.HttpResponseBadRequest('Invalid CSRF token.') - - flow = _get_flow_for_token(client_csrf, request) - - if not flow: - return http.HttpResponseBadRequest('Missing Oauth2 flow.') - - try: - credentials = flow.step2_exchange(code) - except client.FlowExchangeError as exchange_error: - return http.HttpResponseBadRequest( - 'An error has occurred: {0}'.format(exchange_error)) - - get_storage(request).put(credentials) - - signals.oauth2_authorized.send(sender=signals.oauth2_authorized, - request=request, credentials=credentials) - - return shortcuts.redirect(return_url) - - -def oauth2_authorize(request): - """ View to start the OAuth2 Authorization flow. - - This view starts the OAuth2 authorization flow. If scopes is passed in - as a GET URL parameter, it will authorize those scopes, otherwise the - default scopes specified in settings. The return_url can also be - specified as a GET parameter, otherwise the referer header will be - checked, and if that isn't found it will return to the root path. - - Args: - request: The Django request object. - - Returns: - A redirect to Google OAuth2 Authorization. - """ - return_url = request.GET.get('return_url', None) - if not return_url: - return_url = request.META.get('HTTP_REFERER', '/') - - scopes = request.GET.getlist('scopes', django_util.oauth2_settings.scopes) - # Model storage (but not session storage) requires a logged in user - if django_util.oauth2_settings.storage_model: - if not request.user.is_authenticated(): - return redirect('{0}?next={1}'.format( - settings.LOGIN_URL, parse.quote(request.get_full_path()))) - # This checks for the case where we ended up here because of a logged - # out user but we had credentials for it in the first place - else: - user_oauth = django_util.UserOAuth2(request, scopes, return_url) - if user_oauth.has_credentials(): - return redirect(return_url) - - flow = _make_flow(request=request, scopes=scopes, return_url=return_url) - auth_url = flow.step1_get_authorize_url() - return shortcuts.redirect(auth_url) diff --git a/src/oauth2client/contrib/flask_util.py b/src/oauth2client/contrib/flask_util.py deleted file mode 100644 index fabd613b..00000000 --- a/src/oauth2client/contrib/flask_util.py +++ /dev/null @@ -1,557 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for the Flask web framework - -Provides a Flask extension that makes using OAuth2 web server flow easier. -The extension includes views that handle the entire auth flow and a -``@required`` decorator to automatically ensure that user credentials are -available. - - -Configuration -============= - -To configure, you'll need a set of OAuth2 web application credentials from the -`Google Developer's Console `__. - -.. code-block:: python - - from oauth2client.contrib.flask_util import UserOAuth2 - - app = Flask(__name__) - - app.config['SECRET_KEY'] = 'your-secret-key' - - app.config['GOOGLE_OAUTH2_CLIENT_SECRETS_FILE'] = 'client_secrets.json' - - # or, specify the client id and secret separately - app.config['GOOGLE_OAUTH2_CLIENT_ID'] = 'your-client-id' - app.config['GOOGLE_OAUTH2_CLIENT_SECRET'] = 'your-client-secret' - - oauth2 = UserOAuth2(app) - - -Usage -===== - -Once configured, you can use the :meth:`UserOAuth2.required` decorator to -ensure that credentials are available within a view. - -.. code-block:: python - :emphasize-lines: 3,7,10 - - # Note that app.route should be the outermost decorator. - @app.route('/needs_credentials') - @oauth2.required - def example(): - # http is authorized with the user's credentials and can be used - # to make http calls. - http = oauth2.http() - - # Or, you can access the credentials directly - credentials = oauth2.credentials - -If you want credentials to be optional for a view, you can leave the decorator -off and use :meth:`UserOAuth2.has_credentials` to check. - -.. code-block:: python - :emphasize-lines: 3 - - @app.route('/optional') - def optional(): - if oauth2.has_credentials(): - return 'Credentials found!' - else: - return 'No credentials!' - - -When credentials are available, you can use :attr:`UserOAuth2.email` and -:attr:`UserOAuth2.user_id` to access information from the `ID Token -`__, if -available. - -.. code-block:: python - :emphasize-lines: 4 - - @app.route('/info') - @oauth2.required - def info(): - return "Hello, {} ({})".format(oauth2.email, oauth2.user_id) - - -URLs & Trigging Authorization -============================= - -The extension will add two new routes to your application: - - * ``"oauth2.authorize"`` -> ``/oauth2authorize`` - * ``"oauth2.callback"`` -> ``/oauth2callback`` - -When configuring your OAuth2 credentials on the Google Developer's Console, be -sure to add ``http[s]://[your-app-url]/oauth2callback`` as an authorized -callback url. - -Typically you don't not need to use these routes directly, just be sure to -decorate any views that require credentials with ``@oauth2.required``. If -needed, you can trigger authorization at any time by redirecting the user -to the URL returned by :meth:`UserOAuth2.authorize_url`. - -.. code-block:: python - :emphasize-lines: 3 - - @app.route('/login') - def login(): - return oauth2.authorize_url("/") - - -Incremental Auth -================ - -This extension also supports `Incremental Auth `__. To enable it, -configure the extension with ``include_granted_scopes``. - -.. code-block:: python - - oauth2 = UserOAuth2(app, include_granted_scopes=True) - -Then specify any additional scopes needed on the decorator, for example: - -.. code-block:: python - :emphasize-lines: 2,7 - - @app.route('/drive') - @oauth2.required(scopes=["https://www.googleapis.com/auth/drive"]) - def requires_drive(): - ... - - @app.route('/calendar') - @oauth2.required(scopes=["https://www.googleapis.com/auth/calendar"]) - def requires_calendar(): - ... - -The decorator will ensure that the the user has authorized all specified scopes -before allowing them to access the view, and will also ensure that credentials -do not lose any previously authorized scopes. - - -Storage -======= - -By default, the extension uses a Flask session-based storage solution. This -means that credentials are only available for the duration of a session. It -also means that with Flask's default configuration, the credentials will be -visible in the session cookie. It's highly recommended to use database-backed -session and to use https whenever handling user credentials. - -If you need the credentials to be available longer than a user session or -available outside of a request context, you will need to implement your own -:class:`oauth2client.Storage`. -""" - -from functools import wraps -import hashlib -import json -import os -import pickle - -try: - from flask import Blueprint - from flask import _app_ctx_stack - from flask import current_app - from flask import redirect - from flask import request - from flask import session - from flask import url_for - import markupsafe -except ImportError: # pragma: NO COVER - raise ImportError('The flask utilities require flask 0.9 or newer.') - -import six.moves.http_client as httplib - -from oauth2client import client -from oauth2client import clientsecrets -from oauth2client import transport -from oauth2client.contrib import dictionary_storage - - -_DEFAULT_SCOPES = ('email',) -_CREDENTIALS_KEY = 'google_oauth2_credentials' -_FLOW_KEY = 'google_oauth2_flow_{0}' -_CSRF_KEY = 'google_oauth2_csrf_token' - - -def _get_flow_for_token(csrf_token): - """Retrieves the flow instance associated with a given CSRF token from - the Flask session.""" - flow_pickle = session.pop( - _FLOW_KEY.format(csrf_token), None) - - if flow_pickle is None: - return None - else: - return pickle.loads(flow_pickle) - - -class UserOAuth2(object): - """Flask extension for making OAuth 2.0 easier. - - Configuration values: - - * ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` path to a client secrets json - file, obtained from the credentials screen in the Google Developers - console. - * ``GOOGLE_OAUTH2_CLIENT_ID`` the oauth2 credentials' client ID. This - is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` is not - specified. - * ``GOOGLE_OAUTH2_CLIENT_SECRET`` the oauth2 credentials' client - secret. This is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` - is not specified. - - If app is specified, all arguments will be passed along to init_app. - - If no app is specified, then you should call init_app in your application - factory to finish initialization. - """ - - def __init__(self, app=None, *args, **kwargs): - self.app = app - if app is not None: - self.init_app(app, *args, **kwargs) - - def init_app(self, app, scopes=None, client_secrets_file=None, - client_id=None, client_secret=None, authorize_callback=None, - storage=None, **kwargs): - """Initialize this extension for the given app. - - Arguments: - app: A Flask application. - scopes: Optional list of scopes to authorize. - client_secrets_file: Path to a file containing client secrets. You - can also specify the GOOGLE_OAUTH2_CLIENT_SECRETS_FILE config - value. - client_id: If not specifying a client secrets file, specify the - OAuth2 client id. You can also specify the - GOOGLE_OAUTH2_CLIENT_ID config value. You must also provide a - client secret. - client_secret: The OAuth2 client secret. You can also specify the - GOOGLE_OAUTH2_CLIENT_SECRET config value. - authorize_callback: A function that is executed after successful - user authorization. - storage: A oauth2client.client.Storage subclass for storing the - credentials. By default, this is a Flask session based storage. - kwargs: Any additional args are passed along to the Flow - constructor. - """ - self.app = app - self.authorize_callback = authorize_callback - self.flow_kwargs = kwargs - - if storage is None: - storage = dictionary_storage.DictionaryStorage( - session, key=_CREDENTIALS_KEY) - self.storage = storage - - if scopes is None: - scopes = app.config.get('GOOGLE_OAUTH2_SCOPES', _DEFAULT_SCOPES) - self.scopes = scopes - - self._load_config(client_secrets_file, client_id, client_secret) - - app.register_blueprint(self._create_blueprint()) - - def _load_config(self, client_secrets_file, client_id, client_secret): - """Loads oauth2 configuration in order of priority. - - Priority: - 1. Config passed to the constructor or init_app. - 2. Config passed via the GOOGLE_OAUTH2_CLIENT_SECRETS_FILE app - config. - 3. Config passed via the GOOGLE_OAUTH2_CLIENT_ID and - GOOGLE_OAUTH2_CLIENT_SECRET app config. - - Raises: - ValueError if no config could be found. - """ - if client_id and client_secret: - self.client_id, self.client_secret = client_id, client_secret - return - - if client_secrets_file: - self._load_client_secrets(client_secrets_file) - return - - if 'GOOGLE_OAUTH2_CLIENT_SECRETS_FILE' in self.app.config: - self._load_client_secrets( - self.app.config['GOOGLE_OAUTH2_CLIENT_SECRETS_FILE']) - return - - try: - self.client_id, self.client_secret = ( - self.app.config['GOOGLE_OAUTH2_CLIENT_ID'], - self.app.config['GOOGLE_OAUTH2_CLIENT_SECRET']) - except KeyError: - raise ValueError( - 'OAuth2 configuration could not be found. Either specify the ' - 'client_secrets_file or client_id and client_secret or set ' - 'the app configuration variables ' - 'GOOGLE_OAUTH2_CLIENT_SECRETS_FILE or ' - 'GOOGLE_OAUTH2_CLIENT_ID and GOOGLE_OAUTH2_CLIENT_SECRET.') - - def _load_client_secrets(self, filename): - """Loads client secrets from the given filename.""" - client_type, client_info = clientsecrets.loadfile(filename) - if client_type != clientsecrets.TYPE_WEB: - raise ValueError( - 'The flow specified in {0} is not supported.'.format( - client_type)) - - self.client_id = client_info['client_id'] - self.client_secret = client_info['client_secret'] - - def _make_flow(self, return_url=None, **kwargs): - """Creates a Web Server Flow""" - # Generate a CSRF token to prevent malicious requests. - csrf_token = hashlib.sha256(os.urandom(1024)).hexdigest() - - session[_CSRF_KEY] = csrf_token - - state = json.dumps({ - 'csrf_token': csrf_token, - 'return_url': return_url - }) - - kw = self.flow_kwargs.copy() - kw.update(kwargs) - - extra_scopes = kw.pop('scopes', []) - scopes = set(self.scopes).union(set(extra_scopes)) - - flow = client.OAuth2WebServerFlow( - client_id=self.client_id, - client_secret=self.client_secret, - scope=scopes, - state=state, - redirect_uri=url_for('oauth2.callback', _external=True), - **kw) - - flow_key = _FLOW_KEY.format(csrf_token) - session[flow_key] = pickle.dumps(flow) - - return flow - - def _create_blueprint(self): - bp = Blueprint('oauth2', __name__) - bp.add_url_rule('/oauth2authorize', 'authorize', self.authorize_view) - bp.add_url_rule('/oauth2callback', 'callback', self.callback_view) - - return bp - - def authorize_view(self): - """Flask view that starts the authorization flow. - - Starts flow by redirecting the user to the OAuth2 provider. - """ - args = request.args.to_dict() - - # Scopes will be passed as mutliple args, and to_dict() will only - # return one. So, we use getlist() to get all of the scopes. - args['scopes'] = request.args.getlist('scopes') - - return_url = args.pop('return_url', None) - if return_url is None: - return_url = request.referrer or '/' - - flow = self._make_flow(return_url=return_url, **args) - auth_url = flow.step1_get_authorize_url() - - return redirect(auth_url) - - def callback_view(self): - """Flask view that handles the user's return from OAuth2 provider. - - On return, exchanges the authorization code for credentials and stores - the credentials. - """ - if 'error' in request.args: - reason = request.args.get( - 'error_description', request.args.get('error', '')) - reason = markupsafe.escape(reason) - return ('Authorization failed: {0}'.format(reason), - httplib.BAD_REQUEST) - - try: - encoded_state = request.args['state'] - server_csrf = session[_CSRF_KEY] - code = request.args['code'] - except KeyError: - return 'Invalid request', httplib.BAD_REQUEST - - try: - state = json.loads(encoded_state) - client_csrf = state['csrf_token'] - return_url = state['return_url'] - except (ValueError, KeyError): - return 'Invalid request state', httplib.BAD_REQUEST - - if client_csrf != server_csrf: - return 'Invalid request state', httplib.BAD_REQUEST - - flow = _get_flow_for_token(server_csrf) - - if flow is None: - return 'Invalid request state', httplib.BAD_REQUEST - - # Exchange the auth code for credentials. - try: - credentials = flow.step2_exchange(code) - except client.FlowExchangeError as exchange_error: - current_app.logger.exception(exchange_error) - content = 'An error occurred: {0}'.format(exchange_error) - return content, httplib.BAD_REQUEST - - # Save the credentials to the storage. - self.storage.put(credentials) - - if self.authorize_callback: - self.authorize_callback(credentials) - - return redirect(return_url) - - @property - def credentials(self): - """The credentials for the current user or None if unavailable.""" - ctx = _app_ctx_stack.top - - if not hasattr(ctx, _CREDENTIALS_KEY): - ctx.google_oauth2_credentials = self.storage.get() - - return ctx.google_oauth2_credentials - - def has_credentials(self): - """Returns True if there are valid credentials for the current user.""" - if not self.credentials: - return False - # Is the access token expired? If so, do we have an refresh token? - elif (self.credentials.access_token_expired and - not self.credentials.refresh_token): - return False - else: - return True - - @property - def email(self): - """Returns the user's email address or None if there are no credentials. - - The email address is provided by the current credentials' id_token. - This should not be used as unique identifier as the user can change - their email. If you need a unique identifier, use user_id. - """ - if not self.credentials: - return None - try: - return self.credentials.id_token['email'] - except KeyError: - current_app.logger.error( - 'Invalid id_token {0}'.format(self.credentials.id_token)) - - @property - def user_id(self): - """Returns the a unique identifier for the user - - Returns None if there are no credentials. - - The id is provided by the current credentials' id_token. - """ - if not self.credentials: - return None - try: - return self.credentials.id_token['sub'] - except KeyError: - current_app.logger.error( - 'Invalid id_token {0}'.format(self.credentials.id_token)) - - def authorize_url(self, return_url, **kwargs): - """Creates a URL that can be used to start the authorization flow. - - When the user is directed to the URL, the authorization flow will - begin. Once complete, the user will be redirected to the specified - return URL. - - Any kwargs are passed into the flow constructor. - """ - return url_for('oauth2.authorize', return_url=return_url, **kwargs) - - def required(self, decorated_function=None, scopes=None, - **decorator_kwargs): - """Decorator to require OAuth2 credentials for a view. - - If credentials are not available for the current user, then they will - be redirected to the authorization flow. Once complete, the user will - be redirected back to the original page. - """ - - def curry_wrapper(wrapped_function): - @wraps(wrapped_function) - def required_wrapper(*args, **kwargs): - return_url = decorator_kwargs.pop('return_url', request.url) - - requested_scopes = set(self.scopes) - if scopes is not None: - requested_scopes |= set(scopes) - if self.has_credentials(): - requested_scopes |= self.credentials.scopes - - requested_scopes = list(requested_scopes) - - # Does the user have credentials and does the credentials have - # all of the needed scopes? - if (self.has_credentials() and - self.credentials.has_scopes(requested_scopes)): - return wrapped_function(*args, **kwargs) - # Otherwise, redirect to authorization - else: - auth_url = self.authorize_url( - return_url, - scopes=requested_scopes, - **decorator_kwargs) - - return redirect(auth_url) - - return required_wrapper - - if decorated_function: - return curry_wrapper(decorated_function) - else: - return curry_wrapper - - def http(self, *args, **kwargs): - """Returns an authorized http instance. - - Can only be called if there are valid credentials for the user, such - as inside of a view that is decorated with @required. - - Args: - *args: Positional arguments passed to httplib2.Http constructor. - **kwargs: Positional arguments passed to httplib2.Http constructor. - - Raises: - ValueError if no credentials are available. - """ - if not self.credentials: - raise ValueError('No credentials available.') - return self.credentials.authorize( - transport.get_http_object(*args, **kwargs)) diff --git a/src/oauth2client/contrib/gce.py b/src/oauth2client/contrib/gce.py deleted file mode 100644 index aaab15ff..00000000 --- a/src/oauth2client/contrib/gce.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for Google Compute Engine - -Utilities for making it easier to use OAuth 2.0 on Google Compute Engine. -""" - -import logging -import warnings - -from six.moves import http_client - -from oauth2client import client -from oauth2client.contrib import _metadata - - -logger = logging.getLogger(__name__) - -_SCOPES_WARNING = """\ -You have requested explicit scopes to be used with a GCE service account. -Using this argument will have no effect on the actual scopes for tokens -requested. These scopes are set at VM instance creation time and -can't be overridden in the request. -""" - - -class AppAssertionCredentials(client.AssertionCredentials): - """Credentials object for Compute Engine Assertion Grants - - This object will allow a Compute Engine instance to identify itself to - Google and other OAuth 2.0 servers that can verify assertions. It can be - used for the purpose of accessing data stored under an account assigned to - the Compute Engine instance itself. - - This credential does not require a flow to instantiate because it - represents a two legged flow, and therefore has all of the required - information to generate and refresh its own access tokens. - - Note that :attr:`service_account_email` and :attr:`scopes` - will both return None until the credentials have been refreshed. - To check whether credentials have previously been refreshed use - :attr:`invalid`. - """ - - def __init__(self, email=None, *args, **kwargs): - """Constructor for AppAssertionCredentials - - Args: - email: an email that specifies the service account to use. - Only necessary if using custom service accounts - (see https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#createdefaultserviceaccount). - """ - if 'scopes' in kwargs: - warnings.warn(_SCOPES_WARNING) - kwargs['scopes'] = None - - # Assertion type is no longer used, but still in the - # parent class signature. - super(AppAssertionCredentials, self).__init__(None, *args, **kwargs) - - self.service_account_email = email - self.scopes = None - self.invalid = True - - @classmethod - def from_json(cls, json_data): - raise NotImplementedError( - 'Cannot serialize credentials for GCE service accounts.') - - def to_json(self): - raise NotImplementedError( - 'Cannot serialize credentials for GCE service accounts.') - - def retrieve_scopes(self, http): - """Retrieves the canonical list of scopes for this access token. - - Overrides client.Credentials.retrieve_scopes. Fetches scopes info - from the metadata server. - - Args: - http: httplib2.Http, an http object to be used to make the refresh - request. - - Returns: - A set of strings containing the canonical list of scopes. - """ - self._retrieve_info(http) - return self.scopes - - def _retrieve_info(self, http): - """Retrieves service account info for invalid credentials. - - Args: - http: an object to be used to make HTTP requests. - """ - if self.invalid: - info = _metadata.get_service_account_info( - http, - service_account=self.service_account_email or 'default') - self.invalid = False - self.service_account_email = info['email'] - self.scopes = info['scopes'] - - def _refresh(self, http): - """Refreshes the access token. - - Skip all the storage hoops and just refresh using the API. - - Args: - http: an object to be used to make HTTP requests. - - Raises: - HttpAccessTokenRefreshError: When the refresh fails. - """ - try: - self._retrieve_info(http) - self.access_token, self.token_expiry = _metadata.get_token( - http, service_account=self.service_account_email) - except http_client.HTTPException as err: - raise client.HttpAccessTokenRefreshError(str(err)) - - @property - def serialization_data(self): - raise NotImplementedError( - 'Cannot serialize credentials for GCE service accounts.') - - def create_scoped_required(self): - return False - - def sign_blob(self, blob): - """Cryptographically sign a blob (of bytes). - - This method is provided to support a common interface, but - the actual key used for a Google Compute Engine service account - is not available, so it can't be used to sign content. - - Args: - blob: bytes, Message to be signed. - - Raises: - NotImplementedError, always. - """ - raise NotImplementedError( - 'Compute Engine service accounts cannot sign blobs') diff --git a/src/oauth2client/contrib/keyring_storage.py b/src/oauth2client/contrib/keyring_storage.py deleted file mode 100644 index 4af94488..00000000 --- a/src/oauth2client/contrib/keyring_storage.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A keyring based Storage. - -A Storage for Credentials that uses the keyring module. -""" - -import threading - -import keyring - -from oauth2client import client - - -class Storage(client.Storage): - """Store and retrieve a single credential to and from the keyring. - - To use this module you must have the keyring module installed. See - . This is an optional module and is - not installed with oauth2client by default because it does not work on all - the platforms that oauth2client supports, such as Google App Engine. - - The keyring module is a - cross-platform library for access the keyring capabilities of the local - system. The user will be prompted for their keyring password when this - module is used, and the manner in which the user is prompted will vary per - platform. - - Usage:: - - from oauth2client import keyring_storage - - s = keyring_storage.Storage('name_of_application', 'user1') - credentials = s.get() - - """ - - def __init__(self, service_name, user_name): - """Constructor. - - Args: - service_name: string, The name of the service under which the - credentials are stored. - user_name: string, The name of the user to store credentials for. - """ - super(Storage, self).__init__(lock=threading.Lock()) - self._service_name = service_name - self._user_name = user_name - - def locked_get(self): - """Retrieve Credential from file. - - Returns: - oauth2client.client.Credentials - """ - credentials = None - content = keyring.get_password(self._service_name, self._user_name) - - if content is not None: - try: - credentials = client.Credentials.new_from_json(content) - credentials.set_store(self) - except ValueError: - pass - - return credentials - - def locked_put(self, credentials): - """Write Credentials to file. - - Args: - credentials: Credentials, the credentials to store. - """ - keyring.set_password(self._service_name, self._user_name, - credentials.to_json()) - - def locked_delete(self): - """Delete Credentials file. - - Args: - credentials: Credentials, the credentials to store. - """ - keyring.set_password(self._service_name, self._user_name, '') diff --git a/src/oauth2client/contrib/multiprocess_file_storage.py b/src/oauth2client/contrib/multiprocess_file_storage.py deleted file mode 100644 index e9e8c8cd..00000000 --- a/src/oauth2client/contrib/multiprocess_file_storage.py +++ /dev/null @@ -1,355 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Multiprocess file credential storage. - -This module provides file-based storage that supports multiple credentials and -cross-thread and process access. - -This module supersedes the functionality previously found in `multistore_file`. - -This module provides :class:`MultiprocessFileStorage` which: - * Is tied to a single credential via a user-specified key. This key can be - used to distinguish between multiple users, client ids, and/or scopes. - * Can be safely accessed and refreshed across threads and processes. - -Process & thread safety guarantees the following behavior: - * If one thread or process refreshes a credential, subsequent refreshes - from other processes will re-fetch the credentials from the file instead - of performing an http request. - * If two processes or threads attempt to refresh concurrently, only one - will be able to acquire the lock and refresh, with the deadlock caveat - below. - * The interprocess lock will not deadlock, instead, the if a process can - not acquire the interprocess lock within ``INTERPROCESS_LOCK_DEADLINE`` - it will allow refreshing the credential but will not write the updated - credential to disk, This logic happens during every lock cycle - if the - credentials are refreshed again it will retry locking and writing as - normal. - -Usage -===== - -Before using the storage, you need to decide how you want to key the -credentials. A few common strategies include: - - * If you're storing credentials for multiple users in a single file, use - a unique identifier for each user as the key. - * If you're storing credentials for multiple client IDs in a single file, - use the client ID as the key. - * If you're storing multiple credentials for one user, use the scopes as - the key. - * If you have a complicated setup, use a compound key. For example, you - can use a combination of the client ID and scopes as the key. - -Create an instance of :class:`MultiprocessFileStorage` for each credential you -want to store, for example:: - - filename = 'credentials' - key = '{}-{}'.format(client_id, user_id) - storage = MultiprocessFileStorage(filename, key) - -To store the credentials:: - - storage.put(credentials) - -If you're going to continue to use the credentials after storing them, be sure -to call :func:`set_store`:: - - credentials.set_store(storage) - -To retrieve the credentials:: - - storage.get(credentials) - -""" - -import base64 -import json -import logging -import os -import threading - -import fasteners -from six import iteritems - -from oauth2client import _helpers -from oauth2client import client - - -#: The maximum amount of time, in seconds, to wait when acquire the -#: interprocess lock before falling back to read-only mode. -INTERPROCESS_LOCK_DEADLINE = 1 - -logger = logging.getLogger(__name__) -_backends = {} -_backends_lock = threading.Lock() - - -def _create_file_if_needed(filename): - """Creates the an empty file if it does not already exist. - - Returns: - True if the file was created, False otherwise. - """ - if os.path.exists(filename): - return False - else: - # Equivalent to "touch". - open(filename, 'a+b').close() - logger.info('Credential file {0} created'.format(filename)) - return True - - -def _load_credentials_file(credentials_file): - """Load credentials from the given file handle. - - The file is expected to be in this format: - - { - "file_version": 2, - "credentials": { - "key": "base64 encoded json representation of credentials." - } - } - - This function will warn and return empty credentials instead of raising - exceptions. - - Args: - credentials_file: An open file handle. - - Returns: - A dictionary mapping user-defined keys to an instance of - :class:`oauth2client.client.Credentials`. - """ - try: - credentials_file.seek(0) - data = json.load(credentials_file) - except Exception: - logger.warning( - 'Credentials file could not be loaded, will ignore and ' - 'overwrite.') - return {} - - if data.get('file_version') != 2: - logger.warning( - 'Credentials file is not version 2, will ignore and ' - 'overwrite.') - return {} - - credentials = {} - - for key, encoded_credential in iteritems(data.get('credentials', {})): - try: - credential_json = base64.b64decode(encoded_credential) - credential = client.Credentials.new_from_json(credential_json) - credentials[key] = credential - except: - logger.warning( - 'Invalid credential {0} in file, ignoring.'.format(key)) - - return credentials - - -def _write_credentials_file(credentials_file, credentials): - """Writes credentials to a file. - - Refer to :func:`_load_credentials_file` for the format. - - Args: - credentials_file: An open file handle, must be read/write. - credentials: A dictionary mapping user-defined keys to an instance of - :class:`oauth2client.client.Credentials`. - """ - data = {'file_version': 2, 'credentials': {}} - - for key, credential in iteritems(credentials): - credential_json = credential.to_json() - encoded_credential = _helpers._from_bytes(base64.b64encode( - _helpers._to_bytes(credential_json))) - data['credentials'][key] = encoded_credential - - credentials_file.seek(0) - json.dump(data, credentials_file) - credentials_file.truncate() - - -class _MultiprocessStorageBackend(object): - """Thread-local backend for multiprocess storage. - - Each process has only one instance of this backend per file. All threads - share a single instance of this backend. This ensures that all threads - use the same thread lock and process lock when accessing the file. - """ - - def __init__(self, filename): - self._file = None - self._filename = filename - self._process_lock = fasteners.InterProcessLock( - '{0}.lock'.format(filename)) - self._thread_lock = threading.Lock() - self._read_only = False - self._credentials = {} - - def _load_credentials(self): - """(Re-)loads the credentials from the file.""" - if not self._file: - return - - loaded_credentials = _load_credentials_file(self._file) - self._credentials.update(loaded_credentials) - - logger.debug('Read credential file') - - def _write_credentials(self): - if self._read_only: - logger.debug('In read-only mode, not writing credentials.') - return - - _write_credentials_file(self._file, self._credentials) - logger.debug('Wrote credential file {0}.'.format(self._filename)) - - def acquire_lock(self): - self._thread_lock.acquire() - locked = self._process_lock.acquire(timeout=INTERPROCESS_LOCK_DEADLINE) - - if locked: - _create_file_if_needed(self._filename) - self._file = open(self._filename, 'r+') - self._read_only = False - - else: - logger.warn( - 'Failed to obtain interprocess lock for credentials. ' - 'If a credential is being refreshed, other processes may ' - 'not see the updated access token and refresh as well.') - if os.path.exists(self._filename): - self._file = open(self._filename, 'r') - else: - self._file = None - self._read_only = True - - self._load_credentials() - - def release_lock(self): - if self._file is not None: - self._file.close() - self._file = None - - if not self._read_only: - self._process_lock.release() - - self._thread_lock.release() - - def _refresh_predicate(self, credentials): - if credentials is None: - return True - elif credentials.invalid: - return True - elif credentials.access_token_expired: - return True - else: - return False - - def locked_get(self, key): - # Check if the credential is already in memory. - credentials = self._credentials.get(key, None) - - # Use the refresh predicate to determine if the entire store should be - # reloaded. This basically checks if the credentials are invalid - # or expired. This covers the situation where another process has - # refreshed the credentials and this process doesn't know about it yet. - # In that case, this process won't needlessly refresh the credentials. - if self._refresh_predicate(credentials): - self._load_credentials() - credentials = self._credentials.get(key, None) - - return credentials - - def locked_put(self, key, credentials): - self._load_credentials() - self._credentials[key] = credentials - self._write_credentials() - - def locked_delete(self, key): - self._load_credentials() - self._credentials.pop(key, None) - self._write_credentials() - - -def _get_backend(filename): - """A helper method to get or create a backend with thread locking. - - This ensures that only one backend is used per-file per-process, so that - thread and process locks are appropriately shared. - - Args: - filename: The full path to the credential storage file. - - Returns: - An instance of :class:`_MultiprocessStorageBackend`. - """ - filename = os.path.abspath(filename) - - with _backends_lock: - if filename not in _backends: - _backends[filename] = _MultiprocessStorageBackend(filename) - return _backends[filename] - - -class MultiprocessFileStorage(client.Storage): - """Multiprocess file credential storage. - - Args: - filename: The path to the file where credentials will be stored. - key: An arbitrary string used to uniquely identify this set of - credentials. For example, you may use the user's ID as the key or - a combination of the client ID and user ID. - """ - def __init__(self, filename, key): - self._key = key - self._backend = _get_backend(filename) - - def acquire_lock(self): - self._backend.acquire_lock() - - def release_lock(self): - self._backend.release_lock() - - def locked_get(self): - """Retrieves the current credentials from the store. - - Returns: - An instance of :class:`oauth2client.client.Credentials` or `None`. - """ - credential = self._backend.locked_get(self._key) - - if credential is not None: - credential.set_store(self) - - return credential - - def locked_put(self, credentials): - """Writes the given credentials to the store. - - Args: - credentials: an instance of - :class:`oauth2client.client.Credentials`. - """ - return self._backend.locked_put(self._key, credentials) - - def locked_delete(self): - """Deletes the current credentials from the store.""" - return self._backend.locked_delete(self._key) diff --git a/src/oauth2client/contrib/sqlalchemy.py b/src/oauth2client/contrib/sqlalchemy.py deleted file mode 100644 index 7d9fd4b2..00000000 --- a/src/oauth2client/contrib/sqlalchemy.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""OAuth 2.0 utilities for SQLAlchemy. - -Utilities for using OAuth 2.0 in conjunction with a SQLAlchemy. - -Configuration -============= - -In order to use this storage, you'll need to create table -with :class:`oauth2client.contrib.sqlalchemy.CredentialsType` column. -It's recommended to either put this column on some sort of user info -table or put the column in a table with a belongs-to relationship to -a user info table. - -Here's an example of a simple table with a :class:`CredentialsType` -column that's related to a user table by the `user_id` key. - -.. code-block:: python - - from sqlalchemy import Column, ForeignKey, Integer - from sqlalchemy.ext.declarative import declarative_base - from sqlalchemy.orm import relationship - - from oauth2client.contrib.sqlalchemy import CredentialsType - - - Base = declarative_base() - - - class Credentials(Base): - __tablename__ = 'credentials' - - user_id = Column(Integer, ForeignKey('user.id')) - credentials = Column(CredentialsType) - - - class User(Base): - id = Column(Integer, primary_key=True) - # bunch of other columns - credentials = relationship('Credentials') - - -Usage -===== - -With tables ready, you are now able to store credentials in database. -We will reuse tables defined above. - -.. code-block:: python - - from sqlalchemy.orm import Session - - from oauth2client.client import OAuth2Credentials - from oauth2client.contrib.sql_alchemy import Storage - - session = Session() - user = session.query(User).first() - storage = Storage( - session=session, - model_class=Credentials, - # This is the key column used to identify - # the row that stores the credentials. - key_name='user_id', - key_value=user.id, - property_name='credentials', - ) - - # Store - credentials = OAuth2Credentials(...) - storage.put(credentials) - - # Retrieve - credentials = storage.get() - - # Delete - storage.delete() - -""" - -from __future__ import absolute_import - -import sqlalchemy.types - -from oauth2client import client - - -class CredentialsType(sqlalchemy.types.PickleType): - """Type representing credentials. - - Alias for :class:`sqlalchemy.types.PickleType`. - """ - - -class Storage(client.Storage): - """Store and retrieve a single credential to and from SQLAlchemy. - This helper presumes the Credentials - have been stored as a Credentials column - on a db model class. - """ - - def __init__(self, session, model_class, key_name, - key_value, property_name): - """Constructor for Storage. - - Args: - session: An instance of :class:`sqlalchemy.orm.Session`. - model_class: SQLAlchemy declarative mapping. - key_name: string, key name for the entity that has the credentials - key_value: key value for the entity that has the credentials - property_name: A string indicating which property on the - ``model_class`` to store the credentials. - This property must be a - :class:`CredentialsType` column. - """ - super(Storage, self).__init__() - - self.session = session - self.model_class = model_class - self.key_name = key_name - self.key_value = key_value - self.property_name = property_name - - def locked_get(self): - """Retrieve stored credential. - - Returns: - A :class:`oauth2client.Credentials` instance or `None`. - """ - filters = {self.key_name: self.key_value} - query = self.session.query(self.model_class).filter_by(**filters) - entity = query.first() - - if entity: - credential = getattr(entity, self.property_name) - if credential and hasattr(credential, 'set_store'): - credential.set_store(self) - return credential - else: - return None - - def locked_put(self, credentials): - """Write a credentials to the SQLAlchemy datastore. - - Args: - credentials: :class:`oauth2client.Credentials` - """ - filters = {self.key_name: self.key_value} - query = self.session.query(self.model_class).filter_by(**filters) - entity = query.first() - - if not entity: - entity = self.model_class(**filters) - - setattr(entity, self.property_name, credentials) - self.session.add(entity) - - def locked_delete(self): - """Delete credentials from the SQLAlchemy datastore.""" - filters = {self.key_name: self.key_value} - self.session.query(self.model_class).filter_by(**filters).delete() diff --git a/src/oauth2client/contrib/xsrfutil.py b/src/oauth2client/contrib/xsrfutil.py deleted file mode 100644 index 7c3ec035..00000000 --- a/src/oauth2client/contrib/xsrfutil.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2014 the Melange authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper methods for creating & verifying XSRF tokens.""" - -import base64 -import binascii -import hmac -import time - -from oauth2client import _helpers - - -# Delimiter character -DELIMITER = b':' - -# 1 hour in seconds -DEFAULT_TIMEOUT_SECS = 60 * 60 - - -@_helpers.positional(2) -def generate_token(key, user_id, action_id='', when=None): - """Generates a URL-safe token for the given user, action, time tuple. - - Args: - key: secret key to use. - user_id: the user ID of the authenticated user. - action_id: a string identifier of the action they requested - authorization for. - when: the time in seconds since the epoch at which the user was - authorized for this action. If not set the current time is used. - - Returns: - A string XSRF protection token. - """ - digester = hmac.new(_helpers._to_bytes(key, encoding='utf-8')) - digester.update(_helpers._to_bytes(str(user_id), encoding='utf-8')) - digester.update(DELIMITER) - digester.update(_helpers._to_bytes(action_id, encoding='utf-8')) - digester.update(DELIMITER) - when = _helpers._to_bytes(str(when or int(time.time())), encoding='utf-8') - digester.update(when) - digest = digester.digest() - - token = base64.urlsafe_b64encode(digest + DELIMITER + when) - return token - - -@_helpers.positional(3) -def validate_token(key, token, user_id, action_id="", current_time=None): - """Validates that the given token authorizes the user for the action. - - Tokens are invalid if the time of issue is too old or if the token - does not match what generateToken outputs (i.e. the token was forged). - - Args: - key: secret key to use. - token: a string of the token generated by generateToken. - user_id: the user ID of the authenticated user. - action_id: a string identifier of the action they requested - authorization for. - - Returns: - A boolean - True if the user is authorized for the action, False - otherwise. - """ - if not token: - return False - try: - decoded = base64.urlsafe_b64decode(token) - token_time = int(decoded.split(DELIMITER)[-1]) - except (TypeError, ValueError, binascii.Error): - return False - if current_time is None: - current_time = time.time() - # If the token is too old it's not valid. - if current_time - token_time > DEFAULT_TIMEOUT_SECS: - return False - - # The given token should match the generated one with the same time. - expected_token = generate_token(key, user_id, action_id=action_id, - when=token_time) - if len(token) != len(expected_token): - return False - - # Perform constant time comparison to avoid timing attacks - different = 0 - for x, y in zip(bytearray(token), bytearray(expected_token)): - different |= x ^ y - return not different diff --git a/src/oauth2client/crypt.py b/src/oauth2client/crypt.py deleted file mode 100644 index 13260982..00000000 --- a/src/oauth2client/crypt.py +++ /dev/null @@ -1,250 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Crypto-related routines for oauth2client.""" - -import json -import logging -import time - -from oauth2client import _helpers -from oauth2client import _pure_python_crypt - - -RsaSigner = _pure_python_crypt.RsaSigner -RsaVerifier = _pure_python_crypt.RsaVerifier - -CLOCK_SKEW_SECS = 300 # 5 minutes in seconds -AUTH_TOKEN_LIFETIME_SECS = 300 # 5 minutes in seconds -MAX_TOKEN_LIFETIME_SECS = 86400 # 1 day in seconds - -logger = logging.getLogger(__name__) - - -class AppIdentityError(Exception): - """Error to indicate crypto failure.""" - - -def _bad_pkcs12_key_as_pem(*args, **kwargs): - raise NotImplementedError('pkcs12_key_as_pem requires OpenSSL.') - - -try: - from oauth2client import _openssl_crypt - OpenSSLSigner = _openssl_crypt.OpenSSLSigner - OpenSSLVerifier = _openssl_crypt.OpenSSLVerifier - pkcs12_key_as_pem = _openssl_crypt.pkcs12_key_as_pem -except ImportError: # pragma: NO COVER - OpenSSLVerifier = None - OpenSSLSigner = None - pkcs12_key_as_pem = _bad_pkcs12_key_as_pem - -try: - from oauth2client import _pycrypto_crypt - PyCryptoSigner = _pycrypto_crypt.PyCryptoSigner - PyCryptoVerifier = _pycrypto_crypt.PyCryptoVerifier -except ImportError: # pragma: NO COVER - PyCryptoVerifier = None - PyCryptoSigner = None - - -if OpenSSLSigner: - Signer = OpenSSLSigner - Verifier = OpenSSLVerifier -elif PyCryptoSigner: # pragma: NO COVER - Signer = PyCryptoSigner - Verifier = PyCryptoVerifier -else: # pragma: NO COVER - Signer = RsaSigner - Verifier = RsaVerifier - - -def make_signed_jwt(signer, payload, key_id=None): - """Make a signed JWT. - - See http://self-issued.info/docs/draft-jones-json-web-token.html. - - Args: - signer: crypt.Signer, Cryptographic signer. - payload: dict, Dictionary of data to convert to JSON and then sign. - key_id: string, (Optional) Key ID header. - - Returns: - string, The JWT for the payload. - """ - header = {'typ': 'JWT', 'alg': 'RS256'} - if key_id is not None: - header['kid'] = key_id - - segments = [ - _helpers._urlsafe_b64encode(_helpers._json_encode(header)), - _helpers._urlsafe_b64encode(_helpers._json_encode(payload)), - ] - signing_input = b'.'.join(segments) - - signature = signer.sign(signing_input) - segments.append(_helpers._urlsafe_b64encode(signature)) - - logger.debug(str(segments)) - - return b'.'.join(segments) - - -def _verify_signature(message, signature, certs): - """Verifies signed content using a list of certificates. - - Args: - message: string or bytes, The message to verify. - signature: string or bytes, The signature on the message. - certs: iterable, certificates in PEM format. - - Raises: - AppIdentityError: If none of the certificates can verify the message - against the signature. - """ - for pem in certs: - verifier = Verifier.from_string(pem, is_x509_cert=True) - if verifier.verify(message, signature): - return - - # If we have not returned, no certificate confirms the signature. - raise AppIdentityError('Invalid token signature') - - -def _check_audience(payload_dict, audience): - """Checks audience field from a JWT payload. - - Does nothing if the passed in ``audience`` is null. - - Args: - payload_dict: dict, A dictionary containing a JWT payload. - audience: string or NoneType, an audience to check for in - the JWT payload. - - Raises: - AppIdentityError: If there is no ``'aud'`` field in the payload - dictionary but there is an ``audience`` to check. - AppIdentityError: If the ``'aud'`` field in the payload dictionary - does not match the ``audience``. - """ - if audience is None: - return - - audience_in_payload = payload_dict.get('aud') - if audience_in_payload is None: - raise AppIdentityError( - 'No aud field in token: {0}'.format(payload_dict)) - if audience_in_payload != audience: - raise AppIdentityError('Wrong recipient, {0} != {1}: {2}'.format( - audience_in_payload, audience, payload_dict)) - - -def _verify_time_range(payload_dict): - """Verifies the issued at and expiration from a JWT payload. - - Makes sure the current time (in UTC) falls between the issued at and - expiration for the JWT (with some skew allowed for via - ``CLOCK_SKEW_SECS``). - - Args: - payload_dict: dict, A dictionary containing a JWT payload. - - Raises: - AppIdentityError: If there is no ``'iat'`` field in the payload - dictionary. - AppIdentityError: If there is no ``'exp'`` field in the payload - dictionary. - AppIdentityError: If the JWT expiration is too far in the future (i.e. - if the expiration would imply a token lifetime - longer than what is allowed.) - AppIdentityError: If the token appears to have been issued in the - future (up to clock skew). - AppIdentityError: If the token appears to have expired in the past - (up to clock skew). - """ - # Get the current time to use throughout. - now = int(time.time()) - - # Make sure issued at and expiration are in the payload. - issued_at = payload_dict.get('iat') - if issued_at is None: - raise AppIdentityError( - 'No iat field in token: {0}'.format(payload_dict)) - expiration = payload_dict.get('exp') - if expiration is None: - raise AppIdentityError( - 'No exp field in token: {0}'.format(payload_dict)) - - # Make sure the expiration gives an acceptable token lifetime. - if expiration >= now + MAX_TOKEN_LIFETIME_SECS: - raise AppIdentityError( - 'exp field too far in future: {0}'.format(payload_dict)) - - # Make sure (up to clock skew) that the token wasn't issued in the future. - earliest = issued_at - CLOCK_SKEW_SECS - if now < earliest: - raise AppIdentityError('Token used too early, {0} < {1}: {2}'.format( - now, earliest, payload_dict)) - # Make sure (up to clock skew) that the token isn't already expired. - latest = expiration + CLOCK_SKEW_SECS - if now > latest: - raise AppIdentityError('Token used too late, {0} > {1}: {2}'.format( - now, latest, payload_dict)) - - -def verify_signed_jwt_with_certs(jwt, certs, audience=None): - """Verify a JWT against public certs. - - See http://self-issued.info/docs/draft-jones-json-web-token.html. - - Args: - jwt: string, A JWT. - certs: dict, Dictionary where values of public keys in PEM format. - audience: string, The audience, 'aud', that this JWT should contain. If - None then the JWT's 'aud' parameter is not verified. - - Returns: - dict, The deserialized JSON payload in the JWT. - - Raises: - AppIdentityError: if any checks are failed. - """ - jwt = _helpers._to_bytes(jwt) - - if jwt.count(b'.') != 2: - raise AppIdentityError( - 'Wrong number of segments in token: {0}'.format(jwt)) - - header, payload, signature = jwt.split(b'.') - message_to_sign = header + b'.' + payload - signature = _helpers._urlsafe_b64decode(signature) - - # Parse token. - payload_bytes = _helpers._urlsafe_b64decode(payload) - try: - payload_dict = json.loads(_helpers._from_bytes(payload_bytes)) - except: - raise AppIdentityError('Can\'t parse token: {0}'.format(payload_bytes)) - - # Verify that the signature matches the message. - _verify_signature(message_to_sign, signature, certs.values()) - - # Verify the issued at and created times in the payload. - _verify_time_range(payload_dict) - - # Check audience. - _check_audience(payload_dict, audience) - - return payload_dict diff --git a/src/oauth2client/file.py b/src/oauth2client/file.py deleted file mode 100644 index 3551c80d..00000000 --- a/src/oauth2client/file.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for OAuth. - -Utilities for making it easier to work with OAuth 2.0 -credentials. -""" - -import os -import threading - -from oauth2client import _helpers -from oauth2client import client - - -class Storage(client.Storage): - """Store and retrieve a single credential to and from a file.""" - - def __init__(self, filename): - super(Storage, self).__init__(lock=threading.Lock()) - self._filename = filename - - def locked_get(self): - """Retrieve Credential from file. - - Returns: - oauth2client.client.Credentials - - Raises: - IOError if the file is a symbolic link. - """ - credentials = None - _helpers.validate_file(self._filename) - try: - f = open(self._filename, 'rb') - content = f.read() - f.close() - except IOError: - return credentials - - try: - credentials = client.Credentials.new_from_json(content) - credentials.set_store(self) - except ValueError: - pass - - return credentials - - def _create_file_if_needed(self): - """Create an empty file if necessary. - - This method will not initialize the file. Instead it implements a - simple version of "touch" to ensure the file has been created. - """ - if not os.path.exists(self._filename): - old_umask = os.umask(0o177) - try: - open(self._filename, 'a+b').close() - finally: - os.umask(old_umask) - - def locked_put(self, credentials): - """Write Credentials to file. - - Args: - credentials: Credentials, the credentials to store. - - Raises: - IOError if the file is a symbolic link. - """ - self._create_file_if_needed() - _helpers.validate_file(self._filename) - f = open(self._filename, 'w') - f.write(credentials.to_json()) - f.close() - - def locked_delete(self): - """Delete Credentials file. - - Args: - credentials: Credentials, the credentials to store. - """ - os.unlink(self._filename) diff --git a/src/oauth2client/service_account.py b/src/oauth2client/service_account.py deleted file mode 100644 index 540bfaaa..00000000 --- a/src/oauth2client/service_account.py +++ /dev/null @@ -1,685 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""oauth2client Service account credentials class.""" - -import base64 -import copy -import datetime -import json -import time - -import oauth2client -from oauth2client import _helpers -from oauth2client import client -from oauth2client import crypt -from oauth2client import transport - - -_PASSWORD_DEFAULT = 'notasecret' -_PKCS12_KEY = '_private_key_pkcs12' -_PKCS12_ERROR = r""" -This library only implements PKCS#12 support via the pyOpenSSL library. -Either install pyOpenSSL, or please convert the .p12 file -to .pem format: - $ cat key.p12 | \ - > openssl pkcs12 -nodes -nocerts -passin pass:notasecret | \ - > openssl rsa > key.pem -""" - - -class ServiceAccountCredentials(client.AssertionCredentials): - """Service Account credential for OAuth 2.0 signed JWT grants. - - Supports - - * JSON keyfile (typically contains a PKCS8 key stored as - PEM text) - * ``.p12`` key (stores PKCS12 key and certificate) - - Makes an assertion to server using a signed JWT assertion in exchange - for an access token. - - This credential does not require a flow to instantiate because it - represents a two legged flow, and therefore has all of the required - information to generate and refresh its own access tokens. - - Args: - service_account_email: string, The email associated with the - service account. - signer: ``crypt.Signer``, A signer which can be used to sign content. - scopes: List or string, (Optional) Scopes to use when acquiring - an access token. - private_key_id: string, (Optional) Private key identifier. Typically - only used with a JSON keyfile. Can be sent in the - header of a JWT token assertion. - client_id: string, (Optional) Client ID for the project that owns the - service account. - user_agent: string, (Optional) User agent to use when sending - request. - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - revoke_uri: string, URI for revoke endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - kwargs: dict, Extra key-value pairs (both strings) to send in the - payload body when making an assertion. - """ - - MAX_TOKEN_LIFETIME_SECS = 3600 - """Max lifetime of the token (one hour, in seconds).""" - - NON_SERIALIZED_MEMBERS = ( - frozenset(['_signer']) | - client.AssertionCredentials.NON_SERIALIZED_MEMBERS) - """Members that aren't serialized when object is converted to JSON.""" - - # Can be over-ridden by factory constructors. Used for - # serialization/deserialization purposes. - _private_key_pkcs8_pem = None - _private_key_pkcs12 = None - _private_key_password = None - - def __init__(self, - service_account_email, - signer, - scopes='', - private_key_id=None, - client_id=None, - user_agent=None, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI, - **kwargs): - - super(ServiceAccountCredentials, self).__init__( - None, user_agent=user_agent, token_uri=token_uri, - revoke_uri=revoke_uri) - - self._service_account_email = service_account_email - self._signer = signer - self._scopes = _helpers.scopes_to_string(scopes) - self._private_key_id = private_key_id - self.client_id = client_id - self._user_agent = user_agent - self._kwargs = kwargs - - def _to_json(self, strip, to_serialize=None): - """Utility function that creates JSON repr. of a credentials object. - - Over-ride is needed since PKCS#12 keys will not in general be JSON - serializable. - - Args: - strip: array, An array of names of members to exclude from the - JSON. - to_serialize: dict, (Optional) The properties for this object - that will be serialized. This allows callers to - modify before serializing. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - if to_serialize is None: - to_serialize = copy.copy(self.__dict__) - pkcs12_val = to_serialize.get(_PKCS12_KEY) - if pkcs12_val is not None: - to_serialize[_PKCS12_KEY] = base64.b64encode(pkcs12_val) - return super(ServiceAccountCredentials, self)._to_json( - strip, to_serialize=to_serialize) - - @classmethod - def _from_parsed_json_keyfile(cls, keyfile_dict, scopes, - token_uri=None, revoke_uri=None): - """Helper for factory constructors from JSON keyfile. - - Args: - keyfile_dict: dict-like object, The parsed dictionary-like object - containing the contents of the JSON keyfile. - scopes: List or string, Scopes to use when acquiring an - access token. - token_uri: string, URI for OAuth 2.0 provider token endpoint. - If unset and not present in keyfile_dict, defaults - to Google's endpoints. - revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint. - If unset and not present in keyfile_dict, defaults - to Google's endpoints. - - Returns: - ServiceAccountCredentials, a credentials object created from - the keyfile contents. - - Raises: - ValueError, if the credential type is not :data:`SERVICE_ACCOUNT`. - KeyError, if one of the expected keys is not present in - the keyfile. - """ - creds_type = keyfile_dict.get('type') - if creds_type != client.SERVICE_ACCOUNT: - raise ValueError('Unexpected credentials type', creds_type, - 'Expected', client.SERVICE_ACCOUNT) - - service_account_email = keyfile_dict['client_email'] - private_key_pkcs8_pem = keyfile_dict['private_key'] - private_key_id = keyfile_dict['private_key_id'] - client_id = keyfile_dict['client_id'] - if not token_uri: - token_uri = keyfile_dict.get('token_uri', - oauth2client.GOOGLE_TOKEN_URI) - if not revoke_uri: - revoke_uri = keyfile_dict.get('revoke_uri', - oauth2client.GOOGLE_REVOKE_URI) - - signer = crypt.Signer.from_string(private_key_pkcs8_pem) - credentials = cls(service_account_email, signer, scopes=scopes, - private_key_id=private_key_id, - client_id=client_id, token_uri=token_uri, - revoke_uri=revoke_uri) - credentials._private_key_pkcs8_pem = private_key_pkcs8_pem - return credentials - - @classmethod - def from_json_keyfile_name(cls, filename, scopes='', - token_uri=None, revoke_uri=None): - - """Factory constructor from JSON keyfile by name. - - Args: - filename: string, The location of the keyfile. - scopes: List or string, (Optional) Scopes to use when acquiring an - access token. - token_uri: string, URI for OAuth 2.0 provider token endpoint. - If unset and not present in the key file, defaults - to Google's endpoints. - revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint. - If unset and not present in the key file, defaults - to Google's endpoints. - - Returns: - ServiceAccountCredentials, a credentials object created from - the keyfile. - - Raises: - ValueError, if the credential type is not :data:`SERVICE_ACCOUNT`. - KeyError, if one of the expected keys is not present in - the keyfile. - """ - with open(filename, 'r') as file_obj: - client_credentials = json.load(file_obj) - return cls._from_parsed_json_keyfile(client_credentials, scopes, - token_uri=token_uri, - revoke_uri=revoke_uri) - - @classmethod - def from_json_keyfile_dict(cls, keyfile_dict, scopes='', - token_uri=None, revoke_uri=None): - """Factory constructor from parsed JSON keyfile. - - Args: - keyfile_dict: dict-like object, The parsed dictionary-like object - containing the contents of the JSON keyfile. - scopes: List or string, (Optional) Scopes to use when acquiring an - access token. - token_uri: string, URI for OAuth 2.0 provider token endpoint. - If unset and not present in keyfile_dict, defaults - to Google's endpoints. - revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint. - If unset and not present in keyfile_dict, defaults - to Google's endpoints. - - Returns: - ServiceAccountCredentials, a credentials object created from - the keyfile. - - Raises: - ValueError, if the credential type is not :data:`SERVICE_ACCOUNT`. - KeyError, if one of the expected keys is not present in - the keyfile. - """ - return cls._from_parsed_json_keyfile(keyfile_dict, scopes, - token_uri=token_uri, - revoke_uri=revoke_uri) - - @classmethod - def _from_p12_keyfile_contents(cls, service_account_email, - private_key_pkcs12, - private_key_password=None, scopes='', - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI): - """Factory constructor from JSON keyfile. - - Args: - service_account_email: string, The email associated with the - service account. - private_key_pkcs12: string, The contents of a PKCS#12 keyfile. - private_key_password: string, (Optional) Password for PKCS#12 - private key. Defaults to ``notasecret``. - scopes: List or string, (Optional) Scopes to use when acquiring an - access token. - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - revoke_uri: string, URI for revoke endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 - provider can be used. - - Returns: - ServiceAccountCredentials, a credentials object created from - the keyfile. - - Raises: - NotImplementedError if pyOpenSSL is not installed / not the - active crypto library. - """ - if private_key_password is None: - private_key_password = _PASSWORD_DEFAULT - if crypt.Signer is not crypt.OpenSSLSigner: - raise NotImplementedError(_PKCS12_ERROR) - signer = crypt.Signer.from_string(private_key_pkcs12, - private_key_password) - credentials = cls(service_account_email, signer, scopes=scopes, - token_uri=token_uri, revoke_uri=revoke_uri) - credentials._private_key_pkcs12 = private_key_pkcs12 - credentials._private_key_password = private_key_password - return credentials - - @classmethod - def from_p12_keyfile(cls, service_account_email, filename, - private_key_password=None, scopes='', - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI): - - """Factory constructor from JSON keyfile. - - Args: - service_account_email: string, The email associated with the - service account. - filename: string, The location of the PKCS#12 keyfile. - private_key_password: string, (Optional) Password for PKCS#12 - private key. Defaults to ``notasecret``. - scopes: List or string, (Optional) Scopes to use when acquiring an - access token. - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - revoke_uri: string, URI for revoke endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 - provider can be used. - - Returns: - ServiceAccountCredentials, a credentials object created from - the keyfile. - - Raises: - NotImplementedError if pyOpenSSL is not installed / not the - active crypto library. - """ - with open(filename, 'rb') as file_obj: - private_key_pkcs12 = file_obj.read() - return cls._from_p12_keyfile_contents( - service_account_email, private_key_pkcs12, - private_key_password=private_key_password, scopes=scopes, - token_uri=token_uri, revoke_uri=revoke_uri) - - @classmethod - def from_p12_keyfile_buffer(cls, service_account_email, file_buffer, - private_key_password=None, scopes='', - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI): - """Factory constructor from JSON keyfile. - - Args: - service_account_email: string, The email associated with the - service account. - file_buffer: stream, A buffer that implements ``read()`` - and contains the PKCS#12 key contents. - private_key_password: string, (Optional) Password for PKCS#12 - private key. Defaults to ``notasecret``. - scopes: List or string, (Optional) Scopes to use when acquiring an - access token. - token_uri: string, URI for token endpoint. For convenience defaults - to Google's endpoints but any OAuth 2.0 provider can be - used. - revoke_uri: string, URI for revoke endpoint. For convenience - defaults to Google's endpoints but any OAuth 2.0 - provider can be used. - - Returns: - ServiceAccountCredentials, a credentials object created from - the keyfile. - - Raises: - NotImplementedError if pyOpenSSL is not installed / not the - active crypto library. - """ - private_key_pkcs12 = file_buffer.read() - return cls._from_p12_keyfile_contents( - service_account_email, private_key_pkcs12, - private_key_password=private_key_password, scopes=scopes, - token_uri=token_uri, revoke_uri=revoke_uri) - - def _generate_assertion(self): - """Generate the assertion that will be used in the request.""" - now = int(time.time()) - payload = { - 'aud': self.token_uri, - 'scope': self._scopes, - 'iat': now, - 'exp': now + self.MAX_TOKEN_LIFETIME_SECS, - 'iss': self._service_account_email, - } - payload.update(self._kwargs) - return crypt.make_signed_jwt(self._signer, payload, - key_id=self._private_key_id) - - def sign_blob(self, blob): - """Cryptographically sign a blob (of bytes). - - Implements abstract method - :meth:`oauth2client.client.AssertionCredentials.sign_blob`. - - Args: - blob: bytes, Message to be signed. - - Returns: - tuple, A pair of the private key ID used to sign the blob and - the signed contents. - """ - return self._private_key_id, self._signer.sign(blob) - - @property - def service_account_email(self): - """Get the email for the current service account. - - Returns: - string, The email associated with the service account. - """ - return self._service_account_email - - @property - def serialization_data(self): - # NOTE: This is only useful for JSON keyfile. - return { - 'type': 'service_account', - 'client_email': self._service_account_email, - 'private_key_id': self._private_key_id, - 'private_key': self._private_key_pkcs8_pem, - 'client_id': self.client_id, - } - - @classmethod - def from_json(cls, json_data): - """Deserialize a JSON-serialized instance. - - Inverse to :meth:`to_json`. - - Args: - json_data: dict or string, Serialized JSON (as a string or an - already parsed dictionary) representing a credential. - - Returns: - ServiceAccountCredentials from the serialized data. - """ - if not isinstance(json_data, dict): - json_data = json.loads(_helpers._from_bytes(json_data)) - - private_key_pkcs8_pem = None - pkcs12_val = json_data.get(_PKCS12_KEY) - password = None - if pkcs12_val is None: - private_key_pkcs8_pem = json_data['_private_key_pkcs8_pem'] - signer = crypt.Signer.from_string(private_key_pkcs8_pem) - else: - # NOTE: This assumes that private_key_pkcs8_pem is not also - # in the serialized data. This would be very incorrect - # state. - pkcs12_val = base64.b64decode(pkcs12_val) - password = json_data['_private_key_password'] - signer = crypt.Signer.from_string(pkcs12_val, password) - - credentials = cls( - json_data['_service_account_email'], - signer, - scopes=json_data['_scopes'], - private_key_id=json_data['_private_key_id'], - client_id=json_data['client_id'], - user_agent=json_data['_user_agent'], - **json_data['_kwargs'] - ) - if private_key_pkcs8_pem is not None: - credentials._private_key_pkcs8_pem = private_key_pkcs8_pem - if pkcs12_val is not None: - credentials._private_key_pkcs12 = pkcs12_val - if password is not None: - credentials._private_key_password = password - credentials.invalid = json_data['invalid'] - credentials.access_token = json_data['access_token'] - credentials.token_uri = json_data['token_uri'] - credentials.revoke_uri = json_data['revoke_uri'] - token_expiry = json_data.get('token_expiry', None) - if token_expiry is not None: - credentials.token_expiry = datetime.datetime.strptime( - token_expiry, client.EXPIRY_FORMAT) - return credentials - - def create_scoped_required(self): - return not self._scopes - - def create_scoped(self, scopes): - result = self.__class__(self._service_account_email, - self._signer, - scopes=scopes, - private_key_id=self._private_key_id, - client_id=self.client_id, - user_agent=self._user_agent, - **self._kwargs) - result.token_uri = self.token_uri - result.revoke_uri = self.revoke_uri - result._private_key_pkcs8_pem = self._private_key_pkcs8_pem - result._private_key_pkcs12 = self._private_key_pkcs12 - result._private_key_password = self._private_key_password - return result - - def create_with_claims(self, claims): - """Create credentials that specify additional claims. - - Args: - claims: dict, key-value pairs for claims. - - Returns: - ServiceAccountCredentials, a copy of the current service account - credentials with updated claims to use when obtaining access - tokens. - """ - new_kwargs = dict(self._kwargs) - new_kwargs.update(claims) - result = self.__class__(self._service_account_email, - self._signer, - scopes=self._scopes, - private_key_id=self._private_key_id, - client_id=self.client_id, - user_agent=self._user_agent, - **new_kwargs) - result.token_uri = self.token_uri - result.revoke_uri = self.revoke_uri - result._private_key_pkcs8_pem = self._private_key_pkcs8_pem - result._private_key_pkcs12 = self._private_key_pkcs12 - result._private_key_password = self._private_key_password - return result - - def create_delegated(self, sub): - """Create credentials that act as domain-wide delegation of authority. - - Use the ``sub`` parameter as the subject to delegate on behalf of - that user. - - For example:: - - >>> account_sub = 'foo@email.com' - >>> delegate_creds = creds.create_delegated(account_sub) - - Args: - sub: string, An email address that this service account will - act on behalf of (via domain-wide delegation). - - Returns: - ServiceAccountCredentials, a copy of the current service account - updated to act on behalf of ``sub``. - """ - return self.create_with_claims({'sub': sub}) - - -def _datetime_to_secs(utc_time): - # TODO(issue 298): use time_delta.total_seconds() - # time_delta.total_seconds() not supported in Python 2.6 - epoch = datetime.datetime(1970, 1, 1) - time_delta = utc_time - epoch - return time_delta.days * 86400 + time_delta.seconds - - -class _JWTAccessCredentials(ServiceAccountCredentials): - """Self signed JWT credentials. - - Makes an assertion to server using a self signed JWT from service account - credentials. These credentials do NOT use OAuth 2.0 and instead - authenticate directly. - """ - _MAX_TOKEN_LIFETIME_SECS = 3600 - """Max lifetime of the token (one hour, in seconds).""" - - def __init__(self, - service_account_email, - signer, - scopes=None, - private_key_id=None, - client_id=None, - user_agent=None, - token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI, - additional_claims=None): - if additional_claims is None: - additional_claims = {} - super(_JWTAccessCredentials, self).__init__( - service_account_email, - signer, - private_key_id=private_key_id, - client_id=client_id, - user_agent=user_agent, - token_uri=token_uri, - revoke_uri=revoke_uri, - **additional_claims) - - def authorize(self, http): - """Authorize an httplib2.Http instance with a JWT assertion. - - Unless specified, the 'aud' of the assertion will be the base - uri of the request. - - Args: - http: An instance of ``httplib2.Http`` or something that acts - like it. - Returns: - A modified instance of http that was passed in. - Example:: - h = httplib2.Http() - h = credentials.authorize(h) - """ - transport.wrap_http_for_jwt_access(self, http) - return http - - def get_access_token(self, http=None, additional_claims=None): - """Create a signed jwt. - - Args: - http: unused - additional_claims: dict, additional claims to add to - the payload of the JWT. - Returns: - An AccessTokenInfo with the signed jwt - """ - if additional_claims is None: - if self.access_token is None or self.access_token_expired: - self.refresh(None) - return client.AccessTokenInfo( - access_token=self.access_token, expires_in=self._expires_in()) - else: - # Create a 1 time token - token, unused_expiry = self._create_token(additional_claims) - return client.AccessTokenInfo( - access_token=token, expires_in=self._MAX_TOKEN_LIFETIME_SECS) - - def revoke(self, http): - """Cannot revoke JWTAccessCredentials tokens.""" - pass - - def create_scoped_required(self): - # JWTAccessCredentials are unscoped by definition - return True - - def create_scoped(self, scopes, token_uri=oauth2client.GOOGLE_TOKEN_URI, - revoke_uri=oauth2client.GOOGLE_REVOKE_URI): - # Returns an OAuth2 credentials with the given scope - result = ServiceAccountCredentials(self._service_account_email, - self._signer, - scopes=scopes, - private_key_id=self._private_key_id, - client_id=self.client_id, - user_agent=self._user_agent, - token_uri=token_uri, - revoke_uri=revoke_uri, - **self._kwargs) - if self._private_key_pkcs8_pem is not None: - result._private_key_pkcs8_pem = self._private_key_pkcs8_pem - if self._private_key_pkcs12 is not None: - result._private_key_pkcs12 = self._private_key_pkcs12 - if self._private_key_password is not None: - result._private_key_password = self._private_key_password - return result - - def refresh(self, http): - """Refreshes the access_token. - - The HTTP object is unused since no request needs to be made to - get a new token, it can just be generated locally. - - Args: - http: unused HTTP object - """ - self._refresh(None) - - def _refresh(self, http): - """Refreshes the access_token. - - Args: - http: unused HTTP object - """ - self.access_token, self.token_expiry = self._create_token() - - def _create_token(self, additional_claims=None): - now = client._UTCNOW() - lifetime = datetime.timedelta(seconds=self._MAX_TOKEN_LIFETIME_SECS) - expiry = now + lifetime - payload = { - 'iat': _datetime_to_secs(now), - 'exp': _datetime_to_secs(expiry), - 'iss': self._service_account_email, - 'sub': self._service_account_email - } - payload.update(self._kwargs) - if additional_claims is not None: - payload.update(additional_claims) - jwt = crypt.make_signed_jwt(self._signer, payload, - key_id=self._private_key_id) - return jwt.decode('ascii'), expiry diff --git a/src/oauth2client/tools.py b/src/oauth2client/tools.py deleted file mode 100644 index 51669934..00000000 --- a/src/oauth2client/tools.py +++ /dev/null @@ -1,256 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Command-line tools for authenticating via OAuth 2.0 - -Do the OAuth 2.0 Web Server dance for a command line application. Stores the -generated credentials in a common file that is used by other example apps in -the same directory. -""" - -from __future__ import print_function - -import logging -import socket -import sys - -from six.moves import BaseHTTPServer -from six.moves import http_client -from six.moves import input -from six.moves import urllib - -from oauth2client import _helpers -from oauth2client import client - - -__all__ = ['argparser', 'run_flow', 'message_if_missing'] - -_CLIENT_SECRETS_MESSAGE = """WARNING: Please configure OAuth 2.0 - -To make this sample run you will need to populate the client_secrets.json file -found at: - - {file_path} - -with information from the APIs Console . - -""" - -_FAILED_START_MESSAGE = """ -Failed to start a local webserver listening on either port 8080 -or port 8090. Please check your firewall settings and locally -running programs that may be blocking or using those ports. - -Falling back to --noauth_local_webserver and continuing with -authorization. -""" - -_BROWSER_OPENED_MESSAGE = """ -Your browser has been opened to visit: - - {address} - -If your browser is on a different machine then exit and re-run this -application with the command-line parameter - - --noauth_local_webserver -""" - -_GO_TO_LINK_MESSAGE = """ -Go to the following link in your browser: - - {address} -""" - - -def _CreateArgumentParser(): - try: - import argparse - except ImportError: # pragma: NO COVER - return None - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('--auth_host_name', default='localhost', - help='Hostname when running a local web server.') - parser.add_argument('--noauth_local_webserver', action='store_true', - default=False, help='Do not run a local web server.') - parser.add_argument('--auth_host_port', default=[8080, 8090], type=int, - nargs='*', help='Port web server should listen on.') - parser.add_argument( - '--logging_level', default='ERROR', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - help='Set the logging level of detail.') - return parser - - -# argparser is an ArgumentParser that contains command-line options expected -# by tools.run(). Pass it in as part of the 'parents' argument to your own -# ArgumentParser. -argparser = _CreateArgumentParser() - - -class ClientRedirectServer(BaseHTTPServer.HTTPServer): - """A server to handle OAuth 2.0 redirects back to localhost. - - Waits for a single request and parses the query parameters - into query_params and then stops serving. - """ - query_params = {} - - -class ClientRedirectHandler(BaseHTTPServer.BaseHTTPRequestHandler): - """A handler for OAuth 2.0 redirects back to localhost. - - Waits for a single request and parses the query parameters - into the servers query_params and then stops serving. - """ - - def do_GET(self): - """Handle a GET request. - - Parses the query parameters and prints a message - if the flow has completed. Note that we can't detect - if an error occurred. - """ - self.send_response(http_client.OK) - self.send_header('Content-type', 'text/html') - self.end_headers() - parts = urllib.parse.urlparse(self.path) - query = _helpers.parse_unique_urlencoded(parts.query) - self.server.query_params = query - self.wfile.write( - b'Authentication Status') - self.wfile.write( - b'

The authentication flow has completed.

') - self.wfile.write(b'') - - def log_message(self, format, *args): - """Do not log messages to stdout while running as cmd. line program.""" - - -@_helpers.positional(3) -def run_flow(flow, storage, flags=None, http=None): - """Core code for a command-line application. - - The ``run()`` function is called from your application and runs - through all the steps to obtain credentials. It takes a ``Flow`` - argument and attempts to open an authorization server page in the - user's default web browser. The server asks the user to grant your - application access to the user's data. If the user grants access, - the ``run()`` function returns new credentials. The new credentials - are also stored in the ``storage`` argument, which updates the file - associated with the ``Storage`` object. - - It presumes it is run from a command-line application and supports the - following flags: - - ``--auth_host_name`` (string, default: ``localhost``) - Host name to use when running a local web server to handle - redirects during OAuth authorization. - - ``--auth_host_port`` (integer, default: ``[8080, 8090]``) - Port to use when running a local web server to handle redirects - during OAuth authorization. Repeat this option to specify a list - of values. - - ``--[no]auth_local_webserver`` (boolean, default: ``True``) - Run a local web server to handle redirects during OAuth - authorization. - - The tools module defines an ``ArgumentParser`` the already contains the - flag definitions that ``run()`` requires. You can pass that - ``ArgumentParser`` to your ``ArgumentParser`` constructor:: - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - parents=[tools.argparser]) - flags = parser.parse_args(argv) - - Args: - flow: Flow, an OAuth 2.0 Flow to step through. - storage: Storage, a ``Storage`` to store the credential in. - flags: ``argparse.Namespace``, (Optional) The command-line flags. This - is the object returned from calling ``parse_args()`` on - ``argparse.ArgumentParser`` as described above. Defaults - to ``argparser.parse_args()``. - http: An instance of ``httplib2.Http.request`` or something that - acts like it. - - Returns: - Credentials, the obtained credential. - """ - if flags is None: - flags = argparser.parse_args() - logging.getLogger().setLevel(getattr(logging, flags.logging_level)) - if not flags.noauth_local_webserver: - success = False - port_number = 0 - for port in flags.auth_host_port: - port_number = port - try: - httpd = ClientRedirectServer((flags.auth_host_name, port), - ClientRedirectHandler) - except socket.error: - pass - else: - success = True - break - flags.noauth_local_webserver = not success - if not success: - print(_FAILED_START_MESSAGE) - - if not flags.noauth_local_webserver: - oauth_callback = 'http://{host}:{port}/'.format( - host=flags.auth_host_name, port=port_number) - else: - oauth_callback = client.OOB_CALLBACK_URN - flow.redirect_uri = oauth_callback - authorize_url = flow.step1_get_authorize_url() - - if not flags.noauth_local_webserver: - import webbrowser - webbrowser.open(authorize_url, new=1, autoraise=True) - print(_BROWSER_OPENED_MESSAGE.format(address=authorize_url)) - else: - print(_GO_TO_LINK_MESSAGE.format(address=authorize_url)) - - code = None - if not flags.noauth_local_webserver: - httpd.handle_request() - if 'error' in httpd.query_params: - sys.exit('Authentication request was rejected.') - if 'code' in httpd.query_params: - code = httpd.query_params['code'] - else: - print('Failed to find "code" in the query parameters ' - 'of the redirect.') - sys.exit('Try running with --noauth_local_webserver.') - else: - code = input('Enter verification code: ').strip() - - try: - credential = flow.step2_exchange(code, http=http) - except client.FlowExchangeError as e: - sys.exit('Authentication has failed: {0}'.format(e)) - - storage.put(credential) - credential.set_store(storage) - print('Authentication successful.') - - return credential - - -def message_if_missing(filename): - """Helpful message to display if the CLIENT_SECRETS file is missing.""" - return _CLIENT_SECRETS_MESSAGE.format(file_path=filename) diff --git a/src/oauth2client/transport.py b/src/oauth2client/transport.py deleted file mode 100644 index 79a61f1c..00000000 --- a/src/oauth2client/transport.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -import httplib2 -import six -from six.moves import http_client - -from oauth2client import _helpers - - -_LOGGER = logging.getLogger(__name__) -# Properties present in file-like streams / buffers. -_STREAM_PROPERTIES = ('read', 'seek', 'tell') - -# Google Data client libraries may need to set this to [401, 403]. -REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,) - - -class MemoryCache(object): - """httplib2 Cache implementation which only caches locally.""" - - def __init__(self): - self.cache = {} - - def get(self, key): - return self.cache.get(key) - - def set(self, key, value): - self.cache[key] = value - - def delete(self, key): - self.cache.pop(key, None) - - -def get_cached_http(): - """Return an HTTP object which caches results returned. - - This is intended to be used in methods like - oauth2client.client.verify_id_token(), which calls to the same URI - to retrieve certs. - - Returns: - httplib2.Http, an HTTP object with a MemoryCache - """ - return _CACHED_HTTP - - -def get_http_object(*args, **kwargs): - """Return a new HTTP object. - - Args: - *args: tuple, The positional arguments to be passed when - contructing a new HTTP object. - **kwargs: dict, The keyword arguments to be passed when - contructing a new HTTP object. - - Returns: - httplib2.Http, an HTTP object. - """ - return httplib2.Http(*args, **kwargs) - - -def _initialize_headers(headers): - """Creates a copy of the headers. - - Args: - headers: dict, request headers to copy. - - Returns: - dict, the copied headers or a new dictionary if the headers - were None. - """ - return {} if headers is None else dict(headers) - - -def _apply_user_agent(headers, user_agent): - """Adds a user-agent to the headers. - - Args: - headers: dict, request headers to add / modify user - agent within. - user_agent: str, the user agent to add. - - Returns: - dict, the original headers passed in, but modified if the - user agent is not None. - """ - if user_agent is not None: - if 'user-agent' in headers: - headers['user-agent'] = (user_agent + ' ' + headers['user-agent']) - else: - headers['user-agent'] = user_agent - - return headers - - -def clean_headers(headers): - """Forces header keys and values to be strings, i.e not unicode. - - The httplib module just concats the header keys and values in a way that - may make the message header a unicode string, which, if it then tries to - contatenate to a binary request body may result in a unicode decode error. - - Args: - headers: dict, A dictionary of headers. - - Returns: - The same dictionary but with all the keys converted to strings. - """ - clean = {} - try: - for k, v in six.iteritems(headers): - if not isinstance(k, six.binary_type): - k = str(k) - if not isinstance(v, six.binary_type): - v = str(v) - clean[_helpers._to_bytes(k)] = _helpers._to_bytes(v) - except UnicodeEncodeError: - from oauth2client.client import NonAsciiHeaderError - raise NonAsciiHeaderError(k, ': ', v) - return clean - - -def wrap_http_for_auth(credentials, http): - """Prepares an HTTP object's request method for auth. - - Wraps HTTP requests with logic to catch auth failures (typically - identified via a 401 status code). In the event of failure, tries - to refresh the token used and then retry the original request. - - Args: - credentials: Credentials, the credentials used to identify - the authenticated user. - http: httplib2.Http, an http object to be used to make - auth requests. - """ - orig_request_method = http.request - - # The closure that will replace 'httplib2.Http.request'. - def new_request(uri, method='GET', body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None): - if not credentials.access_token: - _LOGGER.info('Attempting refresh to obtain ' - 'initial access_token') - credentials._refresh(orig_request_method) - - # Clone and modify the request headers to add the appropriate - # Authorization header. - headers = _initialize_headers(headers) - credentials.apply(headers) - _apply_user_agent(headers, credentials.user_agent) - - body_stream_position = None - # Check if the body is a file-like stream. - if all(getattr(body, stream_prop, None) for stream_prop in - _STREAM_PROPERTIES): - body_stream_position = body.tell() - - resp, content = request(orig_request_method, uri, method, body, - clean_headers(headers), - redirections, connection_type) - - # A stored token may expire between the time it is retrieved and - # the time the request is made, so we may need to try twice. - max_refresh_attempts = 2 - for refresh_attempt in range(max_refresh_attempts): - if resp.status not in REFRESH_STATUS_CODES: - break - _LOGGER.info('Refreshing due to a %s (attempt %s/%s)', - resp.status, refresh_attempt + 1, - max_refresh_attempts) - credentials._refresh(orig_request_method) - credentials.apply(headers) - if body_stream_position is not None: - body.seek(body_stream_position) - - resp, content = request(orig_request_method, uri, method, body, - clean_headers(headers), - redirections, connection_type) - - return resp, content - - # Replace the request method with our own closure. - http.request = new_request - - # Set credentials as a property of the request method. - http.request.credentials = credentials - - -def wrap_http_for_jwt_access(credentials, http): - """Prepares an HTTP object's request method for JWT access. - - Wraps HTTP requests with logic to catch auth failures (typically - identified via a 401 status code). In the event of failure, tries - to refresh the token used and then retry the original request. - - Args: - credentials: _JWTAccessCredentials, the credentials used to identify - a service account that uses JWT access tokens. - http: httplib2.Http, an http object to be used to make - auth requests. - """ - orig_request_method = http.request - wrap_http_for_auth(credentials, http) - # The new value of ``http.request`` set by ``wrap_http_for_auth``. - authenticated_request_method = http.request - - # The closure that will replace 'httplib2.Http.request'. - def new_request(uri, method='GET', body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None): - if 'aud' in credentials._kwargs: - # Preemptively refresh token, this is not done for OAuth2 - if (credentials.access_token is None or - credentials.access_token_expired): - credentials.refresh(None) - return request(authenticated_request_method, uri, - method, body, headers, redirections, - connection_type) - else: - # If we don't have an 'aud' (audience) claim, - # create a 1-time token with the uri root as the audience - headers = _initialize_headers(headers) - _apply_user_agent(headers, credentials.user_agent) - uri_root = uri.split('?', 1)[0] - token, unused_expiry = credentials._create_token({'aud': uri_root}) - - headers['Authorization'] = 'Bearer ' + token - return request(orig_request_method, uri, method, body, - clean_headers(headers), - redirections, connection_type) - - # Replace the request method with our own closure. - http.request = new_request - - # Set credentials as a property of the request method. - http.request.credentials = credentials - - -def request(http, uri, method='GET', body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None): - """Make an HTTP request with an HTTP object and arguments. - - Args: - http: httplib2.Http, an http object to be used to make requests. - uri: string, The URI to be requested. - method: string, The HTTP method to use for the request. Defaults - to 'GET'. - body: string, The payload / body in HTTP request. By default - there is no payload. - headers: dict, Key-value pairs of request headers. By default - there are no headers. - redirections: int, The number of allowed 203 redirects for - the request. Defaults to 5. - connection_type: httplib.HTTPConnection, a subclass to be used for - establishing connection. If not set, the type - will be determined from the ``uri``. - - Returns: - tuple, a pair of a httplib2.Response with the status code and other - headers and the bytes of the content returned. - """ - # NOTE: Allowing http or http.request is temporary (See Issue 601). - http_callable = getattr(http, 'request', http) - return http_callable(uri, method=method, body=body, headers=headers, - redirections=redirections, - connection_type=connection_type) - - -_CACHED_HTTP = httplib2.Http(MemoryCache()) diff --git a/src/passlib/__init__.py b/src/passlib/__init__.py deleted file mode 100644 index b39aa789..00000000 --- a/src/passlib/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""passlib - suite of password hashing & generation routines""" - -__version__ = '1.7.1' diff --git a/src/passlib/_data/wordsets/bip39.txt b/src/passlib/_data/wordsets/bip39.txt deleted file mode 100644 index e29842e6..00000000 --- a/src/passlib/_data/wordsets/bip39.txt +++ /dev/null @@ -1,2049 +0,0 @@ -abandon -ability -able -about -above -absent -absorb -abstract -absurd -abuse -access -accident -account -accuse -achieve -acid -acoustic -acquire -across -act -action -actor -actress -actual -adapt -add -addict -address -adjust -admit -adult -advance -advice -aerobic -affair -afford -afraid -again -age -agent -agree -ahead -aim -air -airport -aisle -alarm -album -alcohol -alert -alien -all -alley -allow -almost -alone -alpha -already -also -alter -always -amateur -amazing -among -amount -amused -analyst -anchor -ancient -anger -angle -angry -animal -ankle -announce -annual -another -answer -antenna -antique -anxiety -any -apart -apology -appear -apple -approve -april -arch -arctic -area -arena -argue -arm -armed -armor -army -around -arrange -arrest -arrive -arrow -art -artefact -artist -artwork -ask -aspect -assault -asset -assist -assume -asthma -athlete -atom -attack -attend -attitude -attract -auction -audit -august -aunt -author -auto -autumn -average -avocado -avoid -awake -aware -away -awesome -awful -awkward -axis -baby -bachelor -bacon -badge -bag -balance -balcony -ball -bamboo -banana -banner -bar -barely -bargain -barrel -base -basic -basket -battle -beach -bean -beauty -because -become -beef -before -begin -behave -behind -believe -below -belt -bench -benefit -best -betray -better -between -beyond -bicycle -bid -bike -bind -biology -bird -birth -bitter -black -blade -blame -blanket -blast -bleak -bless -blind -blood -blossom -blouse -blue -blur -blush -board -boat -body -boil -bomb -bone -bonus -book -boost -border -boring -borrow -boss -bottom -bounce -box -boy -bracket -brain -brand -brass -brave -bread -breeze -brick -bridge -brief -bright -bring -brisk -broccoli -broken -bronze -broom -brother -brown -brush -bubble -buddy -budget -buffalo -build -bulb -bulk -bullet -bundle -bunker -burden -burger -burst -bus -business -busy -butter -buyer -buzz -cabbage -cabin -cable -cactus -cage -cake -call -calm -camera -camp -can -canal -cancel -candy -cannon -canoe -canvas -canyon -capable -capital -captain -car -carbon -card -cargo -carpet -carry -cart -case -cash -casino -castle -casual -cat -catalog -catch -category -cattle -caught -cause -caution -cave -ceiling -celery -cement -census -century -cereal -certain -chair -chalk -champion -change -chaos -chapter -charge -chase -chat -cheap -check -cheese -chef -cherry -chest -chicken -chief -child -chimney -choice -choose -chronic -chuckle -chunk -churn -cigar -cinnamon -circle -citizen -city -civil -claim -clap -clarify -claw -clay -clean -clerk -clever -click -client -cliff -climb -clinic -clip -clock -clog -close -cloth -cloud -clown -club -clump -cluster -clutch -coach -coast -coconut -code -coffee -coil -coin -collect -color -column -combine -come -comfort -comic -common -company -concert -conduct -confirm -congress -connect -consider -control -convince -cook -cool -copper -copy -coral -core -corn -correct -cost -cotton -couch -country -couple -course -cousin -cover -coyote -crack -cradle -craft -cram -crane -crash -crater -crawl -crazy -cream -credit -creek -crew -cricket -crime -crisp -critic -crop -cross -crouch -crowd -crucial -cruel -cruise -crumble -crunch -crush -cry -crystal -cube -culture -cup -cupboard -curious -current -curtain -curve -cushion -custom -cute -cycle -dad -damage -damp -dance -danger -daring -dash -daughter -dawn -day -deal -debate -debris -decade -december -decide -decline -decorate -decrease -deer -defense -define -defy -degree -delay -deliver -demand -demise -denial -dentist -deny -depart -depend -deposit -depth -deputy -derive -describe -desert -design -desk -despair -destroy -detail -detect -develop -device -devote -diagram -dial -diamond -diary -dice -diesel -diet -differ -digital -dignity -dilemma -dinner -dinosaur -direct -dirt -disagree -discover -disease -dish -dismiss -disorder -display -distance -divert -divide -divorce -dizzy -doctor -document -dog -doll -dolphin -domain -donate -donkey -donor -door -dose -double -dove -draft -dragon -drama -drastic -draw -dream -dress -drift -drill -drink -drip -drive -drop -drum -dry -duck -dumb -dune -during -dust -dutch -duty -dwarf -dynamic -eager -eagle -early -earn -earth -easily -east -easy -echo -ecology -economy -edge -edit -educate -effort -egg -eight -either -elbow -elder -electric -elegant -element -elephant -elevator -elite -else -embark -embody -embrace -emerge -emotion -employ -empower -empty -enable -enact -end -endless -endorse -enemy -energy -enforce -engage -engine -enhance -enjoy -enlist -enough -enrich -enroll -ensure -enter -entire -entry -envelope -episode -equal -equip -era -erase -erode -erosion -error -erupt -escape -essay -essence -estate -eternal -ethics -evidence -evil -evoke -evolve -exact -example -excess -exchange -excite -exclude -excuse -execute -exercise -exhaust -exhibit -exile -exist -exit -exotic -expand -expect -expire -explain -expose -express -extend -extra -eye -eyebrow -fabric -face -faculty -fade -faint -faith -fall -false -fame -family -famous -fan -fancy -fantasy -farm -fashion -fat -fatal -father -fatigue -fault -favorite -feature -february -federal -fee -feed -feel -female -fence -festival -fetch -fever -few -fiber -fiction -field -figure -file -film -filter -final -find -fine -finger -finish -fire -firm -first -fiscal -fish -fit -fitness -fix -flag -flame -flash -flat -flavor -flee -flight -flip -float -flock -floor -flower -fluid -flush -fly -foam -focus -fog -foil -fold -follow -food -foot -force -forest -forget -fork -fortune -forum -forward -fossil -foster -found -fox -fragile -frame -frequent -fresh -friend -fringe -frog -front -frost -frown -frozen -fruit -fuel -fun -funny -furnace -fury -future -gadget -gain -galaxy -gallery -game -gap -garage -garbage -garden -garlic -garment -gas -gasp -gate -gather -gauge -gaze -general -genius -genre -gentle -genuine -gesture -ghost -giant -gift -giggle -ginger -giraffe -girl -give -glad -glance -glare -glass -glide -glimpse -globe -gloom -glory -glove -glow -glue -goat -goddess -gold -good -goose -gorilla -gospel -gossip -govern -gown -grab -grace -grain -grant -grape -grass -gravity -great -green -grid -grief -grit -grocery -group -grow -grunt -guard -guess -guide -guilt -guitar -gun -gym -habit -hair -half -hammer -hamster -hand -happy -harbor -hard -harsh -harvest -hat -have -hawk -hazard -head -health -heart -heavy -hedgehog -height -hello -helmet -help -hen -hero -hidden -high -hill -hint -hip -hire -history -hobby -hockey -hold -hole -holiday -hollow -home -honey -hood -hope -horn -horror -horse -hospital -host -hotel -hour -hover -hub -huge -human -humble -humor -hundred -hungry -hunt -hurdle -hurry -hurt -husband -hybrid -ice -icon -idea -identify -idle -ignore -ill -illegal -illness -image -imitate -immense -immune -impact -impose -improve -impulse -inch -include -income -increase -index -indicate -indoor -industry -infant -inflict -inform -inhale -inherit -initial -inject -injury -inmate -inner -innocent -input -inquiry -insane -insect -inside -inspire -install -intact -interest -into -invest -invite -involve -iron -island -isolate -issue -item -ivory -jacket -jaguar -jar -jazz -jealous -jeans -jelly -jewel -job -join -joke -journey -joy -judge -juice -jump -jungle -junior -junk -just -kangaroo -keen -keep -ketchup -key -kick -kid -kidney -kind -kingdom -kiss -kit -kitchen -kite -kitten -kiwi -knee -knife -knock -know -lab -label -labor -ladder -lady -lake -lamp -language -laptop -large -later -latin -laugh -laundry -lava -law -lawn -lawsuit -layer -lazy -leader -leaf -learn -leave -lecture -left -leg -legal -legend -leisure -lemon -lend -length -lens -leopard -lesson -letter -level -liar -liberty -library -license -life -lift -light -like -limb -limit -link -lion -liquid -list -little -live -lizard -load -loan -lobster -local -lock -logic -lonely -long -loop -lottery -loud -lounge -love -loyal -lucky -luggage -lumber -lunar -lunch -luxury -lyrics -machine -mad -magic -magnet -maid -mail -main -major -make -mammal -man -manage -mandate -mango -mansion -manual -maple -marble -march -margin -marine -market -marriage -mask -mass -master -match -material -math -matrix -matter -maximum -maze -meadow -mean -measure -meat -mechanic -medal -media -melody -melt -member -memory -mention -menu -mercy -merge -merit -merry -mesh -message -metal -method -middle -midnight -milk -million -mimic -mind -minimum -minor -minute -miracle -mirror -misery -miss -mistake -mix -mixed -mixture -mobile -model -modify -mom -moment -monitor -monkey -monster -month -moon -moral -more -morning -mosquito -mother -motion -motor -mountain -mouse -move -movie -much -muffin -mule -multiply -muscle -museum -mushroom -music -must -mutual -myself -mystery -myth -naive -name -napkin -narrow -nasty -nation -nature -near -neck -need -negative -neglect -neither -nephew -nerve -nest -net -network -neutral -never -news -next -nice -night -noble -noise -nominee -noodle -normal -north -nose -notable -note -nothing -notice -novel -now -nuclear -number -nurse -nut -oak -obey -object -oblige -obscure -observe -obtain -obvious -occur -ocean -october -odor -off -offer -office -often -oil -okay -old -olive -olympic -omit -once -one -onion -online -only -open -opera -opinion -oppose -option -orange -orbit -orchard -order -ordinary -organ -orient -original -orphan -ostrich -other -outdoor -outer -output -outside -oval -oven -over -own -owner -oxygen -oyster -ozone -pact -paddle -page -pair -palace -palm -panda -panel -panic -panther -paper -parade -parent -park -parrot -party -pass -patch -path -patient -patrol -pattern -pause -pave -payment -peace -peanut -pear -peasant -pelican -pen -penalty -pencil -people -pepper -perfect -permit -person -pet -phone -photo -phrase -physical -piano -picnic -picture -piece -pig -pigeon -pill -pilot -pink -pioneer -pipe -pistol -pitch -pizza -place -planet -plastic -plate -play -please -pledge -pluck -plug -plunge -poem -poet -point -polar -pole -police -pond -pony -pool -popular -portion -position -possible -post -potato -pottery -poverty -powder -power -practice -praise -predict -prefer -prepare -present -pretty -prevent -price -pride -primary -print -priority -prison -private -prize -problem -process -produce -profit -program -project -promote -proof -property -prosper -protect -proud -provide -public -pudding -pull -pulp -pulse -pumpkin -punch -pupil -puppy -purchase -purity -purpose -purse -push -put -puzzle -pyramid -quality -quantum -quarter -question -quick -quit -quiz -quote -rabbit -raccoon -race -rack -radar -radio -rail -rain -raise -rally -ramp -ranch -random -range -rapid -rare -rate -rather -raven -raw -razor -ready -real -reason -rebel -rebuild -recall -receive -recipe -record -recycle -reduce -reflect -reform -refuse -region -regret -regular -reject -relax -release -relief -rely -remain -remember -remind -remove -render -renew -rent -reopen -repair -repeat -replace -report -require -rescue -resemble -resist -resource -response -result -retire -retreat -return -reunion -reveal -review -reward -rhythm -rib -ribbon -rice -rich -ride -ridge -rifle -right -rigid -ring -riot -ripple -risk -ritual -rival -river -road -roast -robot -robust -rocket -romance -roof -rookie -room -rose -rotate -rough -round -route -royal -rubber -rude -rug -rule -run -runway -rural -sad -saddle -sadness -safe -sail -salad -salmon -salon -salt -salute -same -sample -sand -satisfy -satoshi -sauce -sausage -save -say -scale -scan -scare -scatter -scene -scheme -school -science -scissors -scorpion -scout -scrap -screen -script -scrub -sea -search -season -seat -second -secret -section -security -seed -seek -segment -select -sell -seminar -senior -sense -sentence -series -service -session -settle -setup -seven -shadow -shaft -shallow -share -shed -shell -sheriff -shield -shift -shine -ship -shiver -shock -shoe -shoot -shop -short -shoulder -shove -shrimp -shrug -shuffle -shy -sibling -sick -side -siege -sight -sign -silent -silk -silly -silver -similar -simple -since -sing -siren -sister -situate -six -size -skate -sketch -ski -skill -skin -skirt -skull -slab -slam -sleep -slender -slice -slide -slight -slim -slogan -slot -slow -slush -small -smart -smile -smoke -smooth -snack -snake -snap -sniff -snow -soap -soccer -social -sock -soda -soft -solar -soldier -solid -solution -solve -someone -song -soon -sorry -sort -soul -sound -soup -source -south -space -spare -spatial -spawn -speak -special -speed -spell -spend -sphere -spice -spider -spike -spin -spirit -split -spoil -sponsor -spoon -sport -spot -spray -spread -spring -spy -square -squeeze -squirrel -stable -stadium -staff -stage -stairs -stamp -stand -start -state -stay -steak -steel -stem -step -stereo -stick -still -sting -stock -stomach -stone -stool -story -stove -strategy -street -strike -strong -struggle -student -stuff -stumble -style -subject -submit -subway -success -such -sudden -suffer -sugar -suggest -suit -summer -sun -sunny -sunset -super -supply -supreme -sure -surface -surge -surprise -surround -survey -suspect -sustain -swallow -swamp -swap -swarm -swear -sweet -swift -swim -swing -switch -sword -symbol -symptom -syrup -system -table -tackle -tag -tail -talent -talk -tank -tape -target -task -taste -tattoo -taxi -teach -team -tell -ten -tenant -tennis -tent -term -test -text -thank -that -theme -then -theory -there -they -thing -this -thought -three -thrive -throw -thumb -thunder -ticket -tide -tiger -tilt -timber -time -tiny -tip -tired -tissue -title -toast -tobacco -today -toddler -toe -together -toilet -token -tomato -tomorrow -tone -tongue -tonight -tool -tooth -top -topic -topple -torch -tornado -tortoise -toss -total -tourist -toward -tower -town -toy -track -trade -traffic -tragic -train -transfer -trap -trash -travel -tray -treat -tree -trend -trial -tribe -trick -trigger -trim -trip -trophy -trouble -truck -true -truly -trumpet -trust -truth -try -tube -tuition -tumble -tuna -tunnel -turkey -turn -turtle -twelve -twenty -twice -twin -twist -two -type -typical -ugly -umbrella -unable -unaware -uncle -uncover -under -undo -unfair -unfold -unhappy -uniform -unique -unit -universe -unknown -unlock -until -unusual -unveil -update -upgrade -uphold -upon -upper -upset -urban -urge -usage -use -used -useful -useless -usual -utility -vacant -vacuum -vague -valid -valley -valve -van -vanish -vapor -various -vast -vault -vehicle -velvet -vendor -venture -venue -verb -verify -version -very -vessel -veteran -viable -vibrant -vicious -victory -video -view -village -vintage -violin -virtual -virus -visa -visit -visual -vital -vivid -vocal -voice -void -volcano -volume -vote -voyage -wage -wagon -wait -walk -wall -walnut -want -warfare -warm -warrior -wash -wasp -waste -water -wave -way -wealth -weapon -wear -weasel -weather -web -wedding -weekend -weird -welcome -west -wet -whale -what -wheat -wheel -when -where -whip -whisper -wide -width -wife -wild -will -win -window -wine -wing -wink -winner -winter -wire -wisdom -wise -wish -witness -wolf -woman -wonder -wood -wool -word -work -world -worry -worth -wrap -wreck -wrestle -wrist -write -wrong -yard -year -yellow -you -young -youth -zebra -zero -zone -zoo - diff --git a/src/passlib/_data/wordsets/eff_long.txt b/src/passlib/_data/wordsets/eff_long.txt deleted file mode 100644 index caf71f52..00000000 --- a/src/passlib/_data/wordsets/eff_long.txt +++ /dev/null @@ -1,7776 +0,0 @@ -abacus -abdomen -abdominal -abide -abiding -ability -ablaze -able -abnormal -abrasion -abrasive -abreast -abridge -abroad -abruptly -absence -absentee -absently -absinthe -absolute -absolve -abstain -abstract -absurd -accent -acclaim -acclimate -accompany -account -accuracy -accurate -accustom -acetone -achiness -aching -acid -acorn -acquaint -acquire -acre -acrobat -acronym -acting -action -activate -activator -active -activism -activist -activity -actress -acts -acutely -acuteness -aeration -aerobics -aerosol -aerospace -afar -affair -affected -affecting -affection -affidavit -affiliate -affirm -affix -afflicted -affluent -afford -affront -aflame -afloat -aflutter -afoot -afraid -afterglow -afterlife -aftermath -aftermost -afternoon -aged -ageless -agency -agenda -agent -aggregate -aghast -agile -agility -aging -agnostic -agonize -agonizing -agony -agreeable -agreeably -agreed -agreeing -agreement -aground -ahead -ahoy -aide -aids -aim -ajar -alabaster -alarm -albatross -album -alfalfa -algebra -algorithm -alias -alibi -alienable -alienate -aliens -alike -alive -alkaline -alkalize -almanac -almighty -almost -aloe -aloft -aloha -alone -alongside -aloof -alphabet -alright -although -altitude -alto -aluminum -alumni -always -amaretto -amaze -amazingly -amber -ambiance -ambiguity -ambiguous -ambition -ambitious -ambulance -ambush -amendable -amendment -amends -amenity -amiable -amicably -amid -amigo -amino -amiss -ammonia -ammonium -amnesty -amniotic -among -amount -amperage -ample -amplifier -amplify -amply -amuck -amulet -amusable -amused -amusement -amuser -amusing -anaconda -anaerobic -anagram -anatomist -anatomy -anchor -anchovy -ancient -android -anemia -anemic -aneurism -anew -angelfish -angelic -anger -angled -angler -angles -angling -angrily -angriness -anguished -angular -animal -animate -animating -animation -animator -anime -animosity -ankle -annex -annotate -announcer -annoying -annually -annuity -anointer -another -answering -antacid -antarctic -anteater -antelope -antennae -anthem -anthill -anthology -antibody -antics -antidote -antihero -antiquely -antiques -antiquity -antirust -antitoxic -antitrust -antiviral -antivirus -antler -antonym -antsy -anvil -anybody -anyhow -anymore -anyone -anyplace -anything -anytime -anyway -anywhere -aorta -apache -apostle -appealing -appear -appease -appeasing -appendage -appendix -appetite -appetizer -applaud -applause -apple -appliance -applicant -applied -apply -appointee -appraisal -appraiser -apprehend -approach -approval -approve -apricot -april -apron -aptitude -aptly -aqua -aqueduct -arbitrary -arbitrate -ardently -area -arena -arguable -arguably -argue -arise -armadillo -armband -armchair -armed -armful -armhole -arming -armless -armoire -armored -armory -armrest -army -aroma -arose -around -arousal -arrange -array -arrest -arrival -arrive -arrogance -arrogant -arson -art -ascend -ascension -ascent -ascertain -ashamed -ashen -ashes -ashy -aside -askew -asleep -asparagus -aspect -aspirate -aspire -aspirin -astonish -astound -astride -astrology -astronaut -astronomy -astute -atlantic -atlas -atom -atonable -atop -atrium -atrocious -atrophy -attach -attain -attempt -attendant -attendee -attention -attentive -attest -attic -attire -attitude -attractor -attribute -atypical -auction -audacious -audacity -audible -audibly -audience -audio -audition -augmented -august -authentic -author -autism -autistic -autograph -automaker -automated -automatic -autopilot -available -avalanche -avatar -avenge -avenging -avenue -average -aversion -avert -aviation -aviator -avid -avoid -await -awaken -award -aware -awhile -awkward -awning -awoke -awry -axis -babble -babbling -babied -baboon -backache -backboard -backboned -backdrop -backed -backer -backfield -backfire -backhand -backing -backlands -backlash -backless -backlight -backlit -backlog -backpack -backpedal -backrest -backroom -backshift -backside -backslid -backspace -backspin -backstab -backstage -backtalk -backtrack -backup -backward -backwash -backwater -backyard -bacon -bacteria -bacterium -badass -badge -badland -badly -badness -baffle -baffling -bagel -bagful -baggage -bagged -baggie -bagginess -bagging -baggy -bagpipe -baguette -baked -bakery -bakeshop -baking -balance -balancing -balcony -balmy -balsamic -bamboo -banana -banish -banister -banjo -bankable -bankbook -banked -banker -banking -banknote -bankroll -banner -bannister -banshee -banter -barbecue -barbed -barbell -barber -barcode -barge -bargraph -barista -baritone -barley -barmaid -barman -barn -barometer -barrack -barracuda -barrel -barrette -barricade -barrier -barstool -bartender -barterer -bash -basically -basics -basil -basin -basis -basket -batboy -batch -bath -baton -bats -battalion -battered -battering -battery -batting -battle -bauble -bazooka -blabber -bladder -blade -blah -blame -blaming -blanching -blandness -blank -blaspheme -blasphemy -blast -blatancy -blatantly -blazer -blazing -bleach -bleak -bleep -blemish -blend -bless -blighted -blimp -bling -blinked -blinker -blinking -blinks -blip -blissful -blitz -blizzard -bloated -bloating -blob -blog -bloomers -blooming -blooper -blot -blouse -blubber -bluff -bluish -blunderer -blunt -blurb -blurred -blurry -blurt -blush -blustery -boaster -boastful -boasting -boat -bobbed -bobbing -bobble -bobcat -bobsled -bobtail -bodacious -body -bogged -boggle -bogus -boil -bok -bolster -bolt -bonanza -bonded -bonding -bondless -boned -bonehead -boneless -bonelike -boney -bonfire -bonnet -bonsai -bonus -bony -boogeyman -boogieman -book -boondocks -booted -booth -bootie -booting -bootlace -bootleg -boots -boozy -borax -boring -borough -borrower -borrowing -boss -botanical -botanist -botany -botch -both -bottle -bottling -bottom -bounce -bouncing -bouncy -bounding -boundless -bountiful -bovine -boxcar -boxer -boxing -boxlike -boxy -breach -breath -breeches -breeching -breeder -breeding -breeze -breezy -brethren -brewery -brewing -briar -bribe -brick -bride -bridged -brigade -bright -brilliant -brim -bring -brink -brisket -briskly -briskness -bristle -brittle -broadband -broadcast -broaden -broadly -broadness -broadside -broadways -broiler -broiling -broken -broker -bronchial -bronco -bronze -bronzing -brook -broom -brought -browbeat -brownnose -browse -browsing -bruising -brunch -brunette -brunt -brush -brussels -brute -brutishly -bubble -bubbling -bubbly -buccaneer -bucked -bucket -buckle -buckshot -buckskin -bucktooth -buckwheat -buddhism -buddhist -budding -buddy -budget -buffalo -buffed -buffer -buffing -buffoon -buggy -bulb -bulge -bulginess -bulgur -bulk -bulldog -bulldozer -bullfight -bullfrog -bullhorn -bullion -bullish -bullpen -bullring -bullseye -bullwhip -bully -bunch -bundle -bungee -bunion -bunkbed -bunkhouse -bunkmate -bunny -bunt -busboy -bush -busily -busload -bust -busybody -buzz -cabana -cabbage -cabbie -cabdriver -cable -caboose -cache -cackle -cacti -cactus -caddie -caddy -cadet -cadillac -cadmium -cage -cahoots -cake -calamari -calamity -calcium -calculate -calculus -caliber -calibrate -calm -caloric -calorie -calzone -camcorder -cameo -camera -camisole -camper -campfire -camping -campsite -campus -canal -canary -cancel -candied -candle -candy -cane -canine -canister -cannabis -canned -canning -cannon -cannot -canola -canon -canopener -canopy -canteen -canyon -capable -capably -capacity -cape -capillary -capital -capitol -capped -capricorn -capsize -capsule -caption -captivate -captive -captivity -capture -caramel -carat -caravan -carbon -cardboard -carded -cardiac -cardigan -cardinal -cardstock -carefully -caregiver -careless -caress -caretaker -cargo -caring -carless -carload -carmaker -carnage -carnation -carnival -carnivore -carol -carpenter -carpentry -carpool -carport -carried -carrot -carrousel -carry -cartel -cartload -carton -cartoon -cartridge -cartwheel -carve -carving -carwash -cascade -case -cash -casing -casino -casket -cassette -casually -casualty -catacomb -catalog -catalyst -catalyze -catapult -cataract -catatonic -catcall -catchable -catcher -catching -catchy -caterer -catering -catfight -catfish -cathedral -cathouse -catlike -catnap -catnip -catsup -cattail -cattishly -cattle -catty -catwalk -caucasian -caucus -causal -causation -cause -causing -cauterize -caution -cautious -cavalier -cavalry -caviar -cavity -cedar -celery -celestial -celibacy -celibate -celtic -cement -census -ceramics -ceremony -certainly -certainty -certified -certify -cesarean -cesspool -chafe -chaffing -chain -chair -chalice -challenge -chamber -chamomile -champion -chance -change -channel -chant -chaos -chaperone -chaplain -chapped -chaps -chapter -character -charbroil -charcoal -charger -charging -chariot -charity -charm -charred -charter -charting -chase -chasing -chaste -chastise -chastity -chatroom -chatter -chatting -chatty -cheating -cheddar -cheek -cheer -cheese -cheesy -chef -chemicals -chemist -chemo -cherisher -cherub -chess -chest -chevron -chevy -chewable -chewer -chewing -chewy -chief -chihuahua -childcare -childhood -childish -childless -childlike -chili -chill -chimp -chip -chirping -chirpy -chitchat -chivalry -chive -chloride -chlorine -choice -chokehold -choking -chomp -chooser -choosing -choosy -chop -chosen -chowder -chowtime -chrome -chubby -chuck -chug -chummy -chump -chunk -churn -chute -cider -cilantro -cinch -cinema -cinnamon -circle -circling -circular -circulate -circus -citable -citadel -citation -citizen -citric -citrus -city -civic -civil -clad -claim -clambake -clammy -clamor -clamp -clamshell -clang -clanking -clapped -clapper -clapping -clarify -clarinet -clarity -clash -clasp -class -clatter -clause -clavicle -claw -clay -clean -clear -cleat -cleaver -cleft -clench -clergyman -clerical -clerk -clever -clicker -client -climate -climatic -cling -clinic -clinking -clip -clique -cloak -clobber -clock -clone -cloning -closable -closure -clothes -clothing -cloud -clover -clubbed -clubbing -clubhouse -clump -clumsily -clumsy -clunky -clustered -clutch -clutter -coach -coagulant -coastal -coaster -coasting -coastland -coastline -coat -coauthor -cobalt -cobbler -cobweb -cocoa -coconut -cod -coeditor -coerce -coexist -coffee -cofounder -cognition -cognitive -cogwheel -coherence -coherent -cohesive -coil -coke -cola -cold -coleslaw -coliseum -collage -collapse -collar -collected -collector -collide -collie -collision -colonial -colonist -colonize -colony -colossal -colt -coma -come -comfort -comfy -comic -coming -comma -commence -commend -comment -commerce -commode -commodity -commodore -common -commotion -commute -commuting -compacted -compacter -compactly -compactor -companion -company -compare -compel -compile -comply -component -composed -composer -composite -compost -composure -compound -compress -comprised -computer -computing -comrade -concave -conceal -conceded -concept -concerned -concert -conch -concierge -concise -conclude -concrete -concur -condense -condiment -condition -condone -conducive -conductor -conduit -cone -confess -confetti -confidant -confident -confider -confiding -configure -confined -confining -confirm -conflict -conform -confound -confront -confused -confusing -confusion -congenial -congested -congrats -congress -conical -conjoined -conjure -conjuror -connected -connector -consensus -consent -console -consoling -consonant -constable -constant -constrain -constrict -construct -consult -consumer -consuming -contact -container -contempt -contend -contented -contently -contents -contest -context -contort -contour -contrite -control -contusion -convene -convent -copartner -cope -copied -copier -copilot -coping -copious -copper -copy -coral -cork -cornball -cornbread -corncob -cornea -corned -corner -cornfield -cornflake -cornhusk -cornmeal -cornstalk -corny -coronary -coroner -corporal -corporate -corral -correct -corridor -corrode -corroding -corrosive -corsage -corset -cortex -cosigner -cosmetics -cosmic -cosmos -cosponsor -cost -cottage -cotton -couch -cough -could -countable -countdown -counting -countless -country -county -courier -covenant -cover -coveted -coveting -coyness -cozily -coziness -cozy -crabbing -crabgrass -crablike -crabmeat -cradle -cradling -crafter -craftily -craftsman -craftwork -crafty -cramp -cranberry -crane -cranial -cranium -crank -crate -crave -craving -crawfish -crawlers -crawling -crayfish -crayon -crazed -crazily -craziness -crazy -creamed -creamer -creamlike -crease -creasing -creatable -create -creation -creative -creature -credible -credibly -credit -creed -creme -creole -crepe -crept -crescent -crested -cresting -crestless -crevice -crewless -crewman -crewmate -crib -cricket -cried -crier -crimp -crimson -cringe -cringing -crinkle -crinkly -crisped -crisping -crisply -crispness -crispy -criteria -critter -croak -crock -crook -croon -crop -cross -crouch -crouton -crowbar -crowd -crown -crucial -crudely -crudeness -cruelly -cruelness -cruelty -crumb -crummiest -crummy -crumpet -crumpled -cruncher -crunching -crunchy -crusader -crushable -crushed -crusher -crushing -crust -crux -crying -cryptic -crystal -cubbyhole -cube -cubical -cubicle -cucumber -cuddle -cuddly -cufflink -culinary -culminate -culpable -culprit -cultivate -cultural -culture -cupbearer -cupcake -cupid -cupped -cupping -curable -curator -curdle -cure -curfew -curing -curled -curler -curliness -curling -curly -curry -curse -cursive -cursor -curtain -curtly -curtsy -curvature -curve -curvy -cushy -cusp -cussed -custard -custodian -custody -customary -customer -customize -customs -cut -cycle -cyclic -cycling -cyclist -cylinder -cymbal -cytoplasm -cytoplast -dab -dad -daffodil -dagger -daily -daintily -dainty -dairy -daisy -dallying -dance -dancing -dandelion -dander -dandruff -dandy -danger -dangle -dangling -daredevil -dares -daringly -darkened -darkening -darkish -darkness -darkroom -darling -darn -dart -darwinism -dash -dastardly -data -datebook -dating -daughter -daunting -dawdler -dawn -daybed -daybreak -daycare -daydream -daylight -daylong -dayroom -daytime -dazzler -dazzling -deacon -deafening -deafness -dealer -dealing -dealmaker -dealt -dean -debatable -debate -debating -debit -debrief -debtless -debtor -debug -debunk -decade -decaf -decal -decathlon -decay -deceased -deceit -deceiver -deceiving -december -decency -decent -deception -deceptive -decibel -decidable -decimal -decimeter -decipher -deck -declared -decline -decode -decompose -decorated -decorator -decoy -decrease -decree -dedicate -dedicator -deduce -deduct -deed -deem -deepen -deeply -deepness -deface -defacing -defame -default -defeat -defection -defective -defendant -defender -defense -defensive -deferral -deferred -defiance -defiant -defile -defiling -define -definite -deflate -deflation -deflator -deflected -deflector -defog -deforest -defraud -defrost -deftly -defuse -defy -degraded -degrading -degrease -degree -dehydrate -deity -dejected -delay -delegate -delegator -delete -deletion -delicacy -delicate -delicious -delighted -delirious -delirium -deliverer -delivery -delouse -delta -deluge -delusion -deluxe -demanding -demeaning -demeanor -demise -democracy -democrat -demote -demotion -demystify -denatured -deniable -denial -denim -denote -dense -density -dental -dentist -denture -deny -deodorant -deodorize -departed -departure -depict -deplete -depletion -deplored -deploy -deport -depose -depraved -depravity -deprecate -depress -deprive -depth -deputize -deputy -derail -deranged -derby -derived -desecrate -deserve -deserving -designate -designed -designer -designing -deskbound -desktop -deskwork -desolate -despair -despise -despite -destiny -destitute -destruct -detached -detail -detection -detective -detector -detention -detergent -detest -detonate -detonator -detoxify -detract -deuce -devalue -deviancy -deviant -deviate -deviation -deviator -device -devious -devotedly -devotee -devotion -devourer -devouring -devoutly -dexterity -dexterous -diabetes -diabetic -diabolic -diagnoses -diagnosis -diagram -dial -diameter -diaper -diaphragm -diary -dice -dicing -dictate -dictation -dictator -difficult -diffused -diffuser -diffusion -diffusive -dig -dilation -diligence -diligent -dill -dilute -dime -diminish -dimly -dimmed -dimmer -dimness -dimple -diner -dingbat -dinghy -dinginess -dingo -dingy -dining -dinner -diocese -dioxide -diploma -dipped -dipper -dipping -directed -direction -directive -directly -directory -direness -dirtiness -disabled -disagree -disallow -disarm -disarray -disaster -disband -disbelief -disburse -discard -discern -discharge -disclose -discolor -discount -discourse -discover -discuss -disdain -disengage -disfigure -disgrace -dish -disinfect -disjoin -disk -dislike -disliking -dislocate -dislodge -disloyal -dismantle -dismay -dismiss -dismount -disobey -disorder -disown -disparate -disparity -dispatch -dispense -dispersal -dispersed -disperser -displace -display -displease -disposal -dispose -disprove -dispute -disregard -disrupt -dissuade -distance -distant -distaste -distill -distinct -distort -distract -distress -district -distrust -ditch -ditto -ditzy -dividable -divided -dividend -dividers -dividing -divinely -diving -divinity -divisible -divisibly -division -divisive -divorcee -dizziness -dizzy -doable -docile -dock -doctrine -document -dodge -dodgy -doily -doing -dole -dollar -dollhouse -dollop -dolly -dolphin -domain -domelike -domestic -dominion -dominoes -donated -donation -donator -donor -donut -doodle -doorbell -doorframe -doorknob -doorman -doormat -doornail -doorpost -doorstep -doorstop -doorway -doozy -dork -dormitory -dorsal -dosage -dose -dotted -doubling -douche -dove -down -dowry -doze -drab -dragging -dragonfly -dragonish -dragster -drainable -drainage -drained -drainer -drainpipe -dramatic -dramatize -drank -drapery -drastic -draw -dreaded -dreadful -dreadlock -dreamboat -dreamily -dreamland -dreamless -dreamlike -dreamt -dreamy -drearily -dreary -drench -dress -drew -dribble -dried -drier -drift -driller -drilling -drinkable -drinking -dripping -drippy -drivable -driven -driver -driveway -driving -drizzle -drizzly -drone -drool -droop -drop-down -dropbox -dropkick -droplet -dropout -dropper -drove -drown -drowsily -drudge -drum -dry -dubbed -dubiously -duchess -duckbill -ducking -duckling -ducktail -ducky -duct -dude -duffel -dugout -duh -duke -duller -dullness -duly -dumping -dumpling -dumpster -duo -dupe -duplex -duplicate -duplicity -durable -durably -duration -duress -during -dusk -dust -dutiful -duty -duvet -dwarf -dweeb -dwelled -dweller -dwelling -dwindle -dwindling -dynamic -dynamite -dynasty -dyslexia -dyslexic -each -eagle -earache -eardrum -earflap -earful -earlobe -early -earmark -earmuff -earphone -earpiece -earplugs -earring -earshot -earthen -earthlike -earthling -earthly -earthworm -earthy -earwig -easeful -easel -easiest -easily -easiness -easing -eastbound -eastcoast -easter -eastward -eatable -eaten -eatery -eating -eats -ebay -ebony -ebook -ecard -eccentric -echo -eclair -eclipse -ecologist -ecology -economic -economist -economy -ecosphere -ecosystem -edge -edginess -edging -edgy -edition -editor -educated -education -educator -eel -effective -effects -efficient -effort -eggbeater -egging -eggnog -eggplant -eggshell -egomaniac -egotism -egotistic -either -eject -elaborate -elastic -elated -elbow -eldercare -elderly -eldest -electable -election -elective -elephant -elevate -elevating -elevation -elevator -eleven -elf -eligible -eligibly -eliminate -elite -elitism -elixir -elk -ellipse -elliptic -elm -elongated -elope -eloquence -eloquent -elsewhere -elude -elusive -elves -email -embargo -embark -embassy -embattled -embellish -ember -embezzle -emblaze -emblem -embody -embolism -emboss -embroider -emcee -emerald -emergency -emission -emit -emote -emoticon -emotion -empathic -empathy -emperor -emphases -emphasis -emphasize -emphatic -empirical -employed -employee -employer -emporium -empower -emptier -emptiness -empty -emu -enable -enactment -enamel -enchanted -enchilada -encircle -enclose -enclosure -encode -encore -encounter -encourage -encroach -encrust -encrypt -endanger -endeared -endearing -ended -ending -endless -endnote -endocrine -endorphin -endorse -endowment -endpoint -endurable -endurance -enduring -energetic -energize -energy -enforced -enforcer -engaged -engaging -engine -engorge -engraved -engraver -engraving -engross -engulf -enhance -enigmatic -enjoyable -enjoyably -enjoyer -enjoying -enjoyment -enlarged -enlarging -enlighten -enlisted -enquirer -enrage -enrich -enroll -enslave -ensnare -ensure -entail -entangled -entering -entertain -enticing -entire -entitle -entity -entomb -entourage -entrap -entree -entrench -entrust -entryway -entwine -enunciate -envelope -enviable -enviably -envious -envision -envoy -envy -enzyme -epic -epidemic -epidermal -epidermis -epidural -epilepsy -epileptic -epilogue -epiphany -episode -equal -equate -equation -equator -equinox -equipment -equity -equivocal -eradicate -erasable -erased -eraser -erasure -ergonomic -errand -errant -erratic -error -erupt -escalate -escalator -escapable -escapade -escapist -escargot -eskimo -esophagus -espionage -espresso -esquire -essay -essence -essential -establish -estate -esteemed -estimate -estimator -estranged -estrogen -etching -eternal -eternity -ethanol -ether -ethically -ethics -euphemism -evacuate -evacuee -evade -evaluate -evaluator -evaporate -evasion -evasive -even -everglade -evergreen -everybody -everyday -everyone -evict -evidence -evident -evil -evoke -evolution -evolve -exact -exalted -example -excavate -excavator -exceeding -exception -excess -exchange -excitable -exciting -exclaim -exclude -excluding -exclusion -exclusive -excretion -excretory -excursion -excusable -excusably -excuse -exemplary -exemplify -exemption -exerciser -exert -exes -exfoliate -exhale -exhaust -exhume -exile -existing -exit -exodus -exonerate -exorcism -exorcist -expand -expanse -expansion -expansive -expectant -expedited -expediter -expel -expend -expenses -expensive -expert -expire -expiring -explain -expletive -explicit -explode -exploit -explore -exploring -exponent -exporter -exposable -expose -exposure -express -expulsion -exquisite -extended -extending -extent -extenuate -exterior -external -extinct -extortion -extradite -extras -extrovert -extrude -extruding -exuberant -fable -fabric -fabulous -facebook -facecloth -facedown -faceless -facelift -faceplate -faceted -facial -facility -facing -facsimile -faction -factoid -factor -factsheet -factual -faculty -fade -fading -failing -falcon -fall -false -falsify -fame -familiar -family -famine -famished -fanatic -fancied -fanciness -fancy -fanfare -fang -fanning -fantasize -fantastic -fantasy -fascism -fastball -faster -fasting -fastness -faucet -favorable -favorably -favored -favoring -favorite -fax -feast -federal -fedora -feeble -feed -feel -feisty -feline -felt-tip -feminine -feminism -feminist -feminize -femur -fence -fencing -fender -ferment -fernlike -ferocious -ferocity -ferret -ferris -ferry -fervor -fester -festival -festive -festivity -fetal -fetch -fever -fiber -fiction -fiddle -fiddling -fidelity -fidgeting -fidgety -fifteen -fifth -fiftieth -fifty -figment -figure -figurine -filing -filled -filler -filling -film -filter -filth -filtrate -finale -finalist -finalize -finally -finance -financial -finch -fineness -finer -finicky -finished -finisher -finishing -finite -finless -finlike -fiscally -fit -five -flaccid -flagman -flagpole -flagship -flagstick -flagstone -flail -flakily -flaky -flame -flammable -flanked -flanking -flannels -flap -flaring -flashback -flashbulb -flashcard -flashily -flashing -flashy -flask -flatbed -flatfoot -flatly -flatness -flatten -flattered -flatterer -flattery -flattop -flatware -flatworm -flavored -flavorful -flavoring -flaxseed -fled -fleshed -fleshy -flick -flier -flight -flinch -fling -flint -flip -flirt -float -flock -flogging -flop -floral -florist -floss -flounder -flyable -flyaway -flyer -flying -flyover -flypaper -foam -foe -fog -foil -folic -folk -follicle -follow -fondling -fondly -fondness -fondue -font -food -fool -footage -football -footbath -footboard -footer -footgear -foothill -foothold -footing -footless -footman -footnote -footpad -footpath -footprint -footrest -footsie -footsore -footwear -footwork -fossil -foster -founder -founding -fountain -fox -foyer -fraction -fracture -fragile -fragility -fragment -fragrance -fragrant -frail -frame -framing -frantic -fraternal -frayed -fraying -frays -freckled -freckles -freebase -freebee -freebie -freedom -freefall -freehand -freeing -freeload -freely -freemason -freeness -freestyle -freeware -freeway -freewill -freezable -freezing -freight -french -frenzied -frenzy -frequency -frequent -fresh -fretful -fretted -friction -friday -fridge -fried -friend -frighten -frightful -frigidity -frigidly -frill -fringe -frisbee -frisk -fritter -frivolous -frolic -from -front -frostbite -frosted -frostily -frosting -frostlike -frosty -froth -frown -frozen -fructose -frugality -frugally -fruit -frustrate -frying -gab -gaffe -gag -gainfully -gaining -gains -gala -gallantly -galleria -gallery -galley -gallon -gallows -gallstone -galore -galvanize -gambling -game -gaming -gamma -gander -gangly -gangrene -gangway -gap -garage -garbage -garden -gargle -garland -garlic -garment -garnet -garnish -garter -gas -gatherer -gathering -gating -gauging -gauntlet -gauze -gave -gawk -gazing -gear -gecko -geek -geiger -gem -gender -generic -generous -genetics -genre -gentile -gentleman -gently -gents -geography -geologic -geologist -geology -geometric -geometry -geranium -gerbil -geriatric -germicide -germinate -germless -germproof -gestate -gestation -gesture -getaway -getting -getup -giant -gibberish -giblet -giddily -giddiness -giddy -gift -gigabyte -gigahertz -gigantic -giggle -giggling -giggly -gigolo -gilled -gills -gimmick -girdle -giveaway -given -giver -giving -gizmo -gizzard -glacial -glacier -glade -gladiator -gladly -glamorous -glamour -glance -glancing -glandular -glare -glaring -glass -glaucoma -glazing -gleaming -gleeful -glider -gliding -glimmer -glimpse -glisten -glitch -glitter -glitzy -gloater -gloating -gloomily -gloomy -glorified -glorifier -glorify -glorious -glory -gloss -glove -glowing -glowworm -glucose -glue -gluten -glutinous -glutton -gnarly -gnat -goal -goatskin -goes -goggles -going -goldfish -goldmine -goldsmith -golf -goliath -gonad -gondola -gone -gong -good -gooey -goofball -goofiness -goofy -google -goon -gopher -gore -gorged -gorgeous -gory -gosling -gossip -gothic -gotten -gout -gown -grab -graceful -graceless -gracious -gradation -graded -grader -gradient -grading -gradually -graduate -graffiti -grafted -grafting -grain -granddad -grandkid -grandly -grandma -grandpa -grandson -granite -granny -granola -grant -granular -grape -graph -grapple -grappling -grasp -grass -gratified -gratify -grating -gratitude -gratuity -gravel -graveness -graves -graveyard -gravitate -gravity -gravy -gray -grazing -greasily -greedily -greedless -greedy -green -greeter -greeting -grew -greyhound -grid -grief -grievance -grieving -grievous -grill -grimace -grimacing -grime -griminess -grimy -grinch -grinning -grip -gristle -grit -groggily -groggy -groin -groom -groove -grooving -groovy -grope -ground -grouped -grout -grove -grower -growing -growl -grub -grudge -grudging -grueling -gruffly -grumble -grumbling -grumbly -grumpily -grunge -grunt -guacamole -guidable -guidance -guide -guiding -guileless -guise -gulf -gullible -gully -gulp -gumball -gumdrop -gumminess -gumming -gummy -gurgle -gurgling -guru -gush -gusto -gusty -gutless -guts -gutter -guy -guzzler -gyration -habitable -habitant -habitat -habitual -hacked -hacker -hacking -hacksaw -had -haggler -haiku -half -halogen -halt -halved -halves -hamburger -hamlet -hammock -hamper -hamster -hamstring -handbag -handball -handbook -handbrake -handcart -handclap -handclasp -handcraft -handcuff -handed -handful -handgrip -handgun -handheld -handiness -handiwork -handlebar -handled -handler -handling -handmade -handoff -handpick -handprint -handrail -handsaw -handset -handsfree -handshake -handstand -handwash -handwork -handwoven -handwrite -handyman -hangnail -hangout -hangover -hangup -hankering -hankie -hanky -haphazard -happening -happier -happiest -happily -happiness -happy -harbor -hardcopy -hardcore -hardcover -harddisk -hardened -hardener -hardening -hardhat -hardhead -hardiness -hardly -hardness -hardship -hardware -hardwired -hardwood -hardy -harmful -harmless -harmonica -harmonics -harmonize -harmony -harness -harpist -harsh -harvest -hash -hassle -haste -hastily -hastiness -hasty -hatbox -hatchback -hatchery -hatchet -hatching -hatchling -hate -hatless -hatred -haunt -haven -hazard -hazelnut -hazily -haziness -hazing -hazy -headache -headband -headboard -headcount -headdress -headed -header -headfirst -headgear -heading -headlamp -headless -headlock -headphone -headpiece -headrest -headroom -headscarf -headset -headsman -headstand -headstone -headway -headwear -heap -heat -heave -heavily -heaviness -heaving -hedge -hedging -heftiness -hefty -helium -helmet -helper -helpful -helping -helpless -helpline -hemlock -hemstitch -hence -henchman -henna -herald -herbal -herbicide -herbs -heritage -hermit -heroics -heroism -herring -herself -hertz -hesitancy -hesitant -hesitate -hexagon -hexagram -hubcap -huddle -huddling -huff -hug -hula -hulk -hull -human -humble -humbling -humbly -humid -humiliate -humility -humming -hummus -humongous -humorist -humorless -humorous -humpback -humped -humvee -hunchback -hundredth -hunger -hungrily -hungry -hunk -hunter -hunting -huntress -huntsman -hurdle -hurled -hurler -hurling -hurray -hurricane -hurried -hurry -hurt -husband -hush -husked -huskiness -hut -hybrid -hydrant -hydrated -hydration -hydrogen -hydroxide -hyperlink -hypertext -hyphen -hypnoses -hypnosis -hypnotic -hypnotism -hypnotist -hypnotize -hypocrisy -hypocrite -ibuprofen -ice -iciness -icing -icky -icon -icy -idealism -idealist -idealize -ideally -idealness -identical -identify -identity -ideology -idiocy -idiom -idly -igloo -ignition -ignore -iguana -illicitly -illusion -illusive -image -imaginary -imagines -imaging -imbecile -imitate -imitation -immature -immerse -immersion -imminent -immobile -immodest -immorally -immortal -immovable -immovably -immunity -immunize -impaired -impale -impart -impatient -impeach -impeding -impending -imperfect -imperial -impish -implant -implement -implicate -implicit -implode -implosion -implosive -imply -impolite -important -importer -impose -imposing -impotence -impotency -impotent -impound -imprecise -imprint -imprison -impromptu -improper -improve -improving -improvise -imprudent -impulse -impulsive -impure -impurity -iodine -iodize -ion -ipad -iphone -ipod -irate -irk -iron -irregular -irrigate -irritable -irritably -irritant -irritate -islamic -islamist -isolated -isolating -isolation -isotope -issue -issuing -italicize -italics -item -itinerary -itunes -ivory -ivy -jab -jackal -jacket -jackknife -jackpot -jailbird -jailbreak -jailer -jailhouse -jalapeno -jam -janitor -january -jargon -jarring -jasmine -jaundice -jaunt -java -jawed -jawless -jawline -jaws -jaybird -jaywalker -jazz -jeep -jeeringly -jellied -jelly -jersey -jester -jet -jiffy -jigsaw -jimmy -jingle -jingling -jinx -jitters -jittery -job -jockey -jockstrap -jogger -jogging -john -joining -jokester -jokingly -jolliness -jolly -jolt -jot -jovial -joyfully -joylessly -joyous -joyride -joystick -jubilance -jubilant -judge -judgingly -judicial -judiciary -judo -juggle -juggling -jugular -juice -juiciness -juicy -jujitsu -jukebox -july -jumble -jumbo -jump -junction -juncture -june -junior -juniper -junkie -junkman -junkyard -jurist -juror -jury -justice -justifier -justify -justly -justness -juvenile -kabob -kangaroo -karaoke -karate -karma -kebab -keenly -keenness -keep -keg -kelp -kennel -kept -kerchief -kerosene -kettle -kick -kiln -kilobyte -kilogram -kilometer -kilowatt -kilt -kimono -kindle -kindling -kindly -kindness -kindred -kinetic -kinfolk -king -kinship -kinsman -kinswoman -kissable -kisser -kissing -kitchen -kite -kitten -kitty -kiwi -kleenex -knapsack -knee -knelt -knickers -knoll -koala -kooky -kosher -krypton -kudos -kung -labored -laborer -laboring -laborious -labrador -ladder -ladies -ladle -ladybug -ladylike -lagged -lagging -lagoon -lair -lake -lance -landed -landfall -landfill -landing -landlady -landless -landline -landlord -landmark -landmass -landmine -landowner -landscape -landside -landslide -language -lankiness -lanky -lantern -lapdog -lapel -lapped -lapping -laptop -lard -large -lark -lash -lasso -last -latch -late -lather -latitude -latrine -latter -latticed -launch -launder -laundry -laurel -lavender -lavish -laxative -lazily -laziness -lazy -lecturer -left -legacy -legal -legend -legged -leggings -legible -legibly -legislate -lego -legroom -legume -legwarmer -legwork -lemon -lend -length -lens -lent -leotard -lesser -letdown -lethargic -lethargy -letter -lettuce -level -leverage -levers -levitate -levitator -liability -liable -liberty -librarian -library -licking -licorice -lid -life -lifter -lifting -liftoff -ligament -likely -likeness -likewise -liking -lilac -lilly -lily -limb -limeade -limelight -limes -limit -limping -limpness -line -lingo -linguini -linguist -lining -linked -linoleum -linseed -lint -lion -lip -liquefy -liqueur -liquid -lisp -list -litigate -litigator -litmus -litter -little -livable -lived -lively -liver -livestock -lividly -living -lizard -lubricant -lubricate -lucid -luckily -luckiness -luckless -lucrative -ludicrous -lugged -lukewarm -lullaby -lumber -luminance -luminous -lumpiness -lumping -lumpish -lunacy -lunar -lunchbox -luncheon -lunchroom -lunchtime -lung -lurch -lure -luridness -lurk -lushly -lushness -luster -lustfully -lustily -lustiness -lustrous -lusty -luxurious -luxury -lying -lyrically -lyricism -lyricist -lyrics -macarena -macaroni -macaw -mace -machine -machinist -magazine -magenta -maggot -magical -magician -magma -magnesium -magnetic -magnetism -magnetize -magnifier -magnify -magnitude -magnolia -mahogany -maimed -majestic -majesty -majorette -majority -makeover -maker -makeshift -making -malformed -malt -mama -mammal -mammary -mammogram -manager -managing -manatee -mandarin -mandate -mandatory -mandolin -manger -mangle -mango -mangy -manhandle -manhole -manhood -manhunt -manicotti -manicure -manifesto -manila -mankind -manlike -manliness -manly -manmade -manned -mannish -manor -manpower -mantis -mantra -manual -many -map -marathon -marauding -marbled -marbles -marbling -march -mardi -margarine -margarita -margin -marigold -marina -marine -marital -maritime -marlin -marmalade -maroon -married -marrow -marry -marshland -marshy -marsupial -marvelous -marxism -mascot -masculine -mashed -mashing -massager -masses -massive -mastiff -matador -matchbook -matchbox -matcher -matching -matchless -material -maternal -maternity -math -mating -matriarch -matrimony -matrix -matron -matted -matter -maturely -maturing -maturity -mauve -maverick -maximize -maximum -maybe -mayday -mayflower -moaner -moaning -mobile -mobility -mobilize -mobster -mocha -mocker -mockup -modified -modify -modular -modulator -module -moisten -moistness -moisture -molar -molasses -mold -molecular -molecule -molehill -mollusk -mom -monastery -monday -monetary -monetize -moneybags -moneyless -moneywise -mongoose -mongrel -monitor -monkhood -monogamy -monogram -monologue -monopoly -monorail -monotone -monotype -monoxide -monsieur -monsoon -monstrous -monthly -monument -moocher -moodiness -moody -mooing -moonbeam -mooned -moonlight -moonlike -moonlit -moonrise -moonscape -moonshine -moonstone -moonwalk -mop -morale -morality -morally -morbidity -morbidly -morphine -morphing -morse -mortality -mortally -mortician -mortified -mortify -mortuary -mosaic -mossy -most -mothball -mothproof -motion -motivate -motivator -motive -motocross -motor -motto -mountable -mountain -mounted -mounting -mourner -mournful -mouse -mousiness -moustache -mousy -mouth -movable -move -movie -moving -mower -mowing -much -muck -mud -mug -mulberry -mulch -mule -mulled -mullets -multiple -multiply -multitask -multitude -mumble -mumbling -mumbo -mummified -mummify -mummy -mumps -munchkin -mundane -municipal -muppet -mural -murkiness -murky -murmuring -muscular -museum -mushily -mushiness -mushroom -mushy -music -musket -muskiness -musky -mustang -mustard -muster -mustiness -musty -mutable -mutate -mutation -mute -mutilated -mutilator -mutiny -mutt -mutual -muzzle -myself -myspace -mystified -mystify -myth -nacho -nag -nail -name -naming -nanny -nanometer -nape -napkin -napped -napping -nappy -narrow -nastily -nastiness -national -native -nativity -natural -nature -naturist -nautical -navigate -navigator -navy -nearby -nearest -nearly -nearness -neatly -neatness -nebula -nebulizer -nectar -negate -negation -negative -neglector -negligee -negligent -negotiate -nemeses -nemesis -neon -nephew -nerd -nervous -nervy -nest -net -neurology -neuron -neurosis -neurotic -neuter -neutron -never -next -nibble -nickname -nicotine -niece -nifty -nimble -nimbly -nineteen -ninetieth -ninja -nintendo -ninth -nuclear -nuclei -nucleus -nugget -nullify -number -numbing -numbly -numbness -numeral -numerate -numerator -numeric -numerous -nuptials -nursery -nursing -nurture -nutcase -nutlike -nutmeg -nutrient -nutshell -nuttiness -nutty -nuzzle -nylon -oaf -oak -oasis -oat -obedience -obedient -obituary -object -obligate -obliged -oblivion -oblivious -oblong -obnoxious -oboe -obscure -obscurity -observant -observer -observing -obsessed -obsession -obsessive -obsolete -obstacle -obstinate -obstruct -obtain -obtrusive -obtuse -obvious -occultist -occupancy -occupant -occupier -occupy -ocean -ocelot -octagon -octane -october -octopus -ogle -oil -oink -ointment -okay -old -olive -olympics -omega -omen -ominous -omission -omit -omnivore -onboard -oncoming -ongoing -onion -online -onlooker -only -onscreen -onset -onshore -onslaught -onstage -onto -onward -onyx -oops -ooze -oozy -opacity -opal -open -operable -operate -operating -operation -operative -operator -opium -opossum -opponent -oppose -opposing -opposite -oppressed -oppressor -opt -opulently -osmosis -other -otter -ouch -ought -ounce -outage -outback -outbid -outboard -outbound -outbreak -outburst -outcast -outclass -outcome -outdated -outdoors -outer -outfield -outfit -outflank -outgoing -outgrow -outhouse -outing -outlast -outlet -outline -outlook -outlying -outmatch -outmost -outnumber -outplayed -outpost -outpour -output -outrage -outrank -outreach -outright -outscore -outsell -outshine -outshoot -outsider -outskirts -outsmart -outsource -outspoken -outtakes -outthink -outward -outweigh -outwit -oval -ovary -oven -overact -overall -overarch -overbid -overbill -overbite -overblown -overboard -overbook -overbuilt -overcast -overcoat -overcome -overcook -overcrowd -overdraft -overdrawn -overdress -overdrive -overdue -overeager -overeater -overexert -overfed -overfeed -overfill -overflow -overfull -overgrown -overhand -overhang -overhaul -overhead -overhear -overheat -overhung -overjoyed -overkill -overlabor -overlaid -overlap -overlay -overload -overlook -overlord -overlying -overnight -overpass -overpay -overplant -overplay -overpower -overprice -overrate -overreach -overreact -override -overripe -overrule -overrun -overshoot -overshot -oversight -oversized -oversleep -oversold -overspend -overstate -overstay -overstep -overstock -overstuff -oversweet -overtake -overthrow -overtime -overtly -overtone -overture -overturn -overuse -overvalue -overview -overwrite -owl -oxford -oxidant -oxidation -oxidize -oxidizing -oxygen -oxymoron -oyster -ozone -paced -pacemaker -pacific -pacifier -pacifism -pacifist -pacify -padded -padding -paddle -paddling -padlock -pagan -pager -paging -pajamas -palace -palatable -palm -palpable -palpitate -paltry -pampered -pamperer -pampers -pamphlet -panama -pancake -pancreas -panda -pandemic -pang -panhandle -panic -panning -panorama -panoramic -panther -pantomime -pantry -pants -pantyhose -paparazzi -papaya -paper -paprika -papyrus -parabola -parachute -parade -paradox -paragraph -parakeet -paralegal -paralyses -paralysis -paralyze -paramedic -parameter -paramount -parasail -parasite -parasitic -parcel -parched -parchment -pardon -parish -parka -parking -parkway -parlor -parmesan -parole -parrot -parsley -parsnip -partake -parted -parting -partition -partly -partner -partridge -party -passable -passably -passage -passcode -passenger -passerby -passing -passion -passive -passivism -passover -passport -password -pasta -pasted -pastel -pastime -pastor -pastrami -pasture -pasty -patchwork -patchy -paternal -paternity -path -patience -patient -patio -patriarch -patriot -patrol -patronage -patronize -pauper -pavement -paver -pavestone -pavilion -paving -pawing -payable -payback -paycheck -payday -payee -payer -paying -payment -payphone -payroll -pebble -pebbly -pecan -pectin -peculiar -peddling -pediatric -pedicure -pedigree -pedometer -pegboard -pelican -pellet -pelt -pelvis -penalize -penalty -pencil -pendant -pending -penholder -penknife -pennant -penniless -penny -penpal -pension -pentagon -pentagram -pep -perceive -percent -perch -percolate -perennial -perfected -perfectly -perfume -periscope -perish -perjurer -perjury -perkiness -perky -perm -peroxide -perpetual -perplexed -persecute -persevere -persuaded -persuader -pesky -peso -pessimism -pessimist -pester -pesticide -petal -petite -petition -petri -petroleum -petted -petticoat -pettiness -petty -petunia -phantom -phobia -phoenix -phonebook -phoney -phonics -phoniness -phony -phosphate -photo -phrase -phrasing -placard -placate -placidly -plank -planner -plant -plasma -plaster -plastic -plated -platform -plating -platinum -platonic -platter -platypus -plausible -plausibly -playable -playback -player -playful -playgroup -playhouse -playing -playlist -playmaker -playmate -playoff -playpen -playroom -playset -plaything -playtime -plaza -pleading -pleat -pledge -plentiful -plenty -plethora -plexiglas -pliable -plod -plop -plot -plow -ploy -pluck -plug -plunder -plunging -plural -plus -plutonium -plywood -poach -pod -poem -poet -pogo -pointed -pointer -pointing -pointless -pointy -poise -poison -poker -poking -polar -police -policy -polio -polish -politely -polka -polo -polyester -polygon -polygraph -polymer -poncho -pond -pony -popcorn -pope -poplar -popper -poppy -popsicle -populace -popular -populate -porcupine -pork -porous -porridge -portable -portal -portfolio -porthole -portion -portly -portside -poser -posh -posing -possible -possibly -possum -postage -postal -postbox -postcard -posted -poster -posting -postnasal -posture -postwar -pouch -pounce -pouncing -pound -pouring -pout -powdered -powdering -powdery -power -powwow -pox -praising -prance -prancing -pranker -prankish -prankster -prayer -praying -preacher -preaching -preachy -preamble -precinct -precise -precision -precook -precut -predator -predefine -predict -preface -prefix -preflight -preformed -pregame -pregnancy -pregnant -preheated -prelaunch -prelaw -prelude -premiere -premises -premium -prenatal -preoccupy -preorder -prepaid -prepay -preplan -preppy -preschool -prescribe -preseason -preset -preshow -president -presoak -press -presume -presuming -preteen -pretended -pretender -pretense -pretext -pretty -pretzel -prevail -prevalent -prevent -preview -previous -prewar -prewashed -prideful -pried -primal -primarily -primary -primate -primer -primp -princess -print -prior -prism -prison -prissy -pristine -privacy -private -privatize -prize -proactive -probable -probably -probation -probe -probing -probiotic -problem -procedure -process -proclaim -procreate -procurer -prodigal -prodigy -produce -product -profane -profanity -professed -professor -profile -profound -profusely -progeny -prognosis -program -progress -projector -prologue -prolonged -promenade -prominent -promoter -promotion -prompter -promptly -prone -prong -pronounce -pronto -proofing -proofread -proofs -propeller -properly -property -proponent -proposal -propose -props -prorate -protector -protegee -proton -prototype -protozoan -protract -protrude -proud -provable -proved -proven -provided -provider -providing -province -proving -provoke -provoking -provolone -prowess -prowler -prowling -proximity -proxy -prozac -prude -prudishly -prune -pruning -pry -psychic -public -publisher -pucker -pueblo -pug -pull -pulmonary -pulp -pulsate -pulse -pulverize -puma -pumice -pummel -punch -punctual -punctuate -punctured -pungent -punisher -punk -pupil -puppet -puppy -purchase -pureblood -purebred -purely -pureness -purgatory -purge -purging -purifier -purify -purist -puritan -purity -purple -purplish -purposely -purr -purse -pursuable -pursuant -pursuit -purveyor -pushcart -pushchair -pusher -pushiness -pushing -pushover -pushpin -pushup -pushy -putdown -putt -puzzle -puzzling -pyramid -pyromania -python -quack -quadrant -quail -quaintly -quake -quaking -qualified -qualifier -qualify -quality -qualm -quantum -quarrel -quarry -quartered -quarterly -quarters -quartet -quench -query -quicken -quickly -quickness -quicksand -quickstep -quiet -quill -quilt -quintet -quintuple -quirk -quit -quiver -quizzical -quotable -quotation -quote -rabid -race -racing -racism -rack -racoon -radar -radial -radiance -radiantly -radiated -radiation -radiator -radio -radish -raffle -raft -rage -ragged -raging -ragweed -raider -railcar -railing -railroad -railway -raisin -rake -raking -rally -ramble -rambling -ramp -ramrod -ranch -rancidity -random -ranged -ranger -ranging -ranked -ranking -ransack -ranting -rants -rare -rarity -rascal -rash -rasping -ravage -raven -ravine -raving -ravioli -ravishing -reabsorb -reach -reacquire -reaction -reactive -reactor -reaffirm -ream -reanalyze -reappear -reapply -reappoint -reapprove -rearrange -rearview -reason -reassign -reassure -reattach -reawake -rebalance -rebate -rebel -rebirth -reboot -reborn -rebound -rebuff -rebuild -rebuilt -reburial -rebuttal -recall -recant -recapture -recast -recede -recent -recess -recharger -recipient -recital -recite -reckless -reclaim -recliner -reclining -recluse -reclusive -recognize -recoil -recollect -recolor -reconcile -reconfirm -reconvene -recopy -record -recount -recoup -recovery -recreate -rectal -rectangle -rectified -rectify -recycled -recycler -recycling -reemerge -reenact -reenter -reentry -reexamine -referable -referee -reference -refill -refinance -refined -refinery -refining -refinish -reflected -reflector -reflex -reflux -refocus -refold -reforest -reformat -reformed -reformer -reformist -refract -refrain -refreeze -refresh -refried -refueling -refund -refurbish -refurnish -refusal -refuse -refusing -refutable -refute -regain -regalia -regally -reggae -regime -region -register -registrar -registry -regress -regretful -regroup -regular -regulate -regulator -rehab -reheat -rehire -rehydrate -reimburse -reissue -reiterate -rejoice -rejoicing -rejoin -rekindle -relapse -relapsing -relatable -related -relation -relative -relax -relay -relearn -release -relenting -reliable -reliably -reliance -reliant -relic -relieve -relieving -relight -relish -relive -reload -relocate -relock -reluctant -rely -remake -remark -remarry -rematch -remedial -remedy -remember -reminder -remindful -remission -remix -remnant -remodeler -remold -remorse -remote -removable -removal -removed -remover -removing -rename -renderer -rendering -rendition -renegade -renewable -renewably -renewal -renewed -renounce -renovate -renovator -rentable -rental -rented -renter -reoccupy -reoccur -reopen -reorder -repackage -repacking -repaint -repair -repave -repaying -repayment -repeal -repeated -repeater -repent -rephrase -replace -replay -replica -reply -reporter -repose -repossess -repost -repressed -reprimand -reprint -reprise -reproach -reprocess -reproduce -reprogram -reps -reptile -reptilian -repugnant -repulsion -repulsive -repurpose -reputable -reputably -request -require -requisite -reroute -rerun -resale -resample -rescuer -reseal -research -reselect -reseller -resemble -resend -resent -reset -reshape -reshoot -reshuffle -residence -residency -resident -residual -residue -resigned -resilient -resistant -resisting -resize -resolute -resolved -resonant -resonate -resort -resource -respect -resubmit -result -resume -resupply -resurface -resurrect -retail -retainer -retaining -retake -retaliate -retention -rethink -retinal -retired -retiree -retiring -retold -retool -retorted -retouch -retrace -retract -retrain -retread -retreat -retrial -retrieval -retriever -retry -return -retying -retype -reunion -reunite -reusable -reuse -reveal -reveler -revenge -revenue -reverb -revered -reverence -reverend -reversal -reverse -reversing -reversion -revert -revisable -revise -revision -revisit -revivable -revival -reviver -reviving -revocable -revoke -revolt -revolver -revolving -reward -rewash -rewind -rewire -reword -rework -rewrap -rewrite -rhyme -ribbon -ribcage -rice -riches -richly -richness -rickety -ricotta -riddance -ridden -ride -riding -rifling -rift -rigging -rigid -rigor -rimless -rimmed -rind -rink -rinse -rinsing -riot -ripcord -ripeness -ripening -ripping -ripple -rippling -riptide -rise -rising -risk -risotto -ritalin -ritzy -rival -riverbank -riverbed -riverboat -riverside -riveter -riveting -roamer -roaming -roast -robbing -robe -robin -robotics -robust -rockband -rocker -rocket -rockfish -rockiness -rocking -rocklike -rockslide -rockstar -rocky -rogue -roman -romp -rope -roping -roster -rosy -rotten -rotting -rotunda -roulette -rounding -roundish -roundness -roundup -roundworm -routine -routing -rover -roving -royal -rubbed -rubber -rubbing -rubble -rubdown -ruby -ruckus -rudder -rug -ruined -rule -rumble -rumbling -rummage -rumor -runaround -rundown -runner -running -runny -runt -runway -rupture -rural -ruse -rush -rust -rut -sabbath -sabotage -sacrament -sacred -sacrifice -sadden -saddlebag -saddled -saddling -sadly -sadness -safari -safeguard -safehouse -safely -safeness -saffron -saga -sage -sagging -saggy -said -saint -sake -salad -salami -salaried -salary -saline -salon -saloon -salsa -salt -salutary -salute -salvage -salvaging -salvation -same -sample -sampling -sanction -sanctity -sanctuary -sandal -sandbag -sandbank -sandbar -sandblast -sandbox -sanded -sandfish -sanding -sandlot -sandpaper -sandpit -sandstone -sandstorm -sandworm -sandy -sanitary -sanitizer -sank -santa -sapling -sappiness -sappy -sarcasm -sarcastic -sardine -sash -sasquatch -sassy -satchel -satiable -satin -satirical -satisfied -satisfy -saturate -saturday -sauciness -saucy -sauna -savage -savanna -saved -savings -savior -savor -saxophone -say -scabbed -scabby -scalded -scalding -scale -scaling -scallion -scallop -scalping -scam -scandal -scanner -scanning -scant -scapegoat -scarce -scarcity -scarecrow -scared -scarf -scarily -scariness -scarring -scary -scavenger -scenic -schedule -schematic -scheme -scheming -schilling -schnapps -scholar -science -scientist -scion -scoff -scolding -scone -scoop -scooter -scope -scorch -scorebook -scorecard -scored -scoreless -scorer -scoring -scorn -scorpion -scotch -scoundrel -scoured -scouring -scouting -scouts -scowling -scrabble -scraggly -scrambled -scrambler -scrap -scratch -scrawny -screen -scribble -scribe -scribing -scrimmage -script -scroll -scrooge -scrounger -scrubbed -scrubber -scruffy -scrunch -scrutiny -scuba -scuff -sculptor -sculpture -scurvy -scuttle -secluded -secluding -seclusion -second -secrecy -secret -sectional -sector -secular -securely -security -sedan -sedate -sedation -sedative -sediment -seduce -seducing -segment -seismic -seizing -seldom -selected -selection -selective -selector -self -seltzer -semantic -semester -semicolon -semifinal -seminar -semisoft -semisweet -senate -senator -send -senior -senorita -sensation -sensitive -sensitize -sensually -sensuous -sepia -september -septic -septum -sequel -sequence -sequester -series -sermon -serotonin -serpent -serrated -serve -service -serving -sesame -sessions -setback -setting -settle -settling -setup -sevenfold -seventeen -seventh -seventy -severity -shabby -shack -shaded -shadily -shadiness -shading -shadow -shady -shaft -shakable -shakily -shakiness -shaking -shaky -shale -shallot -shallow -shame -shampoo -shamrock -shank -shanty -shape -shaping -share -sharpener -sharper -sharpie -sharply -sharpness -shawl -sheath -shed -sheep -sheet -shelf -shell -shelter -shelve -shelving -sherry -shield -shifter -shifting -shiftless -shifty -shimmer -shimmy -shindig -shine -shingle -shininess -shining -shiny -ship -shirt -shivering -shock -shone -shoplift -shopper -shopping -shoptalk -shore -shortage -shortcake -shortcut -shorten -shorter -shorthand -shortlist -shortly -shortness -shorts -shortwave -shorty -shout -shove -showbiz -showcase -showdown -shower -showgirl -showing -showman -shown -showoff -showpiece -showplace -showroom -showy -shrank -shrapnel -shredder -shredding -shrewdly -shriek -shrill -shrimp -shrine -shrink -shrivel -shrouded -shrubbery -shrubs -shrug -shrunk -shucking -shudder -shuffle -shuffling -shun -shush -shut -shy -siamese -siberian -sibling -siding -sierra -siesta -sift -sighing -silenced -silencer -silent -silica -silicon -silk -silliness -silly -silo -silt -silver -similarly -simile -simmering -simple -simplify -simply -sincere -sincerity -singer -singing -single -singular -sinister -sinless -sinner -sinuous -sip -siren -sister -sitcom -sitter -sitting -situated -situation -sixfold -sixteen -sixth -sixties -sixtieth -sixtyfold -sizable -sizably -size -sizing -sizzle -sizzling -skater -skating -skedaddle -skeletal -skeleton -skeptic -sketch -skewed -skewer -skid -skied -skier -skies -skiing -skilled -skillet -skillful -skimmed -skimmer -skimming -skimpily -skincare -skinhead -skinless -skinning -skinny -skintight -skipper -skipping -skirmish -skirt -skittle -skydiver -skylight -skyline -skype -skyrocket -skyward -slab -slacked -slacker -slacking -slackness -slacks -slain -slam -slander -slang -slapping -slapstick -slashed -slashing -slate -slather -slaw -sled -sleek -sleep -sleet -sleeve -slept -sliceable -sliced -slicer -slicing -slick -slider -slideshow -sliding -slighted -slighting -slightly -slimness -slimy -slinging -slingshot -slinky -slip -slit -sliver -slobbery -slogan -sloped -sloping -sloppily -sloppy -slot -slouching -slouchy -sludge -slug -slum -slurp -slush -sly -small -smartly -smartness -smasher -smashing -smashup -smell -smelting -smile -smilingly -smirk -smite -smith -smitten -smock -smog -smoked -smokeless -smokiness -smoking -smoky -smolder -smooth -smother -smudge -smudgy -smuggler -smuggling -smugly -smugness -snack -snagged -snaking -snap -snare -snarl -snazzy -sneak -sneer -sneeze -sneezing -snide -sniff -snippet -snipping -snitch -snooper -snooze -snore -snoring -snorkel -snort -snout -snowbird -snowboard -snowbound -snowcap -snowdrift -snowdrop -snowfall -snowfield -snowflake -snowiness -snowless -snowman -snowplow -snowshoe -snowstorm -snowsuit -snowy -snub -snuff -snuggle -snugly -snugness -speak -spearfish -spearhead -spearman -spearmint -species -specimen -specked -speckled -specks -spectacle -spectator -spectrum -speculate -speech -speed -spellbind -speller -spelling -spendable -spender -spending -spent -spew -sphere -spherical -sphinx -spider -spied -spiffy -spill -spilt -spinach -spinal -spindle -spinner -spinning -spinout -spinster -spiny -spiral -spirited -spiritism -spirits -spiritual -splashed -splashing -splashy -splatter -spleen -splendid -splendor -splice -splicing -splinter -splotchy -splurge -spoilage -spoiled -spoiler -spoiling -spoils -spoken -spokesman -sponge -spongy -sponsor -spoof -spookily -spooky -spool -spoon -spore -sporting -sports -sporty -spotless -spotlight -spotted -spotter -spotting -spotty -spousal -spouse -spout -sprain -sprang -sprawl -spray -spree -sprig -spring -sprinkled -sprinkler -sprint -sprite -sprout -spruce -sprung -spry -spud -spur -sputter -spyglass -squabble -squad -squall -squander -squash -squatted -squatter -squatting -squeak -squealer -squealing -squeamish -squeegee -squeeze -squeezing -squid -squiggle -squiggly -squint -squire -squirt -squishier -squishy -stability -stabilize -stable -stack -stadium -staff -stage -staging -stagnant -stagnate -stainable -stained -staining -stainless -stalemate -staleness -stalling -stallion -stamina -stammer -stamp -stand -stank -staple -stapling -starboard -starch -stardom -stardust -starfish -stargazer -staring -stark -starless -starlet -starlight -starlit -starring -starry -starship -starter -starting -startle -startling -startup -starved -starving -stash -state -static -statistic -statue -stature -status -statute -statutory -staunch -stays -steadfast -steadier -steadily -steadying -steam -steed -steep -steerable -steering -steersman -stegosaur -stellar -stem -stench -stencil -step -stereo -sterile -sterility -sterilize -sterling -sternness -sternum -stew -stick -stiffen -stiffly -stiffness -stifle -stifling -stillness -stilt -stimulant -stimulate -stimuli -stimulus -stinger -stingily -stinging -stingray -stingy -stinking -stinky -stipend -stipulate -stir -stitch -stock -stoic -stoke -stole -stomp -stonewall -stoneware -stonework -stoning -stony -stood -stooge -stool -stoop -stoplight -stoppable -stoppage -stopped -stopper -stopping -stopwatch -storable -storage -storeroom -storewide -storm -stout -stove -stowaway -stowing -straddle -straggler -strained -strainer -straining -strangely -stranger -strangle -strategic -strategy -stratus -straw -stray -streak -stream -street -strength -strenuous -strep -stress -stretch -strewn -stricken -strict -stride -strife -strike -striking -strive -striving -strobe -strode -stroller -strongbox -strongly -strongman -struck -structure -strudel -struggle -strum -strung -strut -stubbed -stubble -stubbly -stubborn -stucco -stuck -student -studied -studio -study -stuffed -stuffing -stuffy -stumble -stumbling -stump -stung -stunned -stunner -stunning -stunt -stupor -sturdily -sturdy -styling -stylishly -stylist -stylized -stylus -suave -subarctic -subatomic -subdivide -subdued -subduing -subfloor -subgroup -subheader -subject -sublease -sublet -sublevel -sublime -submarine -submerge -submersed -submitter -subpanel -subpar -subplot -subprime -subscribe -subscript -subsector -subside -subsiding -subsidize -subsidy -subsoil -subsonic -substance -subsystem -subtext -subtitle -subtly -subtotal -subtract -subtype -suburb -subway -subwoofer -subzero -succulent -such -suction -sudden -sudoku -suds -sufferer -suffering -suffice -suffix -suffocate -suffrage -sugar -suggest -suing -suitable -suitably -suitcase -suitor -sulfate -sulfide -sulfite -sulfur -sulk -sullen -sulphate -sulphuric -sultry -superbowl -superglue -superhero -superior -superjet -superman -supermom -supernova -supervise -supper -supplier -supply -support -supremacy -supreme -surcharge -surely -sureness -surface -surfacing -surfboard -surfer -surgery -surgical -surging -surname -surpass -surplus -surprise -surreal -surrender -surrogate -surround -survey -survival -survive -surviving -survivor -sushi -suspect -suspend -suspense -sustained -sustainer -swab -swaddling -swagger -swampland -swan -swapping -swarm -sway -swear -sweat -sweep -swell -swept -swerve -swifter -swiftly -swiftness -swimmable -swimmer -swimming -swimsuit -swimwear -swinger -swinging -swipe -swirl -switch -swivel -swizzle -swooned -swoop -swoosh -swore -sworn -swung -sycamore -sympathy -symphonic -symphony -symptom -synapse -syndrome -synergy -synopses -synopsis -synthesis -synthetic -syrup -system -t-shirt -tabasco -tabby -tableful -tables -tablet -tableware -tabloid -tackiness -tacking -tackle -tackling -tacky -taco -tactful -tactical -tactics -tactile -tactless -tadpole -taekwondo -tag -tainted -take -taking -talcum -talisman -tall -talon -tamale -tameness -tamer -tamper -tank -tanned -tannery -tanning -tantrum -tapeless -tapered -tapering -tapestry -tapioca -tapping -taps -tarantula -target -tarmac -tarnish -tarot -tartar -tartly -tartness -task -tassel -taste -tastiness -tasting -tasty -tattered -tattle -tattling -tattoo -taunt -tavern -thank -that -thaw -theater -theatrics -thee -theft -theme -theology -theorize -thermal -thermos -thesaurus -these -thesis -thespian -thicken -thicket -thickness -thieving -thievish -thigh -thimble -thing -think -thinly -thinner -thinness -thinning -thirstily -thirsting -thirsty -thirteen -thirty -thong -thorn -those -thousand -thrash -thread -threaten -threefold -thrift -thrill -thrive -thriving -throat -throbbing -throng -throttle -throwaway -throwback -thrower -throwing -thud -thumb -thumping -thursday -thus -thwarting -thyself -tiara -tibia -tidal -tidbit -tidiness -tidings -tidy -tiger -tighten -tightly -tightness -tightrope -tightwad -tigress -tile -tiling -till -tilt -timid -timing -timothy -tinderbox -tinfoil -tingle -tingling -tingly -tinker -tinkling -tinsel -tinsmith -tint -tinwork -tiny -tipoff -tipped -tipper -tipping -tiptoeing -tiptop -tiring -tissue -trace -tracing -track -traction -tractor -trade -trading -tradition -traffic -tragedy -trailing -trailside -train -traitor -trance -tranquil -transfer -transform -translate -transpire -transport -transpose -trapdoor -trapeze -trapezoid -trapped -trapper -trapping -traps -trash -travel -traverse -travesty -tray -treachery -treading -treadmill -treason -treat -treble -tree -trekker -tremble -trembling -tremor -trench -trend -trespass -triage -trial -triangle -tribesman -tribunal -tribune -tributary -tribute -triceps -trickery -trickily -tricking -trickle -trickster -tricky -tricolor -tricycle -trident -tried -trifle -trifocals -trillion -trilogy -trimester -trimmer -trimming -trimness -trinity -trio -tripod -tripping -triumph -trivial -trodden -trolling -trombone -trophy -tropical -tropics -trouble -troubling -trough -trousers -trout -trowel -truce -truck -truffle -trump -trunks -trustable -trustee -trustful -trusting -trustless -truth -try -tubby -tubeless -tubular -tucking -tuesday -tug -tuition -tulip -tumble -tumbling -tummy -turban -turbine -turbofan -turbojet -turbulent -turf -turkey -turmoil -turret -turtle -tusk -tutor -tutu -tux -tweak -tweed -tweet -tweezers -twelve -twentieth -twenty -twerp -twice -twiddle -twiddling -twig -twilight -twine -twins -twirl -twistable -twisted -twister -twisting -twisty -twitch -twitter -tycoon -tying -tyke -udder -ultimate -ultimatum -ultra -umbilical -umbrella -umpire -unabashed -unable -unadorned -unadvised -unafraid -unaired -unaligned -unaltered -unarmored -unashamed -unaudited -unawake -unaware -unbaked -unbalance -unbeaten -unbend -unbent -unbiased -unbitten -unblended -unblessed -unblock -unbolted -unbounded -unboxed -unbraided -unbridle -unbroken -unbuckled -unbundle -unburned -unbutton -uncanny -uncapped -uncaring -uncertain -unchain -unchanged -uncharted -uncheck -uncivil -unclad -unclaimed -unclamped -unclasp -uncle -unclip -uncloak -unclog -unclothed -uncoated -uncoiled -uncolored -uncombed -uncommon -uncooked -uncork -uncorrupt -uncounted -uncouple -uncouth -uncover -uncross -uncrown -uncrushed -uncured -uncurious -uncurled -uncut -undamaged -undated -undaunted -undead -undecided -undefined -underage -underarm -undercoat -undercook -undercut -underdog -underdone -underfed -underfeed -underfoot -undergo -undergrad -underhand -underline -underling -undermine -undermost -underpaid -underpass -underpay -underrate -undertake -undertone -undertook -undertow -underuse -underwear -underwent -underwire -undesired -undiluted -undivided -undocked -undoing -undone -undrafted -undress -undrilled -undusted -undying -unearned -unearth -unease -uneasily -uneasy -uneatable -uneaten -unedited -unelected -unending -unengaged -unenvied -unequal -unethical -uneven -unexpired -unexposed -unfailing -unfair -unfasten -unfazed -unfeeling -unfiled -unfilled -unfitted -unfitting -unfixable -unfixed -unflawed -unfocused -unfold -unfounded -unframed -unfreeze -unfrosted -unfrozen -unfunded -unglazed -ungloved -unglue -ungodly -ungraded -ungreased -unguarded -unguided -unhappily -unhappy -unharmed -unhealthy -unheard -unhearing -unheated -unhelpful -unhidden -unhinge -unhitched -unholy -unhook -unicorn -unicycle -unified -unifier -uniformed -uniformly -unify -unimpeded -uninjured -uninstall -uninsured -uninvited -union -uniquely -unisexual -unison -unissued -unit -universal -universe -unjustly -unkempt -unkind -unknotted -unknowing -unknown -unlaced -unlatch -unlawful -unleaded -unlearned -unleash -unless -unleveled -unlighted -unlikable -unlimited -unlined -unlinked -unlisted -unlit -unlivable -unloaded -unloader -unlocked -unlocking -unlovable -unloved -unlovely -unloving -unluckily -unlucky -unmade -unmanaged -unmanned -unmapped -unmarked -unmasked -unmasking -unmatched -unmindful -unmixable -unmixed -unmolded -unmoral -unmovable -unmoved -unmoving -unnamable -unnamed -unnatural -unneeded -unnerve -unnerving -unnoticed -unopened -unopposed -unpack -unpadded -unpaid -unpainted -unpaired -unpaved -unpeeled -unpicked -unpiloted -unpinned -unplanned -unplanted -unpleased -unpledged -unplowed -unplug -unpopular -unproven -unquote -unranked -unrated -unraveled -unreached -unread -unreal -unreeling -unrefined -unrelated -unrented -unrest -unretired -unrevised -unrigged -unripe -unrivaled -unroasted -unrobed -unroll -unruffled -unruly -unrushed -unsaddle -unsafe -unsaid -unsalted -unsaved -unsavory -unscathed -unscented -unscrew -unsealed -unseated -unsecured -unseeing -unseemly -unseen -unselect -unselfish -unsent -unsettled -unshackle -unshaken -unshaved -unshaven -unsheathe -unshipped -unsightly -unsigned -unskilled -unsliced -unsmooth -unsnap -unsocial -unsoiled -unsold -unsolved -unsorted -unspoiled -unspoken -unstable -unstaffed -unstamped -unsteady -unsterile -unstirred -unstitch -unstopped -unstuck -unstuffed -unstylish -unsubtle -unsubtly -unsuited -unsure -unsworn -untagged -untainted -untaken -untamed -untangled -untapped -untaxed -unthawed -unthread -untidy -untie -until -untimed -untimely -untitled -untoasted -untold -untouched -untracked -untrained -untreated -untried -untrimmed -untrue -untruth -unturned -untwist -untying -unusable -unused -unusual -unvalued -unvaried -unvarying -unveiled -unveiling -unvented -unviable -unvisited -unvocal -unwanted -unwarlike -unwary -unwashed -unwatched -unweave -unwed -unwelcome -unwell -unwieldy -unwilling -unwind -unwired -unwitting -unwomanly -unworldly -unworn -unworried -unworthy -unwound -unwoven -unwrapped -unwritten -unzip -upbeat -upchuck -upcoming -upcountry -update -upfront -upgrade -upheaval -upheld -uphill -uphold -uplifted -uplifting -upload -upon -upper -upright -uprising -upriver -uproar -uproot -upscale -upside -upstage -upstairs -upstart -upstate -upstream -upstroke -upswing -uptake -uptight -uptown -upturned -upward -upwind -uranium -urban -urchin -urethane -urgency -urgent -urging -urologist -urology -usable -usage -useable -used -uselessly -user -usher -usual -utensil -utility -utilize -utmost -utopia -utter -vacancy -vacant -vacate -vacation -vagabond -vagrancy -vagrantly -vaguely -vagueness -valiant -valid -valium -valley -valuables -value -vanilla -vanish -vanity -vanquish -vantage -vaporizer -variable -variably -varied -variety -various -varmint -varnish -varsity -varying -vascular -vaseline -vastly -vastness -veal -vegan -veggie -vehicular -velcro -velocity -velvet -vendetta -vending -vendor -veneering -vengeful -venomous -ventricle -venture -venue -venus -verbalize -verbally -verbose -verdict -verify -verse -version -versus -vertebrae -vertical -vertigo -very -vessel -vest -veteran -veto -vexingly -viability -viable -vibes -vice -vicinity -victory -video -viewable -viewer -viewing -viewless -viewpoint -vigorous -village -villain -vindicate -vineyard -vintage -violate -violation -violator -violet -violin -viper -viral -virtual -virtuous -virus -visa -viscosity -viscous -viselike -visible -visibly -vision -visiting -visitor -visor -vista -vitality -vitalize -vitally -vitamins -vivacious -vividly -vividness -vixen -vocalist -vocalize -vocally -vocation -voice -voicing -void -volatile -volley -voltage -volumes -voter -voting -voucher -vowed -vowel -voyage -wackiness -wad -wafer -waffle -waged -wager -wages -waggle -wagon -wake -waking -walk -walmart -walnut -walrus -waltz -wand -wannabe -wanted -wanting -wasabi -washable -washbasin -washboard -washbowl -washcloth -washday -washed -washer -washhouse -washing -washout -washroom -washstand -washtub -wasp -wasting -watch -water -waviness -waving -wavy -whacking -whacky -wham -wharf -wheat -whenever -whiff -whimsical -whinny -whiny -whisking -whoever -whole -whomever -whoopee -whooping -whoops -why -wick -widely -widen -widget -widow -width -wieldable -wielder -wife -wifi -wikipedia -wildcard -wildcat -wilder -wildfire -wildfowl -wildland -wildlife -wildly -wildness -willed -willfully -willing -willow -willpower -wilt -wimp -wince -wincing -wind -wing -winking -winner -winnings -winter -wipe -wired -wireless -wiring -wiry -wisdom -wise -wish -wisplike -wispy -wistful -wizard -wobble -wobbling -wobbly -wok -wolf -wolverine -womanhood -womankind -womanless -womanlike -womanly -womb -woof -wooing -wool -woozy -word -work -worried -worrier -worrisome -worry -worsening -worshiper -worst -wound -woven -wow -wrangle -wrath -wreath -wreckage -wrecker -wrecking -wrench -wriggle -wriggly -wrinkle -wrinkly -wrist -writing -written -wrongdoer -wronged -wrongful -wrongly -wrongness -wrought -xbox -xerox -yahoo -yam -yanking -yapping -yard -yarn -yeah -yearbook -yearling -yearly -yearning -yeast -yelling -yelp -yen -yesterday -yiddish -yield -yin -yippee -yo-yo -yodel -yoga -yogurt -yonder -yoyo -yummy -zap -zealous -zebra -zen -zeppelin -zero -zestfully -zesty -zigzagged -zipfile -zipping -zippy -zips -zit -zodiac -zombie -zone -zoning -zookeeper -zoologist -zoology -zoom diff --git a/src/passlib/_data/wordsets/eff_prefixed.txt b/src/passlib/_data/wordsets/eff_prefixed.txt deleted file mode 100644 index 9ac732fe..00000000 --- a/src/passlib/_data/wordsets/eff_prefixed.txt +++ /dev/null @@ -1,1296 +0,0 @@ -aardvark -abandoned -abbreviate -abdomen -abhorrence -abiding -abnormal -abrasion -absorbing -abundant -abyss -academy -accountant -acetone -achiness -acid -acoustics -acquire -acrobat -actress -acuteness -aerosol -aesthetic -affidavit -afloat -afraid -aftershave -again -agency -aggressor -aghast -agitate -agnostic -agonizing -agreeing -aidless -aimlessly -ajar -alarmclock -albatross -alchemy -alfalfa -algae -aliens -alkaline -almanac -alongside -alphabet -already -also -altitude -aluminum -always -amazingly -ambulance -amendment -amiable -ammunition -amnesty -amoeba -amplifier -amuser -anagram -anchor -android -anesthesia -angelfish -animal -anklet -announcer -anonymous -answer -antelope -anxiety -anyplace -aorta -apartment -apnea -apostrophe -apple -apricot -aquamarine -arachnid -arbitrate -ardently -arena -argument -aristocrat -armchair -aromatic -arrowhead -arsonist -artichoke -asbestos -ascend -aseptic -ashamed -asinine -asleep -asocial -asparagus -astronaut -asymmetric -atlas -atmosphere -atom -atrocious -attic -atypical -auctioneer -auditorium -augmented -auspicious -automobile -auxiliary -avalanche -avenue -aviator -avocado -awareness -awhile -awkward -awning -awoke -axially -azalea -babbling -backpack -badass -bagpipe -bakery -balancing -bamboo -banana -barracuda -basket -bathrobe -bazooka -blade -blender -blimp -blouse -blurred -boatyard -bobcat -body -bogusness -bohemian -boiler -bonnet -boots -borough -bossiness -bottle -bouquet -boxlike -breath -briefcase -broom -brushes -bubblegum -buckle -buddhist -buffalo -bullfrog -bunny -busboy -buzzard -cabin -cactus -cadillac -cafeteria -cage -cahoots -cajoling -cakewalk -calculator -camera -canister -capsule -carrot -cashew -cathedral -caucasian -caviar -ceasefire -cedar -celery -cement -census -ceramics -cesspool -chalkboard -cheesecake -chimney -chlorine -chopsticks -chrome -chute -cilantro -cinnamon -circle -cityscape -civilian -clay -clergyman -clipboard -clock -clubhouse -coathanger -cobweb -coconut -codeword -coexistent -coffeecake -cognitive -cohabitate -collarbone -computer -confetti -copier -cornea -cosmetics -cotton -couch -coverless -coyote -coziness -crawfish -crewmember -crib -croissant -crumble -crystal -cubical -cucumber -cuddly -cufflink -cuisine -culprit -cup -curry -cushion -cuticle -cybernetic -cyclist -cylinder -cymbal -cynicism -cypress -cytoplasm -dachshund -daffodil -dagger -dairy -dalmatian -dandelion -dartboard -dastardly -datebook -daughter -dawn -daytime -dazzler -dealer -debris -decal -dedicate -deepness -defrost -degree -dehydrator -deliverer -democrat -dentist -deodorant -depot -deranged -desktop -detergent -device -dexterity -diamond -dibs -dictionary -diffuser -digit -dilated -dimple -dinnerware -dioxide -diploma -directory -dishcloth -ditto -dividers -dizziness -doctor -dodge -doll -dominoes -donut -doorstep -dorsal -double -downstairs -dozed -drainpipe -dresser -driftwood -droppings -drum -dryer -dubiously -duckling -duffel -dugout -dumpster -duplex -durable -dustpan -dutiful -duvet -dwarfism -dwelling -dwindling -dynamite -dyslexia -eagerness -earlobe -easel -eavesdrop -ebook -eccentric -echoless -eclipse -ecosystem -ecstasy -edged -editor -educator -eelworm -eerie -effects -eggnog -egomaniac -ejection -elastic -elbow -elderly -elephant -elfishly -eliminator -elk -elliptical -elongated -elsewhere -elusive -elves -emancipate -embroidery -emcee -emerald -emission -emoticon -emperor -emulate -enactment -enchilada -endorphin -energy -enforcer -engine -enhance -enigmatic -enjoyably -enlarged -enormous -enquirer -enrollment -ensemble -entryway -enunciate -envoy -enzyme -epidemic -equipment -erasable -ergonomic -erratic -eruption -escalator -eskimo -esophagus -espresso -essay -estrogen -etching -eternal -ethics -etiquette -eucalyptus -eulogy -euphemism -euthanize -evacuation -evergreen -evidence -evolution -exam -excerpt -exerciser -exfoliate -exhale -exist -exorcist -explode -exquisite -exterior -exuberant -fabric -factory -faded -failsafe -falcon -family -fanfare -fasten -faucet -favorite -feasibly -february -federal -feedback -feigned -feline -femur -fence -ferret -festival -fettuccine -feudalist -feverish -fiberglass -fictitious -fiddle -figurine -fillet -finalist -fiscally -fixture -flashlight -fleshiness -flight -florist -flypaper -foamless -focus -foggy -folksong -fondue -footpath -fossil -fountain -fox -fragment -freeway -fridge -frosting -fruit -fryingpan -gadget -gainfully -gallstone -gamekeeper -gangway -garlic -gaslight -gathering -gauntlet -gearbox -gecko -gem -generator -geographer -gerbil -gesture -getaway -geyser -ghoulishly -gibberish -giddiness -giftshop -gigabyte -gimmick -giraffe -giveaway -gizmo -glasses -gleeful -glisten -glove -glucose -glycerin -gnarly -gnomish -goatskin -goggles -goldfish -gong -gooey -gorgeous -gosling -gothic -gourmet -governor -grape -greyhound -grill -groundhog -grumbling -guacamole -guerrilla -guitar -gullible -gumdrop -gurgling -gusto -gutless -gymnast -gynecology -gyration -habitat -hacking -haggard -haiku -halogen -hamburger -handgun -happiness -hardhat -hastily -hatchling -haughty -hazelnut -headband -hedgehog -hefty -heinously -helmet -hemoglobin -henceforth -herbs -hesitation -hexagon -hubcap -huddling -huff -hugeness -hullabaloo -human -hunter -hurricane -hushing -hyacinth -hybrid -hydrant -hygienist -hypnotist -ibuprofen -icepack -icing -iconic -identical -idiocy -idly -igloo -ignition -iguana -illuminate -imaging -imbecile -imitator -immigrant -imprint -iodine -ionosphere -ipad -iphone -iridescent -irksome -iron -irrigation -island -isotope -issueless -italicize -itemizer -itinerary -itunes -ivory -jabbering -jackrabbit -jaguar -jailhouse -jalapeno -jamboree -janitor -jarring -jasmine -jaundice -jawbreaker -jaywalker -jazz -jealous -jeep -jelly -jeopardize -jersey -jetski -jezebel -jiffy -jigsaw -jingling -jobholder -jockstrap -jogging -john -joinable -jokingly -journal -jovial -joystick -jubilant -judiciary -juggle -juice -jujitsu -jukebox -jumpiness -junkyard -juror -justifying -juvenile -kabob -kamikaze -kangaroo -karate -kayak -keepsake -kennel -kerosene -ketchup -khaki -kickstand -kilogram -kimono -kingdom -kiosk -kissing -kite -kleenex -knapsack -kneecap -knickers -koala -krypton -laboratory -ladder -lakefront -lantern -laptop -laryngitis -lasagna -latch -laundry -lavender -laxative -lazybones -lecturer -leftover -leggings -leisure -lemon -length -leopard -leprechaun -lettuce -leukemia -levers -lewdness -liability -library -licorice -lifeboat -lightbulb -likewise -lilac -limousine -lint -lioness -lipstick -liquid -listless -litter -liverwurst -lizard -llama -luau -lubricant -lucidity -ludicrous -luggage -lukewarm -lullaby -lumberjack -lunchbox -luridness -luscious -luxurious -lyrics -macaroni -maestro -magazine -mahogany -maimed -majority -makeover -malformed -mammal -mango -mapmaker -marbles -massager -matchstick -maverick -maximum -mayonnaise -moaning -mobilize -moccasin -modify -moisture -molecule -momentum -monastery -moonshine -mortuary -mosquito -motorcycle -mousetrap -movie -mower -mozzarella -muckiness -mudflow -mugshot -mule -mummy -mundane -muppet -mural -mustard -mutation -myriad -myspace -myth -nail -namesake -nanosecond -napkin -narrator -nastiness -natives -nautically -navigate -nearest -nebula -nectar -nefarious -negotiator -neither -nemesis -neoliberal -nephew -nervously -nest -netting -neuron -nevermore -nextdoor -nicotine -niece -nimbleness -nintendo -nirvana -nuclear -nugget -nuisance -nullify -numbing -nuptials -nursery -nutcracker -nylon -oasis -oat -obediently -obituary -object -obliterate -obnoxious -observer -obtain -obvious -occupation -oceanic -octopus -ocular -office -oftentimes -oiliness -ointment -older -olympics -omissible -omnivorous -oncoming -onion -onlooker -onstage -onward -onyx -oomph -opaquely -opera -opium -opossum -opponent -optical -opulently -oscillator -osmosis -ostrich -otherwise -ought -outhouse -ovation -oven -owlish -oxford -oxidize -oxygen -oyster -ozone -pacemaker -padlock -pageant -pajamas -palm -pamphlet -pantyhose -paprika -parakeet -passport -patio -pauper -pavement -payphone -pebble -peculiarly -pedometer -pegboard -pelican -penguin -peony -pepperoni -peroxide -pesticide -petroleum -pewter -pharmacy -pheasant -phonebook -phrasing -physician -plank -pledge -plotted -plug -plywood -pneumonia -podiatrist -poetic -pogo -poison -poking -policeman -poncho -popcorn -porcupine -postcard -poultry -powerboat -prairie -pretzel -princess -propeller -prune -pry -pseudo -psychopath -publisher -pucker -pueblo -pulley -pumpkin -punchbowl -puppy -purse -pushup -putt -puzzle -pyramid -python -quarters -quesadilla -quilt -quote -racoon -radish -ragweed -railroad -rampantly -rancidity -rarity -raspberry -ravishing -rearrange -rebuilt -receipt -reentry -refinery -register -rehydrate -reimburse -rejoicing -rekindle -relic -remote -renovator -reopen -reporter -request -rerun -reservoir -retriever -reunion -revolver -rewrite -rhapsody -rhetoric -rhino -rhubarb -rhyme -ribbon -riches -ridden -rigidness -rimmed -riptide -riskily -ritzy -riverboat -roamer -robe -rocket -romancer -ropelike -rotisserie -roundtable -royal -rubber -rudderless -rugby -ruined -rulebook -rummage -running -rupture -rustproof -sabotage -sacrifice -saddlebag -saffron -sainthood -saltshaker -samurai -sandworm -sapphire -sardine -sassy -satchel -sauna -savage -saxophone -scarf -scenario -schoolbook -scientist -scooter -scrapbook -sculpture -scythe -secretary -sedative -segregator -seismology -selected -semicolon -senator -septum -sequence -serpent -sesame -settler -severely -shack -shelf -shirt -shovel -shrimp -shuttle -shyness -siamese -sibling -siesta -silicon -simmering -singles -sisterhood -sitcom -sixfold -sizable -skateboard -skeleton -skies -skulk -skylight -slapping -sled -slingshot -sloth -slumbering -smartphone -smelliness -smitten -smokestack -smudge -snapshot -sneezing -sniff -snowsuit -snugness -speakers -sphinx -spider -splashing -sponge -sprout -spur -spyglass -squirrel -statue -steamboat -stingray -stopwatch -strawberry -student -stylus -suave -subway -suction -suds -suffocate -sugar -suitcase -sulphur -superstore -surfer -sushi -swan -sweatshirt -swimwear -sword -sycamore -syllable -symphony -synagogue -syringes -systemize -tablespoon -taco -tadpole -taekwondo -tagalong -takeout -tallness -tamale -tanned -tapestry -tarantula -tastebud -tattoo -tavern -thaw -theater -thimble -thorn -throat -thumb -thwarting -tiara -tidbit -tiebreaker -tiger -timid -tinsel -tiptoeing -tirade -tissue -tractor -tree -tripod -trousers -trucks -tryout -tubeless -tuesday -tugboat -tulip -tumbleweed -tupperware -turtle -tusk -tutorial -tuxedo -tweezers -twins -tyrannical -ultrasound -umbrella -umpire -unarmored -unbuttoned -uncle -underwear -unevenness -unflavored -ungloved -unhinge -unicycle -unjustly -unknown -unlocking -unmarked -unnoticed -unopened -unpaved -unquenched -unroll -unscrewing -untied -unusual -unveiled -unwrinkled -unyielding -unzip -upbeat -upcountry -update -upfront -upgrade -upholstery -upkeep -upload -uppercut -upright -upstairs -uptown -upwind -uranium -urban -urchin -urethane -urgent -urologist -username -usher -utensil -utility -utmost -utopia -utterance -vacuum -vagrancy -valuables -vanquished -vaporizer -varied -vaseline -vegetable -vehicle -velcro -vendor -vertebrae -vestibule -veteran -vexingly -vicinity -videogame -viewfinder -vigilante -village -vinegar -violin -viperfish -virus -visor -vitamins -vivacious -vixen -vocalist -vogue -voicemail -volleyball -voucher -voyage -vulnerable -waffle -wagon -wakeup -walrus -wanderer -wasp -water -waving -wheat -whisper -wholesaler -wick -widow -wielder -wifeless -wikipedia -wildcat -windmill -wipeout -wired -wishbone -wizardry -wobbliness -wolverine -womb -woolworker -workbasket -wound -wrangle -wreckage -wristwatch -wrongdoing -xerox -xylophone -yacht -yahoo -yard -yearbook -yesterday -yiddish -yield -yo-yo -yodel -yogurt -yuppie -zealot -zebra -zeppelin -zestfully -zigzagged -zillion -zipping -zirconium -zodiac -zombie -zookeeper -zucchini diff --git a/src/passlib/_data/wordsets/eff_short.txt b/src/passlib/_data/wordsets/eff_short.txt deleted file mode 100644 index 4c8baa4c..00000000 --- a/src/passlib/_data/wordsets/eff_short.txt +++ /dev/null @@ -1,1296 +0,0 @@ -acid -acorn -acre -acts -afar -affix -aged -agent -agile -aging -agony -ahead -aide -aids -aim -ajar -alarm -alias -alibi -alien -alike -alive -aloe -aloft -aloha -alone -amend -amino -ample -amuse -angel -anger -angle -ankle -apple -april -apron -aqua -area -arena -argue -arise -armed -armor -army -aroma -array -arson -art -ashen -ashes -atlas -atom -attic -audio -avert -avoid -awake -award -awoke -axis -bacon -badge -bagel -baggy -baked -baker -balmy -banjo -barge -barn -bash -basil -bask -batch -bath -baton -bats -blade -blank -blast -blaze -bleak -blend -bless -blimp -blink -bloat -blob -blog -blot -blunt -blurt -blush -boast -boat -body -boil -bok -bolt -boned -boney -bonus -bony -book -booth -boots -boss -botch -both -boxer -breed -bribe -brick -bride -brim -bring -brink -brisk -broad -broil -broke -brook -broom -brush -buck -bud -buggy -bulge -bulk -bully -bunch -bunny -bunt -bush -bust -busy -buzz -cable -cache -cadet -cage -cake -calm -cameo -canal -candy -cane -canon -cape -card -cargo -carol -carry -carve -case -cash -cause -cedar -chain -chair -chant -chaos -charm -chase -cheek -cheer -chef -chess -chest -chew -chief -chili -chill -chip -chomp -chop -chow -chuck -chump -chunk -churn -chute -cider -cinch -city -civic -civil -clad -claim -clamp -clap -clash -clasp -class -claw -clay -clean -clear -cleat -cleft -clerk -click -cling -clink -clip -cloak -clock -clone -cloth -cloud -clump -coach -coast -coat -cod -coil -coke -cola -cold -colt -coma -come -comic -comma -cone -cope -copy -coral -cork -cost -cot -couch -cough -cover -cozy -craft -cramp -crane -crank -crate -crave -crawl -crazy -creme -crepe -crept -crib -cried -crisp -crook -crop -cross -crowd -crown -crumb -crush -crust -cub -cult -cupid -cure -curl -curry -curse -curve -curvy -cushy -cut -cycle -dab -dad -daily -dairy -daisy -dance -dandy -darn -dart -dash -data -date -dawn -deaf -deal -dean -debit -debt -debug -decaf -decal -decay -deck -decor -decoy -deed -delay -denim -dense -dent -depth -derby -desk -dial -diary -dice -dig -dill -dime -dimly -diner -dingy -disco -dish -disk -ditch -ditzy -dizzy -dock -dodge -doing -doll -dome -donor -donut -dose -dot -dove -down -dowry -doze -drab -drama -drank -draw -dress -dried -drift -drill -drive -drone -droop -drove -drown -drum -dry -duck -duct -dude -dug -duke -duo -dusk -dust -duty -dwarf -dwell -eagle -early -earth -easel -east -eaten -eats -ebay -ebony -ebook -echo -edge -eel -eject -elbow -elder -elf -elk -elm -elope -elude -elves -email -emit -empty -emu -enter -entry -envoy -equal -erase -error -erupt -essay -etch -evade -even -evict -evil -evoke -exact -exit -fable -faced -fact -fade -fall -false -fancy -fang -fax -feast -feed -femur -fence -fend -ferry -fetal -fetch -fever -fiber -fifth -fifty -film -filth -final -finch -fit -five -flag -flaky -flame -flap -flask -fled -flick -fling -flint -flip -flirt -float -flock -flop -floss -flyer -foam -foe -fog -foil -folic -folk -food -fool -found -fox -foyer -frail -frame -fray -fresh -fried -frill -frisk -from -front -frost -froth -frown -froze -fruit -gag -gains -gala -game -gap -gas -gave -gear -gecko -geek -gem -genre -gift -gig -gills -given -giver -glad -glass -glide -gloss -glove -glow -glue -goal -going -golf -gong -good -gooey -goofy -gore -gown -grab -grain -grant -grape -graph -grasp -grass -grave -gravy -gray -green -greet -grew -grid -grief -grill -grip -grit -groom -grope -growl -grub -grunt -guide -gulf -gulp -gummy -guru -gush -gut -guy -habit -half -halo -halt -happy -harm -hash -hasty -hatch -hate -haven -hazel -hazy -heap -heat -heave -hedge -hefty -help -herbs -hers -hub -hug -hula -hull -human -humid -hump -hung -hunk -hunt -hurry -hurt -hush -hut -ice -icing -icon -icy -igloo -image -ion -iron -islam -issue -item -ivory -ivy -jab -jam -jaws -jazz -jeep -jelly -jet -jiffy -job -jog -jolly -jolt -jot -joy -judge -juice -juicy -july -jumbo -jump -junky -juror -jury -keep -keg -kept -kick -kilt -king -kite -kitty -kiwi -knee -knelt -koala -kung -ladle -lady -lair -lake -lance -land -lapel -large -lash -lasso -last -latch -late -lazy -left -legal -lemon -lend -lens -lent -level -lever -lid -life -lift -lilac -lily -limb -limes -line -lint -lion -lip -list -lived -liver -lunar -lunch -lung -lurch -lure -lurk -lying -lyric -mace -maker -malt -mama -mango -manor -many -map -march -mardi -marry -mash -match -mate -math -moan -mocha -moist -mold -mom -moody -mop -morse -most -motor -motto -mount -mouse -mousy -mouth -move -movie -mower -mud -mug -mulch -mule -mull -mumbo -mummy -mural -muse -music -musky -mute -nacho -nag -nail -name -nanny -nap -navy -near -neat -neon -nerd -nest -net -next -niece -ninth -nutty -oak -oasis -oat -ocean -oil -old -olive -omen -onion -only -ooze -opal -open -opera -opt -otter -ouch -ounce -outer -oval -oven -owl -ozone -pace -pagan -pager -palm -panda -panic -pants -panty -paper -park -party -pasta -patch -path -patio -payer -pecan -penny -pep -perch -perky -perm -pest -petal -petri -petty -photo -plank -plant -plaza -plead -plot -plow -pluck -plug -plus -poach -pod -poem -poet -pogo -point -poise -poker -polar -polio -polka -polo -pond -pony -poppy -pork -poser -pouch -pound -pout -power -prank -press -print -prior -prism -prize -probe -prong -proof -props -prude -prune -pry -pug -pull -pulp -pulse -puma -punch -punk -pupil -puppy -purr -purse -push -putt -quack -quake -query -quiet -quill -quilt -quit -quota -quote -rabid -race -rack -radar -radio -raft -rage -raid -rail -rake -rally -ramp -ranch -range -rank -rant -rash -raven -reach -react -ream -rebel -recap -relax -relay -relic -remix -repay -repel -reply -rerun -reset -rhyme -rice -rich -ride -rigid -rigor -rinse -riot -ripen -rise -risk -ritzy -rival -river -roast -robe -robin -rock -rogue -roman -romp -rope -rover -royal -ruby -rug -ruin -rule -runny -rush -rust -rut -sadly -sage -said -saint -salad -salon -salsa -salt -same -sandy -santa -satin -sauna -saved -savor -sax -say -scale -scam -scan -scare -scarf -scary -scoff -scold -scoop -scoot -scope -score -scorn -scout -scowl -scrap -scrub -scuba -scuff -sect -sedan -self -send -sepia -serve -set -seven -shack -shade -shady -shaft -shaky -sham -shape -share -sharp -shed -sheep -sheet -shelf -shell -shine -shiny -ship -shirt -shock -shop -shore -shout -shove -shown -showy -shred -shrug -shun -shush -shut -shy -sift -silk -silly -silo -sip -siren -sixth -size -skate -skew -skid -skier -skies -skip -skirt -skit -sky -slab -slack -slain -slam -slang -slash -slate -slaw -sled -sleek -sleep -sleet -slept -slice -slick -slimy -sling -slip -slit -slob -slot -slug -slum -slurp -slush -small -smash -smell -smile -smirk -smog -snack -snap -snare -snarl -sneak -sneer -sniff -snore -snort -snout -snowy -snub -snuff -speak -speed -spend -spent -spew -spied -spill -spiny -spoil -spoke -spoof -spool -spoon -sport -spot -spout -spray -spree -spur -squad -squat -squid -stack -staff -stage -stain -stall -stamp -stand -stank -stark -start -stash -state -stays -steam -steep -stem -step -stew -stick -sting -stir -stock -stole -stomp -stony -stood -stool -stoop -stop -storm -stout -stove -straw -stray -strut -stuck -stud -stuff -stump -stung -stunt -suds -sugar -sulk -surf -sushi -swab -swan -swarm -sway -swear -sweat -sweep -swell -swept -swim -swing -swipe -swirl -swoop -swore -syrup -tacky -taco -tag -take -tall -talon -tamer -tank -taper -taps -tarot -tart -task -taste -tasty -taunt -thank -thaw -theft -theme -thigh -thing -think -thong -thorn -those -throb -thud -thumb -thump -thus -tiara -tidal -tidy -tiger -tile -tilt -tint -tiny -trace -track -trade -train -trait -trap -trash -tray -treat -tree -trek -trend -trial -tribe -trick -trio -trout -truce -truck -trump -trunk -try -tug -tulip -tummy -turf -tusk -tutor -tutu -tux -tweak -tweet -twice -twine -twins -twirl -twist -uncle -uncut -undo -unify -union -unit -untie -upon -upper -urban -used -user -usher -utter -value -vapor -vegan -venue -verse -vest -veto -vice -video -view -viral -virus -visa -visor -vixen -vocal -voice -void -volt -voter -vowel -wad -wafer -wager -wages -wagon -wake -walk -wand -wasp -watch -water -wavy -wheat -whiff -whole -whoop -wick -widen -widow -width -wife -wifi -wilt -wimp -wind -wing -wink -wipe -wired -wiry -wise -wish -wispy -wok -wolf -womb -wool -woozy -word -work -worry -wound -woven -wrath -wreck -wrist -xerox -yahoo -yam -yard -year -yeast -yelp -yield -yo-yo -yodel -yoga -yoyo -yummy -zebra -zero -zesty -zippy -zone -zoom diff --git a/src/passlib/_setup/__init__.py b/src/passlib/_setup/__init__.py deleted file mode 100644 index 38819437..00000000 --- a/src/passlib/_setup/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""passlib.setup - helpers used by passlib's setup.py script""" diff --git a/src/passlib/_setup/docdist.py b/src/passlib/_setup/docdist.py deleted file mode 100644 index 19c4dc12..00000000 --- a/src/passlib/_setup/docdist.py +++ /dev/null @@ -1,87 +0,0 @@ -"""custom command to build doc.zip file""" -#============================================================================= -# imports -#============================================================================= -# core -import os -from distutils import dir_util -from distutils.cmd import Command -from distutils.errors import * -from distutils.spawn import spawn -# local -__all__ = [ - "docdist" -] -#============================================================================= -# command -#============================================================================= -class docdist(Command): - - description = "create zip file containing standalone html docs" - - user_options = [ - ('build-dir=', None, 'Build directory'), - ('dist-dir=', 'd', - "directory to put the source distribution archive(s) in " - "[default: dist]"), - ('format=', 'f', - "archive format to create (tar, ztar, gztar, zip)"), - ('sign', 's', 'sign files using gpg'), - ('identity=', 'i', 'GPG identity used to sign files'), - ] - - def initialize_options(self): - self.build_dir = None - self.dist_dir = None - self.format = None - self.keep_temp = False - self.sign = False - self.identity = None - - def finalize_options(self): - if self.identity and not self.sign: - raise DistutilsOptionError( - "Must use --sign for --identity to have meaning" - ) - if self.build_dir is None: - cmd = self.get_finalized_command('build') - self.build_dir = os.path.join(cmd.build_base, 'docdist') - if not self.dist_dir: - self.dist_dir = "dist" - if not self.format: - self.format = "zip" - - def run(self): - # call build sphinx to build docs - self.run_command("build_sphinx") - cmd = self.get_finalized_command("build_sphinx") - source_dir = cmd.builder_target_dir - - # copy to directory with appropriate name - dist = self.distribution - arc_name = "%s-docs-%s" % (dist.get_name(), dist.get_version()) - tmp_dir = os.path.join(self.build_dir, arc_name) - if os.path.exists(tmp_dir): - dir_util.remove_tree(tmp_dir, dry_run=self.dry_run) - self.copy_tree(source_dir, tmp_dir, preserve_symlinks=True) - - # make archive from dir - arc_base = os.path.join(self.dist_dir, arc_name) - self.arc_filename = self.make_archive(arc_base, self.format, - self.build_dir) - - # Sign if requested - if self.sign: - gpg_args = ["gpg", "--detach-sign", "-a", self.arc_filename] - if self.identity: - gpg_args[2:2] = ["--local-user", self.identity] - spawn(gpg_args, - dry_run=self.dry_run) - - # cleanup - if not self.keep_temp: - dir_util.remove_tree(tmp_dir, dry_run=self.dry_run) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/_setup/stamp.py b/src/passlib/_setup/stamp.py deleted file mode 100644 index 8a68658d..00000000 --- a/src/passlib/_setup/stamp.py +++ /dev/null @@ -1,57 +0,0 @@ -"""update version string during build""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import os -import re -import time -from distutils.dist import Distribution -# pkg -# local -__all__ = [ - "stamp_source", - "stamp_distutils_output", -] -#============================================================================= -# helpers -#============================================================================= -def get_command_class(opts, name): - return opts['cmdclass'].get(name) or Distribution().get_command_class(name) - -def stamp_source(base_dir, version, dry_run=False): - """update version string in passlib dist""" - path = os.path.join(base_dir, "passlib", "__init__.py") - with open(path) as fh: - input = fh.read() - output, count = re.subn('(?m)^__version__\s*=.*$', - '__version__ = ' + repr(version), - input) - assert count == 1, "failed to replace version string" - if not dry_run: - os.unlink(path) # sdist likes to use hardlinks - with open(path, "w") as fh: - fh.write(output) - -def stamp_distutils_output(opts, version): - - # subclass buildpy to update version string in source - _build_py = get_command_class(opts, "build_py") - class build_py(_build_py): - def build_packages(self): - _build_py.build_packages(self) - stamp_source(self.build_lib, version, self.dry_run) - opts['cmdclass']['build_py'] = build_py - - # subclass sdist to do same thing - _sdist = get_command_class(opts, "sdist") - class sdist(_sdist): - def make_release_tree(self, base_dir, files): - _sdist.make_release_tree(self, base_dir, files) - stamp_source(base_dir, version, self.dry_run) - opts['cmdclass']['sdist'] = sdist - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/apache.py b/src/passlib/apache.py deleted file mode 100644 index 04a64d00..00000000 --- a/src/passlib/apache.py +++ /dev/null @@ -1,1246 +0,0 @@ -"""passlib.apache - apache password support""" -# XXX: relocate this to passlib.ext.apache? -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -import os -from warnings import warn -# site -# pkg -from passlib import exc, registry -from passlib.context import CryptContext -from passlib.exc import ExpectedStringError -from passlib.hash import htdigest -from passlib.utils import render_bytes, to_bytes, is_ascii_codec -from passlib.utils.decor import deprecated_method -from passlib.utils.compat import join_bytes, unicode, BytesIO, PY3 -# local -__all__ = [ - 'HtpasswdFile', - 'HtdigestFile', -] - -#============================================================================= -# constants & support -#============================================================================= -_UNSET = object() - -_BCOLON = b":" -_BHASH = b"#" - -# byte values that aren't allowed in fields. -_INVALID_FIELD_CHARS = b":\n\r\t\x00" - -#: _CommonFile._source token types -_SKIPPED = "skipped" -_RECORD = "record" - -#============================================================================= -# common helpers -#============================================================================= -class _CommonFile(object): - """common framework for HtpasswdFile & HtdigestFile""" - #=================================================================== - # instance attrs - #=================================================================== - - # charset encoding used by file (defaults to utf-8) - encoding = None - - # whether users() and other public methods should return unicode or bytes? - # (defaults to False under PY2, True under PY3) - return_unicode = None - - # if bound to local file, these will be set. - _path = None # local file path - _mtime = None # mtime when last loaded, or 0 - - # if true, automatically save to local file after changes are made. - autosave = False - - # dict mapping key -> value for all records in database. - # (e.g. user => hash for Htpasswd) - _records = None - - #: list of tokens for recreating original file contents when saving. if present, - #: will be sequence of (_SKIPPED, b"whitespace/comments") and (_RECORD, ) tuples. - _source = None - - #=================================================================== - # alt constuctors - #=================================================================== - @classmethod - def from_string(cls, data, **kwds): - """create new object from raw string. - - :type data: unicode or bytes - :arg data: - database to load, as single string. - - :param \*\*kwds: - all other keywords are the same as in the class constructor - """ - if 'path' in kwds: - raise TypeError("'path' not accepted by from_string()") - self = cls(**kwds) - self.load_string(data) - return self - - @classmethod - def from_path(cls, path, **kwds): - """create new object from file, without binding object to file. - - :type path: str - :arg path: - local filepath to load from - - :param \*\*kwds: - all other keywords are the same as in the class constructor - """ - self = cls(**kwds) - self.load(path) - return self - - #=================================================================== - # init - #=================================================================== - def __init__(self, path=None, new=False, autoload=True, autosave=False, - encoding="utf-8", return_unicode=PY3, - ): - # set encoding - if not encoding: - warn("``encoding=None`` is deprecated as of Passlib 1.6, " - "and will cause a ValueError in Passlib 1.8, " - "use ``return_unicode=False`` instead.", - DeprecationWarning, stacklevel=2) - encoding = "utf-8" - return_unicode = False - elif not is_ascii_codec(encoding): - # htpasswd/htdigest files assumes 1-byte chars, and use ":" separator, - # so only ascii-compatible encodings are allowed. - raise ValueError("encoding must be 7-bit ascii compatible") - self.encoding = encoding - - # set other attrs - self.return_unicode = return_unicode - self.autosave = autosave - self._path = path - self._mtime = 0 - - # init db - if not autoload: - warn("``autoload=False`` is deprecated as of Passlib 1.6, " - "and will be removed in Passlib 1.8, use ``new=True`` instead", - DeprecationWarning, stacklevel=2) - new = True - if path and not new: - self.load() - else: - self._records = {} - self._source = [] - - def __repr__(self): - tail = '' - if self.autosave: - tail += ' autosave=True' - if self._path: - tail += ' path=%r' % self._path - if self.encoding != "utf-8": - tail += ' encoding=%r' % self.encoding - return "<%s 0x%0x%s>" % (self.__class__.__name__, id(self), tail) - - # NOTE: ``path`` is a property so that ``_mtime`` is wiped when it's set. - - @property - def path(self): - return self._path - - @path.setter - def path(self, value): - if value != self._path: - self._mtime = 0 - self._path = value - - @property - def mtime(self): - """modify time when last loaded (if bound to a local file)""" - return self._mtime - - #=================================================================== - # loading - #=================================================================== - def load_if_changed(self): - """Reload from ``self.path`` only if file has changed since last load""" - if not self._path: - raise RuntimeError("%r is not bound to a local file" % self) - if self._mtime and self._mtime == os.path.getmtime(self._path): - return False - self.load() - return True - - def load(self, path=None, force=True): - """Load state from local file. - If no path is specified, attempts to load from ``self.path``. - - :type path: str - :arg path: local file to load from - - :type force: bool - :param force: - if ``force=False``, only load from ``self.path`` if file - has changed since last load. - - .. deprecated:: 1.6 - This keyword will be removed in Passlib 1.8; - Applications should use :meth:`load_if_changed` instead. - """ - if path is not None: - with open(path, "rb") as fh: - self._mtime = 0 - self._load_lines(fh) - elif not force: - warn("%(name)s.load(force=False) is deprecated as of Passlib 1.6," - "and will be removed in Passlib 1.8; " - "use %(name)s.load_if_changed() instead." % - dict(name=self.__class__.__name__), - DeprecationWarning, stacklevel=2) - return self.load_if_changed() - elif self._path: - with open(self._path, "rb") as fh: - self._mtime = os.path.getmtime(self._path) - self._load_lines(fh) - else: - raise RuntimeError("%s().path is not set, an explicit path is required" % - self.__class__.__name__) - return True - - def load_string(self, data): - """Load state from unicode or bytes string, replacing current state""" - data = to_bytes(data, self.encoding, "data") - self._mtime = 0 - self._load_lines(BytesIO(data)) - - def _load_lines(self, lines): - """load from sequence of lists""" - parse = self._parse_record - records = {} - source = [] - skipped = b'' - for idx, line in enumerate(lines): - # NOTE: per htpasswd source (https://github.com/apache/httpd/blob/trunk/support/htpasswd.c), - # lines with only whitespace, or with "#" as first non-whitespace char, - # are left alone / ignored. - tmp = line.lstrip() - if not tmp or tmp.startswith(_BHASH): - skipped += line - continue - - # parse valid line - key, value = parse(line, idx+1) - - # NOTE: if multiple entries for a key, we use the first one, - # which seems to match htpasswd source - if key in records: - log.warning("username occurs multiple times in source file: %r" % key) - skipped += line - continue - - # flush buffer of skipped whitespace lines - if skipped: - source.append((_SKIPPED, skipped)) - skipped = b'' - - # store new user line - records[key] = value - source.append((_RECORD, key)) - - # don't bother preserving trailing whitespace, but do preserve trailing comments - if skipped.rstrip(): - source.append((_SKIPPED, skipped)) - - # NOTE: not replacing ._records until parsing succeeds, so loading is atomic. - self._records = records - self._source = source - - def _parse_record(self, record, lineno): # pragma: no cover - abstract method - """parse line of file into (key, value) pair""" - raise NotImplementedError("should be implemented in subclass") - - def _set_record(self, key, value): - """ - helper for setting record which takes care of inserting source line if needed; - - :returns: - bool if key already present - """ - records = self._records - existing = (key in records) - records[key] = value - if not existing: - self._source.append((_RECORD, key)) - return existing - - #=================================================================== - # saving - #=================================================================== - def _autosave(self): - """subclass helper to call save() after any changes""" - if self.autosave and self._path: - self.save() - - def save(self, path=None): - """Save current state to file. - If no path is specified, attempts to save to ``self.path``. - """ - if path is not None: - with open(path, "wb") as fh: - fh.writelines(self._iter_lines()) - elif self._path: - self.save(self._path) - self._mtime = os.path.getmtime(self._path) - else: - raise RuntimeError("%s().path is not set, cannot autosave" % - self.__class__.__name__) - - def to_string(self): - """Export current state as a string of bytes""" - return join_bytes(self._iter_lines()) - - # def clean(self): - # """ - # discard any comments or whitespace that were being preserved from the source file, - # and re-sort keys in alphabetical order - # """ - # self._source = [(_RECORD, key) for key in sorted(self._records)] - # self._autosave() - - def _iter_lines(self): - """iterator yielding lines of database""" - # NOTE: this relies on being an OrderedDict so that it outputs - # records in a deterministic order. - records = self._records - if __debug__: - pending = set(records) - for action, content in self._source: - if action == _SKIPPED: - # 'content' is whitespace/comments to write - yield content - else: - assert action == _RECORD - # 'content' is record key - if content not in records: - # record was deleted - # NOTE: doing it lazily like this so deleting & re-adding user - # preserves their original location in the file. - continue - yield self._render_record(content, records[content]) - if __debug__: - pending.remove(content) - if __debug__: - # sanity check that we actually wrote all the records - # (otherwise _source & _records are somehow out of sync) - assert not pending, "failed to write all records: missing=%r" % (pending,) - - def _render_record(self, key, value): # pragma: no cover - abstract method - """given key/value pair, encode as line of file""" - raise NotImplementedError("should be implemented in subclass") - - #=================================================================== - # field encoding - #=================================================================== - def _encode_user(self, user): - """user-specific wrapper for _encode_field()""" - return self._encode_field(user, "user") - - def _encode_realm(self, realm): # pragma: no cover - abstract method - """realm-specific wrapper for _encode_field()""" - return self._encode_field(realm, "realm") - - def _encode_field(self, value, param="field"): - """convert field to internal representation. - - internal representation is always bytes. byte strings are left as-is, - unicode strings encoding using file's default encoding (or ``utf-8`` - if no encoding has been specified). - - :raises UnicodeEncodeError: - if unicode value cannot be encoded using default encoding. - - :raises ValueError: - if resulting byte string contains a forbidden character, - or is too long (>255 bytes). - - :returns: - encoded identifer as bytes - """ - if isinstance(value, unicode): - value = value.encode(self.encoding) - elif not isinstance(value, bytes): - raise ExpectedStringError(value, param) - if len(value) > 255: - raise ValueError("%s must be at most 255 characters: %r" % - (param, value)) - if any(c in _INVALID_FIELD_CHARS for c in value): - raise ValueError("%s contains invalid characters: %r" % - (param, value,)) - return value - - def _decode_field(self, value): - """decode field from internal representation to format - returns by users() method, etc. - - :raises UnicodeDecodeError: - if unicode value cannot be decoded using default encoding. - (usually indicates wrong encoding set for file). - - :returns: - field as unicode or bytes, as appropriate. - """ - assert isinstance(value, bytes), "expected value to be bytes" - if self.return_unicode: - return value.decode(self.encoding) - else: - return value - - # FIXME: htpasswd doc says passwords limited to 255 chars under Windows & MPE, - # and that longer ones are truncated. this may be side-effect of those - # platforms supporting the 'plaintext' scheme. these classes don't currently - # check for this. - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# htpasswd context -# -# This section sets up a CryptContexts to mimic what schemes Apache -# (and the htpasswd tool) should support on the current system. -# -# Apache has long-time supported some basic builtin schemes (listed below), -# as well as the host's crypt() method -- though it's limited to being able -# to *verify* any scheme using that method, but can only generate "des_crypt" hashes. -# -# Apache 2.4 added builtin bcrypt support (even for platforms w/o native support). -# c.f. http://httpd.apache.org/docs/2.4/programs/htpasswd.html vs the 2.2 docs. -#============================================================================= - -#: set of default schemes that (if chosen) should be using bcrypt, -#: but can't due to lack of bcrypt. -_warn_no_bcrypt = set() - -def _init_default_schemes(): - - #: pick strongest one for host - host_best = None - for name in ["bcrypt", "sha256_crypt"]: - if registry.has_os_crypt_support(name): - host_best = name - break - - # check if we have a bcrypt backend -- otherwise issue warning - # XXX: would like to not spam this unless the user *requests* apache 24 - bcrypt = "bcrypt" if registry.has_backend("bcrypt") else None - _warn_no_bcrypt.clear() - if not bcrypt: - _warn_no_bcrypt.update(["portable_apache_24", "host_apache_24", - "linux_apache_24", "portable", "host"]) - - defaults = dict( - # strongest hash builtin to specific apache version - portable_apache_24=bcrypt or "apr_md5_crypt", - portable_apache_22="apr_md5_crypt", - - # strongest hash across current host & specific apache version - host_apache_24=bcrypt or host_best or "apr_md5_crypt", - host_apache_22=host_best or "apr_md5_crypt", - - # strongest hash on a linux host - linux_apache_24=bcrypt or "sha256_crypt", - linux_apache_22="sha256_crypt", - ) - - # set latest-apache version aliases - # XXX: could check for apache install, and pick correct host 22/24 default? - defaults.update( - portable=defaults['portable_apache_24'], - host=defaults['host_apache_24'], - ) - return defaults - -#: dict mapping default alias -> appropriate scheme -htpasswd_defaults = _init_default_schemes() - -def _init_htpasswd_context(): - - # start with schemes built into apache - schemes = [ - # builtin support added in apache 2.4 - # (https://bz.apache.org/bugzilla/show_bug.cgi?id=49288) - "bcrypt", - - # support not "builtin" to apache, instead it requires support through host's crypt(). - # adding them here to allow editing htpasswd under windows and then deploying under unix. - "sha256_crypt", - "sha512_crypt", - "des_crypt", - - # apache default as of 2.2.18, and still default in 2.4 - "apr_md5_crypt", - - # NOTE: apache says ONLY intended for transitioning htpasswd <-> ldap - "ldap_sha1", - - # NOTE: apache says ONLY supported on Windows, Netware, TPF - "plaintext" - ] - - # apache can verify anything supported by the native crypt(), - # though htpasswd tool can only generate a limited set of hashes. - # (this list may overlap w/ builtin apache schemes) - schemes.extend(registry.get_supported_os_crypt_schemes()) - - # hack to remove dups and sort into preferred order - preferred = schemes[:3] + ["apr_md5_crypt"] + schemes - schemes = sorted(set(schemes), key=preferred.index) - - # NOTE: default will change to "portable" in passlib 2.0 - return CryptContext(schemes, default=htpasswd_defaults['portable_apache_22']) - -#: CryptContext configured to match htpasswd -htpasswd_context = _init_htpasswd_context() - -#============================================================================= -# htpasswd editing -#============================================================================= - -class HtpasswdFile(_CommonFile): - """class for reading & writing Htpasswd files. - - The class constructor accepts the following arguments: - - :type path: filepath - :param path: - - Specifies path to htpasswd file, use to implicitly load from and save to. - - This class has two modes of operation: - - 1. It can be "bound" to a local file by passing a ``path`` to the class - constructor. In this case it will load the contents of the file when - created, and the :meth:`load` and :meth:`save` methods will automatically - load from and save to that file if they are called without arguments. - - 2. Alternately, it can exist as an independant object, in which case - :meth:`load` and :meth:`save` will require an explicit path to be - provided whenever they are called. As well, ``autosave`` behavior - will not be available. - - This feature is new in Passlib 1.6, and is the default if no - ``path`` value is provided to the constructor. - - This is also exposed as a readonly instance attribute. - - :type new: bool - :param new: - - Normally, if *path* is specified, :class:`HtpasswdFile` will - immediately load the contents of the file. However, when creating - a new htpasswd file, applications can set ``new=True`` so that - the existing file (if any) will not be loaded. - - .. versionadded:: 1.6 - This feature was previously enabled by setting ``autoload=False``. - That alias has been deprecated, and will be removed in Passlib 1.8 - - :type autosave: bool - :param autosave: - - Normally, any changes made to an :class:`HtpasswdFile` instance - will not be saved until :meth:`save` is explicitly called. However, - if ``autosave=True`` is specified, any changes made will be - saved to disk immediately (assuming *path* has been set). - - This is also exposed as a writeable instance attribute. - - :type encoding: str - :param encoding: - - Optionally specify character encoding used to read/write file - and hash passwords. Defaults to ``utf-8``, though ``latin-1`` - is the only other commonly encountered encoding. - - This is also exposed as a readonly instance attribute. - - :type default_scheme: str - :param default_scheme: - Optionally specify default scheme to use when encoding new passwords. - - This can be any of the schemes with builtin Apache support, - OR natively supported by the host OS's :func:`crypt.crypt` function. - - * Builtin schemes include ``"bcrypt"`` (apache 2.4+), ``"apr_md5_crypt"`, - and ``"des_crypt"``. - - * Schemes commonly supported by Unix hosts - include ``"bcrypt"``, ``"sha256_crypt"``, and ``"des_crypt"``. - - In order to not have to sort out what you should use, - passlib offers a number of aliases, that will resolve - to the most appropriate scheme based on your needs: - - * ``"portable"``, ``"portable_apache_24"`` -- pick scheme that's portable across hosts - running apache >= 2.4. **This will be the default as of Passlib 2.0**. - - * ``"portable_apache_22"`` -- pick scheme that's portable across hosts - running apache >= 2.4. **This is the default up to Passlib 1.9**. - - * ``"host"``, ``"host_apache_24"`` -- pick strongest scheme supported by - apache >= 2.4 and/or host OS. - - * ``"host_apache_22"`` -- pick strongest scheme supported by - apache >= 2.2 and/or host OS. - - .. versionadded:: 1.6 - This keyword was previously named ``default``. That alias - has been deprecated, and will be removed in Passlib 1.8. - - .. versionchanged:: 1.6.3 - - Added support for ``"bcrypt"``, ``"sha256_crypt"``, and ``"portable"`` alias. - - .. versionchanged:: 1.7 - - Added apache 2.4 semantics, and additional aliases. - - :type context: :class:`~passlib.context.CryptContext` - :param context: - :class:`!CryptContext` instance used to create - and verify the hashes found in the htpasswd file. - The default value is a pre-built context which supports all - of the hashes officially allowed in an htpasswd file. - - This is also exposed as a readonly instance attribute. - - .. warning:: - - This option may be used to add support for non-standard hash - formats to an htpasswd file. However, the resulting file - will probably not be usable by another application, - and particularly not by Apache. - - :param autoload: - Set to ``False`` to prevent the constructor from automatically - loaded the file from disk. - - .. deprecated:: 1.6 - This has been replaced by the *new* keyword. - Instead of setting ``autoload=False``, you should use - ``new=True``. Support for this keyword will be removed - in Passlib 1.8. - - :param default: - Change the default algorithm used to hash new passwords. - - .. deprecated:: 1.6 - This has been renamed to *default_scheme* for clarity. - Support for this alias will be removed in Passlib 1.8. - - Loading & Saving - ================ - .. automethod:: load - .. automethod:: load_if_changed - .. automethod:: load_string - .. automethod:: save - .. automethod:: to_string - - Inspection - ================ - .. automethod:: users - .. automethod:: check_password - .. automethod:: get_hash - - Modification - ================ - .. automethod:: set_password - .. automethod:: delete - - Alternate Constructors - ====================== - .. automethod:: from_string - - Attributes - ========== - .. attribute:: path - - Path to local file that will be used as the default - for all :meth:`load` and :meth:`save` operations. - May be written to, initialized by the *path* constructor keyword. - - .. attribute:: autosave - - Writeable flag indicating whether changes will be automatically - written to *path*. - - Errors - ====== - :raises ValueError: - All of the methods in this class will raise a :exc:`ValueError` if - any user name contains a forbidden character (one of ``:\\r\\n\\t\\x00``), - or is longer than 255 characters. - """ - #=================================================================== - # instance attrs - #=================================================================== - - # NOTE: _records map stores for the key, and for the value, - # both in bytes which use self.encoding - - #=================================================================== - # init & serialization - #=================================================================== - def __init__(self, path=None, default_scheme=None, context=htpasswd_context, - **kwds): - if 'default' in kwds: - warn("``default`` is deprecated as of Passlib 1.6, " - "and will be removed in Passlib 1.8, it has been renamed " - "to ``default_scheem``.", - DeprecationWarning, stacklevel=2) - default_scheme = kwds.pop("default") - if default_scheme: - if default_scheme in _warn_no_bcrypt: - warn("HtpasswdFile: no bcrypt backends available, " - "using fallback for default scheme %r" % default_scheme, - exc.PasslibSecurityWarning) - default_scheme = htpasswd_defaults.get(default_scheme, default_scheme) - context = context.copy(default=default_scheme) - self.context = context - super(HtpasswdFile, self).__init__(path, **kwds) - - def _parse_record(self, record, lineno): - # NOTE: should return (user, hash) tuple - result = record.rstrip().split(_BCOLON) - if len(result) != 2: - raise ValueError("malformed htpasswd file (error reading line %d)" - % lineno) - return result - - def _render_record(self, user, hash): - return render_bytes("%s:%s\n", user, hash) - - #=================================================================== - # public methods - #=================================================================== - - def users(self): - """ - Return list of all users in database - """ - return [self._decode_field(user) for user in self._records] - - ##def has_user(self, user): - ## "check whether entry is present for user" - ## return self._encode_user(user) in self._records - - ##def rename(self, old, new): - ## """rename user account""" - ## old = self._encode_user(old) - ## new = self._encode_user(new) - ## hash = self._records.pop(old) - ## self._records[new] = hash - ## self._autosave() - - def set_password(self, user, password): - """Set password for user; adds user if needed. - - :returns: - * ``True`` if existing user was updated. - * ``False`` if user account was added. - - .. versionchanged:: 1.6 - This method was previously called ``update``, it was renamed - to prevent ambiguity with the dictionary method. - The old alias is deprecated, and will be removed in Passlib 1.8. - """ - hash = self.context.hash(password) - return self.set_hash(user, hash) - - @deprecated_method(deprecated="1.6", removed="1.8", - replacement="set_password") - def update(self, user, password): - """set password for user""" - return self.set_password(user, password) - - def get_hash(self, user): - """Return hash stored for user, or ``None`` if user not found. - - .. versionchanged:: 1.6 - This method was previously named ``find``, it was renamed - for clarity. The old name is deprecated, and will be removed - in Passlib 1.8. - """ - try: - return self._records[self._encode_user(user)] - except KeyError: - return None - - def set_hash(self, user, hash): - """ - semi-private helper which allows writing a hash directly; - adds user if needed. - - .. warning:: - does not (currently) do any validation of the hash string - - .. versionadded:: 1.7 - """ - # assert self.context.identify(hash), "unrecognized hash format" - if PY3 and isinstance(hash, str): - hash = hash.encode(self.encoding) - user = self._encode_user(user) - existing = self._set_record(user, hash) - self._autosave() - return existing - - @deprecated_method(deprecated="1.6", removed="1.8", - replacement="get_hash") - def find(self, user): - """return hash for user""" - return self.get_hash(user) - - # XXX: rename to something more explicit, like delete_user()? - def delete(self, user): - """Delete user's entry. - - :returns: - * ``True`` if user deleted. - * ``False`` if user not found. - """ - try: - del self._records[self._encode_user(user)] - except KeyError: - return False - self._autosave() - return True - - def check_password(self, user, password): - """ - Verify password for specified user. - If algorithm marked as deprecated by CryptContext, will automatically be re-hashed. - - :returns: - * ``None`` if user not found. - * ``False`` if user found, but password does not match. - * ``True`` if user found and password matches. - - .. versionchanged:: 1.6 - This method was previously called ``verify``, it was renamed - to prevent ambiguity with the :class:`!CryptContext` method. - The old alias is deprecated, and will be removed in Passlib 1.8. - """ - user = self._encode_user(user) - hash = self._records.get(user) - if hash is None: - return None - if isinstance(password, unicode): - # NOTE: encoding password to match file, making the assumption - # that server will use same encoding to hash the password. - password = password.encode(self.encoding) - ok, new_hash = self.context.verify_and_update(password, hash) - if ok and new_hash is not None: - # rehash user's password if old hash was deprecated - assert user in self._records # otherwise would have to use ._set_record() - self._records[user] = new_hash - self._autosave() - return ok - - @deprecated_method(deprecated="1.6", removed="1.8", - replacement="check_password") - def verify(self, user, password): - """verify password for user""" - return self.check_password(user, password) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# htdigest editing -#============================================================================= -class HtdigestFile(_CommonFile): - """class for reading & writing Htdigest files. - - The class constructor accepts the following arguments: - - :type path: filepath - :param path: - - Specifies path to htdigest file, use to implicitly load from and save to. - - This class has two modes of operation: - - 1. It can be "bound" to a local file by passing a ``path`` to the class - constructor. In this case it will load the contents of the file when - created, and the :meth:`load` and :meth:`save` methods will automatically - load from and save to that file if they are called without arguments. - - 2. Alternately, it can exist as an independant object, in which case - :meth:`load` and :meth:`save` will require an explicit path to be - provided whenever they are called. As well, ``autosave`` behavior - will not be available. - - This feature is new in Passlib 1.6, and is the default if no - ``path`` value is provided to the constructor. - - This is also exposed as a readonly instance attribute. - - :type default_realm: str - :param default_realm: - - If ``default_realm`` is set, all the :class:`HtdigestFile` - methods that require a realm will use this value if one is not - provided explicitly. If unset, they will raise an error stating - that an explicit realm is required. - - This is also exposed as a writeable instance attribute. - - .. versionadded:: 1.6 - - :type new: bool - :param new: - - Normally, if *path* is specified, :class:`HtdigestFile` will - immediately load the contents of the file. However, when creating - a new htpasswd file, applications can set ``new=True`` so that - the existing file (if any) will not be loaded. - - .. versionadded:: 1.6 - This feature was previously enabled by setting ``autoload=False``. - That alias has been deprecated, and will be removed in Passlib 1.8 - - :type autosave: bool - :param autosave: - - Normally, any changes made to an :class:`HtdigestFile` instance - will not be saved until :meth:`save` is explicitly called. However, - if ``autosave=True`` is specified, any changes made will be - saved to disk immediately (assuming *path* has been set). - - This is also exposed as a writeable instance attribute. - - :type encoding: str - :param encoding: - - Optionally specify character encoding used to read/write file - and hash passwords. Defaults to ``utf-8``, though ``latin-1`` - is the only other commonly encountered encoding. - - This is also exposed as a readonly instance attribute. - - :param autoload: - Set to ``False`` to prevent the constructor from automatically - loaded the file from disk. - - .. deprecated:: 1.6 - This has been replaced by the *new* keyword. - Instead of setting ``autoload=False``, you should use - ``new=True``. Support for this keyword will be removed - in Passlib 1.8. - - Loading & Saving - ================ - .. automethod:: load - .. automethod:: load_if_changed - .. automethod:: load_string - .. automethod:: save - .. automethod:: to_string - - Inspection - ========== - .. automethod:: realms - .. automethod:: users - .. automethod:: check_password(user[, realm], password) - .. automethod:: get_hash - - Modification - ============ - .. automethod:: set_password(user[, realm], password) - .. automethod:: delete - .. automethod:: delete_realm - - Alternate Constructors - ====================== - .. automethod:: from_string - - Attributes - ========== - .. attribute:: default_realm - - The default realm that will be used if one is not provided - to methods that require it. By default this is ``None``, - in which case an explicit realm must be provided for every - method call. Can be written to. - - .. attribute:: path - - Path to local file that will be used as the default - for all :meth:`load` and :meth:`save` operations. - May be written to, initialized by the *path* constructor keyword. - - .. attribute:: autosave - - Writeable flag indicating whether changes will be automatically - written to *path*. - - Errors - ====== - :raises ValueError: - All of the methods in this class will raise a :exc:`ValueError` if - any user name or realm contains a forbidden character (one of ``:\\r\\n\\t\\x00``), - or is longer than 255 characters. - """ - #=================================================================== - # instance attrs - #=================================================================== - - # NOTE: _records map stores (,) for the key, - # and as the value, all as bytes. - - # NOTE: unlike htpasswd, this class doesn't use a CryptContext, - # as only one hash format is supported: htdigest. - - # optionally specify default realm that will be used if none - # is provided to a method call. otherwise realm is always required. - default_realm = None - - #=================================================================== - # init & serialization - #=================================================================== - def __init__(self, path=None, default_realm=None, **kwds): - self.default_realm = default_realm - super(HtdigestFile, self).__init__(path, **kwds) - - def _parse_record(self, record, lineno): - result = record.rstrip().split(_BCOLON) - if len(result) != 3: - raise ValueError("malformed htdigest file (error reading line %d)" - % lineno) - user, realm, hash = result - return (user, realm), hash - - def _render_record(self, key, hash): - user, realm = key - return render_bytes("%s:%s:%s\n", user, realm, hash) - - def _require_realm(self, realm): - if realm is None: - realm = self.default_realm - if realm is None: - raise TypeError("you must specify a realm explicitly, " - "or set the default_realm attribute") - return realm - - def _encode_realm(self, realm): - realm = self._require_realm(realm) - return self._encode_field(realm, "realm") - - def _encode_key(self, user, realm): - return self._encode_user(user), self._encode_realm(realm) - - #=================================================================== - # public methods - #=================================================================== - - def realms(self): - """Return list of all realms in database""" - realms = set(key[1] for key in self._records) - return [self._decode_field(realm) for realm in realms] - - def users(self, realm=None): - """Return list of all users in specified realm. - - * uses ``self.default_realm`` if no realm explicitly provided. - * returns empty list if realm not found. - """ - realm = self._encode_realm(realm) - return [self._decode_field(key[0]) for key in self._records - if key[1] == realm] - - ##def has_user(self, user, realm=None): - ## "check if user+realm combination exists" - ## return self._encode_key(user,realm) in self._records - - ##def rename_realm(self, old, new): - ## """rename all accounts in realm""" - ## old = self._encode_realm(old) - ## new = self._encode_realm(new) - ## keys = [key for key in self._records if key[1] == old] - ## for key in keys: - ## hash = self._records.pop(key) - ## self._set_record((key[0], new), hash) - ## self._autosave() - ## return len(keys) - - ##def rename(self, old, new, realm=None): - ## """rename user account""" - ## old = self._encode_user(old) - ## new = self._encode_user(new) - ## realm = self._encode_realm(realm) - ## hash = self._records.pop((old,realm)) - ## self._set_record((new, realm), hash) - ## self._autosave() - - def set_password(self, user, realm=None, password=_UNSET): - """Set password for user; adds user & realm if needed. - - If ``self.default_realm`` has been set, this may be called - with the syntax ``set_password(user, password)``, - otherwise it must be called with all three arguments: - ``set_password(user, realm, password)``. - - :returns: - * ``True`` if existing user was updated - * ``False`` if user account added. - """ - if password is _UNSET: - # called w/ two args - (user, password), use default realm - realm, password = None, realm - realm = self._require_realm(realm) - hash = htdigest.hash(password, user, realm, encoding=self.encoding) - return self.set_hash(user, realm, hash) - - @deprecated_method(deprecated="1.6", removed="1.8", - replacement="set_password") - def update(self, user, realm, password): - """set password for user""" - return self.set_password(user, realm, password) - - def get_hash(self, user, realm=None): - """Return :class:`~passlib.hash.htdigest` hash stored for user. - - * uses ``self.default_realm`` if no realm explicitly provided. - * returns ``None`` if user or realm not found. - - .. versionchanged:: 1.6 - This method was previously named ``find``, it was renamed - for clarity. The old name is deprecated, and will be removed - in Passlib 1.8. - """ - key = self._encode_key(user, realm) - hash = self._records.get(key) - if hash is None: - return None - if PY3: - hash = hash.decode(self.encoding) - return hash - - def set_hash(self, user, realm=None, hash=_UNSET): - """ - semi-private helper which allows writing a hash directly; - adds user & realm if needed. - - If ``self.default_realm`` has been set, this may be called - with the syntax ``set_hash(user, hash)``, - otherwise it must be called with all three arguments: - ``set_hash(user, realm, hash)``. - - .. warning:: - does not (currently) do any validation of the hash string - - .. versionadded:: 1.7 - """ - if hash is _UNSET: - # called w/ two args - (user, hash), use default realm - realm, hash = None, realm - # assert htdigest.identify(hash), "unrecognized hash format" - if PY3 and isinstance(hash, str): - hash = hash.encode(self.encoding) - key = self._encode_key(user, realm) - existing = self._set_record(key, hash) - self._autosave() - return existing - - @deprecated_method(deprecated="1.6", removed="1.8", - replacement="get_hash") - def find(self, user, realm): - """return hash for user""" - return self.get_hash(user, realm) - - # XXX: rename to something more explicit, like delete_user()? - def delete(self, user, realm=None): - """Delete user's entry for specified realm. - - if realm is not specified, uses ``self.default_realm``. - - :returns: - * ``True`` if user deleted, - * ``False`` if user not found in realm. - """ - key = self._encode_key(user, realm) - try: - del self._records[key] - except KeyError: - return False - self._autosave() - return True - - def delete_realm(self, realm): - """Delete all users for specified realm. - - if realm is not specified, uses ``self.default_realm``. - - :returns: number of users deleted (0 if realm not found) - """ - realm = self._encode_realm(realm) - records = self._records - keys = [key for key in records if key[1] == realm] - for key in keys: - del records[key] - self._autosave() - return len(keys) - - def check_password(self, user, realm=None, password=_UNSET): - """Verify password for specified user + realm. - - If ``self.default_realm`` has been set, this may be called - with the syntax ``check_password(user, password)``, - otherwise it must be called with all three arguments: - ``check_password(user, realm, password)``. - - :returns: - * ``None`` if user or realm not found. - * ``False`` if user found, but password does not match. - * ``True`` if user found and password matches. - - .. versionchanged:: 1.6 - This method was previously called ``verify``, it was renamed - to prevent ambiguity with the :class:`!CryptContext` method. - The old alias is deprecated, and will be removed in Passlib 1.8. - """ - if password is _UNSET: - # called w/ two args - (user, password), use default realm - realm, password = None, realm - user = self._encode_user(user) - realm = self._encode_realm(realm) - hash = self._records.get((user,realm)) - if hash is None: - return None - return htdigest.verify(password, hash, user, realm, - encoding=self.encoding) - - @deprecated_method(deprecated="1.6", removed="1.8", - replacement="check_password") - def verify(self, user, realm, password): - """verify password for user""" - return self.check_password(user, realm, password) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/apps.py b/src/passlib/apps.py deleted file mode 100644 index 7c4be06c..00000000 --- a/src/passlib/apps.py +++ /dev/null @@ -1,197 +0,0 @@ -"""passlib.apps""" -#============================================================================= -# imports -#============================================================================= -# core -import logging; log = logging.getLogger(__name__) -from itertools import chain -# site -# pkg -from passlib import hash -from passlib.context import LazyCryptContext -from passlib.utils import sys_bits -# local -__all__ = [ - 'custom_app_context', - 'django_context', - 'ldap_context', 'ldap_nocrypt_context', - 'mysql_context', 'mysql4_context', 'mysql3_context', - 'phpass_context', - 'phpbb3_context', - 'postgres_context', -] - -#============================================================================= -# master containing all identifiable hashes -#============================================================================= -def _load_master_config(): - from passlib.registry import list_crypt_handlers - - # get master list - schemes = list_crypt_handlers() - - # exclude the ones we know have ambiguous or greedy identify() methods. - excluded = [ - # frequently confused for eachother - 'bigcrypt', - 'crypt16', - - # no good identifiers - 'cisco_pix', - 'cisco_type7', - 'htdigest', - 'mysql323', - 'oracle10', - - # all have same size - 'lmhash', - 'msdcc', - 'msdcc2', - 'nthash', - - # plaintext handlers - 'plaintext', - 'ldap_plaintext', - - # disabled handlers - 'django_disabled', - 'unix_disabled', - 'unix_fallback', - ] - for name in excluded: - schemes.remove(name) - - # return config - return dict(schemes=schemes, default="sha256_crypt") -master_context = LazyCryptContext(onload=_load_master_config) - -#============================================================================= -# for quickly bootstrapping new custom applications -#============================================================================= -custom_app_context = LazyCryptContext( - # choose some reasonbly strong schemes - schemes=["sha512_crypt", "sha256_crypt"], - - # set some useful global options - default="sha256_crypt" if sys_bits < 64 else "sha512_crypt", - - # set a good starting point for rounds selection - sha512_crypt__min_rounds = 535000, - sha256_crypt__min_rounds = 535000, - - # if the admin user category is selected, make a much stronger hash, - admin__sha512_crypt__min_rounds = 1024000, - admin__sha256_crypt__min_rounds = 1024000, - ) - -#============================================================================= -# django -#============================================================================= -_django10_schemes = [ - "django_salted_sha1", "django_salted_md5", "django_des_crypt", - "hex_md5", "django_disabled", -] - -django10_context = LazyCryptContext( - schemes=_django10_schemes, - default="django_salted_sha1", - deprecated=["hex_md5"], -) - -_django14_schemes = ["django_pbkdf2_sha256", "django_pbkdf2_sha1", - "django_bcrypt"] + _django10_schemes -django14_context = LazyCryptContext( - schemes=_django14_schemes, - deprecated=_django10_schemes, -) - -_django16_schemes = _django14_schemes[:] -_django16_schemes.insert(1, "django_bcrypt_sha256") -django16_context = LazyCryptContext( - schemes=_django16_schemes, - deprecated=_django10_schemes, -) - -django110_context = LazyCryptContext( - schemes=["django_pbkdf2_sha256", "django_pbkdf2_sha1", - "django_argon2", "django_bcrypt", "django_bcrypt_sha256", - "django_disabled"], -) - -# this will always point to latest version -django_context = django110_context - -#============================================================================= -# ldap -#============================================================================= -std_ldap_schemes = ["ldap_salted_sha1", "ldap_salted_md5", - "ldap_sha1", "ldap_md5", - "ldap_plaintext" ] - -# create context with all std ldap schemes EXCEPT crypt -ldap_nocrypt_context = LazyCryptContext(std_ldap_schemes) - -# create context with all possible std ldap + ldap crypt schemes -def _iter_ldap_crypt_schemes(): - from passlib.utils import unix_crypt_schemes - return ('ldap_' + name for name in unix_crypt_schemes) - -def _iter_ldap_schemes(): - """helper which iterates over supported std ldap schemes""" - return chain(std_ldap_schemes, _iter_ldap_crypt_schemes()) -ldap_context = LazyCryptContext(_iter_ldap_schemes()) - -### create context with all std ldap schemes + crypt schemes for localhost -##def _iter_host_ldap_schemes(): -## "helper which iterates over supported std ldap schemes" -## from passlib.handlers.ldap_digests import get_host_ldap_crypt_schemes -## return chain(std_ldap_schemes, get_host_ldap_crypt_schemes()) -##ldap_host_context = LazyCryptContext(_iter_host_ldap_schemes()) - -#============================================================================= -# mysql -#============================================================================= -mysql3_context = LazyCryptContext(["mysql323"]) -mysql4_context = LazyCryptContext(["mysql41", "mysql323"], deprecated="mysql323") -mysql_context = mysql4_context # tracks latest mysql version supported - -#============================================================================= -# postgres -#============================================================================= -postgres_context = LazyCryptContext(["postgres_md5"]) - -#============================================================================= -# phpass & variants -#============================================================================= -def _create_phpass_policy(**kwds): - """helper to choose default alg based on bcrypt availability""" - kwds['default'] = 'bcrypt' if hash.bcrypt.has_backend() else 'phpass' - return kwds - -phpass_context = LazyCryptContext( - schemes=["bcrypt", "phpass", "bsdi_crypt"], - onload=_create_phpass_policy, - ) - -phpbb3_context = LazyCryptContext(["phpass"], phpass__ident="H") - -# TODO: support the drupal phpass variants (see phpass homepage) - -#============================================================================= -# roundup -#============================================================================= - -_std_roundup_schemes = [ "ldap_hex_sha1", "ldap_hex_md5", "ldap_des_crypt", "roundup_plaintext" ] -roundup10_context = LazyCryptContext(_std_roundup_schemes) - -# NOTE: 'roundup15' really applies to roundup 1.4.17+ -roundup_context = roundup15_context = LazyCryptContext( - schemes=_std_roundup_schemes + [ "ldap_pbkdf2_sha1" ], - deprecated=_std_roundup_schemes, - default = "ldap_pbkdf2_sha1", - ldap_pbkdf2_sha1__default_rounds = 10000, - ) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/context.py b/src/passlib/context.py deleted file mode 100644 index fa700f76..00000000 --- a/src/passlib/context.py +++ /dev/null @@ -1,2632 +0,0 @@ -"""passlib.context - CryptContext implementation""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import re -import logging; log = logging.getLogger(__name__) -import threading -import time -from warnings import warn -# site -# pkg -from passlib.exc import ExpectedStringError, ExpectedTypeError, PasslibConfigWarning -from passlib.registry import get_crypt_handler, _validate_handler_name -from passlib.utils import (handlers as uh, to_bytes, - to_unicode, splitcomma, - as_bool, timer, rng, getrandstr, - ) -from passlib.utils.binary import BASE64_CHARS -from passlib.utils.compat import (iteritems, num_types, irange, - PY2, PY3, unicode, SafeConfigParser, - NativeStringIO, BytesIO, - unicode_or_bytes_types, native_string_types, - ) -from passlib.utils.decor import deprecated_method, memoized_property -# local -__all__ = [ - 'CryptContext', - 'LazyCryptContext', - 'CryptPolicy', -] - -#============================================================================= -# support -#============================================================================= - -# private object to detect unset params -_UNSET = object() - -def _coerce_vary_rounds(value): - """parse vary_rounds string to percent as [0,1) float, or integer""" - if value.endswith("%"): - # XXX: deprecate this in favor of raw float? - return float(value.rstrip("%"))*.01 - try: - return int(value) - except ValueError: - return float(value) - -# set of options which aren't allowed to be set via policy -_forbidden_scheme_options = set(["salt"]) - # 'salt' - not allowed since a fixed salt would defeat the purpose. - -# dict containing funcs used to coerce strings to correct type for scheme option keys. -# NOTE: this isn't really needed any longer, since Handler.using() handles the actual parsing. -# keeping this around for now, though, since it makes context.to_dict() output cleaner. -_coerce_scheme_options = dict( - min_rounds=int, - max_rounds=int, - default_rounds=int, - vary_rounds=_coerce_vary_rounds, - salt_size=int, -) - -def _is_handler_registered(handler): - """detect if handler is registered or a custom handler""" - return get_crypt_handler(handler.name, None) is handler - -@staticmethod -def _always_needs_update(hash, secret=None): - """ - dummy function patched into handler.needs_update() by _CryptConfig - when hash alg has been deprecated for context. - """ - return True - -#: list of keys allowed under wildcard "all" scheme w/o a security warning. -_global_settings = set(["truncate_error", "vary_rounds"]) - -#============================================================================= -# crypt policy -#============================================================================= -_preamble = ("The CryptPolicy class has been deprecated as of " - "Passlib 1.6, and will be removed in Passlib 1.8. ") - -class CryptPolicy(object): - """ - .. deprecated:: 1.6 - This class has been deprecated, and will be removed in Passlib 1.8. - All of its functionality has been rolled into :class:`CryptContext`. - - This class previously stored the configuration options for the - CryptContext class. In the interest of interface simplification, - all of this class' functionality has been rolled into the CryptContext - class itself. - The documentation for this class is now focused on documenting how to - migrate to the new api. Additionally, where possible, the deprecation - warnings issued by the CryptPolicy methods will list the replacement call - that should be used. - - Constructors - ============ - CryptPolicy objects can be constructed directly using any of - the keywords accepted by :class:`CryptContext`. Direct uses of the - :class:`!CryptPolicy` constructor should either pass the keywords - directly into the CryptContext constructor, or to :meth:`CryptContext.update` - if the policy object was being used to update an existing context object. - - In addition to passing in keywords directly, - CryptPolicy objects can be constructed by the following methods: - - .. automethod:: from_path - .. automethod:: from_string - .. automethod:: from_source - .. automethod:: from_sources - .. automethod:: replace - - Introspection - ============= - All of the informational methods provided by this class have been deprecated - by identical or similar methods in the :class:`CryptContext` class: - - .. automethod:: has_schemes - .. automethod:: schemes - .. automethod:: iter_handlers - .. automethod:: get_handler - .. automethod:: get_options - .. automethod:: handler_is_deprecated - .. automethod:: get_min_verify_time - - Exporting - ========= - .. automethod:: iter_config - .. automethod:: to_dict - .. automethod:: to_file - .. automethod:: to_string - - .. note:: - CryptPolicy are immutable. - Use the :meth:`replace` method to mutate existing instances. - - .. deprecated:: 1.6 - """ - #=================================================================== - # class methods - #=================================================================== - @classmethod - def from_path(cls, path, section="passlib", encoding="utf-8"): - """create a CryptPolicy instance from a local file. - - .. deprecated:: 1.6 - - Creating a new CryptContext from a file, which was previously done via - ``CryptContext(policy=CryptPolicy.from_path(path))``, can now be - done via ``CryptContext.from_path(path)``. - See :meth:`CryptContext.from_path` for details. - - Updating an existing CryptContext from a file, which was previously done - ``context.policy = CryptPolicy.from_path(path)``, can now be - done via ``context.load_path(path)``. - See :meth:`CryptContext.load_path` for details. - """ - warn(_preamble + - "Instead of ``CryptPolicy.from_path(path)``, " - "use ``CryptContext.from_path(path)`` " - " or ``context.load_path(path)`` for an existing CryptContext.", - DeprecationWarning, stacklevel=2) - return cls(_internal_context=CryptContext.from_path(path, section, - encoding)) - - @classmethod - def from_string(cls, source, section="passlib", encoding="utf-8"): - """create a CryptPolicy instance from a string. - - .. deprecated:: 1.6 - - Creating a new CryptContext from a string, which was previously done via - ``CryptContext(policy=CryptPolicy.from_string(data))``, can now be - done via ``CryptContext.from_string(data)``. - See :meth:`CryptContext.from_string` for details. - - Updating an existing CryptContext from a string, which was previously done - ``context.policy = CryptPolicy.from_string(data)``, can now be - done via ``context.load(data)``. - See :meth:`CryptContext.load` for details. - """ - warn(_preamble + - "Instead of ``CryptPolicy.from_string(source)``, " - "use ``CryptContext.from_string(source)`` or " - "``context.load(source)`` for an existing CryptContext.", - DeprecationWarning, stacklevel=2) - return cls(_internal_context=CryptContext.from_string(source, section, - encoding)) - - @classmethod - def from_source(cls, source, _warn=True): - """create a CryptPolicy instance from some source. - - this method autodetects the source type, and invokes - the appropriate constructor automatically. it attempts - to detect whether the source is a configuration string, a filepath, - a dictionary, or an existing CryptPolicy instance. - - .. deprecated:: 1.6 - - Create a new CryptContext, which could previously be done via - ``CryptContext(policy=CryptPolicy.from_source(source))``, should - now be done using an explicit method: the :class:`CryptContext` - constructor itself, :meth:`CryptContext.from_path`, - or :meth:`CryptContext.from_string`. - - Updating an existing CryptContext, which could previously be done via - ``context.policy = CryptPolicy.from_source(source)``, should - now be done using an explicit method: :meth:`CryptContext.update`, - or :meth:`CryptContext.load`. - """ - if _warn: - warn(_preamble + - "Instead of ``CryptPolicy.from_source()``, " - "use ``CryptContext.from_string(path)`` " - " or ``CryptContext.from_path(source)``, as appropriate.", - DeprecationWarning, stacklevel=2) - if isinstance(source, CryptPolicy): - return source - elif isinstance(source, dict): - return cls(_internal_context=CryptContext(**source)) - elif not isinstance(source, (bytes,unicode)): - raise TypeError("source must be CryptPolicy, dict, config string, " - "or file path: %r" % (type(source),)) - elif any(c in source for c in "\n\r\t") or not source.strip(" \t./\;:"): - return cls(_internal_context=CryptContext.from_string(source)) - else: - return cls(_internal_context=CryptContext.from_path(source)) - - @classmethod - def from_sources(cls, sources, _warn=True): - """create a CryptPolicy instance by merging multiple sources. - - each source is interpreted as by :meth:`from_source`, - and the results are merged together. - - .. deprecated:: 1.6 - Instead of using this method to merge multiple policies together, - a :class:`CryptContext` instance should be created, and then - the multiple sources merged together via :meth:`CryptContext.load`. - """ - if _warn: - warn(_preamble + - "Instead of ``CryptPolicy.from_sources()``, " - "use the various CryptContext constructors " - " followed by ``context.update()``.", - DeprecationWarning, stacklevel=2) - if len(sources) == 0: - raise ValueError("no sources specified") - if len(sources) == 1: - return cls.from_source(sources[0], _warn=False) - kwds = {} - for source in sources: - kwds.update(cls.from_source(source, _warn=False)._context.to_dict(resolve=True)) - return cls(_internal_context=CryptContext(**kwds)) - - def replace(self, *args, **kwds): - """create a new CryptPolicy, optionally updating parts of the - existing configuration. - - .. deprecated:: 1.6 - Callers of this method should :meth:`CryptContext.update` or - :meth:`CryptContext.copy` instead. - """ - if self._stub_policy: - warn(_preamble + # pragma: no cover -- deprecated & unused - "Instead of ``context.policy.replace()``, " - "use ``context.update()`` or ``context.copy()``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().replace()``, " - "create a CryptContext instance and " - "use ``context.update()`` or ``context.copy()``.", - DeprecationWarning, stacklevel=2) - sources = [ self ] - if args: - sources.extend(args) - if kwds: - sources.append(kwds) - return CryptPolicy.from_sources(sources, _warn=False) - - #=================================================================== - # instance attrs - #=================================================================== - - # internal CryptContext we're wrapping to handle everything - # until this class is removed. - _context = None - - # flag indicating this is wrapper generated by the CryptContext.policy - # attribute, rather than one created independantly by the application. - _stub_policy = False - - #=================================================================== - # init - #=================================================================== - def __init__(self, *args, **kwds): - context = kwds.pop("_internal_context", None) - if context: - assert isinstance(context, CryptContext) - self._context = context - self._stub_policy = kwds.pop("_stub_policy", False) - assert not (args or kwds), "unexpected args: %r %r" % (args,kwds) - else: - if args: - if len(args) != 1: - raise TypeError("only one positional argument accepted") - if kwds: - raise TypeError("cannot specify positional arg and kwds") - kwds = args[0] - warn(_preamble + - "Instead of constructing a CryptPolicy instance, " - "create a CryptContext directly, or use ``context.update()`` " - "and ``context.load()`` to reconfigure existing CryptContext " - "instances.", - DeprecationWarning, stacklevel=2) - self._context = CryptContext(**kwds) - - #=================================================================== - # public interface for examining options - #=================================================================== - def has_schemes(self): - """return True if policy defines *any* schemes for use. - - .. deprecated:: 1.6 - applications should use ``bool(context.schemes())`` instead. - see :meth:`CryptContext.schemes`. - """ - if self._stub_policy: - warn(_preamble + # pragma: no cover -- deprecated & unused - "Instead of ``context.policy.has_schemes()``, " - "use ``bool(context.schemes())``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().has_schemes()``, " - "create a CryptContext instance and " - "use ``bool(context.schemes())``.", - DeprecationWarning, stacklevel=2) - return bool(self._context.schemes()) - - def iter_handlers(self): - """return iterator over handlers defined in policy. - - .. deprecated:: 1.6 - applications should use ``context.schemes(resolve=True))`` instead. - see :meth:`CryptContext.schemes`. - """ - if self._stub_policy: - warn(_preamble + - "Instead of ``context.policy.iter_handlers()``, " - "use ``context.schemes(resolve=True)``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().iter_handlers()``, " - "create a CryptContext instance and " - "use ``context.schemes(resolve=True)``.", - DeprecationWarning, stacklevel=2) - return self._context.schemes(resolve=True, unconfigured=True) - - def schemes(self, resolve=False): - """return list of schemes defined in policy. - - .. deprecated:: 1.6 - applications should use :meth:`CryptContext.schemes` instead. - """ - if self._stub_policy: - warn(_preamble + # pragma: no cover -- deprecated & unused - "Instead of ``context.policy.schemes()``, " - "use ``context.schemes()``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().schemes()``, " - "create a CryptContext instance and " - "use ``context.schemes()``.", - DeprecationWarning, stacklevel=2) - return list(self._context.schemes(resolve=resolve, unconfigured=True)) - - def get_handler(self, name=None, category=None, required=False): - """return handler as specified by name, or default handler. - - .. deprecated:: 1.6 - applications should use :meth:`CryptContext.handler` instead, - though note that the ``required`` keyword has been removed, - and the new method will always act as if ``required=True``. - """ - if self._stub_policy: - warn(_preamble + - "Instead of ``context.policy.get_handler()``, " - "use ``context.handler()``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().get_handler()``, " - "create a CryptContext instance and " - "use ``context.handler()``.", - DeprecationWarning, stacklevel=2) - # CryptContext.handler() doesn't support required=False, - # so wrapping it in try/except - try: - return self._context.handler(name, category, unconfigured=True) - except KeyError: - if required: - raise - else: - return None - - def get_min_verify_time(self, category=None): - """get min_verify_time setting for policy. - - .. deprecated:: 1.6 - min_verify_time option will be removed entirely in passlib 1.8 - - .. versionchanged:: 1.7 - this method now always returns the value automatically - calculated by :meth:`CryptContext.min_verify_time`, - any value specified by policy is ignored. - """ - warn("get_min_verify_time() and min_verify_time option is deprecated and ignored, " - "and will be removed in Passlib 1.8", DeprecationWarning, - stacklevel=2) - return 0 - - def get_options(self, name, category=None): - """return dictionary of options specific to a given handler. - - .. deprecated:: 1.6 - this method has no direct replacement in the 1.6 api, as there - is not a clearly defined use-case. however, examining the output of - :meth:`CryptContext.to_dict` should serve as the closest alternative. - """ - # XXX: might make a public replacement, but need more study of the use cases. - if self._stub_policy: - warn(_preamble + # pragma: no cover -- deprecated & unused - "``context.policy.get_options()`` will no longer be available.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "``CryptPolicy().get_options()`` will no longer be available.", - DeprecationWarning, stacklevel=2) - if hasattr(name, "name"): - name = name.name - return self._context._config._get_record_options_with_flag(name, category)[0] - - def handler_is_deprecated(self, name, category=None): - """check if handler has been deprecated by policy. - - .. deprecated:: 1.6 - this method has no direct replacement in the 1.6 api, as there - is not a clearly defined use-case. however, examining the output of - :meth:`CryptContext.to_dict` should serve as the closest alternative. - """ - # XXX: might make a public replacement, but need more study of the use cases. - if self._stub_policy: - warn(_preamble + - "``context.policy.handler_is_deprecated()`` will no longer be available.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "``CryptPolicy().handler_is_deprecated()`` will no longer be available.", - DeprecationWarning, stacklevel=2) - if hasattr(name, "name"): - name = name.name - return self._context.handler(name, category).deprecated - - #=================================================================== - # serialization - #=================================================================== - - def iter_config(self, ini=False, resolve=False): - """iterate over key/value pairs representing the policy object. - - .. deprecated:: 1.6 - applications should use :meth:`CryptContext.to_dict` instead. - """ - if self._stub_policy: - warn(_preamble + # pragma: no cover -- deprecated & unused - "Instead of ``context.policy.iter_config()``, " - "use ``context.to_dict().items()``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().iter_config()``, " - "create a CryptContext instance and " - "use ``context.to_dict().items()``.", - DeprecationWarning, stacklevel=2) - # hacked code that renders keys & values in manner that approximates - # old behavior. context.to_dict() is much cleaner. - context = self._context - if ini: - def render_key(key): - return context._render_config_key(key).replace("__", ".") - def render_value(value): - if isinstance(value, (list,tuple)): - value = ", ".join(value) - return value - resolve = False - else: - render_key = context._render_config_key - render_value = lambda value: value - return ( - (render_key(key), render_value(value)) - for key, value in context._config.iter_config(resolve) - ) - - def to_dict(self, resolve=False): - """export policy object as dictionary of options. - - .. deprecated:: 1.6 - applications should use :meth:`CryptContext.to_dict` instead. - """ - if self._stub_policy: - warn(_preamble + - "Instead of ``context.policy.to_dict()``, " - "use ``context.to_dict()``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().to_dict()``, " - "create a CryptContext instance and " - "use ``context.to_dict()``.", - DeprecationWarning, stacklevel=2) - return self._context.to_dict(resolve) - - def to_file(self, stream, section="passlib"): # pragma: no cover -- deprecated & unused - """export policy to file. - - .. deprecated:: 1.6 - applications should use :meth:`CryptContext.to_string` instead, - and then write the output to a file as desired. - """ - if self._stub_policy: - warn(_preamble + - "Instead of ``context.policy.to_file(stream)``, " - "use ``stream.write(context.to_string())``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().to_file(stream)``, " - "create a CryptContext instance and " - "use ``stream.write(context.to_string())``.", - DeprecationWarning, stacklevel=2) - out = self._context.to_string(section=section) - if PY2: - out = out.encode("utf-8") - stream.write(out) - - def to_string(self, section="passlib", encoding=None): - """export policy to file. - - .. deprecated:: 1.6 - applications should use :meth:`CryptContext.to_string` instead. - """ - if self._stub_policy: - warn(_preamble + # pragma: no cover -- deprecated & unused - "Instead of ``context.policy.to_string()``, " - "use ``context.to_string()``.", - DeprecationWarning, stacklevel=2) - else: - warn(_preamble + - "Instead of ``CryptPolicy().to_string()``, " - "create a CryptContext instance and " - "use ``context.to_string()``.", - DeprecationWarning, stacklevel=2) - out = self._context.to_string(section=section) - if encoding: - out = out.encode(encoding) - return out - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# _CryptConfig helper class -#============================================================================= -class _CryptConfig(object): - """parses, validates, and stores CryptContext config - - this is a helper used internally by CryptContext to handle - parsing, validation, and serialization of its config options. - split out from the main class, but not made public since - that just complicates interface too much (c.f. CryptPolicy) - - :arg source: config as dict mapping ``(cat,scheme,option) -> value`` - """ - #=================================================================== - # instance attrs - #=================================================================== - - # triple-nested dict which maps scheme -> category -> key -> value, - # storing all hash-specific options - _scheme_options = None - - # double-nested dict which maps key -> category -> value - # storing all CryptContext options - _context_options = None - - # tuple of handler objects - handlers = None - - # tuple of scheme objects in same order as handlers - schemes = None - - # tuple of categories in alphabetical order (not including None) - categories = None - - # set of all context keywords used by active schemes - context_kwds = None - - # dict mapping category -> default scheme - _default_schemes = None - - # dict mapping (scheme, category) -> custom handler - _records = None - - # dict mapping category -> list of custom handler instances for that category, - # in order of schemes(). populated on demand by _get_record_list() - _record_lists = None - - #=================================================================== - # constructor - #=================================================================== - def __init__(self, source): - self._init_scheme_list(source.get((None,None,"schemes"))) - self._init_options(source) - self._init_default_schemes() - self._init_records() - - def _init_scheme_list(self, data): - """initialize .handlers and .schemes attributes""" - handlers = [] - schemes = [] - if isinstance(data, native_string_types): - data = splitcomma(data) - for elem in data or (): - # resolve elem -> handler & scheme - if hasattr(elem, "name"): - handler = elem - scheme = handler.name - _validate_handler_name(scheme) - elif isinstance(elem, native_string_types): - handler = get_crypt_handler(elem) - scheme = handler.name - else: - raise TypeError("scheme must be name or CryptHandler, " - "not %r" % type(elem)) - - # check scheme name isn't already in use - if scheme in schemes: - raise KeyError("multiple handlers with same name: %r" % - (scheme,)) - - # add to handler list - handlers.append(handler) - schemes.append(scheme) - - self.handlers = tuple(handlers) - self.schemes = tuple(schemes) - - #=================================================================== - # lowlevel options - #=================================================================== - - #--------------------------------------------------------------- - # init lowlevel option storage - #--------------------------------------------------------------- - def _init_options(self, source): - """load config dict into internal representation, - and init .categories attr - """ - # prepare dicts & locals - norm_scheme_option = self._norm_scheme_option - norm_context_option = self._norm_context_option - self._scheme_options = scheme_options = {} - self._context_options = context_options = {} - categories = set() - - # load source config into internal storage - for (cat, scheme, key), value in iteritems(source): - categories.add(cat) - explicit_scheme = scheme - if not cat and not scheme and key in _global_settings: - # going forward, not using "__all__" format. instead... - # whitelisting set of keys which should be passed to (all) schemes, - # rather than passed to the CryptContext itself - scheme = "all" - if scheme: - # normalize scheme option - key, value = norm_scheme_option(key, value) - - # e.g. things like "min_rounds" should never be set cross-scheme - # this will be fatal under 2.0. - if scheme == "all" and key not in _global_settings: - warn("The '%s' option should be configured per-algorithm, and not set " - "globally in the context; This will be an error in Passlib 2.0" % - (key,), PasslibConfigWarning) - - # this scheme is going away in 2.0; - # but most keys deserve an extra warning since it impacts security. - if explicit_scheme == "all": - warn("The 'all' scheme is deprecated as of Passlib 1.7, " - "and will be removed in Passlib 2.0; Please configure " - "options on a per-algorithm basis.", DeprecationWarning) - - # store in scheme_options - # map structure: scheme_options[scheme][category][key] = value - try: - category_map = scheme_options[scheme] - except KeyError: - scheme_options[scheme] = {cat: {key: value}} - else: - try: - option_map = category_map[cat] - except KeyError: - category_map[cat] = {key: value} - else: - option_map[key] = value - else: - # normalize context option - if cat and key == "schemes": - raise KeyError("'schemes' context option is not allowed " - "per category") - key, value = norm_context_option(cat, key, value) - if key == "min_verify_time": # ignored in 1.7, to be removed in 1.8 - continue - - # store in context_options - # map structure: context_options[key][category] = value - try: - category_map = context_options[key] - except KeyError: - context_options[key] = {cat: value} - else: - category_map[cat] = value - - # store list of configured categories - categories.discard(None) - self.categories = tuple(sorted(categories)) - - def _norm_scheme_option(self, key, value): - # check for invalid options - if key in _forbidden_scheme_options: - raise KeyError("%r option not allowed in CryptContext " - "configuration" % (key,)) - # coerce strings for certain fields (e.g. min_rounds uses ints) - if isinstance(value, native_string_types): - func = _coerce_scheme_options.get(key) - if func: - value = func(value) - return key, value - - def _norm_context_option(self, cat, key, value): - schemes = self.schemes - if key == "default": - if hasattr(value, "name"): - value = value.name - elif not isinstance(value, native_string_types): - raise ExpectedTypeError(value, "str", "default") - if schemes and value not in schemes: - raise KeyError("default scheme not found in policy") - elif key == "deprecated": - if isinstance(value, native_string_types): - value = splitcomma(value) - elif not isinstance(value, (list,tuple)): - raise ExpectedTypeError(value, "str or seq", "deprecated") - if 'auto' in value: - # XXX: have any statements been made about when this is default? - # should do it in 1.8 at latest. - if len(value) > 1: - raise ValueError("cannot list other schemes if " - "``deprecated=['auto']`` is used") - elif schemes: - # make sure list of deprecated schemes is subset of configured schemes - for scheme in value: - if not isinstance(scheme, native_string_types): - raise ExpectedTypeError(value, "str", "deprecated element") - if scheme not in schemes: - raise KeyError("deprecated scheme not found " - "in policy: %r" % (scheme,)) - elif key == "min_verify_time": - warn("'min_verify_time' was deprecated in Passlib 1.6, is " - "ignored in 1.7, and will be removed in 1.8", - DeprecationWarning) - elif key == "harden_verify": - warn("'harden_verify' is deprecated & ignored as of Passlib 1.7.1, " - " and will be removed in 1.8", - DeprecationWarning) - elif key != "schemes": - raise KeyError("unknown CryptContext keyword: %r" % (key,)) - return key, value - - #--------------------------------------------------------------- - # reading context options - #--------------------------------------------------------------- - def get_context_optionmap(self, key, _default={}): - """return dict mapping category->value for specific context option. - - .. warning:: treat return value as readonly! - """ - return self._context_options.get(key, _default) - - def get_context_option_with_flag(self, category, key): - """return value of specific option, handling category inheritance. - also returns flag indicating whether value is category-specific. - """ - try: - category_map = self._context_options[key] - except KeyError: - return None, False - value = category_map.get(None) - if category: - try: - alt = category_map[category] - except KeyError: - pass - else: - if value is None or alt != value: - return alt, True - return value, False - - #--------------------------------------------------------------- - # reading scheme options - #--------------------------------------------------------------- - def _get_scheme_optionmap(self, scheme, category, default={}): - """return all options for (scheme,category) combination - - .. warning:: treat return value as readonly! - """ - try: - return self._scheme_options[scheme][category] - except KeyError: - return default - - def get_base_handler(self, scheme): - return self.handlers[self.schemes.index(scheme)] - - @staticmethod - def expand_settings(handler): - setting_kwds = handler.setting_kwds - if 'rounds' in handler.setting_kwds: - # XXX: historically this extras won't be listed in setting_kwds - setting_kwds += uh.HasRounds.using_rounds_kwds - return setting_kwds - - # NOTE: this is only used by _get_record_options_with_flag()... - def get_scheme_options_with_flag(self, scheme, category): - """return composite dict of all options set for scheme. - includes options inherited from 'all' and from default category. - result can be modified. - returns (kwds, has_cat_specific_options) - """ - # start out with copy of global options - get_optionmap = self._get_scheme_optionmap - kwds = get_optionmap("all", None).copy() - has_cat_options = False - - # add in category-specific global options - if category: - defkwds = kwds.copy() # <-- used to detect category-specific options - kwds.update(get_optionmap("all", category)) - - # filter out global settings not supported by handler - allowed_settings = self.expand_settings(self.get_base_handler(scheme)) - for key in set(kwds).difference(allowed_settings): - kwds.pop(key) - if category: - for key in set(defkwds).difference(allowed_settings): - defkwds.pop(key) - - # add in default options for scheme - other = get_optionmap(scheme, None) - kwds.update(other) - - # load category-specific options for scheme - if category: - defkwds.update(other) - kwds.update(get_optionmap(scheme, category)) - - # compare default category options to see if there's anything - # category-specific - if kwds != defkwds: - has_cat_options = True - - return kwds, has_cat_options - - #=================================================================== - # deprecated & default schemes - #=================================================================== - def _init_default_schemes(self): - """initialize maps containing default scheme for each category. - - have to do this after _init_options(), since the default scheme - is affected by the list of deprecated schemes. - """ - # init maps & locals - get_optionmap = self.get_context_optionmap - default_map = self._default_schemes = get_optionmap("default").copy() - dep_map = get_optionmap("deprecated") - schemes = self.schemes - if not schemes: - return - - # figure out default scheme - deps = dep_map.get(None) or () - default = default_map.get(None) - if not default: - for scheme in schemes: - if scheme not in deps: - default_map[None] = scheme - break - else: - raise ValueError("must have at least one non-deprecated scheme") - elif default in deps: - raise ValueError("default scheme cannot be deprecated") - - # figure out per-category default schemes, - for cat in self.categories: - cdeps = dep_map.get(cat, deps) - cdefault = default_map.get(cat, default) - if not cdefault: - for scheme in schemes: - if scheme not in cdeps: - default_map[cat] = scheme - break - else: - raise ValueError("must have at least one non-deprecated " - "scheme for %r category" % cat) - elif cdefault in cdeps: - raise ValueError("default scheme for %r category " - "cannot be deprecated" % cat) - - def default_scheme(self, category): - """return default scheme for specific category""" - defaults = self._default_schemes - try: - return defaults[category] - except KeyError: - pass - if not self.schemes: - raise KeyError("no hash schemes configured for this " - "CryptContext instance") - return defaults[None] - - def is_deprecated_with_flag(self, scheme, category): - """is scheme deprecated under particular category?""" - depmap = self.get_context_optionmap("deprecated") - def test(cat): - source = depmap.get(cat, depmap.get(None)) - if source is None: - return None - elif 'auto' in source: - return scheme != self.default_scheme(cat) - else: - return scheme in source - value = test(None) or False - if category: - alt = test(category) - if alt is not None and value != alt: - return alt, True - return value, False - - #=================================================================== - # CryptRecord objects - #=================================================================== - def _init_records(self): - # NOTE: this step handles final validation of settings, - # checking for violations against handler's internal invariants. - # this is why we create all the records now, - # so CryptContext throws error immediately rather than later. - self._record_lists = {} - records = self._records = {} - all_context_kwds = self.context_kwds = set() - get_options = self._get_record_options_with_flag - categories = (None,) + self.categories - for handler in self.handlers: - scheme = handler.name - all_context_kwds.update(handler.context_kwds) - for cat in categories: - kwds, has_cat_options = get_options(scheme, cat) - if cat is None or has_cat_options: - records[scheme, cat] = self._create_record(handler, cat, **kwds) - # NOTE: if handler has no category-specific opts, get_record() - # will automatically use the default category's record. - # NOTE: default records for specific category stored under the - # key (None,category); these are populated on-demand by get_record(). - - @staticmethod - def _create_record(handler, category=None, deprecated=False, **settings): - # create custom handler if needed. - try: - # XXX: relaxed=True is mostly here to retain backwards-compat behavior. - # could make this optional flag in future. - subcls = handler.using(relaxed=True, **settings) - except TypeError as err: - m = re.match(r".* unexpected keyword argument '(.*)'$", str(err)) - if m and m.group(1) in settings: - # translate into KeyError, for backwards compat. - # XXX: push this down to GenericHandler.using() implementation? - key = m.group(1) - raise KeyError("keyword not supported by %s handler: %r" % - (handler.name, key)) - raise - - # using private attrs to store some extra metadata in custom handler - assert subcls is not handler, "expected unique variant of handler" - ##subcls._Context__category = category - subcls._Context__orig_handler = handler - subcls.deprecated = deprecated # attr reserved for this purpose - return subcls - - def _get_record_options_with_flag(self, scheme, category): - """return composite dict of options for given scheme + category. - - this is currently a private method, though some variant - of its output may eventually be made public. - - given a scheme & category, it returns two things: - a set of all the keyword options to pass to :meth:`_create_record`, - and a bool flag indicating whether any of these options - were specific to the named category. if this flag is false, - the options are identical to the options for the default category. - - the options dict includes all the scheme-specific settings, - as well as optional *deprecated* keyword. - """ - # get scheme options - kwds, has_cat_options = self.get_scheme_options_with_flag(scheme, category) - - # throw in deprecated flag - value, not_inherited = self.is_deprecated_with_flag(scheme, category) - if value: - kwds['deprecated'] = True - if not_inherited: - has_cat_options = True - - return kwds, has_cat_options - - def get_record(self, scheme, category): - """return record for specific scheme & category (cached)""" - # NOTE: this is part of the critical path shared by - # all of CryptContext's PasswordHash methods, - # hence all the caching and error checking. - - # quick lookup in cache - try: - return self._records[scheme, category] - except KeyError: - pass - - # type check - if category is not None and not isinstance(category, native_string_types): - if PY2 and isinstance(category, unicode): - # for compatibility with unicode-centric py2 apps - return self.get_record(scheme, category.encode("utf-8")) - raise ExpectedTypeError(category, "str or None", "category") - if scheme is not None and not isinstance(scheme, native_string_types): - raise ExpectedTypeError(scheme, "str or None", "scheme") - - # if scheme=None, - # use record for category's default scheme, and cache result. - if not scheme: - default = self.default_scheme(category) - assert default - record = self._records[None, category] = self.get_record(default, - category) - return record - - # if no record for (scheme, category), - # use record for (scheme, None), and cache result. - if category: - try: - cache = self._records - record = cache[scheme, category] = cache[scheme, None] - return record - except KeyError: - pass - - # scheme not found in configuration for default category - raise KeyError("crypt algorithm not found in policy: %r" % (scheme,)) - - def _get_record_list(self, category=None): - """return list of records for category (cached) - - this is an internal helper used only by identify_record() - """ - # type check of category - handled by _get_record() - # quick lookup in cache - try: - return self._record_lists[category] - except KeyError: - pass - # cache miss - build list from scratch - value = self._record_lists[category] = [ - self.get_record(scheme, category) - for scheme in self.schemes - ] - return value - - def identify_record(self, hash, category, required=True): - """internal helper to identify appropriate custom handler for hash""" - # NOTE: this is part of the critical path shared by - # all of CryptContext's PasswordHash methods, - # hence all the caching and error checking. - # FIXME: if multiple hashes could match (e.g. lmhash vs nthash) - # this will only return first match. might want to do something - # about this in future, but for now only hashes with - # unique identifiers will work properly in a CryptContext. - # XXX: if all handlers have a unique prefix (e.g. all are MCF / LDAP), - # could use dict-lookup to speed up this search. - if not isinstance(hash, unicode_or_bytes_types): - raise ExpectedStringError(hash, "hash") - # type check of category - handled by _get_record_list() - for record in self._get_record_list(category): - if record.identify(hash): - return record - if not required: - return None - elif not self.schemes: - raise KeyError("no crypt algorithms supported") - else: - raise ValueError("hash could not be identified") - - @memoized_property - def disabled_record(self): - for record in self._get_record_list(None): - if record.is_disabled: - return record - raise RuntimeError("no disabled hasher present " - "(perhaps add 'unix_disabled' to list of schemes?)") - - #=================================================================== - # serialization - #=================================================================== - def iter_config(self, resolve=False): - """regenerate original config. - - this is an iterator which yields ``(cat,scheme,option),value`` items, - in the order they generally appear inside an INI file. - if interpreted as a dictionary, it should match the original - keywords passed to the CryptContext (aside from any canonization). - - it's mainly used as the internal backend for most of the public - serialization methods. - """ - # grab various bits of data - scheme_options = self._scheme_options - context_options = self._context_options - scheme_keys = sorted(scheme_options) - context_keys = sorted(context_options) - - # write loaded schemes (may differ from 'schemes' local var) - if 'schemes' in context_keys: - context_keys.remove("schemes") - value = self.handlers if resolve else self.schemes - if value: - yield (None, None, "schemes"), list(value) - - # then run through config for each user category - for cat in (None,) + self.categories: - - # write context options - for key in context_keys: - try: - value = context_options[key][cat] - except KeyError: - pass - else: - if isinstance(value, list): - value = list(value) - yield (cat, None, key), value - - # write per-scheme options for all schemes. - for scheme in scheme_keys: - try: - kwds = scheme_options[scheme][cat] - except KeyError: - pass - else: - for key in sorted(kwds): - yield (cat, scheme, key), kwds[key] - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# main CryptContext class -#============================================================================= -class CryptContext(object): - """Helper for hashing & verifying passwords using multiple algorithms. - - Instances of this class allow applications to choose a specific - set of hash algorithms which they wish to support, set limits and defaults - for the rounds and salt sizes those algorithms should use, flag - which algorithms should be deprecated, and automatically handle - migrating users to stronger hashes when they log in. - - Basic usage:: - - >>> ctx = CryptContext(schemes=[...]) - - See the Passlib online documentation for details and full documentation. - """ - # FIXME: altering the configuration of this object isn't threadsafe, - # but is generally only done during application init, so not a major - # issue (just yet). - - # XXX: would like some way to restrict the categories that are allowed, - # to restrict what the app OR the config can use. - - # XXX: add wrap/unwrap callback hooks so app can mutate hash format? - - # XXX: add method for detecting and warning user about schemes - # which don't have any good distinguishing marks? - # or greedy ones (unix_disabled, plaintext) which are not listed at the end? - - #=================================================================== - # instance attrs - #=================================================================== - - # _CryptConfig instance holding current parsed config - _config = None - - # copy of _config methods, stored in CryptContext instance for speed. - _get_record = None - _identify_record = None - - #=================================================================== - # secondary constructors - #=================================================================== - @classmethod - def _norm_source(cls, source): - """internal helper - accepts string, dict, or context""" - if isinstance(source, dict): - return cls(**source) - elif isinstance(source, cls): - return source - else: - self = cls() - self.load(source) - return self - - @classmethod - def from_string(cls, source, section="passlib", encoding="utf-8"): - """create new CryptContext instance from an INI-formatted string. - - :type source: unicode or bytes - :arg source: - string containing INI-formatted content. - - :type section: str - :param section: - option name of section to read from, defaults to ``"passlib"``. - - :type encoding: str - :arg encoding: - optional encoding used when source is bytes, defaults to ``"utf-8"``. - - :returns: - new :class:`CryptContext` instance, configured based on the - parameters in the *source* string. - - Usage example:: - - >>> from passlib.context import CryptContext - >>> context = CryptContext.from_string(''' - ... [passlib] - ... schemes = sha256_crypt, des_crypt - ... sha256_crypt__default_rounds = 30000 - ... ''') - - .. versionadded:: 1.6 - - .. seealso:: :meth:`to_string`, the inverse of this constructor. - """ - if not isinstance(source, unicode_or_bytes_types): - raise ExpectedTypeError(source, "unicode or bytes", "source") - self = cls(_autoload=False) - self.load(source, section=section, encoding=encoding) - return self - - @classmethod - def from_path(cls, path, section="passlib", encoding="utf-8"): - """create new CryptContext instance from an INI-formatted file. - - this functions exactly the same as :meth:`from_string`, - except that it loads from a local file. - - :type path: str - :arg path: - path to local file containing INI-formatted config. - - :type section: str - :param section: - option name of section to read from, defaults to ``"passlib"``. - - :type encoding: str - :arg encoding: - encoding used to load file, defaults to ``"utf-8"``. - - :returns: - new CryptContext instance, configured based on the parameters - stored in the file *path*. - - .. versionadded:: 1.6 - - .. seealso:: :meth:`from_string` for an equivalent usage example. - """ - self = cls(_autoload=False) - self.load_path(path, section=section, encoding=encoding) - return self - - def copy(self, **kwds): - """Return copy of existing CryptContext instance. - - This function returns a new CryptContext instance whose configuration - is exactly the same as the original, with the exception that any keywords - passed in will take precedence over the original settings. - As an example:: - - >>> from passlib.context import CryptContext - - >>> # given an existing context... - >>> ctx1 = CryptContext(["sha256_crypt", "md5_crypt"]) - - >>> # copy can be used to make a clone, and update - >>> # some of the settings at the same time... - >>> ctx2 = custom_app_context.copy(default="md5_crypt") - - >>> # and the original will be unaffected by the change - >>> ctx1.default_scheme() - "sha256_crypt" - >>> ctx2.default_scheme() - "md5_crypt" - - .. versionadded:: 1.6 - This method was previously named :meth:`!replace`. That alias - has been deprecated, and will be removed in Passlib 1.8. - - .. seealso:: :meth:`update` - """ - # XXX: it would be faster to store ref to self._config, - # but don't want to share config objects til sure - # can rely on them being immutable. - other = CryptContext(_autoload=False) - other.load(self) - if kwds: - other.load(kwds, update=True) - return other - - def using(self, **kwds): - """ - alias for :meth:`copy`, to match PasswordHash.using() - """ - return self.copy(**kwds) - - def replace(self, **kwds): - """deprecated alias of :meth:`copy`""" - warn("CryptContext().replace() has been deprecated in Passlib 1.6, " - "and will be removed in Passlib 1.8, " - "it has been renamed to CryptContext().copy()", - DeprecationWarning, stacklevel=2) - return self.copy(**kwds) - - #=================================================================== - # init - #=================================================================== - def __init__(self, schemes=None, - # keyword only... - policy=_UNSET, # <-- deprecated - _autoload=True, **kwds): - # XXX: add ability to make flag certain contexts as immutable, - # e.g. the builtin passlib ones? - # XXX: add a name or import path for the contexts, to help out repr? - if schemes is not None: - kwds['schemes'] = schemes - if policy is not _UNSET: - warn("The CryptContext ``policy`` keyword has been deprecated as of Passlib 1.6, " - "and will be removed in Passlib 1.8; please use " - "``CryptContext.from_string()` or " - "``CryptContext.from_path()`` instead.", - DeprecationWarning) - if policy is None: - self.load(kwds) - elif isinstance(policy, CryptPolicy): - self.load(policy._context) - self.update(kwds) - else: - raise TypeError("policy must be a CryptPolicy instance") - elif _autoload: - self.load(kwds) - else: - assert not kwds, "_autoload=False and kwds are mutually exclusive" - - # XXX: would this be useful? - ##def __str__(self): - ## if PY3: - ## return self.to_string() - ## else: - ## return self.to_string().encode("utf-8") - - def __repr__(self): - return "" % id(self) - - #=================================================================== - # deprecated policy object - #=================================================================== - def _get_policy(self): - # The CryptPolicy class has been deprecated, so to support any - # legacy accesses, we create a stub policy object so .policy attr - # will continue to work. - # - # the code waits until app accesses a specific policy object attribute - # before issuing deprecation warning, so developer gets method-specific - # suggestion for how to upgrade. - - # NOTE: making a copy of the context so the policy acts like a snapshot, - # to retain the pre-1.6 behavior. - return CryptPolicy(_internal_context=self.copy(), _stub_policy=True) - - def _set_policy(self, policy): - warn("The CryptPolicy class and the ``context.policy`` attribute have " - "been deprecated as of Passlib 1.6, and will be removed in " - "Passlib 1.8; please use the ``context.load()`` and " - "``context.update()`` methods instead.", - DeprecationWarning, stacklevel=2) - if isinstance(policy, CryptPolicy): - self.load(policy._context) - else: - raise TypeError("expected CryptPolicy instance") - - policy = property(_get_policy, _set_policy, - doc="[deprecated] returns CryptPolicy instance " - "tied to this CryptContext") - - #=================================================================== - # loading / updating configuration - #=================================================================== - @staticmethod - def _parse_ini_stream(stream, section, filename): - """helper read INI from stream, extract passlib section as dict""" - # NOTE: this expects a unicode stream under py3, - # and a utf-8 bytes stream under py2, - # allowing the resulting dict to always use native strings. - p = SafeConfigParser() - if PY3: - # python 3.2 deprecated readfp in favor of read_file - p.read_file(stream, filename) - else: - p.readfp(stream, filename) - # XXX: could change load() to accept list of items, - # and skip intermediate dict creation - return dict(p.items(section)) - - def load_path(self, path, update=False, section="passlib", encoding="utf-8"): - """Load new configuration into CryptContext from a local file. - - This function is a wrapper for :meth:`load` which - loads a configuration string from the local file *path*, - instead of an in-memory source. Its behavior and options - are otherwise identical to :meth:`!load` when provided with - an INI-formatted string. - - .. versionadded:: 1.6 - """ - def helper(stream): - kwds = self._parse_ini_stream(stream, section, path) - return self.load(kwds, update=update) - if PY3: - # decode to unicode, which load() expected under py3 - with open(path, "rt", encoding=encoding) as stream: - return helper(stream) - elif encoding in ["utf-8", "ascii"]: - # keep as utf-8 bytes, which load() expects under py2 - with open(path, "rb") as stream: - return helper(stream) - else: - # transcode to utf-8 bytes - with open(path, "rb") as fh: - tmp = fh.read().decode(encoding).encode("utf-8") - return helper(BytesIO(tmp)) - - def load(self, source, update=False, section="passlib", encoding="utf-8"): - """Load new configuration into CryptContext, replacing existing config. - - :arg source: - source of new configuration to load. - this value can be a number of different types: - - * a :class:`!dict` object, or compatible Mapping - - the key/value pairs will be interpreted the same - keywords for the :class:`CryptContext` class constructor. - - * a :class:`!unicode` or :class:`!bytes` string - - this will be interpreted as an INI-formatted file, - and appropriate key/value pairs will be loaded from - the specified *section*. - - * another :class:`!CryptContext` object. - - this will export a snapshot of its configuration - using :meth:`to_dict`. - - :type update: bool - :param update: - By default, :meth:`load` will replace the existing configuration - entirely. If ``update=True``, it will preserve any existing - configuration options that are not overridden by the new source, - much like the :meth:`update` method. - - :type section: str - :param section: - When parsing an INI-formatted string, :meth:`load` will look for - a section named ``"passlib"``. This option allows an alternate - section name to be used. Ignored when loading from a dictionary. - - :type encoding: str - :param encoding: - Encoding to use when decode bytes from string. - Defaults to ``"utf-8"``. Ignoring when loading from a dictionary. - - :raises TypeError: - * If the source cannot be identified. - * If an unknown / malformed keyword is encountered. - - :raises ValueError: - If an invalid keyword value is encountered. - - .. note:: - - If an error occurs during a :meth:`!load` call, the :class:`!CryptContext` - instance will be restored to the configuration it was in before - the :meth:`!load` call was made; this is to ensure it is - *never* left in an inconsistent state due to a load error. - - .. versionadded:: 1.6 - """ - #----------------------------------------------------------- - # autodetect source type, convert to dict - #----------------------------------------------------------- - parse_keys = True - if isinstance(source, unicode_or_bytes_types): - if PY3: - source = to_unicode(source, encoding, param="source") - else: - source = to_bytes(source, "utf-8", source_encoding=encoding, - param="source") - source = self._parse_ini_stream(NativeStringIO(source), section, - "") - elif isinstance(source, CryptContext): - # extract dict directly from config, so it can be merged later - source = dict(source._config.iter_config(resolve=True)) - parse_keys = False - elif not hasattr(source, "items"): - # mappings are left alone, otherwise throw an error. - raise ExpectedTypeError(source, "string or dict", "source") - - # XXX: add support for other iterable types, e.g. sequence of pairs? - - #----------------------------------------------------------- - # parse dict keys into (category, scheme, option) format, - # and merge with existing configuration if needed. - #----------------------------------------------------------- - if parse_keys: - parse = self._parse_config_key - source = dict((parse(key), value) - for key, value in iteritems(source)) - if update and self._config is not None: - # if updating, do nothing if source is empty, - if not source: - return - # otherwise overlay source on top of existing config - tmp = source - source = dict(self._config.iter_config(resolve=True)) - source.update(tmp) - - #----------------------------------------------------------- - # compile into _CryptConfig instance, and update state - #----------------------------------------------------------- - config = _CryptConfig(source) - self._config = config - self._reset_dummy_verify() - self._get_record = config.get_record - self._identify_record = config.identify_record - if config.context_kwds: - # (re-)enable method for this instance (in case ELSE clause below ran last load). - self.__dict__.pop("_strip_unused_context_kwds", None) - else: - # disable method for this instance, it's not needed. - self._strip_unused_context_kwds = None - - @staticmethod - def _parse_config_key(ckey): - """helper used to parse ``cat__scheme__option`` keys into a tuple""" - # split string into 1-3 parts - assert isinstance(ckey, native_string_types) - parts = ckey.replace(".", "__").split("__") - count = len(parts) - if count == 1: - cat, scheme, key = None, None, parts[0] - elif count == 2: - cat = None - scheme, key = parts - elif count == 3: - cat, scheme, key = parts - else: - raise TypeError("keys must have less than 3 separators: %r" % - (ckey,)) - # validate & normalize the parts - if cat == "default": - cat = None - elif not cat and cat is not None: - raise TypeError("empty category: %r" % ckey) - if scheme == "context": - scheme = None - elif not scheme and scheme is not None: - raise TypeError("empty scheme: %r" % ckey) - if not key: - raise TypeError("empty option: %r" % ckey) - return cat, scheme, key - - def update(self, *args, **kwds): - """Helper for quickly changing configuration. - - This acts much like the :meth:`!dict.update` method: - it updates the context's configuration, - replacing the original value(s) for the specified keys, - and preserving the rest. - It accepts any :ref:`keyword ` - accepted by the :class:`!CryptContext` constructor. - - .. versionadded:: 1.6 - - .. seealso:: :meth:`copy` - """ - if args: - if len(args) > 1: - raise TypeError("expected at most one positional argument") - if kwds: - raise TypeError("positional arg and keywords mutually exclusive") - self.load(args[0], update=True) - elif kwds: - self.load(kwds, update=True) - - # XXX: make this public? even just as flag to load? - # FIXME: this function suffered some bitrot in 1.6.1, - # will need to be updated before works again. - ##def _simplify(self): - ## "helper to remove redundant/unused options" - ## # don't do anything if no schemes are defined - ## if not self._schemes: - ## return - ## - ## def strip_items(target, filter): - ## keys = [key for key,value in iteritems(target) - ## if filter(key,value)] - ## for key in keys: - ## del target[key] - ## - ## # remove redundant default. - ## defaults = self._default_schemes - ## if defaults.get(None) == self._schemes[0]: - ## del defaults[None] - ## - ## # remove options for unused schemes. - ## scheme_options = self._scheme_options - ## schemes = self._schemes + ("all",) - ## strip_items(scheme_options, lambda k,v: k not in schemes) - ## - ## # remove rendundant cat defaults. - ## cur = self.default_scheme() - ## strip_items(defaults, lambda k,v: k and v==cur) - ## - ## # remove redundant category deprecations. - ## # TODO: this should work w/ 'auto', but needs closer inspection - ## deprecated = self._deprecated_schemes - ## cur = self._deprecated_schemes.get(None) - ## strip_items(deprecated, lambda k,v: k and v==cur) - ## - ## # remove redundant category options. - ## for scheme, config in iteritems(scheme_options): - ## if None in config: - ## cur = config[None] - ## strip_items(config, lambda k,v: k and v==cur) - ## - ## # XXX: anything else? - - #=================================================================== - # reading configuration - #=================================================================== - def schemes(self, resolve=False, category=None, unconfigured=False): - """return schemes loaded into this CryptContext instance. - - :type resolve: bool - :arg resolve: - if ``True``, will return a tuple of :class:`~passlib.ifc.PasswordHash` - objects instead of their names. - - :returns: - returns tuple of the schemes configured for this context - via the *schemes* option. - - .. versionadded:: 1.6 - This was previously available as ``CryptContext().policy.schemes()`` - - .. seealso:: the :ref:`schemes ` option for usage example. - """ - # XXX: should resolv return records rather than handlers? - # or deprecate resolve keyword completely? - # offering up a .hashers Mapping in v1.8 would be great. - # NOTE: supporting 'category' and 'unconfigured' kwds as of 1.7 - # just to pass through to .handler(), but not documenting them... - # may not need to put them to use. - schemes = self._config.schemes - if resolve: - return tuple(self.handler(scheme, category, unconfigured=unconfigured) - for scheme in schemes) - else: - return schemes - - def default_scheme(self, category=None, resolve=False, unconfigured=False): - """return name of scheme that :meth:`hash` will use by default. - - :type resolve: bool - :arg resolve: - if ``True``, will return a :class:`~passlib.ifc.PasswordHash` - object instead of the name. - - :type category: str or None - :param category: - Optional :ref:`user category `. - If specified, this will return the catgory-specific default scheme instead. - - :returns: - name of the default scheme. - - .. seealso:: the :ref:`default ` option for usage example. - - .. versionadded:: 1.6 - - .. versionchanged:: 1.7 - - This now returns a hasher configured with any CryptContext-specific - options (custom rounds settings, etc). Previously this returned - the base hasher from :mod:`passlib.hash`. - """ - # XXX: deprecate this in favor of .handler() or whatever it's replaced with? - # NOTE: supporting 'unconfigured' kwds as of 1.7 - # just to pass through to .handler(), but not documenting them... - # may not need to put them to use. - hasher = self.handler(None, category, unconfigured=unconfigured) - return hasher if resolve else hasher.name - - # XXX: need to decide if exposing this would be useful in any way - ##def categories(self): - ## """return user-categories with algorithm-specific options in this CryptContext. - ## - ## this will always return a tuple. - ## if no categories besides the default category have been configured, - ## the tuple will be empty. - ## """ - ## return self._config.categories - - # XXX: need to decide if exposing this would be useful to applications - # in any meaningful way that isn't already served by to_dict() - ##def options(self, scheme, category=None): - ## kwds, percat = self._config.get_options(scheme, category) - ## return kwds - - def handler(self, scheme=None, category=None, unconfigured=False): - """helper to resolve name of scheme -> :class:`~passlib.ifc.PasswordHash` object used by scheme. - - :arg scheme: - This should identify the scheme to lookup. - If omitted or set to ``None``, this will return the handler - for the default scheme. - - :arg category: - If a user category is specified, and no scheme is provided, - it will use the default for that category. - Otherwise this parameter is ignored. - - :param unconfigured: - - By default, this returns a handler object whose .hash() - and .needs_update() methods will honor the configured - provided by CryptContext. See ``unconfigured=True`` - to get the underlying handler from before any context-specific - configuration was applied. - - :raises KeyError: - If the scheme does not exist OR is not being used within this context. - - :returns: - :class:`~passlib.ifc.PasswordHash` object used to implement - the named scheme within this context (this will usually - be one of the objects from :mod:`passlib.hash`) - - .. versionadded:: 1.6 - This was previously available as ``CryptContext().policy.get_handler()`` - - .. versionchanged:: 1.7 - - This now returns a hasher configured with any CryptContext-specific - options (custom rounds settings, etc). Previously this returned - the base hasher from :mod:`passlib.hash`. - """ - try: - hasher = self._get_record(scheme, category) - if unconfigured: - return hasher._Context__orig_handler - else: - return hasher - except KeyError: - pass - if self._config.handlers: - raise KeyError("crypt algorithm not found in this " - "CryptContext instance: %r" % (scheme,)) - else: - raise KeyError("no crypt algorithms loaded in this " - "CryptContext instance") - - def _get_unregistered_handlers(self): - """check if any handlers in this context aren't in the global registry""" - return tuple(handler for handler in self._config.handlers - if not _is_handler_registered(handler)) - - @property - def context_kwds(self): - """ - return :class:`!set` containing union of all :ref:`contextual keywords ` - supported by the handlers in this context. - - .. versionadded:: 1.6.6 - """ - return self._config.context_kwds - - #=================================================================== - # exporting config - #=================================================================== - @staticmethod - def _render_config_key(key): - """convert 3-part config key to single string""" - cat, scheme, option = key - if cat: - return "%s__%s__%s" % (cat, scheme or "context", option) - elif scheme: - return "%s__%s" % (scheme, option) - else: - return option - - @staticmethod - def _render_ini_value(key, value): - """render value to string suitable for INI file""" - # convert lists to comma separated lists - # (mainly 'schemes' & 'deprecated') - if isinstance(value, (list,tuple)): - value = ", ".join(value) - - # convert numbers to strings - elif isinstance(value, num_types): - if isinstance(value, float) and key[2] == "vary_rounds": - value = ("%.2f" % value).rstrip("0") if value else "0" - else: - value = str(value) - - assert isinstance(value, native_string_types), \ - "expected string for key: %r %r" % (key, value) - - # escape any percent signs. - return value.replace("%", "%%") - - def to_dict(self, resolve=False): - """Return current configuration as a dictionary. - - :type resolve: bool - :arg resolve: - if ``True``, the ``schemes`` key will contain a list of - a :class:`~passlib.ifc.PasswordHash` objects instead of just - their names. - - This method dumps the current configuration of the CryptContext - instance. The key/value pairs should be in the format accepted - by the :class:`!CryptContext` class constructor, in fact - ``CryptContext(**myctx.to_dict())`` will create an exact copy of ``myctx``. - As an example:: - - >>> # you can dump the configuration of any crypt context... - >>> from passlib.apps import ldap_nocrypt_context - >>> ldap_nocrypt_context.to_dict() - {'schemes': ['ldap_salted_sha1', - 'ldap_salted_md5', - 'ldap_sha1', - 'ldap_md5', - 'ldap_plaintext']} - - .. versionadded:: 1.6 - This was previously available as ``CryptContext().policy.to_dict()`` - - .. seealso:: the :ref:`context-serialization-example` example in the tutorial. - """ - # XXX: should resolve default to conditional behavior - # based on presence of unregistered handlers? - render_key = self._render_config_key - return dict((render_key(key), value) - for key, value in self._config.iter_config(resolve)) - - def _write_to_parser(self, parser, section): - """helper to write to ConfigParser instance""" - render_key = self._render_config_key - render_value = self._render_ini_value - parser.add_section(section) - for k,v in self._config.iter_config(): - v = render_value(k, v) - k = render_key(k) - parser.set(section, k, v) - - def to_string(self, section="passlib"): - """serialize to INI format and return as unicode string. - - :param section: - name of INI section to output, defaults to ``"passlib"``. - - :returns: - CryptContext configuration, serialized to a INI unicode string. - - This function acts exactly like :meth:`to_dict`, except that it - serializes all the contents into a single human-readable string, - which can be hand edited, and/or stored in a file. The - output of this method is accepted by :meth:`from_string`, - :meth:`from_path`, and :meth:`load`. As an example:: - - >>> # you can dump the configuration of any crypt context... - >>> from passlib.apps import ldap_nocrypt_context - >>> print ldap_nocrypt_context.to_string() - [passlib] - schemes = ldap_salted_sha1, ldap_salted_md5, ldap_sha1, ldap_md5, ldap_plaintext - - .. versionadded:: 1.6 - This was previously available as ``CryptContext().policy.to_string()`` - - .. seealso:: the :ref:`context-serialization-example` example in the tutorial. - """ - parser = SafeConfigParser() - self._write_to_parser(parser, section) - buf = NativeStringIO() - parser.write(buf) - unregistered = self._get_unregistered_handlers() - if unregistered: - buf.write(( - "# NOTE: the %s handler(s) are not registered with Passlib,\n" - "# this string may not correctly reproduce the current configuration.\n\n" - ) % ", ".join(repr(handler.name) for handler in unregistered)) - out = buf.getvalue() - if not PY3: - out = out.decode("utf-8") - return out - - # XXX: is this useful enough to enable? - ##def write_to_path(self, path, section="passlib", update=False): - ## "write to INI file" - ## parser = ConfigParser() - ## if update and os.path.exists(path): - ## if not parser.read([path]): - ## raise EnvironmentError("failed to read existing file") - ## parser.remove_section(section) - ## self._write_to_parser(parser, section) - ## fh = file(path, "w") - ## parser.write(fh) - ## fh.close() - - #=================================================================== - # verify() hardening - # NOTE: this entire feature has been disabled. - # all contents of this section are NOOPs as of 1.7.1, - # and will be removed in 1.8. - #=================================================================== - - mvt_estimate_max_samples = 20 - mvt_estimate_min_samples = 10 - mvt_estimate_max_time = 2 - mvt_estimate_resolution = 0.01 - harden_verify = None - min_verify_time = 0 - - def reset_min_verify_time(self): - self._reset_dummy_verify() - - #=================================================================== - # password hash api - #=================================================================== - - # NOTE: all the following methods do is look up the appropriate - # custom handler for a given (scheme,category) combination, - # and hand off the real work to the handler itself, - # which is optimized for the specific (scheme,category) configuration. - # - # The custom handlers are cached inside the _CryptConfig - # instance stored in self._config, and are retrieved - # via get_record() and identify_record(). - # - # _get_record() and _identify_record() are references - # to _config methods of the same name, - # stored in CryptContext for speed. - - def _get_or_identify_record(self, hash, scheme=None, category=None): - """return record based on scheme, or failing that, by identifying hash""" - if scheme: - if not isinstance(hash, unicode_or_bytes_types): - raise ExpectedStringError(hash, "hash") - return self._get_record(scheme, category) - else: - # hash typecheck handled by identify_record() - return self._identify_record(hash, category) - - def _strip_unused_context_kwds(self, kwds, record): - """ - helper which removes any context keywords from **kwds** - that are known to be used by another scheme in this context, - but are NOT supported by handler specified by **record**. - - .. note:: - as optimization, load() will set this method to None on a per-instance basis - if there are no context kwds. - """ - if not kwds: - return - unused_kwds = self._config.context_kwds.difference(record.context_kwds) - for key in unused_kwds: - kwds.pop(key, None) - - def needs_update(self, hash, scheme=None, category=None, secret=None): - """Check if hash needs to be replaced for some reason, - in which case the secret should be re-hashed. - - This function is the core of CryptContext's support for hash migration: - This function takes in a hash string, and checks the scheme, - number of rounds, and other properties against the current policy. - It returns ``True`` if the hash is using a deprecated scheme, - or is otherwise outside of the bounds specified by the policy - (e.g. the number of rounds is lower than :ref:`min_rounds ` - configuration for that algorithm). - If so, the password should be re-hashed using :meth:`hash` - Otherwise, it will return ``False``. - - :type hash: unicode or bytes - :arg hash: - The hash string to examine. - - :type scheme: str or None - :param scheme: - - Optional scheme to use. Scheme must be one of the ones - configured for this context (see the - :ref:`schemes ` option). - If no scheme is specified, it will be identified - based on the value of *hash*. - - .. deprecated:: 1.7 - - Support for this keyword is deprecated, and will be removed in Passlib 2.0. - - :type category: str or None - :param category: - Optional :ref:`user category `. - If specified, this will cause any category-specific defaults to - be used when determining if the hash needs to be updated - (e.g. is below the minimum rounds). - - :type secret: unicode, bytes, or None - :param secret: - Optional secret associated with the provided ``hash``. - This is not required, or even currently used for anything... - it's for forward-compatibility with any future - update checks that might need this information. - If provided, Passlib assumes the secret has already been - verified successfully against the hash. - - .. versionadded:: 1.6 - - :returns: ``True`` if hash should be replaced, otherwise ``False``. - - :raises ValueError: - If the hash did not match any of the configured :meth:`schemes`. - - .. versionadded:: 1.6 - This method was previously named :meth:`hash_needs_update`. - - .. seealso:: the :ref:`context-migration-example` example in the tutorial. - """ - if scheme is not None: - # TODO: offer replacement alternative. - # ``context.handler(scheme).needs_update()`` would work, - # but may deprecate .handler() in passlib 1.8. - warn("CryptContext.needs_update(): 'scheme' keyword is deprecated as of " - "Passlib 1.7, and will be removed in Passlib 2.0", - DeprecationWarning) - record = self._get_or_identify_record(hash, scheme, category) - return record.deprecated or record.needs_update(hash, secret=secret) - - @deprecated_method(deprecated="1.6", removed="2.0", replacement="CryptContext.needs_update()") - def hash_needs_update(self, hash, scheme=None, category=None): - """Legacy alias for :meth:`needs_update`. - - .. deprecated:: 1.6 - This method was renamed to :meth:`!needs_update` in version 1.6. - This alias will be removed in version 2.0, and should only - be used for compatibility with Passlib 1.3 - 1.5. - """ - return self.needs_update(hash, scheme, category) - - @deprecated_method(deprecated="1.7", removed="2.0") - def genconfig(self, scheme=None, category=None, **settings): - """Generate a config string for specified scheme. - - .. deprecated:: 1.7 - - This method will be removed in version 2.0, and should only - be used for compatibility with Passlib 1.3 - 1.6. - """ - record = self._get_record(scheme, category) - strip_unused = self._strip_unused_context_kwds - if strip_unused: - strip_unused(settings, record) - return record.genconfig(**settings) - - @deprecated_method(deprecated="1.7", removed="2.0") - def genhash(self, secret, config, scheme=None, category=None, **kwds): - """Generate hash for the specified secret using another hash. - - .. deprecated:: 1.7 - - This method will be removed in version 2.0, and should only - be used for compatibility with Passlib 1.3 - 1.6. - """ - record = self._get_or_identify_record(config, scheme, category) - strip_unused = self._strip_unused_context_kwds - if strip_unused: - strip_unused(kwds, record) - return record.genhash(secret, config, **kwds) - - def identify(self, hash, category=None, resolve=False, required=False, - unconfigured=False): - """Attempt to identify which algorithm the hash belongs to. - - Note that this will only consider the algorithms - currently configured for this context - (see the :ref:`schemes ` option). - All registered algorithms will be checked, from first to last, - and whichever one positively identifies the hash first will be returned. - - :type hash: unicode or bytes - :arg hash: - The hash string to test. - - :type category: str or None - :param category: - Optional :ref:`user category `. - Ignored by this function, this parameter - is provided for symmetry with the other methods. - - :type resolve: bool - :param resolve: - If ``True``, returns the hash handler itself, - instead of the name of the hash. - - :type required: bool - :param required: - If ``True``, this will raise a ValueError if the hash - cannot be identified, instead of returning ``None``. - - :returns: - The handler which first identifies the hash, - or ``None`` if none of the algorithms identify the hash. - """ - record = self._identify_record(hash, category, required) - if record is None: - return None - elif resolve: - if unconfigured: - return record._Context__orig_handler - else: - return record - else: - return record.name - - def hash(self, secret, scheme=None, category=None, **kwds): - """run secret through selected algorithm, returning resulting hash. - - :type secret: unicode or bytes - :arg secret: - the password to hash. - - :type scheme: str or None - :param scheme: - - Optional scheme to use. Scheme must be one of the ones - configured for this context (see the - :ref:`schemes ` option). - If no scheme is specified, the configured default - will be used. - - .. deprecated:: 1.7 - - Support for this keyword is deprecated, and will be removed in Passlib 2.0. - - :type category: str or None - :param category: - Optional :ref:`user category `. - If specified, this will cause any category-specific defaults to - be used when hashing the password (e.g. different default scheme, - different default rounds values, etc). - - :param \*\*kwds: - All other keyword options are passed to the selected algorithm's - :meth:`PasswordHash.hash() ` method. - - :returns: - The secret as encoded by the specified algorithm and options. - The return value will always be a :class:`!str`. - - :raises TypeError, ValueError: - * If any of the arguments have an invalid type or value. - This includes any keywords passed to the underlying hash's - :meth:`PasswordHash.hash() ` method. - - .. seealso:: the :ref:`context-basic-example` example in the tutorial - """ - # XXX: could insert normalization to preferred unicode encoding here - if scheme is not None: - # TODO: offer replacement alternative. - # ``context.handler(scheme).hash()`` would work, - # but may deprecate .handler() in passlib 1.8. - warn("CryptContext.hash(): 'scheme' keyword is deprecated as of " - "Passlib 1.7, and will be removed in Passlib 2.0", - DeprecationWarning) - record = self._get_record(scheme, category) - strip_unused = self._strip_unused_context_kwds - if strip_unused: - strip_unused(kwds, record) - return record.hash(secret, **kwds) - - @deprecated_method(deprecated="1.7", removed="2.0", replacement="CryptContext.hash()") - def encrypt(self, *args, **kwds): - """ - Legacy alias for :meth:`hash`. - - .. deprecated:: 1.7 - This method was renamed to :meth:`!hash` in version 1.7. - This alias will be removed in version 2.0, and should only - be used for compatibility with Passlib 1.3 - 1.6. - """ - return self.hash(*args, **kwds) - - def verify(self, secret, hash, scheme=None, category=None, **kwds): - """verify secret against an existing hash. - - If no scheme is specified, this will attempt to identify - the scheme based on the contents of the provided hash - (limited to the schemes configured for this context). - It will then check whether the password verifies against the hash. - - :type secret: unicode or bytes - :arg secret: - the secret to verify - - :type hash: unicode or bytes - :arg hash: - hash string to compare to - - if ``None`` is passed in, this will be treated as "never verifying" - - :type scheme: str - :param scheme: - Optionally force context to use specific scheme. - This is usually not needed, as most hashes can be unambiguously - identified. Scheme must be one of the ones configured - for this context - (see the :ref:`schemes ` option). - - .. deprecated:: 1.7 - - Support for this keyword is deprecated, and will be removed in Passlib 2.0. - - :type category: str or None - :param category: - Optional :ref:`user category ` string. - This is mainly used when generating new hashes, it has little - effect when verifying; this keyword is mainly provided for symmetry. - - :param \*\*kwds: - All additional keywords are passed to the appropriate handler, - and should match its :attr:`~passlib.ifc.PasswordHash.context_kwds`. - - :returns: - ``True`` if the password matched the hash, else ``False``. - - :raises ValueError: - * if the hash did not match any of the configured :meth:`schemes`. - - * if any of the arguments have an invalid value (this includes - any keywords passed to the underlying hash's - :meth:`PasswordHash.verify() ` method). - - :raises TypeError: - * if any of the arguments have an invalid type (this includes - any keywords passed to the underlying hash's - :meth:`PasswordHash.verify() ` method). - - .. seealso:: the :ref:`context-basic-example` example in the tutorial - """ - # XXX: could insert normalization to preferred unicode encoding here - # XXX: what about supporting a setter() callback ala django 1.4 ? - if scheme is not None: - # TODO: offer replacement alternative. - # ``context.handler(scheme).verify()`` would work, - # but may deprecate .handler() in passlib 1.8. - warn("CryptContext.verify(): 'scheme' keyword is deprecated as of " - "Passlib 1.7, and will be removed in Passlib 2.0", - DeprecationWarning) - if hash is None: - # convenience feature -- let apps pass in hash=None when user - # isn't found / has no hash; useful because it invokes dummy_verify() - self.dummy_verify() - return False - record = self._get_or_identify_record(hash, scheme, category) - strip_unused = self._strip_unused_context_kwds - if strip_unused: - strip_unused(kwds, record) - return record.verify(secret, hash, **kwds) - - def verify_and_update(self, secret, hash, scheme=None, category=None, **kwds): - """verify password and re-hash the password if needed, all in a single call. - - This is a convenience method which takes care of all the following: - first it verifies the password (:meth:`~CryptContext.verify`), if this is successfull - it checks if the hash needs updating (:meth:`~CryptContext.needs_update`), and if so, - re-hashes the password (:meth:`~CryptContext.hash`), returning the replacement hash. - This series of steps is a very common task for applications - which wish to update deprecated hashes, and this call takes - care of all 3 steps efficiently. - - :type secret: unicode or bytes - :arg secret: - the secret to verify - - :type secret: unicode or bytes - :arg hash: - hash string to compare to. - - if ``None`` is passed in, this will be treated as "never verifying" - - :type scheme: str - :param scheme: - Optionally force context to use specific scheme. - This is usually not needed, as most hashes can be unambiguously - identified. Scheme must be one of the ones configured - for this context - (see the :ref:`schemes ` option). - - .. deprecated:: 1.7 - - Support for this keyword is deprecated, and will be removed in Passlib 2.0. - - :type category: str or None - :param category: - Optional :ref:`user category `. - If specified, this will cause any category-specific defaults to - be used if the password has to be re-hashed. - - :param \*\*kwds: - all additional keywords are passed to the appropriate handler, - and should match that hash's - :attr:`PasswordHash.context_kwds `. - - :returns: - This function returns a tuple containing two elements: - ``(verified, replacement_hash)``. The first is a boolean - flag indicating whether the password verified, - and the second an optional replacement hash. - The tuple will always match one of the following 3 cases: - - * ``(False, None)`` indicates the secret failed to verify. - * ``(True, None)`` indicates the secret verified correctly, - and the hash does not need updating. - * ``(True, str)`` indicates the secret verified correctly, - but the current hash needs to be updated. The :class:`!str` - will be the freshly generated hash, to replace the old one. - - :raises TypeError, ValueError: - For the same reasons as :meth:`verify`. - - .. seealso:: the :ref:`context-migration-example` example in the tutorial. - """ - # XXX: could insert normalization to preferred unicode encoding here. - if scheme is not None: - warn("CryptContext.verify(): 'scheme' keyword is deprecated as of " - "Passlib 1.7, and will be removed in Passlib 2.0", - DeprecationWarning) - if hash is None: - # convenience feature -- let apps pass in hash=None when user - # isn't found / has no hash; useful because it invokes dummy_verify() - self.dummy_verify() - return False, None - record = self._get_or_identify_record(hash, scheme, category) - strip_unused = self._strip_unused_context_kwds - if strip_unused and kwds: - clean_kwds = kwds.copy() - strip_unused(clean_kwds, record) - else: - clean_kwds = kwds - # XXX: if record is default scheme, could extend PasswordHash - # api to combine verify & needs_update to single call, - # potentially saving some round-trip parsing. - # but might make these codepaths more complex... - if not record.verify(secret, hash, **clean_kwds): - return False, None - elif record.deprecated or record.needs_update(hash, secret=secret): - # NOTE: we re-hash with default scheme, not current one. - return True, self.hash(secret, category=category, **kwds) - else: - return True, None - - #=================================================================== - # missing-user helper - #=================================================================== - - #: secret used for dummy_verify() - _dummy_secret = "too many secrets" - - @memoized_property - def _dummy_hash(self): - """ - precalculated hash for dummy_verify() to use - """ - return self.hash(self._dummy_secret) - - def _reset_dummy_verify(self): - """ - flush memoized values used by dummy_verify() - """ - type(self)._dummy_hash.clear_cache(self) - - def dummy_verify(self, elapsed=0): - """ - Helper that applications can call when user wasn't found, - in order to simulate time it would take to hash a password. - - Runs verify() against a dummy hash, to simulate verification - of a real account password. - - :param elapsed: - - .. deprecated:: 1.7.1 - - this option is ignored, and will be removed in passlib 1.8. - - .. versionadded:: 1.7 - """ - self.verify(self._dummy_secret, self._dummy_hash) - return False - - #=================================================================== - # disabled hash support - #=================================================================== - - def is_enabled(self, hash): - """ - test if hash represents a usuable password -- - i.e. does not represent an unusuable password such as ``"!"``, - which is recognized by the :class:`~passlib.hash.unix_disabled` hash. - - :raises ValueError: - if the hash is not recognized - (typically solved by adding ``unix_disabled`` to the list of schemes). - """ - return not self._identify_record(hash, None).is_disabled - - def disable(self, hash=None): - """ - return a string to disable logins for user, - usually by returning a non-verifying string such as ``"!"``. - - :param hash: - Callers can optionally provide the account's existing hash. - Some disabled handlers (such as :class:`!unix_disabled`) - will encode this into the returned value, - so that it can be recovered via :meth:`enable`. - - :raises RuntimeError: - if this function is called w/o a disabled hasher - (such as :class:`~passlib.hash.unix_disabled`) included - in the list of schemes. - - :returns: - hash string which will be recognized as valid by the context, - but is guaranteed to not validate against *any* password. - """ - record = self._config.disabled_record - assert record.is_disabled - return record.disable(hash) - - def enable(self, hash): - """ - inverse of :meth:`disable` -- - attempts to recover original hash which was converted - by a :meth:`!disable` call into a disabled hash -- - thus restoring the user's original password. - - :raises ValueError: - if original hash not present, or if the disabled handler doesn't - support encoding the original hash (e.g. ``django_disabled``) - - :returns: - the original hash. - """ - record = self._identify_record(hash, None) - if record.is_disabled: - # XXX: should we throw error if result can't be identified by context? - return record.enable(hash) - else: - # hash wasn't a disabled hash, so return unchanged - return hash - - #=================================================================== - # eoc - #=================================================================== - -class LazyCryptContext(CryptContext): - """CryptContext subclass which doesn't load handlers until needed. - - This is a subclass of CryptContext which takes in a set of arguments - exactly like CryptContext, but won't import any handlers - (or even parse its arguments) until - the first time one of its methods is accessed. - - :arg schemes: - The first positional argument can be a list of schemes, or omitted, - just like CryptContext. - - :param onload: - - If a callable is passed in via this keyword, - it will be invoked at lazy-load time - with the following signature: - ``onload(**kwds) -> kwds``; - where ``kwds`` is all the additional kwds passed to LazyCryptContext. - It should perform any additional deferred initialization, - and return the final dict of options to be passed to CryptContext. - - .. versionadded:: 1.6 - - :param create_policy: - - .. deprecated:: 1.6 - This option will be removed in Passlib 1.8, - applications should use ``onload`` instead. - - :param kwds: - - All additional keywords are passed to CryptContext; - or to the *onload* function (if provided). - - This is mainly used internally by modules such as :mod:`passlib.apps`, - which define a large number of contexts, but only a few of them will be needed - at any one time. Use of this class saves the memory needed to import - the specified handlers until the context instance is actually accessed. - As well, it allows constructing a context at *module-init* time, - but using :func:`!onload()` to provide dynamic configuration - at *application-run* time. - - .. note:: - This class is only useful if you're referencing handler objects by name, - and don't want them imported until runtime. If you want to have the config - validated before your application runs, or are passing in already-imported - handler instances, you should use :class:`CryptContext` instead. - - .. versionadded:: 1.4 - """ - _lazy_kwds = None - - # NOTE: the way this class works changed in 1.6. - # previously it just called _lazy_init() when ``.policy`` was - # first accessed. now that is done whenever any of the public - # attributes are accessed, and the class itself is changed - # to a regular CryptContext, to remove the overhead once it's unneeded. - - def __init__(self, schemes=None, **kwds): - if schemes is not None: - kwds['schemes'] = schemes - self._lazy_kwds = kwds - - def _lazy_init(self): - kwds = self._lazy_kwds - if 'create_policy' in kwds: - warn("The CryptPolicy class, and LazyCryptContext's " - "``create_policy`` keyword have been deprecated as of " - "Passlib 1.6, and will be removed in Passlib 1.8; " - "please use the ``onload`` keyword instead.", - DeprecationWarning) - create_policy = kwds.pop("create_policy") - result = create_policy(**kwds) - policy = CryptPolicy.from_source(result, _warn=False) - kwds = policy._context.to_dict() - elif 'onload' in kwds: - onload = kwds.pop("onload") - kwds = onload(**kwds) - del self._lazy_kwds - super(LazyCryptContext, self).__init__(**kwds) - self.__class__ = CryptContext - - def __getattribute__(self, attr): - if (not attr.startswith("_") or attr.startswith("__")) and \ - self._lazy_kwds is not None: - self._lazy_init() - return object.__getattribute__(self, attr) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/__init__.py b/src/passlib/crypto/__init__.py deleted file mode 100644 index 89f54847..00000000 --- a/src/passlib/crypto/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""passlib.crypto -- package containing cryptographic primitives used by passlib""" diff --git a/src/passlib/crypto/_blowfish/__init__.py b/src/passlib/crypto/_blowfish/__init__.py deleted file mode 100644 index 1aa1c85f..00000000 --- a/src/passlib/crypto/_blowfish/__init__.py +++ /dev/null @@ -1,169 +0,0 @@ -"""passlib.crypto._blowfish - pure-python eks-blowfish implementation for bcrypt - -This is a pure-python implementation of the EKS-Blowfish algorithm described by -Provos and Mazieres in `A Future-Adaptable Password Scheme -`_. - -This package contains two submodules: - -* ``_blowfish/base.py`` contains a class implementing the eks-blowfish algorithm - using easy-to-examine code. - -* ``_blowfish/unrolled.py`` contains a subclass which replaces some methods - of the original class with sped-up versions, mainly using unrolled loops - and local variables. this is the class which is actually used by - Passlib to perform BCrypt in pure python. - - This module is auto-generated by a script, ``_blowfish/_gen_files.py``. - -Status ------- -This implementation is usable, but is an order of magnitude too slow to be -usable with real security. For "ok" security, BCrypt hashes should have at -least 2**11 rounds (as of 2011). Assuming a desired response time <= 100ms, -this means a BCrypt implementation should get at least 20 rounds/ms in order -to be both usable *and* secure. On a 2 ghz cpu, this implementation gets -roughly 0.09 rounds/ms under CPython (220x too slow), and 1.9 rounds/ms -under PyPy (10x too slow). - -History -------- -While subsequently modified considerly for Passlib, this code was originally -based on `jBcrypt 0.2 `_, which was -released under the BSD license:: - - Copyright (c) 2006 Damien Miller - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -""" -#============================================================================= -# imports -#============================================================================= -# core -from itertools import chain -import struct -# pkg -from passlib.utils import getrandbytes, rng -from passlib.utils.binary import bcrypt64 -from passlib.utils.compat import BytesIO, unicode, u, native_string_types -from passlib.crypto._blowfish.unrolled import BlowfishEngine -# local -__all__ = [ - 'BlowfishEngine', - 'raw_bcrypt', -] - -#============================================================================= -# bcrypt constants -#============================================================================= - -# bcrypt constant data "OrpheanBeholderScryDoubt" as 6 integers -BCRYPT_CDATA = [ - 0x4f727068, 0x65616e42, 0x65686f6c, - 0x64657253, 0x63727944, 0x6f756274 -] - -# struct used to encode ciphertext as digest (last output byte discarded) -digest_struct = struct.Struct(">6I") - -#============================================================================= -# base bcrypt helper -# -# interface designed only for use by passlib.handlers.bcrypt:BCrypt -# probably not suitable for other purposes -#============================================================================= -BNULL = b'\x00' - -def raw_bcrypt(password, ident, salt, log_rounds): - """perform central password hashing step in bcrypt scheme. - - :param password: the password to hash - :param ident: identifier w/ minor version (e.g. 2, 2a) - :param salt: the binary salt to use (encoded in bcrypt-base64) - :param log_rounds: the log2 of the number of rounds (as int) - :returns: bcrypt-base64 encoded checksum - """ - #=================================================================== - # parse inputs - #=================================================================== - - # parse ident - assert isinstance(ident, native_string_types) - add_null_padding = True - if ident == u('2a') or ident == u('2y') or ident == u('2b'): - pass - elif ident == u('2'): - add_null_padding = False - elif ident == u('2x'): - raise ValueError("crypt_blowfish's buggy '2x' hashes are not " - "currently supported") - else: - raise ValueError("unknown ident: %r" % (ident,)) - - # decode & validate salt - assert isinstance(salt, bytes) - salt = bcrypt64.decode_bytes(salt) - if len(salt) < 16: - raise ValueError("Missing salt bytes") - elif len(salt) > 16: - salt = salt[:16] - - # prepare password - assert isinstance(password, bytes) - if add_null_padding: - password += BNULL - - # validate rounds - if log_rounds < 4 or log_rounds > 31: - raise ValueError("Bad number of rounds") - - #=================================================================== - # - # run EKS-Blowfish algorithm - # - # This uses the "enhanced key schedule" step described by - # Provos and Mazieres in "A Future-Adaptable Password Scheme" - # http://www.openbsd.org/papers/bcrypt-paper.ps - # - #=================================================================== - - engine = BlowfishEngine() - - # convert password & salt into list of 18 32-bit integers (72 bytes total). - pass_words = engine.key_to_words(password) - salt_words = engine.key_to_words(salt) - - # truncate salt_words to original 16 byte salt, or loop won't wrap - # correctly when passed to .eks_salted_expand() - salt_words16 = salt_words[:4] - - # do EKS key schedule setup - engine.eks_salted_expand(pass_words, salt_words16) - - # apply password & salt keys to key schedule a bunch more times. - rounds = 1<> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) -""".strip() - -def render_encipher(write, indent=0): - for i in irange(0, 15, 2): - write(indent, """\ - # Feistel substitution on left word (round %(i)d) - r ^= %(left)s ^ p%(i1)d - - # Feistel substitution on right word (round %(i1)d) - l ^= %(right)s ^ p%(i2)d - """, i=i, i1=i+1, i2=i+2, - left=BFSTR, right=BFSTR.replace("l","r"), - ) - -def write_encipher_function(write, indent=0): - write(indent, """\ - def encipher(self, l, r): - \"""blowfish encipher a single 64-bit block encoded as two 32-bit ints\""" - - (p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, - p10, p11, p12, p13, p14, p15, p16, p17) = self.P - S0, S1, S2, S3 = self.S - - l ^= p0 - - """) - render_encipher(write, indent+1) - - write(indent+1, """\ - - return r ^ p17, l - - """) - -def write_expand_function(write, indent=0): - write(indent, """\ - def expand(self, key_words): - \"""unrolled version of blowfish key expansion\""" - ##assert len(key_words) >= 18, "size of key_words must be >= 18" - - P, S = self.P, self.S - S0, S1, S2, S3 = S - - #============================================================= - # integrate key - #============================================================= - """) - for i in irange(18): - write(indent+1, """\ - p%(i)d = P[%(i)d] ^ key_words[%(i)d] - """, i=i) - write(indent+1, """\ - - #============================================================= - # update P - #============================================================= - - #------------------------------------------------ - # update P[0] and P[1] - #------------------------------------------------ - l, r = p0, 0 - - """) - - render_encipher(write, indent+1) - - write(indent+1, """\ - - p0, p1 = l, r = r ^ p17, l - - """) - - for i in irange(2, 18, 2): - write(indent+1, """\ - #------------------------------------------------ - # update P[%(i)d] and P[%(i1)d] - #------------------------------------------------ - l ^= p0 - - """, i=i, i1=i+1) - - render_encipher(write, indent+1) - - write(indent+1, """\ - p%(i)d, p%(i1)d = l, r = r ^ p17, l - - """, i=i, i1=i+1) - - write(indent+1, """\ - - #------------------------------------------------ - # save changes to original P array - #------------------------------------------------ - P[:] = (p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, - p10, p11, p12, p13, p14, p15, p16, p17) - - #============================================================= - # update S - #============================================================= - - for box in S: - j = 0 - while j < 256: - l ^= p0 - - """) - - render_encipher(write, indent+3) - - write(indent+3, """\ - - box[j], box[j+1] = l, r = r ^ p17, l - j += 2 - """) - -#============================================================================= -# main -#============================================================================= - -def main(): - target = os.path.join(os.path.dirname(__file__), "unrolled.py") - fh = file(target, "w") - - def write(indent, msg, **kwds): - literal = kwds.pop("literal", False) - if kwds: - msg %= kwds - if not literal: - msg = textwrap.dedent(msg.rstrip(" ")) - if indent: - msg = indent_block(msg, " " * (indent*4)) - fh.write(msg) - - write(0, """\ - \"""passlib.crypto._blowfish.unrolled - unrolled loop implementation of bcrypt, - autogenerated by _gen_files.py - - currently this override the encipher() and expand() methods - with optimized versions, and leaves the other base.py methods alone. - \""" - #================================================================= - # imports - #================================================================= - # pkg - from passlib.crypto._blowfish.base import BlowfishEngine as _BlowfishEngine - # local - __all__ = [ - "BlowfishEngine", - ] - #================================================================= - # - #================================================================= - class BlowfishEngine(_BlowfishEngine): - - """) - - write_encipher_function(write, indent=1) - write_expand_function(write, indent=1) - - write(0, """\ - #================================================================= - # eoc - #================================================================= - - #================================================================= - # eof - #================================================================= - """) - -if __name__ == "__main__": - main() - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/_blowfish/base.py b/src/passlib/crypto/_blowfish/base.py deleted file mode 100644 index 7b4f2cb4..00000000 --- a/src/passlib/crypto/_blowfish/base.py +++ /dev/null @@ -1,441 +0,0 @@ -"""passlib.crypto._blowfish.base - unoptimized pure-python blowfish engine""" -#============================================================================= -# imports -#============================================================================= -# core -import struct -# pkg -from passlib.utils import repeat_string -# local -__all__ = [ - "BlowfishEngine", -] - -#============================================================================= -# blowfish constants -#============================================================================= -BLOWFISH_P = BLOWFISH_S = None - -def _init_constants(): - global BLOWFISH_P, BLOWFISH_S - - # NOTE: blowfish's spec states these numbers are the hex representation - # of the fractional portion of PI, in order. - - # Initial contents of key schedule - 18 integers - BLOWFISH_P = [ - 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, - 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, - 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, - 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, - 0x9216d5d9, 0x8979fb1b, - ] - - # all 4 blowfish S boxes in one array - 256 integers per S box - BLOWFISH_S = [ - # sbox 1 - [ - 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, - 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, - 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, - 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, - 0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, - 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, - 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, - 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, - 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, - 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, - 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, - 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, - 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, - 0xafd6ba33, 0x6c24cf5c, 0x7a325381, 0x28958677, - 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, - 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, - 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, - 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, - 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, - 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, - 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, - 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, - 0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, - 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, - 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, - 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, - 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, - 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, - 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, - 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, - 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, - 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09, - 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, - 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, - 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, - 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, - 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, - 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, - 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, - 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, - 0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, - 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, - 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, - 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, - 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, - 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, - 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, - 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, - 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, - 0x2f2f2218, 0xbe0e1777, 0xea752dfe, 0x8b021fa1, - 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, - 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, - 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, - 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, - 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, - 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, - 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, - 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, - 0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, - 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, - 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, - 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, - 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, - 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, - ], - # sbox 2 - [ - 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, - 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266, - 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, - 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, - 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, - 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, - 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, - 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1, - 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, - 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, - 0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, - 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, - 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, - 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, - 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, - 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, - 0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf, - 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, - 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, - 0x5512721f, 0x2e6b7124, 0x501adde6, 0x9f84cd87, - 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, - 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, - 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, - 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, - 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, - 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509, - 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, - 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, - 0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, - 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, - 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, - 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, - 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, - 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, - 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, - 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, - 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, - 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf, - 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, - 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, - 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, - 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, - 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, - 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281, - 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, - 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, - 0xcdb30aeb, 0x532e3054, 0x8fd948e4, 0x6dbc3128, - 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, - 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, - 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0, - 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, - 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, - 0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, - 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, - 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, - 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061, - 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, - 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, - 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, - 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, - 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, - 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340, - 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, - 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, - ], - # sbox 3 - [ - 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, - 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, - 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, - 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, - 0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, - 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, - 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, - 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, - 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, - 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, - 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, - 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, - 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, - 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, 0xca7820fb, - 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, - 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, - 0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33, - 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, - 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, - 0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc, - 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, - 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, - 0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, - 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, - 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, - 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, - 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, - 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, - 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, - 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, - 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, - 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b, - 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, - 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, - 0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, - 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, - 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, - 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, - 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, - 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, - 0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, - 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, - 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, - 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, - 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, - 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, - 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2, - 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, - 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, - 0xb77f19b6, 0xe0a9dc09, 0x662d09a1, 0xc4324633, - 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, - 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, - 0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, - 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, - 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, - 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, - 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, - 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, - 0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, - 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, - 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, - 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, - 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, - 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0, - ], - # sbox 4 - [ - 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, - 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe, - 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, - 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, - 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, - 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, - 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, - 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, - 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, - 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, - 0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, - 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, - 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, - 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51, - 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, - 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, - 0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b, - 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, - 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, - 0x15056dd4, 0x88f46dba, 0x03a16125, 0x0564f0bd, - 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, - 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, - 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, - 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, - 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, - 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, - 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, - 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, - 0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, - 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, - 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, - 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, - 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, - 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, - 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, - 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, - 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, - 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd, - 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, - 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, - 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, - 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, - 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, - 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, - 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, - 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, - 0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, 0x3278e964, - 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, - 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, - 0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d, - 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, - 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, - 0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02, - 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, - 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, - 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a, - 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, - 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, - 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, - 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, - 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, - 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, - 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, - 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6, - ] - ] - -#============================================================================= -# engine -#============================================================================= -class BlowfishEngine(object): - - def __init__(self): - if BLOWFISH_P is None: - _init_constants() - self.P = list(BLOWFISH_P) - self.S = [ list(box) for box in BLOWFISH_S ] - - #=================================================================== - # common helpers - #=================================================================== - @staticmethod - def key_to_words(data, size=18): - """convert data to tuple of 4-byte integers, repeating or - truncating data as needed to reach specified size""" - assert isinstance(data, bytes) - dlen = len(data) - if not dlen: - # return all zeros - original C code would just read the NUL after - # the password, so mimicing that behavior for this edge case. - return [0]*size - - # repeat data until it fills up 4*size bytes - data = repeat_string(data, size<<2) - - # unpack - return struct.unpack(">%dI" % (size,), data) - - #=================================================================== - # blowfish routines - #=================================================================== - def encipher(self, l, r): - """loop version of blowfish encipher routine""" - P, S = self.P, self.S - l ^= P[0] - i = 1 - while i < 17: - # Feistel substitution on left word - r = ((((S[0][l >> 24] + S[1][(l >> 16) & 0xff]) ^ S[2][(l >> 8) & 0xff]) + - S[3][l & 0xff]) & 0xffffffff) ^ P[i] ^ r - # swap vars so even rounds do Feistel substition on right word - l, r = r, l - i += 1 - return r ^ P[17], l - - # NOTE: decipher is same as above, just with reversed(P) instead. - - def expand(self, key_words): - """perform stock Blowfish keyschedule setup""" - assert len(key_words) >= 18, "key_words must be at least as large as P" - P, S, encipher = self.P, self.S, self.encipher - - i = 0 - while i < 18: - P[i] ^= key_words[i] - i += 1 - - i = l = r = 0 - while i < 18: - P[i], P[i+1] = l,r = encipher(l,r) - i += 2 - - for box in S: - i = 0 - while i < 256: - box[i], box[i+1] = l,r = encipher(l,r) - i += 2 - - #=================================================================== - # eks-blowfish routines - #=================================================================== - def eks_salted_expand(self, key_words, salt_words): - """perform EKS' salted version of Blowfish keyschedule setup""" - # NOTE: this is the same as expand(), except for the addition - # of the operations involving *salt_words*. - - assert len(key_words) >= 18, "key_words must be at least as large as P" - salt_size = len(salt_words) - assert salt_size, "salt_words must not be empty" - assert not salt_size & 1, "salt_words must have even length" - P, S, encipher = self.P, self.S, self.encipher - - i = 0 - while i < 18: - P[i] ^= key_words[i] - i += 1 - - s = i = l = r = 0 - while i < 18: - l ^= salt_words[s] - r ^= salt_words[s+1] - s += 2 - if s == salt_size: - s = 0 - P[i], P[i+1] = l,r = encipher(l,r) # next() - i += 2 - - for box in S: - i = 0 - while i < 256: - l ^= salt_words[s] - r ^= salt_words[s+1] - s += 2 - if s == salt_size: - s = 0 - box[i], box[i+1] = l,r = encipher(l,r) # next() - i += 2 - - def eks_repeated_expand(self, key_words, salt_words, rounds): - """perform rounds stage of EKS keyschedule setup""" - expand = self.expand - n = 0 - while n < rounds: - expand(key_words) - expand(salt_words) - n += 1 - - def repeat_encipher(self, l, r, count): - """repeatedly apply encipher operation to a block""" - encipher = self.encipher - n = 0 - while n < count: - l, r = encipher(l, r) - n += 1 - return l, r - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/_blowfish/unrolled.py b/src/passlib/crypto/_blowfish/unrolled.py deleted file mode 100644 index 4acf6e11..00000000 --- a/src/passlib/crypto/_blowfish/unrolled.py +++ /dev/null @@ -1,771 +0,0 @@ -"""passlib.crypto._blowfish.unrolled - unrolled loop implementation of bcrypt, -autogenerated by _gen_files.py - -currently this override the encipher() and expand() methods -with optimized versions, and leaves the other base.py methods alone. -""" -#============================================================================= -# imports -#============================================================================= -# pkg -from passlib.crypto._blowfish.base import BlowfishEngine as _BlowfishEngine -# local -__all__ = [ - "BlowfishEngine", -] -#============================================================================= -# -#============================================================================= -class BlowfishEngine(_BlowfishEngine): - - def encipher(self, l, r): - """blowfish encipher a single 64-bit block encoded as two 32-bit ints""" - - (p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, - p10, p11, p12, p13, p14, p15, p16, p17) = self.P - S0, S1, S2, S3 = self.S - - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - - return r ^ p17, l - - def expand(self, key_words): - """unrolled version of blowfish key expansion""" - ##assert len(key_words) >= 18, "size of key_words must be >= 18" - - P, S = self.P, self.S - S0, S1, S2, S3 = S - - #============================================================= - # integrate key - #============================================================= - p0 = P[0] ^ key_words[0] - p1 = P[1] ^ key_words[1] - p2 = P[2] ^ key_words[2] - p3 = P[3] ^ key_words[3] - p4 = P[4] ^ key_words[4] - p5 = P[5] ^ key_words[5] - p6 = P[6] ^ key_words[6] - p7 = P[7] ^ key_words[7] - p8 = P[8] ^ key_words[8] - p9 = P[9] ^ key_words[9] - p10 = P[10] ^ key_words[10] - p11 = P[11] ^ key_words[11] - p12 = P[12] ^ key_words[12] - p13 = P[13] ^ key_words[13] - p14 = P[14] ^ key_words[14] - p15 = P[15] ^ key_words[15] - p16 = P[16] ^ key_words[16] - p17 = P[17] ^ key_words[17] - - #============================================================= - # update P - #============================================================= - - #------------------------------------------------ - # update P[0] and P[1] - #------------------------------------------------ - l, r = p0, 0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - - p0, p1 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[2] and P[3] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p2, p3 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[4] and P[5] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p4, p5 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[6] and P[7] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p6, p7 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[8] and P[9] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p8, p9 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[10] and P[11] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p10, p11 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[12] and P[13] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p12, p13 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[14] and P[15] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p14, p15 = l, r = r ^ p17, l - - #------------------------------------------------ - # update P[16] and P[17] - #------------------------------------------------ - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - p16, p17 = l, r = r ^ p17, l - - - #------------------------------------------------ - # save changes to original P array - #------------------------------------------------ - P[:] = (p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, - p10, p11, p12, p13, p14, p15, p16, p17) - - #============================================================= - # update S - #============================================================= - - for box in S: - j = 0 - while j < 256: - l ^= p0 - - # Feistel substitution on left word (round 0) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p1 - - # Feistel substitution on right word (round 1) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p2 - # Feistel substitution on left word (round 2) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p3 - - # Feistel substitution on right word (round 3) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p4 - # Feistel substitution on left word (round 4) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p5 - - # Feistel substitution on right word (round 5) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p6 - # Feistel substitution on left word (round 6) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p7 - - # Feistel substitution on right word (round 7) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p8 - # Feistel substitution on left word (round 8) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p9 - - # Feistel substitution on right word (round 9) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p10 - # Feistel substitution on left word (round 10) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p11 - - # Feistel substitution on right word (round 11) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p12 - # Feistel substitution on left word (round 12) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p13 - - # Feistel substitution on right word (round 13) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p14 - # Feistel substitution on left word (round 14) - r ^= ((((S0[l >> 24] + S1[(l >> 16) & 0xff]) ^ S2[(l >> 8) & 0xff]) + - S3[l & 0xff]) & 0xffffffff) ^ p15 - - # Feistel substitution on right word (round 15) - l ^= ((((S0[r >> 24] + S1[(r >> 16) & 0xff]) ^ S2[(r >> 8) & 0xff]) + - S3[r & 0xff]) & 0xffffffff) ^ p16 - - box[j], box[j+1] = l, r = r ^ p17, l - j += 2 - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/_md4.py b/src/passlib/crypto/_md4.py deleted file mode 100644 index bdc211fa..00000000 --- a/src/passlib/crypto/_md4.py +++ /dev/null @@ -1,244 +0,0 @@ -""" -passlib.crypto._md4 -- fallback implementation of MD4 - -Helper implementing insecure and obsolete md4 algorithm. -used for NTHASH format, which is also insecure and broken, -since it's just md4(password). - -Implementated based on rfc at http://www.faqs.org/rfcs/rfc1320.html - -.. note:: - - This shouldn't be imported directly, it's merely used conditionally - by ``passlib.crypto.lookup_hash()`` when a native implementation can't be found. -""" - -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify -import struct -# site -from passlib.utils.compat import bascii_to_str, irange, PY3 -# local -__all__ = ["md4"] - -#============================================================================= -# utils -#============================================================================= -def F(x,y,z): - return (x&y) | ((~x) & z) - -def G(x,y,z): - return (x&y) | (x&z) | (y&z) - -##def H(x,y,z): -## return x ^ y ^ z - -MASK_32 = 2**32-1 - -#============================================================================= -# main class -#============================================================================= -class md4(object): - """pep-247 compatible implementation of MD4 hash algorithm - - .. attribute:: digest_size - - size of md4 digest in bytes (16 bytes) - - .. method:: update - - update digest by appending additional content - - .. method:: copy - - create clone of digest object, including current state - - .. method:: digest - - return bytes representing md4 digest of current content - - .. method:: hexdigest - - return hexadecimal version of digest - """ - # FIXME: make this follow hash object PEP better. - # FIXME: this isn't threadsafe - - name = "md4" - digest_size = digestsize = 16 - block_size = 64 - - _count = 0 # number of 64-byte blocks processed so far (not including _buf) - _state = None # list of [a,b,c,d] 32 bit ints used as internal register - _buf = None # data processed in 64 byte blocks, this holds leftover from last update - - def __init__(self, content=None): - self._count = 0 - self._state = [0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476] - self._buf = b'' - if content: - self.update(content) - - # round 1 table - [abcd k s] - _round1 = [ - [0,1,2,3, 0,3], - [3,0,1,2, 1,7], - [2,3,0,1, 2,11], - [1,2,3,0, 3,19], - - [0,1,2,3, 4,3], - [3,0,1,2, 5,7], - [2,3,0,1, 6,11], - [1,2,3,0, 7,19], - - [0,1,2,3, 8,3], - [3,0,1,2, 9,7], - [2,3,0,1, 10,11], - [1,2,3,0, 11,19], - - [0,1,2,3, 12,3], - [3,0,1,2, 13,7], - [2,3,0,1, 14,11], - [1,2,3,0, 15,19], - ] - - # round 2 table - [abcd k s] - _round2 = [ - [0,1,2,3, 0,3], - [3,0,1,2, 4,5], - [2,3,0,1, 8,9], - [1,2,3,0, 12,13], - - [0,1,2,3, 1,3], - [3,0,1,2, 5,5], - [2,3,0,1, 9,9], - [1,2,3,0, 13,13], - - [0,1,2,3, 2,3], - [3,0,1,2, 6,5], - [2,3,0,1, 10,9], - [1,2,3,0, 14,13], - - [0,1,2,3, 3,3], - [3,0,1,2, 7,5], - [2,3,0,1, 11,9], - [1,2,3,0, 15,13], - ] - - # round 3 table - [abcd k s] - _round3 = [ - [0,1,2,3, 0,3], - [3,0,1,2, 8,9], - [2,3,0,1, 4,11], - [1,2,3,0, 12,15], - - [0,1,2,3, 2,3], - [3,0,1,2, 10,9], - [2,3,0,1, 6,11], - [1,2,3,0, 14,15], - - [0,1,2,3, 1,3], - [3,0,1,2, 9,9], - [2,3,0,1, 5,11], - [1,2,3,0, 13,15], - - [0,1,2,3, 3,3], - [3,0,1,2, 11,9], - [2,3,0,1, 7,11], - [1,2,3,0, 15,15], - ] - - def _process(self, block): - """process 64 byte block""" - # unpack block into 16 32-bit ints - X = struct.unpack("<16I", block) - - # clone state - orig = self._state - state = list(orig) - - # round 1 - F function - (x&y)|(~x & z) - for a,b,c,d,k,s in self._round1: - t = (state[a] + F(state[b],state[c],state[d]) + X[k]) & MASK_32 - state[a] = ((t<>(32-s)) - - # round 2 - G function - for a,b,c,d,k,s in self._round2: - t = (state[a] + G(state[b],state[c],state[d]) + X[k] + 0x5a827999) & MASK_32 - state[a] = ((t<>(32-s)) - - # round 3 - H function - x ^ y ^ z - for a,b,c,d,k,s in self._round3: - t = (state[a] + (state[b] ^ state[c] ^ state[d]) + X[k] + 0x6ed9eba1) & MASK_32 - state[a] = ((t<>(32-s)) - - # add back into original state - for i in irange(4): - orig[i] = (orig[i]+state[i]) & MASK_32 - - def update(self, content): - if not isinstance(content, bytes): - if PY3: - raise TypeError("expected bytes") - else: - # replicate behavior of hashlib under py2 - content = content.encode("ascii") - buf = self._buf - if buf: - content = buf + content - idx = 0 - end = len(content) - while True: - next = idx + 64 - if next <= end: - self._process(content[idx:next]) - self._count += 1 - idx = next - else: - self._buf = content[idx:] - return - - def copy(self): - other = md4() - other._count = self._count - other._state = list(self._state) - other._buf = self._buf - return other - - def digest(self): - # NOTE: backing up state so we can restore it after _process is called, - # in case object is updated again (this is only attr altered by this method) - orig = list(self._state) - - # final block: buf + 0x80, - # then 0x00 padding until congruent w/ 56 mod 64 bytes - # then last 8 bytes = msg length in bits - buf = self._buf - msglen = self._count*512 + len(buf)*8 - block = buf + b'\x80' + b'\x00' * ((119-len(buf)) % 64) + \ - struct.pack("<2I", msglen & MASK_32, (msglen>>32) & MASK_32) - if len(block) == 128: - self._process(block[:64]) - self._process(block[64:]) - else: - assert len(block) == 64 - self._process(block) - - # render digest & restore un-finalized state - out = struct.pack("<4I", *self._state) - self._state = orig - return out - - def hexdigest(self): - return bascii_to_str(hexlify(self.digest())) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/des.py b/src/passlib/crypto/des.py deleted file mode 100644 index 3f87aef3..00000000 --- a/src/passlib/crypto/des.py +++ /dev/null @@ -1,848 +0,0 @@ -"""passlib.crypto.des -- DES block encryption routines - -History -======= -These routines (which have since been drastically modified for python) -are based on a Java implementation of the des-crypt algorithm, -found at ``_. - -The copyright & license for that source is as follows:: - - UnixCrypt.java 0.9 96/11/25 - Copyright (c) 1996 Aki Yoshida. All rights reserved. - Permission to use, copy, modify and distribute this software - for non-commercial or commercial purposes and without fee is - hereby granted provided that this copyright notice appears in - all copies. - - --- - - Unix crypt(3C) utility - @version 0.9, 11/25/96 - @author Aki Yoshida - - --- - - modified April 2001 - by Iris Van den Broeke, Daniel Deville - - --- - Unix Crypt. - Implements the one way cryptography used by Unix systems for - simple password protection. - @version $Id: UnixCrypt2.txt,v 1.1.1.1 2005/09/13 22:20:13 christos Exp $ - @author Greg Wilkins (gregw) - -The netbsd des-crypt implementation has some nice notes on how this all works - - http://fxr.googlebit.com/source/lib/libcrypt/crypt.c?v=NETBSD-CURRENT -""" - -# TODO: could use an accelerated C version of this module to speed up lmhash, -# des-crypt, and ext-des-crypt - -#============================================================================= -# imports -#============================================================================= -# core -import struct -# pkg -from passlib import exc -from passlib.utils.compat import join_byte_values, byte_elem_value, \ - irange, irange, int_types -# local -__all__ = [ - "expand_des_key", - "des_encrypt_block", -] - -#============================================================================= -# constants -#============================================================================= - -# masks/upper limits for various integer sizes -INT_24_MASK = 0xffffff -INT_56_MASK = 0xffffffffffffff -INT_64_MASK = 0xffffffffffffffff - -# mask to clear parity bits from 64-bit key -_KDATA_MASK = 0xfefefefefefefefe -_KPARITY_MASK = 0x0101010101010101 - -# mask used to setup key schedule -_KS_MASK = 0xfcfcfcfcffffffff - -#============================================================================= -# static DES tables -#============================================================================= - -# placeholders filled in by _load_tables() -PCXROT = IE3264 = SPE = CF6464 = None - -def _load_tables(): - """delay loading tables until they are actually needed""" - global PCXROT, IE3264, SPE, CF6464 - - #--------------------------------------------------------------- - # Initial key schedule permutation - # PC1ROT - bit reverse, then PC1, then Rotate, then PC2 - #--------------------------------------------------------------- - # NOTE: this was reordered from original table to make perm3264 logic simpler - PC1ROT=( - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000002000, 0x0000000000002000, - 0x0000000000000020, 0x0000000000000020, 0x0000000000002020, 0x0000000000002020, - 0x0000000000000400, 0x0000000000000400, 0x0000000000002400, 0x0000000000002400, - 0x0000000000000420, 0x0000000000000420, 0x0000000000002420, 0x0000000000002420, ), - ( 0x0000000000000000, 0x2000000000000000, 0x0000000400000000, 0x2000000400000000, - 0x0000800000000000, 0x2000800000000000, 0x0000800400000000, 0x2000800400000000, - 0x0008000000000000, 0x2008000000000000, 0x0008000400000000, 0x2008000400000000, - 0x0008800000000000, 0x2008800000000000, 0x0008800400000000, 0x2008800400000000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000040, 0x0000000000000040, - 0x0000000020000000, 0x0000000020000000, 0x0000000020000040, 0x0000000020000040, - 0x0000000000200000, 0x0000000000200000, 0x0000000000200040, 0x0000000000200040, - 0x0000000020200000, 0x0000000020200000, 0x0000000020200040, 0x0000000020200040, ), - ( 0x0000000000000000, 0x0002000000000000, 0x0800000000000000, 0x0802000000000000, - 0x0100000000000000, 0x0102000000000000, 0x0900000000000000, 0x0902000000000000, - 0x4000000000000000, 0x4002000000000000, 0x4800000000000000, 0x4802000000000000, - 0x4100000000000000, 0x4102000000000000, 0x4900000000000000, 0x4902000000000000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000040000, 0x0000000000040000, - 0x0000020000000000, 0x0000020000000000, 0x0000020000040000, 0x0000020000040000, - 0x0000000000000004, 0x0000000000000004, 0x0000000000040004, 0x0000000000040004, - 0x0000020000000004, 0x0000020000000004, 0x0000020000040004, 0x0000020000040004, ), - ( 0x0000000000000000, 0x0000400000000000, 0x0200000000000000, 0x0200400000000000, - 0x0080000000000000, 0x0080400000000000, 0x0280000000000000, 0x0280400000000000, - 0x0000008000000000, 0x0000408000000000, 0x0200008000000000, 0x0200408000000000, - 0x0080008000000000, 0x0080408000000000, 0x0280008000000000, 0x0280408000000000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000010000000, 0x0000000010000000, - 0x0000000000001000, 0x0000000000001000, 0x0000000010001000, 0x0000000010001000, - 0x0000000040000000, 0x0000000040000000, 0x0000000050000000, 0x0000000050000000, - 0x0000000040001000, 0x0000000040001000, 0x0000000050001000, 0x0000000050001000, ), - ( 0x0000000000000000, 0x0000001000000000, 0x0000080000000000, 0x0000081000000000, - 0x1000000000000000, 0x1000001000000000, 0x1000080000000000, 0x1000081000000000, - 0x0004000000000000, 0x0004001000000000, 0x0004080000000000, 0x0004081000000000, - 0x1004000000000000, 0x1004001000000000, 0x1004080000000000, 0x1004081000000000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000080, 0x0000000000000080, - 0x0000000000080000, 0x0000000000080000, 0x0000000000080080, 0x0000000000080080, - 0x0000000000800000, 0x0000000000800000, 0x0000000000800080, 0x0000000000800080, - 0x0000000000880000, 0x0000000000880000, 0x0000000000880080, 0x0000000000880080, ), - ( 0x0000000000000000, 0x0000000008000000, 0x0000002000000000, 0x0000002008000000, - 0x0000100000000000, 0x0000100008000000, 0x0000102000000000, 0x0000102008000000, - 0x0000200000000000, 0x0000200008000000, 0x0000202000000000, 0x0000202008000000, - 0x0000300000000000, 0x0000300008000000, 0x0000302000000000, 0x0000302008000000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000400000, 0x0000000000400000, - 0x0000000004000000, 0x0000000004000000, 0x0000000004400000, 0x0000000004400000, - 0x0000000000000800, 0x0000000000000800, 0x0000000000400800, 0x0000000000400800, - 0x0000000004000800, 0x0000000004000800, 0x0000000004400800, 0x0000000004400800, ), - ( 0x0000000000000000, 0x0000000000008000, 0x0040000000000000, 0x0040000000008000, - 0x0000004000000000, 0x0000004000008000, 0x0040004000000000, 0x0040004000008000, - 0x8000000000000000, 0x8000000000008000, 0x8040000000000000, 0x8040000000008000, - 0x8000004000000000, 0x8000004000008000, 0x8040004000000000, 0x8040004000008000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000004000, 0x0000000000004000, - 0x0000000000000008, 0x0000000000000008, 0x0000000000004008, 0x0000000000004008, - 0x0000000000000010, 0x0000000000000010, 0x0000000000004010, 0x0000000000004010, - 0x0000000000000018, 0x0000000000000018, 0x0000000000004018, 0x0000000000004018, ), - ( 0x0000000000000000, 0x0000000200000000, 0x0001000000000000, 0x0001000200000000, - 0x0400000000000000, 0x0400000200000000, 0x0401000000000000, 0x0401000200000000, - 0x0020000000000000, 0x0020000200000000, 0x0021000000000000, 0x0021000200000000, - 0x0420000000000000, 0x0420000200000000, 0x0421000000000000, 0x0421000200000000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000010000000000, 0x0000010000000000, - 0x0000000100000000, 0x0000000100000000, 0x0000010100000000, 0x0000010100000000, - 0x0000000000100000, 0x0000000000100000, 0x0000010000100000, 0x0000010000100000, - 0x0000000100100000, 0x0000000100100000, 0x0000010100100000, 0x0000010100100000, ), - ( 0x0000000000000000, 0x0000000080000000, 0x0000040000000000, 0x0000040080000000, - 0x0010000000000000, 0x0010000080000000, 0x0010040000000000, 0x0010040080000000, - 0x0000000800000000, 0x0000000880000000, 0x0000040800000000, 0x0000040880000000, - 0x0010000800000000, 0x0010000880000000, 0x0010040800000000, 0x0010040880000000, ), - ) - #--------------------------------------------------------------- - # Subsequent key schedule rotation permutations - # PC2ROT - PC2 inverse, then Rotate, then PC2 - #--------------------------------------------------------------- - # NOTE: this was reordered from original table to make perm3264 logic simpler - PC2ROTA=( - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000200000, 0x0000000000200000, 0x0000000000200000, 0x0000000000200000, - 0x0000000004000000, 0x0000000004000000, 0x0000000004000000, 0x0000000004000000, - 0x0000000004200000, 0x0000000004200000, 0x0000000004200000, 0x0000000004200000, ), - ( 0x0000000000000000, 0x0000000000000800, 0x0000010000000000, 0x0000010000000800, - 0x0000000000002000, 0x0000000000002800, 0x0000010000002000, 0x0000010000002800, - 0x0000000010000000, 0x0000000010000800, 0x0000010010000000, 0x0000010010000800, - 0x0000000010002000, 0x0000000010002800, 0x0000010010002000, 0x0000010010002800, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000100000000, 0x0000000100000000, 0x0000000100000000, 0x0000000100000000, - 0x0000000000800000, 0x0000000000800000, 0x0000000000800000, 0x0000000000800000, - 0x0000000100800000, 0x0000000100800000, 0x0000000100800000, 0x0000000100800000, ), - ( 0x0000000000000000, 0x0000020000000000, 0x0000000080000000, 0x0000020080000000, - 0x0000000000400000, 0x0000020000400000, 0x0000000080400000, 0x0000020080400000, - 0x0000000008000000, 0x0000020008000000, 0x0000000088000000, 0x0000020088000000, - 0x0000000008400000, 0x0000020008400000, 0x0000000088400000, 0x0000020088400000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000000040, 0x0000000000000040, 0x0000000000000040, 0x0000000000000040, - 0x0000000000001000, 0x0000000000001000, 0x0000000000001000, 0x0000000000001000, - 0x0000000000001040, 0x0000000000001040, 0x0000000000001040, 0x0000000000001040, ), - ( 0x0000000000000000, 0x0000000000000010, 0x0000000000000400, 0x0000000000000410, - 0x0000000000000080, 0x0000000000000090, 0x0000000000000480, 0x0000000000000490, - 0x0000000040000000, 0x0000000040000010, 0x0000000040000400, 0x0000000040000410, - 0x0000000040000080, 0x0000000040000090, 0x0000000040000480, 0x0000000040000490, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000080000, 0x0000000000080000, 0x0000000000080000, 0x0000000000080000, - 0x0000000000100000, 0x0000000000100000, 0x0000000000100000, 0x0000000000100000, - 0x0000000000180000, 0x0000000000180000, 0x0000000000180000, 0x0000000000180000, ), - ( 0x0000000000000000, 0x0000000000040000, 0x0000000000000020, 0x0000000000040020, - 0x0000000000000004, 0x0000000000040004, 0x0000000000000024, 0x0000000000040024, - 0x0000000200000000, 0x0000000200040000, 0x0000000200000020, 0x0000000200040020, - 0x0000000200000004, 0x0000000200040004, 0x0000000200000024, 0x0000000200040024, ), - ( 0x0000000000000000, 0x0000000000000008, 0x0000000000008000, 0x0000000000008008, - 0x0010000000000000, 0x0010000000000008, 0x0010000000008000, 0x0010000000008008, - 0x0020000000000000, 0x0020000000000008, 0x0020000000008000, 0x0020000000008008, - 0x0030000000000000, 0x0030000000000008, 0x0030000000008000, 0x0030000000008008, ), - ( 0x0000000000000000, 0x0000400000000000, 0x0000080000000000, 0x0000480000000000, - 0x0000100000000000, 0x0000500000000000, 0x0000180000000000, 0x0000580000000000, - 0x4000000000000000, 0x4000400000000000, 0x4000080000000000, 0x4000480000000000, - 0x4000100000000000, 0x4000500000000000, 0x4000180000000000, 0x4000580000000000, ), - ( 0x0000000000000000, 0x0000000000004000, 0x0000000020000000, 0x0000000020004000, - 0x0001000000000000, 0x0001000000004000, 0x0001000020000000, 0x0001000020004000, - 0x0200000000000000, 0x0200000000004000, 0x0200000020000000, 0x0200000020004000, - 0x0201000000000000, 0x0201000000004000, 0x0201000020000000, 0x0201000020004000, ), - ( 0x0000000000000000, 0x1000000000000000, 0x0004000000000000, 0x1004000000000000, - 0x0002000000000000, 0x1002000000000000, 0x0006000000000000, 0x1006000000000000, - 0x0000000800000000, 0x1000000800000000, 0x0004000800000000, 0x1004000800000000, - 0x0002000800000000, 0x1002000800000000, 0x0006000800000000, 0x1006000800000000, ), - ( 0x0000000000000000, 0x0040000000000000, 0x2000000000000000, 0x2040000000000000, - 0x0000008000000000, 0x0040008000000000, 0x2000008000000000, 0x2040008000000000, - 0x0000001000000000, 0x0040001000000000, 0x2000001000000000, 0x2040001000000000, - 0x0000009000000000, 0x0040009000000000, 0x2000009000000000, 0x2040009000000000, ), - ( 0x0000000000000000, 0x0400000000000000, 0x8000000000000000, 0x8400000000000000, - 0x0000002000000000, 0x0400002000000000, 0x8000002000000000, 0x8400002000000000, - 0x0100000000000000, 0x0500000000000000, 0x8100000000000000, 0x8500000000000000, - 0x0100002000000000, 0x0500002000000000, 0x8100002000000000, 0x8500002000000000, ), - ( 0x0000000000000000, 0x0000800000000000, 0x0800000000000000, 0x0800800000000000, - 0x0000004000000000, 0x0000804000000000, 0x0800004000000000, 0x0800804000000000, - 0x0000000400000000, 0x0000800400000000, 0x0800000400000000, 0x0800800400000000, - 0x0000004400000000, 0x0000804400000000, 0x0800004400000000, 0x0800804400000000, ), - ( 0x0000000000000000, 0x0080000000000000, 0x0000040000000000, 0x0080040000000000, - 0x0008000000000000, 0x0088000000000000, 0x0008040000000000, 0x0088040000000000, - 0x0000200000000000, 0x0080200000000000, 0x0000240000000000, 0x0080240000000000, - 0x0008200000000000, 0x0088200000000000, 0x0008240000000000, 0x0088240000000000, ), - ) - - # NOTE: this was reordered from original table to make perm3264 logic simpler - PC2ROTB=( - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000000400, 0x0000000000000400, 0x0000000000000400, 0x0000000000000400, - 0x0000000000080000, 0x0000000000080000, 0x0000000000080000, 0x0000000000080000, - 0x0000000000080400, 0x0000000000080400, 0x0000000000080400, 0x0000000000080400, ), - ( 0x0000000000000000, 0x0000000000800000, 0x0000000000004000, 0x0000000000804000, - 0x0000000080000000, 0x0000000080800000, 0x0000000080004000, 0x0000000080804000, - 0x0000000000040000, 0x0000000000840000, 0x0000000000044000, 0x0000000000844000, - 0x0000000080040000, 0x0000000080840000, 0x0000000080044000, 0x0000000080844000, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000000008, 0x0000000000000008, 0x0000000000000008, 0x0000000000000008, - 0x0000000040000000, 0x0000000040000000, 0x0000000040000000, 0x0000000040000000, - 0x0000000040000008, 0x0000000040000008, 0x0000000040000008, 0x0000000040000008, ), - ( 0x0000000000000000, 0x0000000020000000, 0x0000000200000000, 0x0000000220000000, - 0x0000000000000080, 0x0000000020000080, 0x0000000200000080, 0x0000000220000080, - 0x0000000000100000, 0x0000000020100000, 0x0000000200100000, 0x0000000220100000, - 0x0000000000100080, 0x0000000020100080, 0x0000000200100080, 0x0000000220100080, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000002000, 0x0000000000002000, 0x0000000000002000, 0x0000000000002000, - 0x0000020000000000, 0x0000020000000000, 0x0000020000000000, 0x0000020000000000, - 0x0000020000002000, 0x0000020000002000, 0x0000020000002000, 0x0000020000002000, ), - ( 0x0000000000000000, 0x0000000000000800, 0x0000000100000000, 0x0000000100000800, - 0x0000000010000000, 0x0000000010000800, 0x0000000110000000, 0x0000000110000800, - 0x0000000000000004, 0x0000000000000804, 0x0000000100000004, 0x0000000100000804, - 0x0000000010000004, 0x0000000010000804, 0x0000000110000004, 0x0000000110000804, ), - ( 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, - 0x0000000000001000, 0x0000000000001000, 0x0000000000001000, 0x0000000000001000, - 0x0000000000000010, 0x0000000000000010, 0x0000000000000010, 0x0000000000000010, - 0x0000000000001010, 0x0000000000001010, 0x0000000000001010, 0x0000000000001010, ), - ( 0x0000000000000000, 0x0000000000000040, 0x0000010000000000, 0x0000010000000040, - 0x0000000000200000, 0x0000000000200040, 0x0000010000200000, 0x0000010000200040, - 0x0000000000008000, 0x0000000000008040, 0x0000010000008000, 0x0000010000008040, - 0x0000000000208000, 0x0000000000208040, 0x0000010000208000, 0x0000010000208040, ), - ( 0x0000000000000000, 0x0000000004000000, 0x0000000008000000, 0x000000000c000000, - 0x0400000000000000, 0x0400000004000000, 0x0400000008000000, 0x040000000c000000, - 0x8000000000000000, 0x8000000004000000, 0x8000000008000000, 0x800000000c000000, - 0x8400000000000000, 0x8400000004000000, 0x8400000008000000, 0x840000000c000000, ), - ( 0x0000000000000000, 0x0002000000000000, 0x0200000000000000, 0x0202000000000000, - 0x1000000000000000, 0x1002000000000000, 0x1200000000000000, 0x1202000000000000, - 0x0008000000000000, 0x000a000000000000, 0x0208000000000000, 0x020a000000000000, - 0x1008000000000000, 0x100a000000000000, 0x1208000000000000, 0x120a000000000000, ), - ( 0x0000000000000000, 0x0000000000400000, 0x0000000000000020, 0x0000000000400020, - 0x0040000000000000, 0x0040000000400000, 0x0040000000000020, 0x0040000000400020, - 0x0800000000000000, 0x0800000000400000, 0x0800000000000020, 0x0800000000400020, - 0x0840000000000000, 0x0840000000400000, 0x0840000000000020, 0x0840000000400020, ), - ( 0x0000000000000000, 0x0080000000000000, 0x0000008000000000, 0x0080008000000000, - 0x2000000000000000, 0x2080000000000000, 0x2000008000000000, 0x2080008000000000, - 0x0020000000000000, 0x00a0000000000000, 0x0020008000000000, 0x00a0008000000000, - 0x2020000000000000, 0x20a0000000000000, 0x2020008000000000, 0x20a0008000000000, ), - ( 0x0000000000000000, 0x0000002000000000, 0x0000040000000000, 0x0000042000000000, - 0x4000000000000000, 0x4000002000000000, 0x4000040000000000, 0x4000042000000000, - 0x0000400000000000, 0x0000402000000000, 0x0000440000000000, 0x0000442000000000, - 0x4000400000000000, 0x4000402000000000, 0x4000440000000000, 0x4000442000000000, ), - ( 0x0000000000000000, 0x0000004000000000, 0x0000200000000000, 0x0000204000000000, - 0x0000080000000000, 0x0000084000000000, 0x0000280000000000, 0x0000284000000000, - 0x0000800000000000, 0x0000804000000000, 0x0000a00000000000, 0x0000a04000000000, - 0x0000880000000000, 0x0000884000000000, 0x0000a80000000000, 0x0000a84000000000, ), - ( 0x0000000000000000, 0x0000000800000000, 0x0000000400000000, 0x0000000c00000000, - 0x0000100000000000, 0x0000100800000000, 0x0000100400000000, 0x0000100c00000000, - 0x0010000000000000, 0x0010000800000000, 0x0010000400000000, 0x0010000c00000000, - 0x0010100000000000, 0x0010100800000000, 0x0010100400000000, 0x0010100c00000000, ), - ( 0x0000000000000000, 0x0100000000000000, 0x0001000000000000, 0x0101000000000000, - 0x0000001000000000, 0x0100001000000000, 0x0001001000000000, 0x0101001000000000, - 0x0004000000000000, 0x0104000000000000, 0x0005000000000000, 0x0105000000000000, - 0x0004001000000000, 0x0104001000000000, 0x0005001000000000, 0x0105001000000000, ), - ) - #--------------------------------------------------------------- - # PCXROT - PC1ROT, PC2ROTA, PC2ROTB listed in order - # of the PC1 rotation schedule, as used by des_setkey - #--------------------------------------------------------------- - ##ROTATES = (1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1) - ##PCXROT = ( - ## PC1ROT, PC2ROTA, PC2ROTB, PC2ROTB, - ## PC2ROTB, PC2ROTB, PC2ROTB, PC2ROTB, - ## PC2ROTA, PC2ROTB, PC2ROTB, PC2ROTB, - ## PC2ROTB, PC2ROTB, PC2ROTB, PC2ROTA, - ## ) - - # NOTE: modified PCXROT to contain entrys broken into pairs, - # to help generate them in format best used by encoder. - PCXROT = ( - (PC1ROT, PC2ROTA), (PC2ROTB, PC2ROTB), - (PC2ROTB, PC2ROTB), (PC2ROTB, PC2ROTB), - (PC2ROTA, PC2ROTB), (PC2ROTB, PC2ROTB), - (PC2ROTB, PC2ROTB), (PC2ROTB, PC2ROTA), - ) - - #--------------------------------------------------------------- - # Bit reverse, intial permupation, expantion - # Initial permutation/expansion table - #--------------------------------------------------------------- - # NOTE: this was reordered from original table to make perm3264 logic simpler - IE3264=( - ( 0x0000000000000000, 0x0000000000800800, 0x0000000000008008, 0x0000000000808808, - 0x0000008008000000, 0x0000008008800800, 0x0000008008008008, 0x0000008008808808, - 0x0000000080080000, 0x0000000080880800, 0x0000000080088008, 0x0000000080888808, - 0x0000008088080000, 0x0000008088880800, 0x0000008088088008, 0x0000008088888808, ), - ( 0x0000000000000000, 0x0080080000000000, 0x0000800800000000, 0x0080880800000000, - 0x0800000000000080, 0x0880080000000080, 0x0800800800000080, 0x0880880800000080, - 0x8008000000000000, 0x8088080000000000, 0x8008800800000000, 0x8088880800000000, - 0x8808000000000080, 0x8888080000000080, 0x8808800800000080, 0x8888880800000080, ), - ( 0x0000000000000000, 0x0000000000001000, 0x0000000000000010, 0x0000000000001010, - 0x0000000010000000, 0x0000000010001000, 0x0000000010000010, 0x0000000010001010, - 0x0000000000100000, 0x0000000000101000, 0x0000000000100010, 0x0000000000101010, - 0x0000000010100000, 0x0000000010101000, 0x0000000010100010, 0x0000000010101010, ), - ( 0x0000000000000000, 0x0000100000000000, 0x0000001000000000, 0x0000101000000000, - 0x1000000000000000, 0x1000100000000000, 0x1000001000000000, 0x1000101000000000, - 0x0010000000000000, 0x0010100000000000, 0x0010001000000000, 0x0010101000000000, - 0x1010000000000000, 0x1010100000000000, 0x1010001000000000, 0x1010101000000000, ), - ( 0x0000000000000000, 0x0000000000002000, 0x0000000000000020, 0x0000000000002020, - 0x0000000020000000, 0x0000000020002000, 0x0000000020000020, 0x0000000020002020, - 0x0000000000200000, 0x0000000000202000, 0x0000000000200020, 0x0000000000202020, - 0x0000000020200000, 0x0000000020202000, 0x0000000020200020, 0x0000000020202020, ), - ( 0x0000000000000000, 0x0000200000000000, 0x0000002000000000, 0x0000202000000000, - 0x2000000000000000, 0x2000200000000000, 0x2000002000000000, 0x2000202000000000, - 0x0020000000000000, 0x0020200000000000, 0x0020002000000000, 0x0020202000000000, - 0x2020000000000000, 0x2020200000000000, 0x2020002000000000, 0x2020202000000000, ), - ( 0x0000000000000000, 0x0000000000004004, 0x0400000000000040, 0x0400000000004044, - 0x0000000040040000, 0x0000000040044004, 0x0400000040040040, 0x0400000040044044, - 0x0000000000400400, 0x0000000000404404, 0x0400000000400440, 0x0400000000404444, - 0x0000000040440400, 0x0000000040444404, 0x0400000040440440, 0x0400000040444444, ), - ( 0x0000000000000000, 0x0000400400000000, 0x0000004004000000, 0x0000404404000000, - 0x4004000000000000, 0x4004400400000000, 0x4004004004000000, 0x4004404404000000, - 0x0040040000000000, 0x0040440400000000, 0x0040044004000000, 0x0040444404000000, - 0x4044040000000000, 0x4044440400000000, 0x4044044004000000, 0x4044444404000000, ), - ) - - #--------------------------------------------------------------- - # Table that combines the S, P, and E operations. - #--------------------------------------------------------------- - SPE=( - ( 0x0080088008200000, 0x0000008008000000, 0x0000000000200020, 0x0080088008200020, - 0x0000000000200000, 0x0080088008000020, 0x0000008008000020, 0x0000000000200020, - 0x0080088008000020, 0x0080088008200000, 0x0000008008200000, 0x0080080000000020, - 0x0080080000200020, 0x0000000000200000, 0x0000000000000000, 0x0000008008000020, - 0x0000008008000000, 0x0000000000000020, 0x0080080000200000, 0x0080088008000000, - 0x0080088008200020, 0x0000008008200000, 0x0080080000000020, 0x0080080000200000, - 0x0000000000000020, 0x0080080000000000, 0x0080088008000000, 0x0000008008200020, - 0x0080080000000000, 0x0080080000200020, 0x0000008008200020, 0x0000000000000000, - 0x0000000000000000, 0x0080088008200020, 0x0080080000200000, 0x0000008008000020, - 0x0080088008200000, 0x0000008008000000, 0x0080080000000020, 0x0080080000200000, - 0x0000008008200020, 0x0080080000000000, 0x0080088008000000, 0x0000000000200020, - 0x0080088008000020, 0x0000000000000020, 0x0000000000200020, 0x0000008008200000, - 0x0080088008200020, 0x0080088008000000, 0x0000008008200000, 0x0080080000200020, - 0x0000000000200000, 0x0080080000000020, 0x0000008008000020, 0x0000000000000000, - 0x0000008008000000, 0x0000000000200000, 0x0080080000200020, 0x0080088008200000, - 0x0000000000000020, 0x0000008008200020, 0x0080080000000000, 0x0080088008000020, ), - ( 0x1000800810004004, 0x0000000000000000, 0x0000800810000000, 0x0000000010004004, - 0x1000000000004004, 0x1000800800000000, 0x0000800800004004, 0x0000800810000000, - 0x0000800800000000, 0x1000000010004004, 0x1000000000000000, 0x0000800800004004, - 0x1000000010000000, 0x0000800810004004, 0x0000000010004004, 0x1000000000000000, - 0x0000000010000000, 0x1000800800004004, 0x1000000010004004, 0x0000800800000000, - 0x1000800810000000, 0x0000000000004004, 0x0000000000000000, 0x1000000010000000, - 0x1000800800004004, 0x1000800810000000, 0x0000800810004004, 0x1000000000004004, - 0x0000000000004004, 0x0000000010000000, 0x1000800800000000, 0x1000800810004004, - 0x1000000010000000, 0x0000800810004004, 0x0000800800004004, 0x1000800810000000, - 0x1000800810004004, 0x1000000010000000, 0x1000000000004004, 0x0000000000000000, - 0x0000000000004004, 0x1000800800000000, 0x0000000010000000, 0x1000000010004004, - 0x0000800800000000, 0x0000000000004004, 0x1000800810000000, 0x1000800800004004, - 0x0000800810004004, 0x0000800800000000, 0x0000000000000000, 0x1000000000004004, - 0x1000000000000000, 0x1000800810004004, 0x0000800810000000, 0x0000000010004004, - 0x1000000010004004, 0x0000000010000000, 0x1000800800000000, 0x0000800800004004, - 0x1000800800004004, 0x1000000000000000, 0x0000000010004004, 0x0000800810000000, ), - ( 0x0000000000400410, 0x0010004004400400, 0x0010000000000000, 0x0010000000400410, - 0x0000004004000010, 0x0000000000400400, 0x0010000000400410, 0x0010004004000000, - 0x0010000000400400, 0x0000004004000000, 0x0000004004400400, 0x0000000000000010, - 0x0010004004400410, 0x0010000000000010, 0x0000000000000010, 0x0000004004400410, - 0x0000000000000000, 0x0000004004000010, 0x0010004004400400, 0x0010000000000000, - 0x0010000000000010, 0x0010004004400410, 0x0000004004000000, 0x0000000000400410, - 0x0000004004400410, 0x0010000000400400, 0x0010004004000010, 0x0000004004400400, - 0x0010004004000000, 0x0000000000000000, 0x0000000000400400, 0x0010004004000010, - 0x0010004004400400, 0x0010000000000000, 0x0000000000000010, 0x0000004004000000, - 0x0010000000000010, 0x0000004004000010, 0x0000004004400400, 0x0010000000400410, - 0x0000000000000000, 0x0010004004400400, 0x0010004004000000, 0x0000004004400410, - 0x0000004004000010, 0x0000000000400400, 0x0010004004400410, 0x0000000000000010, - 0x0010004004000010, 0x0000000000400410, 0x0000000000400400, 0x0010004004400410, - 0x0000004004000000, 0x0010000000400400, 0x0010000000400410, 0x0010004004000000, - 0x0010000000400400, 0x0000000000000000, 0x0000004004400410, 0x0010000000000010, - 0x0000000000400410, 0x0010004004000010, 0x0010000000000000, 0x0000004004400400, ), - ( 0x0800100040040080, 0x0000100000001000, 0x0800000000000080, 0x0800100040041080, - 0x0000000000000000, 0x0000000040041000, 0x0800100000001080, 0x0800000040040080, - 0x0000100040041000, 0x0800000000001080, 0x0000000000001000, 0x0800100000000080, - 0x0800000000001080, 0x0800100040040080, 0x0000000040040000, 0x0000000000001000, - 0x0800000040041080, 0x0000100040040000, 0x0000100000000000, 0x0800000000000080, - 0x0000100040040000, 0x0800100000001080, 0x0000000040041000, 0x0000100000000000, - 0x0800100000000080, 0x0000000000000000, 0x0800000040040080, 0x0000100040041000, - 0x0000100000001000, 0x0800000040041080, 0x0800100040041080, 0x0000000040040000, - 0x0800000040041080, 0x0800100000000080, 0x0000000040040000, 0x0800000000001080, - 0x0000100040040000, 0x0000100000001000, 0x0800000000000080, 0x0000000040041000, - 0x0800100000001080, 0x0000000000000000, 0x0000100000000000, 0x0800000040040080, - 0x0000000000000000, 0x0800000040041080, 0x0000100040041000, 0x0000100000000000, - 0x0000000000001000, 0x0800100040041080, 0x0800100040040080, 0x0000000040040000, - 0x0800100040041080, 0x0800000000000080, 0x0000100000001000, 0x0800100040040080, - 0x0800000040040080, 0x0000100040040000, 0x0000000040041000, 0x0800100000001080, - 0x0800100000000080, 0x0000000000001000, 0x0800000000001080, 0x0000100040041000, ), - ( 0x0000000000800800, 0x0000001000000000, 0x0040040000000000, 0x2040041000800800, - 0x2000001000800800, 0x0040040000800800, 0x2040041000000000, 0x0000001000800800, - 0x0000001000000000, 0x2000000000000000, 0x2000000000800800, 0x0040041000000000, - 0x2040040000800800, 0x2000001000800800, 0x0040041000800800, 0x0000000000000000, - 0x0040041000000000, 0x0000000000800800, 0x2000001000000000, 0x2040040000000000, - 0x0040040000800800, 0x2040041000000000, 0x0000000000000000, 0x2000000000800800, - 0x2000000000000000, 0x2040040000800800, 0x2040041000800800, 0x2000001000000000, - 0x0000001000800800, 0x0040040000000000, 0x2040040000000000, 0x0040041000800800, - 0x0040041000800800, 0x2040040000800800, 0x2000001000000000, 0x0000001000800800, - 0x0000001000000000, 0x2000000000000000, 0x2000000000800800, 0x0040040000800800, - 0x0000000000800800, 0x0040041000000000, 0x2040041000800800, 0x0000000000000000, - 0x2040041000000000, 0x0000000000800800, 0x0040040000000000, 0x2000001000000000, - 0x2040040000800800, 0x0040040000000000, 0x0000000000000000, 0x2040041000800800, - 0x2000001000800800, 0x0040041000800800, 0x2040040000000000, 0x0000001000000000, - 0x0040041000000000, 0x2000001000800800, 0x0040040000800800, 0x2040040000000000, - 0x2000000000000000, 0x2040041000000000, 0x0000001000800800, 0x2000000000800800, ), - ( 0x4004000000008008, 0x4004000020000000, 0x0000000000000000, 0x0000200020008008, - 0x4004000020000000, 0x0000200000000000, 0x4004200000008008, 0x0000000020000000, - 0x4004200000000000, 0x4004200020008008, 0x0000200020000000, 0x0000000000008008, - 0x0000200000008008, 0x4004000000008008, 0x0000000020008008, 0x4004200020000000, - 0x0000000020000000, 0x4004200000008008, 0x4004000020008008, 0x0000000000000000, - 0x0000200000000000, 0x4004000000000000, 0x0000200020008008, 0x4004000020008008, - 0x4004200020008008, 0x0000000020008008, 0x0000000000008008, 0x4004200000000000, - 0x4004000000000000, 0x0000200020000000, 0x4004200020000000, 0x0000200000008008, - 0x4004200000000000, 0x0000000000008008, 0x0000200000008008, 0x4004200020000000, - 0x0000200020008008, 0x4004000020000000, 0x0000000000000000, 0x0000200000008008, - 0x0000000000008008, 0x0000200000000000, 0x4004000020008008, 0x0000000020000000, - 0x4004000020000000, 0x4004200020008008, 0x0000200020000000, 0x4004000000000000, - 0x4004200020008008, 0x0000200020000000, 0x0000000020000000, 0x4004200000008008, - 0x4004000000008008, 0x0000000020008008, 0x4004200020000000, 0x0000000000000000, - 0x0000200000000000, 0x4004000000008008, 0x4004200000008008, 0x0000200020008008, - 0x0000000020008008, 0x4004200000000000, 0x4004000000000000, 0x4004000020008008, ), - ( 0x0000400400000000, 0x0020000000000000, 0x0020000000100000, 0x0400000000100040, - 0x0420400400100040, 0x0400400400000040, 0x0020400400000000, 0x0000000000000000, - 0x0000000000100000, 0x0420000000100040, 0x0420000000000040, 0x0000400400100000, - 0x0400000000000040, 0x0020400400100000, 0x0000400400100000, 0x0420000000000040, - 0x0420000000100040, 0x0000400400000000, 0x0400400400000040, 0x0420400400100040, - 0x0000000000000000, 0x0020000000100000, 0x0400000000100040, 0x0020400400000000, - 0x0400400400100040, 0x0420400400000040, 0x0020400400100000, 0x0400000000000040, - 0x0420400400000040, 0x0400400400100040, 0x0020000000000000, 0x0000000000100000, - 0x0420400400000040, 0x0000400400100000, 0x0400400400100040, 0x0420000000000040, - 0x0000400400000000, 0x0020000000000000, 0x0000000000100000, 0x0400400400100040, - 0x0420000000100040, 0x0420400400000040, 0x0020400400000000, 0x0000000000000000, - 0x0020000000000000, 0x0400000000100040, 0x0400000000000040, 0x0020000000100000, - 0x0000000000000000, 0x0420000000100040, 0x0020000000100000, 0x0020400400000000, - 0x0420000000000040, 0x0000400400000000, 0x0420400400100040, 0x0000000000100000, - 0x0020400400100000, 0x0400000000000040, 0x0400400400000040, 0x0420400400100040, - 0x0400000000100040, 0x0020400400100000, 0x0000400400100000, 0x0400400400000040, ), - ( 0x8008000080082000, 0x0000002080082000, 0x8008002000000000, 0x0000000000000000, - 0x0000002000002000, 0x8008000080080000, 0x0000000080082000, 0x8008002080082000, - 0x8008000000000000, 0x0000000000002000, 0x0000002080080000, 0x8008002000000000, - 0x8008002080080000, 0x8008002000002000, 0x8008000000002000, 0x0000000080082000, - 0x0000002000000000, 0x8008002080080000, 0x8008000080080000, 0x0000002000002000, - 0x8008002080082000, 0x8008000000002000, 0x0000000000000000, 0x0000002080080000, - 0x0000000000002000, 0x0000000080080000, 0x8008002000002000, 0x8008000080082000, - 0x0000000080080000, 0x0000002000000000, 0x0000002080082000, 0x8008000000000000, - 0x0000000080080000, 0x0000002000000000, 0x8008000000002000, 0x8008002080082000, - 0x8008002000000000, 0x0000000000002000, 0x0000000000000000, 0x0000002080080000, - 0x8008000080082000, 0x8008002000002000, 0x0000002000002000, 0x8008000080080000, - 0x0000002080082000, 0x8008000000000000, 0x8008000080080000, 0x0000002000002000, - 0x8008002080082000, 0x0000000080080000, 0x0000000080082000, 0x8008000000002000, - 0x0000002080080000, 0x8008002000000000, 0x8008002000002000, 0x0000000080082000, - 0x8008000000000000, 0x0000002080082000, 0x8008002080080000, 0x0000000000000000, - 0x0000000000002000, 0x8008000080082000, 0x0000002000000000, 0x8008002080080000, ), - ) - - #--------------------------------------------------------------- - # compressed/interleaved => final permutation table - # Compression, final permutation, bit reverse - #--------------------------------------------------------------- - # NOTE: this was reordered from original table to make perm6464 logic simpler - CF6464=( - ( 0x0000000000000000, 0x0000002000000000, 0x0000200000000000, 0x0000202000000000, - 0x0020000000000000, 0x0020002000000000, 0x0020200000000000, 0x0020202000000000, - 0x2000000000000000, 0x2000002000000000, 0x2000200000000000, 0x2000202000000000, - 0x2020000000000000, 0x2020002000000000, 0x2020200000000000, 0x2020202000000000, ), - ( 0x0000000000000000, 0x0000000200000000, 0x0000020000000000, 0x0000020200000000, - 0x0002000000000000, 0x0002000200000000, 0x0002020000000000, 0x0002020200000000, - 0x0200000000000000, 0x0200000200000000, 0x0200020000000000, 0x0200020200000000, - 0x0202000000000000, 0x0202000200000000, 0x0202020000000000, 0x0202020200000000, ), - ( 0x0000000000000000, 0x0000000000000020, 0x0000000000002000, 0x0000000000002020, - 0x0000000000200000, 0x0000000000200020, 0x0000000000202000, 0x0000000000202020, - 0x0000000020000000, 0x0000000020000020, 0x0000000020002000, 0x0000000020002020, - 0x0000000020200000, 0x0000000020200020, 0x0000000020202000, 0x0000000020202020, ), - ( 0x0000000000000000, 0x0000000000000002, 0x0000000000000200, 0x0000000000000202, - 0x0000000000020000, 0x0000000000020002, 0x0000000000020200, 0x0000000000020202, - 0x0000000002000000, 0x0000000002000002, 0x0000000002000200, 0x0000000002000202, - 0x0000000002020000, 0x0000000002020002, 0x0000000002020200, 0x0000000002020202, ), - ( 0x0000000000000000, 0x0000008000000000, 0x0000800000000000, 0x0000808000000000, - 0x0080000000000000, 0x0080008000000000, 0x0080800000000000, 0x0080808000000000, - 0x8000000000000000, 0x8000008000000000, 0x8000800000000000, 0x8000808000000000, - 0x8080000000000000, 0x8080008000000000, 0x8080800000000000, 0x8080808000000000, ), - ( 0x0000000000000000, 0x0000000800000000, 0x0000080000000000, 0x0000080800000000, - 0x0008000000000000, 0x0008000800000000, 0x0008080000000000, 0x0008080800000000, - 0x0800000000000000, 0x0800000800000000, 0x0800080000000000, 0x0800080800000000, - 0x0808000000000000, 0x0808000800000000, 0x0808080000000000, 0x0808080800000000, ), - ( 0x0000000000000000, 0x0000000000000080, 0x0000000000008000, 0x0000000000008080, - 0x0000000000800000, 0x0000000000800080, 0x0000000000808000, 0x0000000000808080, - 0x0000000080000000, 0x0000000080000080, 0x0000000080008000, 0x0000000080008080, - 0x0000000080800000, 0x0000000080800080, 0x0000000080808000, 0x0000000080808080, ), - ( 0x0000000000000000, 0x0000000000000008, 0x0000000000000800, 0x0000000000000808, - 0x0000000000080000, 0x0000000000080008, 0x0000000000080800, 0x0000000000080808, - 0x0000000008000000, 0x0000000008000008, 0x0000000008000800, 0x0000000008000808, - 0x0000000008080000, 0x0000000008080008, 0x0000000008080800, 0x0000000008080808, ), - ( 0x0000000000000000, 0x0000001000000000, 0x0000100000000000, 0x0000101000000000, - 0x0010000000000000, 0x0010001000000000, 0x0010100000000000, 0x0010101000000000, - 0x1000000000000000, 0x1000001000000000, 0x1000100000000000, 0x1000101000000000, - 0x1010000000000000, 0x1010001000000000, 0x1010100000000000, 0x1010101000000000, ), - ( 0x0000000000000000, 0x0000000100000000, 0x0000010000000000, 0x0000010100000000, - 0x0001000000000000, 0x0001000100000000, 0x0001010000000000, 0x0001010100000000, - 0x0100000000000000, 0x0100000100000000, 0x0100010000000000, 0x0100010100000000, - 0x0101000000000000, 0x0101000100000000, 0x0101010000000000, 0x0101010100000000, ), - ( 0x0000000000000000, 0x0000000000000010, 0x0000000000001000, 0x0000000000001010, - 0x0000000000100000, 0x0000000000100010, 0x0000000000101000, 0x0000000000101010, - 0x0000000010000000, 0x0000000010000010, 0x0000000010001000, 0x0000000010001010, - 0x0000000010100000, 0x0000000010100010, 0x0000000010101000, 0x0000000010101010, ), - ( 0x0000000000000000, 0x0000000000000001, 0x0000000000000100, 0x0000000000000101, - 0x0000000000010000, 0x0000000000010001, 0x0000000000010100, 0x0000000000010101, - 0x0000000001000000, 0x0000000001000001, 0x0000000001000100, 0x0000000001000101, - 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x0000000001010101, ), - ( 0x0000000000000000, 0x0000004000000000, 0x0000400000000000, 0x0000404000000000, - 0x0040000000000000, 0x0040004000000000, 0x0040400000000000, 0x0040404000000000, - 0x4000000000000000, 0x4000004000000000, 0x4000400000000000, 0x4000404000000000, - 0x4040000000000000, 0x4040004000000000, 0x4040400000000000, 0x4040404000000000, ), - ( 0x0000000000000000, 0x0000000400000000, 0x0000040000000000, 0x0000040400000000, - 0x0004000000000000, 0x0004000400000000, 0x0004040000000000, 0x0004040400000000, - 0x0400000000000000, 0x0400000400000000, 0x0400040000000000, 0x0400040400000000, - 0x0404000000000000, 0x0404000400000000, 0x0404040000000000, 0x0404040400000000, ), - ( 0x0000000000000000, 0x0000000000000040, 0x0000000000004000, 0x0000000000004040, - 0x0000000000400000, 0x0000000000400040, 0x0000000000404000, 0x0000000000404040, - 0x0000000040000000, 0x0000000040000040, 0x0000000040004000, 0x0000000040004040, - 0x0000000040400000, 0x0000000040400040, 0x0000000040404000, 0x0000000040404040, ), - ( 0x0000000000000000, 0x0000000000000004, 0x0000000000000400, 0x0000000000000404, - 0x0000000000040000, 0x0000000000040004, 0x0000000000040400, 0x0000000000040404, - 0x0000000004000000, 0x0000000004000004, 0x0000000004000400, 0x0000000004000404, - 0x0000000004040000, 0x0000000004040004, 0x0000000004040400, 0x0000000004040404, ), - ) - #=================================================================== - # eof _load_tables() - #=================================================================== - -#============================================================================= -# support -#============================================================================= - -def _permute(c, p): - """Returns the permutation of the given 32-bit or 64-bit code with - the specified permutation table.""" - # NOTE: only difference between 32 & 64 bit permutations - # is that len(p)==8 for 32 bit, and len(p)==16 for 64 bit. - out = 0 - for r in p: - out |= r[c&0xf] - c >>= 4 - return out - -#============================================================================= -# packing & unpacking -#============================================================================= -# FIXME: more properly named _uint8_struct... -_uint64_struct = struct.Struct(">Q") - -def _pack64(value): - return _uint64_struct.pack(value) - -def _unpack64(value): - return _uint64_struct.unpack(value)[0] - -def _pack56(value): - return _uint64_struct.pack(value)[1:] - -def _unpack56(value): - return _uint64_struct.unpack(b'\x00' + value)[0] - -#============================================================================= -# 56->64 key manipulation -#============================================================================= - -##def expand_7bit(value): -## "expand 7-bit integer => 7-bits + 1 odd-parity bit" -## # parity calc adapted from 32-bit even parity alg found at -## # http://graphics.stanford.edu/~seander/bithacks.html#ParityParallel -## assert 0 <= value < 0x80, "value out of range" -## return (value<<1) | (0x9669 >> ((value ^ (value >> 4)) & 0xf)) & 1 - -_EXPAND_ITER = irange(49,-7,-7) - -def expand_des_key(key): - """convert DES from 7 bytes to 8 bytes (by inserting empty parity bits)""" - if isinstance(key, bytes): - if len(key) != 7: - raise ValueError("key must be 7 bytes in size") - elif isinstance(key, int_types): - if key < 0 or key > INT_56_MASK: - raise ValueError("key must be 56-bit non-negative integer") - return _unpack64(expand_des_key(_pack56(key))) - else: - raise exc.ExpectedTypeError(key, "bytes or int", "key") - key = _unpack56(key) - # NOTE: the following would insert correctly-valued parity bits in each key, - # but the parity bit would just be ignored in des_encrypt_block(), - # so not bothering to use it. - # XXX: could make parity-restoring optionally available via flag - ##return join_byte_values(expand_7bit((key >> shift) & 0x7f) - ## for shift in _EXPAND_ITER) - return join_byte_values(((key>>shift) & 0x7f)<<1 for shift in _EXPAND_ITER) - -def shrink_des_key(key): - """convert DES key from 8 bytes to 7 bytes (by discarding the parity bits)""" - if isinstance(key, bytes): - if len(key) != 8: - raise ValueError("key must be 8 bytes in size") - return _pack56(shrink_des_key(_unpack64(key))) - elif isinstance(key, int_types): - if key < 0 or key > INT_64_MASK: - raise ValueError("key must be 64-bit non-negative integer") - else: - raise exc.ExpectedTypeError(key, "bytes or int", "key") - key >>= 1 - result = 0 - offset = 0 - while offset < 56: - result |= (key & 0x7f)<>= 8 - offset += 7 - assert not (result & ~INT_64_MASK) - return result - -#============================================================================= -# des encryption -#============================================================================= -def des_encrypt_block(key, input, salt=0, rounds=1): - """encrypt single block of data using DES, operates on 8-byte strings. - - :arg key: - DES key as 7 byte string, or 8 byte string with parity bits - (parity bit values are ignored). - - :arg input: - plaintext block to encrypt, as 8 byte string. - - :arg salt: - Optional 24-bit integer used to mutate the base DES algorithm in a - manner specific to :class:`~passlib.hash.des_crypt` and its variants. - The default value ``0`` provides the normal (unsalted) DES behavior. - The salt functions as follows: - if the ``i``'th bit of ``salt`` is set, - bits ``i`` and ``i+24`` are swapped in the DES E-box output. - - :arg rounds: - Optional number of rounds of to apply the DES key schedule. - the default (``rounds=1``) provides the normal DES behavior, - but :class:`~passlib.hash.des_crypt` and its variants use - alternate rounds values. - - :raises TypeError: if any of the provided args are of the wrong type. - :raises ValueError: - if any of the input blocks are the wrong size, - or the salt/rounds values are out of range. - - :returns: - resulting 8-byte ciphertext block. - """ - # validate & unpack key - if isinstance(key, bytes): - if len(key) == 7: - key = expand_des_key(key) - elif len(key) != 8: - raise ValueError("key must be 7 or 8 bytes") - key = _unpack64(key) - else: - raise exc.ExpectedTypeError(key, "bytes", "key") - - # validate & unpack input - if isinstance(input, bytes): - if len(input) != 8: - raise ValueError("input block must be 8 bytes") - input = _unpack64(input) - else: - raise exc.ExpectedTypeError(input, "bytes", "input") - - # hand things off to other func - result = des_encrypt_int_block(key, input, salt, rounds) - - # repack result - return _pack64(result) - -def des_encrypt_int_block(key, input, salt=0, rounds=1): - """encrypt single block of data using DES, operates on 64-bit integers. - - this function is essentially the same as :func:`des_encrypt_block`, - except that it operates on integers, and will NOT automatically - expand 56-bit keys if provided (since there's no way to detect them). - - :arg key: - DES key as 64-bit integer (the parity bits are ignored). - - :arg input: - input block as 64-bit integer - - :arg salt: - optional 24-bit integer used to mutate the base DES algorithm. - defaults to ``0`` (no mutation applied). - - :arg rounds: - optional number of rounds of to apply the DES key schedule. - defaults to ``1``. - - :raises TypeError: if any of the provided args are of the wrong type. - :raises ValueError: - if any of the input blocks are the wrong size, - or the salt/rounds values are out of range. - - :returns: - resulting ciphertext as 64-bit integer. - """ - #--------------------------------------------------------------- - # input validation - #--------------------------------------------------------------- - - # validate salt, rounds - if rounds < 1: - raise ValueError("rounds must be positive integer") - if salt < 0 or salt > INT_24_MASK: - raise ValueError("salt must be 24-bit non-negative integer") - - # validate & unpack key - if not isinstance(key, int_types): - raise exc.ExpectedTypeError(key, "int", "key") - elif key < 0 or key > INT_64_MASK: - raise ValueError("key must be 64-bit non-negative integer") - - # validate & unpack input - if not isinstance(input, int_types): - raise exc.ExpectedTypeError(input, "int", "input") - elif input < 0 or input > INT_64_MASK: - raise ValueError("input must be 64-bit non-negative integer") - - #--------------------------------------------------------------- - # DES setup - #--------------------------------------------------------------- - # load tables if not already done - global SPE, PCXROT, IE3264, CF6464 - if PCXROT is None: - _load_tables() - - # load SPE into local vars to speed things up and remove an array access call - SPE0, SPE1, SPE2, SPE3, SPE4, SPE5, SPE6, SPE7 = SPE - - # NOTE: parity bits are ignored completely - # (UTs do fuzz testing to ensure this) - - # generate key schedule - # NOTE: generation was modified to output two elements at a time, - # so that per-round loop could do two passes at once. - def _iter_key_schedule(ks_odd): - """given 64-bit key, iterates over the 8 (even,odd) key schedule pairs""" - for p_even, p_odd in PCXROT: - ks_even = _permute(ks_odd, p_even) - ks_odd = _permute(ks_even, p_odd) - yield ks_even & _KS_MASK, ks_odd & _KS_MASK - ks_list = list(_iter_key_schedule(key)) - - # expand 24 bit salt -> 32 bit per des_crypt & bsdi_crypt - salt = ( - ((salt & 0x00003f) << 26) | - ((salt & 0x000fc0) << 12) | - ((salt & 0x03f000) >> 2) | - ((salt & 0xfc0000) >> 16) - ) - - # init L & R - if input == 0: - L = R = 0 - else: - L = ((input >> 31) & 0xaaaaaaaa) | (input & 0x55555555) - L = _permute(L, IE3264) - - R = ((input >> 32) & 0xaaaaaaaa) | ((input >> 1) & 0x55555555) - R = _permute(R, IE3264) - - #--------------------------------------------------------------- - # main DES loop - run for specified number of rounds - #--------------------------------------------------------------- - while rounds: - rounds -= 1 - - # run over each part of the schedule, 2 parts at a time - for ks_even, ks_odd in ks_list: - k = ((R>>32) ^ R) & salt # use the salt to flip specific bits - B = (k<<32) ^ k ^ R ^ ks_even - - L ^= (SPE0[(B>>58)&0x3f] ^ SPE1[(B>>50)&0x3f] ^ - SPE2[(B>>42)&0x3f] ^ SPE3[(B>>34)&0x3f] ^ - SPE4[(B>>26)&0x3f] ^ SPE5[(B>>18)&0x3f] ^ - SPE6[(B>>10)&0x3f] ^ SPE7[(B>>2)&0x3f]) - - k = ((L>>32) ^ L) & salt # use the salt to flip specific bits - B = (k<<32) ^ k ^ L ^ ks_odd - - R ^= (SPE0[(B>>58)&0x3f] ^ SPE1[(B>>50)&0x3f] ^ - SPE2[(B>>42)&0x3f] ^ SPE3[(B>>34)&0x3f] ^ - SPE4[(B>>26)&0x3f] ^ SPE5[(B>>18)&0x3f] ^ - SPE6[(B>>10)&0x3f] ^ SPE7[(B>>2)&0x3f]) - - # swap L and R - L, R = R, L - - #--------------------------------------------------------------- - # return final result - #--------------------------------------------------------------- - C = ( - ((L>>3) & 0x0f0f0f0f00000000) - | - ((L<<33) & 0xf0f0f0f000000000) - | - ((R>>35) & 0x000000000f0f0f0f) - | - ((R<<1) & 0x00000000f0f0f0f0) - ) - return _permute(C, CF6464) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/digest.py b/src/passlib/crypto/digest.py deleted file mode 100644 index d26f8927..00000000 --- a/src/passlib/crypto/digest.py +++ /dev/null @@ -1,891 +0,0 @@ -"""passlib.crypto.digest -- crytographic helpers used by the password hashes in passlib - -.. versionadded:: 1.7 -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import division -# core -import hashlib -import logging; log = logging.getLogger(__name__) -try: - # new in py3.4 - from hashlib import pbkdf2_hmac as _stdlib_pbkdf2_hmac - if _stdlib_pbkdf2_hmac.__module__ == "hashlib": - # builtin pure-python backends are slightly faster than stdlib's pure python fallback, - # so only using stdlib's version if it's backed by openssl's pbkdf2_hmac() - log.debug("ignoring pure-python hashlib.pbkdf2_hmac()") - _stdlib_pbkdf2_hmac = None -except ImportError: - _stdlib_pbkdf2_hmac = None -import re -import os -from struct import Struct -from warnings import warn -# site -try: - # https://pypi.python.org/pypi/fastpbkdf2/ - from fastpbkdf2 import pbkdf2_hmac as _fast_pbkdf2_hmac -except ImportError: - _fast_pbkdf2_hmac = None -# pkg -from passlib import exc -from passlib.utils import join_bytes, to_native_str, join_byte_values, to_bytes, \ - SequenceMixin -from passlib.utils.compat import irange, int_types, unicode_or_bytes_types, PY3 -from passlib.utils.decor import memoized_property -# local -__all__ = [ - # hash utils - "lookup_hash", - "HashInfo", - "norm_hash_name", - - # hmac utils - "compile_hmac", - - # kdfs - "pbkdf1", - "pbkdf2_hmac", -] - -#============================================================================= -# generic constants -#============================================================================= - -#: max 32-bit value -MAX_UINT32 = (1 << 32) - 1 - -#: max 64-bit value -MAX_UINT64 = (1 << 64) - 1 - -#============================================================================= -# hash utils -#============================================================================= - -#: list of known hash names, used by lookup_hash()'s _norm_hash_name() helper -_known_hash_names = [ - # format: (hashlib/ssl name, iana name or standin, other known aliases ...) - - # hashes with official IANA-assigned names - # (as of 2012-03 - http://www.iana.org/assignments/hash-function-text-names) - ("md2", "md2"), - ("md5", "md5"), - ("sha1", "sha-1"), - ("sha224", "sha-224", "sha2-224"), - ("sha256", "sha-256", "sha2-256"), - ("sha384", "sha-384", "sha2-384"), - ("sha512", "sha-512", "sha2-512"), - - # TODO: add sha3 to this table. - - # hashlib/ssl-supported hashes without official IANA names, - # (hopefully-) compatible stand-ins have been chosen. - ("md4", "md4"), - ("sha", "sha-0", "sha0"), - ("ripemd", "ripemd"), - ("ripemd160", "ripemd-160"), -] - -#: cache of hash info instances used by lookup_hash() -_hash_info_cache = {} - -def _get_hash_aliases(name): - """ - internal helper used by :func:`lookup_hash` -- - normalize arbitrary hash name to hashlib format. - if name not recognized, returns dummy record and issues a warning. - - :arg name: - unnormalized name - - :returns: - tuple with 2+ elements: ``(hashlib_name, iana_name|None, ... 0+ aliases)``. - """ - - # normalize input - orig = name - if not isinstance(name, str): - name = to_native_str(name, 'utf-8', 'hash name') - name = re.sub("[_ /]", "-", name.strip().lower()) - if name.startswith("scram-"): # helper for SCRAM protocol (see passlib.handlers.scram) - name = name[6:] - if name.endswith("-plus"): - name = name[:-5] - - # look through standard names and known aliases - def check_table(name): - for row in _known_hash_names: - if name in row: - return row - result = check_table(name) - if result: - return result - - # try to clean name up some more - m = re.match(r"(?i)^(?P[a-z]+)-?(?P\d)?-?(?P\d{3,4})?$", name) - if m: - # roughly follows "SHA2-256" style format, normalize representation, - # and checked table. - iana_name, rev, size = m.group("name", "rev", "size") - if rev: - iana_name += rev - hashlib_name = iana_name - if size: - iana_name += "-" + size - if rev: - hashlib_name += "_" - hashlib_name += size - result = check_table(iana_name) - if result: - return result - - # not found in table, but roughly recognize format. use names we built up as fallback. - log.info("normalizing unrecognized hash name %r => %r / %r", - orig, hashlib_name, iana_name) - - else: - # just can't make sense of it. return something - iana_name = name - hashlib_name = name.replace("-", "_") - log.warning("normalizing unrecognized hash name and format %r => %r / %r", - orig, hashlib_name, iana_name) - - return hashlib_name, iana_name - - -def _get_hash_const(name): - """ - internal helper used by :func:`lookup_hash` -- - lookup hash constructor by name - - :arg name: - name (normalized to hashlib format, e.g. ``"sha256"``) - - :returns: - hash constructor, e.g. ``hashlib.sha256()``; - or None if hash can't be located. - """ - # check hashlib. for an efficient constructor - if not name.startswith("_") and name not in ("new", "algorithms"): - try: - return getattr(hashlib, name) - except AttributeError: - pass - - # check hashlib.new() in case SSL supports the digest - new_ssl_hash = hashlib.new - try: - # new() should throw ValueError if alg is unknown - new_ssl_hash(name, b"") - except ValueError: - pass - else: - # create wrapper function - # XXX: is there a faster way to wrap this? - def const(msg=b""): - return new_ssl_hash(name, msg) - const.__name__ = name - const.__module__ = "hashlib" - const.__doc__ = ("wrapper for hashlib.new(%r),\n" - "generated by passlib.crypto.digest.lookup_hash()") % name - return const - - # use builtin md4 as fallback when not supported by hashlib - if name == "md4": - from passlib.crypto._md4 import md4 - return md4 - - # XXX: any other modules / registries we should check? - # TODO: add pysha3 support. - - return None - -def lookup_hash(digest, return_unknown=False): - """ - Returns a :class:`HashInfo` record containing information about a given hash function. - Can be used to look up a hash constructor by name, normalize hash name representation, etc. - - :arg digest: - This can be any of: - - * A string containing a :mod:`!hashlib` digest name (e.g. ``"sha256"``), - * A string containing an IANA-assigned hash name, - * A digest constructor function (e.g. ``hashlib.sha256``). - - Case is ignored, underscores are converted to hyphens, - and various other cleanups are made. - - :param return_unknown: - By default, this function will throw an :exc:`~passlib.exc.UnknownHashError` if no hash constructor - can be found. However, if this flag is False, it will instead return a dummy record - without a constructor function. This is mainly used by :func:`norm_hash_name`. - - :returns HashInfo: - :class:`HashInfo` instance containing information about specified digest. - - Multiple calls resolving to the same hash should always - return the same :class:`!HashInfo` instance. - """ - # check for cached entry - cache = _hash_info_cache - try: - return cache[digest] - except (KeyError, TypeError): - # NOTE: TypeError is to catch 'TypeError: unhashable type' (e.g. HashInfo) - pass - - # resolve ``digest`` to ``const`` & ``name_record`` - cache_by_name = True - if isinstance(digest, unicode_or_bytes_types): - # normalize name - name_list = _get_hash_aliases(digest) - name = name_list[0] - assert name - - # if name wasn't normalized to hashlib format, - # get info for normalized name and reuse it. - if name != digest: - info = lookup_hash(name, return_unknown=return_unknown) - if info.const is None: - # pass through dummy record - assert return_unknown - return info - cache[digest] = info - return info - - # else look up constructor - const = _get_hash_const(name) - if const is None: - if return_unknown: - # return a dummy record (but don't cache it, so normal lookup still returns error) - return HashInfo(None, name_list) - else: - raise exc.UnknownHashError(name) - - elif isinstance(digest, HashInfo): - # handle border case where HashInfo is passed in. - return digest - - elif callable(digest): - # try to lookup digest based on it's self-reported name - # (which we trust to be the canonical "hashlib" name) - const = digest - name_list = _get_hash_aliases(const().name) - name = name_list[0] - other_const = _get_hash_const(name) - if other_const is None: - # this is probably a third-party digest we don't know about, - # so just pass it on through, and register reverse lookup for it's name. - pass - - elif other_const is const: - # if we got back same constructor, this is just a known stdlib constructor, - # which was passed in before we had cached it by name. proceed normally. - pass - - else: - # if we got back different object, then ``const`` is something else - # (such as a mock object), in which case we want to skip caching it by name, - # as that would conflict with real hash. - cache_by_name = False - - else: - raise exc.ExpectedTypeError(digest, "digest name or constructor", "digest") - - # create new instance - info = HashInfo(const, name_list) - - # populate cache - cache[const] = info - if cache_by_name: - for name in name_list: - if name: # (skips iana name if it's empty) - assert cache.get(name) in [None, info], "%r already in cache" % name - cache[name] = info - return info - -#: UT helper for clearing internal cache -lookup_hash.clear_cache = _hash_info_cache.clear - - -def norm_hash_name(name, format="hashlib"): - """Normalize hash function name (convenience wrapper for :func:`lookup_hash`). - - :arg name: - Original hash function name. - - This name can be a Python :mod:`~hashlib` digest name, - a SCRAM mechanism name, IANA assigned hash name, etc. - Case is ignored, and underscores are converted to hyphens. - - :param format: - Naming convention to normalize to. - Possible values are: - - * ``"hashlib"`` (the default) - normalizes name to be compatible - with Python's :mod:`!hashlib`. - - * ``"iana"`` - normalizes name to IANA-assigned hash function name. - For hashes which IANA hasn't assigned a name for, this issues a warning, - and then uses a heuristic to return a "best guess" name. - - :returns: - Hash name, returned as native :class:`!str`. - """ - info = lookup_hash(name, return_unknown=True) - if not info.const: - warn("norm_hash_name(): unknown hash: %r" % (name,), exc.PasslibRuntimeWarning) - if format == "hashlib": - return info.name - elif format == "iana": - return info.iana_name - else: - raise ValueError("unknown format: %r" % (format,)) - - -class HashInfo(SequenceMixin): - """ - Record containing information about a given hash algorithm, as returned :func:`lookup_hash`. - - This class exposes the following attributes: - - .. autoattribute:: const - .. autoattribute:: digest_size - .. autoattribute:: block_size - .. autoattribute:: name - .. autoattribute:: iana_name - .. autoattribute:: aliases - - This object can also be treated a 3-element sequence - containing ``(const, digest_size, block_size)``. - """ - #========================================================================= - # instance attrs - #========================================================================= - - #: Canonical / hashlib-compatible name (e.g. ``"sha256"``). - name = None - - #: IANA assigned name (e.g. ``"sha-256"``), may be ``None`` if unknown. - iana_name = None - - #: Tuple of other known aliases (may be empty) - aliases = () - - #: Hash constructor function (e.g. :func:`hashlib.sha256`) - const = None - - #: Hash's digest size - digest_size = None - - #: Hash's block size - block_size = None - - def __init__(self, const, names): - """ - initialize new instance. - :arg const: - hash constructor - :arg names: - list of 2+ names. should be list of ``(name, iana_name, ... 0+ aliases)``. - names must be lower-case. only iana name may be None. - """ - self.name = names[0] - self.iana_name = names[1] - self.aliases = names[2:] - - self.const = const - if const is None: - return - - hash = const() - self.digest_size = hash.digest_size - self.block_size = hash.block_size - - # do sanity check on digest size - if len(hash.digest()) != hash.digest_size: - raise RuntimeError("%r constructor failed sanity check" % self.name) - - # do sanity check on name. - if hash.name != self.name: - warn("inconsistent digest name: %r resolved to %r, which reports name as %r" % - (self.name, const, hash.name), exc.PasslibRuntimeWarning) - - #========================================================================= - # methods - #========================================================================= - def __repr__(self): - return " digest output``. - - However, if ``multipart=True``, the returned function has the signature - ``hmac() -> update, finalize``, where ``update(msg)`` may be called multiple times, - and ``finalize() -> digest_output`` may be repeatedly called at any point to - calculate the HMAC digest so far. - - The returned object will also have a ``digest_info`` attribute, containing - a :class:`lookup_hash` instance for the specified digest. - - This function exists, and has the weird signature it does, in order to squeeze as - provide as much efficiency as possible, by omitting much of the setup cost - and features of the stdlib :mod:`hmac` module. - """ - # all the following was adapted from stdlib's hmac module - - # resolve digest (cached) - digest_info = lookup_hash(digest) - const, digest_size, block_size = digest_info - assert block_size >= 16, "block size too small" - - # prepare key - if not isinstance(key, bytes): - key = to_bytes(key, param="key") - klen = len(key) - if klen > block_size: - key = const(key).digest() - klen = digest_size - if klen < block_size: - key += b'\x00' * (block_size - klen) - - # create pre-initialized hash constructors - _inner_copy = const(key.translate(_TRANS_36)).copy - _outer_copy = const(key.translate(_TRANS_5C)).copy - - if multipart: - # create multi-part function - # NOTE: this is slightly slower than the single-shot version, - # and should only be used if needed. - def hmac(): - """generated by compile_hmac(multipart=True)""" - inner = _inner_copy() - def finalize(): - outer = _outer_copy() - outer.update(inner.digest()) - return outer.digest() - return inner.update, finalize - else: - - # single-shot function - def hmac(msg): - """generated by compile_hmac()""" - inner = _inner_copy() - inner.update(msg) - outer = _outer_copy() - outer.update(inner.digest()) - return outer.digest() - - # add info attr - hmac.digest_info = digest_info - return hmac - -#============================================================================= -# pbkdf1 -#============================================================================= -def pbkdf1(digest, secret, salt, rounds, keylen=None): - """pkcs#5 password-based key derivation v1.5 - - :arg digest: - digest name or constructor. - - :arg secret: - secret to use when generating the key. - may be :class:`!bytes` or :class:`unicode` (encoded using UTF-8). - - :arg salt: - salt string to use when generating key. - may be :class:`!bytes` or :class:`unicode` (encoded using UTF-8). - - :param rounds: - number of rounds to use to generate key. - - :arg keylen: - number of bytes to generate (if omitted / ``None``, uses digest's native size) - - :returns: - raw :class:`bytes` of generated key - - .. note:: - - This algorithm has been deprecated, new code should use PBKDF2. - Among other limitations, ``keylen`` cannot be larger - than the digest size of the specified hash. - """ - # resolve digest - const, digest_size, block_size = lookup_hash(digest) - - # validate secret & salt - secret = to_bytes(secret, param="secret") - salt = to_bytes(salt, param="salt") - - # validate rounds - if not isinstance(rounds, int_types): - raise exc.ExpectedTypeError(rounds, "int", "rounds") - if rounds < 1: - raise ValueError("rounds must be at least 1") - - # validate keylen - if keylen is None: - keylen = digest_size - elif not isinstance(keylen, int_types): - raise exc.ExpectedTypeError(keylen, "int or None", "keylen") - elif keylen < 0: - raise ValueError("keylen must be at least 0") - elif keylen > digest_size: - raise ValueError("keylength too large for digest: %r > %r" % - (keylen, digest_size)) - - # main pbkdf1 loop - block = secret + salt - for _ in irange(rounds): - block = const(block).digest() - return block[:keylen] - -#============================================================================= -# pbkdf2 -#============================================================================= - -_pack_uint32 = Struct(">L").pack - -def pbkdf2_hmac(digest, secret, salt, rounds, keylen=None): - """pkcs#5 password-based key derivation v2.0 using HMAC + arbitrary digest. - - :arg digest: - digest name or constructor. - - :arg secret: - passphrase to use to generate key. - may be :class:`!bytes` or :class:`unicode` (encoded using UTF-8). - - :arg salt: - salt string to use when generating key. - may be :class:`!bytes` or :class:`unicode` (encoded using UTF-8). - - :param rounds: - number of rounds to use to generate key. - - :arg keylen: - number of bytes to generate. - if omitted / ``None``, will use digest's native output size. - - :returns: - raw bytes of generated key - - .. versionchanged:: 1.7 - - This function will use the first available of the following backends: - - * `fastpbk2 `_ - * :func:`hashlib.pbkdf2_hmac` (only available in py2 >= 2.7.8, and py3 >= 3.4) - * builtin pure-python backend - - See :data:`passlib.crypto.digest.PBKDF2_BACKENDS` to determine - which backend(s) are in use. - """ - # validate secret & salt - secret = to_bytes(secret, param="secret") - salt = to_bytes(salt, param="salt") - - # resolve digest - digest_info = lookup_hash(digest) - digest_size = digest_info.digest_size - - # validate rounds - if not isinstance(rounds, int_types): - raise exc.ExpectedTypeError(rounds, "int", "rounds") - if rounds < 1: - raise ValueError("rounds must be at least 1") - - # validate keylen - if keylen is None: - keylen = digest_size - elif not isinstance(keylen, int_types): - raise exc.ExpectedTypeError(keylen, "int or None", "keylen") - elif keylen < 1: - # XXX: could allow keylen=0, but want to be compat w/ stdlib - raise ValueError("keylen must be at least 1") - - # find smallest block count s.t. keylen <= block_count * digest_size; - # make sure block count won't overflow (per pbkdf2 spec) - # this corresponds to throwing error if keylen > digest_size * MAX_UINT32 - # NOTE: stdlib will throw error at lower bound (keylen > MAX_SINT32) - # NOTE: have do this before other backends checked, since fastpbkdf2 raises wrong error - # (InvocationError, not OverflowError) - block_count = (keylen + digest_size - 1) // digest_size - if block_count > MAX_UINT32: - raise OverflowError("keylen too long for digest") - - # - # check for various high-speed backends - # - - # ~3x faster than pure-python backend - # NOTE: have to do this after above guards since fastpbkdf2 lacks bounds checks. - if digest_info.supported_by_fastpbkdf2: - return _fast_pbkdf2_hmac(digest_info.name, secret, salt, rounds, keylen) - - # ~1.4x faster than pure-python backend - # NOTE: have to do this after fastpbkdf2 since hashlib-ssl is slower, - # will support larger number of hashes. - if digest_info.supported_by_hashlib_pbkdf2: - return _stdlib_pbkdf2_hmac(digest_info.name, secret, salt, rounds, keylen) - - # - # otherwise use our own implementation - # - - # generated keyed hmac - keyed_hmac = compile_hmac(digest, secret) - - # get helper to calculate pbkdf2 inner loop efficiently - calc_block = _get_pbkdf2_looper(digest_size) - - # assemble & return result - return join_bytes( - calc_block(keyed_hmac, keyed_hmac(salt + _pack_uint32(i)), rounds) - for i in irange(1, block_count + 1) - )[:keylen] - -#------------------------------------------------------------------------------------- -# pick best choice for pure-python helper -# TODO: consider some alternatives, such as C-accelerated xor_bytes helper if available -#------------------------------------------------------------------------------------- -# NOTE: this env var is only present to support the admin/benchmark_pbkdf2 script -_force_backend = os.environ.get("PASSLIB_PBKDF2_BACKEND") or "any" - -if PY3 and _force_backend in ["any", "from-bytes"]: - from functools import partial - - def _get_pbkdf2_looper(digest_size): - return partial(_pbkdf2_looper, digest_size) - - def _pbkdf2_looper(digest_size, keyed_hmac, digest, rounds): - """ - py3-only implementation of pbkdf2 inner loop; - uses 'int.from_bytes' + integer XOR - """ - from_bytes = int.from_bytes - BIG = "big" # endianess doesn't matter, just has to be consistent - accum = from_bytes(digest, BIG) - for _ in irange(rounds - 1): - digest = keyed_hmac(digest) - accum ^= from_bytes(digest, BIG) - return accum.to_bytes(digest_size, BIG) - - _builtin_backend = "from-bytes" - -elif _force_backend in ["any", "unpack", "from-bytes"]: - from struct import Struct - from passlib.utils import sys_bits - - _have_64_bit = (sys_bits >= 64) - - #: cache used by _get_pbkdf2_looper - _looper_cache = {} - - def _get_pbkdf2_looper(digest_size): - """ - We want a helper function which performs equivalent of the following:: - - def helper(keyed_hmac, digest, rounds): - accum = digest - for _ in irange(rounds - 1): - digest = keyed_hmac(digest) - accum ^= digest - return accum - - However, no efficient way to implement "bytes ^ bytes" in python. - Instead, using approach where we dynamically compile a helper function based - on digest size. Instead of a single `accum` var, this helper breaks the digest - into a series of integers. - - It stores these in a series of`accum_` vars, and performs `accum ^= digest` - by unpacking digest and perform xor for each "accum_ ^= digest_". - this keeps everything in locals, avoiding excessive list creation, encoding or decoding, - etc. - - :param digest_size: - digest size to compile for, in bytes. (must be multiple of 4). - - :return: - helper function with call signature outlined above. - """ - # - # cache helpers - # - try: - return _looper_cache[digest_size] - except KeyError: - pass - - # - # figure out most efficient struct format to unpack digest into list of native ints - # - if _have_64_bit and not digest_size & 0x7: - # digest size multiple of 8, on a 64 bit system -- use array of UINT64 - count = (digest_size >> 3) - fmt = "=%dQ" % count - elif not digest_size & 0x3: - if _have_64_bit: - # digest size multiple of 4, on a 64 bit system -- use array of UINT64 + 1 UINT32 - count = (digest_size >> 3) - fmt = "=%dQI" % count - count += 1 - else: - # digest size multiple of 4, on a 32 bit system -- use array of UINT32 - count = (digest_size >> 2) - fmt = "=%dI" % count - else: - # stopping here, cause no known hashes have digest size that isn't multiple of 4 bytes. - # if needed, could go crazy w/ "H" & "B" - raise NotImplementedError("unsupported digest size: %d" % digest_size) - struct = Struct(fmt) - - # - # build helper source - # - tdict = dict( - digest_size=digest_size, - accum_vars=", ".join("acc_%d" % i for i in irange(count)), - digest_vars=", ".join("dig_%d" % i for i in irange(count)), - ) - - # head of function - source = ( - "def helper(keyed_hmac, digest, rounds):\n" - " '''pbkdf2 loop helper for digest_size={digest_size}'''\n" - " unpack_digest = struct.unpack\n" - " {accum_vars} = unpack_digest(digest)\n" - " for _ in irange(1, rounds):\n" - " digest = keyed_hmac(digest)\n" - " {digest_vars} = unpack_digest(digest)\n" - ).format(**tdict) - - # xor digest - for i in irange(count): - source += " acc_%d ^= dig_%d\n" % (i, i) - - # return result - source += " return struct.pack({accum_vars})\n".format(**tdict) - - # - # compile helper - # - code = compile(source, "", "exec") - gdict = dict(irange=irange, struct=struct) - ldict = dict() - eval(code, gdict, ldict) - helper = ldict['helper'] - if __debug__: - helper.__source__ = source - - # - # store in cache - # - _looper_cache[digest_size] = helper - return helper - - _builtin_backend = "unpack" - -else: - assert _force_backend in ["any", "hexlify"] - - # XXX: older & slower approach that used int(hexlify()), - # keeping it around for a little while just for benchmarking. - - from binascii import hexlify as _hexlify - from passlib.utils import int_to_bytes - - def _get_pbkdf2_looper(digest_size): - return _pbkdf2_looper - - def _pbkdf2_looper(keyed_hmac, digest, rounds): - hexlify = _hexlify - accum = int(hexlify(digest), 16) - for _ in irange(rounds - 1): - digest = keyed_hmac(digest) - accum ^= int(hexlify(digest), 16) - return int_to_bytes(accum, len(digest)) - - _builtin_backend = "hexlify" - -# helper for benchmark script -- disable hashlib, fastpbkdf2 support if builtin requested -if _force_backend == _builtin_backend: - _fast_pbkdf2_hmac = _stdlib_pbkdf2_hmac = None - -# expose info about what backends are active -PBKDF2_BACKENDS = [b for b in [ - "fastpbkdf2" if _fast_pbkdf2_hmac else None, - "hashlib-ssl" if _stdlib_pbkdf2_hmac else None, - "builtin-" + _builtin_backend -] if b] - -# *very* rough estimate of relative speed (compared to sha256 using 'unpack' backend on 64bit arch) -if "fastpbkdf2" in PBKDF2_BACKENDS: - PBKDF2_SPEED_FACTOR = 3 -elif "hashlib-ssl" in PBKDF2_BACKENDS: - PBKDF2_SPEED_FACTOR = 1.4 -else: - # remaining backends have *some* difference in performance, but not enough to matter - PBKDF2_SPEED_FACTOR = 1 - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/crypto/scrypt/__init__.py b/src/passlib/crypto/scrypt/__init__.py deleted file mode 100644 index 16b9feb4..00000000 --- a/src/passlib/crypto/scrypt/__init__.py +++ /dev/null @@ -1,210 +0,0 @@ -"""passlib.utils.scrypt -- scrypt hash frontend and help utilities""" -#========================================================================== -# imports -#========================================================================== -from __future__ import absolute_import -# core -import logging; log = logging.getLogger(__name__) -from warnings import warn -# pkg -from passlib import exc -from passlib.utils import to_bytes -from passlib.utils.compat import PYPY -# local -__all__ =[ - "validate", - "scrypt", -] - -#========================================================================== -# config validation -#========================================================================== - -#: max output length in bytes -MAX_KEYLEN = ((1 << 32) - 1) * 32 - -#: max ``r * p`` limit -MAX_RP = (1 << 30) - 1 - -# TODO: unittests for this function -def validate(n, r, p): - """ - helper which validates a set of scrypt config parameters. - scrypt will take ``O(n * r * p)`` time and ``O(n * r)`` memory. - limitations are that ``n = 2**``, ``n < 2**(16*r)``, ``r * p < 2 ** 30``. - - :param n: scrypt rounds - :param r: scrypt block size - :param p: scrypt parallel factor - """ - if r < 1: - raise ValueError("r must be > 0: r=%r" % r) - - if p < 1: - raise ValueError("p must be > 0: p=%r" % p) - - if r * p > MAX_RP: - # pbkdf2-hmac-sha256 limitation - it will be requested to generate ``p*(2*r)*64`` bytes, - # but pbkdf2 can do max of (2**31-1) blocks, and sha-256 has 32 byte block size... - # so ``(2**31-1)*32 >= p*r*128`` -> ``r*p < 2**30`` - raise ValueError("r * p must be < 2**30: r=%r, p=%r" % (r,p)) - - if n < 2 or n & (n - 1): - raise ValueError("n must be > 1, and a power of 2: n=%r" % n) - - return True - -# TODO: configuration picker (may need psutil for full effect) - -#========================================================================== -# hash frontend -#========================================================================== - -#: backend function used by scrypt(), filled in by _set_backend() -_scrypt = None - -#: name of backend currently in use, exposed for informational purposes. -backend = None - -def scrypt(secret, salt, n, r, p=1, keylen=32): - """run SCrypt key derivation function using specified parameters. - - :arg secret: - passphrase string (unicode is encoded to bytes using utf-8). - - :arg salt: - salt string (unicode is encoded to bytes using utf-8). - - :arg n: - integer 'N' parameter - - :arg r: - integer 'r' parameter - - :arg p: - integer 'p' parameter - - :arg keylen: - number of bytes of key to generate. - defaults to 32 (the internal block size). - - :returns: - a *keylen*-sized bytes instance - - SCrypt imposes a number of constraints on it's input parameters: - - * ``r * p < 2**30`` -- due to a limitation of PBKDF2-HMAC-SHA256. - * ``keylen < (2**32 - 1) * 32`` -- due to a limitation of PBKDF2-HMAC-SHA256. - * ``n`` must a be a power of 2, and > 1 -- internal limitation of scrypt() implementation - - :raises ValueError: if the provided parameters are invalid (see constraints above). - - .. warning:: - - Unless the third-party ``scrypt ``_ package - is installed, passlib will use a builtin pure-python implementation of scrypt, - which is *considerably* slower (and thus requires a much lower / less secure - ``n`` value in order to be usuable). Installing the :mod:`!scrypt` package - is strongly recommended. - """ - validate(n, r, p) - secret = to_bytes(secret, param="secret") - salt = to_bytes(salt, param="salt") - if keylen < 1: - raise ValueError("keylen must be at least 1") - if keylen > MAX_KEYLEN: - raise ValueError("keylen too large, must be <= %d" % MAX_KEYLEN) - return _scrypt(secret, salt, n, r, p, keylen) - - -def _load_builtin_backend(): - """ - Load pure-python scrypt implementation built into passlib. - """ - slowdown = 10 if PYPY else 100 - warn("Using builtin scrypt backend, which is %dx slower than is required " - "for adequate security. Installing scrypt support (via 'pip install scrypt') " - "is strongly recommended" % slowdown, exc.PasslibSecurityWarning) - from ._builtin import ScryptEngine - return ScryptEngine.execute - - -def _load_cffi_backend(): - """ - Try to import the ctypes-based scrypt hash function provided by the - ``scrypt ``_ package. - """ - try: - from scrypt import hash - return hash - except ImportError: - pass - # not available, but check to see if package present but outdated / not installed right - try: - import scrypt - except ImportError as err: - if "scrypt" not in str(err): - # e.g. if cffi isn't set up right - # user should try importing scrypt explicitly to diagnose problem. - warn("'scrypt' package failed to import correctly (possible installation issue?)", - exc.PasslibWarning) - # else: package just isn't installed - else: - warn("'scrypt' package is too old (lacks ``hash()`` method)", exc.PasslibWarning) - return None - - -#: list of potential backends -backend_values = ("scrypt", "builtin") - -#: dict mapping backend name -> loader -_backend_loaders = dict( - scrypt=_load_cffi_backend, # XXX: rename backend constant to "cffi"? - builtin=_load_builtin_backend, -) - - -def _set_backend(name, dryrun=False): - """ - set backend for scrypt(). if name not specified, loads first available. - - :raises ~passlib.exc.MissingBackendError: if backend can't be found - - .. note:: mainly intended to be called by unittests, and scrypt hash handler - """ - if name == "any": - return - elif name == "default": - for name in backend_values: - try: - return _set_backend(name, dryrun=dryrun) - except exc.MissingBackendError: - continue - raise exc.MissingBackendError("no scrypt backends available") - else: - loader = _backend_loaders.get(name) - if not loader: - raise ValueError("unknown scrypt backend: %r" % (name,)) - hash = loader() - if not hash: - raise exc.MissingBackendError("scrypt backend %r not available" % name) - if dryrun: - return - global _scrypt, backend - backend = name - _scrypt = hash - -# initialize backend -_set_backend("default") - - -def _has_backend(name): - try: - _set_backend(name, dryrun=True) - return True - except exc.MissingBackendError: - return False - -#========================================================================== -# eof -#========================================================================== diff --git a/src/passlib/crypto/scrypt/_builtin.py b/src/passlib/crypto/scrypt/_builtin.py deleted file mode 100644 index e9bb305d..00000000 --- a/src/passlib/crypto/scrypt/_builtin.py +++ /dev/null @@ -1,244 +0,0 @@ -"""passlib.utils.scrypt._builtin -- scrypt() kdf in pure-python""" -#========================================================================== -# imports -#========================================================================== -# core -import operator -import struct -# pkg -from passlib.utils.compat import izip -from passlib.crypto.digest import pbkdf2_hmac -from passlib.crypto.scrypt._salsa import salsa20 -# local -__all__ =[ - "ScryptEngine", -] - -#========================================================================== -# scrypt engine -#========================================================================== -class ScryptEngine(object): - """ - helper class used to run scrypt kdf, see scrypt() for frontend - - .. warning:: - this class does NO validation of the input ranges or types. - - it's not intended to be used directly, - but only as a backend for :func:`passlib.utils.scrypt.scrypt()`. - """ - #================================================================= - # instance attrs - #================================================================= - - # primary scrypt config parameters - n = 0 - r = 0 - p = 0 - - # derived values & objects - smix_bytes = 0 - iv_bytes = 0 - bmix_len = 0 - bmix_half_len = 0 - bmix_struct = None - integerify = None - - #================================================================= - # frontend - #================================================================= - @classmethod - def execute(cls, secret, salt, n, r, p, keylen): - """create engine & run scrypt() hash calculation""" - return cls(n, r, p).run(secret, salt, keylen) - - #================================================================= - # init - #================================================================= - def __init__(self, n, r, p): - # store config - self.n = n - self.r = r - self.p = p - self.smix_bytes = r << 7 # num bytes in smix input - 2*r*16*4 - self.iv_bytes = self.smix_bytes * p - self.bmix_len = bmix_len = r << 5 # length of bmix block list - 32*r integers - self.bmix_half_len = r << 4 - assert struct.calcsize("I") == 4 - self.bmix_struct = struct.Struct("<" + str(bmix_len) + "I") - - # use optimized bmix for certain cases - if r == 1: - self.bmix = self._bmix_1 - - # pick best integerify function - integerify(bmix_block) should - # take last 64 bytes of block and return a little-endian integer. - # since it's immediately converted % n, we only have to extract - # the first 32 bytes if n < 2**32 - which due to the current - # internal representation, is already unpacked as a 32-bit int. - if n <= 0xFFFFffff: - integerify = operator.itemgetter(-16) - else: - assert n <= 0xFFFFffffFFFFffff - ig1 = operator.itemgetter(-16) - ig2 = operator.itemgetter(-17) - def integerify(X): - return ig1(X) | (ig2(X)<<32) - self.integerify = integerify - - #================================================================= - # frontend - #================================================================= - def run(self, secret, salt, keylen): - """ - run scrypt kdf for specified secret, salt, and keylen - - .. note:: - - * time cost is ``O(n * r * p)`` - * mem cost is ``O(n * r)`` - """ - # stretch salt into initial byte array via pbkdf2 - iv_bytes = self.iv_bytes - input = pbkdf2_hmac("sha256", secret, salt, rounds=1, keylen=iv_bytes) - - # split initial byte array into 'p' mflen-sized chunks, - # and run each chunk through smix() to generate output chunk. - smix = self.smix - if self.p == 1: - output = smix(input) - else: - # XXX: *could* use threading here, if really high p values encountered, - # but would tradeoff for more memory usage. - smix_bytes = self.smix_bytes - output = b''.join( - smix(input[offset:offset+smix_bytes]) - for offset in range(0, iv_bytes, smix_bytes) - ) - - # stretch final byte array into output via pbkdf2 - return pbkdf2_hmac("sha256", secret, output, rounds=1, keylen=keylen) - - #================================================================= - # smix() helper - #================================================================= - def smix(self, input): - """run SCrypt smix function on a single input block - - :arg input: - byte string containing input data. - interpreted as 32*r little endian 4 byte integers. - - :returns: - byte string containing output data - derived by mixing input using n & r parameters. - - .. note:: time & mem cost are both ``O(n * r)`` - """ - # gather locals - bmix = self.bmix - bmix_struct = self.bmix_struct - integerify = self.integerify - n = self.n - - # parse input into 32*r integers ('X' in scrypt source) - # mem cost -- O(r) - buffer = list(bmix_struct.unpack(input)) - - # starting with initial buffer contents, derive V s.t. - # V[0]=initial_buffer ... V[i] = bmix(V[i-1], V[i-1]) ... V[n-1] = bmix(V[n-2], V[n-2]) - # final buffer contents should equal bmix(V[n-1], V[n-1]) - # - # time cost -- O(n * r) -- n loops, bmix is O(r) - # mem cost -- O(n * r) -- V is n-element array of r-element tuples - # NOTE: could do time / memory tradeoff to shrink size of V - def vgen(): - i = 0 - while i < n: - last = tuple(buffer) - yield last - bmix(last, buffer) - i += 1 - V = list(vgen()) - - # generate result from X & V. - # - # time cost -- O(n * r) -- loops n times, calls bmix() which has O(r) time cost - # mem cost -- O(1) -- allocates nothing, calls bmix() which has O(1) mem cost - get_v_elem = V.__getitem__ - n_mask = n - 1 - i = 0 - while i < n: - j = integerify(buffer) & n_mask - result = tuple(a ^ b for a, b in izip(buffer, get_v_elem(j))) - bmix(result, buffer) - i += 1 - - # # NOTE: we could easily support arbitrary values of ``n``, not just powers of 2, - # # but very few implementations have that ability, so not enabling it for now... - # if not n_is_log_2: - # while i < n: - # j = integerify(buffer) % n - # tmp = tuple(a^b for a,b in izip(buffer, get_v_elem(j))) - # bmix(tmp,buffer) - # i += 1 - - # repack tmp - return bmix_struct.pack(*buffer) - - #================================================================= - # bmix() helper - #================================================================= - def bmix(self, source, target): - """ - block mixing function used by smix() - uses salsa20/8 core to mix block contents. - - :arg source: - source to read from. - should be list of 32*r 4-byte integers - (2*r salsa20 blocks). - - :arg target: - target to write to. - should be list with same size as source. - the existing value of this buffer is ignored. - - .. warning:: - - this operates *in place* on target, - so source & target should NOT be same list. - - .. note:: - - * time cost is ``O(r)`` -- loops 16*r times, salsa20() has ``O(1)`` cost. - - * memory cost is ``O(1)`` -- salsa20() uses 16 x uint4, - all other operations done in-place. - """ - ## assert source is not target - # Y[-1] = B[2r-1], Y[i] = hash( Y[i-1] xor B[i]) - # B' <-- (Y_0, Y_2 ... Y_{2r-2}, Y_1, Y_3 ... Y_{2r-1}) */ - half = self.bmix_half_len # 16*r out of 32*r - start of Y_1 - tmp = source[-16:] # 'X' in scrypt source - siter = iter(source) - j = 0 - while j < half: - jn = j+16 - target[j:jn] = tmp = salsa20(a ^ b for a, b in izip(tmp, siter)) - target[half+j:half+jn] = tmp = salsa20(a ^ b for a, b in izip(tmp, siter)) - j = jn - - def _bmix_1(self, source, target): - """special bmix() method optimized for ``r=1`` case""" - B = source[16:] - target[:16] = tmp = salsa20(a ^ b for a, b in izip(B, iter(source))) - target[16:] = salsa20(a ^ b for a, b in izip(tmp, B)) - - #================================================================= - # eoc - #================================================================= - -#========================================================================== -# eof -#========================================================================== diff --git a/src/passlib/crypto/scrypt/_gen_files.py b/src/passlib/crypto/scrypt/_gen_files.py deleted file mode 100644 index 55ddfae3..00000000 --- a/src/passlib/crypto/scrypt/_gen_files.py +++ /dev/null @@ -1,154 +0,0 @@ -"""passlib.utils.scrypt._gen_files - meta script that generates _salsa.py""" -#========================================================================== -# imports -#========================================================================== -# core -import os -# pkg -# local -#========================================================================== -# constants -#========================================================================== - -_SALSA_OPS = [ - # row = (target idx, source idx 1, source idx 2, rotate) - # interpreted as salsa operation over uint32... - # target = (source1+source2)<> (32 - (b)))) - ##x[ 4] ^= R(x[ 0]+x[12], 7); x[ 8] ^= R(x[ 4]+x[ 0], 9); - ##x[12] ^= R(x[ 8]+x[ 4],13); x[ 0] ^= R(x[12]+x[ 8],18); - ( 4, 0, 12, 7), - ( 8, 4, 0, 9), - ( 12, 8, 4, 13), - ( 0, 12, 8, 18), - - ##x[ 9] ^= R(x[ 5]+x[ 1], 7); x[13] ^= R(x[ 9]+x[ 5], 9); - ##x[ 1] ^= R(x[13]+x[ 9],13); x[ 5] ^= R(x[ 1]+x[13],18); - ( 9, 5, 1, 7), - ( 13, 9, 5, 9), - ( 1, 13, 9, 13), - ( 5, 1, 13, 18), - - ##x[14] ^= R(x[10]+x[ 6], 7); x[ 2] ^= R(x[14]+x[10], 9); - ##x[ 6] ^= R(x[ 2]+x[14],13); x[10] ^= R(x[ 6]+x[ 2],18); - ( 14, 10, 6, 7), - ( 2, 14, 10, 9), - ( 6, 2, 14, 13), - ( 10, 6, 2, 18), - - ##x[ 3] ^= R(x[15]+x[11], 7); x[ 7] ^= R(x[ 3]+x[15], 9); - ##x[11] ^= R(x[ 7]+x[ 3],13); x[15] ^= R(x[11]+x[ 7],18); - ( 3, 15, 11, 7), - ( 7, 3, 15, 9), - ( 11, 7, 3, 13), - ( 15, 11, 7, 18), - - ##/* Operate on rows. */ - ##x[ 1] ^= R(x[ 0]+x[ 3], 7); x[ 2] ^= R(x[ 1]+x[ 0], 9); - ##x[ 3] ^= R(x[ 2]+x[ 1],13); x[ 0] ^= R(x[ 3]+x[ 2],18); - ( 1, 0, 3, 7), - ( 2, 1, 0, 9), - ( 3, 2, 1, 13), - ( 0, 3, 2, 18), - - ##x[ 6] ^= R(x[ 5]+x[ 4], 7); x[ 7] ^= R(x[ 6]+x[ 5], 9); - ##x[ 4] ^= R(x[ 7]+x[ 6],13); x[ 5] ^= R(x[ 4]+x[ 7],18); - ( 6, 5, 4, 7), - ( 7, 6, 5, 9), - ( 4, 7, 6, 13), - ( 5, 4, 7, 18), - - ##x[11] ^= R(x[10]+x[ 9], 7); x[ 8] ^= R(x[11]+x[10], 9); - ##x[ 9] ^= R(x[ 8]+x[11],13); x[10] ^= R(x[ 9]+x[ 8],18); - ( 11, 10, 9, 7), - ( 8, 11, 10, 9), - ( 9, 8, 11, 13), - ( 10, 9, 8, 18), - - ##x[12] ^= R(x[15]+x[14], 7); x[13] ^= R(x[12]+x[15], 9); - ##x[14] ^= R(x[13]+x[12],13); x[15] ^= R(x[14]+x[13],18); - ( 12, 15, 14, 7), - ( 13, 12, 15, 9), - ( 14, 13, 12, 13), - ( 15, 14, 13, 18), -] - -def main(): - target = os.path.join(os.path.dirname(__file__), "_salsa.py") - fh = file(target, "w") - write = fh.write - - VNAMES = ["v%d" % i for i in range(16)] - - PAD = " " * 4 - PAD2 = " " * 8 - PAD3 = " " * 12 - TLIST = ", ".join("b%d" % i for i in range(16)) - VLIST = ", ".join(VNAMES) - kwds = dict( - VLIST=VLIST, - TLIST=TLIST, - ) - - write('''\ -"""passlib.utils.scrypt._salsa - salsa 20/8 core, autogenerated by _gen_salsa.py""" -#================================================================= -# salsa function -#================================================================= - -def salsa20(input): - \"""apply the salsa20/8 core to the provided input - - :args input: input list containing 16 32-bit integers - :returns: result list containing 16 32-bit integers - \""" - - %(TLIST)s = input - %(VLIST)s = \\ - %(TLIST)s - - i = 0 - while i < 4: -''' % kwds) - - for idx, (target, source1, source2, rotate) in enumerate(_SALSA_OPS): - write('''\ - # salsa op %(idx)d: [%(it)d] ^= ([%(is1)d]+[%(is2)d])<<<%(rot1)d - t = (%(src1)s + %(src2)s) & 0xffffffff - %(dst)s ^= ((t & 0x%(rmask)08x) << %(rot1)d) | (t >> %(rot2)d) - -''' % dict( - idx=idx, is1 = source1, is2=source2, it=target, - src1=VNAMES[source1], - src2=VNAMES[source2], - dst=VNAMES[target], - rmask=(1<<(32-rotate))-1, - rot1=rotate, - rot2=32-rotate, - )) - - write('''\ - i += 1 - -''') - - for idx in range(16): - write(PAD + "b%d = (b%d + v%d) & 0xffffffff\n" % (idx,idx,idx)) - - write('''\ - - return %(TLIST)s - -#================================================================= -# eof -#================================================================= -''' % kwds) - -if __name__ == "__main__": - main() - -#========================================================================== -# eof -#========================================================================== diff --git a/src/passlib/crypto/scrypt/_salsa.py b/src/passlib/crypto/scrypt/_salsa.py deleted file mode 100644 index 9112732e..00000000 --- a/src/passlib/crypto/scrypt/_salsa.py +++ /dev/null @@ -1,170 +0,0 @@ -"""passlib.utils.scrypt._salsa - salsa 20/8 core, autogenerated by _gen_salsa.py""" -#================================================================= -# salsa function -#================================================================= - -def salsa20(input): - """apply the salsa20/8 core to the provided input - - :args input: input list containing 16 32-bit integers - :returns: result list containing 16 32-bit integers - """ - - b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 = input - v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 = \ - b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 - - i = 0 - while i < 4: - # salsa op 0: [4] ^= ([0]+[12])<<<7 - t = (v0 + v12) & 0xffffffff - v4 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 1: [8] ^= ([4]+[0])<<<9 - t = (v4 + v0) & 0xffffffff - v8 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 2: [12] ^= ([8]+[4])<<<13 - t = (v8 + v4) & 0xffffffff - v12 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 3: [0] ^= ([12]+[8])<<<18 - t = (v12 + v8) & 0xffffffff - v0 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 4: [9] ^= ([5]+[1])<<<7 - t = (v5 + v1) & 0xffffffff - v9 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 5: [13] ^= ([9]+[5])<<<9 - t = (v9 + v5) & 0xffffffff - v13 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 6: [1] ^= ([13]+[9])<<<13 - t = (v13 + v9) & 0xffffffff - v1 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 7: [5] ^= ([1]+[13])<<<18 - t = (v1 + v13) & 0xffffffff - v5 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 8: [14] ^= ([10]+[6])<<<7 - t = (v10 + v6) & 0xffffffff - v14 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 9: [2] ^= ([14]+[10])<<<9 - t = (v14 + v10) & 0xffffffff - v2 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 10: [6] ^= ([2]+[14])<<<13 - t = (v2 + v14) & 0xffffffff - v6 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 11: [10] ^= ([6]+[2])<<<18 - t = (v6 + v2) & 0xffffffff - v10 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 12: [3] ^= ([15]+[11])<<<7 - t = (v15 + v11) & 0xffffffff - v3 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 13: [7] ^= ([3]+[15])<<<9 - t = (v3 + v15) & 0xffffffff - v7 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 14: [11] ^= ([7]+[3])<<<13 - t = (v7 + v3) & 0xffffffff - v11 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 15: [15] ^= ([11]+[7])<<<18 - t = (v11 + v7) & 0xffffffff - v15 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 16: [1] ^= ([0]+[3])<<<7 - t = (v0 + v3) & 0xffffffff - v1 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 17: [2] ^= ([1]+[0])<<<9 - t = (v1 + v0) & 0xffffffff - v2 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 18: [3] ^= ([2]+[1])<<<13 - t = (v2 + v1) & 0xffffffff - v3 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 19: [0] ^= ([3]+[2])<<<18 - t = (v3 + v2) & 0xffffffff - v0 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 20: [6] ^= ([5]+[4])<<<7 - t = (v5 + v4) & 0xffffffff - v6 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 21: [7] ^= ([6]+[5])<<<9 - t = (v6 + v5) & 0xffffffff - v7 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 22: [4] ^= ([7]+[6])<<<13 - t = (v7 + v6) & 0xffffffff - v4 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 23: [5] ^= ([4]+[7])<<<18 - t = (v4 + v7) & 0xffffffff - v5 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 24: [11] ^= ([10]+[9])<<<7 - t = (v10 + v9) & 0xffffffff - v11 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 25: [8] ^= ([11]+[10])<<<9 - t = (v11 + v10) & 0xffffffff - v8 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 26: [9] ^= ([8]+[11])<<<13 - t = (v8 + v11) & 0xffffffff - v9 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 27: [10] ^= ([9]+[8])<<<18 - t = (v9 + v8) & 0xffffffff - v10 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - # salsa op 28: [12] ^= ([15]+[14])<<<7 - t = (v15 + v14) & 0xffffffff - v12 ^= ((t & 0x01ffffff) << 7) | (t >> 25) - - # salsa op 29: [13] ^= ([12]+[15])<<<9 - t = (v12 + v15) & 0xffffffff - v13 ^= ((t & 0x007fffff) << 9) | (t >> 23) - - # salsa op 30: [14] ^= ([13]+[12])<<<13 - t = (v13 + v12) & 0xffffffff - v14 ^= ((t & 0x0007ffff) << 13) | (t >> 19) - - # salsa op 31: [15] ^= ([14]+[13])<<<18 - t = (v14 + v13) & 0xffffffff - v15 ^= ((t & 0x00003fff) << 18) | (t >> 14) - - i += 1 - - b0 = (b0 + v0) & 0xffffffff - b1 = (b1 + v1) & 0xffffffff - b2 = (b2 + v2) & 0xffffffff - b3 = (b3 + v3) & 0xffffffff - b4 = (b4 + v4) & 0xffffffff - b5 = (b5 + v5) & 0xffffffff - b6 = (b6 + v6) & 0xffffffff - b7 = (b7 + v7) & 0xffffffff - b8 = (b8 + v8) & 0xffffffff - b9 = (b9 + v9) & 0xffffffff - b10 = (b10 + v10) & 0xffffffff - b11 = (b11 + v11) & 0xffffffff - b12 = (b12 + v12) & 0xffffffff - b13 = (b13 + v13) & 0xffffffff - b14 = (b14 + v14) & 0xffffffff - b15 = (b15 + v15) & 0xffffffff - - return b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15 - -#================================================================= -# eof -#================================================================= diff --git a/src/passlib/exc.py b/src/passlib/exc.py deleted file mode 100644 index c4b78b44..00000000 --- a/src/passlib/exc.py +++ /dev/null @@ -1,311 +0,0 @@ -"""passlib.exc -- exceptions & warnings raised by passlib""" -#============================================================================= -# exceptions -#============================================================================= -class UnknownBackendError(ValueError): - """ - Error raised if multi-backend handler doesn't recognize backend name. - Inherits from :exc:`ValueError`. - - .. versionadded:: 1.7 - """ - def __init__(self, hasher, backend): - self.hasher = hasher - self.backend = backend - message = "%s: unknown backend: %r" % (hasher.name, backend) - ValueError.__init__(self, message) - -class MissingBackendError(RuntimeError): - """Error raised if multi-backend handler has no available backends; - or if specifically requested backend is not available. - - :exc:`!MissingBackendError` derives - from :exc:`RuntimeError`, since it usually indicates - lack of an external library or OS feature. - This is primarily raised by handlers which depend on - external libraries (which is currently just - :class:`~passlib.hash.bcrypt`). - """ - -class PasswordSizeError(ValueError): - """ - Error raised if a password exceeds the maximum size allowed - by Passlib (by default, 4096 characters); or if password exceeds - a hash-specific size limitation. - - Many password hash algorithms take proportionately larger amounts of time and/or - memory depending on the size of the password provided. This could present - a potential denial of service (DOS) situation if a maliciously large - password is provided to an application. Because of this, Passlib enforces - a maximum size limit, but one which should be *much* larger - than any legitimate password. :exc:`!PasswordSizeError` derives - from :exc:`!ValueError`. - - .. note:: - Applications wishing to use a different limit should set the - ``PASSLIB_MAX_PASSWORD_SIZE`` environmental variable before - Passlib is loaded. The value can be any large positive integer. - - .. attribute:: max_size - - indicates the maximum allowed size. - - .. versionadded:: 1.6 - """ - - max_size = None - - def __init__(self, max_size, msg=None): - self.max_size = max_size - if msg is None: - msg = "password exceeds maximum allowed size" - ValueError.__init__(self, msg) - - # this also prevents a glibc crypt segfault issue, detailed here ... - # http://www.openwall.com/lists/oss-security/2011/11/15/1 - -class PasswordTruncateError(PasswordSizeError): - """ - Error raised if password would be truncated by hash. - This derives from :exc:`PasswordSizeError` and :exc:`ValueError`. - - Hashers such as :class:`~passlib.hash.bcrypt` can be configured to raises - this error by setting ``truncate_error=True``. - - .. attribute:: max_size - - indicates the maximum allowed size. - - .. versionadded:: 1.7 - """ - - def __init__(self, cls, msg=None): - if msg is None: - msg = ("Password too long (%s truncates to %d characters)" % - (cls.name, cls.truncate_size)) - PasswordSizeError.__init__(self, cls.truncate_size, msg) - -class PasslibSecurityError(RuntimeError): - """ - Error raised if critical security issue is detected - (e.g. an attempt is made to use a vulnerable version of a bcrypt backend). - - .. versionadded:: 1.6.3 - """ - - -class TokenError(ValueError): - """ - Base error raised by v:mod:`passlib.totp` when - a token can't be parsed / isn't valid / etc. - Derives from :exc:`!ValueError`. - - Usually one of the more specific subclasses below will be raised: - - * :class:`MalformedTokenError` -- invalid chars, too few digits - * :class:`InvalidTokenError` -- no match found - * :class:`UsedTokenError` -- match found, but token already used - - .. versionadded:: 1.7 - """ - - #: default message to use if none provided -- subclasses may fill this in - _default_message = 'Token not acceptable' - - def __init__(self, msg=None, *args, **kwds): - if msg is None: - msg = self._default_message - ValueError.__init__(self, msg, *args, **kwds) - - -class MalformedTokenError(TokenError): - """ - Error raised by :mod:`passlib.totp` when a token isn't formatted correctly - (contains invalid characters, wrong number of digits, etc) - """ - _default_message = "Unrecognized token" - - -class InvalidTokenError(TokenError): - """ - Error raised by :mod:`passlib.totp` when a token is formatted correctly, - but doesn't match any tokens within valid range. - """ - _default_message = "Token did not match" - - -class UsedTokenError(TokenError): - """ - Error raised by :mod:`passlib.totp` if a token is reused. - Derives from :exc:`TokenError`. - - .. autoattribute:: expire_time - - .. versionadded:: 1.7 - """ - _default_message = "Token has already been used, please wait for another." - - #: optional value indicating when current counter period will end, - #: and a new token can be generated. - expire_time = None - - def __init__(self, *args, **kwds): - self.expire_time = kwds.pop("expire_time", None) - TokenError.__init__(self, *args, **kwds) - - -class UnknownHashError(ValueError): - """Error raised by :class:`~passlib.crypto.lookup_hash` if hash name is not recognized. - This exception derives from :exc:`!ValueError`. - - .. versionadded:: 1.7 - """ - def __init__(self, name): - self.name = name - ValueError.__init__(self, "unknown hash algorithm: %r" % name) - -#============================================================================= -# warnings -#============================================================================= -class PasslibWarning(UserWarning): - """base class for Passlib's user warnings, - derives from the builtin :exc:`UserWarning`. - - .. versionadded:: 1.6 - """ - -# XXX: there's only one reference to this class, and it will go away in 2.0; -# so can probably remove this along with this / roll this into PasslibHashWarning. -class PasslibConfigWarning(PasslibWarning): - """Warning issued when non-fatal issue is found related to the configuration - of a :class:`~passlib.context.CryptContext` instance. - - This occurs primarily in one of two cases: - - * The CryptContext contains rounds limits which exceed the hard limits - imposed by the underlying algorithm. - * An explicit rounds value was provided which exceeds the limits - imposed by the CryptContext. - - In both of these cases, the code will perform correctly & securely; - but the warning is issued as a sign the configuration may need updating. - - .. versionadded:: 1.6 - """ - -class PasslibHashWarning(PasslibWarning): - """Warning issued when non-fatal issue is found with parameters - or hash string passed to a passlib hash class. - - This occurs primarily in one of two cases: - - * A rounds value or other setting was explicitly provided which - exceeded the handler's limits (and has been clamped - by the :ref:`relaxed` flag). - - * A malformed hash string was encountered which (while parsable) - should be re-encoded. - - .. versionadded:: 1.6 - """ - -class PasslibRuntimeWarning(PasslibWarning): - """Warning issued when something unexpected happens during runtime. - - The fact that it's a warning instead of an error means Passlib - was able to correct for the issue, but that it's anomalous enough - that the developers would love to hear under what conditions it occurred. - - .. versionadded:: 1.6 - """ - -class PasslibSecurityWarning(PasslibWarning): - """Special warning issued when Passlib encounters something - that might affect security. - - .. versionadded:: 1.6 - """ - -#============================================================================= -# error constructors -# -# note: these functions are used by the hashes in Passlib to raise common -# error messages. They are currently just functions which return ValueError, -# rather than subclasses of ValueError, since the specificity isn't needed -# yet; and who wants to import a bunch of error classes when catching -# ValueError will do? -#============================================================================= - -def _get_name(handler): - return handler.name if handler else "" - -#------------------------------------------------------------------------ -# generic helpers -#------------------------------------------------------------------------ -def type_name(value): - """return pretty-printed string containing name of value's type""" - cls = value.__class__ - if cls.__module__ and cls.__module__ not in ["__builtin__", "builtins"]: - return "%s.%s" % (cls.__module__, cls.__name__) - elif value is None: - return 'None' - else: - return cls.__name__ - -def ExpectedTypeError(value, expected, param): - """error message when param was supposed to be one type, but found another""" - # NOTE: value is never displayed, since it may sometimes be a password. - name = type_name(value) - return TypeError("%s must be %s, not %s" % (param, expected, name)) - -def ExpectedStringError(value, param): - """error message when param was supposed to be unicode or bytes""" - return ExpectedTypeError(value, "unicode or bytes", param) - -#------------------------------------------------------------------------ -# hash/verify parameter errors -#------------------------------------------------------------------------ -def MissingDigestError(handler=None): - """raised when verify() method gets passed config string instead of hash""" - name = _get_name(handler) - return ValueError("expected %s hash, got %s config string instead" % - (name, name)) - -def NullPasswordError(handler=None): - """raised by OS crypt() supporting hashes, which forbid NULLs in password""" - name = _get_name(handler) - return ValueError("%s does not allow NULL bytes in password" % name) - -#------------------------------------------------------------------------ -# errors when parsing hashes -#------------------------------------------------------------------------ -def InvalidHashError(handler=None): - """error raised if unrecognized hash provided to handler""" - return ValueError("not a valid %s hash" % _get_name(handler)) - -def MalformedHashError(handler=None, reason=None): - """error raised if recognized-but-malformed hash provided to handler""" - text = "malformed %s hash" % _get_name(handler) - if reason: - text = "%s (%s)" % (text, reason) - return ValueError(text) - -def ZeroPaddedRoundsError(handler=None): - """error raised if hash was recognized but contained zero-padded rounds field""" - return MalformedHashError(handler, "zero-padded rounds") - -#------------------------------------------------------------------------ -# settings / hash component errors -#------------------------------------------------------------------------ -def ChecksumSizeError(handler, raw=False): - """error raised if hash was recognized, but checksum was wrong size""" - # TODO: if handler.use_defaults is set, this came from app-provided value, - # not from parsing a hash string, might want different error msg. - checksum_size = handler.checksum_size - unit = "bytes" if raw else "chars" - reason = "checksum must be exactly %d %s" % (checksum_size, unit) - return MalformedHashError(handler, reason) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/ext/__init__.py b/src/passlib/ext/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/src/passlib/ext/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/passlib/ext/django/__init__.py b/src/passlib/ext/django/__init__.py deleted file mode 100644 index 2dc9b282..00000000 --- a/src/passlib/ext/django/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""passlib.ext.django.models -- monkeypatch django hashing framework - -this plugin monkeypatches django's hashing framework -so that it uses a passlib context object, allowing handling of arbitrary -hashes in Django databases. -""" diff --git a/src/passlib/ext/django/models.py b/src/passlib/ext/django/models.py deleted file mode 100644 index e766c2db..00000000 --- a/src/passlib/ext/django/models.py +++ /dev/null @@ -1,36 +0,0 @@ -"""passlib.ext.django.models -- monkeypatch django hashing framework""" -#============================================================================= -# imports -#============================================================================= -# core -# site -# pkg -from passlib.context import CryptContext -from passlib.ext.django.utils import DjangoContextAdapter -# local -__all__ = ["password_context"] - -#============================================================================= -# global attrs -#============================================================================= - -#: adapter instance used to drive most of this -adapter = DjangoContextAdapter() - -# the context object which this patches contrib.auth to use for password hashing. -# configuration controlled by ``settings.PASSLIB_CONFIG``. -password_context = adapter.context - -#: hook callers should use if context is changed -context_changed = adapter.reset_hashers - -#============================================================================= -# main code -#============================================================================= - -# load config & install monkeypatch -adapter.load_model() - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/ext/django/utils.py b/src/passlib/ext/django/utils.py deleted file mode 100644 index a83cb89a..00000000 --- a/src/passlib/ext/django/utils.py +++ /dev/null @@ -1,1233 +0,0 @@ -"""passlib.ext.django.utils - helper functions used by this plugin""" -#============================================================================= -# imports -#============================================================================= -# core -from functools import update_wrapper, wraps -import logging; log = logging.getLogger(__name__) -import sys -import weakref -from warnings import warn -# site -try: - from django import VERSION as DJANGO_VERSION - log.debug("found django %r installation", DJANGO_VERSION) -except ImportError: - log.debug("django installation not found") - DJANGO_VERSION = () -# pkg -from passlib import exc, registry -from passlib.context import CryptContext -from passlib.exc import PasslibRuntimeWarning -from passlib.utils.compat import get_method_function, iteritems, OrderedDict, unicode -from passlib.utils.decor import memoized_property -# local -__all__ = [ - "DJANGO_VERSION", - "MIN_DJANGO_VERSION", - "get_preset_config", - "get_django_hasher", -] - -#: minimum version supported by passlib.ext.django -MIN_DJANGO_VERSION = (1, 8) - -#============================================================================= -# default policies -#============================================================================= - -# map preset names -> passlib.app attrs -_preset_map = { - "django-1.0": "django10_context", - "django-1.4": "django14_context", - "django-1.6": "django16_context", - "django-latest": "django_context", -} - -def get_preset_config(name): - """Returns configuration string for one of the preset strings - supported by the ``PASSLIB_CONFIG`` setting. - Currently supported presets: - - * ``"passlib-default"`` - default config used by this release of passlib. - * ``"django-default"`` - config matching currently installed django version. - * ``"django-latest"`` - config matching newest django version (currently same as ``"django-1.6"``). - * ``"django-1.0"`` - config used by stock Django 1.0 - 1.3 installs - * ``"django-1.4"`` - config used by stock Django 1.4 installs - * ``"django-1.6"`` - config used by stock Django 1.6 installs - """ - # TODO: add preset which includes HASHERS + PREFERRED_HASHERS, - # after having imported any custom hashers. e.g. "django-current" - if name == "django-default": - if not DJANGO_VERSION: - raise ValueError("can't resolve django-default preset, " - "django not installed") - name = "django-1.6" - if name == "passlib-default": - return PASSLIB_DEFAULT - try: - attr = _preset_map[name] - except KeyError: - raise ValueError("unknown preset config name: %r" % name) - import passlib.apps - return getattr(passlib.apps, attr).to_string() - -# default context used by passlib 1.6 -PASSLIB_DEFAULT = """ -[passlib] - -; list of schemes supported by configuration -; currently all django 1.6, 1.4, and 1.0 hashes, -; and three common modular crypt format hashes. -schemes = - django_pbkdf2_sha256, django_pbkdf2_sha1, django_bcrypt, django_bcrypt_sha256, - django_salted_sha1, django_salted_md5, django_des_crypt, hex_md5, - sha512_crypt, bcrypt, phpass - -; default scheme to use for new hashes -default = django_pbkdf2_sha256 - -; hashes using these schemes will automatically be re-hashed -; when the user logs in (currently all django 1.0 hashes) -deprecated = - django_pbkdf2_sha1, django_salted_sha1, django_salted_md5, - django_des_crypt, hex_md5 - -; sets some common options, including minimum rounds for two primary hashes. -; if a hash has less than this number of rounds, it will be re-hashed. -sha512_crypt__min_rounds = 80000 -django_pbkdf2_sha256__min_rounds = 10000 - -; set somewhat stronger iteration counts for ``User.is_staff`` -staff__sha512_crypt__default_rounds = 100000 -staff__django_pbkdf2_sha256__default_rounds = 12500 - -; and even stronger ones for ``User.is_superuser`` -superuser__sha512_crypt__default_rounds = 120000 -superuser__django_pbkdf2_sha256__default_rounds = 15000 -""" - -#============================================================================= -# helpers -#============================================================================= - -#: prefix used to shoehorn passlib's handler names into django hasher namespace -PASSLIB_WRAPPER_PREFIX = "passlib_" - -#: prefix used by all the django-specific hash formats in passlib; -#: all of these hashes should have a ``.django_name`` attribute. -DJANGO_COMPAT_PREFIX = "django_" - -#: set of hashes w/o "django_" prefix, but which also expose ``.django_name``. -_other_django_hashes = set(["hex_md5"]) - -def _wrap_method(method): - """wrap method object in bare function""" - @wraps(method) - def wrapper(*args, **kwds): - return method(*args, **kwds) - return wrapper - -#============================================================================= -# translator -#============================================================================= -class DjangoTranslator(object): - """ - Object which helps translate passlib hasher objects / names - to and from django hasher objects / names. - - These methods are wrapped in a class so that results can be cached, - but with the ability to have independant caches, since django hasher - names may / may not correspond to the same instance (or even class). - """ - #============================================================================= - # instance attrs - #============================================================================= - - #: CryptContext instance - #: (if any -- generally only set by DjangoContextAdapter subclass) - context = None - - #: internal cache of passlib hasher -> django hasher instance. - #: key stores weakref to passlib hasher. - _django_hasher_cache = None - - #: special case -- unsalted_sha1 - _django_unsalted_sha1 = None - - #: internal cache of django name -> passlib hasher - #: value stores weakrefs to passlib hasher. - _passlib_hasher_cache = None - - #============================================================================= - # init - #============================================================================= - - def __init__(self, context=None, **kwds): - super(DjangoTranslator, self).__init__(**kwds) - if context is not None: - self.context = context - - self._django_hasher_cache = weakref.WeakKeyDictionary() - self._passlib_hasher_cache = weakref.WeakValueDictionary() - - def reset_hashers(self): - self._django_hasher_cache.clear() - self._passlib_hasher_cache.clear() - self._django_unsalted_sha1 = None - - def _get_passlib_hasher(self, passlib_name): - """ - resolve passlib hasher by name, using context if available. - """ - context = self.context - if context is None: - return registry.get_crypt_handler(passlib_name) - else: - return context.handler(passlib_name) - - #============================================================================= - # resolve passlib hasher -> django hasher - #============================================================================= - - def passlib_to_django_name(self, passlib_name): - """ - Convert passlib hasher / name to Django hasher name. - """ - return self.passlib_to_django(passlib_name).algorithm - - # XXX: add option (in class, or call signature) to always return a wrapper, - # rather than native builtin -- would let HashersTest check that - # our own wrapper + implementations are matching up with their tests. - def passlib_to_django(self, passlib_hasher, cached=True): - """ - Convert passlib hasher / name to Django hasher. - - :param passlib_hasher: - passlib hasher / name - - :returns: - django hasher instance - """ - # resolve names to hasher - if not hasattr(passlib_hasher, "name"): - passlib_hasher = self._get_passlib_hasher(passlib_hasher) - - # check cache - if cached: - cache = self._django_hasher_cache - try: - return cache[passlib_hasher] - except KeyError: - pass - result = cache[passlib_hasher] = \ - self.passlib_to_django(passlib_hasher, cached=False) - return result - - # find native equivalent, and return wrapper if there isn't one - django_name = getattr(passlib_hasher, "django_name", None) - if django_name: - return self._create_django_hasher(django_name) - else: - return _PasslibHasherWrapper(passlib_hasher) - - _builtin_django_hashers = dict( - md5="MD5PasswordHasher", - ) - - def _create_django_hasher(self, django_name): - """ - helper to create new django hasher by name. - wraps underlying django methods. - """ - # if we haven't patched django, can use it directly - module = sys.modules.get("passlib.ext.django.models") - if module is None or not module.adapter.patched: - from django.contrib.auth.hashers import get_hasher - return get_hasher(django_name) - - # We've patched django's get_hashers(), so calling django's get_hasher() - # or get_hashers_by_algorithm() would only land us back here. - # As non-ideal workaround, have to use original get_hashers(), - get_hashers = module.adapter._manager.getorig("django.contrib.auth.hashers:get_hashers").__wrapped__ - for hasher in get_hashers(): - if hasher.algorithm == django_name: - return hasher - - # hardcode a few for cases where get_hashers() look won't work. - path = self._builtin_django_hashers.get(django_name) - if path: - if "." not in path: - path = "django.contrib.auth.hashers." + path - from django.utils.module_loading import import_string - return import_string(path)() - - raise ValueError("unknown hasher: %r" % django_name) - - #============================================================================= - # reverse django -> passlib - #============================================================================= - - def django_to_passlib_name(self, django_name): - """ - Convert Django hasher / name to Passlib hasher name. - """ - return self.django_to_passlib(django_name).name - - def django_to_passlib(self, django_name, cached=True): - """ - Convert Django hasher / name to Passlib hasher / name. - If present, CryptContext will be checked instead of main registry. - - :param django_name: - Django hasher class or algorithm name. - "default" allowed if context provided. - - :raises ValueError: - if can't resolve hasher. - - :returns: - passlib hasher or name - """ - # check for django hasher - if hasattr(django_name, "algorithm"): - - # check for passlib adapter - if isinstance(django_name, _PasslibHasherWrapper): - return django_name.passlib_handler - - # resolve django hasher -> name - django_name = django_name.algorithm - - # check cache - if cached: - cache = self._passlib_hasher_cache - try: - return cache[django_name] - except KeyError: - pass - result = cache[django_name] = \ - self.django_to_passlib(django_name, cached=False) - return result - - # check if it's an obviously-wrapped name - if django_name.startswith(PASSLIB_WRAPPER_PREFIX): - passlib_name = django_name[len(PASSLIB_WRAPPER_PREFIX):] - return self._get_passlib_hasher(passlib_name) - - # resolve default - if django_name == "default": - context = self.context - if context is None: - raise TypeError("can't determine default scheme w/ context") - return context.handler() - - # special case: Django uses a separate hasher for "sha1$$digest" - # hashes (unsalted_sha1) and "sha1$salt$digest" (sha1); - # but passlib uses "django_salted_sha1" for both of these. - if django_name == "unsalted_sha1": - django_name = "sha1" - - # resolve name - # XXX: bother caching these lists / mapping? - # not needed in long-term due to cache above. - context = self.context - if context is None: - # check registry - # TODO: should make iteration via registry easier - candidates = ( - registry.get_crypt_handler(passlib_name) - for passlib_name in registry.list_crypt_handlers() - if passlib_name.startswith(DJANGO_COMPAT_PREFIX) or - passlib_name in _other_django_hashes - ) - else: - # check context - candidates = context.schemes(resolve=True) - for handler in candidates: - if getattr(handler, "django_name", None) == django_name: - return handler - - # give up - # NOTE: this should only happen for custom django hashers that we don't - # know the equivalents for. _HasherHandler (below) is work in - # progress that would allow us to at least return a wrapper. - raise ValueError("can't translate django name to passlib name: %r" % - (django_name,)) - - #============================================================================= - # django hasher lookup - #============================================================================= - - def resolve_django_hasher(self, django_name, cached=True): - """ - Take in a django algorithm name, return django hasher. - """ - # check for django hasher - if hasattr(django_name, "algorithm"): - return django_name - - # resolve to passlib hasher - passlib_hasher = self.django_to_passlib(django_name, cached=cached) - - # special case: Django uses a separate hasher for "sha1$$digest" - # hashes (unsalted_sha1) and "sha1$salt$digest" (sha1); - # but passlib uses "django_salted_sha1" for both of these. - # XXX: this isn't ideal way to handle this. would like to do something - # like pass "django_variant=django_name" into passlib_to_django(), - # and have it cache separate hasher there. - # but that creates a LOT of complication in it's cache structure, - # for what is just one special case. - if django_name == "unsalted_sha1" and passlib_hasher.name == "django_salted_sha1": - if not cached: - return self._create_django_hasher(django_name) - result = self._django_unsalted_sha1 - if result is None: - result = self._django_unsalted_sha1 = self._create_django_hasher(django_name) - return result - - # lookup corresponding django hasher - return self.passlib_to_django(passlib_hasher, cached=cached) - - #============================================================================= - # eoc - #============================================================================= - -#============================================================================= -# adapter -#============================================================================= -class DjangoContextAdapter(DjangoTranslator): - """ - Object which tries to adapt a Passlib CryptContext object, - using a Django-hasher compatible API. - - When installed in django, :mod:`!passlib.ext.django` will create - an instance of this class, and then monkeypatch the appropriate - methods into :mod:`!django.contrib.auth` and other appropriate places. - """ - #============================================================================= - # instance attrs - #============================================================================= - - #: CryptContext instance we're wrapping - context = None - - #: ref to original make_password(), - #: needed to generate usuable passwords that match django - _orig_make_password = None - - #: ref to django helper of this name -- not monkeypatched - is_password_usable = None - - #: PatchManager instance used to track installation - _manager = None - - #: whether config=disabled flag was set - enabled = True - - #: patch status - patched = False - - #============================================================================= - # init - #============================================================================= - def __init__(self, context=None, get_user_category=None, **kwds): - - # init log - self.log = logging.getLogger(__name__ + ".DjangoContextAdapter") - - # init parent, filling in default context object - if context is None: - context = CryptContext() - super(DjangoContextAdapter, self).__init__(context=context, **kwds) - - # setup user category - if get_user_category: - assert callable(get_user_category) - self.get_user_category = get_user_category - - # install lru cache wrappers - from django.utils.lru_cache import lru_cache - self.get_hashers = lru_cache()(self.get_hashers) - - # get copy of original make_password - from django.contrib.auth.hashers import make_password - if make_password.__module__.startswith("passlib."): - make_password = _PatchManager.peek_unpatched_func(make_password) - self._orig_make_password = make_password - - # get other django helpers - from django.contrib.auth.hashers import is_password_usable - self.is_password_usable = is_password_usable - - # init manager - mlog = logging.getLogger(__name__ + ".DjangoContextAdapter._manager") - self._manager = _PatchManager(log=mlog) - - def reset_hashers(self): - """ - Wrapper to manually reset django's hasher lookup cache - """ - # resets cache for .get_hashers() & .get_hashers_by_algorithm() - from django.contrib.auth.hashers import reset_hashers - reset_hashers(setting="PASSWORD_HASHERS") - - # reset internal caches - super(DjangoContextAdapter, self).reset_hashers() - - #============================================================================= - # django hashers helpers -- hasher lookup - #============================================================================= - - # lru_cache()'ed by init - def get_hashers(self): - """ - Passlib replacement for get_hashers() -- - Return list of available django hasher classes - """ - passlib_to_django = self.passlib_to_django - return [passlib_to_django(hasher) - for hasher in self.context.schemes(resolve=True)] - - def get_hasher(self, algorithm="default"): - """ - Passlib replacement for get_hasher() -- - Return django hasher by name - """ - return self.resolve_django_hasher(algorithm) - - def identify_hasher(self, encoded): - """ - Passlib replacement for identify_hasher() -- - Identify django hasher based on hash. - """ - handler = self.context.identify(encoded, resolve=True, required=True) - if handler.name == "django_salted_sha1" and encoded.startswith("sha1$$"): - # Django uses a separate hasher for "sha1$$digest" hashes, but - # passlib identifies it as belonging to "sha1$salt$digest" handler. - # We want to resolve to correct django hasher. - return self.get_hasher("unsalted_sha1") - return self.passlib_to_django(handler) - - #============================================================================= - # django.contrib.auth.hashers helpers -- password helpers - #============================================================================= - - def make_password(self, password, salt=None, hasher="default"): - """ - Passlib replacement for make_password() - """ - if password is None: - return self._orig_make_password(None) - # NOTE: relying on hasher coming from context, and thus having - # context-specific config baked into it. - passlib_hasher = self.django_to_passlib(hasher) - if "salt" not in passlib_hasher.setting_kwds: - # ignore salt param even if preset - pass - elif hasher.startswith("unsalted_"): - # Django uses a separate 'unsalted_sha1' hasher for "sha1$$digest", - # but passlib just reuses it's "sha1" handler ("sha1$salt$digest"). To make - # this work, have to explicitly tell the sha1 handler to use an empty salt. - passlib_hasher = passlib_hasher.using(salt="") - elif salt: - # Django make_password() autogenerates a salt if salt is bool False (None / ''), - # so we only pass the keyword on if there's actually a fixed salt. - passlib_hasher = passlib_hasher.using(salt=salt) - return passlib_hasher.hash(password) - - def check_password(self, password, encoded, setter=None, preferred="default"): - """ - Passlib replacement for check_password() - """ - # XXX: this currently ignores "preferred" keyword, since its purpose - # was for hash migration, and that's handled by the context. - if password is None or not self.is_password_usable(encoded): - return False - - # verify password - context = self.context - correct = context.verify(password, encoded) - if not (correct and setter): - return correct - - # check if we need to rehash - if preferred == "default": - if not context.needs_update(encoded, secret=password): - return correct - else: - # Django's check_password() won't call setter() on a - # 'preferred' alg, even if it's otherwise deprecated. To try and - # replicate this behavior if preferred is set, we look up the - # passlib hasher, and call it's original needs_update() method. - # TODO: Solve redundancy that verify() call - # above is already identifying hash. - hasher = self.django_to_passlib(preferred) - if (hasher.identify(encoded) and - not hasher.needs_update(encoded, secret=password)): - # alg is 'preferred' and hash itself doesn't need updating, - # so nothing to do. - return correct - # else: either hash isn't preferred, or it needs updating. - - # call setter to rehash - setter(password) - return correct - - #============================================================================= - # django users helpers - #============================================================================= - - def user_check_password(self, user, password): - """ - Passlib replacement for User.check_password() - """ - if password is None: - return False - hash = user.password - if not self.is_password_usable(hash): - return False - cat = self.get_user_category(user) - ok, new_hash = self.context.verify_and_update(password, hash, - category=cat) - if ok and new_hash is not None: - # migrate to new hash if needed. - user.password = new_hash - user.save() - return ok - - def user_set_password(self, user, password): - """ - Passlib replacement for User.set_password() - """ - if password is None: - user.set_unusable_password() - else: - cat = self.get_user_category(user) - user.password = self.context.hash(password, category=cat) - - def get_user_category(self, user): - """ - Helper for hashing passwords per-user -- - figure out the CryptContext category for specified Django user object. - .. note:: - This may be overridden via PASSLIB_GET_CATEGORY django setting - """ - if user.is_superuser: - return "superuser" - elif user.is_staff: - return "staff" - else: - return None - - #============================================================================= - # patch control - #============================================================================= - - HASHERS_PATH = "django.contrib.auth.hashers" - MODELS_PATH = "django.contrib.auth.models" - USER_CLASS_PATH = MODELS_PATH + ":User" - FORMS_PATH = "django.contrib.auth.forms" - - #: list of locations to patch - patch_locations = [ - # - # User object - # NOTE: could leave defaults alone, but want to have user available - # so that we can support get_user_category() - # - (USER_CLASS_PATH + ".check_password", "user_check_password", dict(method=True)), - (USER_CLASS_PATH + ".set_password", "user_set_password", dict(method=True)), - - # - # Hashers module - # - (HASHERS_PATH + ":", "check_password"), - (HASHERS_PATH + ":", "make_password"), - (HASHERS_PATH + ":", "get_hashers"), - (HASHERS_PATH + ":", "get_hasher"), - (HASHERS_PATH + ":", "identify_hasher"), - - # - # Patch known imports from hashers module - # - (MODELS_PATH + ":", "check_password"), - (MODELS_PATH + ":", "make_password"), - (FORMS_PATH + ":", "get_hasher"), - (FORMS_PATH + ":", "identify_hasher"), - - ] - - def install_patch(self): - """ - Install monkeypatch to replace django hasher framework. - """ - # don't reapply - log = self.log - if self.patched: - log.warning("monkeypatching already applied, refusing to reapply") - return False - - # version check - if DJANGO_VERSION < MIN_DJANGO_VERSION: - raise RuntimeError("passlib.ext.django requires django >= %s" % - (MIN_DJANGO_VERSION,)) - - # log start - log.debug("preparing to monkeypatch django ...") - - # run through patch locations - manager = self._manager - for record in self.patch_locations: - if len(record) == 2: - record += ({},) - target, source, opts = record - if target.endswith((":", ",")): - target += source - value = getattr(self, source) - if opts.get("method"): - # have to wrap our method in a function, - # since we're installing it in a class *as* a method - # XXX: make this a flag for .patch()? - value = _wrap_method(value) - manager.patch(target, value) - - # reset django's caches (e.g. get_hash_by_algorithm) - self.reset_hashers() - - # done! - self.patched = True - log.debug("... finished monkeypatching django") - return True - - def remove_patch(self): - """ - Remove monkeypatch from django hasher framework. - As precaution in case there are lingering refs to context, - context object will be wiped. - - .. warning:: - This may cause problems if any other Django modules have imported - their own copies of the patched functions, though the patched - code has been designed to throw an error as soon as possible in - this case. - """ - log = self.log - manager = self._manager - - if self.patched: - log.debug("removing django monkeypatching...") - manager.unpatch_all(unpatch_conflicts=True) - self.context.load({}) - self.patched = False - self.reset_hashers() - log.debug("...finished removing django monkeypatching") - return True - - if manager.isactive(): # pragma: no cover -- sanity check - log.warning("reverting partial monkeypatching of django...") - manager.unpatch_all() - self.context.load({}) - self.reset_hashers() - log.debug("...finished removing django monkeypatching") - return True - - log.debug("django not monkeypatched") - return False - - #============================================================================= - # loading config - #============================================================================= - - def load_model(self): - """ - Load configuration from django, and install patch. - """ - self._load_settings() - if self.enabled: - try: - self.install_patch() - except: - # try to undo what we can - self.remove_patch() - raise - else: - if self.patched: # pragma: no cover -- sanity check - log.error("didn't expect monkeypatching would be applied!") - self.remove_patch() - log.debug("passlib.ext.django loaded") - - def _load_settings(self): - """ - Update settings from django - """ - from django.conf import settings - - # TODO: would like to add support for inheriting config from a preset - # (or from existing hasher state) and letting PASSLIB_CONFIG - # be an update, not a replacement. - - # TODO: wrap and import any custom hashers as passlib handlers, - # so they could be used in the passlib config. - - # load config from settings - _UNSET = object() - config = getattr(settings, "PASSLIB_CONFIG", _UNSET) - if config is _UNSET: - # XXX: should probably deprecate this alias - config = getattr(settings, "PASSLIB_CONTEXT", _UNSET) - if config is _UNSET: - config = "passlib-default" - if config is None: - warn("setting PASSLIB_CONFIG=None is deprecated, " - "and support will be removed in Passlib 1.8, " - "use PASSLIB_CONFIG='disabled' instead.", - DeprecationWarning) - config = "disabled" - elif not isinstance(config, (unicode, bytes, dict)): - raise exc.ExpectedTypeError(config, "str or dict", "PASSLIB_CONFIG") - - # load custom category func (if any) - get_category = getattr(settings, "PASSLIB_GET_CATEGORY", None) - if get_category and not callable(get_category): - raise exc.ExpectedTypeError(get_category, "callable", "PASSLIB_GET_CATEGORY") - - # check if we've been disabled - if config == "disabled": - self.enabled = False - return - else: - self.__dict__.pop("enabled", None) - - # resolve any preset aliases - if isinstance(config, str) and '\n' not in config: - config = get_preset_config(config) - - # setup category func - if get_category: - self.get_user_category = get_category - else: - self.__dict__.pop("get_category", None) - - # setup context - self.context.load(config) - self.reset_hashers() - - #============================================================================= - # eof - #============================================================================= - -#============================================================================= -# wrapping passlib handlers as django hashers -#============================================================================= -_GEN_SALT_SIGNAL = "--!!!generate-new-salt!!!--" - -class ProxyProperty(object): - """helper that proxies another attribute""" - - def __init__(self, attr): - self.attr = attr - - def __get__(self, obj, cls): - if obj is None: - cls = obj - return getattr(obj, self.attr) - - def __set__(self, obj, value): - setattr(obj, self.attr, value) - - def __delete__(self, obj): - delattr(obj, self.attr) - - -class _PasslibHasherWrapper(object): - """ - adapter which which wraps a :cls:`passlib.ifc.PasswordHash` class, - and provides an interface compatible with the Django hasher API. - - :param passlib_handler: - passlib hash handler (e.g. :cls:`passlib.hash.sha256_crypt`. - """ - #===================================================================== - # instance attrs - #===================================================================== - - #: passlib handler that we're adapting. - passlib_handler = None - - # NOTE: 'rounds' attr will store variable rounds, IF handler supports it. - # 'iterations' will act as proxy, for compatibility with django pbkdf2 hashers. - # rounds = None - # iterations = None - - #===================================================================== - # init - #===================================================================== - def __init__(self, passlib_handler): - # init handler - if getattr(passlib_handler, "django_name", None): - raise ValueError("handlers that reflect an official django " - "hasher shouldn't be wrapped: %r" % - (passlib_handler.name,)) - if passlib_handler.is_disabled: - # XXX: could this be implemented? - raise ValueError("can't wrap disabled-hash handlers: %r" % - (passlib_handler.name)) - self.passlib_handler = passlib_handler - - # init rounds support - if self._has_rounds: - self.rounds = passlib_handler.default_rounds - self.iterations = ProxyProperty("rounds") - - #===================================================================== - # internal methods - #===================================================================== - def __repr__(self): - return "" % self.passlib_handler - - #===================================================================== - # internal properties - #===================================================================== - - @memoized_property - def __name__(self): - return "Passlib_%s_PasswordHasher" % self.passlib_handler.name.title() - - @memoized_property - def _has_rounds(self): - return "rounds" in self.passlib_handler.setting_kwds - - @memoized_property - def _translate_kwds(self): - """ - internal helper for safe_summary() -- - used to translate passlib hash options -> django keywords - """ - out = dict(checksum="hash") - if self._has_rounds and "pbkdf2" in self.passlib_handler.name: - out['rounds'] = 'iterations' - return out - - #===================================================================== - # hasher properties - #===================================================================== - - @memoized_property - def algorithm(self): - return PASSLIB_WRAPPER_PREFIX + self.passlib_handler.name - - #===================================================================== - # hasher api - #===================================================================== - def salt(self): - # NOTE: passlib's handler.hash() should generate new salt each time, - # so this just returns a special constant which tells - # encode() (below) not to pass a salt keyword along. - return _GEN_SALT_SIGNAL - - def verify(self, password, encoded): - return self.passlib_handler.verify(password, encoded) - - def encode(self, password, salt=None, rounds=None, iterations=None): - kwds = {} - if salt is not None and salt != _GEN_SALT_SIGNAL: - kwds['salt'] = salt - if self._has_rounds: - if rounds is not None: - kwds['rounds'] = rounds - elif iterations is not None: - kwds['rounds'] = iterations - else: - kwds['rounds'] = self.rounds - elif rounds is not None or iterations is not None: - warn("%s.hash(): 'rounds' and 'iterations' are ignored" % self.__name__) - handler = self.passlib_handler - if kwds: - handler = handler.using(**kwds) - return handler.hash(password) - - def safe_summary(self, encoded): - from django.contrib.auth.hashers import mask_hash - from django.utils.translation import ugettext_noop as _ - handler = self.passlib_handler - items = [ - # since this is user-facing, we're reporting passlib's name, - # without the distracting PASSLIB_HASHER_PREFIX prepended. - (_('algorithm'), handler.name), - ] - if hasattr(handler, "parsehash"): - kwds = handler.parsehash(encoded, sanitize=mask_hash) - for key, value in iteritems(kwds): - key = self._translate_kwds.get(key, key) - items.append((_(key), value)) - return OrderedDict(items) - - def must_update(self, encoded): - # TODO: would like access CryptContext, would need caller to pass it to get_passlib_hasher(). - # for now (as of passlib 1.6.6), replicating django policy that this returns True - # if 'encoded' hash has different rounds value from self.rounds - if self._has_rounds: - # XXX: could cache this subclass somehow (would have to intercept writes to self.rounds) - # TODO: always call subcls/handler.needs_update() in case there's other things to check - subcls = self.passlib_handler.using(min_rounds=self.rounds, max_rounds=self.rounds) - if subcls.needs_update(encoded): - return True - return False - - #===================================================================== - # eoc - #===================================================================== - -#============================================================================= -# adapting django hashers -> passlib handlers -#============================================================================= -# TODO: this code probably halfway works, mainly just needs -# a routine to read HASHERS and PREFERRED_HASHER. - -##from passlib.registry import register_crypt_handler -##from passlib.utils import classproperty, to_native_str, to_unicode -##from passlib.utils.compat import unicode -## -## -##class _HasherHandler(object): -## "helper for wrapping Hasher instances as passlib handlers" -## # FIXME: this generic wrapper doesn't handle custom settings -## # FIXME: genconfig / genhash not supported. -## -## def __init__(self, hasher): -## self.django_hasher = hasher -## if hasattr(hasher, "iterations"): -## # assume encode() accepts an "iterations" parameter. -## # fake min/max rounds -## self.min_rounds = 1 -## self.max_rounds = 0xFFFFffff -## self.default_rounds = self.django_hasher.iterations -## self.setting_kwds += ("rounds",) -## -## # hasher instance - filled in by constructor -## django_hasher = None -## -## setting_kwds = ("salt",) -## context_kwds = () -## -## @property -## def name(self): -## # XXX: need to make sure this wont' collide w/ builtin django hashes. -## # maybe by renaming this to django compatible aliases? -## return DJANGO_PASSLIB_PREFIX + self.django_name -## -## @property -## def django_name(self): -## # expose this so hasher_to_passlib_name() extracts original name -## return self.django_hasher.algorithm -## -## @property -## def ident(self): -## # this should always be correct, as django relies on ident prefix. -## return unicode(self.django_name + "$") -## -## @property -## def identify(self, hash): -## # this should always work, as django relies on ident prefix. -## return to_unicode(hash, "latin-1", "hash").startswith(self.ident) -## -## @property -## def hash(self, secret, salt=None, **kwds): -## # NOTE: from how make_password() is coded, all hashers -## # should have salt param. but only some will have -## # 'iterations' parameter. -## opts = {} -## if 'rounds' in self.setting_kwds and 'rounds' in kwds: -## opts['iterations'] = kwds.pop("rounds") -## if kwds: -## raise TypeError("unexpected keyword arguments: %r" % list(kwds)) -## if isinstance(secret, unicode): -## secret = secret.encode("utf-8") -## if salt is None: -## salt = self.django_hasher.salt() -## return to_native_str(self.django_hasher(secret, salt, **opts)) -## -## @property -## def verify(self, secret, hash): -## hash = to_native_str(hash, "utf-8", "hash") -## if isinstance(secret, unicode): -## secret = secret.encode("utf-8") -## return self.django_hasher.verify(secret, hash) -## -##def register_hasher(hasher): -## handler = _HasherHandler(hasher) -## register_crypt_handler(handler) -## return handler - -#============================================================================= -# monkeypatch helpers -#============================================================================= -# private singleton indicating lack-of-value -_UNSET = object() - -class _PatchManager(object): - """helper to manage monkeypatches and run sanity checks""" - - # NOTE: this could easily use a dict interface, - # but keeping it distinct to make clear that it's not a dict, - # since it has important side-effects. - - #=================================================================== - # init and support - #=================================================================== - def __init__(self, log=None): - # map of key -> (original value, patched value) - # original value may be _UNSET - self.log = log or logging.getLogger(__name__ + "._PatchManager") - self._state = {} - - def isactive(self): - return bool(self._state) - - # bool value tests if any patches are currently applied. - # NOTE: this behavior is deprecated in favor of .isactive - __bool__ = __nonzero__ = isactive - - def _import_path(self, path): - """retrieve obj and final attribute name from resource path""" - name, attr = path.split(":") - obj = __import__(name, fromlist=[attr], level=0) - while '.' in attr: - head, attr = attr.split(".", 1) - obj = getattr(obj, head) - return obj, attr - - @staticmethod - def _is_same_value(left, right): - """check if two values are the same (stripping method wrappers, etc)""" - return get_method_function(left) == get_method_function(right) - - #=================================================================== - # reading - #=================================================================== - def _get_path(self, key, default=_UNSET): - obj, attr = self._import_path(key) - return getattr(obj, attr, default) - - def get(self, path, default=None): - """return current value for path""" - return self._get_path(path, default) - - def getorig(self, path, default=None): - """return original (unpatched) value for path""" - try: - value, _= self._state[path] - except KeyError: - value = self._get_path(path) - return default if value is _UNSET else value - - def check_all(self, strict=False): - """run sanity check on all keys, issue warning if out of sync""" - same = self._is_same_value - for path, (orig, expected) in iteritems(self._state): - if same(self._get_path(path), expected): - continue - msg = "another library has patched resource: %r" % path - if strict: - raise RuntimeError(msg) - else: - warn(msg, PasslibRuntimeWarning) - - #=================================================================== - # patching - #=================================================================== - def _set_path(self, path, value): - obj, attr = self._import_path(path) - if value is _UNSET: - if hasattr(obj, attr): - delattr(obj, attr) - else: - setattr(obj, attr, value) - - def patch(self, path, value, wrap=False): - """monkeypatch object+attr at to have , stores original""" - assert value != _UNSET - current = self._get_path(path) - try: - orig, expected = self._state[path] - except KeyError: - self.log.debug("patching resource: %r", path) - orig = current - else: - self.log.debug("modifying resource: %r", path) - if not self._is_same_value(current, expected): - warn("overridding resource another library has patched: %r" - % path, PasslibRuntimeWarning) - if wrap: - assert callable(value) - wrapped = orig - wrapped_by = value - def wrapper(*args, **kwds): - return wrapped_by(wrapped, *args, **kwds) - update_wrapper(wrapper, value) - value = wrapper - if callable(value): - # needed by DjangoContextAdapter init - get_method_function(value)._patched_original_value = orig - self._set_path(path, value) - self._state[path] = (orig, value) - - @classmethod - def peek_unpatched_func(cls, value): - return value._patched_original_value - - ##def patch_many(self, **kwds): - ## "override specified resources with new values" - ## for path, value in iteritems(kwds): - ## self.patch(path, value) - - def monkeypatch(self, parent, name=None, enable=True, wrap=False): - """function decorator which patches function of same name in """ - def builder(func): - if enable: - sep = "." if ":" in parent else ":" - path = parent + sep + (name or func.__name__) - self.patch(path, func, wrap=wrap) - return func - if callable(name): - # called in non-decorator mode - func = name - name = None - builder(func) - return None - return builder - - #=================================================================== - # unpatching - #=================================================================== - def unpatch(self, path, unpatch_conflicts=True): - try: - orig, expected = self._state[path] - except KeyError: - return - current = self._get_path(path) - self.log.debug("unpatching resource: %r", path) - if not self._is_same_value(current, expected): - if unpatch_conflicts: - warn("reverting resource another library has patched: %r" - % path, PasslibRuntimeWarning) - else: - warn("not reverting resource another library has patched: %r" - % path, PasslibRuntimeWarning) - del self._state[path] - return - self._set_path(path, orig) - del self._state[path] - - def unpatch_all(self, **kwds): - for key in list(self._state): - self.unpatch(key, **kwds) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/__init__.py b/src/passlib/handlers/__init__.py deleted file mode 100644 index 0a0338c8..00000000 --- a/src/passlib/handlers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""passlib.handlers -- holds implementations of all passlib's builtin hash formats""" diff --git a/src/passlib/handlers/argon2.py b/src/passlib/handlers/argon2.py deleted file mode 100644 index 578c2c51..00000000 --- a/src/passlib/handlers/argon2.py +++ /dev/null @@ -1,825 +0,0 @@ -"""passlib.handlers.argon2 -- argon2 password hash wrapper - -References -========== -* argon2 - - home: https://github.com/P-H-C/phc-winner-argon2 - - whitepaper: https://github.com/P-H-C/phc-winner-argon2/blob/master/argon2-specs.pdf -* argon2 cffi wrapper - - pypi: https://pypi.python.org/pypi/argon2_cffi - - home: https://github.com/hynek/argon2_cffi -* argon2 pure python - - pypi: https://pypi.python.org/pypi/argon2pure - - home: https://github.com/bwesterb/argon2pure -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement, absolute_import -# core -import logging -log = logging.getLogger(__name__) -import re -import types -from warnings import warn -# site -_argon2_cffi = None # loaded below -_argon2pure = None # dynamically imported by _load_backend_argon2pure() -# pkg -from passlib import exc -from passlib.crypto.digest import MAX_UINT32 -from passlib.utils import to_bytes -from passlib.utils.binary import b64s_encode, b64s_decode -from passlib.utils.compat import u, unicode, bascii_to_str -import passlib.utils.handlers as uh -# local -__all__ = [ - "argon2", -] - -#============================================================================= -# import argon2 package (https://pypi.python.org/pypi/argon2_cffi) -#============================================================================= - -# import package -try: - import argon2 as _argon2_cffi -except ImportError: - _argon2_cffi = None - -# get default settings for hasher -_PasswordHasher = getattr(_argon2_cffi, "PasswordHasher", None) -if _PasswordHasher: - # we have argon2_cffi >= 16.0, use their default hasher settings - _default_settings = _PasswordHasher() - _default_version = _argon2_cffi.low_level.ARGON2_VERSION -else: - # use these as our fallback settings (for no backend, or argon2pure) - class _default_settings: - """ - dummy object to use as source of defaults when argon2 mod not present. - synced w/ argon2 16.1 as of 2016-6-16 - """ - time_cost = 2 - memory_cost = 512 - parallelism = 2 - salt_len = 16 - hash_len = 16 - _default_version = 0x13 - -#============================================================================= -# handler -#============================================================================= -class _Argon2Common(uh.SubclassBackendMixin, uh.ParallelismMixin, - uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, - uh.GenericHandler): - """ - Base class which implements brunt of Argon2 code. - This is then subclassed by the various backends, - to override w/ backend-specific methods. - - When a backend is loaded, the bases of the 'argon2' class proper - are modified to prepend the correct backend-specific subclass. - """ - #=================================================================== - # class attrs - #=================================================================== - - #------------------------ - # PasswordHash - #------------------------ - - name = "argon2" - setting_kwds = ("salt", - "salt_size", - "salt_len", # 'salt_size' alias for compat w/ argon2 package - "rounds", - "time_cost", # 'rounds' alias for compat w/ argon2 package - "memory_cost", - "parallelism", - "digest_size", - "hash_len", # 'digest_size' alias for compat w/ argon2 package - ) - - # TODO: could support the optional 'data' parameter, - # but need to research the uses, what a more descriptive name would be, - # and deal w/ fact that argon2_cffi 16.1 doesn't currently support it. - # (argon2_pure does though) - - #------------------------ - # GenericHandler - #------------------------ - ident = u("$argon2i") - checksum_size = _default_settings.hash_len - - # NOTE: from_string() relies on the ordering of these... - ident_values = (u("$argon2i$"), u("$argon2d$")) - - #------------------------ - # HasSalt - #------------------------ - default_salt_size = _default_settings.salt_len - min_salt_size = 8 - max_salt_size = MAX_UINT32 - - #------------------------ - # HasRounds - # TODO: once rounds limit logic is factored out, - # make 'rounds' and 'cost' an alias for 'time_cost' - #------------------------ - default_rounds = _default_settings.time_cost - min_rounds = 1 - max_rounds = MAX_UINT32 - rounds_cost = "linear" - - #------------------------ - # ParalleismMixin - #------------------------ - max_parallelism = (1 << 24) - 1 # from argon2.h / ARGON2_MAX_LANES - - #------------------------ - # custom - #------------------------ - - #: max version support - #: NOTE: this is dependant on the backend, and initialized/modified by set_backend() - max_version = _default_version - - #: minimum version before needs_update() marks the hash; if None, defaults to max_version - min_desired_version = None - - #: minimum valid memory_cost - min_memory_cost = 8 # from argon2.h / ARGON2_MIN_MEMORY - - #: maximum number of threads (-1=unlimited); - #: number of threads used by .hash() will be min(parallelism, max_threads) - max_threads = -1 - - #: global flag signalling argon2pure backend to use threads - #: rather than subprocesses. - pure_use_threads = False - - #=================================================================== - # instance attrs - #=================================================================== - - #: parallelism setting -- class value controls the default - parallelism = _default_settings.parallelism - - #: hash version (int) - #: NOTE: this is modified by set_backend() - version = _default_version - - #: memory cost -- class value controls the default - memory_cost = _default_settings.memory_cost - - #: flag indicating a Type D hash - type_d = False - - #: optional secret data - data = None - - #=================================================================== - # variant constructor - #=================================================================== - - @classmethod - def using(cls, memory_cost=None, salt_len=None, time_cost=None, digest_size=None, - checksum_size=None, hash_len=None, max_threads=None, **kwds): - # support aliases which match argon2 naming convention - if time_cost is not None: - if "rounds" in kwds: - raise TypeError("'time_cost' and 'rounds' are mutually exclusive") - kwds['rounds'] = time_cost - - if salt_len is not None: - if "salt_size" in kwds: - raise TypeError("'salt_len' and 'salt_size' are mutually exclusive") - kwds['salt_size'] = salt_len - - if hash_len is not None: - if digest_size is not None: - raise TypeError("'hash_len' and 'digest_size' are mutually exclusive") - digest_size = hash_len - - if checksum_size is not None: - if digest_size is not None: - raise TypeError("'checksum_size' and 'digest_size' are mutually exclusive") - digest_size = checksum_size - - # create variant - subcls = super(_Argon2Common, cls).using(**kwds) - - # set checksum size - relaxed = kwds.get("relaxed") - if digest_size is not None: - if isinstance(digest_size, uh.native_string_types): - digest_size = int(digest_size) - # NOTE: this isn't *really* digest size minimum, but want to enforce secure minimum. - subcls.checksum_size = uh.norm_integer(subcls, digest_size, min=16, max=MAX_UINT32, - param="digest_size", relaxed=relaxed) - - # set memory cost - if memory_cost is not None: - if isinstance(memory_cost, uh.native_string_types): - memory_cost = int(memory_cost) - subcls.memory_cost = subcls._norm_memory_cost(memory_cost, relaxed=relaxed) - - # validate constraints - subcls._validate_constraints(subcls.memory_cost, subcls.parallelism) - - # set max threads - if max_threads is not None: - if isinstance(max_threads, uh.native_string_types): - max_threads = int(max_threads) - if max_threads < 1 and max_threads != -1: - raise ValueError("max_threads (%d) must be -1 (unlimited), or at least 1." % - (max_threads,)) - subcls.max_threads = max_threads - - return subcls - - @classmethod - def _validate_constraints(cls, memory_cost, parallelism): - # NOTE: this is used by class & instance, hence passing in via arguments. - # could switch and make this a hybrid method. - min_memory_cost = 8 * parallelism - if memory_cost < min_memory_cost: - raise ValueError("%s: memory_cost (%d) is too low, must be at least " - "8 * parallelism (8 * %d = %d)" % - (cls.name, memory_cost, - parallelism, min_memory_cost)) - - #=================================================================== - # public api - #=================================================================== - - @classmethod - def identify(cls, hash): - hash = uh.to_unicode_for_identify(hash) - return hash.startswith(cls.ident_values) - - # hash(), verify(), genhash() -- implemented by backend subclass - - #=================================================================== - # hash parsing / rendering - #=================================================================== - - # info taken from source of decode_string() function in - # - # - # hash format: - # $argon2[$v=]$m=,t=,p=[,keyid=][,data=][$[$]] - # - # NOTE: as of 2016-6-17, the official source (above) lists the "keyid" param in the comments, - # but the actual source of decode_string & encode_string don't mention it at all. - # we're supporting parsing it, but throw NotImplementedError if encountered. - # - # sample hashes: - # v1.0: '$argon2i$m=512,t=2,p=2$5VtWOO3cGWYQHEMaYGbsfQ$AcmqasQgW/wI6wAHAMk4aQ' - # v1.3: '$argon2i$v=19$m=512,t=2,p=2$5VtWOO3cGWYQHEMaYGbsfQ$AcmqasQgW/wI6wAHAMk4aQ' - - #: regex to parse argon hash - _hash_regex = re.compile(br""" - ^ - \$argon2(?P[id])\$ - (?: - v=(?P\d+) - \$ - )? - m=(?P\d+) - , - t=(?P\d+) - , - p=(?P\d+) - (?: - ,keyid=(?P[^,$]+) - )? - (?: - ,data=(?P[^,$]+) - )? - (?: - \$ - (?P[^$]+) - (?: - \$ - (?P.+) - )? - )? - $ - """, re.X) - - @classmethod - def from_string(cls, hash): - # NOTE: assuming hash will be unicode, or use ascii-compatible encoding. - if isinstance(hash, unicode): - hash = hash.encode("utf-8") - if not isinstance(hash, bytes): - raise exc.ExpectedStringError(hash, "hash") - m = cls._hash_regex.match(hash) - if not m: - raise exc.MalformedHashError(cls) - type, version, memory_cost, time_cost, parallelism, keyid, data, salt, digest = \ - m.group("type", "version", "memory_cost", "time_cost", "parallelism", - "keyid", "data", "salt", "digest") - assert type in [b"i", b"d"], "unexpected type code: %r" % (type,) - if keyid: - raise NotImplementedError("argon2 'keyid' parameter not supported") - return cls( - type_d=(type == b"d"), - version=int(version) if version else 0x10, - memory_cost=int(memory_cost), - rounds=int(time_cost), - parallelism=int(parallelism), - salt=b64s_decode(salt) if salt else None, - data=b64s_decode(data) if data else None, - checksum=b64s_decode(digest) if digest else None, - ) - - def to_string(self): - ident = str(self.ident_values[self.type_d]) - version = self.version - if version == 0x10: - vstr = "" - else: - vstr = "v=%d$" % version - data = self.data - if data: - kdstr = ",data=" + bascii_to_str(b64s_encode(self.data)) - else: - kdstr = "" - # NOTE: 'keyid' param currently not supported - return "%s%sm=%d,t=%d,p=%d%s$%s$%s" % (ident, vstr, self.memory_cost, - self.rounds, self.parallelism, - kdstr, - bascii_to_str(b64s_encode(self.salt)), - bascii_to_str(b64s_encode(self.checksum))) - - #=================================================================== - # init - #=================================================================== - def __init__(self, type_d=False, version=None, memory_cost=None, data=None, **kwds): - - # TODO: factor out variable checksum size support into a mixin. - # set checksum size to specific value before _norm_checksum() is called - checksum = kwds.get("checksum") - if checksum is not None: - self.checksum_size = len(checksum) - - # call parent - super(_Argon2Common, self).__init__(**kwds) - - # init type - # NOTE: we don't support *generating* type I hashes, but do support verifying them. - self.type_d = type_d - - # init version - if version is None: - assert uh.validate_default_value(self, self.version, self._norm_version, - param="version") - else: - self.version = self._norm_version(version) - - # init memory cost - if memory_cost is None: - assert uh.validate_default_value(self, self.memory_cost, self._norm_memory_cost, - param="memory_cost") - else: - self.memory_cost = self._norm_memory_cost(memory_cost) - - # init data - if data is None: - assert self.data is None - else: - if not isinstance(data, bytes): - raise uh.exc.ExpectedTypeError(data, "bytes", "data") - self.data = data - - #------------------------------------------------------------------- - # parameter guards - #------------------------------------------------------------------- - - @classmethod - def _norm_version(cls, version): - if not isinstance(version, uh.int_types): - raise uh.exc.ExpectedTypeError(version, "integer", "version") - - # minimum valid version - if version < 0x13 and version != 0x10: - raise ValueError("invalid argon2 hash version: %d" % (version,)) - - # check this isn't past backend's max version - backend = cls.get_backend() - if version > cls.max_version: - raise ValueError("%s: hash version 0x%X not supported by %r backend " - "(max version is 0x%X); try updating or switching backends" % - (cls.name, version, backend, cls.max_version)) - return version - - @classmethod - def _norm_memory_cost(cls, memory_cost, relaxed=False): - return uh.norm_integer(cls, memory_cost, min=cls.min_memory_cost, - param="memory_cost", relaxed=relaxed) - - #=================================================================== - # digest calculation - #=================================================================== - - # NOTE: _calc_checksum implemented by backend subclass - - #=================================================================== - # hash migration - #=================================================================== - - def _calc_needs_update(self, **kwds): - cls = type(self) - if self.type_d: - # type 'd' hashes shouldn't be used for passwords. - return True - minver = cls.min_desired_version - if minver is None or minver > cls.max_version: - minver = cls.max_version - if self.version < minver: - # version is too old. - return True - if self.memory_cost != cls.memory_cost: - return True - if self.checksum_size != cls.checksum_size: - return True - return super(_Argon2Common, self)._calc_needs_update(**kwds) - - #=================================================================== - # backend loading - #=================================================================== - - _no_backend_suggestion = " -- recommend you install one (e.g. 'pip install argon2_cffi')" - - @classmethod - def _finalize_backend_mixin(mixin_cls, name, dryrun): - """ - helper called by from backend mixin classes' _load_backend_mixin() -- - invoked after backend imports have been loaded, and performs - feature detection & testing common to all backends. - """ - max_version = mixin_cls.max_version - assert isinstance(max_version, int) and max_version >= 0x10 - if max_version < 0x13: - warn("%r doesn't support argon2 v1.3, and should be upgraded" % name, - uh.exc.PasslibSecurityWarning) - return True - - @classmethod - def _adapt_backend_error(cls, err, hash=None, self=None): - """ - internal helper invoked when backend has hash/verification error; - used to adapt to passlib message. - """ - backend = cls.get_backend() - - # parse hash to throw error if format was invalid, parameter out of range, etc. - if self is None and hash is not None: - self = cls.from_string(hash) - - # check constraints on parsed object - # XXX: could move this to __init__, but not needed by needs_update calls - if self is not None: - self._validate_constraints(self.memory_cost, self.parallelism) - - # as of cffi 16.1, lacks support in hash_secret(), so genhash() will get here. - # as of cffi 16.2, support removed from verify_secret() as well. - if backend == "argon2_cffi" and self.data is not None: - raise NotImplementedError("argon2_cffi backend doesn't support the 'data' parameter") - - # fallback to reporting a malformed hash - text = str(err) - if text not in [ - "Decoding failed" # argon2_cffi's default message - ]: - reason = "%s reported: %s: hash=%r" % (backend, text, hash) - else: - reason = repr(hash) - raise exc.MalformedHashError(cls, reason=reason) - - #=================================================================== - # eoc - #=================================================================== - -#----------------------------------------------------------------------- -# stub backend -#----------------------------------------------------------------------- -class _NoBackend(_Argon2Common): - """ - mixin used before any backend has been loaded. - contains stubs that force loading of one of the available backends. - """ - #=================================================================== - # primary methods - #=================================================================== - @classmethod - def hash(cls, secret): - cls._stub_requires_backend() - return cls.hash(secret) - - @classmethod - def verify(cls, secret, hash): - cls._stub_requires_backend() - return cls.verify(secret, hash) - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genhash(cls, secret, config): - cls._stub_requires_backend() - return cls.genhash(secret, config) - - #=================================================================== - # digest calculation - #=================================================================== - def _calc_checksum(self, secret): - # NOTE: since argon2_cffi takes care of rendering hash, - # _calc_checksum() is only used by the argon2pure backend. - self._stub_requires_backend() - # NOTE: have to use super() here so that we don't recursively - # call subclass's wrapped _calc_checksum - return super(argon2, self)._calc_checksum(secret) - - #=================================================================== - # eoc - #=================================================================== - -#----------------------------------------------------------------------- -# argon2_cffi backend -#----------------------------------------------------------------------- -class _CffiBackend(_Argon2Common): - """ - argon2_cffi backend - """ - #=================================================================== - # backend loading - #=================================================================== - - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - # we automatically import this at top, so just grab info - if _argon2_cffi is None: - return False - max_version = _argon2_cffi.low_level.ARGON2_VERSION - log.debug("detected 'argon2_cffi' backend, version %r, with support for 0x%x argon2 hashes", - _argon2_cffi.__version__, max_version) - mixin_cls.version = mixin_cls.max_version = max_version - return mixin_cls._finalize_backend_mixin(name, dryrun) - - #=================================================================== - # primary methods - #=================================================================== - @classmethod - def hash(cls, secret): - # TODO: add in 'encoding' support once that's finalized in 1.8 / 1.9. - uh.validate_secret(secret) - secret = to_bytes(secret, "utf-8") - # XXX: doesn't seem to be a way to make this honor max_threads - try: - return bascii_to_str(_argon2_cffi.low_level.hash_secret( - type=_argon2_cffi.low_level.Type.I, - memory_cost=cls.memory_cost, - time_cost=cls.default_rounds, - parallelism=cls.parallelism, - salt=to_bytes(cls._generate_salt()), - hash_len=cls.checksum_size, - secret=secret, - )) - except _argon2_cffi.exceptions.HashingError as err: - raise cls._adapt_backend_error(err) - - @classmethod - def verify(cls, secret, hash): - # TODO: add in 'encoding' support once that's finalized in 1.8 / 1.9. - uh.validate_secret(secret) - secret = to_bytes(secret, "utf-8") - hash = to_bytes(hash, "ascii") - if hash.startswith(b"$argon2d$"): - type = _argon2_cffi.low_level.Type.D - else: - type = _argon2_cffi.low_level.Type.I - # XXX: doesn't seem to be a way to make this honor max_threads - try: - result = _argon2_cffi.low_level.verify_secret(hash, secret, type) - assert result is True - return True - except _argon2_cffi.exceptions.VerifyMismatchError: - return False - except _argon2_cffi.exceptions.VerificationError as err: - raise cls._adapt_backend_error(err, hash=hash) - - # NOTE: deprecated, will be removed in 2.0 - @classmethod - def genhash(cls, secret, config): - # TODO: add in 'encoding' support once that's finalized in 1.8 / 1.9. - uh.validate_secret(secret) - secret = to_bytes(secret, "utf-8") - self = cls.from_string(config) - if self.type_d: - type = _argon2_cffi.low_level.Type.D - else: - type = _argon2_cffi.low_level.Type.I - # XXX: doesn't seem to be a way to make this honor max_threads - try: - result = bascii_to_str(_argon2_cffi.low_level.hash_secret( - type=type, - memory_cost=self.memory_cost, - time_cost=self.rounds, - parallelism=self.parallelism, - salt=to_bytes(self.salt), - hash_len=self.checksum_size, - secret=secret, - version=self.version, - )) - except _argon2_cffi.exceptions.HashingError as err: - raise cls._adapt_backend_error(err, hash=config) - if self.version == 0x10: - # workaround: argon2 0x13 always returns "v=" segment, even for 0x10 hashes - result = result.replace("$v=16$", "$") - return result - - #=================================================================== - # digest calculation - #=================================================================== - def _calc_checksum(self, secret): - raise AssertionError("shouldn't be called under argon2_cffi backend") - - #=================================================================== - # eoc - #=================================================================== - -#----------------------------------------------------------------------- -# argon2pure backend -#----------------------------------------------------------------------- -class _PureBackend(_Argon2Common): - """ - argon2pure backend - """ - #=================================================================== - # backend loading - #=================================================================== - - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - # import argon2pure - global _argon2pure - try: - import argon2pure as _argon2pure - except ImportError: - return False - - # get default / max supported version -- added in v1.2.2 - try: - from argon2pure import ARGON2_DEFAULT_VERSION as max_version - except ImportError: - log.warning("detected 'argon2pure' backend, but package is too old " - "(passlib requires argon2pure >= 1.2.3)") - return False - - log.debug("detected 'argon2pure' backend, with support for 0x%x argon2 hashes", - max_version) - - if not dryrun: - warn("Using argon2pure backend, which is 100x+ slower than is required " - "for adequate security. Installing argon2_cffi (via 'pip install argon2_cffi') " - "is strongly recommended", exc.PasslibSecurityWarning) - - mixin_cls.version = mixin_cls.max_version = max_version - return mixin_cls._finalize_backend_mixin(name, dryrun) - - #=================================================================== - # primary methods - #=================================================================== - - # NOTE: this backend uses default .hash() & .verify() implementations. - - #=================================================================== - # digest calculation - #=================================================================== - def _calc_checksum(self, secret): - # TODO: add in 'encoding' support once that's finalized in 1.8 / 1.9. - uh.validate_secret(secret) - secret = to_bytes(secret, "utf-8") - if self.type_d: - type = _argon2pure.ARGON2D - else: - type = _argon2pure.ARGON2I - kwds = dict( - password=secret, - salt=self.salt, - time_cost=self.rounds, - memory_cost=self.memory_cost, - parallelism=self.parallelism, - tag_length=self.checksum_size, - type_code=type, - version=self.version, - ) - if self.max_threads > 0: - kwds['threads'] = self.max_threads - if self.pure_use_threads: - kwds['use_threads'] = True - if self.data: - kwds['associated_data'] = self.data - # NOTE: should return raw bytes - # NOTE: this may raise _argon2pure.Argon2ParameterError, - # but it if does that, there's a bug in our own parameter checking code. - try: - return _argon2pure.argon2(**kwds) - except _argon2pure.Argon2Error as err: - raise self._adapt_backend_error(err, self=self) - - #=================================================================== - # eoc - #=================================================================== - -class argon2(_NoBackend, _Argon2Common): - """ - This class implements the Argon2 password hash [#argon2-home]_, and follows the :ref:`password-hash-api`. - (This class only supports generating "Type I" argon2 hashes). - - Argon2 supports a variable-length salt, and variable time & memory cost, - and a number of other configurable parameters. - - The :meth:`~passlib.ifc.PasswordHash.replace` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If specified, the length must be between 0-1024 bytes. - If not specified, one will be auto-generated (this is recommended). - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - This corresponds linearly to the amount of time hashing will take. - - :type time_cost: int - :param time_cost: - An alias for **rounds**, for compatibility with underlying argon2 library. - - :param int memory_cost: - Defines the memory usage in kibibytes. - This corresponds linearly to the amount of memory hashing will take. - - :param int parallelism: - Defines the parallelization factor. - *NOTE: this will affect the resulting hash value.* - - :param int digest_size: - Length of the digest in bytes. - - :param int max_threads: - Maximum number of threads that will be used. - -1 means unlimited; otherwise hashing will use ``min(parallelism, max_threads)`` threads. - - .. note:: - - This option is currently only honored by the argon2pure backend. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. todo:: - - * Support configurable threading limits. - """ - #============================================================================= - # backend - #============================================================================= - - # NOTE: the brunt of the argon2 class is implemented in _Argon2Common. - # there are then subclass for each backend (e.g. _PureBackend), - # these are dynamically prepended to this class's bases - # in order to load the appropriate backend. - - #: list of potential backends - backends = ("argon2_cffi", "argon2pure") - - #: flag that this class's bases should be modified by SubclassBackendMixin - _backend_mixin_target = True - - #: map of backend -> mixin class, used by _get_backend_loader() - _backend_mixin_map = { - None: _NoBackend, - "argon2_cffi": _CffiBackend, - "argon2pure": _PureBackend, - } - - #============================================================================= - # - #============================================================================= - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/bcrypt.py b/src/passlib/handlers/bcrypt.py deleted file mode 100644 index e5fbfe0f..00000000 --- a/src/passlib/handlers/bcrypt.py +++ /dev/null @@ -1,1019 +0,0 @@ -"""passlib.bcrypt -- implementation of OpenBSD's BCrypt algorithm. - -TODO: - -* support 2x and altered-2a hashes? - http://www.openwall.com/lists/oss-security/2011/06/27/9 - -* deal with lack of PY3-compatibile c-ext implementation -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement, absolute_import -# core -from base64 import b64encode -from hashlib import sha256 -import os -import re -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -_bcrypt = None # dynamically imported by _load_backend_bcrypt() -_pybcrypt = None # dynamically imported by _load_backend_pybcrypt() -_bcryptor = None # dynamically imported by _load_backend_bcryptor() -# pkg -_builtin_bcrypt = None # dynamically imported by _load_backend_builtin() -from passlib.exc import PasslibHashWarning, PasslibSecurityWarning, PasslibSecurityError -from passlib.utils import safe_crypt, repeat_string, to_bytes, parse_version, \ - rng, getrandstr, test_crypt, to_unicode -from passlib.utils.binary import bcrypt64 -from passlib.utils.compat import u, uascii_to_str, unicode, str_to_uascii -import passlib.utils.handlers as uh - -# local -__all__ = [ - "bcrypt", -] - -#============================================================================= -# support funcs & constants -#============================================================================= -IDENT_2 = u("$2$") -IDENT_2A = u("$2a$") -IDENT_2X = u("$2x$") -IDENT_2Y = u("$2y$") -IDENT_2B = u("$2b$") -_BNULL = b'\x00' - -# reference hash of "test", used in various self-checks -TEST_HASH_2A = b"$2a$04$5BJqKfqMQvV7nS.yUguNcueVirQqDBGaLXSqj.rs.pZPlNR0UX/HK" - -def _detect_pybcrypt(): - """ - internal helper which tries to distinguish pybcrypt vs bcrypt. - - :returns: - True if cext-based py-bcrypt, - False if ffi-based bcrypt, - None if 'bcrypt' module not found. - - .. versionchanged:: 1.6.3 - - Now assuming bcrypt installed, unless py-bcrypt explicitly detected. - Previous releases assumed py-bcrypt by default. - - Making this change since py-bcrypt is (apparently) unmaintained and static, - whereas bcrypt is being actively maintained, and it's internal structure may shift. - """ - # NOTE: this is also used by the unittests. - - # check for module. - try: - import bcrypt - except ImportError: - return None - - # py-bcrypt has a "._bcrypt.__version__" attribute (confirmed for v0.1 - 0.4), - # which bcrypt lacks (confirmed for v1.0 - 2.0) - # "._bcrypt" alone isn't sufficient, since bcrypt 2.0 now has that attribute. - try: - from bcrypt._bcrypt import __version__ - except ImportError: - return False - return True - -#============================================================================= -# backend mixins -#============================================================================= -class _BcryptCommon(uh.SubclassBackendMixin, uh.TruncateMixin, uh.HasManyIdents, - uh.HasRounds, uh.HasSalt, uh.GenericHandler): - """ - Base class which implements brunt of BCrypt code. - This is then subclassed by the various backends, - to override w/ backend-specific methods. - - When a backend is loaded, the bases of the 'bcrypt' class proper - are modified to prepend the correct backend-specific subclass. - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "bcrypt" - setting_kwds = ("salt", "rounds", "ident", "truncate_error") - - #-------------------- - # GenericHandler - #-------------------- - checksum_size = 31 - checksum_chars = bcrypt64.charmap - - #-------------------- - # HasManyIdents - #-------------------- - default_ident = IDENT_2B - ident_values = (IDENT_2, IDENT_2A, IDENT_2X, IDENT_2Y, IDENT_2B) - ident_aliases = {u("2"): IDENT_2, u("2a"): IDENT_2A, u("2y"): IDENT_2Y, - u("2b"): IDENT_2B} - - #-------------------- - # HasSalt - #-------------------- - min_salt_size = max_salt_size = 22 - salt_chars = bcrypt64.charmap - # NOTE: 22nd salt char must be in bcrypt64._padinfo2[1], not full charmap - - #-------------------- - # HasRounds - #-------------------- - default_rounds = 12 # current passlib default - min_rounds = 4 # minimum from bcrypt specification - max_rounds = 31 # 32-bit integer limit (since real_rounds=1< class - - # NOTE: set_backend() will execute the ._load_backend_mixin() - # of the matching mixin class, which will handle backend detection - - # appended to HasManyBackends' "no backends available" error message - _no_backend_suggestion = " -- recommend you install one (e.g. 'pip install bcrypt')" - - @classmethod - def _finalize_backend_mixin(mixin_cls, backend, dryrun): - """ - helper called by from backend mixin classes' _load_backend_mixin() -- - invoked after backend imports have been loaded, and performs - feature detection & testing common to all backends. - """ - #---------------------------------------------------------------- - # setup helpers - #---------------------------------------------------------------- - assert mixin_cls is bcrypt._backend_mixin_map[backend], \ - "_configure_workarounds() invoked from wrong class" - - if mixin_cls._workrounds_initialized: - return True - - verify = mixin_cls.verify - - err_types = (ValueError,) - if _bcryptor: - err_types += (_bcryptor.engine.SaltError,) - - def safe_verify(secret, hash): - """verify() wrapper which traps 'unknown identifier' errors""" - try: - return verify(secret, hash) - except err_types: - # backends without support for given ident will throw various - # errors about unrecognized version: - # pybcrypt, bcrypt -- raises ValueError - # bcryptor -- raises bcryptor.engine.SaltError - return NotImplemented - except AssertionError as err: - # _calc_checksum() code may also throw AssertionError - # if correct hash isn't returned (e.g. 2y hash converted to 2b, - # such as happens with bcrypt 3.0.0) - log.debug("trapped unexpected response from %r backend: verify(%r, %r):", - backend, secret, hash, exc_info=True) - return NotImplemented - - def assert_lacks_8bit_bug(ident): - """ - helper to check for cryptblowfish 8bit bug (fixed in 2y/2b); - even though it's not known to be present in any of passlib's backends. - this is treated as FATAL, because it can easily result in seriously malformed hashes, - and we can't correct for it ourselves. - - test cases from - reference hash is the incorrectly generated $2x$ hash taken from above url - """ - secret = b"\xA3" - bug_hash = ident.encode("ascii") + b"05$/OK.fbVrR/bpIqNJ5ianF.CE5elHaaO4EbggVDjb8P19RukzXSM3e" - if verify(secret, bug_hash): - # NOTE: this only EVER be observed in 2a hashes, - # 2y/2b hashes should have fixed the bug. - # (but we check w/ them anyways). - raise PasslibSecurityError( - "passlib.hash.bcrypt: Your installation of the %r backend is vulnerable to " - "the crypt_blowfish 8-bit bug (CVE-2011-2483), " - "and should be upgraded or replaced with another backend." % backend) - - # if it doesn't have wraparound bug, make sure it *does* handle things - # correctly -- or we're in some weird third case. - correct_hash = ident.encode("ascii") + b"05$/OK.fbVrR/bpIqNJ5ianF.Sa7shbm4.OzKpvFnX1pQLmQW96oUlCq" - if not verify(secret, correct_hash): - raise RuntimeError("%s backend failed to verify %s 8bit hash" % (backend, ident)) - - def detect_wrap_bug(ident): - """ - check for bsd wraparound bug (fixed in 2b) - this is treated as a warning, because it's rare in the field, - and pybcrypt (as of 2015-7-21) is unpatched, but some people may be stuck with it. - - test cases from - - NOTE: reference hash is of password "0"*72 - - NOTE: if in future we need to deliberately create hashes which have this bug, - can use something like 'hashpw(repeat_string(secret[:((1+secret) % 256) or 1]), 72)' - """ - # check if it exhibits wraparound bug - secret = (b"0123456789"*26)[:255] - bug_hash = ident.encode("ascii") + b"04$R1lJ2gkNaoPGdafE.H.16.nVyh2niHsGJhayOHLMiXlI45o8/DU.6" - if verify(secret, bug_hash): - return True - - # if it doesn't have wraparound bug, make sure it *does* handle things - # correctly -- or we're in some weird third case. - correct_hash = ident.encode("ascii") + b"04$R1lJ2gkNaoPGdafE.H.16.1MKHPvmKwryeulRe225LKProWYwt9Oi" - if not verify(secret, correct_hash): - raise RuntimeError("%s backend failed to verify %s wraparound hash" % (backend, ident)) - - return False - - def assert_lacks_wrap_bug(ident): - if not detect_wrap_bug(ident): - return - # should only see in 2a, later idents should NEVER exhibit this bug: - # * 2y implementations should have been free of it - # * 2b was what (supposedly) fixed it - raise RuntimeError("%s backend unexpectedly has wraparound bug for %s" % (backend, ident)) - - #---------------------------------------------------------------- - # check for old 20 support - #---------------------------------------------------------------- - test_hash_20 = b"$2$04$5BJqKfqMQvV7nS.yUguNcuRfMMOXK0xPWavM7pOzjEi5ze5T1k8/S" - result = safe_verify("test", test_hash_20) - if not result: - raise RuntimeError("%s incorrectly rejected $2$ hash" % backend) - elif result is NotImplemented: - mixin_cls._lacks_20_support = True - log.debug("%r backend lacks $2$ support, enabling workaround", backend) - - #---------------------------------------------------------------- - # check for 2a support - #---------------------------------------------------------------- - result = safe_verify("test", TEST_HASH_2A) - if not result: - raise RuntimeError("%s incorrectly rejected $2a$ hash" % backend) - elif result is NotImplemented: - # 2a support is required, and should always be present - raise RuntimeError("%s lacks support for $2a$ hashes" % backend) - else: - assert_lacks_8bit_bug(IDENT_2A) - if detect_wrap_bug(IDENT_2A): - warn("passlib.hash.bcrypt: Your installation of the %r backend is vulnerable to " - "the bsd wraparound bug, " - "and should be upgraded or replaced with another backend " - "(enabling workaround for now)." % backend, - uh.exc.PasslibSecurityWarning) - mixin_cls._has_2a_wraparound_bug = True - - #---------------------------------------------------------------- - # check for 2y support - #---------------------------------------------------------------- - test_hash_2y = TEST_HASH_2A.replace(b"2a", b"2y") - result = safe_verify("test", test_hash_2y) - if not result: - raise RuntimeError("%s incorrectly rejected $2y$ hash" % backend) - elif result is NotImplemented: - mixin_cls._lacks_2y_support = True - log.debug("%r backend lacks $2y$ support, enabling workaround", backend) - else: - # NOTE: Not using this as fallback candidate, - # lacks wide enough support across implementations. - assert_lacks_8bit_bug(IDENT_2Y) - assert_lacks_wrap_bug(IDENT_2Y) - - #---------------------------------------------------------------- - # TODO: check for 2x support - #---------------------------------------------------------------- - - #---------------------------------------------------------------- - # check for 2b support - #---------------------------------------------------------------- - test_hash_2b = TEST_HASH_2A.replace(b"2a", b"2b") - result = safe_verify("test", test_hash_2b) - if not result: - raise RuntimeError("%s incorrectly rejected $2b$ hash" % backend) - elif result is NotImplemented: - mixin_cls._lacks_2b_support = True - log.debug("%r backend lacks $2b$ support, enabling workaround", backend) - else: - mixin_cls._fallback_ident = IDENT_2B - assert_lacks_8bit_bug(IDENT_2B) - assert_lacks_wrap_bug(IDENT_2B) - - # set flag so we don't have to run this again - mixin_cls._workrounds_initialized = True - return True - - #=================================================================== - # digest calculation - #=================================================================== - - # _calc_checksum() defined by backends - - def _prepare_digest_args(self, secret): - """ - common helper for backends to implement _calc_checksum(). - takes in secret, returns (secret, ident) pair, - """ - return self._norm_digest_args(secret, self.ident, new=self.use_defaults) - - @classmethod - def _norm_digest_args(cls, secret, ident, new=False): - # make sure secret is unicode - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - - # check max secret size - uh.validate_secret(secret) - - # check for truncation (during .hash() calls only) - if new: - cls._check_truncate_policy(secret) - - # NOTE: especially important to forbid NULLs for bcrypt, since many - # backends (bcryptor, bcrypt) happily accept them, and then - # silently truncate the password at first NULL they encounter! - if _BNULL in secret: - raise uh.exc.NullPasswordError(cls) - - # TODO: figure out way to skip these tests when not needed... - - # protect from wraparound bug by truncating secret before handing it to the backend. - # bcrypt only uses first 72 bytes anyways. - # NOTE: not needed for 2y/2b, but might use 2a as fallback for them. - if cls._has_2a_wraparound_bug and len(secret) >= 255: - secret = secret[:72] - - # special case handling for variants (ordered most common first) - if ident == IDENT_2A: - # nothing needs to be done. - pass - - elif ident == IDENT_2B: - if cls._lacks_2b_support: - # handle $2b$ hash format even if backend is too old. - # have it generate a 2A/2Y digest, then return it as a 2B hash. - # 2a-only backend could potentially exhibit wraparound bug -- - # but we work around that issue above. - ident = cls._fallback_ident - - elif ident == IDENT_2Y: - if cls._lacks_2y_support: - # handle $2y$ hash format (not supported by BSDs, being phased out on others) - # have it generate a 2A/2B digest, then return it as a 2Y hash. - ident = cls._fallback_ident - - elif ident == IDENT_2: - if cls._lacks_20_support: - # handle legacy $2$ format (not supported by most backends except BSD os_crypt) - # we can fake $2$ behavior using the 2A/2Y/2B algorithm - # by repeating the password until it's at least 72 chars in length. - if secret: - secret = repeat_string(secret, 72) - ident = cls._fallback_ident - - elif ident == IDENT_2X: - - # NOTE: shouldn't get here. - # XXX: could check if backend does actually offer 'support' - raise RuntimeError("$2x$ hashes not currently supported by passlib") - - else: - raise AssertionError("unexpected ident value: %r" % ident) - - return secret, ident - -#----------------------------------------------------------------------- -# stub backend -#----------------------------------------------------------------------- -class _NoBackend(_BcryptCommon): - """ - mixin used before any backend has been loaded. - contains stubs that force loading of one of the available backends. - """ - #=================================================================== - # digest calculation - #=================================================================== - def _calc_checksum(self, secret): - self._stub_requires_backend() - # NOTE: have to use super() here so that we don't recursively - # call subclass's wrapped _calc_checksum, e.g. bcrypt_sha256._calc_checksum - return super(bcrypt, self)._calc_checksum(secret) - - #=================================================================== - # eoc - #=================================================================== - -#----------------------------------------------------------------------- -# bcrypt backend -#----------------------------------------------------------------------- -class _BcryptBackend(_BcryptCommon): - """ - backend which uses 'bcrypt' package - """ - - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - # try to import bcrypt - global _bcrypt - if _detect_pybcrypt(): - # pybcrypt was installed instead - return False - try: - import bcrypt as _bcrypt - except ImportError: # pragma: no cover - return False - try: - version = _bcrypt.__about__.__version__ - except: - log.warning("(trapped) error reading bcrypt version", exc_info=True) - version = '' - - log.debug("detected 'bcrypt' backend, version %r", version) - return mixin_cls._finalize_backend_mixin(name, dryrun) - - # # TODO: would like to implementing verify() directly, - # # to skip need for parsing hash strings. - # # below method has a few edge cases where it chokes though. - # @classmethod - # def verify(cls, secret, hash): - # if isinstance(hash, unicode): - # hash = hash.encode("ascii") - # ident = hash[:hash.index(b"$", 1)+1].decode("ascii") - # if ident not in cls.ident_values: - # raise uh.exc.InvalidHashError(cls) - # secret, eff_ident = cls._norm_digest_args(secret, ident) - # if eff_ident != ident: - # # lacks support for original ident, replace w/ new one. - # hash = eff_ident.encode("ascii") + hash[len(ident):] - # result = _bcrypt.hashpw(secret, hash) - # assert result.startswith(eff_ident) - # return consteq(result, hash) - - def _calc_checksum(self, secret): - # bcrypt behavior: - # secret must be bytes - # config must be ascii bytes - # returns ascii bytes - secret, ident = self._prepare_digest_args(secret) - config = self._get_config(ident) - if isinstance(config, unicode): - config = config.encode("ascii") - hash = _bcrypt.hashpw(secret, config) - assert hash.startswith(config) and len(hash) == len(config)+31, \ - "config mismatch: %r => %r" % (config, hash) - assert isinstance(hash, bytes) - return hash[-31:].decode("ascii") - -#----------------------------------------------------------------------- -# bcryptor backend -#----------------------------------------------------------------------- -class _BcryptorBackend(_BcryptCommon): - """ - backend which uses 'bcryptor' package - """ - - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - # try to import bcryptor - global _bcryptor - try: - import bcryptor as _bcryptor - except ImportError: # pragma: no cover - return False - return mixin_cls._finalize_backend_mixin(name, dryrun) - - def _calc_checksum(self, secret): - # bcryptor behavior: - # py2: unicode secret/hash encoded as ascii bytes before use, - # bytes taken as-is; returns ascii bytes. - # py3: not supported - secret, ident = self._prepare_digest_args(secret) - config = self._get_config(ident) - hash = _bcryptor.engine.Engine(False).hash_key(secret, config) - assert hash.startswith(config) and len(hash) == len(config)+31 - return str_to_uascii(hash[-31:]) - -#----------------------------------------------------------------------- -# pybcrypt backend -#----------------------------------------------------------------------- -class _PyBcryptBackend(_BcryptCommon): - """ - backend which uses 'pybcrypt' package - """ - - #: classwide thread lock used for pybcrypt < 0.3 - _calc_lock = None - - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - # try to import pybcrypt - global _pybcrypt - if not _detect_pybcrypt(): - # not installed, or bcrypt installed instead - return False - try: - import bcrypt as _pybcrypt - except ImportError: # pragma: no cover - return False - - # determine pybcrypt version - try: - version = _pybcrypt._bcrypt.__version__ - except: - log.warning("(trapped) error reading pybcrypt version", exc_info=True) - version = "" - log.debug("detected 'pybcrypt' backend, version %r", version) - - # return calc function based on version - vinfo = parse_version(version) or (0, 0) - if vinfo < (0, 3): - warn("py-bcrypt %s has a major security vulnerability, " - "you should upgrade to py-bcrypt 0.3 immediately." - % version, uh.exc.PasslibSecurityWarning) - if mixin_cls._calc_lock is None: - import threading - mixin_cls._calc_lock = threading.Lock() - mixin_cls._calc_checksum = mixin_cls._calc_checksum_threadsafe.__func__ - - return mixin_cls._finalize_backend_mixin(name, dryrun) - - def _calc_checksum_threadsafe(self, secret): - # as workaround for pybcrypt < 0.3's concurrency issue, - # we wrap everything in a thread lock. as long as bcrypt is only - # used through passlib, this should be safe. - with self._calc_lock: - return self._calc_checksum_raw(secret) - - def _calc_checksum_raw(self, secret): - # py-bcrypt behavior: - # py2: unicode secret/hash encoded as ascii bytes before use, - # bytes taken as-is; returns ascii bytes. - # py3: unicode secret encoded as utf-8 bytes, - # hash encoded as ascii bytes, returns ascii unicode. - secret, ident = self._prepare_digest_args(secret) - config = self._get_config(ident) - hash = _pybcrypt.hashpw(secret, config) - assert hash.startswith(config) and len(hash) == len(config)+31 - return str_to_uascii(hash[-31:]) - - _calc_checksum = _calc_checksum_raw - -#----------------------------------------------------------------------- -# os crypt backend -#----------------------------------------------------------------------- -class _OsCryptBackend(_BcryptCommon): - """ - backend which uses :func:`crypt.crypt` - """ - - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - if not test_crypt("test", TEST_HASH_2A): - return False - return mixin_cls._finalize_backend_mixin(name, dryrun) - - def _calc_checksum(self, secret): - secret, ident = self._prepare_digest_args(secret) - config = self._get_config(ident) - hash = safe_crypt(secret, config) - if hash: - assert hash.startswith(config) and len(hash) == len(config)+31 - return hash[-31:] - else: - # NOTE: Have to raise this error because python3's crypt.crypt() only accepts unicode. - # This means it can't handle any passwords that aren't either unicode - # or utf-8 encoded bytes. However, hashing a password with an alternate - # encoding should be a pretty rare edge case; if user needs it, they can just - # install bcrypt backend. - # XXX: is this the right error type to raise? - # maybe have safe_crypt() not swallow UnicodeDecodeError, and have handlers - # like sha256_crypt trap it if they have alternate method of handling them? - raise uh.exc.MissingBackendError( - "non-utf8 encoded passwords can't be handled by crypt.crypt() under python3, " - "recommend running `pip install bcrypt`.", - ) - -#----------------------------------------------------------------------- -# builtin backend -#----------------------------------------------------------------------- -class _BuiltinBackend(_BcryptCommon): - """ - backend which uses passlib's pure-python implementation - """ - @classmethod - def _load_backend_mixin(mixin_cls, name, dryrun): - from passlib.utils import as_bool - if not as_bool(os.environ.get("PASSLIB_BUILTIN_BCRYPT")): - log.debug("bcrypt 'builtin' backend not enabled via $PASSLIB_BUILTIN_BCRYPT") - return False - global _builtin_bcrypt - from passlib.crypto._blowfish import raw_bcrypt as _builtin_bcrypt - return mixin_cls._finalize_backend_mixin(name, dryrun) - - def _calc_checksum(self, secret): - secret, ident = self._prepare_digest_args(secret) - chk = _builtin_bcrypt(secret, ident[1:-1], - self.salt.encode("ascii"), self.rounds) - return chk.decode("ascii") - -#============================================================================= -# handler -#============================================================================= -class bcrypt(_NoBackend, _BcryptCommon): - """This class implements the BCrypt password hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 22 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 12, must be between 4 and 31, inclusive. - This value is logarithmic, the actual number of iterations used will be :samp:`2**{rounds}` - -- increasing the rounds by +1 will double the amount of time taken. - - :type ident: str - :param ident: - Specifies which version of the BCrypt algorithm will be used when creating a new hash. - Typically this option is not needed, as the default (``"2b"``) is usually the correct choice. - If specified, it must be one of the following: - - * ``"2"`` - the first revision of BCrypt, which suffers from a minor security flaw and is generally not used anymore. - * ``"2a"`` - some implementations suffered from rare security flaws, replaced by 2b. - * ``"2y"`` - format specific to the *crypt_blowfish* BCrypt implementation, - identical to ``"2b"`` in all but name. - * ``"2b"`` - latest revision of the official BCrypt algorithm, current default. - - :param bool truncate_error: - By default, BCrypt will silently truncate passwords larger than 72 bytes. - Setting ``truncate_error=True`` will cause :meth:`~passlib.ifc.PasswordHash.hash` - to raise a :exc:`~passlib.exc.PasswordTruncateError` instead. - - .. versionadded:: 1.7 - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - - .. versionchanged:: 1.6 - This class now supports ``"2y"`` hashes, and recognizes - (but does not support) the broken ``"2x"`` hashes. - (see the :ref:`crypt_blowfish bug ` - for details). - - .. versionchanged:: 1.6 - Added a pure-python backend. - - .. versionchanged:: 1.6.3 - - Added support for ``"2b"`` variant. - - .. versionchanged:: 1.7 - - Now defaults to ``"2b"`` variant. - """ - #============================================================================= - # backend - #============================================================================= - - # NOTE: the brunt of the bcrypt class is implemented in _BcryptCommon. - # there are then subclass for each backend (e.g. _PyBcryptBackend), - # these are dynamically prepended to this class's bases - # in order to load the appropriate backend. - - #: list of potential backends - backends = ("bcrypt", "pybcrypt", "bcryptor", "os_crypt", "builtin") - - #: flag that this class's bases should be modified by SubclassBackendMixin - _backend_mixin_target = True - - #: map of backend -> mixin class, used by _get_backend_loader() - _backend_mixin_map = { - None: _NoBackend, - "bcrypt": _BcryptBackend, - "pybcrypt": _PyBcryptBackend, - "bcryptor": _BcryptorBackend, - "os_crypt": _OsCryptBackend, - "builtin": _BuiltinBackend, - } - - #============================================================================= - # eoc - #============================================================================= - -#============================================================================= -# variants -#============================================================================= -_UDOLLAR = u("$") - -# XXX: it might be better to have all the bcrypt variants share a common base class, -# and have the (django_)bcrypt_sha256 wrappers just proxy bcrypt instead of subclassing it. -class _wrapped_bcrypt(bcrypt): - """ - abstracts out some bits bcrypt_sha256 & django_bcrypt_sha256 share. - - bypass backend-loading wrappers for hash() etc - - disable truncation support, sha256 wrappers don't need it. - """ - setting_kwds = tuple(elem for elem in bcrypt.setting_kwds if elem not in ["truncate_error"]) - truncate_size = None - - # XXX: these will be needed if any bcrypt backends directly implement this... - # @classmethod - # def hash(cls, secret, **kwds): - # # bypass bcrypt backend overriding this method - # # XXX: would wrapping bcrypt make this easier than subclassing it? - # return super(_BcryptCommon, cls).hash(secret, **kwds) - # - # @classmethod - # def verify(cls, secret, hash): - # # bypass bcrypt backend overriding this method - # return super(_BcryptCommon, cls).verify(secret, hash) - # - # @classmethod - # def genhash(cls, secret, hash): - # # bypass bcrypt backend overriding this method - # return super(_BcryptCommon, cls).genhash(secret, hash) - - @classmethod - def _check_truncate_policy(cls, secret): - # disable check performed by bcrypt(), since this doesn't truncate passwords. - pass - -#============================================================================= -# bcrypt sha256 wrapper -#============================================================================= - -class bcrypt_sha256(_wrapped_bcrypt): - """This class implements a composition of BCrypt+SHA256, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept - all the same optional keywords as the base :class:`bcrypt` hash. - - .. versionadded:: 1.6.2 - - .. versionchanged:: 1.7 - - Now defaults to ``"2b"`` variant. - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "bcrypt_sha256" - - #-------------------- - # GenericHandler - #-------------------- - # this is locked at 2a/2b for now. - ident_values = (IDENT_2A, IDENT_2B) - - # clone bcrypt's ident aliases so they can be used here as well... - ident_aliases = (lambda ident_values: dict(item for item in bcrypt.ident_aliases.items() - if item[1] in ident_values))(ident_values) - default_ident = IDENT_2B - - #=================================================================== - # formatting - #=================================================================== - - # sample hash: - # $bcrypt-sha256$2a,6$/3OeRpbOf8/l6nPPRdZPp.$nRiyYqPobEZGdNRBWihQhiFDh1ws1tu - # $bcrypt-sha256$ -- prefix/identifier - # 2a -- bcrypt variant - # , -- field separator - # 6 -- bcrypt work factor - # $ -- section separator - # /3OeRpbOf8/l6nPPRdZPp. -- salt - # $ -- section separator - # nRiyYqPobEZGdNRBWihQhiFDh1ws1tu -- digest - - # XXX: we can't use .ident attr due to bcrypt code using it. - # working around that via prefix. - prefix = u('$bcrypt-sha256$') - - _hash_re = re.compile(r""" - ^ - [$]bcrypt-sha256 - [$](?P2[ab]) - ,(?P\d{1,2}) - [$](?P[^$]{22}) - (?:[$](?P.{31}))? - $ - """, re.X) - - @classmethod - def identify(cls, hash): - hash = uh.to_unicode_for_identify(hash) - if not hash: - return False - return hash.startswith(cls.prefix) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - if not hash.startswith(cls.prefix): - raise uh.exc.InvalidHashError(cls) - m = cls._hash_re.match(hash) - if not m: - raise uh.exc.MalformedHashError(cls) - rounds = m.group("rounds") - if rounds.startswith(uh._UZERO) and rounds != uh._UZERO: - raise uh.exc.ZeroPaddedRoundsError(cls) - return cls(ident=m.group("variant"), - rounds=int(rounds), - salt=m.group("salt"), - checksum=m.group("digest"), - ) - - _template = u("$bcrypt-sha256$%s,%d$%s$%s") - - def to_string(self): - hash = self._template % (self.ident.strip(_UDOLLAR), - self.rounds, self.salt, self.checksum) - return uascii_to_str(hash) - - #=================================================================== - # checksum - #=================================================================== - def _calc_checksum(self, secret): - # NOTE: can't use digest directly, since bcrypt stops at first NULL. - # NOTE: bcrypt doesn't fully mix entropy for bytes 55-72 of password - # (XXX: citation needed), so we don't want key to be > 55 bytes. - # thus, have to use base64 (44 bytes) rather than hex (64 bytes). - # XXX: it's later come out that 55-72 may be ok, so later revision of bcrypt_sha256 - # may switch to hex encoding, since it's simpler to implement elsewhere. - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - - # NOTE: output of b64encode() uses "+/" altchars, "=" padding chars, - # and no leading/trailing whitespace. - key = b64encode(sha256(secret).digest()) - - # hand result off to normal bcrypt algorithm - return super(bcrypt_sha256, self)._calc_checksum(key) - - #=================================================================== - # other - #=================================================================== - - # XXX: have _needs_update() mark the $2a$ ones for upgrading? - # maybe do that after we switch to hex encoding? - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/cisco.py b/src/passlib/handlers/cisco.py deleted file mode 100644 index e715e1ab..00000000 --- a/src/passlib/handlers/cisco.py +++ /dev/null @@ -1,440 +0,0 @@ -""" -passlib.handlers.cisco -- Cisco password hashes -""" -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify, unhexlify -from hashlib import md5 -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import right_pad_string, to_unicode, repeat_string, to_bytes -from passlib.utils.binary import h64 -from passlib.utils.compat import unicode, u, join_byte_values, \ - join_byte_elems, iter_byte_values, uascii_to_str -import passlib.utils.handlers as uh -# local -__all__ = [ - "cisco_pix", - "cisco_asa", - "cisco_type7", -] - -#============================================================================= -# utils -#============================================================================= - -#: dummy bytes used by spoil_digest var in cisco_pix._calc_checksum() -_DUMMY_BYTES = b'\xFF' * 32 - -#============================================================================= -# cisco pix firewall hash -#============================================================================= -class cisco_pix(uh.HasUserContext, uh.StaticHandler): - """ - This class implements the password hash used by older Cisco PIX firewalls, - and follows the :ref:`password-hash-api`. - It does a single round of hashing, and relies on the username - as the salt. - - This class only allows passwords <= 16 bytes, anything larger - will result in a :exc:`~passlib.exc.PasswordSizeError` if passed to :meth:`~cisco_pix.hash`, - and be silently rejected if passed to :meth:`~cisco_pix.verify`. - - The :meth:`~passlib.ifc.PasswordHash.hash`, - :meth:`~passlib.ifc.PasswordHash.genhash`, and - :meth:`~passlib.ifc.PasswordHash.verify` methods - all support the following extra keyword: - - :param str user: - String containing name of user account this password is associated with. - - This is *required* in order to correctly hash passwords associated - with a user account on the Cisco device, as it is used to salt - the hash. - - Conversely, this *must* be omitted or set to ``""`` in order to correctly - hash passwords which don't have an associated user account - (such as the "enable" password). - - .. versionadded:: 1.6 - - .. versionchanged:: 1.7.1 - - Passwords > 16 bytes are now rejected / throw error instead of being silently truncated, - to match Cisco behavior. A number of :ref:`bugs ` were fixed - which caused prior releases to generate unverifiable hashes in certain cases. - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "cisco_pix" - - truncate_size = 16 - - # NOTE: these are the default policy for PasswordHash, - # but want to set them explicitly for now. - truncate_error = True - truncate_verify_reject = True - - #-------------------- - # GenericHandler - #-------------------- - checksum_size = 16 - checksum_chars = uh.HASH64_CHARS - - #-------------------- - # custom - #-------------------- - - #: control flag signalling "cisco_asa" mode, set by cisco_asa class - _is_asa = False - - #=================================================================== - # methods - #=================================================================== - def _calc_checksum(self, secret): - """ - This function implements the "encrypted" hash format used by Cisco - PIX & ASA. It's behavior has been confirmed for ASA 9.6, - but is presumed correct for PIX & other ASA releases, - as it fits with known test vectors, and existing literature. - - While nearly the same, the PIX & ASA hashes have slight differences, - so this function performs differently based on the _is_asa class flag. - Noteable changes from PIX to ASA include password size limit - increased from 16 -> 32, and other internal changes. - """ - # select PIX vs or ASA mode - asa = self._is_asa - - # - # encode secret - # - # per ASA 8.4 documentation, - # http://www.cisco.com/c/en/us/td/docs/security/asa/asa84/configuration/guide/asa_84_cli_config/ref_cli.html#Supported_Character_Sets, - # it supposedly uses UTF-8 -- though some double-encoding issues have - # been observed when trying to actually *set* a non-ascii password - # via ASDM, and access via SSH seems to strip 8-bit chars. - # - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - - # - # check if password too large - # - # Per ASA 9.6 changes listed in - # http://www.cisco.com/c/en/us/td/docs/security/asa/roadmap/asa_new_features.html, - # prior releases had a maximum limit of 32 characters. - # Testing with an ASA 9.6 system bears this out -- - # setting 32-char password for a user account, - # and logins will fail if any chars are appended. - # (ASA 9.6 added new PBKDF2-based hash algorithm, - # which supports larger passwords). - # - # Per PIX documentation - # http://www.cisco.com/en/US/docs/security/pix/pix50/configuration/guide/commands.html, - # it would not allow passwords > 16 chars. - # - # Thus, we unconditionally throw a password size error here, - # as nothing valid can come from a larger password. - # NOTE: assuming PIX has same behavior, but at 16 char limit. - # - spoil_digest = None - if len(secret) > self.truncate_size: - if self.use_defaults: - # called from hash() - msg = "Password too long (%s allows at most %d bytes)" % \ - (self.name, self.truncate_size) - raise uh.exc.PasswordSizeError(self.truncate_size, msg=msg) - else: - # called from verify() -- - # We don't want to throw error, or return early, - # as that would let attacker know too much. Instead, we set a - # flag to add some dummy data into the md5 digest, so that - # output won't match truncated version of secret, or anything - # else that's fixed and predictable. - spoil_digest = secret + _DUMMY_BYTES - - # - # append user to secret - # - # Policy appears to be: - # - # * Nothing appended for enable password (user = "") - # - # * ASA: If user present, but secret is >= 28 chars, nothing appended. - # - # * 1-2 byte users not allowed. - # DEVIATION: we're letting them through, and repeating their - # chars ala 3-char user, to simplify testing. - # Could issue warning in the future though. - # - # * 3 byte user has first char repeated, to pad to 4. - # (observed under ASA 9.6, assuming true elsewhere) - # - # * 4 byte users are used directly. - # - # * 5+ byte users are truncated to 4 bytes. - # - user = self.user - if user: - if isinstance(user, unicode): - user = user.encode("utf-8") - if not asa or len(secret) < 28: - secret += repeat_string(user, 4) - - # - # pad / truncate result to limit - # - # While PIX always pads to 16 bytes, ASA increases to 32 bytes IFF - # secret+user > 16 bytes. This makes PIX & ASA have different results - # where secret size in range(13,16), and user is present -- - # PIX will truncate to 16, ASA will truncate to 32. - # - if asa and len(secret) > 16: - pad_size = 32 - else: - pad_size = 16 - secret = right_pad_string(secret, pad_size) - - # - # md5 digest - # - if spoil_digest: - # make sure digest won't match truncated version of secret - secret += spoil_digest - digest = md5(secret).digest() - - # - # drop every 4th byte - # NOTE: guessing this was done because it makes output exactly - # 16 bytes, which may have been a general 'char password[]' - # size limit under PIX - # - digest = join_byte_elems(c for i, c in enumerate(digest) if (i + 1) & 3) - - # - # encode using Hash64 - # - return h64.encode_bytes(digest).decode("ascii") - - # NOTE: works, but needs UTs. - # @classmethod - # def same_as_pix(cls, secret, user=""): - # """ - # test whether (secret + user) combination should - # have the same hash under PIX and ASA. - # - # mainly present to help unittests. - # """ - # # see _calc_checksum() above for details of this logic. - # size = len(to_bytes(secret, "utf-8")) - # if user and size < 28: - # size += 4 - # return size < 17 - - #=================================================================== - # eoc - #=================================================================== - - -class cisco_asa(cisco_pix): - """ - This class implements the password hash used by Cisco ASA/PIX 7.0 and newer (2005). - Aside from a different internal algorithm, it's use and format is identical - to the older :class:`cisco_pix` class. - - For passwords less than 13 characters, this should be identical to :class:`!cisco_pix`, - but will generate a different hash for most larger inputs - (See the `Format & Algorithm`_ section for the details). - - This class only allows passwords <= 32 bytes, anything larger - will result in a :exc:`~passlib.exc.PasswordSizeError` if passed to :meth:`~cisco_asa.hash`, - and be silently rejected if passed to :meth:`~cisco_asa.verify`. - - .. versionadded:: 1.7 - - .. versionchanged:: 1.7.1 - - Passwords > 32 bytes are now rejected / throw error instead of being silently truncated, - to match Cisco behavior. A number of :ref:`bugs ` were fixed - which caused prior releases to generate unverifiable hashes in certain cases. - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "cisco_asa" - - #-------------------- - # TruncateMixin - #-------------------- - truncate_size = 32 - - #-------------------- - # cisco_pix - #-------------------- - _is_asa = True - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# type 7 -#============================================================================= -class cisco_type7(uh.GenericHandler): - """ - This class implements the "Type 7" password encoding used by Cisco IOS, - and follows the :ref:`password-hash-api`. - It has a simple 4-5 bit salt, but is nonetheless a reversible encoding - instead of a real hash. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: int - :param salt: - This may be an optional salt integer drawn from ``range(0,16)``. - If omitted, one will be chosen at random. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` values that are out of range. - - Note that while this class outputs digests in upper-case hexadecimal, - it will accept lower-case as well. - - This class also provides the following additional method: - - .. automethod:: decode - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "cisco_type7" - setting_kwds = ("salt",) - - #-------------------- - # GenericHandler - #-------------------- - checksum_chars = uh.UPPER_HEX_CHARS - - #-------------------- - # HasSalt - #-------------------- - - # NOTE: encoding could handle max_salt_value=99, but since key is only 52 - # chars in size, not sure what appropriate behavior is for that edge case. - min_salt_value = 0 - max_salt_value = 52 - - #=================================================================== - # methods - #=================================================================== - @classmethod - def using(cls, salt=None, **kwds): - subcls = super(cisco_type7, cls).using(**kwds) - if salt is not None: - salt = subcls._norm_salt(salt, relaxed=kwds.get("relaxed")) - subcls._generate_salt = staticmethod(lambda: salt) - return subcls - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - if len(hash) < 2: - raise uh.exc.InvalidHashError(cls) - salt = int(hash[:2]) # may throw ValueError - return cls(salt=salt, checksum=hash[2:].upper()) - - def __init__(self, salt=None, **kwds): - super(cisco_type7, self).__init__(**kwds) - if salt is not None: - salt = self._norm_salt(salt) - elif self.use_defaults: - salt = self._generate_salt() - assert self._norm_salt(salt) == salt, "generated invalid salt: %r" % (salt,) - else: - raise TypeError("no salt specified") - self.salt = salt - - @classmethod - def _norm_salt(cls, salt, relaxed=False): - """ - validate & normalize salt value. - .. note:: - the salt for this algorithm is an integer 0-52, not a string - """ - if not isinstance(salt, int): - raise uh.exc.ExpectedTypeError(salt, "integer", "salt") - if 0 <= salt <= cls.max_salt_value: - return salt - msg = "salt/offset must be in 0..52 range" - if relaxed: - warn(msg, uh.PasslibHashWarning) - return 0 if salt < 0 else cls.max_salt_value - else: - raise ValueError(msg) - - @staticmethod - def _generate_salt(): - return uh.rng.randint(0, 15) - - def to_string(self): - return "%02d%s" % (self.salt, uascii_to_str(self.checksum)) - - def _calc_checksum(self, secret): - # XXX: no idea what unicode policy is, but all examples are - # 7-bit ascii compatible, so using UTF-8 - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return hexlify(self._cipher(secret, self.salt)).decode("ascii").upper() - - @classmethod - def decode(cls, hash, encoding="utf-8"): - """decode hash, returning original password. - - :arg hash: encoded password - :param encoding: optional encoding to use (defaults to ``UTF-8``). - :returns: password as unicode - """ - self = cls.from_string(hash) - tmp = unhexlify(self.checksum.encode("ascii")) - raw = self._cipher(tmp, self.salt) - return raw.decode(encoding) if encoding else raw - - # type7 uses a xor-based vingere variant, using the following secret key: - _key = u("dsfd;kfoA,.iyewrkldJKDHSUBsgvca69834ncxv9873254k;fg87") - - @classmethod - def _cipher(cls, data, salt): - """xor static key against data - encrypts & decrypts""" - key = cls._key - key_size = len(key) - return join_byte_values( - value ^ ord(key[(salt + idx) % key_size]) - for idx, value in enumerate(iter_byte_values(data)) - ) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/des_crypt.py b/src/passlib/handlers/des_crypt.py deleted file mode 100644 index 9561ab48..00000000 --- a/src/passlib/handlers/des_crypt.py +++ /dev/null @@ -1,607 +0,0 @@ -"""passlib.handlers.des_crypt - traditional unix (DES) crypt and variants""" -#============================================================================= -# imports -#============================================================================= -# core -import re -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import safe_crypt, test_crypt, to_unicode -from passlib.utils.binary import h64, h64big -from passlib.utils.compat import byte_elem_value, u, uascii_to_str, unicode, suppress_cause -from passlib.crypto.des import des_encrypt_int_block -import passlib.utils.handlers as uh -# local -__all__ = [ - "des_crypt", - "bsdi_crypt", - "bigcrypt", - "crypt16", -] - -#============================================================================= -# pure-python backend for des_crypt family -#============================================================================= -_BNULL = b'\x00' - -def _crypt_secret_to_key(secret): - """convert secret to 64-bit DES key. - - this only uses the first 8 bytes of the secret, - and discards the high 8th bit of each byte at that. - a null parity bit is inserted after every 7th bit of the output. - """ - # NOTE: this would set the parity bits correctly, - # but des_encrypt_int_block() would just ignore them... - ##return sum(expand_7bit(byte_elem_value(c) & 0x7f) << (56-i*8) - ## for i, c in enumerate(secret[:8])) - return sum((byte_elem_value(c) & 0x7f) << (57-i*8) - for i, c in enumerate(secret[:8])) - -def _raw_des_crypt(secret, salt): - """pure-python backed for des_crypt""" - assert len(salt) == 2 - - # NOTE: some OSes will accept non-HASH64 characters in the salt, - # but what value they assign these characters varies wildy, - # so just rejecting them outright. - # the same goes for single-character salts... - # some OSes duplicate the char, some insert a '.' char, - # and openbsd does (something) which creates an invalid hash. - salt_value = h64.decode_int12(salt) - - # gotta do something - no official policy since this predates unicode - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - assert isinstance(secret, bytes) - - # forbidding NULL char because underlying crypt() rejects them too. - if _BNULL in secret: - raise uh.exc.NullPasswordError(des_crypt) - - # convert first 8 bytes of secret string into an integer - key_value = _crypt_secret_to_key(secret) - - # run data through des using input of 0 - result = des_encrypt_int_block(key_value, 0, salt_value, 25) - - # run h64 encode on result - return h64big.encode_int64(result) - -def _bsdi_secret_to_key(secret): - """convert secret to DES key used by bsdi_crypt""" - key_value = _crypt_secret_to_key(secret) - idx = 8 - end = len(secret) - while idx < end: - next = idx + 8 - tmp_value = _crypt_secret_to_key(secret[idx:next]) - key_value = des_encrypt_int_block(key_value, key_value) ^ tmp_value - idx = next - return key_value - -def _raw_bsdi_crypt(secret, rounds, salt): - """pure-python backend for bsdi_crypt""" - - # decode salt - salt_value = h64.decode_int24(salt) - - # gotta do something - no official policy since this predates unicode - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - assert isinstance(secret, bytes) - - # forbidding NULL char because underlying crypt() rejects them too. - if _BNULL in secret: - raise uh.exc.NullPasswordError(bsdi_crypt) - - # convert secret string into an integer - key_value = _bsdi_secret_to_key(secret) - - # run data through des using input of 0 - result = des_encrypt_int_block(key_value, 0, salt_value, rounds) - - # run h64 encode on result - return h64big.encode_int64(result) - -#============================================================================= -# handlers -#============================================================================= -class des_crypt(uh.TruncateMixin, uh.HasManyBackends, uh.HasSalt, uh.GenericHandler): - """This class implements the des-crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 2 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :param bool truncate_error: - By default, des_crypt will silently truncate passwords larger than 8 bytes. - Setting ``truncate_error=True`` will cause :meth:`~passlib.ifc.PasswordHash.hash` - to raise a :exc:`~passlib.exc.PasswordTruncateError` instead. - - .. versionadded:: 1.7 - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "des_crypt" - setting_kwds = ("salt", "truncate_error") - - #-------------------- - # GenericHandler - #-------------------- - checksum_chars = uh.HASH64_CHARS - checksum_size = 11 - - #-------------------- - # HasSalt - #-------------------- - min_salt_size = max_salt_size = 2 - salt_chars = uh.HASH64_CHARS - - #-------------------- - # TruncateMixin - #-------------------- - truncate_size = 8 - - #=================================================================== - # formatting - #=================================================================== - # FORMAT: 2 chars of H64-encoded salt + 11 chars of H64-encoded checksum - - _hash_regex = re.compile(u(r""" - ^ - (?P[./a-z0-9]{2}) - (?P[./a-z0-9]{11})? - $"""), re.X|re.I) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - salt, chk = hash[:2], hash[2:] - return cls(salt=salt, checksum=chk or None) - - def to_string(self): - hash = u("%s%s") % (self.salt, self.checksum) - return uascii_to_str(hash) - - #=================================================================== - # digest calculation - #=================================================================== - def _calc_checksum(self, secret): - # check for truncation (during .hash() calls only) - if self.use_defaults: - self._check_truncate_policy(secret) - - return self._calc_checksum_backend(secret) - - #=================================================================== - # backend - #=================================================================== - backends = ("os_crypt", "builtin") - - #--------------------------------------------------------------- - # os_crypt backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_os_crypt(cls): - if test_crypt("test", 'abgOeLfPimXQo'): - cls._set_calc_checksum_backend(cls._calc_checksum_os_crypt) - return True - else: - return False - - def _calc_checksum_os_crypt(self, secret): - # NOTE: we let safe_crypt() encode unicode secret -> utf8; - # no official policy since des-crypt predates unicode - hash = safe_crypt(secret, self.salt) - if hash: - assert hash.startswith(self.salt) and len(hash) == 13 - return hash[2:] - else: - # py3's crypt.crypt() can't handle non-utf8 bytes. - # fallback to builtin alg, which is always available. - return self._calc_checksum_builtin(secret) - - #--------------------------------------------------------------- - # builtin backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_builtin(cls): - cls._set_calc_checksum_backend(cls._calc_checksum_builtin) - return True - - def _calc_checksum_builtin(self, secret): - return _raw_des_crypt(secret, self.salt.encode("ascii")).decode("ascii") - - #=================================================================== - # eoc - #=================================================================== - -class bsdi_crypt(uh.HasManyBackends, uh.HasRounds, uh.HasSalt, uh.GenericHandler): - """This class implements the BSDi-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 4 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 5001, must be between 1 and 16777215, inclusive. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - - .. versionchanged:: 1.6 - :meth:`hash` will now issue a warning if an even number of rounds is used - (see :ref:`bsdi-crypt-security-issues` regarding weak DES keys). - """ - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "bsdi_crypt" - setting_kwds = ("salt", "rounds") - checksum_size = 11 - checksum_chars = uh.HASH64_CHARS - - #--HasSalt-- - min_salt_size = max_salt_size = 4 - salt_chars = uh.HASH64_CHARS - - #--HasRounds-- - default_rounds = 5001 - min_rounds = 1 - max_rounds = 16777215 # (1<<24)-1 - rounds_cost = "linear" - - # NOTE: OpenBSD login.conf reports 7250 as minimum allowed rounds, - # but that seems to be an OS policy, not a algorithm limitation. - - #=================================================================== - # parsing - #=================================================================== - _hash_regex = re.compile(u(r""" - ^ - _ - (?P[./a-z0-9]{4}) - (?P[./a-z0-9]{4}) - (?P[./a-z0-9]{11})? - $"""), re.X|re.I) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - m = cls._hash_regex.match(hash) - if not m: - raise uh.exc.InvalidHashError(cls) - rounds, salt, chk = m.group("rounds", "salt", "chk") - return cls( - rounds=h64.decode_int24(rounds.encode("ascii")), - salt=salt, - checksum=chk, - ) - - def to_string(self): - hash = u("_%s%s%s") % (h64.encode_int24(self.rounds).decode("ascii"), - self.salt, self.checksum) - return uascii_to_str(hash) - - #=================================================================== - # validation - #=================================================================== - - # NOTE: keeping this flag for admin/choose_rounds.py script. - # want to eventually expose rounds logic to that script in better way. - _avoid_even_rounds = True - - @classmethod - def using(cls, **kwds): - subcls = super(bsdi_crypt, cls).using(**kwds) - if not subcls.default_rounds & 1: - # issue warning if caller set an even 'rounds' value. - warn("bsdi_crypt rounds should be odd, as even rounds may reveal weak DES keys", - uh.exc.PasslibSecurityWarning) - return subcls - - @classmethod - def _generate_rounds(cls): - rounds = super(bsdi_crypt, cls)._generate_rounds() - # ensure autogenerated rounds are always odd - # NOTE: doing this even for default_rounds so needs_update() doesn't get - # caught in a loop. - # FIXME: this technically might generate a rounds value 1 larger - # than the requested upper bound - but better to err on side of safety. - return rounds|1 - - #=================================================================== - # migration - #=================================================================== - - def _calc_needs_update(self, **kwds): - # mark bsdi_crypt hashes as deprecated if they have even rounds. - if not self.rounds & 1: - return True - # hand off to base implementation - return super(bsdi_crypt, self)._calc_needs_update(**kwds) - - #=================================================================== - # backends - #=================================================================== - backends = ("os_crypt", "builtin") - - #--------------------------------------------------------------- - # os_crypt backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_os_crypt(cls): - if test_crypt("test", '_/...lLDAxARksGCHin.'): - cls._set_calc_checksum_backend(cls._calc_checksum_os_crypt) - return True - else: - return False - - def _calc_checksum_os_crypt(self, secret): - config = self.to_string() - hash = safe_crypt(secret, config) - if hash: - assert hash.startswith(config[:9]) and len(hash) == 20 - return hash[-11:] - else: - # py3's crypt.crypt() can't handle non-utf8 bytes. - # fallback to builtin alg, which is always available. - return self._calc_checksum_builtin(secret) - - #--------------------------------------------------------------- - # builtin backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_builtin(cls): - cls._set_calc_checksum_backend(cls._calc_checksum_builtin) - return True - - def _calc_checksum_builtin(self, secret): - return _raw_bsdi_crypt(secret, self.rounds, self.salt.encode("ascii")).decode("ascii") - - #=================================================================== - # eoc - #=================================================================== - -class bigcrypt(uh.HasSalt, uh.GenericHandler): - """This class implements the BigCrypt password hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 22 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "bigcrypt" - setting_kwds = ("salt",) - checksum_chars = uh.HASH64_CHARS - # NOTE: checksum chars must be multiple of 11 - - #--HasSalt-- - min_salt_size = max_salt_size = 2 - salt_chars = uh.HASH64_CHARS - - #=================================================================== - # internal helpers - #=================================================================== - _hash_regex = re.compile(u(r""" - ^ - (?P[./a-z0-9]{2}) - (?P([./a-z0-9]{11})+)? - $"""), re.X|re.I) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - m = cls._hash_regex.match(hash) - if not m: - raise uh.exc.InvalidHashError(cls) - salt, chk = m.group("salt", "chk") - return cls(salt=salt, checksum=chk) - - def to_string(self): - hash = u("%s%s") % (self.salt, self.checksum) - return uascii_to_str(hash) - - def _norm_checksum(self, checksum, relaxed=False): - checksum = super(bigcrypt, self)._norm_checksum(checksum, relaxed=relaxed) - if len(checksum) % 11: - raise uh.exc.InvalidHashError(self) - return checksum - - #=================================================================== - # backend - #=================================================================== - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - chk = _raw_des_crypt(secret, self.salt.encode("ascii")) - idx = 8 - end = len(secret) - while idx < end: - next = idx + 8 - chk += _raw_des_crypt(secret[idx:next], chk[-11:-9]) - idx = next - return chk.decode("ascii") - - #=================================================================== - # eoc - #=================================================================== - -class crypt16(uh.TruncateMixin, uh.HasSalt, uh.GenericHandler): - """This class implements the crypt16 password hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 2 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :param bool truncate_error: - By default, crypt16 will silently truncate passwords larger than 16 bytes. - Setting ``truncate_error=True`` will cause :meth:`~passlib.ifc.PasswordHash.hash` - to raise a :exc:`~passlib.exc.PasswordTruncateError` instead. - - .. versionadded:: 1.7 - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "crypt16" - setting_kwds = ("salt", "truncate_error") - - #-------------------- - # GenericHandler - #-------------------- - checksum_size = 22 - checksum_chars = uh.HASH64_CHARS - - #-------------------- - # HasSalt - #-------------------- - min_salt_size = max_salt_size = 2 - salt_chars = uh.HASH64_CHARS - - #-------------------- - # TruncateMixin - #-------------------- - truncate_size = 16 - - #=================================================================== - # internal helpers - #=================================================================== - _hash_regex = re.compile(u(r""" - ^ - (?P[./a-z0-9]{2}) - (?P[./a-z0-9]{22})? - $"""), re.X|re.I) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - m = cls._hash_regex.match(hash) - if not m: - raise uh.exc.InvalidHashError(cls) - salt, chk = m.group("salt", "chk") - return cls(salt=salt, checksum=chk) - - def to_string(self): - hash = u("%s%s") % (self.salt, self.checksum) - return uascii_to_str(hash) - - #=================================================================== - # backend - #=================================================================== - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - - # check for truncation (during .hash() calls only) - if self.use_defaults: - self._check_truncate_policy(secret) - - # parse salt value - try: - salt_value = h64.decode_int12(self.salt.encode("ascii")) - except ValueError: # pragma: no cover - caught by class - raise suppress_cause(ValueError("invalid chars in salt")) - - # convert first 8 byts of secret string into an integer, - key1 = _crypt_secret_to_key(secret) - - # run data through des using input of 0 - result1 = des_encrypt_int_block(key1, 0, salt_value, 20) - - # convert next 8 bytes of secret string into integer (key=0 if secret < 8 chars) - key2 = _crypt_secret_to_key(secret[8:16]) - - # run data through des using input of 0 - result2 = des_encrypt_int_block(key2, 0, salt_value, 5) - - # done - chk = h64big.encode_int64(result1) + h64big.encode_int64(result2) - return chk.decode("ascii") - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/digests.py b/src/passlib/handlers/digests.py deleted file mode 100644 index 37610512..00000000 --- a/src/passlib/handlers/digests.py +++ /dev/null @@ -1,146 +0,0 @@ -"""passlib.handlers.digests - plain hash digests -""" -#============================================================================= -# imports -#============================================================================= -# core -import hashlib -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import to_native_str, to_bytes, render_bytes, consteq -from passlib.utils.compat import unicode, str_to_uascii -import passlib.utils.handlers as uh -from passlib.crypto.digest import lookup_hash -# local -__all__ = [ - "create_hex_hash", - "hex_md4", - "hex_md5", - "hex_sha1", - "hex_sha256", - "hex_sha512", -] - -#============================================================================= -# helpers for hexadecimal hashes -#============================================================================= -class HexDigestHash(uh.StaticHandler): - """this provides a template for supporting passwords stored as plain hexadecimal hashes""" - #=================================================================== - # class attrs - #=================================================================== - _hash_func = None # hash function to use - filled in by create_hex_hash() - checksum_size = None # filled in by create_hex_hash() - checksum_chars = uh.HEX_CHARS - - #=================================================================== - # methods - #=================================================================== - @classmethod - def _norm_hash(cls, hash): - return hash.lower() - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return str_to_uascii(self._hash_func(secret).hexdigest()) - - #=================================================================== - # eoc - #=================================================================== - -def create_hex_hash(digest, module=__name__): - # NOTE: could set digest_name=hash.name for cpython, but not for some other platforms. - info = lookup_hash(digest) - name = "hex_" + info.name - return type(name, (HexDigestHash,), dict( - name=name, - __module__=module, # so ABCMeta won't clobber it - _hash_func=staticmethod(info.const), # sometimes it's a function, sometimes not. so wrap it. - checksum_size=info.digest_size*2, - __doc__="""This class implements a plain hexadecimal %s hash, and follows the :ref:`password-hash-api`. - -It supports no optional or contextual keywords. -""" % (info.name,) - )) - -#============================================================================= -# predefined handlers -#============================================================================= -hex_md4 = create_hex_hash("md4") -hex_md5 = create_hex_hash("md5") -hex_md5.django_name = "unsalted_md5" -hex_sha1 = create_hex_hash("sha1") -hex_sha256 = create_hex_hash("sha256") -hex_sha512 = create_hex_hash("sha512") - -#============================================================================= -# htdigest -#============================================================================= -class htdigest(uh.MinimalHandler): - """htdigest hash function. - - .. todo:: - document this hash - """ - name = "htdigest" - setting_kwds = () - context_kwds = ("user", "realm", "encoding") - default_encoding = "utf-8" - - @classmethod - def hash(cls, secret, user, realm, encoding=None): - # NOTE: this was deliberately written so that raw bytes are passed through - # unchanged, the encoding kwd is only used to handle unicode values. - if not encoding: - encoding = cls.default_encoding - uh.validate_secret(secret) - if isinstance(secret, unicode): - secret = secret.encode(encoding) - user = to_bytes(user, encoding, "user") - realm = to_bytes(realm, encoding, "realm") - data = render_bytes("%s:%s:%s", user, realm, secret) - return hashlib.md5(data).hexdigest() - - @classmethod - def _norm_hash(cls, hash): - """normalize hash to native string, and validate it""" - hash = to_native_str(hash, param="hash") - if len(hash) != 32: - raise uh.exc.MalformedHashError(cls, "wrong size") - for char in hash: - if char not in uh.LC_HEX_CHARS: - raise uh.exc.MalformedHashError(cls, "invalid chars in hash") - return hash - - @classmethod - def verify(cls, secret, hash, user, realm, encoding="utf-8"): - hash = cls._norm_hash(hash) - other = cls.hash(secret, user, realm, encoding) - return consteq(hash, other) - - @classmethod - def identify(cls, hash): - try: - cls._norm_hash(hash) - except ValueError: - return False - return True - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genconfig(cls): - return cls.hash("", "", "") - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genhash(cls, secret, config, user, realm, encoding=None): - # NOTE: 'config' is ignored, as this hash has no salting / other configuration. - # just have to make sure it's valid. - cls._norm_hash(config) - return cls.hash(secret, user, realm, encoding) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/django.py b/src/passlib/handlers/django.py deleted file mode 100644 index 88906f51..00000000 --- a/src/passlib/handlers/django.py +++ /dev/null @@ -1,506 +0,0 @@ -"""passlib.handlers.django- Django password hash support""" -#============================================================================= -# imports -#============================================================================= -# core -from base64 import b64encode -from binascii import hexlify -from hashlib import md5, sha1, sha256 -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.handlers.bcrypt import _wrapped_bcrypt -from passlib.hash import argon2, bcrypt, pbkdf2_sha1, pbkdf2_sha256 -from passlib.utils import to_unicode, rng, getrandstr -from passlib.utils.binary import BASE64_CHARS -from passlib.utils.compat import str_to_uascii, uascii_to_str, unicode, u -from passlib.crypto.digest import pbkdf2_hmac -import passlib.utils.handlers as uh -# local -__all__ = [ - "django_salted_sha1", - "django_salted_md5", - "django_bcrypt", - "django_pbkdf2_sha1", - "django_pbkdf2_sha256", - "django_argon2", - "django_des_crypt", - "django_disabled", -] - -#============================================================================= -# lazy imports & constants -#============================================================================= - -# imported by django_des_crypt._calc_checksum() -des_crypt = None - -def _import_des_crypt(): - global des_crypt - if des_crypt is None: - from passlib.hash import des_crypt - return des_crypt - -# django 1.4's salt charset -SALT_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - -#============================================================================= -# salted hashes -#============================================================================= -class DjangoSaltedHash(uh.HasSalt, uh.GenericHandler): - """base class providing common code for django hashes""" - # name, ident, checksum_size must be set by subclass. - # ident must include "$" suffix. - setting_kwds = ("salt", "salt_size") - - # NOTE: django 1.0-1.3 would accept empty salt strings. - # django 1.4 won't, but this appears to be regression - # (https://code.djangoproject.com/ticket/18144) - # so presumably it will be fixed in a later release. - default_salt_size = 12 - max_salt_size = None - salt_chars = SALT_CHARS - - checksum_chars = uh.LOWER_HEX_CHARS - - @classmethod - def from_string(cls, hash): - salt, chk = uh.parse_mc2(hash, cls.ident, handler=cls) - return cls(salt=salt, checksum=chk) - - def to_string(self): - return uh.render_mc2(self.ident, self.salt, self.checksum) - -# NOTE: only used by PBKDF2 -class DjangoVariableHash(uh.HasRounds, DjangoSaltedHash): - """base class providing common code for django hashes w/ variable rounds""" - setting_kwds = DjangoSaltedHash.setting_kwds + ("rounds",) - - min_rounds = 1 - - @classmethod - def from_string(cls, hash): - rounds, salt, chk = uh.parse_mc3(hash, cls.ident, handler=cls) - return cls(rounds=rounds, salt=salt, checksum=chk) - - def to_string(self): - return uh.render_mc3(self.ident, self.rounds, self.salt, self.checksum) - -class django_salted_sha1(DjangoSaltedHash): - """This class implements Django's Salted SHA1 hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and uses a single round of SHA1. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, a 12 character one will be autogenerated (this is recommended). - If specified, may be any series of characters drawn from the regexp range ``[0-9a-zA-Z]``. - - :type salt_size: int - :param salt_size: - Optional number of characters to use when autogenerating new salts. - Defaults to 12, but can be any positive value. - - This should be compatible with Django 1.4's :class:`!SHA1PasswordHasher` class. - - .. versionchanged: 1.6 - This class now generates 12-character salts instead of 5, - and generated salts uses the character range ``[0-9a-zA-Z]`` instead of - the ``[0-9a-f]``. This is to be compatible with how Django >= 1.4 - generates these hashes; but hashes generated in this manner will still be - correctly interpreted by earlier versions of Django. - """ - name = "django_salted_sha1" - django_name = "sha1" - ident = u("sha1$") - checksum_size = 40 - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return str_to_uascii(sha1(self.salt.encode("ascii") + secret).hexdigest()) - -class django_salted_md5(DjangoSaltedHash): - """This class implements Django's Salted MD5 hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and uses a single round of MD5. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, a 12 character one will be autogenerated (this is recommended). - If specified, may be any series of characters drawn from the regexp range ``[0-9a-zA-Z]``. - - :type salt_size: int - :param salt_size: - Optional number of characters to use when autogenerating new salts. - Defaults to 12, but can be any positive value. - - This should be compatible with the hashes generated by - Django 1.4's :class:`!MD5PasswordHasher` class. - - .. versionchanged: 1.6 - This class now generates 12-character salts instead of 5, - and generated salts uses the character range ``[0-9a-zA-Z]`` instead of - the ``[0-9a-f]``. This is to be compatible with how Django >= 1.4 - generates these hashes; but hashes generated in this manner will still be - correctly interpreted by earlier versions of Django. - """ - name = "django_salted_md5" - django_name = "md5" - ident = u("md5$") - checksum_size = 32 - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return str_to_uascii(md5(self.salt.encode("ascii") + secret).hexdigest()) - -#============================================================================= -# BCrypt -#============================================================================= - -django_bcrypt = uh.PrefixWrapper("django_bcrypt", bcrypt, - prefix=u('bcrypt$'), ident=u("bcrypt$"), - # NOTE: this docstring is duplicated in the docs, since sphinx - # seems to be having trouble reading it via autodata:: - doc="""This class implements Django 1.4's BCrypt wrapper, and follows the :ref:`password-hash-api`. - - This is identical to :class:`!bcrypt` itself, but with - the Django-specific prefix ``"bcrypt$"`` prepended. - - See :doc:`/lib/passlib.hash.bcrypt` for more details, - the usage and behavior is identical. - - This should be compatible with the hashes generated by - Django 1.4's :class:`!BCryptPasswordHasher` class. - - .. versionadded:: 1.6 - """) -django_bcrypt.django_name = "bcrypt" -django_bcrypt._using_clone_attrs += ("django_name",) - -#============================================================================= -# BCRYPT + SHA256 -#============================================================================= - -class django_bcrypt_sha256(_wrapped_bcrypt): - """This class implements Django 1.6's Bcrypt+SHA256 hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - While the algorithm and format is somewhat different, - the api and options for this hash are identical to :class:`!bcrypt` itself, - see :doc:`bcrypt ` for more details. - - .. versionadded:: 1.6.2 - """ - name = "django_bcrypt_sha256" - django_name = "bcrypt_sha256" - _digest = sha256 - - # sample hash: - # bcrypt_sha256$$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu - - # XXX: we can't use .ident attr due to bcrypt code using it. - # working around that via django_prefix - django_prefix = u('bcrypt_sha256$') - - @classmethod - def identify(cls, hash): - hash = uh.to_unicode_for_identify(hash) - if not hash: - return False - return hash.startswith(cls.django_prefix) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - if not hash.startswith(cls.django_prefix): - raise uh.exc.InvalidHashError(cls) - bhash = hash[len(cls.django_prefix):] - if not bhash.startswith("$2"): - raise uh.exc.MalformedHashError(cls) - return super(django_bcrypt_sha256, cls).from_string(bhash) - - def to_string(self): - bhash = super(django_bcrypt_sha256, self).to_string() - return uascii_to_str(self.django_prefix) + bhash - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - secret = hexlify(self._digest(secret).digest()) - return super(django_bcrypt_sha256, self)._calc_checksum(secret) - -#============================================================================= -# PBKDF2 variants -#============================================================================= - -class django_pbkdf2_sha256(DjangoVariableHash): - """This class implements Django's PBKDF2-HMAC-SHA256 hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, a 12 character one will be autogenerated (this is recommended). - If specified, may be any series of characters drawn from the regexp range ``[0-9a-zA-Z]``. - - :type salt_size: int - :param salt_size: - Optional number of characters to use when autogenerating new salts. - Defaults to 12, but can be any positive value. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 29000, but must be within ``range(1,1<<32)``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - This should be compatible with the hashes generated by - Django 1.4's :class:`!PBKDF2PasswordHasher` class. - - .. versionadded:: 1.6 - """ - name = "django_pbkdf2_sha256" - django_name = "pbkdf2_sha256" - ident = u('pbkdf2_sha256$') - min_salt_size = 1 - max_rounds = 0xffffffff # setting at 32-bit limit for now - checksum_chars = uh.PADDED_BASE64_CHARS - checksum_size = 44 # 32 bytes -> base64 - default_rounds = pbkdf2_sha256.default_rounds # NOTE: django 1.6 uses 12000 - _digest = "sha256" - - def _calc_checksum(self, secret): - # NOTE: secret & salt will be encoded using UTF-8 by pbkdf2_hmac() - hash = pbkdf2_hmac(self._digest, secret, self.salt, self.rounds) - return b64encode(hash).rstrip().decode("ascii") - -class django_pbkdf2_sha1(django_pbkdf2_sha256): - """This class implements Django's PBKDF2-HMAC-SHA1 hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, a 12 character one will be autogenerated (this is recommended). - If specified, may be any series of characters drawn from the regexp range ``[0-9a-zA-Z]``. - - :type salt_size: int - :param salt_size: - Optional number of characters to use when autogenerating new salts. - Defaults to 12, but can be any positive value. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 131000, but must be within ``range(1,1<<32)``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - This should be compatible with the hashes generated by - Django 1.4's :class:`!PBKDF2SHA1PasswordHasher` class. - - .. versionadded:: 1.6 - """ - name = "django_pbkdf2_sha1" - django_name = "pbkdf2_sha1" - ident = u('pbkdf2_sha1$') - checksum_size = 28 # 20 bytes -> base64 - default_rounds = pbkdf2_sha1.default_rounds # NOTE: django 1.6 uses 12000 - _digest = "sha1" - -#============================================================================= -# Argon2 -#============================================================================= - -django_argon2 = uh.PrefixWrapper("django_argon2", argon2, - prefix=u('argon2'), ident=u('argon2$argon2i$'), - # NOTE: this docstring is duplicated in the docs, since sphinx - # seems to be having trouble reading it via autodata:: - doc="""This class implements Django 1.10's Argon2 wrapper, and follows the :ref:`password-hash-api`. - - This is identical to :class:`!argon2` itself, but with - the Django-specific prefix ``"argon2$"`` prepended. - - See :doc:`argon2 ` for more details, - the usage and behavior is identical. - - This should be compatible with the hashes generated by - Django 1.10's :class:`!Argon2PasswordHasher` class. - - .. versionadded:: 1.7 - """) -django_argon2.django_name = "argon2" -django_argon2._using_clone_attrs += ("django_name",) - -#============================================================================= -# DES -#============================================================================= -class django_des_crypt(uh.TruncateMixin, uh.HasSalt, uh.GenericHandler): - """This class implements Django's :class:`des_crypt` wrapper, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 2 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :param bool truncate_error: - By default, django_des_crypt will silently truncate passwords larger than 8 bytes. - Setting ``truncate_error=True`` will cause :meth:`~passlib.ifc.PasswordHash.hash` - to raise a :exc:`~passlib.exc.PasswordTruncateError` instead. - - .. versionadded:: 1.7 - - This should be compatible with the hashes generated by - Django 1.4's :class:`!CryptPasswordHasher` class. - Note that Django only supports this hash on Unix systems - (though :class:`!django_des_crypt` is available cross-platform - under Passlib). - - .. versionchanged:: 1.6 - This class will now accept hashes with empty salt strings, - since Django 1.4 generates them this way. - """ - name = "django_des_crypt" - django_name = "crypt" - setting_kwds = ("salt", "salt_size", "truncate_error") - ident = u("crypt$") - checksum_chars = salt_chars = uh.HASH64_CHARS - checksum_size = 11 - min_salt_size = default_salt_size = 2 - truncate_size = 8 - - # NOTE: regarding duplicate salt field: - # - # django 1.0 had a "crypt$$" hash format, - # used [a-z0-9] to generate a 5 char salt, stored it in salt1, - # duplicated the first two chars of salt1 as salt2. - # it would throw an error if salt1 was empty. - # - # django 1.4 started generating 2 char salt using the full alphabet, - # left salt1 empty, and only paid attention to salt2. - # - # in order to be compatible with django 1.0, the hashes generated - # by this function will always include salt1, unless the following - # class-level field is disabled (mainly used for testing) - use_duplicate_salt = True - - @classmethod - def from_string(cls, hash): - salt, chk = uh.parse_mc2(hash, cls.ident, handler=cls) - if chk: - # chk should be full des_crypt hash - if not salt: - # django 1.4 always uses empty salt field, - # so extract salt from des_crypt hash - salt = chk[:2] - elif salt[:2] != chk[:2]: - # django 1.0 stored 5 chars in salt field, and duplicated - # the first two chars in . we keep the full salt, - # but make sure the first two chars match as sanity check. - raise uh.exc.MalformedHashError(cls, - "first two digits of salt and checksum must match") - # in all cases, strip salt chars from - chk = chk[2:] - return cls(salt=salt, checksum=chk) - - def to_string(self): - salt = self.salt - chk = salt[:2] + self.checksum - if self.use_duplicate_salt: - # filling in salt field, so that we're compatible with django 1.0 - return uh.render_mc2(self.ident, salt, chk) - else: - # django 1.4+ style hash - return uh.render_mc2(self.ident, "", chk) - - def _calc_checksum(self, secret): - # NOTE: we lazily import des_crypt, - # since most django deploys won't use django_des_crypt - global des_crypt - if des_crypt is None: - _import_des_crypt() - # check for truncation (during .hash() calls only) - if self.use_defaults: - self._check_truncate_policy(secret) - return des_crypt(salt=self.salt[:2])._calc_checksum(secret) - -class django_disabled(uh.ifc.DisabledHash, uh.StaticHandler): - """This class provides disabled password behavior for Django, and follows the :ref:`password-hash-api`. - - This class does not implement a hash, but instead - claims the special hash string ``"!"`` which Django uses - to indicate an account's password has been disabled. - - * newly encrypted passwords will hash to ``"!"``. - * it rejects all passwords. - - .. note:: - - Django 1.6 prepends a randomly generated 40-char alphanumeric string - to each unusuable password. This class recognizes such strings, - but for backwards compatibility, still returns ``"!"``. - - See ``_ for why - Django appends an alphanumeric string. - - .. versionchanged:: 1.6.2 added Django 1.6 support - - .. versionchanged:: 1.7 started appending an alphanumeric string. - """ - name = "django_disabled" - _hash_prefix = u("!") - suffix_length = 40 - - # XXX: move this to StaticHandler, or wherever _hash_prefix is being used? - @classmethod - def identify(cls, hash): - hash = uh.to_unicode_for_identify(hash) - return hash.startswith(cls._hash_prefix) - - def _calc_checksum(self, secret): - # generate random suffix to match django's behavior - return getrandstr(rng, BASE64_CHARS[:-2], self.suffix_length) - - @classmethod - def verify(cls, secret, hash): - uh.validate_secret(secret) - if not cls.identify(hash): - raise uh.exc.InvalidHashError(cls) - return False - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/fshp.py b/src/passlib/handlers/fshp.py deleted file mode 100644 index db13e745..00000000 --- a/src/passlib/handlers/fshp.py +++ /dev/null @@ -1,214 +0,0 @@ -"""passlib.handlers.fshp -""" - -#============================================================================= -# imports -#============================================================================= -# core -from base64 import b64encode, b64decode -import re -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import to_unicode -import passlib.utils.handlers as uh -from passlib.utils.compat import bascii_to_str, iteritems, u,\ - unicode -from passlib.crypto.digest import pbkdf1 -# local -__all__ = [ - 'fshp', -] -#============================================================================= -# sha1-crypt -#============================================================================= -class fshp(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """This class implements the FSHP password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :param salt: - Optional raw salt string. - If not specified, one will be autogenerated (this is recommended). - - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 16 bytes, but can be any non-negative value. - - :param rounds: - Optional number of rounds to use. - Defaults to 480000, must be between 1 and 4294967295, inclusive. - - :param variant: - Optionally specifies variant of FSHP to use. - - * ``0`` - uses SHA-1 digest (deprecated). - * ``1`` - uses SHA-2/256 digest (default). - * ``2`` - uses SHA-2/384 digest. - * ``3`` - uses SHA-2/512 digest. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "fshp" - setting_kwds = ("salt", "salt_size", "rounds", "variant") - checksum_chars = uh.PADDED_BASE64_CHARS - ident = u("{FSHP") - # checksum_size is property() that depends on variant - - #--HasRawSalt-- - default_salt_size = 16 # current passlib default, FSHP uses 8 - max_salt_size = None - - #--HasRounds-- - # FIXME: should probably use different default rounds - # based on the variant. setting for default variant (sha256) for now. - default_rounds = 480000 # current passlib default, FSHP uses 4096 - min_rounds = 1 # set by FSHP - max_rounds = 4294967295 # 32-bit integer limit - not set by FSHP - rounds_cost = "linear" - - #--variants-- - default_variant = 1 - _variant_info = { - # variant: (hash name, digest size) - 0: ("sha1", 20), - 1: ("sha256", 32), - 2: ("sha384", 48), - 3: ("sha512", 64), - } - _variant_aliases = dict( - [(unicode(k),k) for k in _variant_info] + - [(v[0],k) for k,v in iteritems(_variant_info)] - ) - - #=================================================================== - # configuration - #=================================================================== - @classmethod - def using(cls, variant=None, **kwds): - subcls = super(fshp, cls).using(**kwds) - if variant is not None: - subcls.default_variant = cls._norm_variant(variant) - return subcls - - #=================================================================== - # instance attrs - #=================================================================== - variant = None - - #=================================================================== - # init - #=================================================================== - def __init__(self, variant=None, **kwds): - # NOTE: variant must be set first, since it controls checksum size, etc. - self.use_defaults = kwds.get("use_defaults") # load this early - if variant is not None: - variant = self._norm_variant(variant) - elif self.use_defaults: - variant = self.default_variant - assert self._norm_variant(variant) == variant, "invalid default variant: %r" % (variant,) - else: - raise TypeError("no variant specified") - self.variant = variant - super(fshp, self).__init__(**kwds) - - @classmethod - def _norm_variant(cls, variant): - if isinstance(variant, bytes): - variant = variant.decode("ascii") - if isinstance(variant, unicode): - try: - variant = cls._variant_aliases[variant] - except KeyError: - raise ValueError("invalid fshp variant") - if not isinstance(variant, int): - raise TypeError("fshp variant must be int or known alias") - if variant not in cls._variant_info: - raise ValueError("invalid fshp variant") - return variant - - @property - def checksum_alg(self): - return self._variant_info[self.variant][0] - - @property - def checksum_size(self): - return self._variant_info[self.variant][1] - - #=================================================================== - # formatting - #=================================================================== - - _hash_regex = re.compile(u(r""" - ^ - \{FSHP - (\d+)\| # variant - (\d+)\| # salt size - (\d+)\} # rounds - ([a-zA-Z0-9+/]+={0,3}) # digest - $"""), re.X) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - m = cls._hash_regex.match(hash) - if not m: - raise uh.exc.InvalidHashError(cls) - variant, salt_size, rounds, data = m.group(1,2,3,4) - variant = int(variant) - salt_size = int(salt_size) - rounds = int(rounds) - try: - data = b64decode(data.encode("ascii")) - except TypeError: - raise uh.exc.MalformedHashError(cls) - salt = data[:salt_size] - chk = data[salt_size:] - return cls(salt=salt, checksum=chk, rounds=rounds, variant=variant) - - def to_string(self): - chk = self.checksum - salt = self.salt - data = bascii_to_str(b64encode(salt+chk)) - return "{FSHP%d|%d|%d}%s" % (self.variant, len(salt), self.rounds, data) - - #=================================================================== - # backend - #=================================================================== - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - # NOTE: for some reason, FSHP uses pbkdf1 with password & salt reversed. - # this has only a minimal impact on security, - # but it is worth noting this deviation. - return pbkdf1( - digest=self.checksum_alg, - secret=self.salt, - salt=secret, - rounds=self.rounds, - keylen=self.checksum_size, - ) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/ldap_digests.py b/src/passlib/handlers/ldap_digests.py deleted file mode 100644 index 4356f071..00000000 --- a/src/passlib/handlers/ldap_digests.py +++ /dev/null @@ -1,272 +0,0 @@ -"""passlib.handlers.digests - plain hash digests -""" -#============================================================================= -# imports -#============================================================================= -# core -from base64 import b64encode, b64decode -from hashlib import md5, sha1 -import logging; log = logging.getLogger(__name__) -import re -# site -# pkg -from passlib.handlers.misc import plaintext -from passlib.utils import unix_crypt_schemes, to_unicode -from passlib.utils.compat import uascii_to_str, unicode, u -from passlib.utils.decor import classproperty -import passlib.utils.handlers as uh -# local -__all__ = [ - "ldap_plaintext", - "ldap_md5", - "ldap_sha1", - "ldap_salted_md5", - "ldap_salted_sha1", - - ##"get_active_ldap_crypt_schemes", - "ldap_des_crypt", - "ldap_bsdi_crypt", - "ldap_md5_crypt", - "ldap_sha1_crypt" - "ldap_bcrypt", - "ldap_sha256_crypt", - "ldap_sha512_crypt", -] - -#============================================================================= -# ldap helpers -#============================================================================= -class _Base64DigestHelper(uh.StaticHandler): - """helper for ldap_md5 / ldap_sha1""" - # XXX: could combine this with hex digests in digests.py - - ident = None # required - prefix identifier - _hash_func = None # required - hash function - _hash_regex = None # required - regexp to recognize hash - checksum_chars = uh.PADDED_BASE64_CHARS - - @classproperty - def _hash_prefix(cls): - """tell StaticHandler to strip ident from checksum""" - return cls.ident - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - chk = self._hash_func(secret).digest() - return b64encode(chk).decode("ascii") - -class _SaltedBase64DigestHelper(uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """helper for ldap_salted_md5 / ldap_salted_sha1""" - setting_kwds = ("salt", "salt_size") - checksum_chars = uh.PADDED_BASE64_CHARS - - ident = None # required - prefix identifier - _hash_func = None # required - hash function - _hash_regex = None # required - regexp to recognize hash - min_salt_size = max_salt_size = 4 - - # NOTE: openldap implementation uses 4 byte salt, - # but it's been reported (issue 30) that some servers use larger salts. - # the semi-related rfc3112 recommends support for up to 16 byte salts. - min_salt_size = 4 - default_salt_size = 4 - max_salt_size = 16 - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - m = cls._hash_regex.match(hash) - if not m: - raise uh.exc.InvalidHashError(cls) - try: - data = b64decode(m.group("tmp").encode("ascii")) - except TypeError: - raise uh.exc.MalformedHashError(cls) - cs = cls.checksum_size - assert cs - return cls(checksum=data[:cs], salt=data[cs:]) - - def to_string(self): - data = self.checksum + self.salt - hash = self.ident + b64encode(data).decode("ascii") - return uascii_to_str(hash) - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return self._hash_func(secret + self.salt).digest() - -#============================================================================= -# implementations -#============================================================================= -class ldap_md5(_Base64DigestHelper): - """This class stores passwords using LDAP's plain MD5 format, and follows the :ref:`password-hash-api`. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods have no optional keywords. - """ - name = "ldap_md5" - ident = u("{MD5}") - _hash_func = md5 - _hash_regex = re.compile(u(r"^\{MD5\}(?P[+/a-zA-Z0-9]{22}==)$")) - -class ldap_sha1(_Base64DigestHelper): - """This class stores passwords using LDAP's plain SHA1 format, and follows the :ref:`password-hash-api`. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods have no optional keywords. - """ - name = "ldap_sha1" - ident = u("{SHA}") - _hash_func = sha1 - _hash_regex = re.compile(u(r"^\{SHA\}(?P[+/a-zA-Z0-9]{27}=)$")) - -class ldap_salted_md5(_SaltedBase64DigestHelper): - """This class stores passwords using LDAP's salted MD5 format, and follows the :ref:`password-hash-api`. - - It supports a 4-16 byte salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it may be any 4-16 byte string. - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 4 bytes for compatibility with the LDAP spec, - but some systems use larger salts, and Passlib supports - any value between 4-16. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - - .. versionchanged:: 1.6 - This format now supports variable length salts, instead of a fix 4 bytes. - """ - name = "ldap_salted_md5" - ident = u("{SMD5}") - checksum_size = 16 - _hash_func = md5 - _hash_regex = re.compile(u(r"^\{SMD5\}(?P[+/a-zA-Z0-9]{27,}={0,2})$")) - -class ldap_salted_sha1(_SaltedBase64DigestHelper): - """This class stores passwords using LDAP's salted SHA1 format, and follows the :ref:`password-hash-api`. - - It supports a 4-16 byte salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it may be any 4-16 byte string. - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 4 bytes for compatibility with the LDAP spec, - but some systems use larger salts, and Passlib supports - any value between 4-16. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - - .. versionchanged:: 1.6 - This format now supports variable length salts, instead of a fix 4 bytes. - """ - name = "ldap_salted_sha1" - ident = u("{SSHA}") - checksum_size = 20 - _hash_func = sha1 - _hash_regex = re.compile(u(r"^\{SSHA\}(?P[+/a-zA-Z0-9]{32,}={0,2})$")) - -class ldap_plaintext(plaintext): - """This class stores passwords in plaintext, and follows the :ref:`password-hash-api`. - - This class acts much like the generic :class:`!passlib.hash.plaintext` handler, - except that it will identify a hash only if it does NOT begin with the ``{XXX}`` identifier prefix - used by RFC2307 passwords. - - The :meth:`~passlib.ifc.PasswordHash.hash`, :meth:`~passlib.ifc.PasswordHash.genhash`, and :meth:`~passlib.ifc.PasswordHash.verify` methods all require the - following additional contextual keyword: - - :type encoding: str - :param encoding: - This controls the character encoding to use (defaults to ``utf-8``). - - This encoding will be used to encode :class:`!unicode` passwords - under Python 2, and decode :class:`!bytes` hashes under Python 3. - - .. versionchanged:: 1.6 - The ``encoding`` keyword was added. - """ - # NOTE: this subclasses plaintext, since all it does differently - # is override identify() - - name = "ldap_plaintext" - _2307_pat = re.compile(u(r"^\{\w+\}.*$")) - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genconfig(cls): - # Overridding plaintext.genconfig() since it returns "", - # but have to return non-empty value due to identify() below - return "!" - - @classmethod - def identify(cls, hash): - # NOTE: identifies all strings EXCEPT those with {XXX} prefix - hash = uh.to_unicode_for_identify(hash) - return bool(hash) and cls._2307_pat.match(hash) is None - -#============================================================================= -# {CRYPT} wrappers -# the following are wrappers around the base crypt algorithms, -# which add the ldap required {CRYPT} prefix -#============================================================================= -ldap_crypt_schemes = [ 'ldap_' + name for name in unix_crypt_schemes ] - -def _init_ldap_crypt_handlers(): - # NOTE: I don't like to implicitly modify globals() like this, - # but don't want to write out all these handlers out either :) - g = globals() - for wname in unix_crypt_schemes: - name = 'ldap_' + wname - g[name] = uh.PrefixWrapper(name, wname, prefix=u("{CRYPT}"), lazy=True) - del g -_init_ldap_crypt_handlers() - -##_lcn_host = None -##def get_host_ldap_crypt_schemes(): -## global _lcn_host -## if _lcn_host is None: -## from passlib.hosts import host_context -## schemes = host_context.schemes() -## _lcn_host = [ -## "ldap_" + name -## for name in unix_crypt_names -## if name in schemes -## ] -## return _lcn_host - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/md5_crypt.py b/src/passlib/handlers/md5_crypt.py deleted file mode 100644 index 993db4d2..00000000 --- a/src/passlib/handlers/md5_crypt.py +++ /dev/null @@ -1,346 +0,0 @@ -"""passlib.handlers.md5_crypt - md5-crypt algorithm""" -#============================================================================= -# imports -#============================================================================= -# core -from hashlib import md5 -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import safe_crypt, test_crypt, repeat_string -from passlib.utils.binary import h64 -from passlib.utils.compat import unicode, u -import passlib.utils.handlers as uh -# local -__all__ = [ - "md5_crypt", - "apr_md5_crypt", -] - -#============================================================================= -# pure-python backend -#============================================================================= -_BNULL = b"\x00" -_MD5_MAGIC = b"$1$" -_APR_MAGIC = b"$apr1$" - -# pre-calculated offsets used to speed up C digest stage (see notes below). -# sequence generated using the following: - ##perms_order = "p,pp,ps,psp,sp,spp".split(",") - ##def offset(i): - ## key = (("p" if i % 2 else "") + ("s" if i % 3 else "") + - ## ("p" if i % 7 else "") + ("" if i % 2 else "p")) - ## return perms_order.index(key) - ##_c_digest_offsets = [(offset(i), offset(i+1)) for i in range(0,42,2)] -_c_digest_offsets = ( - (0, 3), (5, 1), (5, 3), (1, 2), (5, 1), (5, 3), (1, 3), - (4, 1), (5, 3), (1, 3), (5, 0), (5, 3), (1, 3), (5, 1), - (4, 3), (1, 3), (5, 1), (5, 2), (1, 3), (5, 1), (5, 3), - ) - -# map used to transpose bytes when encoding final digest -_transpose_map = (12, 6, 0, 13, 7, 1, 14, 8, 2, 15, 9, 3, 5, 10, 4, 11) - -def _raw_md5_crypt(pwd, salt, use_apr=False): - """perform raw md5-crypt calculation - - this function provides a pure-python implementation of the internals - for the MD5-Crypt algorithms; it doesn't handle any of the - parsing/validation of the hash strings themselves. - - :arg pwd: password chars/bytes to hash - :arg salt: salt chars to use - :arg use_apr: use apache variant - - :returns: - encoded checksum chars - """ - # NOTE: regarding 'apr' format: - # really, apache? you had to invent a whole new "$apr1$" format, - # when all you did was change the ident incorporated into the hash? - # would love to find webpage explaining why just using a portable - # implementation of $1$ wasn't sufficient. *nothing else* was changed. - - #=================================================================== - # init & validate inputs - #=================================================================== - - # validate secret - # XXX: not sure what official unicode policy is, using this as default - if isinstance(pwd, unicode): - pwd = pwd.encode("utf-8") - assert isinstance(pwd, bytes), "pwd not unicode or bytes" - if _BNULL in pwd: - raise uh.exc.NullPasswordError(md5_crypt) - pwd_len = len(pwd) - - # validate salt - should have been taken care of by caller - assert isinstance(salt, unicode), "salt not unicode" - salt = salt.encode("ascii") - assert len(salt) < 9, "salt too large" - # NOTE: spec says salts larger than 8 bytes should be truncated, - # instead of causing an error. this function assumes that's been - # taken care of by the handler class. - - # load APR specific constants - if use_apr: - magic = _APR_MAGIC - else: - magic = _MD5_MAGIC - - #=================================================================== - # digest B - used as subinput to digest A - #=================================================================== - db = md5(pwd + salt + pwd).digest() - - #=================================================================== - # digest A - used to initialize first round of digest C - #=================================================================== - # start out with pwd + magic + salt - a_ctx = md5(pwd + magic + salt) - a_ctx_update = a_ctx.update - - # add pwd_len bytes of b, repeating b as many times as needed. - a_ctx_update(repeat_string(db, pwd_len)) - - # add null chars & first char of password - # NOTE: this may have historically been a bug, - # where they meant to use db[0] instead of B_NULL, - # but the original code memclear'ed db, - # and now all implementations have to use this. - i = pwd_len - evenchar = pwd[:1] - while i: - a_ctx_update(_BNULL if i & 1 else evenchar) - i >>= 1 - - # finish A - da = a_ctx.digest() - - #=================================================================== - # digest C - for a 1000 rounds, combine A, S, and P - # digests in various ways; in order to burn CPU time. - #=================================================================== - - # NOTE: the original MD5-Crypt implementation performs the C digest - # calculation using the following loop: - # - ##dc = da - ##i = 0 - ##while i < rounds: - ## tmp_ctx = md5(pwd if i & 1 else dc) - ## if i % 3: - ## tmp_ctx.update(salt) - ## if i % 7: - ## tmp_ctx.update(pwd) - ## tmp_ctx.update(dc if i & 1 else pwd) - ## dc = tmp_ctx.digest() - ## i += 1 - # - # The code Passlib uses (below) implements an equivalent algorithm, - # it's just been heavily optimized to pre-calculate a large number - # of things beforehand. It works off of a couple of observations - # about the original algorithm: - # - # 1. each round is a combination of 'dc', 'salt', and 'pwd'; and the exact - # combination is determined by whether 'i' a multiple of 2,3, and/or 7. - # 2. since lcm(2,3,7)==42, the series of combinations will repeat - # every 42 rounds. - # 3. even rounds 0-40 consist of 'hash(dc + round-specific-constant)'; - # while odd rounds 1-41 consist of hash(round-specific-constant + dc) - # - # Using these observations, the following code... - # * calculates the round-specific combination of salt & pwd for each round 0-41 - # * runs through as many 42-round blocks as possible (23) - # * runs through as many pairs of rounds as needed for remaining rounds (17) - # * this results in the required 42*23+2*17=1000 rounds required by md5_crypt. - # - # this cuts out a lot of the control overhead incurred when running the - # original loop 1000 times in python, resulting in ~20% increase in - # speed under CPython (though still 2x slower than glibc crypt) - - # prepare the 6 combinations of pwd & salt which are needed - # (order of 'perms' must match how _c_digest_offsets was generated) - pwd_pwd = pwd+pwd - pwd_salt = pwd+salt - perms = [pwd, pwd_pwd, pwd_salt, pwd_salt+pwd, salt+pwd, salt+pwd_pwd] - - # build up list of even-round & odd-round constants, - # and store in 21-element list as (even,odd) pairs. - data = [ (perms[even], perms[odd]) for even, odd in _c_digest_offsets] - - # perform 23 blocks of 42 rounds each (for a total of 966 rounds) - dc = da - blocks = 23 - while blocks: - for even, odd in data: - dc = md5(odd + md5(dc + even).digest()).digest() - blocks -= 1 - - # perform 17 more pairs of rounds (34 more rounds, for a total of 1000) - for even, odd in data[:17]: - dc = md5(odd + md5(dc + even).digest()).digest() - - #=================================================================== - # encode digest using appropriate transpose map - #=================================================================== - return h64.encode_transposed_bytes(dc, _transpose_map).decode("ascii") - -#============================================================================= -# handler -#============================================================================= -class _MD5_Common(uh.HasSalt, uh.GenericHandler): - """common code for md5_crypt and apr_md5_crypt""" - #=================================================================== - # class attrs - #=================================================================== - # name - set in subclass - setting_kwds = ("salt", "salt_size") - # ident - set in subclass - checksum_size = 22 - checksum_chars = uh.HASH64_CHARS - - max_salt_size = 8 - salt_chars = uh.HASH64_CHARS - - #=================================================================== - # methods - #=================================================================== - - @classmethod - def from_string(cls, hash): - salt, chk = uh.parse_mc2(hash, cls.ident, handler=cls) - return cls(salt=salt, checksum=chk) - - def to_string(self): - return uh.render_mc2(self.ident, self.salt, self.checksum) - - # _calc_checksum() - provided by subclass - - #=================================================================== - # eoc - #=================================================================== - -class md5_crypt(uh.HasManyBackends, _MD5_Common): - """This class implements the MD5-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 0-8 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type salt_size: int - :param salt_size: - Optional number of characters to use when autogenerating new salts. - Defaults to 8, but can be any value between 0 and 8. - (This is mainly needed when generating Cisco-compatible hashes, - which require ``salt_size=4``). - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - name = "md5_crypt" - ident = u("$1$") - - #=================================================================== - # methods - #=================================================================== - # FIXME: can't find definitive policy on how md5-crypt handles non-ascii. - # all backends currently coerce -> utf-8 - - backends = ("os_crypt", "builtin") - - #--------------------------------------------------------------- - # os_crypt backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_os_crypt(cls): - if test_crypt("test", '$1$test$pi/xDtU5WFVRqYS6BMU8X/'): - cls._set_calc_checksum_backend(cls._calc_checksum_os_crypt) - return True - else: - return False - - def _calc_checksum_os_crypt(self, secret): - config = self.ident + self.salt - hash = safe_crypt(secret, config) - if hash: - assert hash.startswith(config) and len(hash) == len(config) + 23 - return hash[-22:] - else: - # py3's crypt.crypt() can't handle non-utf8 bytes. - # fallback to builtin alg, which is always available. - return self._calc_checksum_builtin(secret) - - #--------------------------------------------------------------- - # builtin backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_builtin(cls): - cls._set_calc_checksum_backend(cls._calc_checksum_builtin) - return True - - def _calc_checksum_builtin(self, secret): - return _raw_md5_crypt(secret, self.salt) - - #=================================================================== - # eoc - #=================================================================== - -class apr_md5_crypt(_MD5_Common): - """This class implements the Apr-MD5-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 0-8 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - name = "apr_md5_crypt" - ident = u("$apr1$") - - #=================================================================== - # methods - #=================================================================== - def _calc_checksum(self, secret): - return _raw_md5_crypt(secret, self.salt, use_apr=True) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/misc.py b/src/passlib/handlers/misc.py deleted file mode 100644 index 44abc343..00000000 --- a/src/passlib/handlers/misc.py +++ /dev/null @@ -1,269 +0,0 @@ -"""passlib.handlers.misc - misc generic handlers -""" -#============================================================================= -# imports -#============================================================================= -# core -import sys -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import to_native_str, str_consteq -from passlib.utils.compat import unicode, u, unicode_or_bytes_types -import passlib.utils.handlers as uh -# local -__all__ = [ - "unix_disabled", - "unix_fallback", - "plaintext", -] - -#============================================================================= -# handler -#============================================================================= -class unix_fallback(uh.ifc.DisabledHash, uh.StaticHandler): - """This class provides the fallback behavior for unix shadow files, and follows the :ref:`password-hash-api`. - - This class does not implement a hash, but instead provides fallback - behavior as found in /etc/shadow on most unix variants. - If used, should be the last scheme in the context. - - * this class will positively identify all hash strings. - * for security, passwords will always hash to ``!``. - * it rejects all passwords if the hash is NOT an empty string (``!`` or ``*`` are frequently used). - * by default it rejects all passwords if the hash is an empty string, - but if ``enable_wildcard=True`` is passed to verify(), - all passwords will be allowed through if the hash is an empty string. - - .. deprecated:: 1.6 - This has been deprecated due to its "wildcard" feature, - and will be removed in Passlib 1.8. Use :class:`unix_disabled` instead. - """ - name = "unix_fallback" - context_kwds = ("enable_wildcard",) - - @classmethod - def identify(cls, hash): - if isinstance(hash, unicode_or_bytes_types): - return True - else: - raise uh.exc.ExpectedStringError(hash, "hash") - - def __init__(self, enable_wildcard=False, **kwds): - warn("'unix_fallback' is deprecated, " - "and will be removed in Passlib 1.8; " - "please use 'unix_disabled' instead.", - DeprecationWarning) - super(unix_fallback, self).__init__(**kwds) - self.enable_wildcard = enable_wildcard - - def _calc_checksum(self, secret): - if self.checksum: - # NOTE: hash will generally be "!", but we want to preserve - # it in case it's something else, like "*". - return self.checksum - else: - return u("!") - - @classmethod - def verify(cls, secret, hash, enable_wildcard=False): - uh.validate_secret(secret) - if not isinstance(hash, unicode_or_bytes_types): - raise uh.exc.ExpectedStringError(hash, "hash") - elif hash: - return False - else: - return enable_wildcard - -_MARKER_CHARS = u("*!") -_MARKER_BYTES = b"*!" - -class unix_disabled(uh.ifc.DisabledHash, uh.MinimalHandler): - """This class provides disabled password behavior for unix shadow files, - and follows the :ref:`password-hash-api`. - - This class does not implement a hash, but instead matches the "disabled account" - strings found in ``/etc/shadow`` on most Unix variants. "encrypting" a password - will simply return the disabled account marker. It will reject all passwords, - no matter the hash string. The :meth:`~passlib.ifc.PasswordHash.hash` - method supports one optional keyword: - - :type marker: str - :param marker: - Optional marker string which overrides the platform default - used to indicate a disabled account. - - If not specified, this will default to ``"*"`` on BSD systems, - and use the Linux default ``"!"`` for all other platforms. - (:attr:`!unix_disabled.default_marker` will contain the default value) - - .. versionadded:: 1.6 - This class was added as a replacement for the now-deprecated - :class:`unix_fallback` class, which had some undesirable features. - """ - name = "unix_disabled" - setting_kwds = ("marker",) - context_kwds = () - - _disable_prefixes = tuple(str(_MARKER_CHARS)) - - # TODO: rename attr to 'marker'... - if 'bsd' in sys.platform: # pragma: no cover -- runtime detection - default_marker = u("*") - else: - # use the linux default for other systems - # (glibc also supports adding old hash after the marker - # so it can be restored later). - default_marker = u("!") - - @classmethod - def using(cls, marker=None, **kwds): - subcls = super(unix_disabled, cls).using(**kwds) - if marker is not None: - if not cls.identify(marker): - raise ValueError("invalid marker: %r" % marker) - subcls.default_marker = marker - return subcls - - @classmethod - def identify(cls, hash): - # NOTE: technically, anything in the /etc/shadow password field - # which isn't valid crypt() output counts as "disabled". - # but that's rather ambiguous, and it's hard to predict what - # valid output is for unknown crypt() implementations. - # so to be on the safe side, we only match things *known* - # to be disabled field indicators, and will add others - # as they are found. things beginning w/ "$" should *never* match. - # - # things currently matched: - # * linux uses "!" - # * bsd uses "*" - # * linux may use "!" + hash to disable but preserve original hash - # * linux counts empty string as "any password"; - # this code recognizes it, but treats it the same as "!" - if isinstance(hash, unicode): - start = _MARKER_CHARS - elif isinstance(hash, bytes): - start = _MARKER_BYTES - else: - raise uh.exc.ExpectedStringError(hash, "hash") - return not hash or hash[0] in start - - @classmethod - def verify(cls, secret, hash): - uh.validate_secret(secret) - if not cls.identify(hash): # handles typecheck - raise uh.exc.InvalidHashError(cls) - return False - - @classmethod - def hash(cls, secret, **kwds): - if kwds: - uh.warn_hash_settings_deprecation(cls, kwds) - return cls.using(**kwds).hash(secret) - uh.validate_secret(secret) - marker = cls.default_marker - assert marker and cls.identify(marker) - return to_native_str(marker, param="marker") - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genhash(cls, secret, config, marker=None): - if not cls.identify(config): - raise uh.exc.InvalidHashError(cls) - elif config: - # preserve the existing str,since it might contain a disabled password hash ("!" + hash) - uh.validate_secret(secret) - return to_native_str(config, param="config") - else: - if marker is not None: - cls = cls.using(marker=marker) - return cls.hash(secret) - - @classmethod - def disable(cls, hash=None): - out = cls.hash("") - if hash is not None: - hash = to_native_str(hash, param="hash") - if cls.identify(hash): - # extract original hash, so that we normalize marker - hash = cls.enable(hash) - if hash: - out += hash - return out - - @classmethod - def enable(cls, hash): - hash = to_native_str(hash, param="hash") - for prefix in cls._disable_prefixes: - if hash.startswith(prefix): - orig = hash[len(prefix):] - if orig: - return orig - else: - raise ValueError("cannot restore original hash") - raise uh.exc.InvalidHashError(cls) - -class plaintext(uh.MinimalHandler): - """This class stores passwords in plaintext, and follows the :ref:`password-hash-api`. - - The :meth:`~passlib.ifc.PasswordHash.hash`, :meth:`~passlib.ifc.PasswordHash.genhash`, and :meth:`~passlib.ifc.PasswordHash.verify` methods all require the - following additional contextual keyword: - - :type encoding: str - :param encoding: - This controls the character encoding to use (defaults to ``utf-8``). - - This encoding will be used to encode :class:`!unicode` passwords - under Python 2, and decode :class:`!bytes` hashes under Python 3. - - .. versionchanged:: 1.6 - The ``encoding`` keyword was added. - """ - # NOTE: this is subclassed by ldap_plaintext - - name = "plaintext" - setting_kwds = () - context_kwds = ("encoding",) - default_encoding = "utf-8" - - @classmethod - def identify(cls, hash): - if isinstance(hash, unicode_or_bytes_types): - return True - else: - raise uh.exc.ExpectedStringError(hash, "hash") - - @classmethod - def hash(cls, secret, encoding=None): - uh.validate_secret(secret) - if not encoding: - encoding = cls.default_encoding - return to_native_str(secret, encoding, "secret") - - @classmethod - def verify(cls, secret, hash, encoding=None): - if not encoding: - encoding = cls.default_encoding - hash = to_native_str(hash, encoding, "hash") - if not cls.identify(hash): - raise uh.exc.InvalidHashError(cls) - return str_consteq(cls.hash(secret, encoding), hash) - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genconfig(cls): - return cls.hash("") - - @uh.deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genhash(cls, secret, config, encoding=None): - # NOTE: 'config' is ignored, as this hash has no salting / etc - if not cls.identify(config): - raise uh.exc.InvalidHashError(cls) - return cls.hash(secret, encoding=encoding) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/mssql.py b/src/passlib/handlers/mssql.py deleted file mode 100644 index b060b365..00000000 --- a/src/passlib/handlers/mssql.py +++ /dev/null @@ -1,244 +0,0 @@ -"""passlib.handlers.mssql - MS-SQL Password Hash - -Notes -===== -MS-SQL has used a number of hash algs over the years, -most of which were exposed through the undocumented -'pwdencrypt' and 'pwdcompare' sql functions. - -Known formats -------------- -6.5 - snefru hash, ascii encoded password - no examples found - -7.0 - snefru hash, unicode (what encoding?) - saw ref that these blobs were 16 bytes in size - no examples found - -2000 - byte string using displayed as 0x hex, using 0x0100 prefix. - contains hashes of password and upper-case password. - -2007 - same as 2000, but without the upper-case hash. - -refs ----------- -https://blogs.msdn.com/b/lcris/archive/2007/04/30/sql-server-2005-about-login-password-hashes.aspx?Redirected=true -http://us.generation-nt.com/securing-passwords-hash-help-35429432.html -http://forum.md5decrypter.co.uk/topic230-mysql-and-mssql-get-password-hashes.aspx -http://www.theregister.co.uk/2002/07/08/cracking_ms_sql_server_passwords/ -""" -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify, unhexlify -from hashlib import sha1 -import re -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import consteq -from passlib.utils.compat import bascii_to_str, unicode, u -import passlib.utils.handlers as uh -# local -__all__ = [ - "mssql2000", - "mssql2005", -] - -#============================================================================= -# mssql 2000 -#============================================================================= -def _raw_mssql(secret, salt): - assert isinstance(secret, unicode) - assert isinstance(salt, bytes) - return sha1(secret.encode("utf-16-le") + salt).digest() - -BIDENT = b"0x0100" -##BIDENT2 = b("\x01\x00") -UIDENT = u("0x0100") - -def _ident_mssql(hash, csize, bsize): - """common identify for mssql 2000/2005""" - if isinstance(hash, unicode): - if len(hash) == csize and hash.startswith(UIDENT): - return True - elif isinstance(hash, bytes): - if len(hash) == csize and hash.startswith(BIDENT): - return True - ##elif len(hash) == bsize and hash.startswith(BIDENT2): # raw bytes - ## return True - else: - raise uh.exc.ExpectedStringError(hash, "hash") - return False - -def _parse_mssql(hash, csize, bsize, handler): - """common parser for mssql 2000/2005; returns 4 byte salt + checksum""" - if isinstance(hash, unicode): - if len(hash) == csize and hash.startswith(UIDENT): - try: - return unhexlify(hash[6:].encode("utf-8")) - except TypeError: # throw when bad char found - pass - elif isinstance(hash, bytes): - # assumes ascii-compat encoding - assert isinstance(hash, bytes) - if len(hash) == csize and hash.startswith(BIDENT): - try: - return unhexlify(hash[6:]) - except TypeError: # throw when bad char found - pass - ##elif len(hash) == bsize and hash.startswith(BIDENT2): # raw bytes - ## return hash[2:] - else: - raise uh.exc.ExpectedStringError(hash, "hash") - raise uh.exc.InvalidHashError(handler) - -class mssql2000(uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """This class implements the password hash used by MS-SQL 2000, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 4 bytes in length. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - """ - #=================================================================== - # algorithm information - #=================================================================== - name = "mssql2000" - setting_kwds = ("salt",) - checksum_size = 40 - min_salt_size = max_salt_size = 4 - - #=================================================================== - # formatting - #=================================================================== - - # 0100 - 2 byte identifier - # 4 byte salt - # 20 byte checksum - # 20 byte checksum - # = 46 bytes - # encoded '0x' + 92 chars = 94 - - @classmethod - def identify(cls, hash): - return _ident_mssql(hash, 94, 46) - - @classmethod - def from_string(cls, hash): - data = _parse_mssql(hash, 94, 46, cls) - return cls(salt=data[:4], checksum=data[4:]) - - def to_string(self): - raw = self.salt + self.checksum - # raw bytes format - BIDENT2 + raw - return "0x0100" + bascii_to_str(hexlify(raw).upper()) - - def _calc_checksum(self, secret): - if isinstance(secret, bytes): - secret = secret.decode("utf-8") - salt = self.salt - return _raw_mssql(secret, salt) + _raw_mssql(secret.upper(), salt) - - @classmethod - def verify(cls, secret, hash): - # NOTE: we only compare against the upper-case hash - # XXX: add 'full' just to verify both checksums? - uh.validate_secret(secret) - self = cls.from_string(hash) - chk = self.checksum - if chk is None: - raise uh.exc.MissingDigestError(cls) - if isinstance(secret, bytes): - secret = secret.decode("utf-8") - result = _raw_mssql(secret.upper(), self.salt) - return consteq(result, chk[20:]) - -#============================================================================= -# handler -#============================================================================= -class mssql2005(uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """This class implements the password hash used by MS-SQL 2005, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 4 bytes in length. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - """ - #=================================================================== - # algorithm information - #=================================================================== - name = "mssql2005" - setting_kwds = ("salt",) - - checksum_size = 20 - min_salt_size = max_salt_size = 4 - - #=================================================================== - # formatting - #=================================================================== - - # 0x0100 - 2 byte identifier - # 4 byte salt - # 20 byte checksum - # = 26 bytes - # encoded '0x' + 52 chars = 54 - - @classmethod - def identify(cls, hash): - return _ident_mssql(hash, 54, 26) - - @classmethod - def from_string(cls, hash): - data = _parse_mssql(hash, 54, 26, cls) - return cls(salt=data[:4], checksum=data[4:]) - - def to_string(self): - raw = self.salt + self.checksum - # raw bytes format - BIDENT2 + raw - return "0x0100" + bascii_to_str(hexlify(raw)).upper() - - def _calc_checksum(self, secret): - if isinstance(secret, bytes): - secret = secret.decode("utf-8") - return _raw_mssql(secret, self.salt) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/mysql.py b/src/passlib/handlers/mysql.py deleted file mode 100644 index 4a712535..00000000 --- a/src/passlib/handlers/mysql.py +++ /dev/null @@ -1,128 +0,0 @@ -"""passlib.handlers.mysql - -MySQL 3.2.3 / OLD_PASSWORD() - - This implements Mysql's OLD_PASSWORD algorithm, introduced in version 3.2.3, deprecated in version 4.1. - - See :mod:`passlib.handlers.mysql_41` for the new algorithm was put in place in version 4.1 - - This algorithm is known to be very insecure, and should only be used to verify existing password hashes. - - http://djangosnippets.org/snippets/1508/ - -MySQL 4.1.1 / NEW PASSWORD - This implements Mysql new PASSWORD algorithm, introduced in version 4.1. - - This function is unsalted, and therefore not very secure against rainbow attacks. - It should only be used when dealing with mysql passwords, - for all other purposes, you should use a salted hash function. - - Description taken from http://dev.mysql.com/doc/refman/6.0/en/password-hashing.html -""" -#============================================================================= -# imports -#============================================================================= -# core -from hashlib import sha1 -import re -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import to_native_str -from passlib.utils.compat import bascii_to_str, unicode, u, \ - byte_elem_value, str_to_uascii -import passlib.utils.handlers as uh -# local -__all__ = [ - 'mysql323', - 'mysq41', -] - -#============================================================================= -# backend -#============================================================================= -class mysql323(uh.StaticHandler): - """This class implements the MySQL 3.2.3 password hash, and follows the :ref:`password-hash-api`. - - It has no salt and a single fixed round. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept no optional keywords. - """ - #=================================================================== - # class attrs - #=================================================================== - name = "mysql323" - checksum_size = 16 - checksum_chars = uh.HEX_CHARS - - #=================================================================== - # methods - #=================================================================== - @classmethod - def _norm_hash(cls, hash): - return hash.lower() - - def _calc_checksum(self, secret): - # FIXME: no idea if mysql has a policy about handling unicode passwords - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - - MASK_32 = 0xffffffff - MASK_31 = 0x7fffffff - WHITE = b' \t' - - nr1 = 0x50305735 - nr2 = 0x12345671 - add = 7 - for c in secret: - if c in WHITE: - continue - tmp = byte_elem_value(c) - nr1 ^= ((((nr1 & 63)+add)*tmp) + (nr1 << 8)) & MASK_32 - nr2 = (nr2+((nr2 << 8) ^ nr1)) & MASK_32 - add = (add+tmp) & MASK_32 - return u("%08x%08x") % (nr1 & MASK_31, nr2 & MASK_31) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# handler -#============================================================================= -class mysql41(uh.StaticHandler): - """This class implements the MySQL 4.1 password hash, and follows the :ref:`password-hash-api`. - - It has no salt and a single fixed round. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept no optional keywords. - """ - #=================================================================== - # class attrs - #=================================================================== - name = "mysql41" - _hash_prefix = u("*") - checksum_chars = uh.HEX_CHARS - checksum_size = 40 - - #=================================================================== - # methods - #=================================================================== - @classmethod - def _norm_hash(cls, hash): - return hash.upper() - - def _calc_checksum(self, secret): - # FIXME: no idea if mysql has a policy about handling unicode passwords - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return str_to_uascii(sha1(sha1(secret).digest()).hexdigest()).upper() - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/oracle.py b/src/passlib/handlers/oracle.py deleted file mode 100644 index a094f372..00000000 --- a/src/passlib/handlers/oracle.py +++ /dev/null @@ -1,172 +0,0 @@ -"""passlib.handlers.oracle - Oracle DB Password Hashes""" -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify, unhexlify -from hashlib import sha1 -import re -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import to_unicode, xor_bytes -from passlib.utils.compat import irange, u, \ - uascii_to_str, unicode, str_to_uascii -from passlib.crypto.des import des_encrypt_block -import passlib.utils.handlers as uh -# local -__all__ = [ - "oracle10g", - "oracle11g" -] - -#============================================================================= -# oracle10 -#============================================================================= -def des_cbc_encrypt(key, value, iv=b'\x00' * 8, pad=b'\x00'): - """performs des-cbc encryption, returns only last block. - - this performs a specific DES-CBC encryption implementation - as needed by the Oracle10 hash. it probably won't be useful for - other purposes as-is. - - input value is null-padded to multiple of 8 bytes. - - :arg key: des key as bytes - :arg value: value to encrypt, as bytes. - :param iv: optional IV - :param pad: optional pad byte - - :returns: last block of DES-CBC encryption of all ``value``'s byte blocks. - """ - value += pad * (-len(value) % 8) # null pad to multiple of 8 - hash = iv # start things off - for offset in irange(0,len(value),8): - chunk = xor_bytes(hash, value[offset:offset+8]) - hash = des_encrypt_block(key, chunk) - return hash - -# magic string used as initial des key by oracle10 -ORACLE10_MAGIC = b"\x01\x23\x45\x67\x89\xAB\xCD\xEF" - -class oracle10(uh.HasUserContext, uh.StaticHandler): - """This class implements the password hash used by Oracle up to version 10g, and follows the :ref:`password-hash-api`. - - It does a single round of hashing, and relies on the username as the salt. - - The :meth:`~passlib.ifc.PasswordHash.hash`, :meth:`~passlib.ifc.PasswordHash.genhash`, and :meth:`~passlib.ifc.PasswordHash.verify` methods all require the - following additional contextual keywords: - - :type user: str - :param user: name of oracle user account this password is associated with. - """ - #=================================================================== - # algorithm information - #=================================================================== - name = "oracle10" - checksum_chars = uh.HEX_CHARS - checksum_size = 16 - - #=================================================================== - # methods - #=================================================================== - @classmethod - def _norm_hash(cls, hash): - return hash.upper() - - def _calc_checksum(self, secret): - # FIXME: not sure how oracle handles unicode. - # online docs about 10g hash indicate it puts ascii chars - # in a 2-byte encoding w/ the high byte set to null. - # they don't say how it handles other chars, or what encoding. - # - # so for now, encoding secret & user to utf-16-be, - # since that fits, and if secret/user is bytes, - # we assume utf-8, and decode first. - # - # this whole mess really needs someone w/ an oracle system, - # and some answers :) - if isinstance(secret, bytes): - secret = secret.decode("utf-8") - user = to_unicode(self.user, "utf-8", param="user") - input = (user+secret).upper().encode("utf-16-be") - hash = des_cbc_encrypt(ORACLE10_MAGIC, input) - hash = des_cbc_encrypt(hash, input) - return hexlify(hash).decode("ascii").upper() - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# oracle11 -#============================================================================= -class oracle11(uh.HasSalt, uh.GenericHandler): - """This class implements the Oracle11g password hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 20 hexadecimal characters. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "oracle11" - setting_kwds = ("salt",) - checksum_size = 40 - checksum_chars = uh.UPPER_HEX_CHARS - - #--HasSalt-- - min_salt_size = max_salt_size = 20 - salt_chars = uh.UPPER_HEX_CHARS - - - #=================================================================== - # methods - #=================================================================== - _hash_regex = re.compile(u("^S:(?P[0-9a-f]{40})(?P[0-9a-f]{20})$"), re.I) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - m = cls._hash_regex.match(hash) - if not m: - raise uh.exc.InvalidHashError(cls) - salt, chk = m.group("salt", "chk") - return cls(salt=salt, checksum=chk.upper()) - - def to_string(self): - chk = self.checksum - hash = u("S:%s%s") % (chk.upper(), self.salt.upper()) - return uascii_to_str(hash) - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - chk = sha1(secret + unhexlify(self.salt.encode("ascii"))).hexdigest() - return str_to_uascii(chk).upper() - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/pbkdf2.py b/src/passlib/handlers/pbkdf2.py deleted file mode 100644 index 274278d8..00000000 --- a/src/passlib/handlers/pbkdf2.py +++ /dev/null @@ -1,475 +0,0 @@ -"""passlib.handlers.pbkdf - PBKDF2 based hashes""" -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify, unhexlify -from base64 import b64encode, b64decode -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import to_unicode -from passlib.utils.binary import ab64_decode, ab64_encode -from passlib.utils.compat import str_to_bascii, u, uascii_to_str, unicode -from passlib.crypto.digest import pbkdf2_hmac -import passlib.utils.handlers as uh -# local -__all__ = [ - "pbkdf2_sha1", - "pbkdf2_sha256", - "pbkdf2_sha512", - "cta_pbkdf2_sha1", - "dlitz_pbkdf2_sha1", - "grub_pbkdf2_sha512", -] - -#============================================================================= -# -#============================================================================= -class Pbkdf2DigestHandler(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """base class for various pbkdf2_{digest} algorithms""" - #=================================================================== - # class attrs - #=================================================================== - - #--GenericHandler-- - setting_kwds = ("salt", "salt_size", "rounds") - checksum_chars = uh.HASH64_CHARS - - #--HasSalt-- - default_salt_size = 16 - max_salt_size = 1024 - - #--HasRounds-- - default_rounds = None # set by subclass - min_rounds = 1 - max_rounds = 0xffffffff # setting at 32-bit limit for now - rounds_cost = "linear" - - #--this class-- - _digest = None # name of subclass-specified hash - - # NOTE: max_salt_size and max_rounds are arbitrarily chosen to provide sanity check. - # the underlying pbkdf2 specifies no bounds for either. - - # NOTE: defaults chosen to be at least as large as pbkdf2 rfc recommends... - # >8 bytes of entropy in salt, >1000 rounds - # increased due to time since rfc established - - #=================================================================== - # methods - #=================================================================== - - @classmethod - def from_string(cls, hash): - rounds, salt, chk = uh.parse_mc3(hash, cls.ident, handler=cls) - salt = ab64_decode(salt.encode("ascii")) - if chk: - chk = ab64_decode(chk.encode("ascii")) - return cls(rounds=rounds, salt=salt, checksum=chk) - - def to_string(self): - salt = ab64_encode(self.salt).decode("ascii") - chk = ab64_encode(self.checksum).decode("ascii") - return uh.render_mc3(self.ident, self.rounds, salt, chk) - - def _calc_checksum(self, secret): - # NOTE: pbkdf2_hmac() will encode secret & salt using UTF8 - return pbkdf2_hmac(self._digest, secret, self.salt, self.rounds, self.checksum_size) - -def create_pbkdf2_hash(hash_name, digest_size, rounds=12000, ident=None, module=__name__): - """create new Pbkdf2DigestHandler subclass for a specific hash""" - name = 'pbkdf2_' + hash_name - if ident is None: - ident = u("$pbkdf2-%s$") % (hash_name,) - base = Pbkdf2DigestHandler - return type(name, (base,), dict( - __module__=module, # so ABCMeta won't clobber it. - name=name, - ident=ident, - _digest = hash_name, - default_rounds=rounds, - checksum_size=digest_size, - encoded_checksum_size=(digest_size*4+2)//3, - __doc__="""This class implements a generic ``PBKDF2-HMAC-%(digest)s``-based password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt bytes. - If specified, the length must be between 0-1024 bytes. - If not specified, a %(dsc)d byte salt will be autogenerated (this is recommended). - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to %(dsc)d bytes, but can be any value between 0 and 1024. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to %(dr)d, but must be within ``range(1,1<<32)``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ % dict(digest=hash_name.upper(), dsc=base.default_salt_size, dr=rounds) - )) - -#------------------------------------------------------------------------ -# derived handlers -#------------------------------------------------------------------------ -pbkdf2_sha1 = create_pbkdf2_hash("sha1", 20, 131000, ident=u("$pbkdf2$")) -pbkdf2_sha256 = create_pbkdf2_hash("sha256", 32, 29000) -pbkdf2_sha512 = create_pbkdf2_hash("sha512", 64, 25000) - -ldap_pbkdf2_sha1 = uh.PrefixWrapper("ldap_pbkdf2_sha1", pbkdf2_sha1, "{PBKDF2}", "$pbkdf2$", ident=True) -ldap_pbkdf2_sha256 = uh.PrefixWrapper("ldap_pbkdf2_sha256", pbkdf2_sha256, "{PBKDF2-SHA256}", "$pbkdf2-sha256$", ident=True) -ldap_pbkdf2_sha512 = uh.PrefixWrapper("ldap_pbkdf2_sha512", pbkdf2_sha512, "{PBKDF2-SHA512}", "$pbkdf2-sha512$", ident=True) - -#============================================================================= -# cryptacular's pbkdf2 hash -#============================================================================= - -# bytes used by cta hash for base64 values 63 & 64 -CTA_ALTCHARS = b"-_" - -class cta_pbkdf2_sha1(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """This class implements Cryptacular's PBKDF2-based crypt algorithm, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt bytes. - If specified, it may be any length. - If not specified, a one will be autogenerated (this is recommended). - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 16 bytes, but can be any value between 0 and 1024. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 60000, must be within ``range(1,1<<32)``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "cta_pbkdf2_sha1" - setting_kwds = ("salt", "salt_size", "rounds") - ident = u("$p5k2$") - checksum_size = 20 - - # NOTE: max_salt_size and max_rounds are arbitrarily chosen to provide a - # sanity check. underlying algorithm (and reference implementation) - # allows effectively unbounded values for both of these parameters. - - #--HasSalt-- - default_salt_size = 16 - max_salt_size = 1024 - - #--HasRounds-- - default_rounds = pbkdf2_sha1.default_rounds - min_rounds = 1 - max_rounds = 0xffffffff # setting at 32-bit limit for now - rounds_cost = "linear" - - #=================================================================== - # formatting - #=================================================================== - - # hash $p5k2$1000$ZxK4ZBJCfQg=$jJZVscWtO--p1-xIZl6jhO2LKR0= - # ident $p5k2$ - # rounds 1000 - # salt ZxK4ZBJCfQg= - # chk jJZVscWtO--p1-xIZl6jhO2LKR0= - # NOTE: rounds in hex - - @classmethod - def from_string(cls, hash): - # NOTE: passlib deviation - forbidding zero-padded rounds - rounds, salt, chk = uh.parse_mc3(hash, cls.ident, rounds_base=16, handler=cls) - salt = b64decode(salt.encode("ascii"), CTA_ALTCHARS) - if chk: - chk = b64decode(chk.encode("ascii"), CTA_ALTCHARS) - return cls(rounds=rounds, salt=salt, checksum=chk) - - def to_string(self): - salt = b64encode(self.salt, CTA_ALTCHARS).decode("ascii") - chk = b64encode(self.checksum, CTA_ALTCHARS).decode("ascii") - return uh.render_mc3(self.ident, self.rounds, salt, chk, rounds_base=16) - - #=================================================================== - # backend - #=================================================================== - def _calc_checksum(self, secret): - # NOTE: pbkdf2_hmac() will encode secret & salt using utf-8 - return pbkdf2_hmac("sha1", secret, self.salt, self.rounds, 20) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# dlitz's pbkdf2 hash -#============================================================================= -class dlitz_pbkdf2_sha1(uh.HasRounds, uh.HasSalt, uh.GenericHandler): - """This class implements Dwayne Litzenberger's PBKDF2-based crypt algorithm, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If specified, it may be any length, but must use the characters in the regexp range ``[./0-9A-Za-z]``. - If not specified, a 16 character salt will be autogenerated (this is recommended). - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 16 bytes, but can be any value between 0 and 1024. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 60000, must be within ``range(1,1<<32)``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "dlitz_pbkdf2_sha1" - setting_kwds = ("salt", "salt_size", "rounds") - ident = u("$p5k2$") - _stub_checksum = u("0" * 48 + "=") - - # NOTE: max_salt_size and max_rounds are arbitrarily chosen to provide a - # sanity check. underlying algorithm (and reference implementation) - # allows effectively unbounded values for both of these parameters. - - #--HasSalt-- - default_salt_size = 16 - max_salt_size = 1024 - salt_chars = uh.HASH64_CHARS - - #--HasRounds-- - # NOTE: for security, the default here is set to match pbkdf2_sha1, - # even though this hash's extra block makes it twice as slow. - default_rounds = pbkdf2_sha1.default_rounds - min_rounds = 1 - max_rounds = 0xffffffff # setting at 32-bit limit for now - rounds_cost = "linear" - - #=================================================================== - # formatting - #=================================================================== - - # hash $p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g - # ident $p5k2$ - # rounds c - # salt u9HvcT4d - # chk Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g - # rounds in lowercase hex, no zero padding - - @classmethod - def from_string(cls, hash): - rounds, salt, chk = uh.parse_mc3(hash, cls.ident, rounds_base=16, - default_rounds=400, handler=cls) - return cls(rounds=rounds, salt=salt, checksum=chk) - - def to_string(self): - rounds = self.rounds - if rounds == 400: - rounds = None # omit rounds measurement if == 400 - return uh.render_mc3(self.ident, rounds, self.salt, self.checksum, rounds_base=16) - - def _get_config(self): - rounds = self.rounds - if rounds == 400: - rounds = None # omit rounds measurement if == 400 - return uh.render_mc3(self.ident, rounds, self.salt, None, rounds_base=16) - - #=================================================================== - # backend - #=================================================================== - def _calc_checksum(self, secret): - # NOTE: pbkdf2_hmac() will encode secret & salt using utf-8 - salt = self._get_config() - result = pbkdf2_hmac("sha1", secret, salt, self.rounds, 24) - return ab64_encode(result).decode("ascii") - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# crowd -#============================================================================= -class atlassian_pbkdf2_sha1(uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """This class implements the PBKDF2 hash used by Atlassian. - - It supports a fixed-length salt, and a fixed number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt bytes. - If specified, the length must be exactly 16 bytes. - If not specified, a salt will be autogenerated (this is recommended). - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include - ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #--GenericHandler-- - name = "atlassian_pbkdf2_sha1" - setting_kwds =("salt",) - ident = u("{PKCS5S2}") - checksum_size = 32 - - #--HasRawSalt-- - min_salt_size = max_salt_size = 16 - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - ident = cls.ident - if not hash.startswith(ident): - raise uh.exc.InvalidHashError(cls) - data = b64decode(hash[len(ident):].encode("ascii")) - salt, chk = data[:16], data[16:] - return cls(salt=salt, checksum=chk) - - def to_string(self): - data = self.salt + self.checksum - hash = self.ident + b64encode(data).decode("ascii") - return uascii_to_str(hash) - - def _calc_checksum(self, secret): - # TODO: find out what crowd's policy is re: unicode - # crowd seems to use a fixed number of rounds. - # NOTE: pbkdf2_hmac() will encode secret & salt using utf-8 - return pbkdf2_hmac("sha1", secret, self.salt, 10000, 32) - -#============================================================================= -# grub -#============================================================================= -class grub_pbkdf2_sha512(uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.GenericHandler): - """This class implements Grub's pbkdf2-hmac-sha512 hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: bytes - :param salt: - Optional salt bytes. - If specified, the length must be between 0-1024 bytes. - If not specified, a 64 byte salt will be autogenerated (this is recommended). - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 64 bytes, but can be any value between 0 and 1024. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 19000, but must be within ``range(1,1<<32)``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - name = "grub_pbkdf2_sha512" - setting_kwds = ("salt", "salt_size", "rounds") - - ident = u("grub.pbkdf2.sha512.") - checksum_size = 64 - - # NOTE: max_salt_size and max_rounds are arbitrarily chosen to provide a - # sanity check. the underlying pbkdf2 specifies no bounds for either, - # and it's not clear what grub specifies. - - default_salt_size = 64 - max_salt_size = 1024 - - default_rounds = pbkdf2_sha512.default_rounds - min_rounds = 1 - max_rounds = 0xffffffff # setting at 32-bit limit for now - rounds_cost = "linear" - - @classmethod - def from_string(cls, hash): - rounds, salt, chk = uh.parse_mc3(hash, cls.ident, sep=u("."), - handler=cls) - salt = unhexlify(salt.encode("ascii")) - if chk: - chk = unhexlify(chk.encode("ascii")) - return cls(rounds=rounds, salt=salt, checksum=chk) - - def to_string(self): - salt = hexlify(self.salt).decode("ascii").upper() - chk = hexlify(self.checksum).decode("ascii").upper() - return uh.render_mc3(self.ident, self.rounds, salt, chk, sep=u(".")) - - def _calc_checksum(self, secret): - # TODO: find out what grub's policy is re: unicode - # NOTE: pbkdf2_hmac() will encode secret & salt using utf-8 - return pbkdf2_hmac("sha512", secret, self.salt, self.rounds, 64) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/phpass.py b/src/passlib/handlers/phpass.py deleted file mode 100644 index 6736f0f2..00000000 --- a/src/passlib/handlers/phpass.py +++ /dev/null @@ -1,135 +0,0 @@ -"""passlib.handlers.phpass - PHPass Portable Crypt - -phppass located - http://www.openwall.com/phpass/ -algorithm described - http://www.openwall.com/articles/PHP-Users-Passwords - -phpass context - blowfish, bsdi_crypt, phpass -""" -#============================================================================= -# imports -#============================================================================= -# core -from hashlib import md5 -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils.binary import h64 -from passlib.utils.compat import u, uascii_to_str, unicode -import passlib.utils.handlers as uh -# local -__all__ = [ - "phpass", -] - -#============================================================================= -# phpass -#============================================================================= -class phpass(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.GenericHandler): - """This class implements the PHPass Portable Hash, and follows the :ref:`password-hash-api`. - - It supports a fixed-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 8 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 19, must be between 7 and 30, inclusive. - This value is logarithmic, the actual number of iterations used will be :samp:`2**{rounds}`. - - :type ident: str - :param ident: - phpBB3 uses ``H`` instead of ``P`` for its identifier, - this may be set to ``H`` in order to generate phpBB3 compatible hashes. - it defaults to ``P``. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "phpass" - setting_kwds = ("salt", "rounds", "ident") - checksum_chars = uh.HASH64_CHARS - - #--HasSalt-- - min_salt_size = max_salt_size = 8 - salt_chars = uh.HASH64_CHARS - - #--HasRounds-- - default_rounds = 19 - min_rounds = 7 - max_rounds = 30 - rounds_cost = "log2" - - #--HasManyIdents-- - default_ident = u("$P$") - ident_values = (u("$P$"), u("$H$")) - ident_aliases = {u("P"):u("$P$"), u("H"):u("$H$")} - - #=================================================================== - # formatting - #=================================================================== - - #$P$9IQRaTwmfeRo7ud9Fh4E2PdI0S3r.L0 - # $P$ - # 9 - # IQRaTwmf - # eRo7ud9Fh4E2PdI0S3r.L0 - - @classmethod - def from_string(cls, hash): - ident, data = cls._parse_ident(hash) - rounds, salt, chk = data[0], data[1:9], data[9:] - return cls( - ident=ident, - rounds=h64.decode_int6(rounds.encode("ascii")), - salt=salt, - checksum=chk or None, - ) - - def to_string(self): - hash = u("%s%s%s%s") % (self.ident, - h64.encode_int6(self.rounds).decode("ascii"), - self.salt, - self.checksum or u('')) - return uascii_to_str(hash) - - #=================================================================== - # backend - #=================================================================== - def _calc_checksum(self, secret): - # FIXME: can't find definitive policy on how phpass handles non-ascii. - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - real_rounds = 1<`_ - hash names. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - - In addition to the standard :ref:`password-hash-api` methods, - this class also provides the following methods for manipulating Passlib - scram hashes in ways useful for pluging into a SCRAM protocol stack: - - .. automethod:: extract_digest_info - .. automethod:: extract_digest_algs - .. automethod:: derive_digest - """ - #=================================================================== - # class attrs - #=================================================================== - - # NOTE: unlike most GenericHandler classes, the 'checksum' attr of - # ScramHandler is actually a map from digest_name -> digest, so - # many of the standard methods have been overridden. - - # NOTE: max_salt_size and max_rounds are arbitrarily chosen to provide - # a sanity check; the underlying pbkdf2 specifies no bounds for either. - - #--GenericHandler-- - name = "scram" - setting_kwds = ("salt", "salt_size", "rounds", "algs") - ident = u("$scram$") - - #--HasSalt-- - default_salt_size = 12 - max_salt_size = 1024 - - #--HasRounds-- - default_rounds = 100000 - min_rounds = 1 - max_rounds = 2**32-1 - rounds_cost = "linear" - - #--custom-- - - # default algorithms when creating new hashes. - default_algs = ["sha-1", "sha-256", "sha-512"] - - # list of algs verify prefers to use, in order. - _verify_algs = ["sha-256", "sha-512", "sha-224", "sha-384", "sha-1"] - - #=================================================================== - # instance attrs - #=================================================================== - - # 'checksum' is different from most GenericHandler subclasses, - # in that it contains a dict mapping from alg -> digest, - # or None if no checksum present. - - # list of algorithms to create/compare digests for. - algs = None - - #=================================================================== - # scram frontend helpers - #=================================================================== - @classmethod - def extract_digest_info(cls, hash, alg): - """return (salt, rounds, digest) for specific hash algorithm. - - :type hash: str - :arg hash: - :class:`!scram` hash stored for desired user - - :type alg: str - :arg alg: - Name of digest algorithm (e.g. ``"sha-1"``) requested by client. - - This value is run through :func:`~passlib.crypto.digest.norm_hash_name`, - so it is case-insensitive, and can be the raw SCRAM - mechanism name (e.g. ``"SCRAM-SHA-1"``), the IANA name, - or the hashlib name. - - :raises KeyError: - If the hash does not contain an entry for the requested digest - algorithm. - - :returns: - A tuple containing ``(salt, rounds, digest)``, - where *digest* matches the raw bytes returned by - SCRAM's :func:`Hi` function for the stored password, - the provided *salt*, and the iteration count (*rounds*). - *salt* and *digest* are both raw (unencoded) bytes. - """ - # XXX: this could be sped up by writing custom parsing routine - # that just picks out relevant digest, and doesn't bother - # with full structure validation each time it's called. - alg = norm_hash_name(alg, 'iana') - self = cls.from_string(hash) - chkmap = self.checksum - if not chkmap: - raise ValueError("scram hash contains no digests") - return self.salt, self.rounds, chkmap[alg] - - @classmethod - def extract_digest_algs(cls, hash, format="iana"): - """Return names of all algorithms stored in a given hash. - - :type hash: str - :arg hash: - The :class:`!scram` hash to parse - - :type format: str - :param format: - This changes the naming convention used by the - returned algorithm names. By default the names - are IANA-compatible; possible values are ``"iana"`` or ``"hashlib"``. - - :returns: - Returns a list of digest algorithms; e.g. ``["sha-1"]`` - """ - # XXX: this could be sped up by writing custom parsing routine - # that just picks out relevant names, and doesn't bother - # with full structure validation each time it's called. - algs = cls.from_string(hash).algs - if format == "iana": - return algs - else: - return [norm_hash_name(alg, format) for alg in algs] - - @classmethod - def derive_digest(cls, password, salt, rounds, alg): - """helper to create SaltedPassword digest for SCRAM. - - This performs the step in the SCRAM protocol described as:: - - SaltedPassword := Hi(Normalize(password), salt, i) - - :type password: unicode or utf-8 bytes - :arg password: password to run through digest - - :type salt: bytes - :arg salt: raw salt data - - :type rounds: int - :arg rounds: number of iterations. - - :type alg: str - :arg alg: name of digest to use (e.g. ``"sha-1"``). - - :returns: - raw bytes of ``SaltedPassword`` - """ - if isinstance(password, bytes): - password = password.decode("utf-8") - # NOTE: pbkdf2_hmac() will encode secret & salt using utf-8, - # and handle normalizing alg name. - return pbkdf2_hmac(alg, saslprep(password), salt, rounds) - - #=================================================================== - # serialization - #=================================================================== - - @classmethod - def from_string(cls, hash): - hash = to_native_str(hash, "ascii", "hash") - if not hash.startswith("$scram$"): - raise uh.exc.InvalidHashError(cls) - parts = hash[7:].split("$") - if len(parts) != 3: - raise uh.exc.MalformedHashError(cls) - rounds_str, salt_str, chk_str = parts - - # decode rounds - rounds = int(rounds_str) - if rounds_str != str(rounds): # forbid zero padding, etc. - raise uh.exc.MalformedHashError(cls) - - # decode salt - try: - salt = ab64_decode(salt_str.encode("ascii")) - except TypeError: - raise uh.exc.MalformedHashError(cls) - - # decode algs/digest list - if not chk_str: - # scram hashes MUST have something here. - raise uh.exc.MalformedHashError(cls) - elif "=" in chk_str: - # comma-separated list of 'alg=digest' pairs - algs = None - chkmap = {} - for pair in chk_str.split(","): - alg, digest = pair.split("=") - try: - chkmap[alg] = ab64_decode(digest.encode("ascii")) - except TypeError: - raise uh.exc.MalformedHashError(cls) - else: - # comma-separated list of alg names, no digests - algs = chk_str - chkmap = None - - # return new object - return cls( - rounds=rounds, - salt=salt, - checksum=chkmap, - algs=algs, - ) - - def to_string(self): - salt = bascii_to_str(ab64_encode(self.salt)) - chkmap = self.checksum - chk_str = ",".join( - "%s=%s" % (alg, bascii_to_str(ab64_encode(chkmap[alg]))) - for alg in self.algs - ) - return '$scram$%d$%s$%s' % (self.rounds, salt, chk_str) - - #=================================================================== - # variant constructor - #=================================================================== - @classmethod - def using(cls, default_algs=None, algs=None, **kwds): - # parse aliases - if algs is not None: - assert default_algs is None - default_algs = algs - - # create subclass - subcls = super(scram, cls).using(**kwds) - - # fill in algs - if default_algs is not None: - subcls.default_algs = cls._norm_algs(default_algs) - return subcls - - #=================================================================== - # init - #=================================================================== - def __init__(self, algs=None, **kwds): - super(scram, self).__init__(**kwds) - - # init algs - digest_map = self.checksum - if algs is not None: - if digest_map is not None: - raise RuntimeError("checksum & algs kwds are mutually exclusive") - algs = self._norm_algs(algs) - elif digest_map is not None: - # derive algs list from digest map (if present). - algs = self._norm_algs(digest_map.keys()) - elif self.use_defaults: - algs = list(self.default_algs) - assert self._norm_algs(algs) == algs, "invalid default algs: %r" % (algs,) - else: - raise TypeError("no algs list specified") - self.algs = algs - - def _norm_checksum(self, checksum, relaxed=False): - if not isinstance(checksum, dict): - raise uh.exc.ExpectedTypeError(checksum, "dict", "checksum") - for alg, digest in iteritems(checksum): - if alg != norm_hash_name(alg, 'iana'): - raise ValueError("malformed algorithm name in scram hash: %r" % - (alg,)) - if len(alg) > 9: - raise ValueError("SCRAM limits algorithm names to " - "9 characters: %r" % (alg,)) - if not isinstance(digest, bytes): - raise uh.exc.ExpectedTypeError(digest, "raw bytes", "digests") - # TODO: verify digest size (if digest is known) - if 'sha-1' not in checksum: - # NOTE: required because of SCRAM spec. - raise ValueError("sha-1 must be in algorithm list of scram hash") - return checksum - - @classmethod - def _norm_algs(cls, algs): - """normalize algs parameter""" - if isinstance(algs, native_string_types): - algs = splitcomma(algs) - algs = sorted(norm_hash_name(alg, 'iana') for alg in algs) - if any(len(alg)>9 for alg in algs): - raise ValueError("SCRAM limits alg names to max of 9 characters") - if 'sha-1' not in algs: - # NOTE: required because of SCRAM spec (rfc 5802) - raise ValueError("sha-1 must be in algorithm list of scram hash") - return algs - - #=================================================================== - # migration - #=================================================================== - def _calc_needs_update(self, **kwds): - # marks hashes as deprecated if they don't include at least all default_algs. - # XXX: should we deprecate if they aren't exactly the same, - # to permit removing legacy hashes? - if not set(self.algs).issuperset(self.default_algs): - return True - - # hand off to base implementation - return super(scram, self)._calc_needs_update(**kwds) - - #=================================================================== - # digest methods - #=================================================================== - def _calc_checksum(self, secret, alg=None): - rounds = self.rounds - salt = self.salt - hash = self.derive_digest - if alg: - # if requested, generate digest for specific alg - return hash(secret, salt, rounds, alg) - else: - # by default, return dict containing digests for all algs - return dict( - (alg, hash(secret, salt, rounds, alg)) - for alg in self.algs - ) - - @classmethod - def verify(cls, secret, hash, full=False): - uh.validate_secret(secret) - self = cls.from_string(hash) - chkmap = self.checksum - if not chkmap: - raise ValueError("expected %s hash, got %s config string instead" % - (cls.name, cls.name)) - - # NOTE: to make the verify method efficient, we just calculate hash - # of shortest digest by default. apps can pass in "full=True" to - # check entire hash for consistency. - if full: - correct = failed = False - for alg, digest in iteritems(chkmap): - other = self._calc_checksum(secret, alg) - # NOTE: could do this length check in norm_algs(), - # but don't need to be that strict, and want to be able - # to parse hashes containing algs not supported by platform. - # it's fine if we fail here though. - if len(digest) != len(other): - raise ValueError("mis-sized %s digest in scram hash: %r != %r" - % (alg, len(digest), len(other))) - if consteq(other, digest): - correct = True - else: - failed = True - if correct and failed: - raise ValueError("scram hash verified inconsistently, " - "may be corrupted") - else: - return correct - else: - # XXX: should this just always use sha1 hash? would be faster. - # otherwise only verify against one hash, pick one w/ best security. - for alg in self._verify_algs: - if alg in chkmap: - other = self._calc_checksum(secret, alg) - return consteq(other, chkmap[alg]) - # there should always be sha-1 at the very least, - # or something went wrong inside _norm_algs() - raise AssertionError("sha-1 digest not found!") - - #=================================================================== - # - #=================================================================== - -#============================================================================= -# code used for testing scram against protocol examples during development. -#============================================================================= -##def _test_reference_scram(): -## "quick hack testing scram reference vectors" -## # NOTE: "n,," is GS2 header - see https://tools.ietf.org/html/rfc5801 -## from passlib.utils.compat import print_ -## -## engine = _scram_engine( -## alg="sha-1", -## salt='QSXCR+Q6sek8bf92'.decode("base64"), -## rounds=4096, -## password=u("pencil"), -## ) -## print_(engine.digest.encode("base64").rstrip()) -## -## msg = engine.format_auth_msg( -## username="user", -## client_nonce = "fyko+d2lbbFgONRv9qkxdawL", -## server_nonce = "3rfcNHYJY1ZVvWVs7j", -## header='c=biws', -## ) -## -## cp = engine.get_encoded_client_proof(msg) -## assert cp == "v0X8v3Bz2T0CJGbJQyF0X+HI4Ts=", cp -## -## ss = engine.get_encoded_server_sig(msg) -## assert ss == "rmF9pqV8S7suAoZWja4dJRkFsKQ=", ss -## -##class _scram_engine(object): -## """helper class for verifying scram hash behavior -## against SCRAM protocol examples. not officially part of Passlib. -## -## takes in alg, salt, rounds, and a digest or password. -## -## can calculate the various keys & messages of the scram protocol. -## -## """ -## #========================================================= -## # init -## #========================================================= -## -## @classmethod -## def from_string(cls, hash, alg): -## "create record from scram hash, for given alg" -## return cls(alg, *scram.extract_digest_info(hash, alg)) -## -## def __init__(self, alg, salt, rounds, digest=None, password=None): -## self.alg = norm_hash_name(alg) -## self.salt = salt -## self.rounds = rounds -## self.password = password -## if password: -## data = scram.derive_digest(password, salt, rounds, alg) -## if digest and data != digest: -## raise ValueError("password doesn't match digest") -## else: -## digest = data -## elif not digest: -## raise TypeError("must provide password or digest") -## self.digest = digest -## -## #========================================================= -## # frontend methods -## #========================================================= -## def get_hash(self, data): -## "return hash of raw data" -## return hashlib.new(iana_to_hashlib(self.alg), data).digest() -## -## def get_client_proof(self, msg): -## "return client proof of specified auth msg text" -## return xor_bytes(self.client_key, self.get_client_sig(msg)) -## -## def get_encoded_client_proof(self, msg): -## return self.get_client_proof(msg).encode("base64").rstrip() -## -## def get_client_sig(self, msg): -## "return client signature of specified auth msg text" -## return self.get_hmac(self.stored_key, msg) -## -## def get_server_sig(self, msg): -## "return server signature of specified auth msg text" -## return self.get_hmac(self.server_key, msg) -## -## def get_encoded_server_sig(self, msg): -## return self.get_server_sig(msg).encode("base64").rstrip() -## -## def format_server_response(self, client_nonce, server_nonce): -## return 'r={client_nonce}{server_nonce},s={salt},i={rounds}'.format( -## client_nonce=client_nonce, -## server_nonce=server_nonce, -## rounds=self.rounds, -## salt=self.encoded_salt, -## ) -## -## def format_auth_msg(self, username, client_nonce, server_nonce, -## header='c=biws'): -## return ( -## 'n={username},r={client_nonce}' -## ',' -## 'r={client_nonce}{server_nonce},s={salt},i={rounds}' -## ',' -## '{header},r={client_nonce}{server_nonce}' -## ).format( -## username=username, -## client_nonce=client_nonce, -## server_nonce=server_nonce, -## salt=self.encoded_salt, -## rounds=self.rounds, -## header=header, -## ) -## -## #========================================================= -## # helpers to calculate & cache constant data -## #========================================================= -## def _calc_get_hmac(self): -## return get_prf("hmac-" + iana_to_hashlib(self.alg))[0] -## -## def _calc_client_key(self): -## return self.get_hmac(self.digest, b("Client Key")) -## -## def _calc_stored_key(self): -## return self.get_hash(self.client_key) -## -## def _calc_server_key(self): -## return self.get_hmac(self.digest, b("Server Key")) -## -## def _calc_encoded_salt(self): -## return self.salt.encode("base64").rstrip() -## -## #========================================================= -## # hacks for calculated attributes -## #========================================================= -## -## def __getattr__(self, attr): -## if not attr.startswith("_"): -## f = getattr(self, "_calc_" + attr, None) -## if f: -## value = f() -## setattr(self, attr, value) -## return value -## raise AttributeError("attribute not found") -## -## def __dir__(self): -## cdir = dir(self.__class__) -## attrs = set(cdir) -## attrs.update(self.__dict__) -## attrs.update(attr[6:] for attr in cdir -## if attr.startswith("_calc_")) -## return sorted(attrs) -## #========================================================= -## # eoc -## #========================================================= - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/scrypt.py b/src/passlib/handlers/scrypt.py deleted file mode 100644 index 1686fda5..00000000 --- a/src/passlib/handlers/scrypt.py +++ /dev/null @@ -1,383 +0,0 @@ -"""passlib.handlers.scrypt -- scrypt password hash""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement, absolute_import -# core -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.crypto import scrypt as _scrypt -from passlib.utils import h64, to_bytes -from passlib.utils.binary import h64, b64s_decode, b64s_encode -from passlib.utils.compat import u, bascii_to_str, suppress_cause -from passlib.utils.decor import classproperty -import passlib.utils.handlers as uh -# local -__all__ = [ - "scrypt", -] - -#============================================================================= -# scrypt format identifiers -#============================================================================= - -IDENT_SCRYPT = u("$scrypt$") # identifier used by passlib -IDENT_7 = u("$7$") # used by official scrypt spec - -_UDOLLAR = u("$") - -#============================================================================= -# handler -#============================================================================= -class scrypt(uh.ParallelismMixin, uh.HasRounds, uh.HasRawSalt, uh.HasRawChecksum, uh.HasManyIdents, - uh.GenericHandler): - """This class implements an SCrypt-based password [#scrypt-home]_ hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, a variable number of rounds, - as well as some custom tuning parameters unique to scrypt (see below). - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If specified, the length must be between 0-1024 bytes. - If not specified, one will be auto-generated (this is recommended). - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 16 bytes, but can be any value between 0 and 1024. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 16, but must be within ``range(1,32)``. - - .. warning:: - - Unlike many hash algorithms, increasing the rounds value - will increase both the time *and memory* required to hash a password. - - :type block_size: int - :param block_size: - Optional block size to pass to scrypt hash function (the ``r`` parameter). - Useful for tuning scrypt to optimal performance for your CPU architecture. - Defaults to 8. - - :type parallelism: int - :param parallelism: - Optional parallelism to pass to scrypt hash function (the ``p`` parameter). - Defaults to 1. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. note:: - - The underlying scrypt hash function has a number of limitations - on it's parameter values, which forbids certain combinations of settings. - The requirements are: - - * ``linear_rounds = 2**`` - * ``linear_rounds < 2**(16 * block_size)`` - * ``block_size * parallelism <= 2**30-1`` - - .. todo:: - - This class currently does not support configuring default values - for ``block_size`` or ``parallelism`` via a :class:`~passlib.context.CryptContext` - configuration. - """ - - #=================================================================== - # class attrs - #=================================================================== - - #------------------------ - # PasswordHash - #------------------------ - name = "scrypt" - setting_kwds = ("ident", "salt", "salt_size", "rounds", "block_size", "parallelism") - - #------------------------ - # GenericHandler - #------------------------ - # NOTE: scrypt supports arbitrary output sizes. since it's output runs through - # pbkdf2-hmac-sha256 before returning, and this could be raised eventually... - # but a 256-bit digest is more than sufficient for password hashing. - # XXX: make checksum size configurable? could merge w/ argon2 code that does this. - checksum_size = 32 - - #------------------------ - # HasManyIdents - #------------------------ - default_ident = IDENT_SCRYPT - ident_values = (IDENT_SCRYPT, IDENT_7) - - #------------------------ - # HasRawSalt - #------------------------ - default_salt_size = 16 - max_salt_size = 1024 - - #------------------------ - # HasRounds - #------------------------ - # TODO: would like to dynamically pick this based on system - default_rounds = 16 - min_rounds = 1 - max_rounds = 31 # limited by scrypt alg - rounds_cost = "log2" - - # TODO: make default block size configurable via using(), and deprecatable via .needs_update() - - #=================================================================== - # instance attrs - #=================================================================== - - #: default parallelism setting (min=1 currently hardcoded in mixin) - parallelism = 1 - - #: default block size setting - block_size = 8 - - #=================================================================== - # variant constructor - #=================================================================== - - @classmethod - def using(cls, block_size=None, **kwds): - subcls = super(scrypt, cls).using(**kwds) - if block_size is not None: - if isinstance(block_size, uh.native_string_types): - block_size = int(block_size) - subcls.block_size = subcls._norm_block_size(block_size, relaxed=kwds.get("relaxed")) - - # make sure param combination is valid for scrypt() - try: - _scrypt.validate(1 << cls.default_rounds, cls.block_size, cls.parallelism) - except ValueError as err: - raise suppress_cause(ValueError("scrypt: invalid settings combination: " + str(err))) - - return subcls - - #=================================================================== - # parsing - #=================================================================== - - @classmethod - def from_string(cls, hash): - return cls(**cls.parse(hash)) - - @classmethod - def parse(cls, hash): - ident, suffix = cls._parse_ident(hash) - func = getattr(cls, "_parse_%s_string" % ident.strip(_UDOLLAR), None) - if func: - return func(suffix) - else: - raise uh.exc.InvalidHashError(cls) - - # - # passlib's format: - # $scrypt$ln=,r=,p=

$[$] - # where: - # logN, r, p -- decimal-encoded positive integer, no zero-padding - # logN -- log cost setting - # r -- block size setting (usually 8) - # p -- parallelism setting (usually 1) - # salt, digest -- b64-nopad encoded bytes - # - - @classmethod - def _parse_scrypt_string(cls, suffix): - # break params, salt, and digest sections - parts = suffix.split("$") - if len(parts) == 3: - params, salt, digest = parts - elif len(parts) == 2: - params, salt = parts - digest = None - else: - raise uh.exc.MalformedHashError(cls, "malformed hash") - - # break params apart - parts = params.split(",") - if len(parts) == 3: - nstr, bstr, pstr = parts - assert nstr.startswith("ln=") - assert bstr.startswith("r=") - assert pstr.startswith("p=") - else: - raise uh.exc.MalformedHashError(cls, "malformed settings field") - - return dict( - ident=IDENT_SCRYPT, - rounds=int(nstr[3:]), - block_size=int(bstr[2:]), - parallelism=int(pstr[2:]), - salt=b64s_decode(salt.encode("ascii")), - checksum=b64s_decode(digest.encode("ascii")) if digest else None, - ) - - # - # official format specification defined at - # https://gitlab.com/jas/scrypt-unix-crypt/blob/master/unix-scrypt.txt - # format: - # $7$[$] - # 0 12345 67890 1 - # where: - # All bytes use h64-little-endian encoding - # N: 6-bit log cost setting - # r: 30-bit block size setting - # p: 30-bit parallelism setting - # salt: variable length salt bytes - # digest: fixed 32-byte digest - # - - @classmethod - def _parse_7_string(cls, suffix): - # XXX: annoyingly, official spec embeds salt *raw*, yet doesn't specify a hash encoding. - # so assuming only h64 chars are valid for salt, and are ASCII encoded. - - # split into params & digest - parts = suffix.encode("ascii").split(b"$") - if len(parts) == 2: - params, digest = parts - elif len(parts) == 1: - params, = parts - digest = None - else: - raise uh.exc.MalformedHashError() - - # parse params & return - if len(params) < 11: - raise uh.exc.MalformedHashError(cls, "params field too short") - return dict( - ident=IDENT_7, - rounds=h64.decode_int6(params[:1]), - block_size=h64.decode_int30(params[1:6]), - parallelism=h64.decode_int30(params[6:11]), - salt=params[11:], - checksum=h64.decode_bytes(digest) if digest else None, - ) - - #=================================================================== - # formatting - #=================================================================== - def to_string(self): - ident = self.ident - if ident == IDENT_SCRYPT: - return "$scrypt$ln=%d,r=%d,p=%d$%s$%s" % ( - self.rounds, - self.block_size, - self.parallelism, - bascii_to_str(b64s_encode(self.salt)), - bascii_to_str(b64s_encode(self.checksum)), - ) - else: - assert ident == IDENT_7 - salt = self.salt - try: - salt.decode("ascii") - except UnicodeDecodeError: - raise suppress_cause(NotImplementedError("scrypt $7$ hashes dont support non-ascii salts")) - return bascii_to_str(b"".join([ - b"$7$", - h64.encode_int6(self.rounds), - h64.encode_int30(self.block_size), - h64.encode_int30(self.parallelism), - self.salt, - b"$", - h64.encode_bytes(self.checksum) - ])) - - #=================================================================== - # init - #=================================================================== - def __init__(self, block_size=None, **kwds): - super(scrypt, self).__init__(**kwds) - - # init block size - if block_size is None: - assert uh.validate_default_value(self, self.block_size, self._norm_block_size, - param="block_size") - else: - self.block_size = self._norm_block_size(block_size) - - # NOTE: if hash contains invalid complex constraint, relying on error - # being raised by scrypt call in _calc_checksum() - - @classmethod - def _norm_block_size(cls, block_size, relaxed=False): - return uh.norm_integer(cls, block_size, min=1, param="block_size", relaxed=relaxed) - - def _generate_salt(self): - salt = super(scrypt, self)._generate_salt() - if self.ident == IDENT_7: - # this format doesn't support non-ascii salts. - # as workaround, we take raw bytes, encoded to base64 - salt = b64s_encode(salt) - return salt - - #=================================================================== - # backend configuration - # NOTE: this following HasManyBackends' API, but provides it's own implementation, - # which actually switches the backend that 'passlib.crypto.scrypt.scrypt()' uses. - #=================================================================== - - @classproperty - def backends(cls): - return _scrypt.backend_values - - @classmethod - def get_backend(cls): - return _scrypt.backend - - @classmethod - def has_backend(cls, name="any"): - try: - cls.set_backend(name, dryrun=True) - return True - except uh.exc.MissingBackendError: - return False - - @classmethod - def set_backend(cls, name="any", dryrun=False): - _scrypt._set_backend(name, dryrun=dryrun) - - #=================================================================== - # digest calculation - #=================================================================== - def _calc_checksum(self, secret): - secret = to_bytes(secret, param="secret") - return _scrypt.scrypt(secret, self.salt, n=(1 << self.rounds), r=self.block_size, - p=self.parallelism, keylen=self.checksum_size) - - #=================================================================== - # hash migration - #=================================================================== - - def _calc_needs_update(self, **kwds): - """ - mark hash as needing update if rounds is outside desired bounds. - """ - # XXX: for now, marking all hashes which don't have matching block_size setting - if self.block_size != type(self).block_size: - return True - return super(scrypt, self)._calc_needs_update(**kwds) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/sha1_crypt.py b/src/passlib/handlers/sha1_crypt.py deleted file mode 100644 index d3e972cf..00000000 --- a/src/passlib/handlers/sha1_crypt.py +++ /dev/null @@ -1,158 +0,0 @@ -"""passlib.handlers.sha1_crypt -""" - -#============================================================================= -# imports -#============================================================================= - -# core -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import safe_crypt, test_crypt -from passlib.utils.binary import h64 -from passlib.utils.compat import u, unicode, irange -from passlib.crypto.digest import compile_hmac -import passlib.utils.handlers as uh -# local -__all__ = [ -] -#============================================================================= -# sha1-crypt -#============================================================================= -_BNULL = b'\x00' - -class sha1_crypt(uh.HasManyBackends, uh.HasRounds, uh.HasSalt, uh.GenericHandler): - """This class implements the SHA1-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, an 8 character one will be autogenerated (this is recommended). - If specified, it must be 0-64 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type salt_size: int - :param salt_size: - Optional number of bytes to use when autogenerating new salts. - Defaults to 8 bytes, but can be any value between 0 and 64. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 480000, must be between 1 and 4294967295, inclusive. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - - #=================================================================== - # class attrs - #=================================================================== - #--GenericHandler-- - name = "sha1_crypt" - setting_kwds = ("salt", "salt_size", "rounds") - ident = u("$sha1$") - checksum_size = 28 - checksum_chars = uh.HASH64_CHARS - - #--HasSalt-- - default_salt_size = 8 - max_salt_size = 64 - salt_chars = uh.HASH64_CHARS - - #--HasRounds-- - default_rounds = 480000 # current passlib default - min_rounds = 1 # really, this should be higher. - max_rounds = 4294967295 # 32-bit integer limit - rounds_cost = "linear" - - #=================================================================== - # formatting - #=================================================================== - @classmethod - def from_string(cls, hash): - rounds, salt, chk = uh.parse_mc3(hash, cls.ident, handler=cls) - return cls(rounds=rounds, salt=salt, checksum=chk) - - def to_string(self, config=False): - chk = None if config else self.checksum - return uh.render_mc3(self.ident, self.rounds, self.salt, chk) - - #=================================================================== - # backend - #=================================================================== - backends = ("os_crypt", "builtin") - - #--------------------------------------------------------------- - # os_crypt backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_os_crypt(cls): - if test_crypt("test", '$sha1$1$Wq3GL2Vp$C8U25GvfHS8qGHim' - 'ExLaiSFlGkAe'): - cls._set_calc_checksum_backend(cls._calc_checksum_os_crypt) - return True - else: - return False - - def _calc_checksum_os_crypt(self, secret): - config = self.to_string(config=True) - hash = safe_crypt(secret, config) - if hash: - assert hash.startswith(config) and len(hash) == len(config) + 29 - return hash[-28:] - else: - # py3's crypt.crypt() can't handle non-utf8 bytes. - # fallback to builtin alg, which is always available. - return self._calc_checksum_builtin(secret) - - #--------------------------------------------------------------- - # builtin backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_builtin(cls): - cls._set_calc_checksum_backend(cls._calc_checksum_builtin) - return True - - def _calc_checksum_builtin(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - if _BNULL in secret: - raise uh.exc.NullPasswordError(self) - rounds = self.rounds - # NOTE: this seed value is NOT the same as the config string - result = (u("%s$sha1$%s") % (self.salt, rounds)).encode("ascii") - # NOTE: this algorithm is essentially PBKDF1, modified to use HMAC. - keyed_hmac = compile_hmac("sha1", secret) - for _ in irange(rounds): - result = keyed_hmac(result) - return h64.encode_transposed_bytes(result, self._chk_offsets).decode("ascii") - - _chk_offsets = [ - 2,1,0, - 5,4,3, - 8,7,6, - 11,10,9, - 14,13,12, - 17,16,15, - 0,19,18, - ] - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/sha2_crypt.py b/src/passlib/handlers/sha2_crypt.py deleted file mode 100644 index 807de5eb..00000000 --- a/src/passlib/handlers/sha2_crypt.py +++ /dev/null @@ -1,519 +0,0 @@ -"""passlib.handlers.sha2_crypt - SHA256-Crypt / SHA512-Crypt""" -#============================================================================= -# imports -#============================================================================= -# core -import hashlib -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils import safe_crypt, test_crypt, \ - repeat_string, to_unicode -from passlib.utils.binary import h64 -from passlib.utils.compat import byte_elem_value, u, \ - uascii_to_str, unicode -import passlib.utils.handlers as uh -# local -__all__ = [ - "sha512_crypt", - "sha256_crypt", -] - -#============================================================================= -# pure-python backend, used by both sha256_crypt & sha512_crypt -# when crypt.crypt() backend is not available. -#============================================================================= -_BNULL = b'\x00' - -# pre-calculated offsets used to speed up C digest stage (see notes below). -# sequence generated using the following: - ##perms_order = "p,pp,ps,psp,sp,spp".split(",") - ##def offset(i): - ## key = (("p" if i % 2 else "") + ("s" if i % 3 else "") + - ## ("p" if i % 7 else "") + ("" if i % 2 else "p")) - ## return perms_order.index(key) - ##_c_digest_offsets = [(offset(i), offset(i+1)) for i in range(0,42,2)] -_c_digest_offsets = ( - (0, 3), (5, 1), (5, 3), (1, 2), (5, 1), (5, 3), (1, 3), - (4, 1), (5, 3), (1, 3), (5, 0), (5, 3), (1, 3), (5, 1), - (4, 3), (1, 3), (5, 1), (5, 2), (1, 3), (5, 1), (5, 3), - ) - -# map used to transpose bytes when encoding final sha256_crypt digest -_256_transpose_map = ( - 20, 10, 0, 11, 1, 21, 2, 22, 12, 23, 13, 3, 14, 4, 24, 5, - 25, 15, 26, 16, 6, 17, 7, 27, 8, 28, 18, 29, 19, 9, 30, 31, -) - -# map used to transpose bytes when encoding final sha512_crypt digest -_512_transpose_map = ( - 42, 21, 0, 1, 43, 22, 23, 2, 44, 45, 24, 3, 4, 46, 25, 26, - 5, 47, 48, 27, 6, 7, 49, 28, 29, 8, 50, 51, 30, 9, 10, 52, - 31, 32, 11, 53, 54, 33, 12, 13, 55, 34, 35, 14, 56, 57, 36, 15, - 16, 58, 37, 38, 17, 59, 60, 39, 18, 19, 61, 40, 41, 20, 62, 63, -) - -def _raw_sha2_crypt(pwd, salt, rounds, use_512=False): - """perform raw sha256-crypt / sha512-crypt - - this function provides a pure-python implementation of the internals - for the SHA256-Crypt and SHA512-Crypt algorithms; it doesn't - handle any of the parsing/validation of the hash strings themselves. - - :arg pwd: password chars/bytes to hash - :arg salt: salt chars to use - :arg rounds: linear rounds cost - :arg use_512: use sha512-crypt instead of sha256-crypt mode - - :returns: - encoded checksum chars - """ - #=================================================================== - # init & validate inputs - #=================================================================== - - # NOTE: the setup portion of this algorithm scales ~linearly in time - # with the size of the password, making it vulnerable to a DOS from - # unreasonably large inputs. the following code has some optimizations - # which would make things even worse, using O(pwd_len**2) memory - # when calculating digest P. - # - # to mitigate these two issues: 1) this code switches to a - # O(pwd_len)-memory algorithm for passwords that are much larger - # than average, and 2) Passlib enforces a library-wide max limit on - # the size of passwords it will allow, to prevent this algorithm and - # others from being DOSed in this way (see passlib.exc.PasswordSizeError - # for details). - - # validate secret - if isinstance(pwd, unicode): - # XXX: not sure what official unicode policy is, using this as default - pwd = pwd.encode("utf-8") - assert isinstance(pwd, bytes) - if _BNULL in pwd: - raise uh.exc.NullPasswordError(sha512_crypt if use_512 else sha256_crypt) - pwd_len = len(pwd) - - # validate rounds - assert 1000 <= rounds <= 999999999, "invalid rounds" - # NOTE: spec says out-of-range rounds should be clipped, instead of - # causing an error. this function assumes that's been taken care of - # by the handler class. - - # validate salt - assert isinstance(salt, unicode), "salt not unicode" - salt = salt.encode("ascii") - salt_len = len(salt) - assert salt_len < 17, "salt too large" - # NOTE: spec says salts larger than 16 bytes should be truncated, - # instead of causing an error. this function assumes that's been - # taken care of by the handler class. - - # load sha256/512 specific constants - if use_512: - hash_const = hashlib.sha512 - transpose_map = _512_transpose_map - else: - hash_const = hashlib.sha256 - transpose_map = _256_transpose_map - - #=================================================================== - # digest B - used as subinput to digest A - #=================================================================== - db = hash_const(pwd + salt + pwd).digest() - - #=================================================================== - # digest A - used to initialize first round of digest C - #=================================================================== - # start out with pwd + salt - a_ctx = hash_const(pwd + salt) - a_ctx_update = a_ctx.update - - # add pwd_len bytes of b, repeating b as many times as needed. - a_ctx_update(repeat_string(db, pwd_len)) - - # for each bit in pwd_len: add b if it's 1, or pwd if it's 0 - i = pwd_len - while i: - a_ctx_update(db if i & 1 else pwd) - i >>= 1 - - # finish A - da = a_ctx.digest() - - #=================================================================== - # digest P from password - used instead of password itself - # when calculating digest C. - #=================================================================== - if pwd_len < 96: - # this method is faster under python, but uses O(pwd_len**2) memory; - # so we don't use it for larger passwords to avoid a potential DOS. - dp = repeat_string(hash_const(pwd * pwd_len).digest(), pwd_len) - else: - # this method is slower under python, but uses a fixed amount of memory. - tmp_ctx = hash_const(pwd) - tmp_ctx_update = tmp_ctx.update - i = pwd_len-1 - while i: - tmp_ctx_update(pwd) - i -= 1 - dp = repeat_string(tmp_ctx.digest(), pwd_len) - assert len(dp) == pwd_len - - #=================================================================== - # digest S - used instead of salt itself when calculating digest C - #=================================================================== - ds = hash_const(salt * (16 + byte_elem_value(da[0]))).digest()[:salt_len] - assert len(ds) == salt_len, "salt_len somehow > hash_len!" - - #=================================================================== - # digest C - for a variable number of rounds, combine A, S, and P - # digests in various ways; in order to burn CPU time. - #=================================================================== - - # NOTE: the original SHA256/512-Crypt specification performs the C digest - # calculation using the following loop: - # - ##dc = da - ##i = 0 - ##while i < rounds: - ## tmp_ctx = hash_const(dp if i & 1 else dc) - ## if i % 3: - ## tmp_ctx.update(ds) - ## if i % 7: - ## tmp_ctx.update(dp) - ## tmp_ctx.update(dc if i & 1 else dp) - ## dc = tmp_ctx.digest() - ## i += 1 - # - # The code Passlib uses (below) implements an equivalent algorithm, - # it's just been heavily optimized to pre-calculate a large number - # of things beforehand. It works off of a couple of observations - # about the original algorithm: - # - # 1. each round is a combination of 'dc', 'ds', and 'dp'; determined - # by the whether 'i' a multiple of 2,3, and/or 7. - # 2. since lcm(2,3,7)==42, the series of combinations will repeat - # every 42 rounds. - # 3. even rounds 0-40 consist of 'hash(dc + round-specific-constant)'; - # while odd rounds 1-41 consist of hash(round-specific-constant + dc) - # - # Using these observations, the following code... - # * calculates the round-specific combination of ds & dp for each round 0-41 - # * runs through as many 42-round blocks as possible - # * runs through as many pairs of rounds as possible for remaining rounds - # * performs once last round if the total rounds should be odd. - # - # this cuts out a lot of the control overhead incurred when running the - # original loop 40,000+ times in python, resulting in ~20% increase in - # speed under CPython (though still 2x slower than glibc crypt) - - # prepare the 6 combinations of ds & dp which are needed - # (order of 'perms' must match how _c_digest_offsets was generated) - dp_dp = dp+dp - dp_ds = dp+ds - perms = [dp, dp_dp, dp_ds, dp_ds+dp, ds+dp, ds+dp_dp] - - # build up list of even-round & odd-round constants, - # and store in 21-element list as (even,odd) pairs. - data = [ (perms[even], perms[odd]) for even, odd in _c_digest_offsets] - - # perform as many full 42-round blocks as possible - dc = da - blocks, tail = divmod(rounds, 42) - while blocks: - for even, odd in data: - dc = hash_const(odd + hash_const(dc + even).digest()).digest() - blocks -= 1 - - # perform any leftover rounds - if tail: - # perform any pairs of rounds - pairs = tail>>1 - for even, odd in data[:pairs]: - dc = hash_const(odd + hash_const(dc + even).digest()).digest() - - # if rounds was odd, do one last round (since we started at 0, - # last round will be an even-numbered round) - if tail & 1: - dc = hash_const(dc + data[pairs][0]).digest() - - #=================================================================== - # encode digest using appropriate transpose map - #=================================================================== - return h64.encode_transposed_bytes(dc, transpose_map).decode("ascii") - -#============================================================================= -# handlers -#============================================================================= -_UROUNDS = u("rounds=") -_UDOLLAR = u("$") -_UZERO = u("0") - -class _SHA2_Common(uh.HasManyBackends, uh.HasRounds, uh.HasSalt, - uh.GenericHandler): - """class containing common code shared by sha256_crypt & sha512_crypt""" - #=================================================================== - # class attrs - #=================================================================== - # name - set by subclass - setting_kwds = ("salt", "rounds", "implicit_rounds", "salt_size") - # ident - set by subclass - checksum_chars = uh.HASH64_CHARS - # checksum_size - set by subclass - - max_salt_size = 16 - salt_chars = uh.HASH64_CHARS - - min_rounds = 1000 # bounds set by spec - max_rounds = 999999999 # bounds set by spec - rounds_cost = "linear" - - _cdb_use_512 = False # flag for _calc_digest_builtin() - _rounds_prefix = None # ident + _UROUNDS - - #=================================================================== - # methods - #=================================================================== - implicit_rounds = False - - def __init__(self, implicit_rounds=None, **kwds): - super(_SHA2_Common, self).__init__(**kwds) - # if user calls hash() w/ 5000 rounds, default to compact form. - if implicit_rounds is None: - implicit_rounds = (self.use_defaults and self.rounds == 5000) - self.implicit_rounds = implicit_rounds - - def _parse_salt(self, salt): - # required per SHA2-crypt spec -- truncate config salts rather than throwing error - return self._norm_salt(salt, relaxed=self.checksum is None) - - def _parse_rounds(self, rounds): - # required per SHA2-crypt spec -- clip config rounds rather than throwing error - return self._norm_rounds(rounds, relaxed=self.checksum is None) - - @classmethod - def from_string(cls, hash): - # basic format this parses - - # $5$[rounds=$][$] - - # TODO: this *could* use uh.parse_mc3(), except that the rounds - # portion has a slightly different grammar. - - # convert to unicode, check for ident prefix, split on dollar signs. - hash = to_unicode(hash, "ascii", "hash") - ident = cls.ident - if not hash.startswith(ident): - raise uh.exc.InvalidHashError(cls) - assert len(ident) == 3 - parts = hash[3:].split(_UDOLLAR) - - # extract rounds value - if parts[0].startswith(_UROUNDS): - assert len(_UROUNDS) == 7 - rounds = parts.pop(0)[7:] - if rounds.startswith(_UZERO) and rounds != _UZERO: - raise uh.exc.ZeroPaddedRoundsError(cls) - rounds = int(rounds) - implicit_rounds = False - else: - rounds = 5000 - implicit_rounds = True - - # rest should be salt and checksum - if len(parts) == 2: - salt, chk = parts - elif len(parts) == 1: - salt = parts[0] - chk = None - else: - raise uh.exc.MalformedHashError(cls) - - # return new object - return cls( - rounds=rounds, - salt=salt, - checksum=chk or None, - implicit_rounds=implicit_rounds, - ) - - def to_string(self): - if self.rounds == 5000 and self.implicit_rounds: - hash = u("%s%s$%s") % (self.ident, self.salt, - self.checksum or u('')) - else: - hash = u("%srounds=%d$%s$%s") % (self.ident, self.rounds, - self.salt, self.checksum or u('')) - return uascii_to_str(hash) - - #=================================================================== - # backends - #=================================================================== - backends = ("os_crypt", "builtin") - - #--------------------------------------------------------------- - # os_crypt backend - #--------------------------------------------------------------- - - #: test hash for OS detection -- provided by subclass - _test_hash = None - - @classmethod - def _load_backend_os_crypt(cls): - if test_crypt(*cls._test_hash): - cls._set_calc_checksum_backend(cls._calc_checksum_os_crypt) - return True - else: - return False - - def _calc_checksum_os_crypt(self, secret): - hash = safe_crypt(secret, self.to_string()) - if hash: - # NOTE: avoiding full parsing routine via from_string().checksum, - # and just extracting the bit we need. - cs = self.checksum_size - assert hash.startswith(self.ident) and hash[-cs-1] == _UDOLLAR - return hash[-cs:] - else: - # py3's crypt.crypt() can't handle non-utf8 bytes. - # fallback to builtin alg, which is always available. - return self._calc_checksum_builtin(secret) - - #--------------------------------------------------------------- - # builtin backend - #--------------------------------------------------------------- - @classmethod - def _load_backend_builtin(cls): - cls._set_calc_checksum_backend(cls._calc_checksum_builtin) - return True - - def _calc_checksum_builtin(self, secret): - return _raw_sha2_crypt(secret, self.salt, self.rounds, - self._cdb_use_512) - - #=================================================================== - # eoc - #=================================================================== - -class sha256_crypt(_SHA2_Common): - """This class implements the SHA256-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 0-16 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 535000, must be between 1000 and 999999999, inclusive. - - :type implicit_rounds: bool - :param implicit_rounds: - this is an internal option which generally doesn't need to be touched. - - this flag determines whether the hash should omit the rounds parameter - when encoding it to a string; this is only permitted by the spec for rounds=5000, - and the flag is ignored otherwise. the spec requires the two different - encodings be preserved as they are, instead of normalizing them. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - name = "sha256_crypt" - ident = u("$5$") - checksum_size = 43 - # NOTE: using 25/75 weighting of builtin & os_crypt backends - default_rounds = 535000 - - #=================================================================== - # backends - #=================================================================== - _test_hash = ("test", "$5$rounds=1000$test$QmQADEXMG8POI5W" - "Dsaeho0P36yK3Tcrgboabng6bkb/") - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# sha 512 crypt -#============================================================================= -class sha512_crypt(_SHA2_Common): - """This class implements the SHA512-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, one will be autogenerated (this is recommended). - If specified, it must be 0-16 characters, drawn from the regexp range ``[./0-9A-Za-z]``. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 656000, must be between 1000 and 999999999, inclusive. - - :type implicit_rounds: bool - :param implicit_rounds: - this is an internal option which generally doesn't need to be touched. - - this flag determines whether the hash should omit the rounds parameter - when encoding it to a string; this is only permitted by the spec for rounds=5000, - and the flag is ignored otherwise. the spec requires the two different - encodings be preserved as they are, instead of normalizing them. - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - - #=================================================================== - # class attrs - #=================================================================== - name = "sha512_crypt" - ident = u("$6$") - checksum_size = 86 - _cdb_use_512 = True - # NOTE: using 25/75 weighting of builtin & os_crypt backends - default_rounds = 656000 - - #=================================================================== - # backend - #=================================================================== - _test_hash = ("test", "$6$rounds=1000$test$2M/Lx6Mtobqj" - "Ljobw0Wmo4Q5OFx5nVLJvmgseatA6oMn" - "yWeBdRDx4DU.1H3eGmse6pgsOgDisWBG" - "I5c7TZauS0") - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/sun_md5_crypt.py b/src/passlib/handlers/sun_md5_crypt.py deleted file mode 100644 index 0eeb4e74..00000000 --- a/src/passlib/handlers/sun_md5_crypt.py +++ /dev/null @@ -1,363 +0,0 @@ -"""passlib.handlers.sun_md5_crypt - Sun's Md5 Crypt, used on Solaris - -.. warning:: - - This implementation may not reproduce - the original Solaris behavior in some border cases. - See documentation for details. -""" - -#============================================================================= -# imports -#============================================================================= -# core -from hashlib import md5 -import re -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import to_unicode -from passlib.utils.binary import h64 -from passlib.utils.compat import byte_elem_value, irange, u, \ - uascii_to_str, unicode, str_to_bascii -import passlib.utils.handlers as uh -# local -__all__ = [ - "sun_md5_crypt", -] - -#============================================================================= -# backend -#============================================================================= -# constant data used by alg - Hamlet act 3 scene 1 + null char -# exact bytes as in http://www.ibiblio.org/pub/docs/books/gutenberg/etext98/2ws2610.txt -# from Project Gutenberg. - -MAGIC_HAMLET = ( - b"To be, or not to be,--that is the question:--\n" - b"Whether 'tis nobler in the mind to suffer\n" - b"The slings and arrows of outrageous fortune\n" - b"Or to take arms against a sea of troubles,\n" - b"And by opposing end them?--To die,--to sleep,--\n" - b"No more; and by a sleep to say we end\n" - b"The heartache, and the thousand natural shocks\n" - b"That flesh is heir to,--'tis a consummation\n" - b"Devoutly to be wish'd. To die,--to sleep;--\n" - b"To sleep! perchance to dream:--ay, there's the rub;\n" - b"For in that sleep of death what dreams may come,\n" - b"When we have shuffled off this mortal coil,\n" - b"Must give us pause: there's the respect\n" - b"That makes calamity of so long life;\n" - b"For who would bear the whips and scorns of time,\n" - b"The oppressor's wrong, the proud man's contumely,\n" - b"The pangs of despis'd love, the law's delay,\n" - b"The insolence of office, and the spurns\n" - b"That patient merit of the unworthy takes,\n" - b"When he himself might his quietus make\n" - b"With a bare bodkin? who would these fardels bear,\n" - b"To grunt and sweat under a weary life,\n" - b"But that the dread of something after death,--\n" - b"The undiscover'd country, from whose bourn\n" - b"No traveller returns,--puzzles the will,\n" - b"And makes us rather bear those ills we have\n" - b"Than fly to others that we know not of?\n" - b"Thus conscience does make cowards of us all;\n" - b"And thus the native hue of resolution\n" - b"Is sicklied o'er with the pale cast of thought;\n" - b"And enterprises of great pith and moment,\n" - b"With this regard, their currents turn awry,\n" - b"And lose the name of action.--Soft you now!\n" - b"The fair Ophelia!--Nymph, in thy orisons\n" - b"Be all my sins remember'd.\n\x00" #<- apparently null at end of C string is included (test vector won't pass otherwise) -) - -# NOTE: these sequences are pre-calculated iteration ranges used by X & Y loops w/in rounds function below -xr = irange(7) -_XY_ROUNDS = [ - tuple((i,i,i+3) for i in xr), # xrounds 0 - tuple((i,i+1,i+4) for i in xr), # xrounds 1 - tuple((i,i+8,(i+11)&15) for i in xr), # yrounds 0 - tuple((i,(i+9)&15, (i+12)&15) for i in xr), # yrounds 1 -] -del xr - -def raw_sun_md5_crypt(secret, rounds, salt): - """given secret & salt, return encoded sun-md5-crypt checksum""" - global MAGIC_HAMLET - assert isinstance(secret, bytes) - assert isinstance(salt, bytes) - - # validate rounds - if rounds <= 0: - rounds = 0 - real_rounds = 4096 + rounds - # NOTE: spec seems to imply max 'rounds' is 2**32-1 - - # generate initial digest to start off round 0. - # NOTE: algorithm 'salt' includes full config string w/ trailing "$" - result = md5(secret + salt).digest() - assert len(result) == 16 - - # NOTE: many things in this function have been inlined (to speed up the loop - # as much as possible), to the point that this code barely resembles - # the algorithm as described in the docs. in particular: - # - # * all accesses to a given bit have been inlined using the formula - # rbitval(bit) = (rval((bit>>3) & 15) >> (bit & 7)) & 1 - # - # * the calculation of coinflip value R has been inlined - # - # * the conditional division of coinflip value V has been inlined as - # a shift right of 0 or 1. - # - # * the i, i+3, etc iterations are precalculated in lists. - # - # * the round-based conditional division of x & y is now performed - # by choosing an appropriate precalculated list, so that it only - # calculates the 7 bits which will actually be used. - # - X_ROUNDS_0, X_ROUNDS_1, Y_ROUNDS_0, Y_ROUNDS_1 = _XY_ROUNDS - - # NOTE: % appears to be *slightly* slower than &, so we prefer & if possible - - round = 0 - while round < real_rounds: - # convert last result byte string to list of byte-ints for easy access - rval = [ byte_elem_value(c) for c in result ].__getitem__ - - # build up X bit by bit - x = 0 - xrounds = X_ROUNDS_1 if (rval((round>>3) & 15)>>(round & 7)) & 1 else X_ROUNDS_0 - for i, ia, ib in xrounds: - a = rval(ia) - b = rval(ib) - v = rval((a >> (b % 5)) & 15) >> ((b>>(a&7)) & 1) - x |= ((rval((v>>3)&15)>>(v&7))&1) << i - - # build up Y bit by bit - y = 0 - yrounds = Y_ROUNDS_1 if (rval(((round+64)>>3) & 15)>>(round & 7)) & 1 else Y_ROUNDS_0 - for i, ia, ib in yrounds: - a = rval(ia) - b = rval(ib) - v = rval((a >> (b % 5)) & 15) >> ((b>>(a&7)) & 1) - y |= ((rval((v>>3)&15)>>(v&7))&1) << i - - # extract x'th and y'th bit, xoring them together to yeild "coin flip" - coin = ((rval(x>>3) >> (x&7)) ^ (rval(y>>3) >> (y&7))) & 1 - - # construct hash for this round - h = md5(result) - if coin: - h.update(MAGIC_HAMLET) - h.update(unicode(round).encode("ascii")) - result = h.digest() - - round += 1 - - # encode output - return h64.encode_transposed_bytes(result, _chk_offsets) - -# NOTE: same offsets as md5_crypt -_chk_offsets = ( - 12,6,0, - 13,7,1, - 14,8,2, - 15,9,3, - 5,10,4, - 11, -) - -#============================================================================= -# handler -#============================================================================= -class sun_md5_crypt(uh.HasRounds, uh.HasSalt, uh.GenericHandler): - """This class implements the Sun-MD5-Crypt password hash, and follows the :ref:`password-hash-api`. - - It supports a variable-length salt, and a variable number of rounds. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts the following optional keywords: - - :type salt: str - :param salt: - Optional salt string. - If not specified, a salt will be autogenerated (this is recommended). - If specified, it must be drawn from the regexp range ``[./0-9A-Za-z]``. - - :type salt_size: int - :param salt_size: - If no salt is specified, this parameter can be used to specify - the size (in characters) of the autogenerated salt. - It currently defaults to 8. - - :type rounds: int - :param rounds: - Optional number of rounds to use. - Defaults to 34000, must be between 0 and 4294963199, inclusive. - - :type bare_salt: bool - :param bare_salt: - Optional flag used to enable an alternate salt digest behavior - used by some hash strings in this scheme. - This flag can be ignored by most users. - Defaults to ``False``. - (see :ref:`smc-bare-salt` for details). - - :type relaxed: bool - :param relaxed: - By default, providing an invalid value for one of the other - keywords will result in a :exc:`ValueError`. If ``relaxed=True``, - and the error can be corrected, a :exc:`~passlib.exc.PasslibHashWarning` - will be issued instead. Correctable errors include ``rounds`` - that are too small or too large, and ``salt`` strings that are too long. - - .. versionadded:: 1.6 - """ - #=================================================================== - # class attrs - #=================================================================== - name = "sun_md5_crypt" - setting_kwds = ("salt", "rounds", "bare_salt", "salt_size") - checksum_chars = uh.HASH64_CHARS - checksum_size = 22 - - # NOTE: docs say max password length is 255. - # release 9u2 - - # NOTE: not sure if original crypt has a salt size limit, - # all instances that have been seen use 8 chars. - default_salt_size = 8 - max_salt_size = None - salt_chars = uh.HASH64_CHARS - - default_rounds = 34000 # current passlib default - min_rounds = 0 - max_rounds = 4294963199 ##2**32-1-4096 - # XXX: ^ not sure what it does if past this bound... does 32 int roll over? - rounds_cost = "linear" - - ident_values = (u("$md5$"), u("$md5,")) - - #=================================================================== - # instance attrs - #=================================================================== - bare_salt = False # flag to indicate legacy hashes that lack "$$" suffix - - #=================================================================== - # constructor - #=================================================================== - def __init__(self, bare_salt=False, **kwds): - self.bare_salt = bare_salt - super(sun_md5_crypt, self).__init__(**kwds) - - #=================================================================== - # internal helpers - #=================================================================== - @classmethod - def identify(cls, hash): - hash = uh.to_unicode_for_identify(hash) - return hash.startswith(cls.ident_values) - - @classmethod - def from_string(cls, hash): - hash = to_unicode(hash, "ascii", "hash") - - # - # detect if hash specifies rounds value. - # if so, parse and validate it. - # by end, set 'rounds' to int value, and 'tail' containing salt+chk - # - if hash.startswith(u("$md5$")): - rounds = 0 - salt_idx = 5 - elif hash.startswith(u("$md5,rounds=")): - idx = hash.find(u("$"), 12) - if idx == -1: - raise uh.exc.MalformedHashError(cls, "unexpected end of rounds") - rstr = hash[12:idx] - try: - rounds = int(rstr) - except ValueError: - raise uh.exc.MalformedHashError(cls, "bad rounds") - if rstr != unicode(rounds): - raise uh.exc.ZeroPaddedRoundsError(cls) - if rounds == 0: - # NOTE: not sure if this is forbidden by spec or not; - # but allowing it would complicate things, - # and it should never occur anyways. - raise uh.exc.MalformedHashError(cls, "explicit zero rounds") - salt_idx = idx+1 - else: - raise uh.exc.InvalidHashError(cls) - - # - # salt/checksum separation is kinda weird, - # to deal cleanly with some backward-compatible workarounds - # implemented by original implementation. - # - chk_idx = hash.rfind(u("$"), salt_idx) - if chk_idx == -1: - # ''-config for $-hash - salt = hash[salt_idx:] - chk = None - bare_salt = True - elif chk_idx == len(hash)-1: - if chk_idx > salt_idx and hash[-2] == u("$"): - raise uh.exc.MalformedHashError(cls, "too many '$' separators") - # $-config for $$-hash - salt = hash[salt_idx:-1] - chk = None - bare_salt = False - elif chk_idx > 0 and hash[chk_idx-1] == u("$"): - # $$-hash - salt = hash[salt_idx:chk_idx-1] - chk = hash[chk_idx+1:] - bare_salt = False - else: - # $-hash - salt = hash[salt_idx:chk_idx] - chk = hash[chk_idx+1:] - bare_salt = True - - return cls( - rounds=rounds, - salt=salt, - checksum=chk, - bare_salt=bare_salt, - ) - - def to_string(self, _withchk=True): - ss = u('') if self.bare_salt else u('$') - rounds = self.rounds - if rounds > 0: - hash = u("$md5,rounds=%d$%s%s") % (rounds, self.salt, ss) - else: - hash = u("$md5$%s%s") % (self.salt, ss) - if _withchk: - chk = self.checksum - hash = u("%s$%s") % (hash, chk) - return uascii_to_str(hash) - - #=================================================================== - # primary interface - #=================================================================== - # TODO: if we're on solaris, check for native crypt() support. - # this will require extra testing, to make sure native crypt - # actually behaves correctly. of particular importance: - # when using ""-config, make sure to append "$x" to string. - - def _calc_checksum(self, secret): - # NOTE: no reference for how sun_md5_crypt handles unicode - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - config = str_to_bascii(self.to_string(_withchk=False)) - return raw_sun_md5_crypt(secret, self.rounds, config).decode("ascii") - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/handlers/windows.py b/src/passlib/handlers/windows.py deleted file mode 100644 index e17beba4..00000000 --- a/src/passlib/handlers/windows.py +++ /dev/null @@ -1,334 +0,0 @@ -"""passlib.handlers.nthash - Microsoft Windows -related hashes""" -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify -import logging; log = logging.getLogger(__name__) -from warnings import warn -# site -# pkg -from passlib.utils import to_unicode, right_pad_string -from passlib.utils.compat import unicode -from passlib.crypto.digest import lookup_hash -md4 = lookup_hash("md4").const -import passlib.utils.handlers as uh -# local -__all__ = [ - "lmhash", - "nthash", - "bsd_nthash", - "msdcc", - "msdcc2", -] - -#============================================================================= -# lanman hash -#============================================================================= -class lmhash(uh.TruncateMixin, uh.HasEncodingContext, uh.StaticHandler): - """This class implements the Lan Manager Password hash, and follows the :ref:`password-hash-api`. - - It has no salt and a single fixed round. - - The :meth:`~passlib.ifc.PasswordHash.using` method accepts a single - optional keyword: - - :param bool truncate_error: - By default, this will silently truncate passwords larger than 14 bytes. - Setting ``truncate_error=True`` will cause :meth:`~passlib.ifc.PasswordHash.hash` - to raise a :exc:`~passlib.exc.PasswordTruncateError` instead. - - .. versionadded:: 1.7 - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.verify` methods accept a single - optional keyword: - - :type encoding: str - :param encoding: - - This specifies what character encoding LMHASH should use when - calculating digest. It defaults to ``cp437``, the most - common encoding encountered. - - Note that while this class outputs digests in lower-case hexadecimal, - it will accept upper-case as well. - """ - #=================================================================== - # class attrs - #=================================================================== - - #-------------------- - # PasswordHash - #-------------------- - name = "lmhash" - setting_kwds = ("truncate_error",) - - #-------------------- - # GenericHandler - #-------------------- - checksum_chars = uh.HEX_CHARS - checksum_size = 32 - - #-------------------- - # TruncateMixin - #-------------------- - truncate_size = 14 - - #-------------------- - # custom - #-------------------- - default_encoding = "cp437" - - #=================================================================== - # methods - #=================================================================== - @classmethod - def _norm_hash(cls, hash): - return hash.lower() - - def _calc_checksum(self, secret): - # check for truncation (during .hash() calls only) - if self.use_defaults: - self._check_truncate_policy(secret) - - return hexlify(self.raw(secret, self.encoding)).decode("ascii") - - # magic constant used by LMHASH - _magic = b"KGS!@#$%" - - @classmethod - def raw(cls, secret, encoding=None): - """encode password using LANMAN hash algorithm. - - :type secret: unicode or utf-8 encoded bytes - :arg secret: secret to hash - :type encoding: str - :arg encoding: - optional encoding to use for unicode inputs. - this defaults to ``cp437``, which is the - common case for most situations. - - :returns: returns string of raw bytes - """ - if not encoding: - encoding = cls.default_encoding - # some nice empircal data re: different encodings is at... - # http://www.openwall.com/lists/john-dev/2011/08/01/2 - # http://www.freerainbowtables.com/phpBB3/viewtopic.php?t=387&p=12163 - from passlib.crypto.des import des_encrypt_block - MAGIC = cls._magic - if isinstance(secret, unicode): - # perform uppercasing while we're still unicode, - # to give a better shot at getting non-ascii chars right. - # (though some codepages do NOT upper-case the same as unicode). - secret = secret.upper().encode(encoding) - elif isinstance(secret, bytes): - # FIXME: just trusting ascii upper will work? - # and if not, how to do codepage specific case conversion? - # we could decode first using , - # but *that* might not always be right. - secret = secret.upper() - else: - raise TypeError("secret must be unicode or bytes") - secret = right_pad_string(secret, 14) - return des_encrypt_block(secret[0:7], MAGIC) + \ - des_encrypt_block(secret[7:14], MAGIC) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# ntlm hash -#============================================================================= -class nthash(uh.StaticHandler): - """This class implements the NT Password hash, and follows the :ref:`password-hash-api`. - - It has no salt and a single fixed round. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept no optional keywords. - - Note that while this class outputs lower-case hexadecimal digests, - it will accept upper-case digests as well. - """ - #=================================================================== - # class attrs - #=================================================================== - name = "nthash" - checksum_chars = uh.HEX_CHARS - checksum_size = 32 - - #=================================================================== - # methods - #=================================================================== - @classmethod - def _norm_hash(cls, hash): - return hash.lower() - - def _calc_checksum(self, secret): - return hexlify(self.raw(secret)).decode("ascii") - - @classmethod - def raw(cls, secret): - """encode password using MD4-based NTHASH algorithm - - :arg secret: secret as unicode or utf-8 encoded bytes - - :returns: returns string of raw bytes - """ - secret = to_unicode(secret, "utf-8", param="secret") - # XXX: found refs that say only first 128 chars are used. - return md4(secret.encode("utf-16-le")).digest() - - @classmethod - def raw_nthash(cls, secret, hex=False): - warn("nthash.raw_nthash() is deprecated, and will be removed " - "in Passlib 1.8, please use nthash.raw() instead", - DeprecationWarning) - ret = nthash.raw(secret) - return hexlify(ret).decode("ascii") if hex else ret - - #=================================================================== - # eoc - #=================================================================== - -bsd_nthash = uh.PrefixWrapper("bsd_nthash", nthash, prefix="$3$$", ident="$3$$", - doc="""The class support FreeBSD's representation of NTHASH - (which is compatible with the :ref:`modular-crypt-format`), - and follows the :ref:`password-hash-api`. - - It has no salt and a single fixed round. - - The :meth:`~passlib.ifc.PasswordHash.hash` and :meth:`~passlib.ifc.PasswordHash.genconfig` methods accept no optional keywords. - """) - -##class ntlm_pair(object): -## "combined lmhash & nthash" -## name = "ntlm_pair" -## setting_kwds = () -## _hash_regex = re.compile(u"^(?P[0-9a-f]{32}):(?P[0-9][a-f]{32})$", -## re.I) -## -## @classmethod -## def identify(cls, hash): -## hash = to_unicode(hash, "latin-1", "hash") -## return len(hash) == 65 and cls._hash_regex.match(hash) is not None -## -## @classmethod -## def hash(cls, secret, config=None): -## if config is not None and not cls.identify(config): -## raise uh.exc.InvalidHashError(cls) -## return lmhash.hash(secret) + ":" + nthash.hash(secret) -## -## @classmethod -## def verify(cls, secret, hash): -## hash = to_unicode(hash, "ascii", "hash") -## m = cls._hash_regex.match(hash) -## if not m: -## raise uh.exc.InvalidHashError(cls) -## lm, nt = m.group("lm", "nt") -## # NOTE: verify against both in case encoding issue -## # causes one not to match. -## return lmhash.verify(secret, lm) or nthash.verify(secret, nt) - -#============================================================================= -# msdcc v1 -#============================================================================= -class msdcc(uh.HasUserContext, uh.StaticHandler): - """This class implements Microsoft's Domain Cached Credentials password hash, - and follows the :ref:`password-hash-api`. - - It has a fixed number of rounds, and uses the associated - username as the salt. - - The :meth:`~passlib.ifc.PasswordHash.hash`, :meth:`~passlib.ifc.PasswordHash.genhash`, and :meth:`~passlib.ifc.PasswordHash.verify` methods - have the following optional keywords: - - :type user: str - :param user: - String containing name of user account this password is associated with. - This is required to properly calculate the hash. - - This keyword is case-insensitive, and should contain just the username - (e.g. ``Administrator``, not ``SOMEDOMAIN\\Administrator``). - - Note that while this class outputs lower-case hexadecimal digests, - it will accept upper-case digests as well. - """ - name = "msdcc" - checksum_chars = uh.HEX_CHARS - checksum_size = 32 - - @classmethod - def _norm_hash(cls, hash): - return hash.lower() - - def _calc_checksum(self, secret): - return hexlify(self.raw(secret, self.user)).decode("ascii") - - @classmethod - def raw(cls, secret, user): - """encode password using mscash v1 algorithm - - :arg secret: secret as unicode or utf-8 encoded bytes - :arg user: username to use as salt - - :returns: returns string of raw bytes - """ - secret = to_unicode(secret, "utf-8", param="secret").encode("utf-16-le") - user = to_unicode(user, "utf-8", param="user").lower().encode("utf-16-le") - return md4(md4(secret).digest() + user).digest() - -#============================================================================= -# msdcc2 aka mscash2 -#============================================================================= -class msdcc2(uh.HasUserContext, uh.StaticHandler): - """This class implements version 2 of Microsoft's Domain Cached Credentials - password hash, and follows the :ref:`password-hash-api`. - - It has a fixed number of rounds, and uses the associated - username as the salt. - - The :meth:`~passlib.ifc.PasswordHash.hash`, :meth:`~passlib.ifc.PasswordHash.genhash`, and :meth:`~passlib.ifc.PasswordHash.verify` methods - have the following extra keyword: - - :type user: str - :param user: - String containing name of user account this password is associated with. - This is required to properly calculate the hash. - - This keyword is case-insensitive, and should contain just the username - (e.g. ``Administrator``, not ``SOMEDOMAIN\\Administrator``). - """ - name = "msdcc2" - checksum_chars = uh.HEX_CHARS - checksum_size = 32 - - @classmethod - def _norm_hash(cls, hash): - return hash.lower() - - def _calc_checksum(self, secret): - return hexlify(self.raw(secret, self.user)).decode("ascii") - - @classmethod - def raw(cls, secret, user): - """encode password using msdcc v2 algorithm - - :type secret: unicode or utf-8 bytes - :arg secret: secret - - :type user: str - :arg user: username to use as salt - - :returns: returns string of raw bytes - """ - from passlib.crypto.digest import pbkdf2_hmac - secret = to_unicode(secret, "utf-8", param="secret").encode("utf-16-le") - user = to_unicode(user, "utf-8", param="user").lower().encode("utf-16-le") - tmp = md4(md4(secret).digest() + user).digest() - return pbkdf2_hmac("sha1", tmp, user, 10240, 16) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/hash.py b/src/passlib/hash.py deleted file mode 100644 index 9b724488..00000000 --- a/src/passlib/hash.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -passlib.hash - proxy object mapping hash scheme names -> handlers - -================== -***** NOTICE ***** -================== - -This module does not actually contain any hashes. This file -is a stub that replaces itself with a proxy object. - -This proxy object (passlib.registry._PasslibRegistryProxy) -handles lazy-loading hashes as they are requested. - -The actual implementation of the various hashes is store elsewhere, -mainly in the submodules of the ``passlib.handlers`` subpackage. -""" - -#============================================================================= -# import proxy object and replace this module -#============================================================================= - -# XXX: if any platform has problem w/ lazy modules, could support 'non-lazy' -# version which just imports all schemes known to list_crypt_handlers() - -from passlib.registry import _proxy -import sys -sys.modules[__name__] = _proxy - -#============================================================================= -# HACK: the following bit of code is unreachable, but it's presence seems to -# help make autocomplete work for certain IDEs such as PyCharm. -# this list is automatically regenerated using $SOURCE/admin/regen.py -#============================================================================= - -#---------------------------------------------------- -# begin autocomplete hack (autogenerated 2016-11-10) -#---------------------------------------------------- -if False: - from passlib.handlers.argon2 import argon2 - from passlib.handlers.bcrypt import bcrypt, bcrypt_sha256 - from passlib.handlers.cisco import cisco_asa, cisco_pix, cisco_type7 - from passlib.handlers.des_crypt import bigcrypt, bsdi_crypt, crypt16, des_crypt - from passlib.handlers.digests import hex_md4, hex_md5, hex_sha1, hex_sha256, hex_sha512, htdigest - from passlib.handlers.django import django_bcrypt, django_bcrypt_sha256, django_des_crypt, django_disabled, django_pbkdf2_sha1, django_pbkdf2_sha256, django_salted_md5, django_salted_sha1 - from passlib.handlers.fshp import fshp - from passlib.handlers.ldap_digests import ldap_bcrypt, ldap_bsdi_crypt, ldap_des_crypt, ldap_md5, ldap_md5_crypt, ldap_plaintext, ldap_salted_md5, ldap_salted_sha1, ldap_sha1, ldap_sha1_crypt, ldap_sha256_crypt, ldap_sha512_crypt - from passlib.handlers.md5_crypt import apr_md5_crypt, md5_crypt - from passlib.handlers.misc import plaintext, unix_disabled, unix_fallback - from passlib.handlers.mssql import mssql2000, mssql2005 - from passlib.handlers.mysql import mysql323, mysql41 - from passlib.handlers.oracle import oracle10, oracle11 - from passlib.handlers.pbkdf2 import atlassian_pbkdf2_sha1, cta_pbkdf2_sha1, dlitz_pbkdf2_sha1, grub_pbkdf2_sha512, ldap_pbkdf2_sha1, ldap_pbkdf2_sha256, ldap_pbkdf2_sha512, pbkdf2_sha1, pbkdf2_sha256, pbkdf2_sha512 - from passlib.handlers.phpass import phpass - from passlib.handlers.postgres import postgres_md5 - from passlib.handlers.roundup import ldap_hex_md5, ldap_hex_sha1, roundup_plaintext - from passlib.handlers.scram import scram - from passlib.handlers.scrypt import scrypt - from passlib.handlers.sha1_crypt import sha1_crypt - from passlib.handlers.sha2_crypt import sha256_crypt, sha512_crypt - from passlib.handlers.sun_md5_crypt import sun_md5_crypt - from passlib.handlers.windows import bsd_nthash, lmhash, msdcc, msdcc2, nthash -#---------------------------------------------------- -# end autocomplete hack -#---------------------------------------------------- - -#============================================================================= -# eoc -#============================================================================= diff --git a/src/passlib/hosts.py b/src/passlib/hosts.py deleted file mode 100644 index 1f137a26..00000000 --- a/src/passlib/hosts.py +++ /dev/null @@ -1,106 +0,0 @@ -"""passlib.hosts""" -#============================================================================= -# imports -#============================================================================= -# core -from warnings import warn -# pkg -from passlib.context import LazyCryptContext -from passlib.exc import PasslibRuntimeWarning -from passlib import registry -from passlib.utils import has_crypt, unix_crypt_schemes -# local -__all__ = [ - "linux_context", "linux2_context", - "openbsd_context", - "netbsd_context", - "freebsd_context", - "host_context", -] - -#============================================================================= -# linux support -#============================================================================= - -# known platform names - linux2 - -linux_context = linux2_context = LazyCryptContext( - schemes = [ "sha512_crypt", "sha256_crypt", "md5_crypt", - "des_crypt", "unix_disabled" ], - deprecated = [ "des_crypt" ], - ) - -#============================================================================= -# bsd support -#============================================================================= - -# known platform names - -# freebsd2 -# freebsd3 -# freebsd4 -# freebsd5 -# freebsd6 -# freebsd7 -# -# netbsd1 - -# referencing source via -http://fxr.googlebit.com -# freebsd 6,7,8 - des, md5, bcrypt, bsd_nthash -# netbsd - des, ext, md5, bcrypt, sha1 -# openbsd - des, ext, md5, bcrypt - -freebsd_context = LazyCryptContext(["bcrypt", "md5_crypt", "bsd_nthash", - "des_crypt", "unix_disabled"]) - -openbsd_context = LazyCryptContext(["bcrypt", "md5_crypt", "bsdi_crypt", - "des_crypt", "unix_disabled"]) - -netbsd_context = LazyCryptContext(["bcrypt", "sha1_crypt", "md5_crypt", - "bsdi_crypt", "des_crypt", "unix_disabled"]) - -# XXX: include darwin in this list? it's got a BSD crypt variant, -# but that's not what it uses for user passwords. - -#============================================================================= -# current host -#============================================================================= -if registry.os_crypt_present: - # NOTE: this is basically mimicing the output of os crypt(), - # except that it uses passlib's (usually stronger) defaults settings, - # and can be inspected and used much more flexibly. - - def _iter_os_crypt_schemes(): - """helper which iterates over supported os_crypt schemes""" - out = registry.get_supported_os_crypt_schemes() - if out: - # only offer disabled handler if there's another scheme in front, - # as this can't actually hash any passwords - out += ("unix_disabled",) - return out - - host_context = LazyCryptContext(_iter_os_crypt_schemes()) - -#============================================================================= -# other platforms -#============================================================================= - -# known platform strings - -# aix3 -# aix4 -# atheos -# beos5 -# darwin -# generic -# hp-ux11 -# irix5 -# irix6 -# mac -# next3 -# os2emx -# riscos -# sunos5 -# unixware7 - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/ifc.py b/src/passlib/ifc.py deleted file mode 100644 index 1a1aef2d..00000000 --- a/src/passlib/ifc.py +++ /dev/null @@ -1,353 +0,0 @@ -"""passlib.ifc - abstract interfaces used by Passlib""" -#============================================================================= -# imports -#============================================================================= -# core -import logging; log = logging.getLogger(__name__) -import sys -# site -# pkg -from passlib.utils.decor import deprecated_method -# local -__all__ = [ - "PasswordHash", -] - -#============================================================================= -# 2/3 compatibility helpers -#============================================================================= -def recreate_with_metaclass(meta): - """class decorator that re-creates class using metaclass""" - def builder(cls): - if meta is type(cls): - return cls - return meta(cls.__name__, cls.__bases__, cls.__dict__.copy()) - return builder - -#============================================================================= -# PasswordHash interface -#============================================================================= -from abc import ABCMeta, abstractmethod, abstractproperty - -# TODO: make this actually use abstractproperty(), -# now that we dropped py25, 'abc' is always available. - -# XXX: rename to PasswordHasher? - -@recreate_with_metaclass(ABCMeta) -class PasswordHash(object): - """This class describes an abstract interface which all password hashes - in Passlib adhere to. Under Python 2.6 and up, this is an actual - Abstract Base Class built using the :mod:`!abc` module. - - See the Passlib docs for full documentation. - """ - #=================================================================== - # class attributes - #=================================================================== - - #--------------------------------------------------------------- - # general information - #--------------------------------------------------------------- - ##name - ##setting_kwds - ##context_kwds - - #: flag which indicates this hasher matches a "disabled" hash - #: (e.g. unix_disabled, or django_disabled); and doesn't actually - #: depend on the provided password. - is_disabled = False - - #: Should be None, or a positive integer indicating hash - #: doesn't support secrets larger than this value. - #: Whether hash throws error or silently truncates secret - #: depends on .truncate_error and .truncate_verify_reject flags below. - #: NOTE: calls may treat as boolean, since value will never be 0. - #: .. versionadded:: 1.7 - #: .. TODO: passlib 1.8: deprecate/rename this attr to "max_secret_size"? - truncate_size = None - - # NOTE: these next two default to the optimistic "ideal", - # most hashes in passlib have to default to False - # for backward compat and/or expected behavior with existing hashes. - - #: If True, .hash() should throw a :exc:`~passlib.exc.PasswordSizeError` for - #: any secrets larger than .truncate_size. Many hashers default to False - #: for historical / compatibility purposes, indicating they will silently - #: truncate instead. All such hashers SHOULD support changing - #: the policy via ``.using(truncate_error=True)``. - #: .. versionadded:: 1.7 - #: .. TODO: passlib 1.8: deprecate/rename this attr to "truncate_hash_error"? - truncate_error = True - - #: If True, .verify() should reject secrets larger than max_password_size. - #: Many hashers default to False for historical / compatibility purposes, - #: indicating they will match on the truncated portion instead. - #: .. versionadded:: 1.7.1 - truncate_verify_reject = True - - #--------------------------------------------------------------- - # salt information -- if 'salt' in setting_kwds - #--------------------------------------------------------------- - ##min_salt_size - ##max_salt_size - ##default_salt_size - ##salt_chars - ##default_salt_chars - - #--------------------------------------------------------------- - # rounds information -- if 'rounds' in setting_kwds - #--------------------------------------------------------------- - ##min_rounds - ##max_rounds - ##default_rounds - ##rounds_cost - - #--------------------------------------------------------------- - # encoding info -- if 'encoding' in context_kwds - #--------------------------------------------------------------- - ##default_encoding - - #=================================================================== - # primary methods - #=================================================================== - @classmethod - @abstractmethod - def hash(cls, secret, # * - **setting_and_context_kwds): # pragma: no cover -- abstract method - r""" - Hash secret, returning result. - Should handle generating salt, etc, and should return string - containing identifier, salt & other configuration, as well as digest. - - :param \*\*settings_kwds: - - Pass in settings to customize configuration of resulting hash. - - .. deprecated:: 1.7 - - Starting with Passlib 1.7, callers should no longer pass settings keywords - (e.g. ``rounds`` or ``salt`` directly to :meth:`!hash`); should use - ``.using(**settings).hash(secret)`` construction instead. - - Support will be removed in Passlib 2.0. - - :param \*\*context_kwds: - - Specific algorithms may require context-specific information (such as the user login). - """ - # FIXME: need stub for classes that define .encrypt() instead ... - # this should call .encrypt(), and check for recursion back to here. - raise NotImplementedError("must be implemented by subclass") - - @deprecated_method(deprecated="1.7", removed="2.0", replacement=".hash()") - @classmethod - def encrypt(cls, *args, **kwds): - """ - Legacy alias for :meth:`hash`. - - .. deprecated:: 1.7 - This method was renamed to :meth:`!hash` in version 1.7. - This alias will be removed in version 2.0, and should only - be used for compatibility with Passlib 1.3 - 1.6. - """ - return cls.hash(*args, **kwds) - - # XXX: could provide default implementation which hands value to - # hash(), and then does constant-time comparision on the result - # (after making both are same string type) - @classmethod - @abstractmethod - def verify(cls, secret, hash, **context_kwds): # pragma: no cover -- abstract method - """verify secret against hash, returns True/False""" - raise NotImplementedError("must be implemented by subclass") - - #=================================================================== - # configuration - #=================================================================== - @classmethod - @abstractmethod - def using(cls, relaxed=False, **kwds): - """ - Return another hasher object (typically a subclass of the current one), - which integrates the configuration options specified by ``kwds``. - This should *always* return a new object, even if no configuration options are changed. - - .. todo:: - - document which options are accepted. - - :returns: - typically returns a subclass for most hasher implementations. - - .. todo:: - - add this method to main documentation. - """ - raise NotImplementedError("must be implemented by subclass") - - #=================================================================== - # migration - #=================================================================== - @classmethod - def needs_update(cls, hash, secret=None): - """ - check if hash's configuration is outside desired bounds, - or contains some other internal option which requires - updating the password hash. - - :param hash: - hash string to examine - - :param secret: - optional secret known to have verified against the provided hash. - (this is used by some hashes to detect legacy algorithm mistakes). - - :return: - whether secret needs re-hashing. - - .. versionadded:: 1.7 - """ - # by default, always report that we don't need update - return False - - #=================================================================== - # additional methods - #=================================================================== - @classmethod - @abstractmethod - def identify(cls, hash): # pragma: no cover -- abstract method - """check if hash belongs to this scheme, returns True/False""" - raise NotImplementedError("must be implemented by subclass") - - @deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genconfig(cls, **setting_kwds): # pragma: no cover -- abstract method - """ - compile settings into a configuration string for genhash() - - .. deprecated:: 1.7 - - As of 1.7, this method is deprecated, and slated for complete removal in Passlib 2.0. - - For all known real-world uses, hashing a constant string - should provide equivalent functionality. - - This deprecation may be reversed if a use-case presents itself in the mean time. - """ - # NOTE: this fallback runs full hash alg, w/ whatever cost param is passed along. - # implementations (esp ones w/ variable cost) will want to subclass this - # with a constant-time implementation that just renders a config string. - if cls.context_kwds: - raise NotImplementedError("must be implemented by subclass") - return cls.using(**setting_kwds).hash("") - - @deprecated_method(deprecated="1.7", removed="2.0") - @classmethod - def genhash(cls, secret, config, **context): - """ - generated hash for secret, using settings from config/hash string - - .. deprecated:: 1.7 - - As of 1.7, this method is deprecated, and slated for complete removal in Passlib 2.0. - - This deprecation may be reversed if a use-case presents itself in the mean time. - """ - # XXX: if hashes reliably offered a .parse() method, could make a fallback for this. - raise NotImplementedError("must be implemented by subclass") - - #=================================================================== - # undocumented methods / attributes - #=================================================================== - # the following entry points are used internally by passlib, - # and aren't documented as part of the exposed interface. - # they are subject to change between releases, - # but are documented here so there's a list of them *somewhere*. - - #--------------------------------------------------------------- - # extra metdata - #--------------------------------------------------------------- - - #: this attribute shouldn't be used by hashers themselves, - #: it's reserved for the CryptContext to track which hashers are deprecated. - #: Note the context will only set this on objects it owns (and generated by .using()), - #: and WONT set it on global objects. - #: [added in 1.7] - #: TODO: document this, or at least the use of testing for - #: 'CryptContext().handler().deprecated' - deprecated = False - - #: optionally present if hasher corresponds to format built into Django. - #: this attribute (if not None) should be the Django 'algorithm' name. - #: also indicates to passlib.ext.django that (when installed in django), - #: django's native hasher should be used in preference to this one. - ## django_name - - #--------------------------------------------------------------- - # checksum information - defined for many hashes - #--------------------------------------------------------------- - ## checksum_chars - ## checksum_size - - #--------------------------------------------------------------- - # experimental methods - #--------------------------------------------------------------- - - ##@classmethod - ##def normhash(cls, hash): - ## """helper to clean up non-canonic instances of hash. - ## currently only provided by bcrypt() to fix an historical passlib issue. - ## """ - - # experimental helper to parse hash into components. - ##@classmethod - ##def parsehash(cls, hash, checksum=True, sanitize=False): - ## """helper to parse hash into components, returns dict""" - - # experiment helper to estimate bitsize of different hashes, - # implement for GenericHandler, but may be currently be off for some hashes. - # want to expand this into a way to programmatically compare - # "strengths" of different hashes and hash algorithms. - # still needs to have some factor for estimate relative cost per round, - # ala in the style of the scrypt whitepaper. - ##@classmethod - ##def bitsize(cls, **kwds): - ## """returns dict mapping component -> bits contributed. - ## components currently include checksum, salt, rounds. - ## """ - - #=================================================================== - # eoc - #=================================================================== - -class DisabledHash(PasswordHash): - """ - extended disabled-hash methods; only need be present if .disabled = True - """ - - is_disabled = True - - @classmethod - def disable(cls, hash=None): - """ - return string representing a 'disabled' hash; - optionally including previously enabled hash - (this is up to the individual scheme). - """ - # default behavior: ignore original hash, return standalone marker - return cls.hash("") - - @classmethod - def enable(cls, hash): - """ - given a disabled-hash string, - extract previously-enabled hash if one is present, - otherwise raises ValueError - """ - # default behavior: no way to restore original hash - raise ValueError("cannot restore original hash") - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/pwd.py b/src/passlib/pwd.py deleted file mode 100644 index 52e1e64c..00000000 --- a/src/passlib/pwd.py +++ /dev/null @@ -1,804 +0,0 @@ -"""passlib.pwd -- password generation helpers""" -#============================================================================= -# imports -#============================================================================= -from __future__ import absolute_import, division, print_function, unicode_literals -# core -import codecs -from collections import defaultdict, MutableMapping -from math import ceil, log as logf -import logging; log = logging.getLogger(__name__) -import pkg_resources -import os -# site -# pkg -from passlib import exc -from passlib.utils.compat import PY2, irange, itervalues, int_types -from passlib.utils import rng, getrandstr, to_unicode -from passlib.utils.decor import memoized_property -# local -__all__ = [ - "genword", "default_charsets", - "genphrase", "default_wordsets", -] - -#============================================================================= -# constants -#============================================================================= - -# XXX: rename / publically document this map? -entropy_aliases = dict( - # barest protection from throttled online attack - unsafe=12, - - # some protection from unthrottled online attack - weak=24, - - # some protection from offline attacks - fair=36, - - # reasonable protection from offline attacks - strong=48, - - # very good protection from offline attacks - secure=60, -) - -#============================================================================= -# internal helpers -#============================================================================= - -def _superclasses(obj, cls): - """return remaining classes in object's MRO after cls""" - mro = type(obj).__mro__ - return mro[mro.index(cls)+1:] - - -def _self_info_rate(source): - """ - returns 'rate of self-information' -- - i.e. average (per-symbol) entropy of the sequence **source**, - where probability of a given symbol occurring is calculated based on - the number of occurrences within the sequence itself. - - if all elements of the source are unique, this should equal ``log(len(source), 2)``. - - :arg source: - iterable containing 0+ symbols - (e.g. list of strings or ints, string of characters, etc). - - :returns: - float bits of entropy - """ - try: - size = len(source) - except TypeError: - # if len() doesn't work, calculate size by summing counts later - size = None - counts = defaultdict(int) - for char in source: - counts[char] += 1 - if size is None: - values = counts.values() - size = sum(values) - else: - values = itervalues(counts) - if not size: - return 0 - # NOTE: the following performs ``- sum(value / size * logf(value / size, 2) for value in values)``, - # it just does so with as much pulled out of the sum() loop as possible... - return logf(size, 2) - sum(value * logf(value, 2) for value in values) / size - - -# def _total_self_info(source): -# """ -# return total self-entropy of a sequence -# (the average entropy per symbol * size of sequence) -# """ -# return _self_info_rate(source) * len(source) - - -def _open_asset_path(path, encoding=None): - """ - :param asset_path: - string containing absolute path to file, - or package-relative path using format - ``"python.module:relative/file/path"``. - - :returns: - filehandle opened in 'rb' mode - (unless encoding explicitly specified) - """ - if encoding: - return codecs.getreader(encoding)(_open_asset_path(path)) - if os.path.isabs(path): - return open(path, "rb") - package, sep, subpath = path.partition(":") - if not sep: - raise ValueError("asset path must be absolute file path " - "or use 'pkg.name:sub/path' format: %r" % (path,)) - return pkg_resources.resource_stream(package, subpath) - - -#: type aliases -_sequence_types = (list, tuple) -_set_types = (set, frozenset) - -#: set of elements that ensure_unique() has validated already. -_ensure_unique_cache = set() - - -def _ensure_unique(source, param="source"): - """ - helper for generators -- - Throws ValueError if source elements aren't unique. - Error message will display (abbreviated) repr of the duplicates in a string/list - """ - # check cache to speed things up for frozensets / tuples / strings - cache = _ensure_unique_cache - hashable = True - try: - if source in cache: - return True - except TypeError: - hashable = False - - # check if it has dup elements - if isinstance(source, _set_types) or len(set(source)) == len(source): - if hashable: - try: - cache.add(source) - except TypeError: - # XXX: under pypy, "list() in set()" above doesn't throw TypeError, - # but trying to add unhashable it to a set *does*. - pass - return True - - # build list of duplicate values - seen = set() - dups = set() - for elem in source: - (dups if elem in seen else seen).add(elem) - dups = sorted(dups) - trunc = 8 - if len(dups) > trunc: - trunc = 5 - dup_repr = ", ".join(repr(str(word)) for word in dups[:trunc]) - if len(dups) > trunc: - dup_repr += ", ... plus %d others" % (len(dups) - trunc) - - # throw error - raise ValueError("`%s` cannot contain duplicate elements: %s" % - (param, dup_repr)) - -#============================================================================= -# base generator class -#============================================================================= -class SequenceGenerator(object): - """ - Base class used by word & phrase generators. - - These objects take a series of options, corresponding - to those of the :func:`generate` function. - They act as callables which can be used to generate a password - or a list of 1+ passwords. They also expose some read-only - informational attributes. - - Parameters - ---------- - :param entropy: - Optionally specify the amount of entropy the resulting passwords - should contain (as measured with respect to the generator itself). - This will be used to auto-calculate the required password size. - - :param length: - Optionally specify the length of password to generate, - measured as count of whatever symbols the subclass uses (characters or words). - Note if ``entropy`` requires a larger minimum length, - that will be used instead. - - :param rng: - Optionally provide a custom RNG source to use. - Should be an instance of :class:`random.Random`, - defaults to :class:`random.SystemRandom`. - - Attributes - ---------- - .. autoattribute:: length - .. autoattribute:: symbol_count - .. autoattribute:: entropy_per_symbol - .. autoattribute:: entropy - - Subclassing - ----------- - Subclasses must implement the ``.__next__()`` method, - and set ``.symbol_count`` before calling base ``__init__`` method. - """ - #============================================================================= - # instance attrs - #============================================================================= - - #: requested size of final password - length = None - - #: requested entropy of final password - requested_entropy = "strong" - - #: random number source to use - rng = rng - - #: number of potential symbols (must be filled in by subclass) - symbol_count = None - - #============================================================================= - # init - #============================================================================= - def __init__(self, entropy=None, length=None, rng=None, **kwds): - - # make sure subclass set things up correctly - assert self.symbol_count is not None, "subclass must set .symbol_count" - - # init length & requested entropy - if entropy is not None or length is None: - if entropy is None: - entropy = self.requested_entropy - entropy = entropy_aliases.get(entropy, entropy) - if entropy <= 0: - raise ValueError("`entropy` must be positive number") - min_length = int(ceil(entropy / self.entropy_per_symbol)) - if length is None or length < min_length: - length = min_length - - self.requested_entropy = entropy - - if length < 1: - raise ValueError("`length` must be positive integer") - self.length = length - - # init other common options - if rng is not None: - self.rng = rng - - # hand off to parent - if kwds and _superclasses(self, SequenceGenerator) == (object,): - raise TypeError("Unexpected keyword(s): %s" % ", ".join(kwds.keys())) - super(SequenceGenerator, self).__init__(**kwds) - - #============================================================================= - # informational helpers - #============================================================================= - - @memoized_property - def entropy_per_symbol(self): - """ - Average entropy per symbol (assuming all symbols have equal probability) - """ - return logf(self.symbol_count, 2) - - @memoized_property - def entropy(self): - """ - Effective entropy of generated passwords. - - This value will always be a multiple of :attr:`entropy_per_symbol`. - If entropy is specified in constructor, :attr:`length` will be chosen so - so that this value is the smallest multiple >= :attr:`requested_entropy`. - """ - return self.length * self.entropy_per_symbol - - #============================================================================= - # generation - #============================================================================= - def __next__(self): - """main generation function, should create one password/phrase""" - raise NotImplementedError("implement in subclass") - - def __call__(self, returns=None): - """ - frontend used by genword() / genphrase() to create passwords - """ - if returns is None: - return next(self) - elif isinstance(returns, int_types): - return [next(self) for _ in irange(returns)] - elif returns is iter: - return self - else: - raise exc.ExpectedTypeError(returns, ", int, or ", "returns") - - def __iter__(self): - return self - - if PY2: - def next(self): - return self.__next__() - - #============================================================================= - # eoc - #============================================================================= - -#============================================================================= -# default charsets -#============================================================================= - -#: global dict of predefined characters sets -default_charsets = dict( - # ascii letters, digits, and some punctuation - ascii_72='0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^&*?/', - - # ascii letters and digits - ascii_62='0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ', - - # ascii_50, without visually similar '1IiLl', '0Oo', '5S', '8B' - ascii_50='234679abcdefghjkmnpqrstuvwxyzACDEFGHJKMNPQRTUVWXYZ', - - # lower case hexadecimal - hex='0123456789abcdef', -) - -#============================================================================= -# password generator -#============================================================================= - -class WordGenerator(SequenceGenerator): - """ - Class which generates passwords by randomly choosing from a string of unique characters. - - Parameters - ---------- - :param chars: - custom character string to draw from. - - :param charset: - predefined charset to draw from. - - :param \*\*kwds: - all other keywords passed to the :class:`SequenceGenerator` parent class. - - Attributes - ---------- - .. autoattribute:: chars - .. autoattribute:: charset - .. autoattribute:: default_charsets - """ - #============================================================================= - # instance attrs - #============================================================================= - - #: Predefined character set in use (set to None for instances using custom 'chars') - charset = "ascii_62" - - #: string of chars to draw from -- usually filled in from charset - chars = None - - #============================================================================= - # init - #============================================================================= - def __init__(self, chars=None, charset=None, **kwds): - - # init chars and charset - if chars: - if charset: - raise TypeError("`chars` and `charset` are mutually exclusive") - else: - if not charset: - charset = self.charset - assert charset - chars = default_charsets[charset] - self.charset = charset - chars = to_unicode(chars, param="chars") - _ensure_unique(chars, param="chars") - self.chars = chars - - # hand off to parent - super(WordGenerator, self).__init__(**kwds) - # log.debug("WordGenerator(): entropy/char=%r", self.entropy_per_symbol) - - #============================================================================= - # informational helpers - #============================================================================= - - @memoized_property - def symbol_count(self): - return len(self.chars) - - #============================================================================= - # generation - #============================================================================= - - def __next__(self): - # XXX: could do things like optionally ensure certain character groups - # (e.g. letters & punctuation) are included - return getrandstr(self.rng, self.chars, self.length) - - #============================================================================= - # eoc - #============================================================================= - - -def genword(entropy=None, length=None, returns=None, **kwds): - """Generate one or more random passwords. - - This function uses :mod:`random.SystemRandom` to generate - one or more passwords using various character sets. - The complexity of the password can be specified - by size, or by the desired amount of entropy. - - Usage Example:: - - >>> # generate a random alphanumeric string with 48 bits of entropy (the default) - >>> from passlib import pwd - >>> pwd.genword() - 'DnBHvDjMK6' - - >>> # generate a random hexadecimal string with 52 bits of entropy - >>> pwd.genword(entropy=52, charset="hex") - '310f1a7ac793f' - - :param entropy: - Strength of resulting password, measured in 'guessing entropy' bits. - An appropriate **length** value will be calculated - based on the requested entropy amount, and the size of the character set. - - This can be a positive integer, or one of the following preset - strings: ``"weak"`` (24), ``"fair"`` (36), - ``"strong"`` (48), and ``"secure"`` (56). - - If neither this or **length** is specified, **entropy** will default - to ``"strong"`` (48). - - :param length: - Size of resulting password, measured in characters. - If omitted, the size is auto-calculated based on the **entropy** parameter. - - If both **entropy** and **length** are specified, - the stronger value will be used. - - :param returns: - Controls what this function returns: - - * If ``None`` (the default), this function will generate a single password. - * If an integer, this function will return a list containing that many passwords. - * If the ``iter`` constant, will return an iterator that yields passwords. - - :param chars: - - Optionally specify custom string of characters to use when randomly - generating a password. This option cannot be combined with **charset**. - - :param charset: - - The predefined character set to draw from (if not specified by **chars**). - There are currently four presets available: - - * ``"ascii_62"`` (the default) -- all digits and ascii upper & lowercase letters. - Provides ~5.95 entropy per character. - - * ``"ascii_50"`` -- subset which excludes visually similar characters - (``1IiLl0Oo5S8B``). Provides ~5.64 entropy per character. - - * ``"ascii_72"`` -- all digits and ascii upper & lowercase letters, - as well as some punctuation. Provides ~6.17 entropy per character. - - * ``"hex"`` -- Lower case hexadecimal. Providers 4 bits of entropy per character. - - :returns: - :class:`!unicode` string containing randomly generated password; - or list of 1+ passwords if :samp:`returns={int}` is specified. - """ - gen = WordGenerator(length=length, entropy=entropy, **kwds) - return gen(returns) - -#============================================================================= -# default wordsets -#============================================================================= - -def _load_wordset(asset_path): - """ - load wordset from compressed datafile within package data. - file should be utf-8 encoded - - :param asset_path: - string containing absolute path to wordset file, - or "python.module:relative/file/path". - - :returns: - tuple of words, as loaded from specified words file. - """ - # open resource file, convert to tuple of words (strip blank lines & ws) - with _open_asset_path(asset_path, "utf-8") as fh: - gen = (word.strip() for word in fh) - words = tuple(word for word in gen if word) - - # NOTE: works but not used - # # detect if file uses " " format, and strip numeric prefix - # def extract(row): - # idx, word = row.replace("\t", " ").split(" ", 1) - # if not idx.isdigit(): - # raise ValueError("row is not dice index + word") - # return word - # try: - # extract(words[-1]) - # except ValueError: - # pass - # else: - # words = tuple(extract(word) for word in words) - - log.debug("loaded %d-element wordset from %r", len(words), asset_path) - return words - - -class WordsetDict(MutableMapping): - """ - Special mapping used to store dictionary of wordsets. - Different from a regular dict in that some wordsets - may be lazy-loaded from an asset path. - """ - - #: dict of key -> asset path - paths = None - - #: dict of key -> value - _loaded = None - - def __init__(self, *args, **kwds): - self.paths = {} - self._loaded = {} - super(WordsetDict, self).__init__(*args, **kwds) - - def __getitem__(self, key): - try: - return self._loaded[key] - except KeyError: - pass - path = self.paths[key] - value = self._loaded[key] = _load_wordset(path) - return value - - def set_path(self, key, path): - """ - set asset path to lazy-load wordset from. - """ - self.paths[key] = path - - def __setitem__(self, key, value): - self._loaded[key] = value - - def __delitem__(self, key): - if key in self: - del self._loaded[key] - self.paths.pop(key, None) - else: - del self.paths[key] - - @property - def _keyset(self): - keys = set(self._loaded) - keys.update(self.paths) - return keys - - def __iter__(self): - return iter(self._keyset) - - def __len__(self): - return len(self._keyset) - - # NOTE: speeds things up, and prevents contains from lazy-loading - def __contains__(self, key): - return key in self._loaded or key in self.paths - - -#: dict of predefined word sets. -#: key is name of wordset, value should be sequence of words. -default_wordsets = WordsetDict() - -# register the wordsets built into passlib -for name in "eff_long eff_short eff_prefixed bip39".split(): - default_wordsets.set_path(name, "passlib:_data/wordsets/%s.txt" % name) - -#============================================================================= -# passphrase generator -#============================================================================= -class PhraseGenerator(SequenceGenerator): - """class which generates passphrases by randomly choosing - from a list of unique words. - - :param wordset: - wordset to draw from. - :param preset: - name of preset wordlist to use instead of ``wordset``. - :param spaces: - whether to insert spaces between words in output (defaults to ``True``). - :param \*\*kwds: - all other keywords passed to the :class:`SequenceGenerator` parent class. - - .. autoattribute:: wordset - """ - #============================================================================= - # instance attrs - #============================================================================= - - #: predefined wordset to use - wordset = "eff_long" - - #: list of words to draw from - words = None - - #: separator to use when joining words - sep = " " - - #============================================================================= - # init - #============================================================================= - def __init__(self, wordset=None, words=None, sep=None, **kwds): - - # load wordset - if words is not None: - if wordset is not None: - raise TypeError("`words` and `wordset` are mutually exclusive") - else: - if wordset is None: - wordset = self.wordset - assert wordset - words = default_wordsets[wordset] - self.wordset = wordset - - # init words - if not isinstance(words, _sequence_types): - words = tuple(words) - _ensure_unique(words, param="words") - self.words = words - - # init separator - if sep is None: - sep = self.sep - sep = to_unicode(sep, param="sep") - self.sep = sep - - # hand off to parent - super(PhraseGenerator, self).__init__(**kwds) - ##log.debug("PhraseGenerator(): entropy/word=%r entropy/char=%r min_chars=%r", - ## self.entropy_per_symbol, self.entropy_per_char, self.min_chars) - - #============================================================================= - # informational helpers - #============================================================================= - - @memoized_property - def symbol_count(self): - return len(self.words) - - #============================================================================= - # generation - #============================================================================= - - def __next__(self): - words = (self.rng.choice(self.words) for _ in irange(self.length)) - return self.sep.join(words) - - #============================================================================= - # eoc - #============================================================================= - - -def genphrase(entropy=None, length=None, returns=None, **kwds): - """Generate one or more random password / passphrases. - - This function uses :mod:`random.SystemRandom` to generate - one or more passwords; it can be configured to generate - alphanumeric passwords, or full english phrases. - The complexity of the password can be specified - by size, or by the desired amount of entropy. - - Usage Example:: - - >>> # generate random phrase with 48 bits of entropy - >>> from passlib import pwd - >>> pwd.genphrase() - 'gangly robbing salt shove' - - >>> # generate a random phrase with 52 bits of entropy - >>> # using a particular wordset - >>> pwd.genword(entropy=52, wordset="bip39") - 'wheat dilemma reward rescue diary' - - :param entropy: - Strength of resulting password, measured in 'guessing entropy' bits. - An appropriate **length** value will be calculated - based on the requested entropy amount, and the size of the word set. - - This can be a positive integer, or one of the following preset - strings: ``"weak"`` (24), ``"fair"`` (36), - ``"strong"`` (48), and ``"secure"`` (56). - - If neither this or **length** is specified, **entropy** will default - to ``"strong"`` (48). - - :param length: - Length of resulting password, measured in words. - If omitted, the size is auto-calculated based on the **entropy** parameter. - - If both **entropy** and **length** are specified, - the stronger value will be used. - - :param returns: - Controls what this function returns: - - * If ``None`` (the default), this function will generate a single password. - * If an integer, this function will return a list containing that many passwords. - * If the ``iter`` builtin, will return an iterator that yields passwords. - - :param words: - - Optionally specifies a list/set of words to use when randomly generating a passphrase. - This option cannot be combined with **wordset**. - - :param wordset: - - The predefined word set to draw from (if not specified by **words**). - There are currently four presets available: - - ``"eff_long"`` (the default) - - Wordset containing 7776 english words of ~7 letters. - Constructed by the EFF, it offers ~12.9 bits of entropy per word. - - This wordset (and the other ``"eff_"`` wordsets) - were `created by the EFF `_ - to aid in generating passwords. See their announcement page - for more details about the design & properties of these wordsets. - - ``"eff_short"`` - - Wordset containing 1296 english words of ~4.5 letters. - Constructed by the EFF, it offers ~10.3 bits of entropy per word. - - ``"eff_prefixed"`` - - Wordset containing 1296 english words of ~8 letters, - selected so that they each have a unique 3-character prefix. - Constructed by the EFF, it offers ~10.3 bits of entropy per word. - - ``"bip39"`` - - Wordset of 2048 english words of ~5 letters, - selected so that they each have a unique 4-character prefix. - Published as part of Bitcoin's `BIP 39 `_, - this wordset has exactly 11 bits of entropy per word. - - This list offers words that are typically shorter than ``"eff_long"`` - (at the cost of slightly less entropy); and much shorter than - ``"eff_prefixed"`` (at the cost of a longer unique prefix). - - :param sep: - Optional separator to use when joining words. - Defaults to ``" "`` (a space), but can be an empty string, a hyphen, etc. - - :returns: - :class:`!unicode` string containing randomly generated passphrase; - or list of 1+ passphrases if :samp:`returns={int}` is specified. - """ - gen = PhraseGenerator(entropy=entropy, length=length, **kwds) - return gen(returns) - -#============================================================================= -# strength measurement -# -# NOTE: -# for a little while, had rough draft of password strength measurement alg here. -# but not sure if there's value in yet another measurement algorithm, -# that's not just duplicating the effort of libraries like zxcbn. -# may revive it later, but for now, leaving some refs to others out there: -# * NIST 800-63 has simple alg -# * zxcvbn (https://tech.dropbox.com/2012/04/zxcvbn-realistic-password-strength-estimation/) -# might also be good, and has approach similar to composite approach i was already thinking about, -# but much more well thought out. -# * passfault (https://github.com/c-a-m/passfault) looks thorough, -# but may have licensing issues, plus porting to python looks like very big job :( -# * give a look at running things through zlib - might be able to cheaply -# catch extra redundancies. -#============================================================================= - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/registry.py b/src/passlib/registry.py deleted file mode 100644 index d4ef6d06..00000000 --- a/src/passlib/registry.py +++ /dev/null @@ -1,542 +0,0 @@ -"""passlib.registry - registry for password hash handlers""" -#============================================================================= -# imports -#============================================================================= -# core -import re -import logging; log = logging.getLogger(__name__) -from warnings import warn -# pkg -from passlib import exc -from passlib.exc import ExpectedTypeError, PasslibWarning -from passlib.ifc import PasswordHash -from passlib.utils import ( - is_crypt_handler, has_crypt as os_crypt_present, - unix_crypt_schemes as os_crypt_schemes, -) -from passlib.utils.compat import unicode_or_str -from passlib.utils.decor import memoize_single_value -# local -__all__ = [ - "register_crypt_handler_path", - "register_crypt_handler", - "get_crypt_handler", - "list_crypt_handlers", -] - -#============================================================================= -# proxy object used in place of 'passlib.hash' module -#============================================================================= -class _PasslibRegistryProxy(object): - """proxy module passlib.hash - - this module is in fact an object which lazy-loads - the requested password hash algorithm from wherever it has been stored. - it acts as a thin wrapper around :func:`passlib.registry.get_crypt_handler`. - """ - __name__ = "passlib.hash" - __package__ = None - - def __getattr__(self, attr): - if attr.startswith("_"): - raise AttributeError("missing attribute: %r" % (attr,)) - handler = get_crypt_handler(attr, None) - if handler: - return handler - else: - raise AttributeError("unknown password hash: %r" % (attr,)) - - def __setattr__(self, attr, value): - if attr.startswith("_"): - # writing to private attributes should behave normally. - # (required so GAE can write to the __loader__ attribute). - object.__setattr__(self, attr, value) - else: - # writing to public attributes should be treated - # as attempting to register a handler. - register_crypt_handler(value, _attr=attr) - - def __repr__(self): - return "" - - def __dir__(self): - # this adds in lazy-loaded handler names, - # otherwise this is the standard dir() implementation. - attrs = set(dir(self.__class__)) - attrs.update(self.__dict__) - attrs.update(_locations) - return sorted(attrs) - -# create single instance - available publically as 'passlib.hash' -_proxy = _PasslibRegistryProxy() - -#============================================================================= -# internal registry state -#============================================================================= - -# singleton uses to detect omitted keywords -_UNSET = object() - -# dict mapping name -> loaded handlers (just uses proxy object's internal dict) -_handlers = _proxy.__dict__ - -# dict mapping names -> import path for lazy loading. -# * import path should be "module.path" or "module.path:attr" -# * if attr omitted, "name" used as default. -_locations = dict( - # NOTE: this is a hardcoded list of the handlers built into passlib, - # applications should call register_crypt_handler_path() - apr_md5_crypt = "passlib.handlers.md5_crypt", - argon2 = "passlib.handlers.argon2", - atlassian_pbkdf2_sha1 = "passlib.handlers.pbkdf2", - bcrypt = "passlib.handlers.bcrypt", - bcrypt_sha256 = "passlib.handlers.bcrypt", - bigcrypt = "passlib.handlers.des_crypt", - bsd_nthash = "passlib.handlers.windows", - bsdi_crypt = "passlib.handlers.des_crypt", - cisco_pix = "passlib.handlers.cisco", - cisco_asa = "passlib.handlers.cisco", - cisco_type7 = "passlib.handlers.cisco", - cta_pbkdf2_sha1 = "passlib.handlers.pbkdf2", - crypt16 = "passlib.handlers.des_crypt", - des_crypt = "passlib.handlers.des_crypt", - django_argon2 = "passlib.handlers.django", - django_bcrypt = "passlib.handlers.django", - django_bcrypt_sha256 = "passlib.handlers.django", - django_pbkdf2_sha256 = "passlib.handlers.django", - django_pbkdf2_sha1 = "passlib.handlers.django", - django_salted_sha1 = "passlib.handlers.django", - django_salted_md5 = "passlib.handlers.django", - django_des_crypt = "passlib.handlers.django", - django_disabled = "passlib.handlers.django", - dlitz_pbkdf2_sha1 = "passlib.handlers.pbkdf2", - fshp = "passlib.handlers.fshp", - grub_pbkdf2_sha512 = "passlib.handlers.pbkdf2", - hex_md4 = "passlib.handlers.digests", - hex_md5 = "passlib.handlers.digests", - hex_sha1 = "passlib.handlers.digests", - hex_sha256 = "passlib.handlers.digests", - hex_sha512 = "passlib.handlers.digests", - htdigest = "passlib.handlers.digests", - ldap_plaintext = "passlib.handlers.ldap_digests", - ldap_md5 = "passlib.handlers.ldap_digests", - ldap_sha1 = "passlib.handlers.ldap_digests", - ldap_hex_md5 = "passlib.handlers.roundup", - ldap_hex_sha1 = "passlib.handlers.roundup", - ldap_salted_md5 = "passlib.handlers.ldap_digests", - ldap_salted_sha1 = "passlib.handlers.ldap_digests", - ldap_des_crypt = "passlib.handlers.ldap_digests", - ldap_bsdi_crypt = "passlib.handlers.ldap_digests", - ldap_md5_crypt = "passlib.handlers.ldap_digests", - ldap_bcrypt = "passlib.handlers.ldap_digests", - ldap_sha1_crypt = "passlib.handlers.ldap_digests", - ldap_sha256_crypt = "passlib.handlers.ldap_digests", - ldap_sha512_crypt = "passlib.handlers.ldap_digests", - ldap_pbkdf2_sha1 = "passlib.handlers.pbkdf2", - ldap_pbkdf2_sha256 = "passlib.handlers.pbkdf2", - ldap_pbkdf2_sha512 = "passlib.handlers.pbkdf2", - lmhash = "passlib.handlers.windows", - md5_crypt = "passlib.handlers.md5_crypt", - msdcc = "passlib.handlers.windows", - msdcc2 = "passlib.handlers.windows", - mssql2000 = "passlib.handlers.mssql", - mssql2005 = "passlib.handlers.mssql", - mysql323 = "passlib.handlers.mysql", - mysql41 = "passlib.handlers.mysql", - nthash = "passlib.handlers.windows", - oracle10 = "passlib.handlers.oracle", - oracle11 = "passlib.handlers.oracle", - pbkdf2_sha1 = "passlib.handlers.pbkdf2", - pbkdf2_sha256 = "passlib.handlers.pbkdf2", - pbkdf2_sha512 = "passlib.handlers.pbkdf2", - phpass = "passlib.handlers.phpass", - plaintext = "passlib.handlers.misc", - postgres_md5 = "passlib.handlers.postgres", - roundup_plaintext = "passlib.handlers.roundup", - scram = "passlib.handlers.scram", - scrypt = "passlib.handlers.scrypt", - sha1_crypt = "passlib.handlers.sha1_crypt", - sha256_crypt = "passlib.handlers.sha2_crypt", - sha512_crypt = "passlib.handlers.sha2_crypt", - sun_md5_crypt = "passlib.handlers.sun_md5_crypt", - unix_disabled = "passlib.handlers.misc", - unix_fallback = "passlib.handlers.misc", -) - -# master regexp for detecting valid handler names -_name_re = re.compile("^[a-z][a-z0-9_]+[a-z0-9]$") - -# names which aren't allowed for various reasons -# (mainly keyword conflicts in CryptContext) -_forbidden_names = frozenset(["onload", "policy", "context", "all", - "default", "none", "auto"]) - -#============================================================================= -# registry frontend functions -#============================================================================= -def _validate_handler_name(name): - """helper to validate handler name - - :raises ValueError: - * if empty name - * if name not lower case - * if name contains double underscores - * if name is reserved (e.g. ``context``, ``all``). - """ - if not name: - raise ValueError("handler name cannot be empty: %r" % (name,)) - if name.lower() != name: - raise ValueError("name must be lower-case: %r" % (name,)) - if not _name_re.match(name): - raise ValueError("invalid name (must be 3+ characters, " - " begin with a-z, and contain only underscore, a-z, " - "0-9): %r" % (name,)) - if '__' in name: - raise ValueError("name may not contain double-underscores: %r" % - (name,)) - if name in _forbidden_names: - raise ValueError("that name is not allowed: %r" % (name,)) - return True - -def register_crypt_handler_path(name, path): - """register location to lazy-load handler when requested. - - custom hashes may be registered via :func:`register_crypt_handler`, - or they may be registered by this function, - which will delay actually importing and loading the handler - until a call to :func:`get_crypt_handler` is made for the specified name. - - :arg name: name of handler - :arg path: module import path - - the specified module path should contain a password hash handler - called :samp:`{name}`, or the path may contain a colon, - specifying the module and module attribute to use. - for example, the following would cause ``get_handler("myhash")`` to look - for a class named ``myhash`` within the ``myapp.helpers`` module:: - - >>> from passlib.registry import registry_crypt_handler_path - >>> registry_crypt_handler_path("myhash", "myapp.helpers") - - ...while this form would cause ``get_handler("myhash")`` to look - for a class name ``MyHash`` within the ``myapp.helpers`` module:: - - >>> from passlib.registry import registry_crypt_handler_path - >>> registry_crypt_handler_path("myhash", "myapp.helpers:MyHash") - """ - # validate name - _validate_handler_name(name) - - # validate path - if path.startswith("."): - raise ValueError("path cannot start with '.'") - if ':' in path: - if path.count(':') > 1: - raise ValueError("path cannot have more than one ':'") - if path.find('.', path.index(':')) > -1: - raise ValueError("path cannot have '.' to right of ':'") - - # store location - _locations[name] = path - log.debug("registered path to %r handler: %r", name, path) - -def register_crypt_handler(handler, force=False, _attr=None): - """register password hash handler. - - this method immediately registers a handler with the internal passlib registry, - so that it will be returned by :func:`get_crypt_handler` when requested. - - :arg handler: the password hash handler to register - :param force: force override of existing handler (defaults to False) - :param _attr: - [internal kwd] if specified, ensures ``handler.name`` - matches this value, or raises :exc:`ValueError`. - - :raises TypeError: - if the specified object does not appear to be a valid handler. - - :raises ValueError: - if the specified object's name (or other required attributes) - contain invalid values. - - :raises KeyError: - if a (different) handler was already registered with - the same name, and ``force=True`` was not specified. - """ - # validate handler - if not is_crypt_handler(handler): - raise ExpectedTypeError(handler, "password hash handler", "handler") - if not handler: - raise AssertionError("``bool(handler)`` must be True") - - # validate name - name = handler.name - _validate_handler_name(name) - if _attr and _attr != name: - raise ValueError("handlers must be stored only under their own name (%r != %r)" % - (_attr, name)) - - # check for existing handler - other = _handlers.get(name) - if other: - if other is handler: - log.debug("same %r handler already registered: %r", name, handler) - return - elif force: - log.warning("overriding previously registered %r handler: %r", - name, other) - else: - raise KeyError("another %r handler has already been registered: %r" % - (name, other)) - - # register handler - _handlers[name] = handler - log.debug("registered %r handler: %r", name, handler) - -def get_crypt_handler(name, default=_UNSET): - """return handler for specified password hash scheme. - - this method looks up a handler for the specified scheme. - if the handler is not already loaded, - it checks if the location is known, and loads it first. - - :arg name: name of handler to return - :param default: optional default value to return if no handler with specified name is found. - - :raises KeyError: if no handler matching that name is found, and no default specified, a KeyError will be raised. - - :returns: handler attached to name, or default value (if specified). - """ - # catch invalid names before we check _handlers, - # since it's a module dict, and exposes things like __package__, etc. - if name.startswith("_"): - if default is _UNSET: - raise KeyError("invalid handler name: %r" % (name,)) - else: - return default - - # check if handler is already loaded - try: - return _handlers[name] - except KeyError: - pass - - # normalize name (and if changed, check dict again) - assert isinstance(name, unicode_or_str), "name must be string instance" - alt = name.replace("-","_").lower() - if alt != name: - warn("handler names should be lower-case, and use underscores instead " - "of hyphens: %r => %r" % (name, alt), PasslibWarning, - stacklevel=2) - name = alt - - # try to load using new name - try: - return _handlers[name] - except KeyError: - pass - - # check if lazy load mapping has been specified for this driver - path = _locations.get(name) - if path: - if ':' in path: - modname, modattr = path.split(":") - else: - modname, modattr = path, name - ##log.debug("loading %r handler from path: '%s:%s'", name, modname, modattr) - - # try to load the module - any import errors indicate runtime config, usually - # either missing package, or bad path provided to register_crypt_handler_path() - mod = __import__(modname, fromlist=[modattr], level=0) - - # first check if importing module triggered register_crypt_handler(), - # (this is discouraged due to its magical implicitness) - handler = _handlers.get(name) - if handler: - # XXX: issue deprecation warning here? - assert is_crypt_handler(handler), "unexpected object: name=%r object=%r" % (name, handler) - return handler - - # then get real handler & register it - handler = getattr(mod, modattr) - register_crypt_handler(handler, _attr=name) - return handler - - # fail! - if default is _UNSET: - raise KeyError("no crypt handler found for algorithm: %r" % (name,)) - else: - return default - -def list_crypt_handlers(loaded_only=False): - """return sorted list of all known crypt handler names. - - :param loaded_only: if ``True``, only returns names of handlers which have actually been loaded. - - :returns: list of names of all known handlers - """ - names = set(_handlers) - if not loaded_only: - names.update(_locations) - # strip private attrs out of namespace and sort. - # TODO: make _handlers a separate list, so we don't have module namespace mixed in. - return sorted(name for name in names if not name.startswith("_")) - -# NOTE: these two functions mainly exist just for the unittests... - -def _has_crypt_handler(name, loaded_only=False): - """check if handler name is known. - - this is only useful for two cases: - - * quickly checking if handler has already been loaded - * checking if handler exists, without actually loading it - - :arg name: name of handler - :param loaded_only: if ``True``, returns False if handler exists but hasn't been loaded - """ - return (name in _handlers) or (not loaded_only and name in _locations) - -def _unload_handler_name(name, locations=True): - """unloads a handler from the registry. - - .. warning:: - - this is an internal function, - used only by the unittests. - - if loaded handler is found with specified name, it's removed. - if path to lazy load handler is found, it's removed. - - missing names are a noop. - - :arg name: name of handler to unload - :param locations: if False, won't purge registered handler locations (default True) - """ - if name in _handlers: - del _handlers[name] - if locations and name in _locations: - del _locations[name] - -#============================================================================= -# inspection helpers -#============================================================================= - -#------------------------------------------------------------------ -# general -#------------------------------------------------------------------ - -# TODO: needs UTs -def _resolve(hasher, param="value"): - """ - internal helper to resolve argument to hasher object - """ - if is_crypt_handler(hasher): - return hasher - elif isinstance(hasher, unicode_or_str): - return get_crypt_handler(hasher) - else: - raise exc.ExpectedTypeError(hasher, unicode_or_str, param) - - -#: backend aliases -ANY = "any" -BUILTIN = "builtin" -OS_CRYPT = "os_crypt" - -# TODO: needs UTs -def has_backend(hasher, backend=ANY, safe=False): - """ - Test if specified backend is available for hasher. - - :param hasher: - Hasher name or object. - - :param backend: - Name of backend, or ``"any"`` if any backend will do. - For hashers without multiple backends, will pretend - they have a single backend named ``"builtin"``. - - :param safe: - By default, throws error if backend is unknown. - If ``safe=True``, will just return false value. - - :raises ValueError: - * if hasher name is unknown. - * if backend is unknown to hasher, and safe=False. - - :return: - True if backend available, False if not available, - and None if unknown + safe=True. - """ - hasher = _resolve(hasher) - - if backend == ANY: - if not hasattr(hasher, "get_backend"): - # single backend, assume it's loaded - return True - - # multiple backends, check at least one is loadable - try: - hasher.get_backend() - return True - except exc.MissingBackendError: - return False - - # test for specific backend - if hasattr(hasher, "has_backend"): - # multiple backends - if safe and backend not in hasher.backends: - return None - return hasher.has_backend(backend) - - # single builtin backend - if backend == BUILTIN: - return True - elif safe: - return None - else: - raise exc.UnknownBackendError(hasher, backend) - -#------------------------------------------------------------------ -# os crypt -#------------------------------------------------------------------ - -# TODO: move unix_crypt_schemes list to here. -# os_crypt_schemes -- alias for unix_crypt_schemes above - - -# TODO: needs UTs -@memoize_single_value -def get_supported_os_crypt_schemes(): - """ - return tuple of schemes which :func:`crypt.crypt` natively supports. - """ - if not os_crypt_present: - return () - cache = tuple(name for name in os_crypt_schemes - if get_crypt_handler(name).has_backend(OS_CRYPT)) - if not cache: # pragma: no cover -- sanity check - # no idea what OS this could happen on... - warn("crypt.crypt() function is present, but doesn't support any " - "formats known to passlib!", exc.PasslibRuntimeWarning) - return cache - - -# TODO: needs UTs -def has_os_crypt_support(hasher): - """ - check if hash is supported by native :func:`crypt.crypt` function. - if :func:`crypt.crypt` is not present, will always return False. - - :param hasher: - name or hasher object. - - :returns bool: - True if hash format is supported by OS, else False. - """ - return os_crypt_present and has_backend(hasher, OS_CRYPT, safe=True) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/__init__.py b/src/passlib/tests/__init__.py deleted file mode 100644 index 389da76e..00000000 --- a/src/passlib/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""passlib tests""" diff --git a/src/passlib/tests/__main__.py b/src/passlib/tests/__main__.py deleted file mode 100644 index 24245768..00000000 --- a/src/passlib/tests/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -import os -from nose import run -run( - defaultTest=os.path.dirname(__file__), -) - diff --git a/src/passlib/tests/_test_bad_register.py b/src/passlib/tests/_test_bad_register.py deleted file mode 100644 index f0683fcc..00000000 --- a/src/passlib/tests/_test_bad_register.py +++ /dev/null @@ -1,15 +0,0 @@ -"""helper for method in test_registry.py""" -from passlib.registry import register_crypt_handler -import passlib.utils.handlers as uh - -class dummy_bad(uh.StaticHandler): - name = "dummy_bad" - -class alt_dummy_bad(uh.StaticHandler): - name = "dummy_bad" - -# NOTE: if passlib.tests is being run from symlink (e.g. via gaeunit), -# this module may be imported a second time as test._test_bad_registry. -# we don't want it to do anything in that case. -if __name__.startswith("passlib.tests"): - register_crypt_handler(alt_dummy_bad) diff --git a/src/passlib/tests/backports.py b/src/passlib/tests/backports.py deleted file mode 100644 index c93b599e..00000000 --- a/src/passlib/tests/backports.py +++ /dev/null @@ -1,65 +0,0 @@ -"""backports of needed unittest2 features""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -import re -import sys -##from warnings import warn -# site -# pkg -from passlib.utils.compat import PY26 -# local -__all__ = [ - "TestCase", - "skip", "skipIf", "skipUnless" -] - -#============================================================================= -# import latest unittest module available -#============================================================================= -try: - import unittest2 as unittest -except ImportError: - if PY26: - raise ImportError("Passlib's tests require 'unittest2' under Python 2.6 (as of Passlib 1.7)") - # python 2.7 and python 3.2 both have unittest2 features (at least, the ones we use) - import unittest - -#============================================================================= -# unittest aliases -#============================================================================= -skip = unittest.skip -skipIf = unittest.skipIf -skipUnless = unittest.skipUnless -SkipTest = unittest.SkipTest - -#============================================================================= -# custom test harness -#============================================================================= -class TestCase(unittest.TestCase): - """backports a number of unittest2 features in TestCase""" - - #=================================================================== - # backport some unittest2 names - #=================================================================== - - #--------------------------------------------------------------- - # backport assertRegex() alias from 3.2 to 2.7 - # was present in 2.7 under an alternate name - #--------------------------------------------------------------- - if not hasattr(unittest.TestCase, "assertRegex"): - assertRegex = unittest.TestCase.assertRegexpMatches - - if not hasattr(unittest.TestCase, "assertRaisesRegex"): - assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/sample1.cfg b/src/passlib/tests/sample1.cfg deleted file mode 100644 index 56e3ae8e..00000000 --- a/src/passlib/tests/sample1.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[passlib] -schemes = des_crypt, md5_crypt, bsdi_crypt, sha512_crypt -default = md5_crypt -all__vary_rounds = 0.1 -bsdi_crypt__default_rounds = 25001 -bsdi_crypt__max_rounds = 30001 -sha512_crypt__max_rounds = 50000 -sha512_crypt__min_rounds = 40000 - diff --git a/src/passlib/tests/sample1b.cfg b/src/passlib/tests/sample1b.cfg deleted file mode 100644 index 542a6036..00000000 --- a/src/passlib/tests/sample1b.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[passlib] -schemes = des_crypt, md5_crypt, bsdi_crypt, sha512_crypt -default = md5_crypt -all__vary_rounds = 0.1 -bsdi_crypt__default_rounds = 25001 -bsdi_crypt__max_rounds = 30001 -sha512_crypt__max_rounds = 50000 -sha512_crypt__min_rounds = 40000 - diff --git a/src/passlib/tests/sample1c.cfg b/src/passlib/tests/sample1c.cfg deleted file mode 100644 index a5033eb9..00000000 Binary files a/src/passlib/tests/sample1c.cfg and /dev/null differ diff --git a/src/passlib/tests/sample_config_1s.cfg b/src/passlib/tests/sample_config_1s.cfg deleted file mode 100644 index 495a13ea..00000000 --- a/src/passlib/tests/sample_config_1s.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[passlib] -schemes = des_crypt, md5_crypt, bsdi_crypt, sha512_crypt -default = md5_crypt -all.vary_rounds = 10%% -bsdi_crypt.max_rounds = 30000 -bsdi_crypt.default_rounds = 25000 -sha512_crypt.max_rounds = 50000 -sha512_crypt.min_rounds = 40000 diff --git a/src/passlib/tests/test_apache.py b/src/passlib/tests/test_apache.py deleted file mode 100644 index c734c990..00000000 --- a/src/passlib/tests/test_apache.py +++ /dev/null @@ -1,651 +0,0 @@ -"""tests for passlib.apache -- (c) Assurance Technologies 2008-2011""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -from logging import getLogger -import os -# site -# pkg -from passlib import apache -from passlib.exc import MissingBackendError -from passlib.utils.compat import irange -from passlib.tests.utils import TestCase, get_file, set_file, ensure_mtime_changed -from passlib.utils.compat import u -from passlib.utils import to_bytes -# module -log = getLogger(__name__) - -def backdate_file_mtime(path, offset=10): - """backdate file's mtime by specified amount""" - # NOTE: this is used so we can test code which detects mtime changes, - # without having to actually *pause* for that long. - atime = os.path.getatime(path) - mtime = os.path.getmtime(path)-offset - os.utime(path, (atime, mtime)) - -#============================================================================= -# htpasswd -#============================================================================= -class HtpasswdFileTest(TestCase): - """test HtpasswdFile class""" - descriptionPrefix = "HtpasswdFile" - - # sample with 4 users - sample_01 = (b'user2:2CHkkwa2AtqGs\n' - b'user3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\n' - b'user4:pass4\n' - b'user1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\n') - - # sample 1 with user 1, 2 deleted; 4 changed - sample_02 = b'user3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\n' - - # sample 1 with user2 updated, user 1 first entry removed, and user 5 added - sample_03 = (b'user2:pass2x\n' - b'user3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\n' - b'user4:pass4\n' - b'user1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\n' - b'user5:pass5\n') - - # standalone sample with 8-bit username - sample_04_utf8 = b'user\xc3\xa6:2CHkkwa2AtqGs\n' - sample_04_latin1 = b'user\xe6:2CHkkwa2AtqGs\n' - - sample_dup = b'user1:pass1\nuser1:pass2\n' - - # sample with bcrypt & sha256_crypt hashes - sample_05 = (b'user2:2CHkkwa2AtqGs\n' - b'user3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\n' - b'user4:pass4\n' - b'user1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\n' - b'user5:$2a$12$yktDxraxijBZ360orOyCOePFGhuis/umyPNJoL5EbsLk.s6SWdrRO\n' - b'user6:$5$rounds=110000$cCRp/xUUGVgwR4aP$' - b'p0.QKFS5qLNRqw1/47lXYiAcgIjJK.WjCO8nrEKuUK.\n') - - def test_00_constructor_autoload(self): - """test constructor autoload""" - # check with existing file - path = self.mktemp() - set_file(path, self.sample_01) - ht = apache.HtpasswdFile(path) - self.assertEqual(ht.to_string(), self.sample_01) - self.assertEqual(ht.path, path) - self.assertTrue(ht.mtime) - - # check changing path - ht.path = path + "x" - self.assertEqual(ht.path, path + "x") - self.assertFalse(ht.mtime) - - # check new=True - ht = apache.HtpasswdFile(path, new=True) - self.assertEqual(ht.to_string(), b"") - self.assertEqual(ht.path, path) - self.assertFalse(ht.mtime) - - # check autoload=False (deprecated alias for new=True) - with self.assertWarningList("``autoload=False`` is deprecated"): - ht = apache.HtpasswdFile(path, autoload=False) - self.assertEqual(ht.to_string(), b"") - self.assertEqual(ht.path, path) - self.assertFalse(ht.mtime) - - # check missing file - os.remove(path) - self.assertRaises(IOError, apache.HtpasswdFile, path) - - # NOTE: "default_scheme" option checked via set_password() test, among others - - def test_00_from_path(self): - path = self.mktemp() - set_file(path, self.sample_01) - ht = apache.HtpasswdFile.from_path(path) - self.assertEqual(ht.to_string(), self.sample_01) - self.assertEqual(ht.path, None) - self.assertFalse(ht.mtime) - - def test_01_delete(self): - """test delete()""" - ht = apache.HtpasswdFile.from_string(self.sample_01) - self.assertTrue(ht.delete("user1")) # should delete both entries - self.assertTrue(ht.delete("user2")) - self.assertFalse(ht.delete("user5")) # user not present - self.assertEqual(ht.to_string(), self.sample_02) - - # invalid user - self.assertRaises(ValueError, ht.delete, "user:") - - def test_01_delete_autosave(self): - path = self.mktemp() - sample = b'user1:pass1\nuser2:pass2\n' - set_file(path, sample) - - ht = apache.HtpasswdFile(path) - ht.delete("user1") - self.assertEqual(get_file(path), sample) - - ht = apache.HtpasswdFile(path, autosave=True) - ht.delete("user1") - self.assertEqual(get_file(path), b"user2:pass2\n") - - def test_02_set_password(self): - """test set_password()""" - ht = apache.HtpasswdFile.from_string( - self.sample_01, default_scheme="plaintext") - self.assertTrue(ht.set_password("user2", "pass2x")) - self.assertFalse(ht.set_password("user5", "pass5")) - self.assertEqual(ht.to_string(), self.sample_03) - - # test legacy default kwd - with self.assertWarningList("``default`` is deprecated"): - ht = apache.HtpasswdFile.from_string(self.sample_01, default="plaintext") - self.assertTrue(ht.set_password("user2", "pass2x")) - self.assertFalse(ht.set_password("user5", "pass5")) - self.assertEqual(ht.to_string(), self.sample_03) - - # invalid user - self.assertRaises(ValueError, ht.set_password, "user:", "pass") - - # test that legacy update() still works - with self.assertWarningList("update\(\) is deprecated"): - ht.update("user2", "test") - self.assertTrue(ht.check_password("user2", "test")) - - def test_02_set_password_autosave(self): - path = self.mktemp() - sample = b'user1:pass1\n' - set_file(path, sample) - - ht = apache.HtpasswdFile(path) - ht.set_password("user1", "pass2") - self.assertEqual(get_file(path), sample) - - ht = apache.HtpasswdFile(path, default_scheme="plaintext", autosave=True) - ht.set_password("user1", "pass2") - self.assertEqual(get_file(path), b"user1:pass2\n") - - def test_02_set_password_default_scheme(self): - """test set_password() -- default_scheme""" - - def check(scheme): - ht = apache.HtpasswdFile(default_scheme=scheme) - ht.set_password("user1", "pass1") - return ht.context.identify(ht.get_hash("user1")) - - # explicit scheme - self.assertEqual(check("sha256_crypt"), "sha256_crypt") - self.assertEqual(check("des_crypt"), "des_crypt") - - # unknown scheme - self.assertRaises(KeyError, check, "xxx") - - # alias resolution - self.assertEqual(check("portable"), apache.htpasswd_defaults["portable"]) - self.assertEqual(check("portable_apache_22"), apache.htpasswd_defaults["portable_apache_22"]) - self.assertEqual(check("host_apache_22"), apache.htpasswd_defaults["host_apache_22"]) - - # default - self.assertEqual(check(None), apache.htpasswd_defaults["portable_apache_22"]) - - def test_03_users(self): - """test users()""" - ht = apache.HtpasswdFile.from_string(self.sample_01) - ht.set_password("user5", "pass5") - ht.delete("user3") - ht.set_password("user3", "pass3") - self.assertEqual(sorted(ht.users()), ["user1", "user2", "user3", "user4", "user5"]) - - def test_04_check_password(self): - """test check_password()""" - ht = apache.HtpasswdFile.from_string(self.sample_05) - self.assertRaises(TypeError, ht.check_password, 1, 'pass9') - self.assertTrue(ht.check_password("user9","pass9") is None) - - # users 1..6 of sample_01 run through all the main hash formats, - # to make sure they're recognized. - for i in irange(1, 7): - i = str(i) - try: - self.assertTrue(ht.check_password("user"+i, "pass"+i)) - self.assertTrue(ht.check_password("user"+i, "pass9") is False) - except MissingBackendError: - if i == "5": - # user5 uses bcrypt, which is apparently not available right now - continue - raise - - self.assertRaises(ValueError, ht.check_password, "user:", "pass") - - # test that legacy verify() still works - with self.assertWarningList(["verify\(\) is deprecated"]*2): - self.assertTrue(ht.verify("user1", "pass1")) - self.assertFalse(ht.verify("user1", "pass2")) - - def test_05_load(self): - """test load()""" - # setup empty file - path = self.mktemp() - set_file(path, "") - backdate_file_mtime(path, 5) - ha = apache.HtpasswdFile(path, default_scheme="plaintext") - self.assertEqual(ha.to_string(), b"") - - # make changes, check load_if_changed() does nothing - ha.set_password("user1", "pass1") - ha.load_if_changed() - self.assertEqual(ha.to_string(), b"user1:pass1\n") - - # change file - set_file(path, self.sample_01) - ha.load_if_changed() - self.assertEqual(ha.to_string(), self.sample_01) - - # make changes, check load() overwrites them - ha.set_password("user5", "pass5") - ha.load() - self.assertEqual(ha.to_string(), self.sample_01) - - # test load w/ no path - hb = apache.HtpasswdFile() - self.assertRaises(RuntimeError, hb.load) - self.assertRaises(RuntimeError, hb.load_if_changed) - - # test load w/ dups and explicit path - set_file(path, self.sample_dup) - hc = apache.HtpasswdFile() - hc.load(path) - self.assertTrue(hc.check_password('user1','pass1')) - - # NOTE: load_string() tested via from_string(), which is used all over this file - - def test_06_save(self): - """test save()""" - # load from file - path = self.mktemp() - set_file(path, self.sample_01) - ht = apache.HtpasswdFile(path) - - # make changes, check they saved - ht.delete("user1") - ht.delete("user2") - ht.save() - self.assertEqual(get_file(path), self.sample_02) - - # test save w/ no path - hb = apache.HtpasswdFile(default_scheme="plaintext") - hb.set_password("user1", "pass1") - self.assertRaises(RuntimeError, hb.save) - - # test save w/ explicit path - hb.save(path) - self.assertEqual(get_file(path), b"user1:pass1\n") - - def test_07_encodings(self): - """test 'encoding' kwd""" - # test bad encodings cause failure in constructor - self.assertRaises(ValueError, apache.HtpasswdFile, encoding="utf-16") - - # check sample utf-8 - ht = apache.HtpasswdFile.from_string(self.sample_04_utf8, encoding="utf-8", - return_unicode=True) - self.assertEqual(ht.users(), [ u("user\u00e6") ]) - - # test deprecated encoding=None - with self.assertWarningList("``encoding=None`` is deprecated"): - ht = apache.HtpasswdFile.from_string(self.sample_04_utf8, encoding=None) - self.assertEqual(ht.users(), [ b'user\xc3\xa6' ]) - - # check sample latin-1 - ht = apache.HtpasswdFile.from_string(self.sample_04_latin1, - encoding="latin-1", return_unicode=True) - self.assertEqual(ht.users(), [ u("user\u00e6") ]) - - def test_08_get_hash(self): - """test get_hash()""" - ht = apache.HtpasswdFile.from_string(self.sample_01) - self.assertEqual(ht.get_hash("user3"), b"{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=") - self.assertEqual(ht.get_hash("user4"), b"pass4") - self.assertEqual(ht.get_hash("user5"), None) - - with self.assertWarningList("find\(\) is deprecated"): - self.assertEqual(ht.find("user4"), b"pass4") - - def test_09_to_string(self): - """test to_string""" - - # check with known sample - ht = apache.HtpasswdFile.from_string(self.sample_01) - self.assertEqual(ht.to_string(), self.sample_01) - - # test blank - ht = apache.HtpasswdFile() - self.assertEqual(ht.to_string(), b"") - - def test_10_repr(self): - ht = apache.HtpasswdFile("fakepath", autosave=True, new=True, encoding="latin-1") - repr(ht) - - def test_11_malformed(self): - self.assertRaises(ValueError, apache.HtpasswdFile.from_string, - b'realm:user1:pass1\n') - self.assertRaises(ValueError, apache.HtpasswdFile.from_string, - b'pass1\n') - - def test_12_from_string(self): - # forbid path kwd - self.assertRaises(TypeError, apache.HtpasswdFile.from_string, - b'', path=None) - - def test_13_whitespace(self): - """whitespace & comment handling""" - - # per htpasswd source (https://github.com/apache/httpd/blob/trunk/support/htpasswd.c), - # lines that match "^\s*(#.*)?$" should be ignored - source = to_bytes( - '\n' - 'user2:pass2\n' - 'user4:pass4\n' - 'user7:pass7\r\n' - ' \t \n' - 'user1:pass1\n' - ' # legacy users\n' - '#user6:pass6\n' - 'user5:pass5\n\n' - ) - - # loading should see all users (except user6, who was commented out) - ht = apache.HtpasswdFile.from_string(source) - self.assertEqual(sorted(ht.users()), ["user1", "user2", "user4", "user5", "user7"]) - - # update existing user - ht.set_hash("user4", "althash4") - self.assertEqual(sorted(ht.users()), ["user1", "user2", "user4", "user5", "user7"]) - - # add a new user - ht.set_hash("user6", "althash6") - self.assertEqual(sorted(ht.users()), ["user1", "user2", "user4", "user5", "user6", "user7"]) - - # delete existing user - ht.delete("user7") - self.assertEqual(sorted(ht.users()), ["user1", "user2", "user4", "user5", "user6"]) - - # re-serialization should preserve whitespace - target = to_bytes( - '\n' - 'user2:pass2\n' - 'user4:althash4\n' - ' \t \n' - 'user1:pass1\n' - ' # legacy users\n' - '#user6:pass6\n' - 'user5:pass5\n' - 'user6:althash6\n' - ) - self.assertEqual(ht.to_string(), target) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# htdigest -#============================================================================= -class HtdigestFileTest(TestCase): - """test HtdigestFile class""" - descriptionPrefix = "HtdigestFile" - - # sample with 4 users - sample_01 = (b'user2:realm:549d2a5f4659ab39a80dac99e159ab19\n' - b'user3:realm:a500bb8c02f6a9170ae46af10c898744\n' - b'user4:realm:ab7b5d5f28ccc7666315f508c7358519\n' - b'user1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n') - - # sample 1 with user 1, 2 deleted; 4 changed - sample_02 = (b'user3:realm:a500bb8c02f6a9170ae46af10c898744\n' - b'user4:realm:ab7b5d5f28ccc7666315f508c7358519\n') - - # sample 1 with user2 updated, user 1 first entry removed, and user 5 added - sample_03 = (b'user2:realm:5ba6d8328943c23c64b50f8b29566059\n' - b'user3:realm:a500bb8c02f6a9170ae46af10c898744\n' - b'user4:realm:ab7b5d5f28ccc7666315f508c7358519\n' - b'user1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n' - b'user5:realm:03c55fdc6bf71552356ad401bdb9af19\n') - - # standalone sample with 8-bit username & realm - sample_04_utf8 = b'user\xc3\xa6:realm\xc3\xa6:549d2a5f4659ab39a80dac99e159ab19\n' - sample_04_latin1 = b'user\xe6:realm\xe6:549d2a5f4659ab39a80dac99e159ab19\n' - - def test_00_constructor_autoload(self): - """test constructor autoload""" - # check with existing file - path = self.mktemp() - set_file(path, self.sample_01) - ht = apache.HtdigestFile(path) - self.assertEqual(ht.to_string(), self.sample_01) - - # check without autoload - ht = apache.HtdigestFile(path, new=True) - self.assertEqual(ht.to_string(), b"") - - # check missing file - os.remove(path) - self.assertRaises(IOError, apache.HtdigestFile, path) - - # NOTE: default_realm option checked via other tests. - - def test_01_delete(self): - """test delete()""" - ht = apache.HtdigestFile.from_string(self.sample_01) - self.assertTrue(ht.delete("user1", "realm")) - self.assertTrue(ht.delete("user2", "realm")) - self.assertFalse(ht.delete("user5", "realm")) - self.assertFalse(ht.delete("user3", "realm5")) - self.assertEqual(ht.to_string(), self.sample_02) - - # invalid user - self.assertRaises(ValueError, ht.delete, "user:", "realm") - - # invalid realm - self.assertRaises(ValueError, ht.delete, "user", "realm:") - - def test_01_delete_autosave(self): - path = self.mktemp() - set_file(path, self.sample_01) - - ht = apache.HtdigestFile(path) - self.assertTrue(ht.delete("user1", "realm")) - self.assertFalse(ht.delete("user3", "realm5")) - self.assertFalse(ht.delete("user5", "realm")) - self.assertEqual(get_file(path), self.sample_01) - - ht.autosave = True - self.assertTrue(ht.delete("user2", "realm")) - self.assertEqual(get_file(path), self.sample_02) - - def test_02_set_password(self): - """test update()""" - ht = apache.HtdigestFile.from_string(self.sample_01) - self.assertTrue(ht.set_password("user2", "realm", "pass2x")) - self.assertFalse(ht.set_password("user5", "realm", "pass5")) - self.assertEqual(ht.to_string(), self.sample_03) - - # default realm - self.assertRaises(TypeError, ht.set_password, "user2", "pass3") - ht.default_realm = "realm2" - ht.set_password("user2", "pass3") - ht.check_password("user2", "realm2", "pass3") - - # invalid user - self.assertRaises(ValueError, ht.set_password, "user:", "realm", "pass") - self.assertRaises(ValueError, ht.set_password, "u"*256, "realm", "pass") - - # invalid realm - self.assertRaises(ValueError, ht.set_password, "user", "realm:", "pass") - self.assertRaises(ValueError, ht.set_password, "user", "r"*256, "pass") - - # test that legacy update() still works - with self.assertWarningList("update\(\) is deprecated"): - ht.update("user2", "realm2", "test") - self.assertTrue(ht.check_password("user2", "test")) - - # TODO: test set_password autosave - - def test_03_users(self): - """test users()""" - ht = apache.HtdigestFile.from_string(self.sample_01) - ht.set_password("user5", "realm", "pass5") - ht.delete("user3", "realm") - ht.set_password("user3", "realm", "pass3") - self.assertEqual(sorted(ht.users("realm")), ["user1", "user2", "user3", "user4", "user5"]) - - self.assertRaises(TypeError, ht.users, 1) - - def test_04_check_password(self): - """test check_password()""" - ht = apache.HtdigestFile.from_string(self.sample_01) - self.assertRaises(TypeError, ht.check_password, 1, 'realm', 'pass5') - self.assertRaises(TypeError, ht.check_password, 'user', 1, 'pass5') - self.assertIs(ht.check_password("user5", "realm","pass5"), None) - for i in irange(1,5): - i = str(i) - self.assertTrue(ht.check_password("user"+i, "realm", "pass"+i)) - self.assertIs(ht.check_password("user"+i, "realm", "pass5"), False) - - # default realm - self.assertRaises(TypeError, ht.check_password, "user5", "pass5") - ht.default_realm = "realm" - self.assertTrue(ht.check_password("user1", "pass1")) - self.assertIs(ht.check_password("user5", "pass5"), None) - - # test that legacy verify() still works - with self.assertWarningList(["verify\(\) is deprecated"]*2): - self.assertTrue(ht.verify("user1", "realm", "pass1")) - self.assertFalse(ht.verify("user1", "realm", "pass2")) - - # invalid user - self.assertRaises(ValueError, ht.check_password, "user:", "realm", "pass") - - def test_05_load(self): - """test load()""" - # setup empty file - path = self.mktemp() - set_file(path, "") - backdate_file_mtime(path, 5) - ha = apache.HtdigestFile(path) - self.assertEqual(ha.to_string(), b"") - - # make changes, check load_if_changed() does nothing - ha.set_password("user1", "realm", "pass1") - ha.load_if_changed() - self.assertEqual(ha.to_string(), b'user1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n') - - # change file - set_file(path, self.sample_01) - ha.load_if_changed() - self.assertEqual(ha.to_string(), self.sample_01) - - # make changes, check load_if_changed overwrites them - ha.set_password("user5", "realm", "pass5") - ha.load() - self.assertEqual(ha.to_string(), self.sample_01) - - # test load w/ no path - hb = apache.HtdigestFile() - self.assertRaises(RuntimeError, hb.load) - self.assertRaises(RuntimeError, hb.load_if_changed) - - # test load w/ explicit path - hc = apache.HtdigestFile() - hc.load(path) - self.assertEqual(hc.to_string(), self.sample_01) - - # change file, test deprecated force=False kwd - ensure_mtime_changed(path) - set_file(path, "") - with self.assertWarningList(r"load\(force=False\) is deprecated"): - ha.load(force=False) - self.assertEqual(ha.to_string(), b"") - - def test_06_save(self): - """test save()""" - # load from file - path = self.mktemp() - set_file(path, self.sample_01) - ht = apache.HtdigestFile(path) - - # make changes, check they saved - ht.delete("user1", "realm") - ht.delete("user2", "realm") - ht.save() - self.assertEqual(get_file(path), self.sample_02) - - # test save w/ no path - hb = apache.HtdigestFile() - hb.set_password("user1", "realm", "pass1") - self.assertRaises(RuntimeError, hb.save) - - # test save w/ explicit path - hb.save(path) - self.assertEqual(get_file(path), hb.to_string()) - - def test_07_realms(self): - """test realms() & delete_realm()""" - ht = apache.HtdigestFile.from_string(self.sample_01) - - self.assertEqual(ht.delete_realm("x"), 0) - self.assertEqual(ht.realms(), ['realm']) - - self.assertEqual(ht.delete_realm("realm"), 4) - self.assertEqual(ht.realms(), []) - self.assertEqual(ht.to_string(), b"") - - def test_08_get_hash(self): - """test get_hash()""" - ht = apache.HtdigestFile.from_string(self.sample_01) - self.assertEqual(ht.get_hash("user3", "realm"), "a500bb8c02f6a9170ae46af10c898744") - self.assertEqual(ht.get_hash("user4", "realm"), "ab7b5d5f28ccc7666315f508c7358519") - self.assertEqual(ht.get_hash("user5", "realm"), None) - - with self.assertWarningList("find\(\) is deprecated"): - self.assertEqual(ht.find("user4", "realm"), "ab7b5d5f28ccc7666315f508c7358519") - - def test_09_encodings(self): - """test encoding parameter""" - # test bad encodings cause failure in constructor - self.assertRaises(ValueError, apache.HtdigestFile, encoding="utf-16") - - # check sample utf-8 - ht = apache.HtdigestFile.from_string(self.sample_04_utf8, encoding="utf-8", return_unicode=True) - self.assertEqual(ht.realms(), [ u("realm\u00e6") ]) - self.assertEqual(ht.users(u("realm\u00e6")), [ u("user\u00e6") ]) - - # check sample latin-1 - ht = apache.HtdigestFile.from_string(self.sample_04_latin1, encoding="latin-1", return_unicode=True) - self.assertEqual(ht.realms(), [ u("realm\u00e6") ]) - self.assertEqual(ht.users(u("realm\u00e6")), [ u("user\u00e6") ]) - - def test_10_to_string(self): - """test to_string()""" - - # check sample - ht = apache.HtdigestFile.from_string(self.sample_01) - self.assertEqual(ht.to_string(), self.sample_01) - - # check blank - ht = apache.HtdigestFile() - self.assertEqual(ht.to_string(), b"") - - def test_11_malformed(self): - self.assertRaises(ValueError, apache.HtdigestFile.from_string, - b'realm:user1:pass1:other\n') - self.assertRaises(ValueError, apache.HtdigestFile.from_string, - b'user1:pass1\n') - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_apps.py b/src/passlib/tests/test_apps.py deleted file mode 100644 index 167437f5..00000000 --- a/src/passlib/tests/test_apps.py +++ /dev/null @@ -1,139 +0,0 @@ -"""test passlib.apps""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib import apps, hash as hashmod -from passlib.tests.utils import TestCase -# module - -#============================================================================= -# test predefined app contexts -#============================================================================= -class AppsTest(TestCase): - """perform general tests to make sure contexts work""" - # NOTE: these tests are not really comprehensive, - # since they would do little but duplicate - # the presets in apps.py - # - # they mainly try to ensure no typos - # or dynamic behavior foul-ups. - - def test_master_context(self): - ctx = apps.master_context - self.assertGreater(len(ctx.schemes()), 50) - - def test_custom_app_context(self): - ctx = apps.custom_app_context - self.assertEqual(ctx.schemes(), ("sha512_crypt", "sha256_crypt")) - for hash in [ - ('$6$rounds=41128$VoQLvDjkaZ6L6BIE$4pt.1Ll1XdDYduEwEYPCMOBiR6W6' - 'znsyUEoNlcVXpv2gKKIbQolgmTGe6uEEVJ7azUxuc8Tf7zV9SD2z7Ij751'), - ('$5$rounds=31817$iZGmlyBQ99JSB5n6$p4E.pdPBWx19OajgjLRiOW0itGny' - 'xDGgMlDcOsfaI17'), - ]: - self.assertTrue(ctx.verify("test", hash)) - - def test_django16_context(self): - ctx = apps.django16_context - for hash in [ - 'pbkdf2_sha256$29000$ZsgquwnCyBs2$fBxRQpfKd2PIeMxtkKPy0h7SrnrN+EU/cm67aitoZ2s=', - 'sha1$0d082$cdb462ae8b6be8784ef24b20778c4d0c82d5957f', - 'md5$b887a$37767f8a745af10612ad44c80ff52e92', - 'crypt$95a6d$95x74hLDQKXI2', - '098f6bcd4621d373cade4e832627b4f6', - ]: - self.assertTrue(ctx.verify("test", hash)) - - self.assertEqual(ctx.identify("!"), "django_disabled") - self.assertFalse(ctx.verify("test", "!")) - - def test_django_context(self): - ctx = apps.django_context - for hash in [ - 'pbkdf2_sha256$29000$ZsgquwnCyBs2$fBxRQpfKd2PIeMxtkKPy0h7SrnrN+EU/cm67aitoZ2s=', - ]: - self.assertTrue(ctx.verify("test", hash)) - - self.assertEqual(ctx.identify("!"), "django_disabled") - self.assertFalse(ctx.verify("test", "!")) - - def test_ldap_nocrypt_context(self): - ctx = apps.ldap_nocrypt_context - for hash in [ - '{SSHA}cPusOzd6d5n3OjSVK3R329ZGCNyFcC7F', - 'test', - ]: - self.assertTrue(ctx.verify("test", hash)) - - self.assertIs(ctx.identify('{CRYPT}$5$rounds=31817$iZGmlyBQ99JSB5' - 'n6$p4E.pdPBWx19OajgjLRiOW0itGnyxDGgMlDcOsfaI17'), None) - - def test_ldap_context(self): - ctx = apps.ldap_context - for hash in [ - ('{CRYPT}$5$rounds=31817$iZGmlyBQ99JSB5n6$p4E.pdPBWx19OajgjLRiOW0' - 'itGnyxDGgMlDcOsfaI17'), - '{SSHA}cPusOzd6d5n3OjSVK3R329ZGCNyFcC7F', - 'test', - ]: - self.assertTrue(ctx.verify("test", hash)) - - def test_ldap_mysql_context(self): - ctx = apps.mysql_context - for hash in [ - '*94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29', - '378b243e220ca493', - ]: - self.assertTrue(ctx.verify("test", hash)) - - def test_postgres_context(self): - ctx = apps.postgres_context - hash = 'md55d9c68c6c50ed3d02a2fcf54f63993b6' - self.assertTrue(ctx.verify("test", hash, user='user')) - - def test_phppass_context(self): - ctx = apps.phpass_context - for hash in [ - '$P$8Ja1vJsKa5qyy/b3mCJGXM7GyBnt6..', - '$H$8b95CoYQnQ9Y6fSTsACyphNh5yoM02.', - '_cD..aBxeRhYFJvtUvsI', - ]: - self.assertTrue(ctx.verify("test", hash)) - - h1 = "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS" - if hashmod.bcrypt.has_backend(): - self.assertTrue(ctx.verify("test", h1)) - self.assertEqual(ctx.default_scheme(), "bcrypt") - self.assertEqual(ctx.handler().name, "bcrypt") - else: - self.assertEqual(ctx.identify(h1), "bcrypt") - self.assertEqual(ctx.default_scheme(), "phpass") - self.assertEqual(ctx.handler().name, "phpass") - - def test_phpbb3_context(self): - ctx = apps.phpbb3_context - for hash in [ - '$P$8Ja1vJsKa5qyy/b3mCJGXM7GyBnt6..', - '$H$8b95CoYQnQ9Y6fSTsACyphNh5yoM02.', - ]: - self.assertTrue(ctx.verify("test", hash)) - self.assertTrue(ctx.hash("test").startswith("$H$")) - - def test_roundup_context(self): - ctx = apps.roundup_context - for hash in [ - '{PBKDF2}9849$JMTYu3eOUSoFYExprVVqbQ$N5.gV.uR1.BTgLSvi0qyPiRlGZ0', - '{SHA}a94a8fe5ccb19ba61c4c0873d391e987982fbbd3', - '{CRYPT}dptOmKDriOGfU', - '{plaintext}test', - ]: - self.assertTrue(ctx.verify("test", hash)) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_context.py b/src/passlib/tests/test_context.py deleted file mode 100644 index 89fb9fe2..00000000 --- a/src/passlib/tests/test_context.py +++ /dev/null @@ -1,1796 +0,0 @@ -"""tests for passlib.context""" -#============================================================================= -# imports -#============================================================================= -# core -from __future__ import with_statement -from passlib.utils.compat import PY3 -if PY3: - from configparser import NoSectionError -else: - from ConfigParser import NoSectionError -import datetime -from functools import partial -import logging; log = logging.getLogger(__name__) -import os -import warnings -# site -# pkg -from passlib import hash -from passlib.context import CryptContext, LazyCryptContext -from passlib.exc import PasslibConfigWarning, PasslibHashWarning -from passlib.utils import tick, to_unicode -from passlib.utils.compat import irange, u, unicode, str_to_uascii, PY2, PY26 -import passlib.utils.handlers as uh -from passlib.tests.utils import (TestCase, set_file, TICK_RESOLUTION, - quicksleep, time_call, handler_derived_from) -from passlib.registry import (register_crypt_handler_path, - _has_crypt_handler as has_crypt_handler, - _unload_handler_name as unload_handler_name, - get_crypt_handler, - ) -# local -#============================================================================= -# support -#============================================================================= -here = os.path.abspath(os.path.dirname(__file__)) - -def merge_dicts(first, *args, **kwds): - target = first.copy() - for arg in args: - target.update(arg) - if kwds: - target.update(kwds) - return target - -#============================================================================= -# -#============================================================================= -class CryptContextTest(TestCase): - descriptionPrefix = "CryptContext" - - # TODO: these unittests could really use a good cleanup - # and reorganizing, to ensure they're getting everything. - - #=================================================================== - # sample configurations used in tests - #=================================================================== - - #--------------------------------------------------------------- - # sample 1 - typical configuration - #--------------------------------------------------------------- - sample_1_schemes = ["des_crypt", "md5_crypt", "bsdi_crypt", "sha512_crypt"] - sample_1_handlers = [get_crypt_handler(name) for name in sample_1_schemes] - - sample_1_dict = dict( - schemes = sample_1_schemes, - default = "md5_crypt", - all__vary_rounds = 0.1, - bsdi_crypt__max_rounds = 30001, - bsdi_crypt__default_rounds = 25001, - sha512_crypt__max_rounds = 50000, - sha512_crypt__min_rounds = 40000, - ) - - sample_1_resolved_dict = merge_dicts(sample_1_dict, - schemes = sample_1_handlers) - - sample_1_unnormalized = u("""\ -[passlib] -schemes = des_crypt, md5_crypt, bsdi_crypt, sha512_crypt -default = md5_crypt -; this is using %... -all__vary_rounds = 10%% -bsdi_crypt__default_rounds = 25001 -bsdi_crypt__max_rounds = 30001 -sha512_crypt__max_rounds = 50000 -sha512_crypt__min_rounds = 40000 -""") - - sample_1_unicode = u("""\ -[passlib] -schemes = des_crypt, md5_crypt, bsdi_crypt, sha512_crypt -default = md5_crypt -all__vary_rounds = 0.1 -bsdi_crypt__default_rounds = 25001 -bsdi_crypt__max_rounds = 30001 -sha512_crypt__max_rounds = 50000 -sha512_crypt__min_rounds = 40000 - -""") - - #--------------------------------------------------------------- - # sample 1 external files - #--------------------------------------------------------------- - - # sample 1 string with '\n' linesep - sample_1_path = os.path.join(here, "sample1.cfg") - - # sample 1 with '\r\n' linesep - sample_1b_unicode = sample_1_unicode.replace(u("\n"), u("\r\n")) - sample_1b_path = os.path.join(here, "sample1b.cfg") - - # sample 1 using UTF-16 and alt section - sample_1c_bytes = sample_1_unicode.replace(u("[passlib]"), - u("[mypolicy]")).encode("utf-16") - sample_1c_path = os.path.join(here, "sample1c.cfg") - - # enable to regenerate sample files - if False: - set_file(sample_1_path, sample_1_unicode) - set_file(sample_1b_path, sample_1b_unicode) - set_file(sample_1c_path, sample_1c_bytes) - - #--------------------------------------------------------------- - # sample 2 & 12 - options patch - #--------------------------------------------------------------- - sample_2_dict = dict( - # using this to test full replacement of existing options - bsdi_crypt__min_rounds = 29001, - bsdi_crypt__max_rounds = 35001, - bsdi_crypt__default_rounds = 31001, - # using this to test partial replacement of existing options - sha512_crypt__min_rounds=45000, - ) - - sample_2_unicode = """\ -[passlib] -bsdi_crypt__min_rounds = 29001 -bsdi_crypt__max_rounds = 35001 -bsdi_crypt__default_rounds = 31001 -sha512_crypt__min_rounds = 45000 -""" - - # sample 2 overlayed on top of sample 1 - sample_12_dict = merge_dicts(sample_1_dict, sample_2_dict) - - #--------------------------------------------------------------- - # sample 3 & 123 - just changing default from sample 1 - #--------------------------------------------------------------- - sample_3_dict = dict( - default="sha512_crypt", - ) - - # sample 3 overlayed on 2 overlayed on 1 - sample_123_dict = merge_dicts(sample_12_dict, sample_3_dict) - - #--------------------------------------------------------------- - # sample 4 - used by api tests - #--------------------------------------------------------------- - sample_4_dict = dict( - schemes = [ "des_crypt", "md5_crypt", "phpass", "bsdi_crypt", - "sha256_crypt"], - deprecated = [ "des_crypt", ], - default = "sha256_crypt", - bsdi_crypt__max_rounds = 31, - bsdi_crypt__default_rounds = 25, - bsdi_crypt__vary_rounds = 0, - sha256_crypt__max_rounds = 3000, - sha256_crypt__min_rounds = 2000, - sha256_crypt__default_rounds = 3000, - phpass__ident = "H", - phpass__default_rounds = 7, - ) - - #=================================================================== - # setup - #=================================================================== - def setUp(self): - super(CryptContextTest, self).setUp() - warnings.filterwarnings("ignore", "The 'all' scheme is deprecated.*") - warnings.filterwarnings("ignore", ".*'scheme' keyword is deprecated as of Passlib 1.7.*") - - #=================================================================== - # constructors - #=================================================================== - def test_01_constructor(self): - """test class constructor""" - - # test blank constructor works correctly - ctx = CryptContext() - self.assertEqual(ctx.to_dict(), {}) - - # test sample 1 with scheme=names - ctx = CryptContext(**self.sample_1_dict) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 with scheme=handlers - ctx = CryptContext(**self.sample_1_resolved_dict) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 2: options w/o schemes - ctx = CryptContext(**self.sample_2_dict) - self.assertEqual(ctx.to_dict(), self.sample_2_dict) - - # test sample 3: default only - ctx = CryptContext(**self.sample_3_dict) - self.assertEqual(ctx.to_dict(), self.sample_3_dict) - - # test unicode scheme names (issue 54) - ctx = CryptContext(schemes=[u("sha256_crypt")]) - self.assertEqual(ctx.schemes(), ("sha256_crypt",)) - - def test_02_from_string(self): - """test from_string() constructor""" - # test sample 1 unicode - ctx = CryptContext.from_string(self.sample_1_unicode) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 with unnormalized inputs - ctx = CryptContext.from_string(self.sample_1_unnormalized) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 utf-8 - ctx = CryptContext.from_string(self.sample_1_unicode.encode("utf-8")) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 w/ '\r\n' linesep - ctx = CryptContext.from_string(self.sample_1b_unicode) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 using UTF-16 and alt section - ctx = CryptContext.from_string(self.sample_1c_bytes, section="mypolicy", - encoding="utf-16") - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test wrong type - self.assertRaises(TypeError, CryptContext.from_string, None) - - # test missing section - self.assertRaises(NoSectionError, CryptContext.from_string, - self.sample_1_unicode, section="fakesection") - - def test_03_from_path(self): - """test from_path() constructor""" - # make sure sample files exist - if not os.path.exists(self.sample_1_path): - raise RuntimeError("can't find data file: %r" % self.sample_1_path) - - # test sample 1 - ctx = CryptContext.from_path(self.sample_1_path) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 w/ '\r\n' linesep - ctx = CryptContext.from_path(self.sample_1b_path) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test sample 1 encoding using UTF-16 and alt section - ctx = CryptContext.from_path(self.sample_1c_path, section="mypolicy", - encoding="utf-16") - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test missing file - self.assertRaises(EnvironmentError, CryptContext.from_path, - os.path.join(here, "sample1xxx.cfg")) - - # test missing section - self.assertRaises(NoSectionError, CryptContext.from_path, - self.sample_1_path, section="fakesection") - - def test_04_copy(self): - """test copy() method""" - cc1 = CryptContext(**self.sample_1_dict) - - # overlay sample 2 onto copy - cc2 = cc1.copy(**self.sample_2_dict) - self.assertEqual(cc1.to_dict(), self.sample_1_dict) - self.assertEqual(cc2.to_dict(), self.sample_12_dict) - - # check that repeating overlay makes no change - cc2b = cc2.copy(**self.sample_2_dict) - self.assertEqual(cc1.to_dict(), self.sample_1_dict) - self.assertEqual(cc2b.to_dict(), self.sample_12_dict) - - # overlay sample 3 on copy - cc3 = cc2.copy(**self.sample_3_dict) - self.assertEqual(cc3.to_dict(), self.sample_123_dict) - - # test empty copy creates separate copy - cc4 = cc1.copy() - self.assertIsNot(cc4, cc1) - self.assertEqual(cc1.to_dict(), self.sample_1_dict) - self.assertEqual(cc4.to_dict(), self.sample_1_dict) - - # ... and that modifying copy doesn't affect original - cc4.update(**self.sample_2_dict) - self.assertEqual(cc1.to_dict(), self.sample_1_dict) - self.assertEqual(cc4.to_dict(), self.sample_12_dict) - - def test_09_repr(self): - """test repr()""" - cc1 = CryptContext(**self.sample_1_dict) - # NOTE: "0x-1234" format used by Pyston 0.5.1 - self.assertRegex(repr(cc1), "^$") - - #=================================================================== - # modifiers - #=================================================================== - def test_10_load(self): - """test load() / load_path() method""" - # NOTE: load() is the workhorse that handles all policy parsing, - # compilation, and validation. most of its features are tested - # elsewhere, since all the constructors and modifiers are just - # wrappers for it. - - # source_type 'auto' - ctx = CryptContext() - - # detect dict - ctx.load(self.sample_1_dict) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # detect unicode string - ctx.load(self.sample_1_unicode) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # detect bytes string - ctx.load(self.sample_1_unicode.encode("utf-8")) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # anything else - TypeError - self.assertRaises(TypeError, ctx.load, None) - - # NOTE: load_path() tested by from_path() - # NOTE: additional string tests done by from_string() - - # update flag - tested by update() method tests - # encoding keyword - tested by from_string() & from_path() - # section keyword - tested by from_string() & from_path() - - # test load empty - ctx = CryptContext(**self.sample_1_dict) - ctx.load({}, update=True) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # multiple loads should clear the state - ctx = CryptContext() - ctx.load(self.sample_1_dict) - ctx.load(self.sample_2_dict) - self.assertEqual(ctx.to_dict(), self.sample_2_dict) - - def test_11_load_rollback(self): - """test load() errors restore old state""" - # create initial context - cc = CryptContext(["des_crypt", "sha256_crypt"], - sha256_crypt__default_rounds=5000, - all__vary_rounds=0.1, - ) - result = cc.to_string() - - # do an update operation that should fail during parsing - # XXX: not sure what the right error type is here. - self.assertRaises(TypeError, cc.update, too__many__key__parts=True) - self.assertEqual(cc.to_string(), result) - - # do an update operation that should fail during extraction - # FIXME: this isn't failing even in broken case, need to figure out - # way to ensure some keys come after this one. - self.assertRaises(KeyError, cc.update, fake_context_option=True) - self.assertEqual(cc.to_string(), result) - - # do an update operation that should fail during compilation - self.assertRaises(ValueError, cc.update, sha256_crypt__min_rounds=10000) - self.assertEqual(cc.to_string(), result) - - def test_12_update(self): - """test update() method""" - - # empty overlay - ctx = CryptContext(**self.sample_1_dict) - ctx.update() - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - - # test basic overlay - ctx = CryptContext(**self.sample_1_dict) - ctx.update(**self.sample_2_dict) - self.assertEqual(ctx.to_dict(), self.sample_12_dict) - - # ... and again - ctx.update(**self.sample_3_dict) - self.assertEqual(ctx.to_dict(), self.sample_123_dict) - - # overlay w/ dict arg - ctx = CryptContext(**self.sample_1_dict) - ctx.update(self.sample_2_dict) - self.assertEqual(ctx.to_dict(), self.sample_12_dict) - - # overlay w/ string - ctx = CryptContext(**self.sample_1_dict) - ctx.update(self.sample_2_unicode) - self.assertEqual(ctx.to_dict(), self.sample_12_dict) - - # too many args - self.assertRaises(TypeError, ctx.update, {}, {}) - self.assertRaises(TypeError, ctx.update, {}, schemes=['des_crypt']) - - # wrong arg type - self.assertRaises(TypeError, ctx.update, None) - - #=================================================================== - # option parsing - #=================================================================== - def test_20_options(self): - """test basic option parsing""" - def parse(**kwds): - return CryptContext(**kwds).to_dict() - - # - # common option parsing tests - # - - # test keys with blank fields are rejected - # blank option - self.assertRaises(TypeError, CryptContext, __=0.1) - self.assertRaises(TypeError, CryptContext, default__scheme__='x') - - # blank scheme - self.assertRaises(TypeError, CryptContext, __option='x') - self.assertRaises(TypeError, CryptContext, default____option='x') - - # blank category - self.assertRaises(TypeError, CryptContext, __scheme__option='x') - - # test keys with too many field are rejected - self.assertRaises(TypeError, CryptContext, - category__scheme__option__invalid = 30000) - - # keys with mixed separators should be handled correctly. - # (testing actual data, not to_dict(), since re-render hid original bug) - self.assertRaises(KeyError, parse, - **{"admin.context__schemes":"md5_crypt"}) - ctx = CryptContext(**{"schemes":"md5_crypt,des_crypt", - "admin.context__default":"des_crypt"}) - self.assertEqual(ctx.default_scheme("admin"), "des_crypt") - - # - # context option -specific tests - # - - # test context option key parsing - result = dict(default="md5_crypt") - self.assertEqual(parse(default="md5_crypt"), result) - self.assertEqual(parse(context__default="md5_crypt"), result) - self.assertEqual(parse(default__context__default="md5_crypt"), result) - self.assertEqual(parse(**{"context.default":"md5_crypt"}), result) - self.assertEqual(parse(**{"default.context.default":"md5_crypt"}), result) - - # test context option key parsing w/ category - result = dict(admin__context__default="md5_crypt") - self.assertEqual(parse(admin__context__default="md5_crypt"), result) - self.assertEqual(parse(**{"admin.context.default":"md5_crypt"}), result) - - # - # hash option -specific tests - # - - # test hash option key parsing - result = dict(all__vary_rounds=0.1) - self.assertEqual(parse(all__vary_rounds=0.1), result) - self.assertEqual(parse(default__all__vary_rounds=0.1), result) - self.assertEqual(parse(**{"all.vary_rounds":0.1}), result) - self.assertEqual(parse(**{"default.all.vary_rounds":0.1}), result) - - # test hash option key parsing w/ category - result = dict(admin__all__vary_rounds=0.1) - self.assertEqual(parse(admin__all__vary_rounds=0.1), result) - self.assertEqual(parse(**{"admin.all.vary_rounds":0.1}), result) - - # settings not allowed if not in hash.setting_kwds - ctx = CryptContext(["phpass", "md5_crypt"], phpass__ident="P") - self.assertRaises(KeyError, ctx.copy, md5_crypt__ident="P") - - # hash options 'salt' and 'rounds' not allowed - self.assertRaises(KeyError, CryptContext, schemes=["des_crypt"], - des_crypt__salt="xx") - self.assertRaises(KeyError, CryptContext, schemes=["des_crypt"], - all__salt="xx") - - def test_21_schemes(self): - """test 'schemes' context option parsing""" - - # schemes can be empty - cc = CryptContext(schemes=None) - self.assertEqual(cc.schemes(), ()) - - # schemes can be list of names - cc = CryptContext(schemes=["des_crypt", "md5_crypt"]) - self.assertEqual(cc.schemes(), ("des_crypt", "md5_crypt")) - - # schemes can be comma-sep string - cc = CryptContext(schemes=" des_crypt, md5_crypt, ") - self.assertEqual(cc.schemes(), ("des_crypt", "md5_crypt")) - - # schemes can be list of handlers - cc = CryptContext(schemes=[hash.des_crypt, hash.md5_crypt]) - self.assertEqual(cc.schemes(), ("des_crypt", "md5_crypt")) - - # scheme must be name or handler - self.assertRaises(TypeError, CryptContext, schemes=[uh.StaticHandler]) - - # handlers must have a name - class nameless(uh.StaticHandler): - name = None - self.assertRaises(ValueError, CryptContext, schemes=[nameless]) - - # names must be unique - class dummy_1(uh.StaticHandler): - name = 'dummy_1' - self.assertRaises(KeyError, CryptContext, schemes=[dummy_1, dummy_1]) - - # schemes not allowed per-category - self.assertRaises(KeyError, CryptContext, - admin__context__schemes=["md5_crypt"]) - - def test_22_deprecated(self): - """test 'deprecated' context option parsing""" - def getdep(ctx, category=None): - return [name for name in ctx.schemes() - if ctx.handler(name, category).deprecated] - - # no schemes - all deprecated values allowed - cc = CryptContext(deprecated=["md5_crypt"]) - cc.update(schemes=["md5_crypt", "des_crypt"]) - self.assertEqual(getdep(cc),["md5_crypt"]) - - # deprecated values allowed if subset of schemes - cc = CryptContext(deprecated=["md5_crypt"], schemes=["md5_crypt", "des_crypt"]) - self.assertEqual(getdep(cc), ["md5_crypt"]) - - # can be handler - # XXX: allow handlers in deprecated list? not for now. - self.assertRaises(TypeError, CryptContext, deprecated=[hash.md5_crypt], - schemes=["md5_crypt", "des_crypt"]) -## cc = CryptContext(deprecated=[hash.md5_crypt], schemes=["md5_crypt", "des_crypt"]) -## self.assertEqual(getdep(cc), ["md5_crypt"]) - - # comma sep list - cc = CryptContext(deprecated="md5_crypt,des_crypt", schemes=["md5_crypt", "des_crypt", "sha256_crypt"]) - self.assertEqual(getdep(cc), ["md5_crypt", "des_crypt"]) - - # values outside of schemes not allowed - self.assertRaises(KeyError, CryptContext, schemes=['des_crypt'], - deprecated=['md5_crypt']) - - # deprecating ALL schemes should cause ValueError - self.assertRaises(ValueError, CryptContext, - schemes=['des_crypt'], - deprecated=['des_crypt']) - self.assertRaises(ValueError, CryptContext, - schemes=['des_crypt', 'md5_crypt'], - admin__context__deprecated=['des_crypt', 'md5_crypt']) - - # deprecating explicit default scheme should cause ValueError - - # ... default listed as deprecated - self.assertRaises(ValueError, CryptContext, - schemes=['des_crypt', 'md5_crypt'], - default="md5_crypt", - deprecated="md5_crypt") - - # ... global default deprecated per-category - self.assertRaises(ValueError, CryptContext, - schemes=['des_crypt', 'md5_crypt'], - default="md5_crypt", - admin__context__deprecated="md5_crypt") - - # ... category default deprecated globally - self.assertRaises(ValueError, CryptContext, - schemes=['des_crypt', 'md5_crypt'], - admin__context__default="md5_crypt", - deprecated="md5_crypt") - - # ... category default deprecated in category - self.assertRaises(ValueError, CryptContext, - schemes=['des_crypt', 'md5_crypt'], - admin__context__default="md5_crypt", - admin__context__deprecated="md5_crypt") - - # category deplist should shadow default deplist - CryptContext( - schemes=['des_crypt', 'md5_crypt'], - deprecated="md5_crypt", - admin__context__default="md5_crypt", - admin__context__deprecated=[]) - - # wrong type - self.assertRaises(TypeError, CryptContext, deprecated=123) - - # deprecated per-category - cc = CryptContext(deprecated=["md5_crypt"], - schemes=["md5_crypt", "des_crypt"], - admin__context__deprecated=["des_crypt"], - ) - self.assertEqual(getdep(cc), ["md5_crypt"]) - self.assertEqual(getdep(cc, "user"), ["md5_crypt"]) - self.assertEqual(getdep(cc, "admin"), ["des_crypt"]) - - # blank per-category deprecated list, shadowing default list - cc = CryptContext(deprecated=["md5_crypt"], - schemes=["md5_crypt", "des_crypt"], - admin__context__deprecated=[], - ) - self.assertEqual(getdep(cc), ["md5_crypt"]) - self.assertEqual(getdep(cc, "user"), ["md5_crypt"]) - self.assertEqual(getdep(cc, "admin"), []) - - def test_23_default(self): - """test 'default' context option parsing""" - - # anything allowed if no schemes - self.assertEqual(CryptContext(default="md5_crypt").to_dict(), - dict(default="md5_crypt")) - - # default allowed if in scheme list - ctx = CryptContext(default="md5_crypt", schemes=["des_crypt", "md5_crypt"]) - self.assertEqual(ctx.default_scheme(), "md5_crypt") - - # default can be handler - # XXX: sure we want to allow this ? maybe deprecate in future. - ctx = CryptContext(default=hash.md5_crypt, schemes=["des_crypt", "md5_crypt"]) - self.assertEqual(ctx.default_scheme(), "md5_crypt") - - # implicit default should be first non-deprecated scheme - ctx = CryptContext(schemes=["des_crypt", "md5_crypt"]) - self.assertEqual(ctx.default_scheme(), "des_crypt") - ctx.update(deprecated="des_crypt") - self.assertEqual(ctx.default_scheme(), "md5_crypt") - - # error if not in scheme list - self.assertRaises(KeyError, CryptContext, schemes=['des_crypt'], - default='md5_crypt') - - # wrong type - self.assertRaises(TypeError, CryptContext, default=1) - - # per-category - ctx = CryptContext(default="des_crypt", - schemes=["des_crypt", "md5_crypt"], - admin__context__default="md5_crypt") - self.assertEqual(ctx.default_scheme(), "des_crypt") - self.assertEqual(ctx.default_scheme("user"), "des_crypt") - self.assertEqual(ctx.default_scheme("admin"), "md5_crypt") - - def test_24_vary_rounds(self): - """test 'vary_rounds' hash option parsing""" - def parse(v): - return CryptContext(all__vary_rounds=v).to_dict()['all__vary_rounds'] - - # floats should be preserved - self.assertEqual(parse(0.1), 0.1) - self.assertEqual(parse('0.1'), 0.1) - - # 'xx%' should be converted to float - self.assertEqual(parse('10%'), 0.1) - - # ints should be preserved - self.assertEqual(parse(1000), 1000) - self.assertEqual(parse('1000'), 1000) - - #=================================================================== - # inspection & serialization - #=================================================================== - - def assertHandlerDerivedFrom(self, handler, base, msg=None): - self.assertTrue(handler_derived_from(handler, base), msg=msg) - - def test_30_schemes(self): - """test schemes() method""" - # NOTE: also checked under test_21 - - # test empty - ctx = CryptContext() - self.assertEqual(ctx.schemes(), ()) - self.assertEqual(ctx.schemes(resolve=True), ()) - - # test sample 1 - ctx = CryptContext(**self.sample_1_dict) - self.assertEqual(ctx.schemes(), tuple(self.sample_1_schemes)) - self.assertEqual(ctx.schemes(resolve=True, unconfigured=True), tuple(self.sample_1_handlers)) - for result, correct in zip(ctx.schemes(resolve=True), self.sample_1_handlers): - self.assertTrue(handler_derived_from(result, correct)) - - # test sample 2 - ctx = CryptContext(**self.sample_2_dict) - self.assertEqual(ctx.schemes(), ()) - - def test_31_default_scheme(self): - """test default_scheme() method""" - # NOTE: also checked under test_23 - - # test empty - ctx = CryptContext() - self.assertRaises(KeyError, ctx.default_scheme) - - # test sample 1 - ctx = CryptContext(**self.sample_1_dict) - self.assertEqual(ctx.default_scheme(), "md5_crypt") - self.assertEqual(ctx.default_scheme(resolve=True, unconfigured=True), hash.md5_crypt) - self.assertHandlerDerivedFrom(ctx.default_scheme(resolve=True), hash.md5_crypt) - - # test sample 2 - ctx = CryptContext(**self.sample_2_dict) - self.assertRaises(KeyError, ctx.default_scheme) - - # test defaults to first in scheme - ctx = CryptContext(schemes=self.sample_1_schemes) - self.assertEqual(ctx.default_scheme(), "des_crypt") - - # categories tested under test_23 - - def test_32_handler(self): - """test handler() method""" - - # default for empty - ctx = CryptContext() - self.assertRaises(KeyError, ctx.handler) - self.assertRaises(KeyError, ctx.handler, "md5_crypt") - - # default for sample 1 - ctx = CryptContext(**self.sample_1_dict) - self.assertEqual(ctx.handler(unconfigured=True), hash.md5_crypt) - self.assertHandlerDerivedFrom(ctx.handler(), hash.md5_crypt) - - # by name - self.assertEqual(ctx.handler("des_crypt", unconfigured=True), hash.des_crypt) - self.assertHandlerDerivedFrom(ctx.handler("des_crypt"), hash.des_crypt) - - # name not in schemes - self.assertRaises(KeyError, ctx.handler, "mysql323") - - # check handler() honors category default - ctx = CryptContext("sha256_crypt,md5_crypt", admin__context__default="md5_crypt") - self.assertEqual(ctx.handler(unconfigured=True), hash.sha256_crypt) - self.assertHandlerDerivedFrom(ctx.handler(), hash.sha256_crypt) - - self.assertEqual(ctx.handler(category="staff", unconfigured=True), hash.sha256_crypt) - self.assertHandlerDerivedFrom(ctx.handler(category="staff"), hash.sha256_crypt) - - self.assertEqual(ctx.handler(category="admin", unconfigured=True), hash.md5_crypt) - self.assertHandlerDerivedFrom(ctx.handler(category="staff"), hash.sha256_crypt) - - # test unicode category strings are accepted under py2 - if PY2: - self.assertEqual(ctx.handler(category=u("staff"), unconfigured=True), hash.sha256_crypt) - self.assertEqual(ctx.handler(category=u("admin"), unconfigured=True), hash.md5_crypt) - - def test_33_options(self): - """test internal _get_record_options() method""" - - def options(ctx, scheme, category=None): - return ctx._config._get_record_options_with_flag(scheme, category)[0] - - # this checks that (3 schemes, 3 categories) inherit options correctly. - # the 'user' category is not present in the options. - cc4 = CryptContext( - truncate_error=True, - schemes = [ "sha512_crypt", "des_crypt", "bsdi_crypt"], - deprecated = ["sha512_crypt", "des_crypt"], - all__vary_rounds = 0.1, - bsdi_crypt__vary_rounds=0.2, - sha512_crypt__max_rounds = 20000, - admin__context__deprecated = [ "des_crypt", "bsdi_crypt" ], - admin__all__vary_rounds = 0.05, - admin__bsdi_crypt__vary_rounds=0.3, - admin__sha512_crypt__max_rounds = 40000, - ) - self.assertEqual(cc4._config.categories, ("admin",)) - - # - # sha512_crypt - # NOTE: 'truncate_error' shouldn't be passed along... - # - self.assertEqual(options(cc4, "sha512_crypt"), dict( - deprecated=True, - vary_rounds=0.1, # inherited from all__ - max_rounds=20000, - )) - - self.assertEqual(options(cc4, "sha512_crypt", "user"), dict( - deprecated=True, # unconfigured category inherits from default - vary_rounds=0.1, - max_rounds=20000, - )) - - self.assertEqual(options(cc4, "sha512_crypt", "admin"), dict( - # NOT deprecated - context option overridden per-category - vary_rounds=0.05, # global overridden per-cateogry - max_rounds=40000, # overridden per-category - )) - - # - # des_crypt - # NOTE: vary_rounds shouldn't be passed along... - # - self.assertEqual(options(cc4, "des_crypt"), dict( - deprecated=True, - truncate_error=True, - )) - - self.assertEqual(options(cc4, "des_crypt", "user"), dict( - deprecated=True, # unconfigured category inherits from default - truncate_error=True, - )) - - self.assertEqual(options(cc4, "des_crypt", "admin"), dict( - deprecated=True, # unchanged though overidden - truncate_error=True, - )) - - # - # bsdi_crypt - # - self.assertEqual(options(cc4, "bsdi_crypt"), dict( - vary_rounds=0.2, # overridden from all__vary_rounds - )) - - self.assertEqual(options(cc4, "bsdi_crypt", "user"), dict( - vary_rounds=0.2, # unconfigured category inherits from default - )) - - self.assertEqual(options(cc4, "bsdi_crypt", "admin"), dict( - vary_rounds=0.3, - deprecated=True, # deprecation set per-category - )) - - def test_34_to_dict(self): - """test to_dict() method""" - # NOTE: this is tested all throughout this test case. - ctx = CryptContext(**self.sample_1_dict) - self.assertEqual(ctx.to_dict(), self.sample_1_dict) - self.assertEqual(ctx.to_dict(resolve=True), self.sample_1_resolved_dict) - - def test_35_to_string(self): - """test to_string() method""" - - # create ctx and serialize - ctx = CryptContext(**self.sample_1_dict) - dump = ctx.to_string() - - # check ctx->string returns canonical format. - # NOTE: ConfigParser for PY26 doesn't use OrderedDict, - # making to_string()'s ordering unpredictable... - # so we skip this test under PY26. - if not PY26: - self.assertEqual(dump, self.sample_1_unicode) - - # check ctx->string->ctx->dict returns original - ctx2 = CryptContext.from_string(dump) - self.assertEqual(ctx2.to_dict(), self.sample_1_dict) - - # test section kwd is honored - other = ctx.to_string(section="password-security") - self.assertEqual(other, dump.replace("[passlib]","[password-security]")) - - # test unmanaged handler warning - from passlib.tests.test_utils_handlers import UnsaltedHash - ctx3 = CryptContext([UnsaltedHash, "md5_crypt"]) - dump = ctx3.to_string() - self.assertRegex(dump, r"# NOTE: the 'unsalted_test_hash' handler\(s\)" - r" are not registered with Passlib") - - #=================================================================== - # password hash api - #=================================================================== - nonstring_vectors = [ - (None, {}), - (None, {"scheme": "des_crypt"}), - (1, {}), - ((), {}), - ] - - def test_40_basic(self): - """test basic hash/identify/verify functionality""" - handlers = [hash.md5_crypt, hash.des_crypt, hash.bsdi_crypt] - cc = CryptContext(handlers, bsdi_crypt__default_rounds=5) - - # run through handlers - for crypt in handlers: - h = cc.hash("test", scheme=crypt.name) - self.assertEqual(cc.identify(h), crypt.name) - self.assertEqual(cc.identify(h, resolve=True, unconfigured=True), crypt) - self.assertHandlerDerivedFrom(cc.identify(h, resolve=True), crypt) - self.assertTrue(cc.verify('test', h)) - self.assertFalse(cc.verify('notest', h)) - - # test default - h = cc.hash("test") - self.assertEqual(cc.identify(h), "md5_crypt") - - # test genhash - h = cc.genhash('secret', cc.genconfig()) - self.assertEqual(cc.identify(h), 'md5_crypt') - - h = cc.genhash('secret', cc.genconfig(), scheme='md5_crypt') - self.assertEqual(cc.identify(h), 'md5_crypt') - - self.assertRaises(ValueError, cc.genhash, 'secret', cc.genconfig(), scheme="des_crypt") - - def test_41_genconfig(self): - """test genconfig() method""" - cc = CryptContext(schemes=["md5_crypt", "phpass"], - phpass__ident="H", - phpass__default_rounds=7, - admin__phpass__ident="P", - ) - - # uses default scheme - self.assertTrue(cc.genconfig().startswith("$1$")) - - # override scheme - self.assertTrue(cc.genconfig(scheme="phpass").startswith("$H$5")) - - # category override - self.assertTrue(cc.genconfig(scheme="phpass", category="admin").startswith("$P$5")) - self.assertTrue(cc.genconfig(scheme="phpass", category="staff").startswith("$H$5")) - - # override scheme & custom settings - self.assertEqual( - cc.genconfig(scheme="phpass", salt='.'*8, rounds=8, ident='P'), - '$P$6........22zGEuacuPOqEpYPDeR0R/', # NOTE: config string generated w/ rounds=1 - ) - - #-------------------------------------------------------------- - # border cases - #-------------------------------------------------------------- - - # test unicode category strings are accepted under py2 - # this tests basic _get_record() used by hash/genhash/verify. - # we have to omit scheme=xxx so codepath is tested fully - if PY2: - c2 = cc.copy(default="phpass") - self.assertTrue(c2.genconfig(category=u("admin")).startswith("$P$5")) - self.assertTrue(c2.genconfig(category=u("staff")).startswith("$H$5")) - - # throws error without schemes - self.assertRaises(KeyError, CryptContext().genconfig) - self.assertRaises(KeyError, CryptContext().genconfig, scheme='md5_crypt') - - # bad scheme values - self.assertRaises(KeyError, cc.genconfig, scheme="fake") # XXX: should this be ValueError? - self.assertRaises(TypeError, cc.genconfig, scheme=1, category='staff') - self.assertRaises(TypeError, cc.genconfig, scheme=1) - - # bad category values - self.assertRaises(TypeError, cc.genconfig, category=1) - - - def test_42_genhash(self): - """test genhash() method""" - - #-------------------------------------------------------------- - # border cases - #-------------------------------------------------------------- - - # rejects non-string secrets - cc = CryptContext(["des_crypt"]) - hash = cc.hash('stub') - for secret, kwds in self.nonstring_vectors: - self.assertRaises(TypeError, cc.genhash, secret, hash, **kwds) - - # rejects non-string config strings - cc = CryptContext(["des_crypt"]) - for config, kwds in self.nonstring_vectors: - if hash is None: - # NOTE: as of 1.7, genhash is just wrapper for hash(), - # and handles genhash(secret, None) fine. - continue - self.assertRaises(TypeError, cc.genhash, 'secret', config, **kwds) - - # rejects config=None, even if default scheme lacks config string - cc = CryptContext(["mysql323"]) - self.assertRaises(TypeError, cc.genhash, "stub", None) - - # throws error without schemes - self.assertRaises(KeyError, CryptContext().genhash, 'secret', 'hash') - - # bad scheme values - self.assertRaises(KeyError, cc.genhash, 'secret', hash, scheme="fake") # XXX: should this be ValueError? - self.assertRaises(TypeError, cc.genhash, 'secret', hash, scheme=1) - - # bad category values - self.assertRaises(TypeError, cc.genconfig, 'secret', hash, category=1) - - def test_43_hash(self,): - """test hash() method""" - # XXX: what more can we test here that isn't deprecated - # or handled under another test (e.g. context kwds?) - - # respects rounds - cc = CryptContext(**self.sample_4_dict) - hash = cc.hash("password") - self.assertTrue(hash.startswith("$5$rounds=3000$")) - self.assertTrue(cc.verify("password", hash)) - self.assertFalse(cc.verify("passwordx", hash)) - - # make default > max throws error if attempted - # XXX: move this to copy() test? - self.assertRaises(ValueError, cc.copy, - sha256_crypt__default_rounds=4000) - - # rejects non-string secrets - cc = CryptContext(["des_crypt"]) - for secret, kwds in self.nonstring_vectors: - self.assertRaises(TypeError, cc.hash, secret, **kwds) - - # throws error without schemes - self.assertRaises(KeyError, CryptContext().hash, 'secret') - - # bad category values - self.assertRaises(TypeError, cc.hash, 'secret', category=1) - - def test_43_hash_legacy(self, use_16_legacy=False): - """test hash() method -- legacy 'scheme' and settings keywords""" - cc = CryptContext(**self.sample_4_dict) - - # TODO: should migrate these tests elsewhere, or remove them. - # can be replaced with following equivalent: - # - # def wrapper(secret, scheme=None, category=None, **kwds): - # handler = cc.handler(scheme, category) - # if kwds: - # handler = handler.using(**kwds) - # return handler.hash(secret) - # - # need to make sure bits being tested here are tested - # under the tests for the equivalent methods called above, - # and then discard the rest of these under 2.0. - - # hash specific settings - with self.assertWarningList(["passing settings to.*is deprecated"]): - self.assertEqual( - cc.hash("password", scheme="phpass", salt='.'*8), - '$H$5........De04R5Egz0aq8Tf.1eVhY/', - ) - with self.assertWarningList(["passing settings to.*is deprecated"]): - self.assertEqual( - cc.hash("password", scheme="phpass", salt='.'*8, ident="P"), - '$P$5........De04R5Egz0aq8Tf.1eVhY/', - ) - - # NOTE: more thorough job of rounds limits done below. - - # min rounds - with self.assertWarningList(["passing settings to.*is deprecated"]): - self.assertEqual( - cc.hash("password", rounds=1999, salt="nacl"), - '$5$rounds=1999$nacl$nmfwJIxqj0csloAAvSER0B8LU0ERCAbhmMug4Twl609', - ) - - with self.assertWarningList(["passing settings to.*is deprecated"]): - self.assertEqual( - cc.hash("password", rounds=2001, salt="nacl"), - '$5$rounds=2001$nacl$8PdeoPL4aXQnJ0woHhqgIw/efyfCKC2WHneOpnvF.31' - ) - # NOTE: max rounds, etc tested in genconfig() - - # bad scheme values - self.assertRaises(KeyError, cc.hash, 'secret', scheme="fake") # XXX: should this be ValueError? - self.assertRaises(TypeError, cc.hash, 'secret', scheme=1) - - def test_44_identify(self): - """test identify() border cases""" - handlers = ["md5_crypt", "des_crypt", "bsdi_crypt"] - cc = CryptContext(handlers, bsdi_crypt__default_rounds=5) - - # check unknown hash - self.assertEqual(cc.identify('$9$232323123$1287319827'), None) - self.assertRaises(ValueError, cc.identify, '$9$232323123$1287319827', required=True) - - #-------------------------------------------------------------- - # border cases - #-------------------------------------------------------------- - - # rejects non-string hashes - cc = CryptContext(["des_crypt"]) - for hash, kwds in self.nonstring_vectors: - self.assertRaises(TypeError, cc.identify, hash, **kwds) - - # throws error without schemes - cc = CryptContext() - self.assertIs(cc.identify('hash'), None) - self.assertRaises(KeyError, cc.identify, 'hash', required=True) - - # bad category values - self.assertRaises(TypeError, cc.identify, None, category=1) - - def test_45_verify(self): - """test verify() scheme kwd""" - handlers = ["md5_crypt", "des_crypt", "bsdi_crypt"] - cc = CryptContext(handlers, bsdi_crypt__default_rounds=5) - - h = hash.md5_crypt.hash("test") - - # check base verify - self.assertTrue(cc.verify("test", h)) - self.assertTrue(not cc.verify("notest", h)) - - # check verify using right alg - self.assertTrue(cc.verify('test', h, scheme='md5_crypt')) - self.assertTrue(not cc.verify('notest', h, scheme='md5_crypt')) - - # check verify using wrong alg - self.assertRaises(ValueError, cc.verify, 'test', h, scheme='bsdi_crypt') - - #-------------------------------------------------------------- - # border cases - #-------------------------------------------------------------- - - # unknown hash should throw error - self.assertRaises(ValueError, cc.verify, 'stub', '$6$232323123$1287319827') - - # rejects non-string secrets - cc = CryptContext(["des_crypt"]) - h = refhash = cc.hash('stub') - for secret, kwds in self.nonstring_vectors: - self.assertRaises(TypeError, cc.verify, secret, h, **kwds) - - # always treat hash=None as False - self.assertFalse(cc.verify(secret, None)) - - # rejects non-string hashes - cc = CryptContext(["des_crypt"]) - for h, kwds in self.nonstring_vectors: - if h is None: - continue - self.assertRaises(TypeError, cc.verify, 'secret', h, **kwds) - - # throws error without schemes - self.assertRaises(KeyError, CryptContext().verify, 'secret', 'hash') - - # bad scheme values - self.assertRaises(KeyError, cc.verify, 'secret', refhash, scheme="fake") # XXX: should this be ValueError? - self.assertRaises(TypeError, cc.verify, 'secret', refhash, scheme=1) - - # bad category values - self.assertRaises(TypeError, cc.verify, 'secret', refhash, category=1) - - def test_46_needs_update(self): - """test needs_update() method""" - cc = CryptContext(**self.sample_4_dict) - - # check deprecated scheme - self.assertTrue(cc.needs_update('9XXD4trGYeGJA')) - self.assertFalse(cc.needs_update('$1$J8HC2RCr$HcmM.7NxB2weSvlw2FgzU0')) - - # check min rounds - self.assertTrue(cc.needs_update('$5$rounds=1999$jD81UCoo.zI.UETs$Y7qSTQ6mTiU9qZB4fRr43wRgQq4V.5AAf7F97Pzxey/')) - self.assertFalse(cc.needs_update('$5$rounds=2000$228SSRje04cnNCaQ$YGV4RYu.5sNiBvorQDlO0WWQjyJVGKBcJXz3OtyQ2u8')) - - # check max rounds - self.assertFalse(cc.needs_update('$5$rounds=3000$fS9iazEwTKi7QPW4$VasgBC8FqlOvD7x2HhABaMXCTh9jwHclPA9j5YQdns.')) - self.assertTrue(cc.needs_update('$5$rounds=3001$QlFHHifXvpFX4PLs$/0ekt7lSs/lOikSerQ0M/1porEHxYq7W/2hdFpxA3fA')) - - #-------------------------------------------------------------- - # test hash.needs_update() interface - #-------------------------------------------------------------- - check_state = [] - class dummy(uh.StaticHandler): - name = 'dummy' - _hash_prefix = '@' - - @classmethod - def needs_update(cls, hash, secret=None): - check_state.append((hash, secret)) - return secret == "nu" - - def _calc_checksum(self, secret): - from hashlib import md5 - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return str_to_uascii(md5(secret).hexdigest()) - - # calling needs_update should query callback - ctx = CryptContext([dummy]) - hash = refhash = dummy.hash("test") - self.assertFalse(ctx.needs_update(hash)) - self.assertEqual(check_state, [(hash,None)]) - del check_state[:] - - # now with a password - self.assertFalse(ctx.needs_update(hash, secret='bob')) - self.assertEqual(check_state, [(hash,'bob')]) - del check_state[:] - - # now when it returns True - self.assertTrue(ctx.needs_update(hash, secret='nu')) - self.assertEqual(check_state, [(hash,'nu')]) - del check_state[:] - - #-------------------------------------------------------------- - # border cases - #-------------------------------------------------------------- - - # rejects non-string hashes - cc = CryptContext(["des_crypt"]) - for hash, kwds in self.nonstring_vectors: - self.assertRaises(TypeError, cc.needs_update, hash, **kwds) - - # throws error without schemes - self.assertRaises(KeyError, CryptContext().needs_update, 'hash') - - # bad scheme values - self.assertRaises(KeyError, cc.needs_update, refhash, scheme="fake") # XXX: should this be ValueError? - self.assertRaises(TypeError, cc.needs_update, refhash, scheme=1) - - # bad category values - self.assertRaises(TypeError, cc.needs_update, refhash, category=1) - - def test_47_verify_and_update(self): - """test verify_and_update()""" - cc = CryptContext(**self.sample_4_dict) - - # create some hashes - h1 = cc.handler("des_crypt").hash("password") - h2 = cc.handler("sha256_crypt").hash("password") - - # check bad password, deprecated hash - ok, new_hash = cc.verify_and_update("wrongpass", h1) - self.assertFalse(ok) - self.assertIs(new_hash, None) - - # check bad password, good hash - ok, new_hash = cc.verify_and_update("wrongpass", h2) - self.assertFalse(ok) - self.assertIs(new_hash, None) - - # check right password, deprecated hash - ok, new_hash = cc.verify_and_update("password", h1) - self.assertTrue(ok) - self.assertTrue(cc.identify(new_hash), "sha256_crypt") - - # check right password, good hash - ok, new_hash = cc.verify_and_update("password", h2) - self.assertTrue(ok) - self.assertIs(new_hash, None) - - #-------------------------------------------------------------- - # border cases - #-------------------------------------------------------------- - - # rejects non-string secrets - cc = CryptContext(["des_crypt"]) - hash = refhash = cc.hash('stub') - for secret, kwds in self.nonstring_vectors: - self.assertRaises(TypeError, cc.verify_and_update, secret, hash, **kwds) - - # always treat hash=None as False - self.assertEqual(cc.verify_and_update(secret, None), (False, None)) - - # rejects non-string hashes - cc = CryptContext(["des_crypt"]) - for hash, kwds in self.nonstring_vectors: - if hash is None: - continue - self.assertRaises(TypeError, cc.verify_and_update, 'secret', hash, **kwds) - - # throws error without schemes - self.assertRaises(KeyError, CryptContext().verify_and_update, 'secret', 'hash') - - # bad scheme values - self.assertRaises(KeyError, cc.verify_and_update, 'secret', refhash, scheme="fake") # XXX: should this be ValueError? - self.assertRaises(TypeError, cc.verify_and_update, 'secret', refhash, scheme=1) - - # bad category values - self.assertRaises(TypeError, cc.verify_and_update, 'secret', refhash, category=1) - - def test_48_context_kwds(self): - """hash(), verify(), and verify_and_update() -- discard unused context keywords""" - - # setup test case - # NOTE: postgres_md5 hash supports 'user' context kwd, which is used for this test. - from passlib.hash import des_crypt, md5_crypt, postgres_md5 - des_hash = des_crypt.hash("stub") - pg_root_hash = postgres_md5.hash("stub", user="root") - pg_admin_hash = postgres_md5.hash("stub", user="admin") - - #------------------------------------------------------------ - # case 1: contextual kwds not supported by any hash in CryptContext - #------------------------------------------------------------ - cc1 = CryptContext([des_crypt, md5_crypt]) - self.assertEqual(cc1.context_kwds, set()) - - # des_scrypt should work w/o any contextual kwds - self.assertTrue(des_crypt.identify(cc1.hash("stub")), "des_crypt") - self.assertTrue(cc1.verify("stub", des_hash)) - self.assertEqual(cc1.verify_and_update("stub", des_hash), (True, None)) - - # des_crypt should throw error due to unknown context keyword - with self.assertWarningList(["passing settings to.*is deprecated"]): - self.assertRaises(TypeError, cc1.hash, "stub", user="root") - self.assertRaises(TypeError, cc1.verify, "stub", des_hash, user="root") - self.assertRaises(TypeError, cc1.verify_and_update, "stub", des_hash, user="root") - - #------------------------------------------------------------ - # case 2: at least one contextual kwd supported by non-default hash - #------------------------------------------------------------ - cc2 = CryptContext([des_crypt, postgres_md5]) - self.assertEqual(cc2.context_kwds, set(["user"])) - - # verify des_crypt works w/o "user" kwd - self.assertTrue(des_crypt.identify(cc2.hash("stub")), "des_crypt") - self.assertTrue(cc2.verify("stub", des_hash)) - self.assertEqual(cc2.verify_and_update("stub", des_hash), (True, None)) - - # verify des_crypt ignores "user" kwd - self.assertTrue(des_crypt.identify(cc2.hash("stub", user="root")), "des_crypt") - self.assertTrue(cc2.verify("stub", des_hash, user="root")) - self.assertEqual(cc2.verify_and_update("stub", des_hash, user="root"), (True, None)) - - # verify error with unknown kwd - with self.assertWarningList(["passing settings to.*is deprecated"]): - self.assertRaises(TypeError, cc2.hash, "stub", badkwd="root") - self.assertRaises(TypeError, cc2.verify, "stub", des_hash, badkwd="root") - self.assertRaises(TypeError, cc2.verify_and_update, "stub", des_hash, badkwd="root") - - #------------------------------------------------------------ - # case 3: at least one contextual kwd supported by default hash - #------------------------------------------------------------ - cc3 = CryptContext([postgres_md5, des_crypt], deprecated="auto") - self.assertEqual(cc3.context_kwds, set(["user"])) - - # postgres_md5 should have error w/o context kwd - self.assertRaises(TypeError, cc3.hash, "stub") - self.assertRaises(TypeError, cc3.verify, "stub", pg_root_hash) - self.assertRaises(TypeError, cc3.verify_and_update, "stub", pg_root_hash) - - # postgres_md5 should work w/ context kwd - self.assertEqual(cc3.hash("stub", user="root"), pg_root_hash) - self.assertTrue(cc3.verify("stub", pg_root_hash, user="root")) - self.assertEqual(cc3.verify_and_update("stub", pg_root_hash, user="root"), (True, None)) - - # verify_and_update() should fail against wrong user - self.assertEqual(cc3.verify_and_update("stub", pg_root_hash, user="admin"), (False, None)) - - # verify_and_update() should pass all context kwds through when rehashing - self.assertEqual(cc3.verify_and_update("stub", des_hash, user="root"), - (True, pg_root_hash)) - - #=================================================================== - # rounds options - #=================================================================== - - # TODO: now that rounds generation has moved out of _CryptRecord to HasRounds, - # this should just test that we're passing right options to handler.using(), - # and that resulting handler has right settings. - # Can then just let HasRounds tests (which are a copy of this) deal with things. - - # NOTE: the follow tests check how _CryptRecord handles - # the min/max/default/vary_rounds options, via the output of - # genconfig(). it's assumed hash() takes the same codepath. - - def test_50_rounds_limits(self): - """test rounds limits""" - cc = CryptContext(schemes=["sha256_crypt"], - sha256_crypt__min_rounds=2000, - sha256_crypt__max_rounds=3000, - sha256_crypt__default_rounds=2500, - ) - - # stub digest returned by sha256_crypt's genconfig calls.. - STUB = '...........................................' - - #-------------------------------------------------- - # settings should have been applied to custom handler, - # it should take care of the rest - #-------------------------------------------------- - custom_handler = cc._get_record("sha256_crypt", None) - self.assertEqual(custom_handler.min_desired_rounds, 2000) - self.assertEqual(custom_handler.max_desired_rounds, 3000) - self.assertEqual(custom_handler.default_rounds, 2500) - - #-------------------------------------------------- - # min_rounds - #-------------------------------------------------- - - # set below handler minimum - with self.assertWarningList([PasslibHashWarning]*2): - c2 = cc.copy(sha256_crypt__min_rounds=500, sha256_crypt__max_rounds=None, - sha256_crypt__default_rounds=500) - self.assertEqual(c2.genconfig(salt="nacl"), "$5$rounds=1000$nacl$" + STUB) - - # below policy minimum - # NOTE: formerly issued a warning in passlib 1.6, now just a wrapper for .replace() - with self.assertWarningList([]): - self.assertEqual( - cc.genconfig(rounds=1999, salt="nacl"), '$5$rounds=1999$nacl$' + STUB) - - # equal to policy minimum - self.assertEqual( - cc.genconfig(rounds=2000, salt="nacl"), '$5$rounds=2000$nacl$' + STUB) - - # above policy minimum - self.assertEqual( - cc.genconfig(rounds=2001, salt="nacl"), '$5$rounds=2001$nacl$' + STUB) - - #-------------------------------------------------- - # max rounds - #-------------------------------------------------- - - # set above handler max - with self.assertWarningList([PasslibHashWarning]*2): - c2 = cc.copy(sha256_crypt__max_rounds=int(1e9)+500, sha256_crypt__min_rounds=None, - sha256_crypt__default_rounds=int(1e9)+500) - - self.assertEqual(c2.genconfig(salt="nacl"), "$5$rounds=999999999$nacl$" + STUB) - - # above policy max - # NOTE: formerly issued a warning in passlib 1.6, now just a wrapper for .using() - with self.assertWarningList([]): - self.assertEqual( - cc.genconfig(rounds=3001, salt="nacl"), '$5$rounds=3001$nacl$' + STUB) - - # equal policy max - self.assertEqual( - cc.genconfig(rounds=3000, salt="nacl"), '$5$rounds=3000$nacl$' + STUB) - - # below policy max - self.assertEqual( - cc.genconfig(rounds=2999, salt="nacl"), '$5$rounds=2999$nacl$' + STUB) - - #-------------------------------------------------- - # default_rounds - #-------------------------------------------------- - - # explicit default rounds - self.assertEqual(cc.genconfig(salt="nacl"), '$5$rounds=2500$nacl$' + STUB) - - # fallback default rounds - use handler's - df = hash.sha256_crypt.default_rounds - c2 = cc.copy(sha256_crypt__default_rounds=None, sha256_crypt__max_rounds=df<<1) - self.assertEqual(c2.genconfig(salt="nacl"), '$5$rounds=%d$nacl$%s' % (df, STUB)) - - # fallback default rounds - use handler's, but clipped to max rounds - c2 = cc.copy(sha256_crypt__default_rounds=None, sha256_crypt__max_rounds=3000) - self.assertEqual(c2.genconfig(salt="nacl"), '$5$rounds=3000$nacl$' + STUB) - - # TODO: test default falls back to mx / mn if handler has no default. - - # default rounds - out of bounds - self.assertRaises(ValueError, cc.copy, sha256_crypt__default_rounds=1999) - cc.copy(sha256_crypt__default_rounds=2000) - cc.copy(sha256_crypt__default_rounds=3000) - self.assertRaises(ValueError, cc.copy, sha256_crypt__default_rounds=3001) - - #-------------------------------------------------- - # border cases - #-------------------------------------------------- - - # invalid min/max bounds - c2 = CryptContext(schemes=["sha256_crypt"]) - # NOTE: as of v1.7, these are clipped w/ a warning instead... - # self.assertRaises(ValueError, c2.copy, sha256_crypt__min_rounds=-1) - # self.assertRaises(ValueError, c2.copy, sha256_crypt__max_rounds=-1) - self.assertRaises(ValueError, c2.copy, sha256_crypt__min_rounds=2000, - sha256_crypt__max_rounds=1999) - - # test bad values - self.assertRaises(ValueError, CryptContext, sha256_crypt__min_rounds='x') - self.assertRaises(ValueError, CryptContext, sha256_crypt__max_rounds='x') - self.assertRaises(ValueError, CryptContext, all__vary_rounds='x') - self.assertRaises(ValueError, CryptContext, sha256_crypt__default_rounds='x') - - # test bad types rejected - bad = datetime.datetime.now() # picked cause can't be compared to int - self.assertRaises(TypeError, CryptContext, "sha256_crypt", sha256_crypt__min_rounds=bad) - self.assertRaises(TypeError, CryptContext, "sha256_crypt", sha256_crypt__max_rounds=bad) - self.assertRaises(TypeError, CryptContext, "sha256_crypt", all__vary_rounds=bad) - self.assertRaises(TypeError, CryptContext, "sha256_crypt", sha256_crypt__default_rounds=bad) - - def test_51_linear_vary_rounds(self): - """test linear vary rounds""" - cc = CryptContext(schemes=["sha256_crypt"], - sha256_crypt__min_rounds=1995, - sha256_crypt__max_rounds=2005, - sha256_crypt__default_rounds=2000, - ) - - # test negative - self.assertRaises(ValueError, cc.copy, all__vary_rounds=-1) - self.assertRaises(ValueError, cc.copy, all__vary_rounds="-1%") - self.assertRaises(ValueError, cc.copy, all__vary_rounds="101%") - - # test static - c2 = cc.copy(all__vary_rounds=0) - self.assertEqual(c2._get_record("sha256_crypt", None).vary_rounds, 0) - self.assert_rounds_range(c2, "sha256_crypt", 2000, 2000) - - c2 = cc.copy(all__vary_rounds="0%") - self.assertEqual(c2._get_record("sha256_crypt", None).vary_rounds, 0) - self.assert_rounds_range(c2, "sha256_crypt", 2000, 2000) - - # test absolute - c2 = cc.copy(all__vary_rounds=1) - self.assertEqual(c2._get_record("sha256_crypt", None).vary_rounds, 1) - self.assert_rounds_range(c2, "sha256_crypt", 1999, 2001) - c2 = cc.copy(all__vary_rounds=100) - self.assertEqual(c2._get_record("sha256_crypt", None).vary_rounds, 100) - self.assert_rounds_range(c2, "sha256_crypt", 1995, 2005) - - # test relative - c2 = cc.copy(all__vary_rounds="0.1%") - self.assertEqual(c2._get_record("sha256_crypt", None).vary_rounds, 0.001) - self.assert_rounds_range(c2, "sha256_crypt", 1998, 2002) - c2 = cc.copy(all__vary_rounds="100%") - self.assertEqual(c2._get_record("sha256_crypt", None).vary_rounds, 1.0) - self.assert_rounds_range(c2, "sha256_crypt", 1995, 2005) - - def test_52_log2_vary_rounds(self): - """test log2 vary rounds""" - cc = CryptContext(schemes=["bcrypt"], - bcrypt__min_rounds=15, - bcrypt__max_rounds=25, - bcrypt__default_rounds=20, - ) - - # test negative - self.assertRaises(ValueError, cc.copy, all__vary_rounds=-1) - self.assertRaises(ValueError, cc.copy, all__vary_rounds="-1%") - self.assertRaises(ValueError, cc.copy, all__vary_rounds="101%") - - # test static - c2 = cc.copy(all__vary_rounds=0) - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 0) - self.assert_rounds_range(c2, "bcrypt", 20, 20) - - c2 = cc.copy(all__vary_rounds="0%") - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 0) - self.assert_rounds_range(c2, "bcrypt", 20, 20) - - # test absolute - c2 = cc.copy(all__vary_rounds=1) - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 1) - self.assert_rounds_range(c2, "bcrypt", 19, 21) - c2 = cc.copy(all__vary_rounds=100) - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 100) - self.assert_rounds_range(c2, "bcrypt", 15, 25) - - # test relative - should shift over at 50% mark - c2 = cc.copy(all__vary_rounds="1%") - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 0.01) - self.assert_rounds_range(c2, "bcrypt", 20, 20) - - c2 = cc.copy(all__vary_rounds="49%") - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 0.49) - self.assert_rounds_range(c2, "bcrypt", 20, 20) - - c2 = cc.copy(all__vary_rounds="50%") - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 0.5) - self.assert_rounds_range(c2, "bcrypt", 19, 20) - - c2 = cc.copy(all__vary_rounds="100%") - self.assertEqual(c2._get_record("bcrypt", None).vary_rounds, 1.0) - self.assert_rounds_range(c2, "bcrypt", 15, 21) - - def assert_rounds_range(self, context, scheme, lower, upper): - """helper to check vary_rounds covers specified range""" - # NOTE: this runs enough times the min and max *should* be hit, - # though there's a faint chance it will randomly fail. - handler = context.handler(scheme) - salt = handler.default_salt_chars[0:1] * handler.max_salt_size - seen = set() - for i in irange(300): - h = context.genconfig(scheme, salt=salt) - r = handler.from_string(h).rounds - seen.add(r) - self.assertEqual(min(seen), lower, "vary_rounds had wrong lower limit:") - self.assertEqual(max(seen), upper, "vary_rounds had wrong upper limit:") - - #=================================================================== - # harden_verify / min_verify_time - #=================================================================== - def test_harden_verify_parsing(self): - """harden_verify -- parsing""" - warnings.filterwarnings("ignore", ".*harden_verify.*", - category=DeprecationWarning) - - # valid values - ctx = CryptContext(schemes=["sha256_crypt"]) - self.assertEqual(ctx.harden_verify, None) - self.assertEqual(ctx.using(harden_verify="").harden_verify, None) - self.assertEqual(ctx.using(harden_verify="true").harden_verify, None) - self.assertEqual(ctx.using(harden_verify="false").harden_verify, None) - - def test_dummy_verify(self): - """ - dummy_verify() method - """ - # check dummy_verify() takes expected time - expected = 0.05 - accuracy = 0.2 - handler = DelayHash.using() - handler.delay = expected - ctx = CryptContext(schemes=[handler]) - ctx.dummy_verify() # prime the memoized helpers - elapsed, _ = time_call(ctx.dummy_verify) - self.assertAlmostEqual(elapsed, expected, delta=expected * accuracy) - - # TODO: test dummy_verify() invoked by .verify() when hash is None, - # and same for .verify_and_update() - - #=================================================================== - # feature tests - #=================================================================== - def test_61_autodeprecate(self): - """test deprecated='auto' is handled correctly""" - - def getstate(ctx, category=None): - return [ctx.handler(scheme, category).deprecated for scheme in ctx.schemes()] - - # correctly reports default - ctx = CryptContext("sha256_crypt,md5_crypt,des_crypt", deprecated="auto") - self.assertEqual(getstate(ctx, None), [False, True, True]) - self.assertEqual(getstate(ctx, "admin"), [False, True, True]) - - # correctly reports changed default - ctx.update(default="md5_crypt") - self.assertEqual(getstate(ctx, None), [True, False, True]) - self.assertEqual(getstate(ctx, "admin"), [True, False, True]) - - # category default is handled correctly - ctx.update(admin__context__default="des_crypt") - self.assertEqual(getstate(ctx, None), [True, False, True]) - self.assertEqual(getstate(ctx, "admin"), [True, True, False]) - - # handles 1 scheme - ctx = CryptContext(["sha256_crypt"], deprecated="auto") - self.assertEqual(getstate(ctx, None), [False]) - self.assertEqual(getstate(ctx, "admin"), [False]) - - # disallow auto & other deprecated schemes at same time. - self.assertRaises(ValueError, CryptContext, "sha256_crypt,md5_crypt", - deprecated="auto,md5_crypt") - self.assertRaises(ValueError, CryptContext, "sha256_crypt,md5_crypt", - deprecated="md5_crypt,auto") - - def test_disabled_hashes(self): - """disabled hash support""" - # - # init ref info - # - from passlib.hash import md5_crypt, unix_disabled - - ctx = CryptContext(["des_crypt"]) - ctx2 = CryptContext(["des_crypt", "unix_disabled"]) - h_ref = ctx.hash("foo") - h_other = md5_crypt.hash('foo') - - # - # ctx.disable() - # - - # test w/o disabled hash support - self.assertRaisesRegex(RuntimeError, "no disabled hasher present", - ctx.disable) - self.assertRaisesRegex(RuntimeError, "no disabled hasher present", - ctx.disable, h_ref) - self.assertRaisesRegex(RuntimeError, "no disabled hasher present", - ctx.disable, h_other) - - # test w/ disabled hash support - h_dis = ctx2.disable() - self.assertEqual(h_dis, unix_disabled.default_marker) - h_dis_ref = ctx2.disable(h_ref) - self.assertEqual(h_dis_ref, unix_disabled.default_marker + h_ref) - - h_dis_other = ctx2.disable(h_other) - self.assertEqual(h_dis_other, unix_disabled.default_marker + h_other) - - # don't double-wrap existing disabled hash - self.assertEqual(ctx2.disable(h_dis_ref), h_dis_ref) - - # - # ctx.is_enabled() - # - - # test w/o disabled hash support - self.assertTrue(ctx.is_enabled(h_ref)) - HASH_NOT_IDENTIFIED = "hash could not be identified" - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.is_enabled, h_other) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.is_enabled, h_dis) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.is_enabled, h_dis_ref) - - # test w/ disabled hash support - self.assertTrue(ctx2.is_enabled(h_ref)) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.is_enabled, h_other) - self.assertFalse(ctx2.is_enabled(h_dis)) - self.assertFalse(ctx2.is_enabled(h_dis_ref)) - - # - # ctx.enable() - # - - # test w/o disabled hash support - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.enable, "") - self.assertRaises(TypeError, ctx.enable, None) - self.assertEqual(ctx.enable(h_ref), h_ref) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.enable, h_other) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.enable, h_dis) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.enable, h_dis_ref) - - # test w/ disabled hash support - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx.enable, "") - self.assertRaises(TypeError, ctx2.enable, None) - self.assertEqual(ctx2.enable(h_ref), h_ref) - self.assertRaisesRegex(ValueError, HASH_NOT_IDENTIFIED, - ctx2.enable, h_other) - self.assertRaisesRegex(ValueError, "cannot restore original hash", - ctx2.enable, h_dis) - self.assertEqual(ctx2.enable(h_dis_ref), h_ref) - - #=================================================================== - # eoc - #=================================================================== - -import hashlib, time - -class DelayHash(uh.StaticHandler): - """dummy hasher which delays by specified amount""" - name = "delay_hash" - checksum_chars = uh.LOWER_HEX_CHARS - checksum_size = 40 - delay = 0 - _hash_prefix = u("$x$") - - def _calc_checksum(self, secret): - time.sleep(self.delay) - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - return str_to_uascii(hashlib.sha1(b"prefix" + secret).hexdigest()) - -#============================================================================= -# LazyCryptContext -#============================================================================= -class dummy_2(uh.StaticHandler): - name = "dummy_2" - -class LazyCryptContextTest(TestCase): - descriptionPrefix = "LazyCryptContext" - - def setUp(self): - # make sure this isn't registered before OR after - unload_handler_name("dummy_2") - self.addCleanup(unload_handler_name, "dummy_2") - - def test_kwd_constructor(self): - """test plain kwds""" - self.assertFalse(has_crypt_handler("dummy_2")) - register_crypt_handler_path("dummy_2", "passlib.tests.test_context") - - cc = LazyCryptContext(iter(["dummy_2", "des_crypt"]), deprecated=["des_crypt"]) - - self.assertFalse(has_crypt_handler("dummy_2", True)) - - self.assertEqual(cc.schemes(), ("dummy_2", "des_crypt")) - self.assertTrue(cc.handler("des_crypt").deprecated) - - self.assertTrue(has_crypt_handler("dummy_2", True)) - - def test_callable_constructor(self): - self.assertFalse(has_crypt_handler("dummy_2")) - register_crypt_handler_path("dummy_2", "passlib.tests.test_context") - - def onload(flag=False): - self.assertTrue(flag) - return dict(schemes=iter(["dummy_2", "des_crypt"]), deprecated=["des_crypt"]) - - cc = LazyCryptContext(onload=onload, flag=True) - - self.assertFalse(has_crypt_handler("dummy_2", True)) - - self.assertEqual(cc.schemes(), ("dummy_2", "des_crypt")) - self.assertTrue(cc.handler("des_crypt").deprecated) - - self.assertTrue(has_crypt_handler("dummy_2", True)) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_context_deprecated.py b/src/passlib/tests/test_context_deprecated.py deleted file mode 100644 index 0f76624c..00000000 --- a/src/passlib/tests/test_context_deprecated.py +++ /dev/null @@ -1,743 +0,0 @@ -"""tests for passlib.context - -this file is a clone of the 1.5 test_context.py, -containing the tests using the legacy CryptPolicy api. -it's being preserved here to ensure the old api doesn't break -(until Passlib 1.8, when this and the legacy api will be removed). -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -from logging import getLogger -import os -import warnings -# site -try: - from pkg_resources import resource_filename -except ImportError: - resource_filename = None -# pkg -from passlib import hash -from passlib.context import CryptContext, CryptPolicy, LazyCryptContext -from passlib.utils import to_bytes, to_unicode -import passlib.utils.handlers as uh -from passlib.tests.utils import TestCase, set_file -from passlib.registry import (register_crypt_handler_path, - _has_crypt_handler as has_crypt_handler, - _unload_handler_name as unload_handler_name, - ) -# module -log = getLogger(__name__) - -#============================================================================= -# -#============================================================================= -class CryptPolicyTest(TestCase): - """test CryptPolicy object""" - - # TODO: need to test user categories w/in all this - - descriptionPrefix = "CryptPolicy" - - #=================================================================== - # sample crypt policies used for testing - #=================================================================== - - #--------------------------------------------------------------- - # sample 1 - average config file - #--------------------------------------------------------------- - # NOTE: copy of this is stored in file passlib/tests/sample_config_1s.cfg - sample_config_1s = """\ -[passlib] -schemes = des_crypt, md5_crypt, bsdi_crypt, sha512_crypt -default = md5_crypt -all.vary_rounds = 10%% -bsdi_crypt.max_rounds = 30000 -bsdi_crypt.default_rounds = 25000 -sha512_crypt.max_rounds = 50000 -sha512_crypt.min_rounds = 40000 -""" - sample_config_1s_path = os.path.abspath(os.path.join( - os.path.dirname(__file__), "sample_config_1s.cfg")) - if not os.path.exists(sample_config_1s_path) and resource_filename: - # in case we're zipped up in an egg. - sample_config_1s_path = resource_filename("passlib.tests", - "sample_config_1s.cfg") - - # make sure sample_config_1s uses \n linesep - tests rely on this - assert sample_config_1s.startswith("[passlib]\nschemes") - - sample_config_1pd = dict( - schemes = [ "des_crypt", "md5_crypt", "bsdi_crypt", "sha512_crypt"], - default = "md5_crypt", - # NOTE: not maintaining backwards compat for rendering to "10%" - all__vary_rounds = 0.1, - bsdi_crypt__max_rounds = 30000, - bsdi_crypt__default_rounds = 25000, - sha512_crypt__max_rounds = 50000, - sha512_crypt__min_rounds = 40000, - ) - - sample_config_1pid = { - "schemes": "des_crypt, md5_crypt, bsdi_crypt, sha512_crypt", - "default": "md5_crypt", - # NOTE: not maintaining backwards compat for rendering to "10%" - "all.vary_rounds": 0.1, - "bsdi_crypt.max_rounds": 30000, - "bsdi_crypt.default_rounds": 25000, - "sha512_crypt.max_rounds": 50000, - "sha512_crypt.min_rounds": 40000, - } - - sample_config_1prd = dict( - schemes = [ hash.des_crypt, hash.md5_crypt, hash.bsdi_crypt, hash.sha512_crypt], - default = "md5_crypt", # NOTE: passlib <= 1.5 was handler obj. - # NOTE: not maintaining backwards compat for rendering to "10%" - all__vary_rounds = 0.1, - bsdi_crypt__max_rounds = 30000, - bsdi_crypt__default_rounds = 25000, - sha512_crypt__max_rounds = 50000, - sha512_crypt__min_rounds = 40000, - ) - - #--------------------------------------------------------------- - # sample 2 - partial policy & result of overlay on sample 1 - #--------------------------------------------------------------- - sample_config_2s = """\ -[passlib] -bsdi_crypt.min_rounds = 29000 -bsdi_crypt.max_rounds = 35000 -bsdi_crypt.default_rounds = 31000 -sha512_crypt.min_rounds = 45000 -""" - - sample_config_2pd = dict( - # using this to test full replacement of existing options - bsdi_crypt__min_rounds = 29000, - bsdi_crypt__max_rounds = 35000, - bsdi_crypt__default_rounds = 31000, - # using this to test partial replacement of existing options - sha512_crypt__min_rounds=45000, - ) - - sample_config_12pd = dict( - schemes = [ "des_crypt", "md5_crypt", "bsdi_crypt", "sha512_crypt"], - default = "md5_crypt", - # NOTE: not maintaining backwards compat for rendering to "10%" - all__vary_rounds = 0.1, - bsdi_crypt__min_rounds = 29000, - bsdi_crypt__max_rounds = 35000, - bsdi_crypt__default_rounds = 31000, - sha512_crypt__max_rounds = 50000, - sha512_crypt__min_rounds=45000, - ) - - #--------------------------------------------------------------- - # sample 3 - just changing default - #--------------------------------------------------------------- - sample_config_3pd = dict( - default="sha512_crypt", - ) - - sample_config_123pd = dict( - schemes = [ "des_crypt", "md5_crypt", "bsdi_crypt", "sha512_crypt"], - default = "sha512_crypt", - # NOTE: not maintaining backwards compat for rendering to "10%" - all__vary_rounds = 0.1, - bsdi_crypt__min_rounds = 29000, - bsdi_crypt__max_rounds = 35000, - bsdi_crypt__default_rounds = 31000, - sha512_crypt__max_rounds = 50000, - sha512_crypt__min_rounds=45000, - ) - - #--------------------------------------------------------------- - # sample 4 - category specific - #--------------------------------------------------------------- - sample_config_4s = """ -[passlib] -schemes = sha512_crypt -all.vary_rounds = 10%% -default.sha512_crypt.max_rounds = 20000 -admin.all.vary_rounds = 5%% -admin.sha512_crypt.max_rounds = 40000 -""" - - sample_config_4pd = dict( - schemes = [ "sha512_crypt" ], - # NOTE: not maintaining backwards compat for rendering to "10%" - all__vary_rounds = 0.1, - sha512_crypt__max_rounds = 20000, - # NOTE: not maintaining backwards compat for rendering to "5%" - admin__all__vary_rounds = 0.05, - admin__sha512_crypt__max_rounds = 40000, - ) - - #--------------------------------------------------------------- - # sample 5 - to_string & deprecation testing - #--------------------------------------------------------------- - sample_config_5s = sample_config_1s + """\ -deprecated = des_crypt -admin__context__deprecated = des_crypt, bsdi_crypt -""" - - sample_config_5pd = sample_config_1pd.copy() - sample_config_5pd.update( - deprecated = [ "des_crypt" ], - admin__context__deprecated = [ "des_crypt", "bsdi_crypt" ], - ) - - sample_config_5pid = sample_config_1pid.copy() - sample_config_5pid.update({ - "deprecated": "des_crypt", - "admin.context.deprecated": "des_crypt, bsdi_crypt", - }) - - sample_config_5prd = sample_config_1prd.copy() - sample_config_5prd.update({ - # XXX: should deprecated return the actual handlers in this case? - # would have to modify how policy stores info, for one. - "deprecated": ["des_crypt"], - "admin__context__deprecated": ["des_crypt", "bsdi_crypt"], - }) - - #=================================================================== - # constructors - #=================================================================== - def setUp(self): - TestCase.setUp(self) - warnings.filterwarnings("ignore", - r"The CryptPolicy class has been deprecated") - warnings.filterwarnings("ignore", - r"the method.*hash_needs_update.*is deprecated") - warnings.filterwarnings("ignore", "The 'all' scheme is deprecated.*") - warnings.filterwarnings("ignore", "bsdi_crypt rounds should be odd") - - def test_00_constructor(self): - """test CryptPolicy() constructor""" - policy = CryptPolicy(**self.sample_config_1pd) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - policy = CryptPolicy(self.sample_config_1pd) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - self.assertRaises(TypeError, CryptPolicy, {}, {}) - self.assertRaises(TypeError, CryptPolicy, {}, dummy=1) - - # check key with too many separators is rejected - self.assertRaises(TypeError, CryptPolicy, - schemes = [ "des_crypt", "md5_crypt", "bsdi_crypt", "sha512_crypt"], - bad__key__bsdi_crypt__max_rounds = 30000, - ) - - # check nameless handler rejected - class nameless(uh.StaticHandler): - name = None - self.assertRaises(ValueError, CryptPolicy, schemes=[nameless]) - - # check scheme must be name or crypt handler - self.assertRaises(TypeError, CryptPolicy, schemes=[uh.StaticHandler]) - - # check name conflicts are rejected - class dummy_1(uh.StaticHandler): - name = 'dummy_1' - self.assertRaises(KeyError, CryptPolicy, schemes=[dummy_1, dummy_1]) - - # with unknown deprecated value - self.assertRaises(KeyError, CryptPolicy, - schemes=['des_crypt'], - deprecated=['md5_crypt']) - - # with unknown default value - self.assertRaises(KeyError, CryptPolicy, - schemes=['des_crypt'], - default='md5_crypt') - - def test_01_from_path_simple(self): - """test CryptPolicy.from_path() constructor""" - # NOTE: this is separate so it can also run under GAE - - # test preset stored in existing file - path = self.sample_config_1s_path - policy = CryptPolicy.from_path(path) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test if path missing - self.assertRaises(EnvironmentError, CryptPolicy.from_path, path + 'xxx') - - def test_01_from_path(self): - """test CryptPolicy.from_path() constructor with encodings""" - path = self.mktemp() - - # test "\n" linesep - set_file(path, self.sample_config_1s) - policy = CryptPolicy.from_path(path) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test "\r\n" linesep - set_file(path, self.sample_config_1s.replace("\n","\r\n")) - policy = CryptPolicy.from_path(path) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test with custom encoding - uc2 = to_bytes(self.sample_config_1s, "utf-16", source_encoding="utf-8") - set_file(path, uc2) - policy = CryptPolicy.from_path(path, encoding="utf-16") - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - def test_02_from_string(self): - """test CryptPolicy.from_string() constructor""" - # test "\n" linesep - policy = CryptPolicy.from_string(self.sample_config_1s) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test "\r\n" linesep - policy = CryptPolicy.from_string( - self.sample_config_1s.replace("\n","\r\n")) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test with unicode - data = to_unicode(self.sample_config_1s) - policy = CryptPolicy.from_string(data) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test with non-ascii-compatible encoding - uc2 = to_bytes(self.sample_config_1s, "utf-16", source_encoding="utf-8") - policy = CryptPolicy.from_string(uc2, encoding="utf-16") - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # test category specific options - policy = CryptPolicy.from_string(self.sample_config_4s) - self.assertEqual(policy.to_dict(), self.sample_config_4pd) - - def test_03_from_source(self): - """test CryptPolicy.from_source() constructor""" - # pass it a path - policy = CryptPolicy.from_source(self.sample_config_1s_path) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # pass it a string - policy = CryptPolicy.from_source(self.sample_config_1s) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # pass it a dict (NOTE: make a copy to detect in-place modifications) - policy = CryptPolicy.from_source(self.sample_config_1pd.copy()) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # pass it existing policy - p2 = CryptPolicy.from_source(policy) - self.assertIs(policy, p2) - - # pass it something wrong - self.assertRaises(TypeError, CryptPolicy.from_source, 1) - self.assertRaises(TypeError, CryptPolicy.from_source, []) - - def test_04_from_sources(self): - """test CryptPolicy.from_sources() constructor""" - - # pass it empty list - self.assertRaises(ValueError, CryptPolicy.from_sources, []) - - # pass it one-element list - policy = CryptPolicy.from_sources([self.sample_config_1s]) - self.assertEqual(policy.to_dict(), self.sample_config_1pd) - - # pass multiple sources - policy = CryptPolicy.from_sources( - [ - self.sample_config_1s_path, - self.sample_config_2s, - self.sample_config_3pd, - ]) - self.assertEqual(policy.to_dict(), self.sample_config_123pd) - - def test_05_replace(self): - """test CryptPolicy.replace() constructor""" - - p1 = CryptPolicy(**self.sample_config_1pd) - - # check overlaying sample 2 - p2 = p1.replace(**self.sample_config_2pd) - self.assertEqual(p2.to_dict(), self.sample_config_12pd) - - # check repeating overlay makes no change - p2b = p2.replace(**self.sample_config_2pd) - self.assertEqual(p2b.to_dict(), self.sample_config_12pd) - - # check overlaying sample 3 - p3 = p2.replace(self.sample_config_3pd) - self.assertEqual(p3.to_dict(), self.sample_config_123pd) - - def test_06_forbidden(self): - """test CryptPolicy() forbidden kwds""" - - # salt not allowed to be set - self.assertRaises(KeyError, CryptPolicy, - schemes=["des_crypt"], - des_crypt__salt="xx", - ) - self.assertRaises(KeyError, CryptPolicy, - schemes=["des_crypt"], - all__salt="xx", - ) - - # schemes not allowed for category - self.assertRaises(KeyError, CryptPolicy, - schemes=["des_crypt"], - user__context__schemes=["md5_crypt"], - ) - - #=================================================================== - # reading - #=================================================================== - def test_10_has_schemes(self): - """test has_schemes() method""" - - p1 = CryptPolicy(**self.sample_config_1pd) - self.assertTrue(p1.has_schemes()) - - p3 = CryptPolicy(**self.sample_config_3pd) - self.assertTrue(not p3.has_schemes()) - - def test_11_iter_handlers(self): - """test iter_handlers() method""" - - p1 = CryptPolicy(**self.sample_config_1pd) - s = self.sample_config_1prd['schemes'] - self.assertEqual(list(p1.iter_handlers()), s) - - p3 = CryptPolicy(**self.sample_config_3pd) - self.assertEqual(list(p3.iter_handlers()), []) - - def test_12_get_handler(self): - """test get_handler() method""" - - p1 = CryptPolicy(**self.sample_config_1pd) - - # check by name - self.assertIs(p1.get_handler("bsdi_crypt"), hash.bsdi_crypt) - - # check by missing name - self.assertIs(p1.get_handler("sha256_crypt"), None) - self.assertRaises(KeyError, p1.get_handler, "sha256_crypt", required=True) - - # check default - self.assertIs(p1.get_handler(), hash.md5_crypt) - - def test_13_get_options(self): - """test get_options() method""" - - p12 = CryptPolicy(**self.sample_config_12pd) - - self.assertEqual(p12.get_options("bsdi_crypt"),dict( - # NOTE: not maintaining backwards compat for rendering to "10%" - vary_rounds = 0.1, - min_rounds = 29000, - max_rounds = 35000, - default_rounds = 31000, - )) - - self.assertEqual(p12.get_options("sha512_crypt"),dict( - # NOTE: not maintaining backwards compat for rendering to "10%" - vary_rounds = 0.1, - min_rounds = 45000, - max_rounds = 50000, - )) - - p4 = CryptPolicy.from_string(self.sample_config_4s) - self.assertEqual(p4.get_options("sha512_crypt"), dict( - # NOTE: not maintaining backwards compat for rendering to "10%" - vary_rounds=0.1, - max_rounds=20000, - )) - - self.assertEqual(p4.get_options("sha512_crypt", "user"), dict( - # NOTE: not maintaining backwards compat for rendering to "10%" - vary_rounds=0.1, - max_rounds=20000, - )) - - self.assertEqual(p4.get_options("sha512_crypt", "admin"), dict( - # NOTE: not maintaining backwards compat for rendering to "5%" - vary_rounds=0.05, - max_rounds=40000, - )) - - def test_14_handler_is_deprecated(self): - """test handler_is_deprecated() method""" - pa = CryptPolicy(**self.sample_config_1pd) - pb = CryptPolicy(**self.sample_config_5pd) - - self.assertFalse(pa.handler_is_deprecated("des_crypt")) - self.assertFalse(pa.handler_is_deprecated(hash.bsdi_crypt)) - self.assertFalse(pa.handler_is_deprecated("sha512_crypt")) - - self.assertTrue(pb.handler_is_deprecated("des_crypt")) - self.assertFalse(pb.handler_is_deprecated(hash.bsdi_crypt)) - self.assertFalse(pb.handler_is_deprecated("sha512_crypt")) - - # check categories as well - self.assertTrue(pb.handler_is_deprecated("des_crypt", "user")) - self.assertFalse(pb.handler_is_deprecated("bsdi_crypt", "user")) - self.assertTrue(pb.handler_is_deprecated("des_crypt", "admin")) - self.assertTrue(pb.handler_is_deprecated("bsdi_crypt", "admin")) - - # check deprecation is overridden per category - pc = CryptPolicy( - schemes=["md5_crypt", "des_crypt"], - deprecated=["md5_crypt"], - user__context__deprecated=["des_crypt"], - ) - self.assertTrue(pc.handler_is_deprecated("md5_crypt")) - self.assertFalse(pc.handler_is_deprecated("des_crypt")) - self.assertFalse(pc.handler_is_deprecated("md5_crypt", "user")) - self.assertTrue(pc.handler_is_deprecated("des_crypt", "user")) - - def test_15_min_verify_time(self): - """test get_min_verify_time() method""" - # silence deprecation warnings for min verify time - warnings.filterwarnings("ignore", category=DeprecationWarning) - - pa = CryptPolicy() - self.assertEqual(pa.get_min_verify_time(), 0) - self.assertEqual(pa.get_min_verify_time('admin'), 0) - - pb = pa.replace(min_verify_time=.1) - self.assertEqual(pb.get_min_verify_time(), 0) - self.assertEqual(pb.get_min_verify_time('admin'), 0) - - #=================================================================== - # serialization - #=================================================================== - def test_20_iter_config(self): - """test iter_config() method""" - p5 = CryptPolicy(**self.sample_config_5pd) - self.assertEqual(dict(p5.iter_config()), self.sample_config_5pd) - self.assertEqual(dict(p5.iter_config(resolve=True)), self.sample_config_5prd) - self.assertEqual(dict(p5.iter_config(ini=True)), self.sample_config_5pid) - - def test_21_to_dict(self): - """test to_dict() method""" - p5 = CryptPolicy(**self.sample_config_5pd) - self.assertEqual(p5.to_dict(), self.sample_config_5pd) - self.assertEqual(p5.to_dict(resolve=True), self.sample_config_5prd) - - def test_22_to_string(self): - """test to_string() method""" - pa = CryptPolicy(**self.sample_config_5pd) - s = pa.to_string() # NOTE: can't compare string directly, ordering etc may not match - pb = CryptPolicy.from_string(s) - self.assertEqual(pb.to_dict(), self.sample_config_5pd) - - s = pa.to_string(encoding="latin-1") - self.assertIsInstance(s, bytes) - - #=================================================================== - # - #=================================================================== - -#============================================================================= -# CryptContext -#============================================================================= -class CryptContextTest(TestCase): - """test CryptContext class""" - descriptionPrefix = "CryptContext" - - def setUp(self): - TestCase.setUp(self) - warnings.filterwarnings("ignore", - r"CryptContext\(\)\.replace\(\) has been deprecated.*") - warnings.filterwarnings("ignore", - r"The CryptContext ``policy`` keyword has been deprecated.*") - warnings.filterwarnings("ignore", ".*(CryptPolicy|context\.policy).*(has|have) been deprecated.*") - warnings.filterwarnings("ignore", - r"the method.*hash_needs_update.*is deprecated") - - #=================================================================== - # constructor - #=================================================================== - def test_00_constructor(self): - """test constructor""" - # create crypt context using handlers - cc = CryptContext([hash.md5_crypt, hash.bsdi_crypt, hash.des_crypt]) - c,b,a = cc.policy.iter_handlers() - self.assertIs(a, hash.des_crypt) - self.assertIs(b, hash.bsdi_crypt) - self.assertIs(c, hash.md5_crypt) - - # create context using names - cc = CryptContext(["md5_crypt", "bsdi_crypt", "des_crypt"]) - c,b,a = cc.policy.iter_handlers() - self.assertIs(a, hash.des_crypt) - self.assertIs(b, hash.bsdi_crypt) - self.assertIs(c, hash.md5_crypt) - - # policy kwd - policy = cc.policy - cc = CryptContext(policy=policy) - self.assertEqual(cc.to_dict(), policy.to_dict()) - - cc = CryptContext(policy=policy, default="bsdi_crypt") - self.assertNotEqual(cc.to_dict(), policy.to_dict()) - self.assertEqual(cc.to_dict(), dict(schemes=["md5_crypt","bsdi_crypt","des_crypt"], - default="bsdi_crypt")) - - self.assertRaises(TypeError, setattr, cc, 'policy', None) - self.assertRaises(TypeError, CryptContext, policy='x') - - def test_01_replace(self): - """test replace()""" - - cc = CryptContext(["md5_crypt", "bsdi_crypt", "des_crypt"]) - self.assertIs(cc.policy.get_handler(), hash.md5_crypt) - - cc2 = cc.replace() - self.assertIsNot(cc2, cc) - # NOTE: was not able to maintain backward compatibility with this... - ##self.assertIs(cc2.policy, cc.policy) - - cc3 = cc.replace(default="bsdi_crypt") - self.assertIsNot(cc3, cc) - # NOTE: was not able to maintain backward compatibility with this... - ##self.assertIs(cc3.policy, cc.policy) - self.assertIs(cc3.policy.get_handler(), hash.bsdi_crypt) - - def test_02_no_handlers(self): - """test no handlers""" - - # check constructor... - cc = CryptContext() - self.assertRaises(KeyError, cc.identify, 'hash', required=True) - self.assertRaises(KeyError, cc.hash, 'secret') - self.assertRaises(KeyError, cc.verify, 'secret', 'hash') - - # check updating policy after the fact... - cc = CryptContext(['md5_crypt']) - p = CryptPolicy(schemes=[]) - cc.policy = p - - self.assertRaises(KeyError, cc.identify, 'hash', required=True) - self.assertRaises(KeyError, cc.hash, 'secret') - self.assertRaises(KeyError, cc.verify, 'secret', 'hash') - - #=================================================================== - # policy adaptation - #=================================================================== - sample_policy_1 = dict( - schemes = [ "des_crypt", "md5_crypt", "phpass", "bsdi_crypt", - "sha256_crypt"], - deprecated = [ "des_crypt", ], - default = "sha256_crypt", - bsdi_crypt__max_rounds = 30, - bsdi_crypt__default_rounds = 25, - bsdi_crypt__vary_rounds = 0, - sha256_crypt__max_rounds = 3000, - sha256_crypt__min_rounds = 2000, - sha256_crypt__default_rounds = 3000, - phpass__ident = "H", - phpass__default_rounds = 7, - ) - - def test_12_hash_needs_update(self): - """test hash_needs_update() method""" - cc = CryptContext(**self.sample_policy_1) - - # check deprecated scheme - self.assertTrue(cc.hash_needs_update('9XXD4trGYeGJA')) - self.assertFalse(cc.hash_needs_update('$1$J8HC2RCr$HcmM.7NxB2weSvlw2FgzU0')) - - # check min rounds - self.assertTrue(cc.hash_needs_update('$5$rounds=1999$jD81UCoo.zI.UETs$Y7qSTQ6mTiU9qZB4fRr43wRgQq4V.5AAf7F97Pzxey/')) - self.assertFalse(cc.hash_needs_update('$5$rounds=2000$228SSRje04cnNCaQ$YGV4RYu.5sNiBvorQDlO0WWQjyJVGKBcJXz3OtyQ2u8')) - - # check max rounds - self.assertFalse(cc.hash_needs_update('$5$rounds=3000$fS9iazEwTKi7QPW4$VasgBC8FqlOvD7x2HhABaMXCTh9jwHclPA9j5YQdns.')) - self.assertTrue(cc.hash_needs_update('$5$rounds=3001$QlFHHifXvpFX4PLs$/0ekt7lSs/lOikSerQ0M/1porEHxYq7W/2hdFpxA3fA')) - - #=================================================================== - # border cases - #=================================================================== - def test_30_nonstring_hash(self): - """test non-string hash values cause error""" - warnings.filterwarnings("ignore", ".*needs_update.*'scheme' keyword is deprecated.*") - - # - # test hash=None or some other non-string causes TypeError - # and that explicit-scheme code path behaves the same. - # - cc = CryptContext(["des_crypt"]) - for hash, kwds in [ - (None, {}), - # NOTE: 'scheme' kwd is deprecated... - (None, {"scheme": "des_crypt"}), - (1, {}), - ((), {}), - ]: - - self.assertRaises(TypeError, cc.hash_needs_update, hash, **kwds) - - cc2 = CryptContext(["mysql323"]) - self.assertRaises(TypeError, cc2.hash_needs_update, None) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# LazyCryptContext -#============================================================================= -class dummy_2(uh.StaticHandler): - name = "dummy_2" - -class LazyCryptContextTest(TestCase): - descriptionPrefix = "LazyCryptContext" - - def setUp(self): - TestCase.setUp(self) - - # make sure this isn't registered before OR after - unload_handler_name("dummy_2") - self.addCleanup(unload_handler_name, "dummy_2") - - # silence some warnings - warnings.filterwarnings("ignore", - r"CryptContext\(\)\.replace\(\) has been deprecated") - warnings.filterwarnings("ignore", ".*(CryptPolicy|context\.policy).*(has|have) been deprecated.*") - - def test_kwd_constructor(self): - """test plain kwds""" - self.assertFalse(has_crypt_handler("dummy_2")) - register_crypt_handler_path("dummy_2", "passlib.tests.test_context") - - cc = LazyCryptContext(iter(["dummy_2", "des_crypt"]), deprecated=["des_crypt"]) - - self.assertFalse(has_crypt_handler("dummy_2", True)) - - self.assertTrue(cc.policy.handler_is_deprecated("des_crypt")) - self.assertEqual(cc.policy.schemes(), ["dummy_2", "des_crypt"]) - - self.assertTrue(has_crypt_handler("dummy_2", True)) - - def test_callable_constructor(self): - """test create_policy() hook, returning CryptPolicy""" - self.assertFalse(has_crypt_handler("dummy_2")) - register_crypt_handler_path("dummy_2", "passlib.tests.test_context") - - def create_policy(flag=False): - self.assertTrue(flag) - return CryptPolicy(schemes=iter(["dummy_2", "des_crypt"]), deprecated=["des_crypt"]) - - cc = LazyCryptContext(create_policy=create_policy, flag=True) - - self.assertFalse(has_crypt_handler("dummy_2", True)) - - self.assertTrue(cc.policy.handler_is_deprecated("des_crypt")) - self.assertEqual(cc.policy.schemes(), ["dummy_2", "des_crypt"]) - - self.assertTrue(has_crypt_handler("dummy_2", True)) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_crypto_builtin_md4.py b/src/passlib/tests/test_crypto_builtin_md4.py deleted file mode 100644 index 0aca1eb0..00000000 --- a/src/passlib/tests/test_crypto_builtin_md4.py +++ /dev/null @@ -1,160 +0,0 @@ -"""passlib.tests -- unittests for passlib.crypto._md4""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement, division -# core -from binascii import hexlify -import hashlib -# site -# pkg -# module -from passlib.utils.compat import bascii_to_str, PY3, u -from passlib.crypto.digest import lookup_hash -from passlib.tests.utils import TestCase, skipUnless -# local -__all__ = [ - "_Common_MD4_Test", - "MD4_Builtin_Test", - "MD4_SSL_Test", -] -#============================================================================= -# test pure-python MD4 implementation -#============================================================================= -class _Common_MD4_Test(TestCase): - """common code for testing md4 backends""" - - vectors = [ - # input -> hex digest - # test vectors from http://www.faqs.org/rfcs/rfc1320.html - A.5 - (b"", "31d6cfe0d16ae931b73c59d7e0c089c0"), - (b"a", "bde52cb31de33e46245e05fbdbd6fb24"), - (b"abc", "a448017aaf21d8525fc10ae87aa6729d"), - (b"message digest", "d9130a8164549fe818874806e1c7014b"), - (b"abcdefghijklmnopqrstuvwxyz", "d79e1c308aa5bbcdeea8ed63df412da9"), - (b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "043f8582f241db351ce627e153e7f0e4"), - (b"12345678901234567890123456789012345678901234567890123456789012345678901234567890", "e33b4ddc9c38f2199c3e7b164fcc0536"), - ] - - def get_md4_const(self): - """ - get md4 constructor -- - overridden by subclasses to use alternate backends. - """ - return lookup_hash("md4").const - - def test_attrs(self): - """informational attributes""" - h = self.get_md4_const()() - self.assertEqual(h.name, "md4") - self.assertEqual(h.digest_size, 16) - self.assertEqual(h.block_size, 64) - - def test_md4_update(self): - """update() method""" - md4 = self.get_md4_const() - h = md4(b'') - self.assertEqual(h.hexdigest(), "31d6cfe0d16ae931b73c59d7e0c089c0") - - h.update(b'a') - self.assertEqual(h.hexdigest(), "bde52cb31de33e46245e05fbdbd6fb24") - - h.update(b'bcdefghijklmnopqrstuvwxyz') - self.assertEqual(h.hexdigest(), "d79e1c308aa5bbcdeea8ed63df412da9") - - if PY3: - # reject unicode, hash should return digest of b'' - h = md4() - self.assertRaises(TypeError, h.update, u('a')) - self.assertEqual(h.hexdigest(), "31d6cfe0d16ae931b73c59d7e0c089c0") - else: - # coerce unicode to ascii, hash should return digest of b'a' - h = md4() - h.update(u('a')) - self.assertEqual(h.hexdigest(), "bde52cb31de33e46245e05fbdbd6fb24") - - def test_md4_hexdigest(self): - """hexdigest() method""" - md4 = self.get_md4_const() - for input, hex in self.vectors: - out = md4(input).hexdigest() - self.assertEqual(out, hex) - - def test_md4_digest(self): - """digest() method""" - md4 = self.get_md4_const() - for input, hex in self.vectors: - out = bascii_to_str(hexlify(md4(input).digest())) - self.assertEqual(out, hex) - - def test_md4_copy(self): - """copy() method""" - md4 = self.get_md4_const() - h = md4(b'abc') - - h2 = h.copy() - h2.update(b'def') - self.assertEqual(h2.hexdigest(), '804e7f1c2586e50b49ac65db5b645131') - - h.update(b'ghi') - self.assertEqual(h.hexdigest(), 'c5225580bfe176f6deeee33dee98732c') - - -#------------------------------------------------------------------------ -# create subclasses to test various backends -#------------------------------------------------------------------------ - -def has_native_md4(): # pragma: no cover -- runtime detection - """ - check if hashlib natively supports md4. - """ - try: - hashlib.new("md4") - return True - except ValueError: - # not supported - ssl probably missing (e.g. ironpython) - return False - - -@skipUnless(has_native_md4(), "hashlib lacks ssl/md4 support") -class MD4_SSL_Test(_Common_MD4_Test): - descriptionPrefix = "hashlib.new('md4')" - - # NOTE: we trust ssl got md4 implementation right, - # this is more to test our test is correct :) - - def setUp(self): - super(MD4_SSL_Test, self).setUp() - - # make sure we're using right constructor. - self.assertEqual(self.get_md4_const().__module__, "hashlib") - - -class MD4_Builtin_Test(_Common_MD4_Test): - descriptionPrefix = "passlib.crypto._md4.md4()" - - def setUp(self): - super(MD4_Builtin_Test, self).setUp() - - if has_native_md4(): - - # Temporarily make lookup_hash() use builtin pure-python implementation, - # by monkeypatching hashlib.new() to ensure we fall back to passlib's md4 class. - orig = hashlib.new - def wrapper(name, *args): - if name == "md4": - raise ValueError("md4 disabled for testing") - return orig(name, *args) - self.patchAttr(hashlib, "new", wrapper) - - # flush cache before & after test, since we're mucking with it. - lookup_hash.clear_cache() - self.addCleanup(lookup_hash.clear_cache) - - # make sure we're using right constructor. - self.assertEqual(self.get_md4_const().__module__, "passlib.crypto._md4") - - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_crypto_des.py b/src/passlib/tests/test_crypto_des.py deleted file mode 100644 index ab31845e..00000000 --- a/src/passlib/tests/test_crypto_des.py +++ /dev/null @@ -1,194 +0,0 @@ -"""passlib.tests -- unittests for passlib.crypto.des""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement, division -# core -from functools import partial -# site -# pkg -# module -from passlib.utils import getrandbytes -from passlib.tests.utils import TestCase - -#============================================================================= -# test DES routines -#============================================================================= -class DesTest(TestCase): - descriptionPrefix = "passlib.crypto.des" - - # test vectors taken from http://www.skepticfiles.org/faq/testdes.htm - des_test_vectors = [ - # key, plaintext, ciphertext - (0x0000000000000000, 0x0000000000000000, 0x8CA64DE9C1B123A7), - (0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0x7359B2163E4EDC58), - (0x3000000000000000, 0x1000000000000001, 0x958E6E627A05557B), - (0x1111111111111111, 0x1111111111111111, 0xF40379AB9E0EC533), - (0x0123456789ABCDEF, 0x1111111111111111, 0x17668DFC7292532D), - (0x1111111111111111, 0x0123456789ABCDEF, 0x8A5AE1F81AB8F2DD), - (0x0000000000000000, 0x0000000000000000, 0x8CA64DE9C1B123A7), - (0xFEDCBA9876543210, 0x0123456789ABCDEF, 0xED39D950FA74BCC4), - (0x7CA110454A1A6E57, 0x01A1D6D039776742, 0x690F5B0D9A26939B), - (0x0131D9619DC1376E, 0x5CD54CA83DEF57DA, 0x7A389D10354BD271), - (0x07A1133E4A0B2686, 0x0248D43806F67172, 0x868EBB51CAB4599A), - (0x3849674C2602319E, 0x51454B582DDF440A, 0x7178876E01F19B2A), - (0x04B915BA43FEB5B6, 0x42FD443059577FA2, 0xAF37FB421F8C4095), - (0x0113B970FD34F2CE, 0x059B5E0851CF143A, 0x86A560F10EC6D85B), - (0x0170F175468FB5E6, 0x0756D8E0774761D2, 0x0CD3DA020021DC09), - (0x43297FAD38E373FE, 0x762514B829BF486A, 0xEA676B2CB7DB2B7A), - (0x07A7137045DA2A16, 0x3BDD119049372802, 0xDFD64A815CAF1A0F), - (0x04689104C2FD3B2F, 0x26955F6835AF609A, 0x5C513C9C4886C088), - (0x37D06BB516CB7546, 0x164D5E404F275232, 0x0A2AEEAE3FF4AB77), - (0x1F08260D1AC2465E, 0x6B056E18759F5CCA, 0xEF1BF03E5DFA575A), - (0x584023641ABA6176, 0x004BD6EF09176062, 0x88BF0DB6D70DEE56), - (0x025816164629B007, 0x480D39006EE762F2, 0xA1F9915541020B56), - (0x49793EBC79B3258F, 0x437540C8698F3CFA, 0x6FBF1CAFCFFD0556), - (0x4FB05E1515AB73A7, 0x072D43A077075292, 0x2F22E49BAB7CA1AC), - (0x49E95D6D4CA229BF, 0x02FE55778117F12A, 0x5A6B612CC26CCE4A), - (0x018310DC409B26D6, 0x1D9D5C5018F728C2, 0x5F4C038ED12B2E41), - (0x1C587F1C13924FEF, 0x305532286D6F295A, 0x63FAC0D034D9F793), - (0x0101010101010101, 0x0123456789ABCDEF, 0x617B3A0CE8F07100), - (0x1F1F1F1F0E0E0E0E, 0x0123456789ABCDEF, 0xDB958605F8C8C606), - (0xE0FEE0FEF1FEF1FE, 0x0123456789ABCDEF, 0xEDBFD1C66C29CCC7), - (0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0x355550B2150E2451), - (0xFFFFFFFFFFFFFFFF, 0x0000000000000000, 0xCAAAAF4DEAF1DBAE), - (0x0123456789ABCDEF, 0x0000000000000000, 0xD5D44FF720683D0D), - (0xFEDCBA9876543210, 0xFFFFFFFFFFFFFFFF, 0x2A2BB008DF97C2F2), - ] - - def test_01_expand(self): - """expand_des_key()""" - from passlib.crypto.des import expand_des_key, shrink_des_key, \ - _KDATA_MASK, INT_56_MASK - - # make sure test vectors are preserved (sans parity bits) - # uses ints, bytes are tested under # 02 - for key1, _, _ in self.des_test_vectors: - key2 = shrink_des_key(key1) - key3 = expand_des_key(key2) - # NOTE: this assumes expand_des_key() sets parity bits to 0 - self.assertEqual(key3, key1 & _KDATA_MASK) - - # type checks - self.assertRaises(TypeError, expand_des_key, 1.0) - - # too large - self.assertRaises(ValueError, expand_des_key, INT_56_MASK+1) - self.assertRaises(ValueError, expand_des_key, b"\x00"*8) - - # too small - self.assertRaises(ValueError, expand_des_key, -1) - self.assertRaises(ValueError, expand_des_key, b"\x00"*6) - - def test_02_shrink(self): - """shrink_des_key()""" - from passlib.crypto.des import expand_des_key, shrink_des_key, INT_64_MASK - rng = self.getRandom() - - # make sure reverse works for some random keys - # uses bytes, ints are tested under # 01 - for i in range(20): - key1 = getrandbytes(rng, 7) - key2 = expand_des_key(key1) - key3 = shrink_des_key(key2) - self.assertEqual(key3, key1) - - # type checks - self.assertRaises(TypeError, shrink_des_key, 1.0) - - # too large - self.assertRaises(ValueError, shrink_des_key, INT_64_MASK+1) - self.assertRaises(ValueError, shrink_des_key, b"\x00"*9) - - # too small - self.assertRaises(ValueError, shrink_des_key, -1) - self.assertRaises(ValueError, shrink_des_key, b"\x00"*7) - - def _random_parity(self, key): - """randomize parity bits""" - from passlib.crypto.des import _KDATA_MASK, _KPARITY_MASK, INT_64_MASK - rng = self.getRandom() - return (key & _KDATA_MASK) | (rng.randint(0,INT_64_MASK) & _KPARITY_MASK) - - def test_03_encrypt_bytes(self): - """des_encrypt_block()""" - from passlib.crypto.des import (des_encrypt_block, shrink_des_key, - _pack64, _unpack64) - - # run through test vectors - for key, plaintext, correct in self.des_test_vectors: - # convert to bytes - key = _pack64(key) - plaintext = _pack64(plaintext) - correct = _pack64(correct) - - # test 64-bit key - result = des_encrypt_block(key, plaintext) - self.assertEqual(result, correct, "key=%r plaintext=%r:" % - (key, plaintext)) - - # test 56-bit version - key2 = shrink_des_key(key) - result = des_encrypt_block(key2, plaintext) - self.assertEqual(result, correct, "key=%r shrink(key)=%r plaintext=%r:" % - (key, key2, plaintext)) - - # test with random parity bits - for _ in range(20): - key3 = _pack64(self._random_parity(_unpack64(key))) - result = des_encrypt_block(key3, plaintext) - self.assertEqual(result, correct, "key=%r rndparity(key)=%r plaintext=%r:" % - (key, key3, plaintext)) - - # check invalid keys - stub = b'\x00' * 8 - self.assertRaises(TypeError, des_encrypt_block, 0, stub) - self.assertRaises(ValueError, des_encrypt_block, b'\x00'*6, stub) - - # check invalid input - self.assertRaises(TypeError, des_encrypt_block, stub, 0) - self.assertRaises(ValueError, des_encrypt_block, stub, b'\x00'*7) - - # check invalid salts - self.assertRaises(ValueError, des_encrypt_block, stub, stub, salt=-1) - self.assertRaises(ValueError, des_encrypt_block, stub, stub, salt=1<<24) - - # check invalid rounds - self.assertRaises(ValueError, des_encrypt_block, stub, stub, 0, rounds=0) - - def test_04_encrypt_ints(self): - """des_encrypt_int_block()""" - from passlib.crypto.des import des_encrypt_int_block - - # run through test vectors - for key, plaintext, correct in self.des_test_vectors: - # test 64-bit key - result = des_encrypt_int_block(key, plaintext) - self.assertEqual(result, correct, "key=%r plaintext=%r:" % - (key, plaintext)) - - # test with random parity bits - for _ in range(20): - key3 = self._random_parity(key) - result = des_encrypt_int_block(key3, plaintext) - self.assertEqual(result, correct, "key=%r rndparity(key)=%r plaintext=%r:" % - (key, key3, plaintext)) - - # check invalid keys - self.assertRaises(TypeError, des_encrypt_int_block, b'\x00', 0) - self.assertRaises(ValueError, des_encrypt_int_block, -1, 0) - - # check invalid input - self.assertRaises(TypeError, des_encrypt_int_block, 0, b'\x00') - self.assertRaises(ValueError, des_encrypt_int_block, 0, -1) - - # check invalid salts - self.assertRaises(ValueError, des_encrypt_int_block, 0, 0, salt=-1) - self.assertRaises(ValueError, des_encrypt_int_block, 0, 0, salt=1<<24) - - # check invalid rounds - self.assertRaises(ValueError, des_encrypt_int_block, 0, 0, 0, rounds=0) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_crypto_digest.py b/src/passlib/tests/test_crypto_digest.py deleted file mode 100644 index 65b4a4db..00000000 --- a/src/passlib/tests/test_crypto_digest.py +++ /dev/null @@ -1,499 +0,0 @@ -"""tests for passlib.utils.(des|pbkdf2|md4)""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement, division -# core -from binascii import hexlify -import hashlib -import warnings -# site -# pkg -# module -from passlib.utils.compat import PY3, u, JYTHON -from passlib.tests.utils import TestCase, TEST_MODE, skipUnless, hb - -#============================================================================= -# test assorted crypto helpers -#============================================================================= -class HashInfoTest(TestCase): - """test various crypto functions""" - descriptionPrefix = "passlib.crypto.digest" - - #: list of formats norm_hash_name() should support - norm_hash_formats = ["hashlib", "iana"] - - #: test cases for norm_hash_name() - #: each row contains (iana name, hashlib name, ... 0+ unnormalized names) - norm_hash_samples = [ - # real hashes - ("md5", "md5", "SCRAM-MD5-PLUS", "MD-5"), - ("sha1", "sha-1", "SCRAM-SHA-1", "SHA1"), - ("sha256", "sha-256", "SHA_256", "sha2-256"), - ("ripemd", "ripemd", "SCRAM-RIPEMD", "RIPEMD"), - ("ripemd160", "ripemd-160", "SCRAM-RIPEMD-160", "RIPEmd160"), - - # fake hashes (to check if fallback normalization behaves sanely) - ("sha4_256", "sha4-256", "SHA4-256", "SHA-4-256"), - ("test128", "test-128", "TEST128"), - ("test2", "test2", "TEST-2"), - ("test3_128", "test3-128", "TEST-3-128"), - ] - - def test_norm_hash_name(self): - """norm_hash_name()""" - from itertools import chain - from passlib.crypto.digest import norm_hash_name, _known_hash_names - - # snapshot warning state, ignore unknown hash warnings - ctx = warnings.catch_warnings() - ctx.__enter__() - self.addCleanup(ctx.__exit__) - warnings.filterwarnings("ignore", '.*unknown hash') - - # test string types - self.assertEqual(norm_hash_name(u("MD4")), "md4") - self.assertEqual(norm_hash_name(b"MD4"), "md4") - self.assertRaises(TypeError, norm_hash_name, None) - - # test selected results - for row in chain(_known_hash_names, self.norm_hash_samples): - for idx, format in enumerate(self.norm_hash_formats): - correct = row[idx] - for value in row: - result = norm_hash_name(value, format) - self.assertEqual(result, correct, - "name=%r, format=%r:" % (value, - format)) - - def test_lookup_hash_ctor(self): - """lookup_hash() -- constructor""" - from passlib.crypto.digest import lookup_hash - - # invalid/unknown names should be rejected - self.assertRaises(ValueError, lookup_hash, "new") - self.assertRaises(ValueError, lookup_hash, "__name__") - self.assertRaises(ValueError, lookup_hash, "sha4") - - # 1. should return hashlib builtin if found - self.assertEqual(lookup_hash("md5"), (hashlib.md5, 16, 64)) - - # 2. should return wrapper around hashlib.new() if found - try: - hashlib.new("sha") - has_sha = True - except ValueError: - has_sha = False - if has_sha: - record = lookup_hash("sha") - const = record[0] - self.assertEqual(record, (const, 20, 64)) - self.assertEqual(hexlify(const(b"abc").digest()), - b"0164b8a914cd2a5e74c4f7ff082c4d97f1edf880") - - else: - self.assertRaises(ValueError, lookup_hash, "sha") - - # 3. should fall back to builtin md4 - try: - hashlib.new("md4") - has_md4 = True - except ValueError: - has_md4 = False - record = lookup_hash("md4") - const = record[0] - if not has_md4: - from passlib.crypto._md4 import md4 - self.assertIs(const, md4) - self.assertEqual(record, (const, 16, 64)) - self.assertEqual(hexlify(const(b"abc").digest()), - b"a448017aaf21d8525fc10ae87aa6729d") - - # 4. unknown names should be rejected - self.assertRaises(ValueError, lookup_hash, "xxx256") - - # should memoize records - self.assertIs(lookup_hash("md5"), lookup_hash("md5")) - - def test_lookup_hash_metadata(self): - """lookup_hash() -- metadata""" - - from passlib.crypto.digest import lookup_hash - - # quick test of metadata using known reference - sha256 - info = lookup_hash("sha256") - self.assertEqual(info.name, "sha256") - self.assertEqual(info.iana_name, "sha-256") - self.assertEqual(info.block_size, 64) - self.assertEqual(info.digest_size, 32) - self.assertIs(lookup_hash("SHA2-256"), info) - - # quick test of metadata using known reference - md5 - info = lookup_hash("md5") - self.assertEqual(info.name, "md5") - self.assertEqual(info.iana_name, "md5") - self.assertEqual(info.block_size, 64) - self.assertEqual(info.digest_size, 16) - - def test_lookup_hash_alt_types(self): - """lookup_hash() -- alternate types""" - - from passlib.crypto.digest import lookup_hash - - info = lookup_hash("sha256") - self.assertIs(lookup_hash(info), info) - self.assertIs(lookup_hash(info.const), info) - - self.assertRaises(TypeError, lookup_hash, 123) - - # TODO: write full test of compile_hmac() -- currently relying on pbkdf2_hmac() tests - -#============================================================================= -# test PBKDF1 support -#============================================================================= -class Pbkdf1_Test(TestCase): - """test kdf helpers""" - descriptionPrefix = "passlib.crypto.digest.pbkdf1" - - pbkdf1_tests = [ - # (password, salt, rounds, keylen, hash, result) - - # - # from http://www.di-mgt.com.au/cryptoKDFs.html - # - (b'password', hb('78578E5A5D63CB06'), 1000, 16, 'sha1', hb('dc19847e05c64d2faf10ebfb4a3d2a20')), - - # - # custom - # - (b'password', b'salt', 1000, 0, 'md5', b''), - (b'password', b'salt', 1000, 1, 'md5', hb('84')), - (b'password', b'salt', 1000, 8, 'md5', hb('8475c6a8531a5d27')), - (b'password', b'salt', 1000, 16, 'md5', hb('8475c6a8531a5d27e386cd496457812c')), - (b'password', b'salt', 1000, None, 'md5', hb('8475c6a8531a5d27e386cd496457812c')), - (b'password', b'salt', 1000, None, 'sha1', hb('4a8fd48e426ed081b535be5769892fa396293efb')), - ] - if not JYTHON: # FIXME: find out why not jython, or reenable this. - pbkdf1_tests.append( - (b'password', b'salt', 1000, None, 'md4', hb('f7f2e91100a8f96190f2dd177cb26453')) - ) - - def test_known(self): - """test reference vectors""" - from passlib.crypto.digest import pbkdf1 - for secret, salt, rounds, keylen, digest, correct in self.pbkdf1_tests: - result = pbkdf1(digest, secret, salt, rounds, keylen) - self.assertEqual(result, correct) - - def test_border(self): - """test border cases""" - from passlib.crypto.digest import pbkdf1 - def helper(secret=b'secret', salt=b'salt', rounds=1, keylen=1, hash='md5'): - return pbkdf1(hash, secret, salt, rounds, keylen) - helper() - - # salt/secret wrong type - self.assertRaises(TypeError, helper, secret=1) - self.assertRaises(TypeError, helper, salt=1) - - # non-existent hashes - self.assertRaises(ValueError, helper, hash='missing') - - # rounds < 1 and wrong type - self.assertRaises(ValueError, helper, rounds=0) - self.assertRaises(TypeError, helper, rounds='1') - - # keylen < 0, keylen > block_size, and wrong type - self.assertRaises(ValueError, helper, keylen=-1) - self.assertRaises(ValueError, helper, keylen=17, hash='md5') - self.assertRaises(TypeError, helper, keylen='1') - -#============================================================================= -# test PBKDF2-HMAC support -#============================================================================= - -# import the test subject -from passlib.crypto.digest import pbkdf2_hmac, PBKDF2_BACKENDS - -# NOTE: relying on tox to verify this works under all the various backends. -class Pbkdf2Test(TestCase): - """test pbkdf2() support""" - descriptionPrefix = "passlib.crypto.digest.pbkdf2_hmac() " % ", ".join(PBKDF2_BACKENDS) - - pbkdf2_test_vectors = [ - # (result, secret, salt, rounds, keylen, digest="sha1") - - # - # from rfc 3962 - # - - # test case 1 / 128 bit - ( - hb("cdedb5281bb2f801565a1122b2563515"), - b"password", b"ATHENA.MIT.EDUraeburn", 1, 16 - ), - - # test case 2 / 128 bit - ( - hb("01dbee7f4a9e243e988b62c73cda935d"), - b"password", b"ATHENA.MIT.EDUraeburn", 2, 16 - ), - - # test case 2 / 256 bit - ( - hb("01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86"), - b"password", b"ATHENA.MIT.EDUraeburn", 2, 32 - ), - - # test case 3 / 256 bit - ( - hb("5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13"), - b"password", b"ATHENA.MIT.EDUraeburn", 1200, 32 - ), - - # test case 4 / 256 bit - ( - hb("d1daa78615f287e6a1c8b120d7062a493f98d203e6be49a6adf4fa574b6e64ee"), - b"password", b'\x12\x34\x56\x78\x78\x56\x34\x12', 5, 32 - ), - - # test case 5 / 256 bit - ( - hb("139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1"), - b"X"*64, b"pass phrase equals block size", 1200, 32 - ), - - # test case 6 / 256 bit - ( - hb("9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a"), - b"X"*65, b"pass phrase exceeds block size", 1200, 32 - ), - - # - # from rfc 6070 - # - ( - hb("0c60c80f961f0e71f3a9b524af6012062fe037a6"), - b"password", b"salt", 1, 20, - ), - - ( - hb("ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"), - b"password", b"salt", 2, 20, - ), - - ( - hb("4b007901b765489abead49d926f721d065a429c1"), - b"password", b"salt", 4096, 20, - ), - - # just runs too long - could enable if ALL option is set - ##( - ## - ## hb("eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"), - ## "password", "salt", 16777216, 20, - ##), - - ( - hb("3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038"), - b"passwordPASSWORDpassword", - b"saltSALTsaltSALTsaltSALTsaltSALTsalt", - 4096, 25, - ), - - ( - hb("56fa6aa75548099dcc37d7f03425e0c3"), - b"pass\00word", b"sa\00lt", 4096, 16, - ), - - # - # from example in http://grub.enbug.org/Authentication - # - ( - hb("887CFF169EA8335235D8004242AA7D6187A41E3187DF0CE14E256D85ED" - "97A97357AAA8FF0A3871AB9EEFF458392F462F495487387F685B7472FC" - "6C29E293F0A0"), - b"hello", - hb("9290F727ED06C38BA4549EF7DE25CF5642659211B7FC076F2D28FEFD71" - "784BB8D8F6FB244A8CC5C06240631B97008565A120764C0EE9C2CB0073" - "994D79080136"), - 10000, 64, "sha512" - ), - - # - # test vectors from fastpbkdf2 - # - ( - hb('55ac046e56e3089fec1691c22544b605f94185216dde0465e68b9d57c20dacbc' - '49ca9cccf179b645991664b39d77ef317c71b845b1e30bd509112041d3a19783'), - b'passwd', b'salt', 1, 64, 'sha256', - ), - - ( - hb('4ddcd8f60b98be21830cee5ef22701f9641a4418d04c0414aeff08876b34ab56' - 'a1d425a1225833549adb841b51c9b3176a272bdebba1d078478f62b397f33c8d'), - b'Password', b'NaCl', 80000, 64, 'sha256', - ), - - ( - hb('120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b'), - b'password', b'salt', 1, 32, 'sha256', - ), - - ( - hb('ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43'), - b'password', b'salt', 2, 32, 'sha256', - ), - - ( - hb('c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a'), - b'password', b'salt', 4096, 32, 'sha256', - ), - - ( - hb('348c89dbcbd32b2f32d814b8116e84cf2b17347ebc1800181c4e2a1fb8dd53e1c' - '635518c7dac47e9'), - b'passwordPASSWORDpassword', b'saltSALTsaltSALTsaltSALTsaltSALTsalt', - 4096, 40, 'sha256', - ), - - ( - hb('9e83f279c040f2a11aa4a02b24c418f2d3cb39560c9627fa4f47e3bcc2897c3d'), - b'', b'salt', 1024, 32, 'sha256', - ), - - ( - hb('ea5808411eb0c7e830deab55096cee582761e22a9bc034e3ece925225b07bf46'), - b'password', b'', 1024, 32, 'sha256', - ), - - ( - hb('89b69d0516f829893c696226650a8687'), - b'pass\x00word', b'sa\x00lt', 4096, 16, 'sha256', - ), - - ( - hb('867f70cf1ade02cff3752599a3a53dc4af34c7a669815ae5d513554e1c8cf252'), - b'password', b'salt', 1, 32, 'sha512', - ), - - ( - hb('e1d9c16aa681708a45f5c7c4e215ceb66e011a2e9f0040713f18aefdb866d53c'), - b'password', b'salt', 2, 32, 'sha512', - ), - - ( - hb('d197b1b33db0143e018b12f3d1d1479e6cdebdcc97c5c0f87f6902e072f457b5'), - b'password', b'salt', 4096, 32, 'sha512', - ), - - ( - hb('6e23f27638084b0f7ea1734e0d9841f55dd29ea60a834466f3396bac801fac1eeb' - '63802f03a0b4acd7603e3699c8b74437be83ff01ad7f55dac1ef60f4d56480c35e' - 'e68fd52c6936'), - b'passwordPASSWORDpassword', b'saltSALTsaltSALTsaltSALTsaltSALTsalt', - 1, 72, 'sha512', - ), - - ( - hb('0c60c80f961f0e71f3a9b524af6012062fe037a6'), - b'password', b'salt', 1, 20, 'sha1', - ), - - # - # custom tests - # - ( - hb('e248fb6b13365146f8ac6307cc222812'), - b"secret", b"salt", 10, 16, "sha1", - ), - ( - hb('e248fb6b13365146f8ac6307cc2228127872da6d'), - b"secret", b"salt", 10, None, "sha1", - ), - ( - hb('b1d5485772e6f76d5ebdc11b38d3eff0a5b2bd50dc11f937e86ecacd0cd40d1b' - '9113e0734e3b76a3'), - b"secret", b"salt", 62, 40, "md5", - ), - ( - hb('ea014cc01f78d3883cac364bb5d054e2be238fb0b6081795a9d84512126e3129' - '062104d2183464c4'), - b"secret", b"salt", 62, 40, "md4", - ), - ] - - def test_known(self): - """test reference vectors""" - for row in self.pbkdf2_test_vectors: - correct, secret, salt, rounds, keylen = row[:5] - digest = row[5] if len(row) == 6 else "sha1" - result = pbkdf2_hmac(digest, secret, salt, rounds, keylen) - self.assertEqual(result, correct) - - def test_backends(self): - """verify expected backends are present""" - from passlib.crypto.digest import PBKDF2_BACKENDS - - # check for fastpbkdf2 - try: - import fastpbkdf2 - has_fastpbkdf2 = True - except ImportError: - has_fastpbkdf2 = False - self.assertEqual("fastpbkdf2" in PBKDF2_BACKENDS, has_fastpbkdf2) - - # check for hashlib - try: - from hashlib import pbkdf2_hmac - has_hashlib_ssl = pbkdf2_hmac.__module__ != "hashlib" - except ImportError: - has_hashlib_ssl = False - self.assertEqual("hashlib-ssl" in PBKDF2_BACKENDS, has_hashlib_ssl) - - # check for appropriate builtin - from passlib.utils.compat import PY3 - if PY3: - self.assertIn("builtin-from-bytes", PBKDF2_BACKENDS) - else: - # XXX: only true as long as this is preferred over hexlify - self.assertIn("builtin-unpack", PBKDF2_BACKENDS) - - def test_border(self): - """test border cases""" - def helper(secret=b'password', salt=b'salt', rounds=1, keylen=None, digest="sha1"): - return pbkdf2_hmac(digest, secret, salt, rounds, keylen) - helper() - - # invalid rounds - self.assertRaises(ValueError, helper, rounds=-1) - self.assertRaises(ValueError, helper, rounds=0) - self.assertRaises(TypeError, helper, rounds='x') - - # invalid keylen - helper(keylen=1) - self.assertRaises(ValueError, helper, keylen=-1) - self.assertRaises(ValueError, helper, keylen=0) - # NOTE: hashlib actually throws error for keylen>=MAX_SINT32, - # but pbkdf2 forbids anything > MAX_UINT32 * digest_size - self.assertRaises(OverflowError, helper, keylen=20*(2**32-1)+1) - self.assertRaises(TypeError, helper, keylen='x') - - # invalid secret/salt type - self.assertRaises(TypeError, helper, salt=5) - self.assertRaises(TypeError, helper, secret=5) - - # invalid hash - self.assertRaises(ValueError, helper, digest='foo') - self.assertRaises(TypeError, helper, digest=5) - - def test_default_keylen(self): - """test keylen==None""" - def helper(secret=b'password', salt=b'salt', rounds=1, keylen=None, digest="sha1"): - return pbkdf2_hmac(digest, secret, salt, rounds, keylen) - self.assertEqual(len(helper(digest='sha1')), 20) - self.assertEqual(len(helper(digest='sha256')), 32) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_crypto_scrypt.py b/src/passlib/tests/test_crypto_scrypt.py deleted file mode 100644 index 96666679..00000000 --- a/src/passlib/tests/test_crypto_scrypt.py +++ /dev/null @@ -1,601 +0,0 @@ -"""tests for passlib.utils.scrypt""" -#============================================================================= -# imports -#============================================================================= -# core -from binascii import hexlify -import hashlib -import logging; log = logging.getLogger(__name__) -import struct -import warnings -warnings.filterwarnings("ignore", ".*using builtin scrypt backend.*") -# site -# pkg -from passlib import exc -from passlib.utils import getrandbytes -from passlib.utils.compat import PYPY, u, bascii_to_str -from passlib.utils.decor import classproperty -from passlib.tests.utils import TestCase, skipUnless, TEST_MODE, hb -# subject -from passlib.crypto import scrypt as scrypt_mod -# local -__all__ = [ - "ScryptEngineTest", - "BuiltinScryptTest", - "FastScryptTest", -] - -#============================================================================= -# support functions -#============================================================================= -def hexstr(data): - """return bytes as hex str""" - return bascii_to_str(hexlify(data)) - -def unpack_uint32_list(data, check_count=None): - """unpack bytes as list of uint32 values""" - count = len(data) // 4 - assert check_count is None or check_count == count - return struct.unpack("<%dI" % count, data) - -def seed_bytes(seed, count): - """ - generate random reference bytes from specified seed. - used to generate some predictable test vectors. - """ - if hasattr(seed, "encode"): - seed = seed.encode("ascii") - buf = b'' - i = 0 - while len(buf) < count: - buf += hashlib.sha256(seed + struct.pack("" % cls.backend - backend = None - - #============================================================================= - # setup - #============================================================================= - def setUp(self): - assert self.backend - scrypt_mod._set_backend(self.backend) - super(_CommonScryptTest, self).setUp() - - #============================================================================= - # reference vectors - #============================================================================= - - reference_vectors = [ - # entry format: (secret, salt, n, r, p, keylen, result) - - #------------------------------------------------------------------------ - # test vectors from scrypt whitepaper -- - # http://www.tarsnap.com/scrypt/scrypt.pdf, appendix b - # - # also present in (expired) scrypt rfc draft -- - # https://tools.ietf.org/html/draft-josefsson-scrypt-kdf-01, section 11 - #------------------------------------------------------------------------ - ("", "", 16, 1, 1, 64, hb(""" - 77 d6 57 62 38 65 7b 20 3b 19 ca 42 c1 8a 04 97 - f1 6b 48 44 e3 07 4a e8 df df fa 3f ed e2 14 42 - fc d0 06 9d ed 09 48 f8 32 6a 75 3a 0f c8 1f 17 - e8 d3 e0 fb 2e 0d 36 28 cf 35 e2 0c 38 d1 89 06 - """)), - - ("password", "NaCl", 1024, 8, 16, 64, hb(""" - fd ba be 1c 9d 34 72 00 78 56 e7 19 0d 01 e9 fe - 7c 6a d7 cb c8 23 78 30 e7 73 76 63 4b 37 31 62 - 2e af 30 d9 2e 22 a3 88 6f f1 09 27 9d 98 30 da - c7 27 af b9 4a 83 ee 6d 83 60 cb df a2 cc 06 40 - """)), - - # NOTE: the following are skipped for all backends unless TEST_MODE="full" - - ("pleaseletmein", "SodiumChloride", 16384, 8, 1, 64, hb(""" - 70 23 bd cb 3a fd 73 48 46 1c 06 cd 81 fd 38 eb - fd a8 fb ba 90 4f 8e 3e a9 b5 43 f6 54 5d a1 f2 - d5 43 29 55 61 3f 0f cf 62 d4 97 05 24 2a 9a f9 - e6 1e 85 dc 0d 65 1e 40 df cf 01 7b 45 57 58 87 - """)), - - # NOTE: the following are always skipped for the builtin backend, - # (just takes too long to be worth it) - - ("pleaseletmein", "SodiumChloride", 1048576, 8, 1, 64, hb(""" - 21 01 cb 9b 6a 51 1a ae ad db be 09 cf 70 f8 81 - ec 56 8d 57 4a 2f fd 4d ab e5 ee 98 20 ad aa 47 - 8e 56 fd 8f 4b a5 d0 9f fa 1c 6d 92 7c 40 f4 c3 - 37 30 40 49 e8 a9 52 fb cb f4 5c 6f a7 7a 41 a4 - """)), - ] - - def test_reference_vectors(self): - """reference vectors""" - for secret, salt, n, r, p, keylen, result in self.reference_vectors: - if n >= 1024 and TEST_MODE(max="default"): - # skip large values unless we're running full test suite - continue - if n > 16384 and self.backend == "builtin": - # skip largest vector for builtin, takes WAAY too long - # (46s under pypy, ~5m under cpython) - continue - log.debug("scrypt reference vector: %r %r n=%r r=%r p=%r", secret, salt, n, r, p) - self.assertEqual(scrypt_mod.scrypt(secret, salt, n, r, p, keylen), result) - - #============================================================================= - # fuzz testing - #============================================================================= - - _already_tested_others = None - - def test_other_backends(self): - """compare output to other backends""" - # only run once, since test is symetric. - # maybe this means it should go somewhere else? - if self._already_tested_others: - raise self.skipTest("already run under %r backend test" % self._already_tested_others) - self._already_tested_others = self.backend - rng = self.getRandom() - - # get available backends - orig = scrypt_mod.backend - available = set(name for name in scrypt_mod.backend_values - if scrypt_mod._has_backend(name)) - scrypt_mod._set_backend(orig) - available.discard(self.backend) - if not available: - raise self.skipTest("no other backends found") - - warnings.filterwarnings("ignore", "(?i)using builtin scrypt backend", - category=exc.PasslibSecurityWarning) - - # generate some random options, and cross-check output - for _ in range(10): - # NOTE: keeping values low due to builtin test - secret = getrandbytes(rng, rng.randint(0, 64)) - salt = getrandbytes(rng, rng.randint(0, 64)) - n = 1 << rng.randint(1, 10) - r = rng.randint(1, 8) - p = rng.randint(1, 3) - ks = rng.randint(1, 64) - previous = None - backends = set() - for name in available: - scrypt_mod._set_backend(name) - self.assertNotIn(scrypt_mod._scrypt, backends) - backends.add(scrypt_mod._scrypt) - result = hexstr(scrypt_mod.scrypt(secret, salt, n, r, p, ks)) - self.assertEqual(len(result), 2*ks) - if previous is not None: - self.assertEqual(result, previous, - msg="%r output differs from others %r: %r" % - (name, available, [secret, salt, n, r, p, ks])) - - #============================================================================= - # test input types - #============================================================================= - def test_backend(self): - """backend management""" - # clobber backend - scrypt_mod.backend = None - scrypt_mod._scrypt = None - self.assertRaises(TypeError, scrypt_mod.scrypt, 's', 's', 2, 2, 2, 16) - - # reload backend - scrypt_mod._set_backend(self.backend) - self.assertEqual(scrypt_mod.backend, self.backend) - scrypt_mod.scrypt('s', 's', 2, 2, 2, 16) - - # throw error for unknown backend - self.assertRaises(ValueError, scrypt_mod._set_backend, 'xxx') - self.assertEqual(scrypt_mod.backend, self.backend) - - def test_secret_param(self): - """'secret' parameter""" - - def run_scrypt(secret): - return hexstr(scrypt_mod.scrypt(secret, "salt", 2, 2, 2, 16)) - - # unicode - TEXT = u("abc\u00defg") - self.assertEqual(run_scrypt(TEXT), '05717106997bfe0da42cf4779a2f8bd8') - - # utf8 bytes - TEXT_UTF8 = b'abc\xc3\x9efg' - self.assertEqual(run_scrypt(TEXT_UTF8), '05717106997bfe0da42cf4779a2f8bd8') - - # latin1 bytes - TEXT_LATIN1 = b'abc\xdefg' - self.assertEqual(run_scrypt(TEXT_LATIN1), '770825d10eeaaeaf98e8a3c40f9f441d') - - # accept empty string - self.assertEqual(run_scrypt(""), 'ca1399e5fae5d3b9578dcd2b1faff6e2') - - # reject other types - self.assertRaises(TypeError, run_scrypt, None) - self.assertRaises(TypeError, run_scrypt, 1) - - def test_salt_param(self): - """'salt' parameter""" - - def run_scrypt(salt): - return hexstr(scrypt_mod.scrypt("secret", salt, 2, 2, 2, 16)) - - # unicode - TEXT = u("abc\u00defg") - self.assertEqual(run_scrypt(TEXT), 'a748ec0f4613929e9e5f03d1ab741d88') - - # utf8 bytes - TEXT_UTF8 = b'abc\xc3\x9efg' - self.assertEqual(run_scrypt(TEXT_UTF8), 'a748ec0f4613929e9e5f03d1ab741d88') - - # latin1 bytes - TEXT_LATIN1 = b'abc\xdefg' - self.assertEqual(run_scrypt(TEXT_LATIN1), '91d056fb76fb6e9a7d1cdfffc0a16cd1') - - # reject other types - self.assertRaises(TypeError, run_scrypt, None) - self.assertRaises(TypeError, run_scrypt, 1) - - def test_n_param(self): - """'n' (rounds) parameter""" - - def run_scrypt(n): - return hexstr(scrypt_mod.scrypt("secret", "salt", n, 2, 2, 16)) - - # must be > 1, and a power of 2 - self.assertRaises(ValueError, run_scrypt, -1) - self.assertRaises(ValueError, run_scrypt, 0) - self.assertRaises(ValueError, run_scrypt, 1) - self.assertEqual(run_scrypt(2), 'dacf2bca255e2870e6636fa8c8957a66') - self.assertRaises(ValueError, run_scrypt, 3) - self.assertRaises(ValueError, run_scrypt, 15) - self.assertEqual(run_scrypt(16), '0272b8fc72bc54b1159340ed99425233') - - def test_r_param(self): - """'r' (block size) parameter""" - def run_scrypt(r, n=2, p=2): - return hexstr(scrypt_mod.scrypt("secret", "salt", n, r, p, 16)) - - # must be > 1 - self.assertRaises(ValueError, run_scrypt, -1) - self.assertRaises(ValueError, run_scrypt, 0) - self.assertEqual(run_scrypt(1), '3d630447d9f065363b8a79b0b3670251') - self.assertEqual(run_scrypt(2), 'dacf2bca255e2870e6636fa8c8957a66') - self.assertEqual(run_scrypt(5), '114f05e985a903c27237b5578e763736') - - # reject r*p >= 2**30 - self.assertRaises(ValueError, run_scrypt, (1<<30), p=1) - self.assertRaises(ValueError, run_scrypt, (1<<30) / 2, p=2) - - def test_p_param(self): - """'p' (parallelism) parameter""" - def run_scrypt(p, n=2, r=2): - return hexstr(scrypt_mod.scrypt("secret", "salt", n, r, p, 16)) - - # must be > 1 - self.assertRaises(ValueError, run_scrypt, -1) - self.assertRaises(ValueError, run_scrypt, 0) - self.assertEqual(run_scrypt(1), 'f2960ea8b7d48231fcec1b89b784a6fa') - self.assertEqual(run_scrypt(2), 'dacf2bca255e2870e6636fa8c8957a66') - self.assertEqual(run_scrypt(5), '848a0eeb2b3543e7f543844d6ca79782') - - # reject r*p >= 2**30 - self.assertRaises(ValueError, run_scrypt, (1<<30), r=1) - self.assertRaises(ValueError, run_scrypt, (1<<30) / 2, r=2) - - def test_keylen_param(self): - """'keylen' parameter""" - rng = self.getRandom() - - def run_scrypt(keylen): - return hexstr(scrypt_mod.scrypt("secret", "salt", 2, 2, 2, keylen)) - - # must be > 0 - self.assertRaises(ValueError, run_scrypt, -1) - self.assertRaises(ValueError, run_scrypt, 0) - self.assertEqual(run_scrypt(1), 'da') - - # pick random value - ksize = rng.randint(0, 1 << 10) - self.assertEqual(len(run_scrypt(ksize)), 2*ksize) # 2 hex chars per output - - # one more than upper bound - self.assertRaises(ValueError, run_scrypt, ((2**32) - 1) * 32 + 1) - - #============================================================================= - # eoc - #============================================================================= - -# NOTE: builtin version runs VERY slow (except under PyPy, where it's only 11x slower), -# so skipping under quick test mode. -@skipUnless(PYPY or TEST_MODE(min="default"), "skipped under current test mode") -class BuiltinScryptTest(_CommonScryptTest): - backend = "builtin" - - def setUp(self): - super(BuiltinScryptTest, self).setUp() - warnings.filterwarnings("ignore", "(?i)using builtin scrypt backend", - category=exc.PasslibSecurityWarning) - - def test_missing_backend(self): - """backend management -- missing backend""" - if _can_import_scrypt(): - raise self.skipTest("'scrypt' backend is present") - self.assertRaises(exc.MissingBackendError, scrypt_mod._set_backend, 'scrypt') - -def _can_import_scrypt(): - """check if scrypt package is importable""" - try: - import scrypt - except ImportError as err: - if "scrypt" in str(err): - return False - raise - return True - -@skipUnless(_can_import_scrypt(), "'scrypt' package not found") -class ScryptPackageTest(_CommonScryptTest): - backend = "scrypt" - - def test_default_backend(self): - """backend management -- default backend""" - scrypt_mod._set_backend("default") - self.assertEqual(scrypt_mod.backend, "scrypt") - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_ext_django.py b/src/passlib/tests/test_ext_django.py deleted file mode 100644 index 25513fc4..00000000 --- a/src/passlib/tests/test_ext_django.py +++ /dev/null @@ -1,805 +0,0 @@ -"""test passlib.ext.django""" -#============================================================================= -# imports -#============================================================================= -# core -from __future__ import absolute_import, division, print_function -import logging; log = logging.getLogger(__name__) -import sys -# site -# pkg -from passlib import apps as _apps, exc, registry -from passlib.apps import django10_context, django14_context, django16_context -from passlib.context import CryptContext -from passlib.ext.django.utils import ( - DJANGO_VERSION, MIN_DJANGO_VERSION, DjangoTranslator, -) -from passlib.utils.compat import iteritems, get_method_function, u -from passlib.utils.decor import memoized_property -# tests -from passlib.tests.utils import TestCase, TEST_MODE, handler_derived_from -from passlib.tests.test_handlers import get_handler_case, conditionally_available_hashes -# local -__all__ = [ - "DjangoBehaviorTest", - "ExtensionBehaviorTest", - "_ExtensionSupport", -] -#============================================================================= -# configure django settings for testcases -#============================================================================= - -# whether we have supported django version -has_min_django = DJANGO_VERSION >= MIN_DJANGO_VERSION - -# import and configure empty django settings -# NOTE: we don't want to set up entirety of django, so not using django.setup() directly. -# instead, manually configuring the settings, and setting it up w/ no apps installed. -# in future, may need to alter this so we call django.setup() after setting -# DJANGO_SETTINGS_MODULE to a custom settings module w/ a dummy django app. -if has_min_django: - # - # initialize django settings manually - # - from django.conf import settings, LazySettings - - if not isinstance(settings, LazySettings): - # this probably means django globals have been configured already, - # which we don't want, since test cases reset and manipulate settings. - raise RuntimeError("expected django.conf.settings to be LazySettings: %r" % (settings,)) - - # else configure a blank settings instance for the unittests - if not settings.configured: - settings.configure() - - # - # init django apps w/ NO installed apps. - # NOTE: required for django >= 1.9 - # - from django.apps import apps - apps.populate(["django.contrib.contenttypes", "django.contrib.auth"]) - -#============================================================================= -# support funcs -#============================================================================= - -# flag for update_settings() to remove specified key entirely -UNSET = object() - -def update_settings(**kwds): - """helper to update django settings from kwds""" - for k,v in iteritems(kwds): - if v is UNSET: - if hasattr(settings, k): - delattr(settings, k) - else: - setattr(settings, k, v) - -if has_min_django: - from django.contrib.auth.models import User - - class FakeUser(User): - """mock user object for use in testing""" - # NOTE: this mainly just overrides .save() to test commit behavior. - - # NOTE: .Meta.app_label required for django >= 1.9 - class Meta: - app_label = __name__ - - @memoized_property - def saved_passwords(self): - return [] - - def pop_saved_passwords(self): - try: - return self.saved_passwords[:] - finally: - del self.saved_passwords[:] - - def save(self, update_fields=None): - # NOTE: ignoring update_fields for test purposes - self.saved_passwords.append(self.password) - -def create_mock_setter(): - state = [] - def setter(password): - state.append(password) - def popstate(): - try: - return state[:] - finally: - del state[:] - setter.popstate = popstate - return setter - -#============================================================================= -# work up stock django config -#============================================================================= - -# build config dict that matches stock django -# XXX: move these to passlib.apps? -if DJANGO_VERSION >= (1, 10): - stock_config = _apps.django110_context.to_dict() - stock_rounds = 30000 -elif DJANGO_VERSION >= (1, 9): - stock_config = _apps.django16_context.to_dict() - stock_rounds = 24000 -else: # 1.8 - stock_config = _apps.django16_context.to_dict() - stock_rounds = 20000 - -stock_config.update( - deprecated="auto", - django_pbkdf2_sha1__default_rounds=stock_rounds, - django_pbkdf2_sha256__default_rounds=stock_rounds, -) - -# override sample hashes used in test cases -from passlib.hash import django_pbkdf2_sha256 -sample_hashes = dict( - django_pbkdf2_sha256=("not a password", django_pbkdf2_sha256 - .using(rounds=stock_config.get("django_pbkdf2_sha256__default_rounds")) - .hash("not a password")) -) - -#============================================================================= -# test utils -#============================================================================= -class _ExtensionSupport(object): - """support funcs for loading/unloading extension""" - #=================================================================== - # support funcs - #=================================================================== - @classmethod - def _iter_patch_candidates(cls): - """helper to scan for monkeypatches. - - returns tuple containing: - * object (module or class) - * attribute of object - * value of attribute - * whether it should or should not be patched - """ - # XXX: this and assert_unpatched() could probably be refactored to use - # the PatchManager class to do the heavy lifting. - from django.contrib.auth import models, hashers - user_attrs = ["check_password", "set_password"] - model_attrs = ["check_password", "make_password"] - hasher_attrs = ["check_password", "make_password", "get_hasher", "identify_hasher", - "get_hashers"] - objs = [(models, model_attrs), - (models.User, user_attrs), - (hashers, hasher_attrs), - ] - for obj, patched in objs: - for attr in dir(obj): - if attr.startswith("_"): - continue - value = obj.__dict__.get(attr, UNSET) # can't use getattr() due to GAE - if value is UNSET and attr not in patched: - continue - value = get_method_function(value) - source = getattr(value, "__module__", None) - if source: - yield obj, attr, source, (attr in patched) - - #=================================================================== - # verify current patch state - #=================================================================== - def assert_unpatched(self): - """test that django is in unpatched state""" - # make sure we aren't currently patched - mod = sys.modules.get("passlib.ext.django.models") - self.assertFalse(mod and mod.adapter.patched, "patch should not be enabled") - - # make sure no objects have been replaced, by checking __module__ - for obj, attr, source, patched in self._iter_patch_candidates(): - if patched: - self.assertTrue(source.startswith("django.contrib.auth."), - "obj=%r attr=%r was not reverted: %r" % - (obj, attr, source)) - else: - self.assertFalse(source.startswith("passlib."), - "obj=%r attr=%r should not have been patched: %r" % - (obj, attr, source)) - - def assert_patched(self, context=None): - """helper to ensure django HAS been patched, and is using specified config""" - # make sure we're currently patched - mod = sys.modules.get("passlib.ext.django.models") - self.assertTrue(mod and mod.adapter.patched, "patch should have been enabled") - - # make sure only the expected objects have been patched - for obj, attr, source, patched in self._iter_patch_candidates(): - if patched: - self.assertTrue(source == "passlib.ext.django.utils", - "obj=%r attr=%r should have been patched: %r" % - (obj, attr, source)) - else: - self.assertFalse(source.startswith("passlib."), - "obj=%r attr=%r should not have been patched: %r" % - (obj, attr, source)) - - # check context matches - if context is not None: - context = CryptContext._norm_source(context) - self.assertEqual(mod.password_context.to_dict(resolve=True), - context.to_dict(resolve=True)) - - #=================================================================== - # load / unload the extension (and verify it worked) - #=================================================================== - _config_keys = ["PASSLIB_CONFIG", "PASSLIB_CONTEXT", "PASSLIB_GET_CATEGORY"] - def load_extension(self, check=True, **kwds): - """helper to load extension with specified config & patch django""" - self.unload_extension() - if check: - config = kwds.get("PASSLIB_CONFIG") or kwds.get("PASSLIB_CONTEXT") - for key in self._config_keys: - kwds.setdefault(key, UNSET) - update_settings(**kwds) - import passlib.ext.django.models - if check: - self.assert_patched(context=config) - - def unload_extension(self): - """helper to remove patches and unload extension""" - # remove patches and unload module - mod = sys.modules.get("passlib.ext.django.models") - if mod: - mod.adapter.remove_patch() - del sys.modules["passlib.ext.django.models"] - # wipe config from django settings - update_settings(**dict((key, UNSET) for key in self._config_keys)) - # check everything's gone - self.assert_unpatched() - - #=================================================================== - # eoc - #=================================================================== - -# XXX: rename to ExtensionFixture? -class _ExtensionTest(TestCase, _ExtensionSupport): - - def setUp(self): - super(_ExtensionTest, self).setUp() - - self.require_TEST_MODE("default") - - if not DJANGO_VERSION: - raise self.skipTest("Django not installed") - elif not has_min_django: - raise self.skipTest("Django version too old") - - # reset to baseline, and verify it worked - self.unload_extension() - - # and do the same when the test exits - self.addCleanup(self.unload_extension) - -#============================================================================= -# extension tests -#============================================================================= -class DjangoBehaviorTest(_ExtensionTest): - """tests model to verify it matches django's behavior""" - descriptionPrefix = "verify django behavior" - patched = False - config = stock_config - - # NOTE: if this test fails, it means we're not accounting for - # some part of django's hashing logic, or that this is - # running against an untested version of django with a new - # hashing policy. - - @property - def context(self): - return CryptContext._norm_source(self.config) - - def assert_unusable_password(self, user): - """check that user object is set to 'unusable password' constant""" - self.assertTrue(user.password.startswith("!")) - self.assertFalse(user.has_usable_password()) - self.assertEqual(user.pop_saved_passwords(), []) - - def assert_valid_password(self, user, hash=UNSET, saved=None): - """check that user object has a usuable password hash. - - :param hash: optionally check it has this exact hash - :param saved: check that mock commit history - for user.password matches this list - """ - if hash is UNSET: - self.assertNotEqual(user.password, "!") - self.assertNotEqual(user.password, None) - else: - self.assertEqual(user.password, hash) - self.assertTrue(user.has_usable_password(), - "hash should be usable: %r" % (user.password,)) - self.assertEqual(user.pop_saved_passwords(), - [] if saved is None else [saved]) - - def test_config(self): - """test hashing interface - - this function is run against both the actual django code, to - verify the assumptions of the unittests are correct; - and run against the passlib extension, to verify it matches - those assumptions. - """ - patched, config = self.patched, self.config - # this tests the following methods: - # User.set_password() - # User.check_password() - # make_password() -- 1.4 only - # check_password() - # identify_hasher() - # User.has_usable_password() - # User.set_unusable_password() - # XXX: this take a while to run. what could be trimmed? - - # TODO: get_hasher() - - #======================================================= - # setup helpers & imports - #======================================================= - ctx = self.context - setter = create_mock_setter() - PASS1 = "toomanysecrets" - WRONG1 = "letmein" - - from django.contrib.auth.hashers import (check_password, make_password, - is_password_usable, identify_hasher) - - #======================================================= - # make sure extension is configured correctly - #======================================================= - if patched: - # contexts should match - from passlib.ext.django.models import password_context - self.assertEqual(password_context.to_dict(resolve=True), - ctx.to_dict(resolve=True)) - - # should have patched both places - from django.contrib.auth.models import check_password as check_password2 - self.assertEqual(check_password2, check_password) - - #======================================================= - # default algorithm - #======================================================= - # User.set_password() should use default alg - user = FakeUser() - user.set_password(PASS1) - self.assertTrue(ctx.handler().verify(PASS1, user.password)) - self.assert_valid_password(user) - - # User.check_password() - n/a - - # make_password() should use default alg - hash = make_password(PASS1) - self.assertTrue(ctx.handler().verify(PASS1, hash)) - - # check_password() - n/a - - #======================================================= - # empty password behavior - #======================================================= - - # User.set_password() should use default alg - user = FakeUser() - user.set_password('') - hash = user.password - self.assertTrue(ctx.handler().verify('', hash)) - self.assert_valid_password(user, hash) - - # User.check_password() should return True - self.assertTrue(user.check_password("")) - self.assert_valid_password(user, hash) - - # no make_password() - - # check_password() should return True - self.assertTrue(check_password("", hash)) - - #======================================================= - # 'unusable flag' behavior - #======================================================= - - # sanity check via user.set_unusable_password() - user = FakeUser() - user.set_unusable_password() - self.assert_unusable_password(user) - - # ensure User.set_password() sets unusable flag - user = FakeUser() - user.set_password(None) - self.assert_unusable_password(user) - - # User.check_password() should always fail - self.assertFalse(user.check_password(None)) - self.assertFalse(user.check_password('None')) - self.assertFalse(user.check_password('')) - self.assertFalse(user.check_password(PASS1)) - self.assertFalse(user.check_password(WRONG1)) - self.assert_unusable_password(user) - - # make_password() should also set flag - self.assertTrue(make_password(None).startswith("!")) - - # check_password() should return False (didn't handle disabled under 1.3) - self.assertFalse(check_password(PASS1, '!')) - - # identify_hasher() and is_password_usable() should reject it - self.assertFalse(is_password_usable(user.password)) - self.assertRaises(ValueError, identify_hasher, user.password) - - #======================================================= - # hash=None - #======================================================= - # User.set_password() - n/a - - # User.check_password() - returns False - user = FakeUser() - user.password = None - self.assertFalse(user.check_password(PASS1)) - self.assertFalse(user.has_usable_password()) - - # make_password() - n/a - - # check_password() - error - self.assertFalse(check_password(PASS1, None)) - - # identify_hasher() - error - self.assertRaises(TypeError, identify_hasher, None) - - #======================================================= - # empty & invalid hash values - # NOTE: django 1.5 behavior change due to django ticket 18453 - # NOTE: passlib integration tries to match current django version - #======================================================= - for hash in ("", # empty hash - "$789$foo", # empty identifier - ): - # User.set_password() - n/a - - # User.check_password() - # As of django 1.5, blank OR invalid hash returns False - user = FakeUser() - user.password = hash - self.assertFalse(user.check_password(PASS1)) - - # verify hash wasn't changed/upgraded during check_password() call - self.assertEqual(user.password, hash) - self.assertEqual(user.pop_saved_passwords(), []) - - # User.has_usable_password() - self.assertFalse(user.has_usable_password()) - - # make_password() - n/a - - # check_password() - self.assertFalse(check_password(PASS1, hash)) - - # identify_hasher() - throws error - self.assertRaises(ValueError, identify_hasher, hash) - - #======================================================= - # run through all the schemes in the context, - # testing various bits of per-scheme behavior. - #======================================================= - for scheme in ctx.schemes(): - #------------------------------------------------------- - # setup constants & imports, pick a sample secret/hash combo - #------------------------------------------------------- - handler = ctx.handler(scheme) - deprecated = ctx.handler(scheme).deprecated - assert not deprecated or scheme != ctx.default_scheme() - try: - testcase = get_handler_case(scheme) - except exc.MissingBackendError: - assert scheme in conditionally_available_hashes - continue - assert handler_derived_from(handler, testcase.handler) - if handler.is_disabled: - continue - if not registry.has_backend(handler): - # TODO: move this above get_handler_case(), - # and omit MissingBackendError check. - assert scheme in ["django_bcrypt", "django_bcrypt_sha256", "django_argon2"], \ - "%r scheme should always have active backend" % scheme - continue - try: - secret, hash = sample_hashes[scheme] - except KeyError: - get_sample_hash = testcase("setUp").get_sample_hash - while True: - secret, hash = get_sample_hash() - if secret: # don't select blank passwords - break - other = 'dontletmein' - - # User.set_password() - n/a - - #------------------------------------------------------- - # User.check_password()+migration against known hash - #------------------------------------------------------- - user = FakeUser() - user.password = hash - - # check against invalid password - self.assertFalse(user.check_password(None)) - ##self.assertFalse(user.check_password('')) - self.assertFalse(user.check_password(other)) - self.assert_valid_password(user, hash) - - # check against valid password - self.assertTrue(user.check_password(secret)) - - # check if it upgraded the hash - # NOTE: needs_update kept separate in case we need to test rounds. - needs_update = deprecated - if needs_update: - self.assertNotEqual(user.password, hash) - self.assertFalse(handler.identify(user.password)) - self.assertTrue(ctx.handler().verify(secret, user.password)) - self.assert_valid_password(user, saved=user.password) - else: - self.assert_valid_password(user, hash) - - # don't need to check rest for most deployments - if TEST_MODE(max="default"): - continue - - #------------------------------------------------------- - # make_password() correctly selects algorithm - #------------------------------------------------------- - alg = DjangoTranslator().passlib_to_django_name(scheme) - hash2 = make_password(secret, hasher=alg) - self.assertTrue(handler.verify(secret, hash2)) - - #------------------------------------------------------- - # check_password()+setter against known hash - #------------------------------------------------------- - # should call setter only if it needs_update - self.assertTrue(check_password(secret, hash, setter=setter)) - self.assertEqual(setter.popstate(), [secret] if needs_update else []) - - # should not call setter - self.assertFalse(check_password(other, hash, setter=setter)) - self.assertEqual(setter.popstate(), []) - - ### check preferred kwd is ignored (feature we don't currently support fully) - ##self.assertTrue(check_password(secret, hash, setter=setter, preferred='fooey')) - ##self.assertEqual(setter.popstate(), [secret]) - - # TODO: get_hasher() - - #------------------------------------------------------- - # identify_hasher() recognizes known hash - #------------------------------------------------------- - self.assertTrue(is_password_usable(hash)) - name = DjangoTranslator().django_to_passlib_name(identify_hasher(hash).algorithm) - self.assertEqual(name, scheme) - -class ExtensionBehaviorTest(DjangoBehaviorTest): - """test model to verify passlib.ext.django conforms to it""" - descriptionPrefix = "verify extension behavior" - patched = True - config = dict( - schemes="sha256_crypt,md5_crypt,des_crypt", - deprecated="des_crypt", - ) - - def setUp(self): - super(ExtensionBehaviorTest, self).setUp() - self.load_extension(PASSLIB_CONFIG=self.config) - -class DjangoExtensionTest(_ExtensionTest): - """test the ``passlib.ext.django`` plugin""" - descriptionPrefix = "passlib.ext.django plugin" - - #=================================================================== - # monkeypatch testing - #=================================================================== - def test_00_patch_control(self): - """test set_django_password_context patch/unpatch""" - - # check config="disabled" - self.load_extension(PASSLIB_CONFIG="disabled", check=False) - self.assert_unpatched() - - # check legacy config=None - with self.assertWarningList("PASSLIB_CONFIG=None is deprecated"): - self.load_extension(PASSLIB_CONFIG=None, check=False) - self.assert_unpatched() - - # try stock django 1.0 context - self.load_extension(PASSLIB_CONFIG="django-1.0", check=False) - self.assert_patched(context=django10_context) - - # try to remove patch - self.unload_extension() - - # patch to use stock django 1.4 context - self.load_extension(PASSLIB_CONFIG="django-1.4", check=False) - self.assert_patched(context=django14_context) - - # try to remove patch again - self.unload_extension() - - def test_01_overwrite_detection(self): - """test detection of foreign monkeypatching""" - # NOTE: this sets things up, and spot checks two methods, - # this should be enough to verify patch manager is working. - # TODO: test unpatch behavior honors flag. - - # configure plugin to use sample context - config = "[passlib]\nschemes=des_crypt\n" - self.load_extension(PASSLIB_CONFIG=config) - - # setup helpers - import django.contrib.auth.models as models - from passlib.ext.django.models import adapter - def dummy(): - pass - - # mess with User.set_password, make sure it's detected - orig = models.User.set_password - models.User.set_password = dummy - with self.assertWarningList("another library has patched.*User\.set_password"): - adapter._manager.check_all() - models.User.set_password = orig - - # mess with models.check_password, make sure it's detected - orig = models.check_password - models.check_password = dummy - with self.assertWarningList("another library has patched.*models:check_password"): - adapter._manager.check_all() - models.check_password = orig - - def test_02_handler_wrapper(self): - """test Hasher-compatible handler wrappers""" - from django.contrib.auth import hashers - - passlib_to_django = DjangoTranslator().passlib_to_django - - # should return native django hasher if available - if DJANGO_VERSION > (1,10): - self.assertRaises(ValueError, passlib_to_django, "hex_md5") - else: - hasher = passlib_to_django("hex_md5") - self.assertIsInstance(hasher, hashers.UnsaltedMD5PasswordHasher) - - hasher = passlib_to_django("django_bcrypt") - self.assertIsInstance(hasher, hashers.BCryptPasswordHasher) - - # otherwise should return wrapper - from passlib.hash import sha256_crypt - hasher = passlib_to_django("sha256_crypt") - self.assertEqual(hasher.algorithm, "passlib_sha256_crypt") - - # and wrapper should return correct hash - encoded = hasher.encode("stub") - self.assertTrue(sha256_crypt.verify("stub", encoded)) - self.assertTrue(hasher.verify("stub", encoded)) - self.assertFalse(hasher.verify("xxxx", encoded)) - - # test wrapper accepts options - encoded = hasher.encode("stub", "abcd"*4, rounds=1234) - self.assertEqual(encoded, "$5$rounds=1234$abcdabcdabcdabcd$" - "v2RWkZQzctPdejyRqmmTDQpZN6wTh7.RUy9zF2LftT6") - self.assertEqual(hasher.safe_summary(encoded), - {'algorithm': 'sha256_crypt', - 'salt': u('abcdab**********'), - 'rounds': 1234, - 'hash': u('v2RWkZ*************************************'), - }) - - #=================================================================== - # PASSLIB_CONFIG settings - #=================================================================== - def test_11_config_disabled(self): - """test PASSLIB_CONFIG='disabled'""" - # test config=None (deprecated) - with self.assertWarningList("PASSLIB_CONFIG=None is deprecated"): - self.load_extension(PASSLIB_CONFIG=None, check=False) - self.assert_unpatched() - - # test disabled config - self.load_extension(PASSLIB_CONFIG="disabled", check=False) - self.assert_unpatched() - - def test_12_config_presets(self): - """test PASSLIB_CONFIG=''""" - # test django presets - self.load_extension(PASSLIB_CONTEXT="django-default", check=False) - ctx = django16_context - self.assert_patched(ctx) - - self.load_extension(PASSLIB_CONFIG="django-1.0", check=False) - self.assert_patched(django10_context) - - self.load_extension(PASSLIB_CONFIG="django-1.4", check=False) - self.assert_patched(django14_context) - - def test_13_config_defaults(self): - """test PASSLIB_CONFIG default behavior""" - # check implicit default - from passlib.ext.django.utils import PASSLIB_DEFAULT - default = CryptContext.from_string(PASSLIB_DEFAULT) - self.load_extension() - self.assert_patched(PASSLIB_DEFAULT) - - # check default preset - self.load_extension(PASSLIB_CONTEXT="passlib-default", check=False) - self.assert_patched(PASSLIB_DEFAULT) - - # check explicit string - self.load_extension(PASSLIB_CONTEXT=PASSLIB_DEFAULT, check=False) - self.assert_patched(PASSLIB_DEFAULT) - - def test_14_config_invalid(self): - """test PASSLIB_CONFIG type checks""" - update_settings(PASSLIB_CONTEXT=123, PASSLIB_CONFIG=UNSET) - self.assertRaises(TypeError, __import__, 'passlib.ext.django.models') - - self.unload_extension() - update_settings(PASSLIB_CONFIG="missing-preset", PASSLIB_CONTEXT=UNSET) - self.assertRaises(ValueError, __import__, 'passlib.ext.django.models') - - #=================================================================== - # PASSLIB_GET_CATEGORY setting - #=================================================================== - def test_21_category_setting(self): - """test PASSLIB_GET_CATEGORY parameter""" - # define config where rounds can be used to detect category - config = dict( - schemes = ["sha256_crypt"], - sha256_crypt__default_rounds = 1000, - staff__sha256_crypt__default_rounds = 2000, - superuser__sha256_crypt__default_rounds = 3000, - ) - from passlib.hash import sha256_crypt - - def run(**kwds): - """helper to take in user opts, return rounds used in password""" - user = FakeUser(**kwds) - user.set_password("stub") - return sha256_crypt.from_string(user.password).rounds - - # test default get_category - self.load_extension(PASSLIB_CONFIG=config) - self.assertEqual(run(), 1000) - self.assertEqual(run(is_staff=True), 2000) - self.assertEqual(run(is_superuser=True), 3000) - - # test patch uses explicit get_category function - def get_category(user): - return user.first_name or None - self.load_extension(PASSLIB_CONTEXT=config, - PASSLIB_GET_CATEGORY=get_category) - self.assertEqual(run(), 1000) - self.assertEqual(run(first_name='other'), 1000) - self.assertEqual(run(first_name='staff'), 2000) - self.assertEqual(run(first_name='superuser'), 3000) - - # test patch can disable get_category entirely - def get_category(user): - return None - self.load_extension(PASSLIB_CONTEXT=config, - PASSLIB_GET_CATEGORY=get_category) - self.assertEqual(run(), 1000) - self.assertEqual(run(first_name='other'), 1000) - self.assertEqual(run(first_name='staff', is_staff=True), 1000) - self.assertEqual(run(first_name='superuser', is_superuser=True), 1000) - - # test bad value - self.assertRaises(TypeError, self.load_extension, PASSLIB_CONTEXT=config, - PASSLIB_GET_CATEGORY='x') - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_ext_django_source.py b/src/passlib/tests/test_ext_django_source.py deleted file mode 100644 index 4b42e59b..00000000 --- a/src/passlib/tests/test_ext_django_source.py +++ /dev/null @@ -1,250 +0,0 @@ -""" -test passlib.ext.django against django source tests -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import absolute_import, division, print_function -# core -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils.compat import suppress_cause -from passlib.ext.django.utils import DJANGO_VERSION, DjangoTranslator, _PasslibHasherWrapper -# tests -from passlib.tests.utils import TestCase, TEST_MODE -from .test_ext_django import ( - has_min_django, stock_config, _ExtensionSupport, -) -if has_min_django: - from .test_ext_django import settings -# local -__all__ = [ - "HashersTest", -] -#============================================================================= -# HashersTest -- -# hack up the some of the real django tests to run w/ extension loaded, -# to ensure we mimic their behavior. -# however, the django tests were moved out of the package, and into a source-only location -# as of django 1.7. so we disable tests from that point on unless test-runner specifies -#============================================================================= - -#: ref to django unittest root module (if found) -test_hashers_mod = None - -#: message about why test module isn't present (if not found) -hashers_skip_msg = None - -#---------------------------------------------------------------------- -# try to load django's tests/auth_tests/test_hasher.py module, -# or note why we failed. -#---------------------------------------------------------------------- -if TEST_MODE(max="quick"): - hashers_skip_msg = "requires >= 'default' test mode" - -elif has_min_django: - import os - import sys - source_path = os.environ.get("PASSLIB_TESTS_DJANGO_SOURCE_PATH") - - if source_path: - if not os.path.exists(source_path): - raise EnvironmentError("django source path not found: %r" % source_path) - if not all(os.path.exists(os.path.join(source_path, name)) - for name in ["django", "tests"]): - raise EnvironmentError("invalid django source path: %r" % source_path) - log.info("using django tests from source path: %r", source_path) - tests_path = os.path.join(source_path, "tests") - sys.path.insert(0, tests_path) - try: - from auth_tests import test_hashers as test_hashers_mod - except ImportError as err: - raise suppress_cause( - EnvironmentError("error trying to import django tests " - "from source path (%r): %r" % - (source_path, err))) - finally: - sys.path.remove(tests_path) - - else: - hashers_skip_msg = "requires PASSLIB_TESTS_DJANGO_SOURCE_PATH to be set" - - if TEST_MODE("full"): - # print warning so user knows what's happening - sys.stderr.write("\nWARNING: $PASSLIB_TESTS_DJANGO_SOURCE_PATH is not set; " - "can't run Django's own unittests against passlib.ext.django\n") - -elif DJANGO_VERSION: - hashers_skip_msg = "django version too old" - -else: - hashers_skip_msg = "django not installed" - -#---------------------------------------------------------------------- -# if found module, create wrapper to run django's own tests, -# but with passlib monkeypatched in. -#---------------------------------------------------------------------- -if test_hashers_mod: - from django.core.signals import setting_changed - from django.dispatch import receiver - from django.utils.module_loading import import_string - from passlib.utils.compat import get_unbound_method_function - - class HashersTest(test_hashers_mod.TestUtilsHashPass, _ExtensionSupport): - """ - Run django's hasher unittests against passlib's extension - and workalike implementations - """ - - #================================================================== - # helpers - #================================================================== - - # port patchAttr() helper method from passlib.tests.utils.TestCase - patchAttr = get_unbound_method_function(TestCase.patchAttr) - - #================================================================== - # custom setup - #================================================================== - def setUp(self): - #--------------------------------------------------------- - # install passlib.ext.django adapter, and get context - #--------------------------------------------------------- - self.load_extension(PASSLIB_CONTEXT=stock_config, check=False) - from passlib.ext.django.models import adapter - context = adapter.context - - #--------------------------------------------------------- - # patch tests module to use our versions of patched funcs - # (which should be installed in hashers module) - #--------------------------------------------------------- - from django.contrib.auth import hashers - for attr in ["make_password", - "check_password", - "identify_hasher", - "is_password_usable", - "get_hasher"]: - self.patchAttr(test_hashers_mod, attr, getattr(hashers, attr)) - - #--------------------------------------------------------- - # django tests expect empty django_des_crypt salt field - #--------------------------------------------------------- - from passlib.hash import django_des_crypt - self.patchAttr(django_des_crypt, "use_duplicate_salt", False) - - #--------------------------------------------------------- - # install receiver to update scheme list if test changes settings - #--------------------------------------------------------- - django_to_passlib_name = DjangoTranslator().django_to_passlib_name - - @receiver(setting_changed, weak=False) - def update_schemes(**kwds): - if kwds and kwds['setting'] != 'PASSWORD_HASHERS': - return - assert context is adapter.context - schemes = [ - django_to_passlib_name(import_string(hash_path)()) - for hash_path in settings.PASSWORD_HASHERS - ] - # workaround for a few tests that only specify hex_md5, - # but test for django_salted_md5 format. - if "hex_md5" in schemes and "django_salted_md5" not in schemes: - schemes.append("django_salted_md5") - schemes.append("django_disabled") - context.update(schemes=schemes, deprecated="auto") - adapter.reset_hashers() - - self.addCleanup(setting_changed.disconnect, update_schemes) - - update_schemes() - - #--------------------------------------------------------- - # need password_context to keep up to date with django_hasher.iterations, - # which is frequently patched by django tests. - # - # HACK: to fix this, inserting wrapper around a bunch of context - # methods so that any time adapter calls them, - # attrs are resynced first. - #--------------------------------------------------------- - - def update_rounds(): - """ - sync django hasher config -> passlib hashers - """ - for handler in context.schemes(resolve=True): - if 'rounds' not in handler.setting_kwds: - continue - hasher = adapter.passlib_to_django(handler) - if isinstance(hasher, _PasslibHasherWrapper): - continue - rounds = getattr(hasher, "rounds", None) or \ - getattr(hasher, "iterations", None) - if rounds is None: - continue - # XXX: this doesn't modify the context, which would - # cause other weirdness (since it would replace handler factories completely, - # instead of just updating their state) - handler.min_desired_rounds = handler.max_desired_rounds = handler.default_rounds = rounds - - _in_update = [False] - - def update_wrapper(wrapped, *args, **kwds): - """ - wrapper around arbitrary func, that first triggers sync - """ - if not _in_update[0]: - _in_update[0] = True - try: - update_rounds() - finally: - _in_update[0] = False - return wrapped(*args, **kwds) - - # sync before any context call - for attr in ["schemes", "handler", "default_scheme", "hash", - "verify", "needs_update", "verify_and_update"]: - self.patchAttr(context, attr, update_wrapper, wrap=True) - - # sync whenever adapter tries to resolve passlib hasher - self.patchAttr(adapter, "django_to_passlib", update_wrapper, wrap=True) - - def tearDown(self): - # NOTE: could rely on addCleanup() instead, but need py26 compat - self.unload_extension() - super(HashersTest, self).tearDown() - - #================================================================== - # skip a few methods that can't be replicated properly - # *want to minimize these as much as possible* - #================================================================== - - _OMIT = lambda self: self.skipTest("omitted by passlib") - - # XXX: this test registers two classes w/ same algorithm id, - # something we don't support -- how does django sanely handle - # that anyways? get_hashers_by_algorithm() should throw KeyError, right? - test_pbkdf2_upgrade_new_hasher = _OMIT - - # TODO: support wrapping django's harden-runtime feature? - # would help pass their tests. - test_check_password_calls_harden_runtime = _OMIT - test_bcrypt_harden_runtime = _OMIT - test_pbkdf2_harden_runtime = _OMIT - - #================================================================== - # eoc - #================================================================== - -else: - # otherwise leave a stub so test log tells why test was skipped. - - class HashersTest(TestCase): - - def test_external_django_hasher_tests(self): - """external django hasher tests""" - raise self.skipTest(hashers_skip_msg) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers.py b/src/passlib/tests/test_handlers.py deleted file mode 100644 index 718eb40c..00000000 --- a/src/passlib/tests/test_handlers.py +++ /dev/null @@ -1,1687 +0,0 @@ -"""passlib.tests.test_handlers - tests for passlib hash algorithms""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -import os -import sys -import warnings -# site -# pkg -from passlib import hash -from passlib.utils import repeat_string -from passlib.utils.compat import irange, PY3, u, get_method_function -from passlib.tests.utils import TestCase, HandlerCase, skipUnless, \ - TEST_MODE, UserHandlerMixin, EncodingHandlerMixin -# module - -#============================================================================= -# constants & support -#============================================================================= - -# some common unicode passwords which used as test cases -UPASS_WAV = u('\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2') -UPASS_USD = u("\u20AC\u00A5$") -UPASS_TABLE = u("t\u00e1\u0411\u2113\u0259") - -PASS_TABLE_UTF8 = b't\xc3\xa1\xd0\x91\xe2\x84\x93\xc9\x99' # utf-8 - -# handlers which support multiple backends, but don't have multi-backend tests. -_omitted_backend_tests = ["django_bcrypt", "django_bcrypt_sha256", "django_argon2"] - -#: modules where get_handler_case() should search for test cases. -_handler_test_modules = [ - "test_handlers", - "test_handlers_argon2", - "test_handlers_bcrypt", - "test_handlers_cisco", - "test_handlers_django", - "test_handlers_pbkdf2", - "test_handlers_scrypt", -] - -def get_handler_case(scheme): - """return HandlerCase instance for scheme, used by other tests""" - from passlib.registry import get_crypt_handler - handler = get_crypt_handler(scheme) - if hasattr(handler, "backends") and scheme not in _omitted_backend_tests: - # NOTE: will throw MissingBackendError if none are installed. - backend = handler.get_backend() - name = "%s_%s_test" % (scheme, backend) - else: - name = "%s_test" % scheme - for module in _handler_test_modules: - modname = "passlib.tests." + module - __import__(modname) - mod = sys.modules[modname] - try: - return getattr(mod, name) - except AttributeError: - pass - raise KeyError("test case %r not found" % name) - -#: hashes which there may not be a backend available for, -#: and get_handler_case() may (correctly) throw a MissingBackendError -conditionally_available_hashes = ["argon2", "bcrypt", "bcrypt_sha256"] - -#============================================================================= -# apr md5 crypt -#============================================================================= -class apr_md5_crypt_test(HandlerCase): - handler = hash.apr_md5_crypt - - known_correct_hashes = [ - # - # http://httpd.apache.org/docs/2.2/misc/password_encryptions.html - # - ('myPassword', '$apr1$r31.....$HqJZimcKQFAMYayBlzkrA/'), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, '$apr1$bzYrOHUx$a1FcpXuQDJV3vPY20CS6N1'), - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash ----\/ - '$apr1$r31.....$HqJZimcKQFAMYayBlzkrA!' - ] - -#============================================================================= -# bigcrypt -#============================================================================= -class bigcrypt_test(HandlerCase): - handler = hash.bigcrypt - - # TODO: find an authoritative source of test vectors - known_correct_hashes = [ - - # - # various docs & messages on the web. - # - ("passphrase", "qiyh4XPJGsOZ2MEAyLkfWqeQ"), - ("This is very long passwd", "f8.SVpL2fvwjkAnxn8/rgTkwvrif6bjYB5c"), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, 'SEChBAyMbMNhgGLyP7kD1HZU'), - ] - - known_unidentified_hashes = [ - # one char short (10 % 11) - "qiyh4XPJGsOZ2MEAyLkfWqe" - - # one char too many (1 % 11) - "f8.SVpL2fvwjkAnxn8/rgTkwvrif6bjYB5cd" - ] - - # omit des_crypt from known_other since it's a valid bigcrypt hash too. - known_other_hashes = [row for row in HandlerCase.known_other_hashes - if row[0] != "des_crypt"] - - def test_90_internal(self): - # check that _norm_checksum() also validates checksum size. - # (current code uses regex in parser) - self.assertRaises(ValueError, hash.bigcrypt, use_defaults=True, - checksum=u('yh4XPJGsOZ')) - -#============================================================================= -# bsdi crypt -#============================================================================= -class _bsdi_crypt_test(HandlerCase): - """test BSDiCrypt algorithm""" - handler = hash.bsdi_crypt - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('U*U*U*U*', '_J9..CCCCXBrJUJV154M'), - ('U*U***U', '_J9..CCCCXUhOBTXzaiE'), - ('U*U***U*', '_J9..CCCC4gQ.mB/PffM'), - ('*U*U*U*U', '_J9..XXXXvlzQGqpPPdk'), - ('*U*U*U*U*', '_J9..XXXXsqM/YSSP..Y'), - ('*U*U*U*U*U*U*U*U', '_J9..XXXXVL7qJCnku0I'), - ('*U*U*U*U*U*U*U*U*', '_J9..XXXXAj8cFbP5scI'), - ('ab1234567', '_J9..SDizh.vll5VED9g'), - ('cr1234567', '_J9..SDizRjWQ/zePPHc'), - ('zxyDPWgydbQjgq', '_J9..SDizxmRI1GjnQuE'), - ('726 even', '_K9..SaltNrQgIYUAeoY'), - ('', '_J9..SDSD5YGyRCr4W4c'), - - # - # custom - # - (" ", "_K1..crsmZxOLzfJH8iw"), - ("my", '_KR/.crsmykRplHbAvwA'), # <-- to detect old 12-bit rounds bug - ("my socra", "_K1..crsmf/9NzZr1fLM"), - ("my socrates", '_K1..crsmOv1rbde9A9o'), - ("my socrates note", "_K1..crsm/2qeAhdISMA"), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '_7C/.ABw0WIKy0ILVqo2'), - ] - known_unidentified_hashes = [ - # bad char in otherwise correctly formatted hash - # \/ - "_K1.!crsmZxOLzfJH8iw" - ] - - platform_crypt_support = [ - ("freebsd|openbsd|netbsd|darwin", True), - ("linux|solaris", False), - ] - - def test_77_fuzz_input(self, **kwds): - # we want to generate even rounds to verify it's correct, but want to ignore warnings - warnings.filterwarnings("ignore", "bsdi_crypt rounds should be odd.*") - super(_bsdi_crypt_test, self).test_77_fuzz_input(**kwds) - - def test_needs_update_w_even_rounds(self): - """needs_update() should flag even rounds""" - handler = self.handler - even_hash = '_Y/../cG0zkJa6LY6k4c' - odd_hash = '_Z/..TgFg0/ptQtpAgws' - secret = 'test' - - # don't issue warning - self.assertTrue(handler.verify(secret, even_hash)) - self.assertTrue(handler.verify(secret, odd_hash)) - - # *do* signal as needing updates - self.assertTrue(handler.needs_update(even_hash)) - self.assertFalse(handler.needs_update(odd_hash)) - - # new hashes shouldn't have even rounds - new_hash = handler.hash("stub") - self.assertFalse(handler.needs_update(new_hash)) - -# create test cases for specific backends -bsdi_crypt_os_crypt_test = _bsdi_crypt_test.create_backend_case("os_crypt") -bsdi_crypt_builtin_test = _bsdi_crypt_test.create_backend_case("builtin") - -#============================================================================= -# crypt16 -#============================================================================= -class crypt16_test(HandlerCase): - handler = hash.crypt16 - - # TODO: find an authortative source of test vectors - known_correct_hashes = [ - # - # from messages around the web, including - # http://seclists.org/bugtraq/1999/Mar/76 - # - ("passphrase", "qi8H8R7OM4xMUNMPuRAZxlY."), - ("printf", "aaCjFz4Sh8Eg2QSqAReePlq6"), - ("printf", "AA/xje2RyeiSU0iBY3PDwjYo"), - ("LOLOAQICI82QB4IP", "/.FcK3mad6JwYt8LVmDqz9Lc"), - ("LOLOAQICI", "/.FcK3mad6JwYSaRHJoTPzY2"), - ("LOLOAQIC", "/.FcK3mad6JwYelhbtlysKy6"), - ("L", "/.CIu/PzYCkl6elhbtlysKy6"), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, 'YeDc9tKkkmDvwP7buzpwhoqQ'), - ] - -#============================================================================= -# des crypt -#============================================================================= -class _des_crypt_test(HandlerCase): - """test des-crypt algorithm""" - handler = hash.des_crypt - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('U*U*U*U*', 'CCNf8Sbh3HDfQ'), - ('U*U***U', 'CCX.K.MFy4Ois'), - ('U*U***U*', 'CC4rMpbg9AMZ.'), - ('*U*U*U*U', 'XXxzOu6maQKqQ'), - ('', 'SDbsugeBiC58A'), - - # - # custom - # - ('', 'OgAwTx2l6NADI'), - (' ', '/Hk.VPuwQTXbc'), - ('test', 'N1tQbOFcM5fpg'), - ('Compl3X AlphaNu3meric', 'um.Wguz3eVCx2'), - ('4lpHa N|_|M3r1K W/ Cur5Es: #$%(*)(*%#', 'sNYqfOyauIyic'), - ('AlOtBsOl', 'cEpWz5IUCShqM'), - - # ensures utf-8 used for unicode - (u('hell\u00D6'), 'saykDgk3BPZ9E'), - ] - known_unidentified_hashes = [ - # bad char in otherwise correctly formatted hash - #\/ - '!gAwTx2l6NADI', - - # wrong size - 'OgAwTx2l6NAD', - 'OgAwTx2l6NADIj', - ] - - platform_crypt_support = [ - ("freebsd|openbsd|netbsd|linux|solaris|darwin", True), - ] - -# create test cases for specific backends -des_crypt_os_crypt_test = _des_crypt_test.create_backend_case("os_crypt") -des_crypt_builtin_test = _des_crypt_test.create_backend_case("builtin") - -#============================================================================= -# fshp -#============================================================================= -class fshp_test(HandlerCase): - """test fshp algorithm""" - handler = hash.fshp - - known_correct_hashes = [ - # - # test vectors from FSHP reference implementation - # https://github.com/bdd/fshp-is-not-secure-anymore/blob/master/python/test.py - # - ('test', '{FSHP0|0|1}qUqP5cyxm6YcTAhz05Hph5gvu9M='), - - ('test', - '{FSHP1|8|4096}MTIzNDU2NzjTdHcmoXwNc0f' - 'f9+ArUHoN0CvlbPZpxFi1C6RDM/MHSA==' - ), - - ('OrpheanBeholderScryDoubt', - '{FSHP1|8|4096}GVSUFDAjdh0vBosn1GUhz' - 'GLHP7BmkbCZVH/3TQqGIjADXpc+6NCg3g==' - ), - ('ExecuteOrder66', - '{FSHP3|16|8192}0aY7rZQ+/PR+Rd5/I9ss' - 'RM7cjguyT8ibypNaSp/U1uziNO3BVlg5qPU' - 'ng+zHUDQC3ao/JbzOnIBUtAeWHEy7a2vZeZ' - '7jAwyJJa2EqOsq4Io=' - ), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, '{FSHP1|16|16384}9v6/l3Lu/d9by5nznpOS' - 'cqQo8eKu/b/CKli3RCkgYg4nRTgZu5y659YV8cCZ68UL'), - ] - - known_unidentified_hashes = [ - # incorrect header - '{FSHX0|0|1}qUqP5cyxm6YcTAhz05Hph5gvu9M=', - 'FSHP0|0|1}qUqP5cyxm6YcTAhz05Hph5gvu9M=', - ] - - known_malformed_hashes = [ - # bad base64 padding - '{FSHP0|0|1}qUqP5cyxm6YcTAhz05Hph5gvu9M', - - # wrong salt size - '{FSHP0|1|1}qUqP5cyxm6YcTAhz05Hph5gvu9M=', - - # bad rounds - '{FSHP0|0|A}qUqP5cyxm6YcTAhz05Hph5gvu9M=', - ] - - def test_90_variant(self): - """test variant keyword""" - handler = self.handler - kwds = dict(salt=b'a', rounds=1) - - # accepts ints - handler(variant=1, **kwds) - - # accepts bytes or unicode - handler(variant=u('1'), **kwds) - handler(variant=b'1', **kwds) - - # aliases - handler(variant=u('sha256'), **kwds) - handler(variant=b'sha256', **kwds) - - # rejects None - self.assertRaises(TypeError, handler, variant=None, **kwds) - - # rejects other types - self.assertRaises(TypeError, handler, variant=complex(1,1), **kwds) - - # invalid variant - self.assertRaises(ValueError, handler, variant='9', **kwds) - self.assertRaises(ValueError, handler, variant=9, **kwds) - -#============================================================================= -# hex digests -#============================================================================= -class hex_md4_test(HandlerCase): - handler = hash.hex_md4 - known_correct_hashes = [ - ("password", '8a9d093f14f8701df17732b2bb182c74'), - (UPASS_TABLE, '876078368c47817ce5f9115f3a42cf74'), - ] - -class hex_md5_test(HandlerCase): - handler = hash.hex_md5 - known_correct_hashes = [ - ("password", '5f4dcc3b5aa765d61d8327deb882cf99'), - (UPASS_TABLE, '05473f8a19f66815e737b33264a0d0b0'), - ] - -class hex_sha1_test(HandlerCase): - handler = hash.hex_sha1 - known_correct_hashes = [ - ("password", '5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8'), - (UPASS_TABLE, 'e059b2628e3a3e2de095679de9822c1d1466e0f0'), - ] - -class hex_sha256_test(HandlerCase): - handler = hash.hex_sha256 - known_correct_hashes = [ - ("password", '5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8'), - (UPASS_TABLE, '6ed729e19bf24d3d20f564375820819932029df05547116cfc2cc868a27b4493'), - ] - -class hex_sha512_test(HandlerCase): - handler = hash.hex_sha512 - known_correct_hashes = [ - ("password", 'b109f3bbbc244eb82441917ed06d618b9008dd09b3befd1b5e07394c' - '706a8bb980b1d7785e5976ec049b46df5f1326af5a2ea6d103fd07c95385ffab0cac' - 'bc86'), - (UPASS_TABLE, 'd91bb0a23d66dca07a1781fd63ae6a05f6919ee5fc368049f350c9f' - '293b078a18165d66097cf0d89fdfbeed1ad6e7dba2344e57348cd6d51308c843a06f' - '29caf'), - ] - -#============================================================================= -# htdigest hash -#============================================================================= -class htdigest_test(UserHandlerMixin, HandlerCase): - handler = hash.htdigest - - known_correct_hashes = [ - # secret, user, realm - - # from RFC 2617 - (("Circle Of Life", "Mufasa", "testrealm@host.com"), - '939e7578ed9e3c518a452acee763bce9'), - - # custom - ((UPASS_TABLE, UPASS_USD, UPASS_WAV), - '4dabed2727d583178777fab468dd1f17'), - ] - - known_unidentified_hashes = [ - # bad char \/ - currently rejecting upper hex chars, may change - '939e7578edAe3c518a452acee763bce9', - - # bad char \/ - '939e7578edxe3c518a452acee763bce9', - ] - - def test_80_user(self): - raise self.skipTest("test case doesn't support 'realm' keyword") - - def populate_context(self, secret, kwds): - """insert username into kwds""" - if isinstance(secret, tuple): - secret, user, realm = secret - else: - user, realm = "user", "realm" - kwds.setdefault("user", user) - kwds.setdefault("realm", realm) - return secret - -#============================================================================= -# ldap hashes -#============================================================================= -class ldap_md5_test(HandlerCase): - handler = hash.ldap_md5 - known_correct_hashes = [ - ("helloworld", '{MD5}/F4DjTilcDIIVEHn/nAQsA=='), - (UPASS_TABLE, '{MD5}BUc/ihn2aBXnN7MyZKDQsA=='), - ] - -class ldap_sha1_test(HandlerCase): - handler = hash.ldap_sha1 - known_correct_hashes = [ - ("helloworld", '{SHA}at+xg6SiyUovktq1redipHiJpaE='), - (UPASS_TABLE, '{SHA}4FmyYo46Pi3glWed6YIsHRRm4PA='), - ] - -class ldap_salted_md5_test(HandlerCase): - handler = hash.ldap_salted_md5 - known_correct_hashes = [ - ("testing1234", '{SMD5}UjFY34os/pnZQ3oQOzjqGu4yeXE='), - (UPASS_TABLE, '{SMD5}Z0ioJ58LlzUeRxm3K6JPGAvBGIM='), - - # alternate salt sizes (8, 15, 16) - ('test', '{SMD5}LnuZPJhiaY95/4lmVFpg548xBsD4P4cw'), - ('test', '{SMD5}XRlncfRzvGi0FDzgR98tUgBg7B3jXOs9p9S615qTkg=='), - ('test', '{SMD5}FbAkzOMOxRbMp6Nn4hnZuel9j9Gas7a2lvI+x5hT6j0='), - ] - - known_malformed_hashes = [ - # salt too small (3) - '{SMD5}IGVhwK+anvspmfDt2t0vgGjt/Q==', - - # incorrect base64 encoding - '{SMD5}LnuZPJhiaY95/4lmVFpg548xBsD4P4c', - '{SMD5}LnuZPJhiaY95/4lmVFpg548xBsD4P4cw' - '{SMD5}LnuZPJhiaY95/4lmVFpg548xBsD4P4cw=', - '{SMD5}LnuZPJhiaY95/4lmV=pg548xBsD4P4cw', - '{SMD5}LnuZPJhiaY95/4lmVFpg548xBsD4P===', - ] - -class ldap_salted_sha1_test(HandlerCase): - handler = hash.ldap_salted_sha1 - known_correct_hashes = [ - ("testing123", '{SSHA}0c0blFTXXNuAMHECS4uxrj3ZieMoWImr'), - ("secret", "{SSHA}0H+zTv8o4MR4H43n03eCsvw1luG8LdB7"), - (UPASS_TABLE, '{SSHA}3yCSD1nLZXznra4N8XzZgAL+s1sQYsx5'), - - # alternate salt sizes (8, 15, 16) - ('test', '{SSHA}P90+qijSp8MJ1tN25j5o1PflUvlqjXHOGeOckw=='), - ('test', '{SSHA}/ZMF5KymNM+uEOjW+9STKlfCFj51bg3BmBNCiPHeW2ttbU0='), - ('test', '{SSHA}Pfx6Vf48AT9x3FVv8znbo8WQkEVSipHSWovxXmvNWUvp/d/7'), - ] - - known_malformed_hashes = [ - # salt too small (3) - '{SSHA}ZQK3Yvtvl6wtIRoISgMGPkcWU7Nfq5U=', - - # incorrect base64 encoding - '{SSHA}P90+qijSp8MJ1tN25j5o1PflUvlqjXHOGeOck', - '{SSHA}P90+qijSp8MJ1tN25j5o1PflUvlqjXHOGeOckw=', - '{SSHA}P90+qijSp8MJ1tN25j5o1Pf=UvlqjXHOGeOckw==', - '{SSHA}P90+qijSp8MJ1tN25j5o1PflUvlqjXHOGeOck===', - ] - -class ldap_plaintext_test(HandlerCase): - # TODO: integrate EncodingHandlerMixin - handler = hash.ldap_plaintext - known_correct_hashes = [ - ("password", 'password'), - (UPASS_TABLE, UPASS_TABLE if PY3 else PASS_TABLE_UTF8), - (PASS_TABLE_UTF8, UPASS_TABLE if PY3 else PASS_TABLE_UTF8), - ] - known_unidentified_hashes = [ - "{FOO}bar", - - # NOTE: this hash currently rejects the empty string. - "", - ] - - known_other_hashes = [ - ("ldap_md5", "{MD5}/F4DjTilcDIIVEHn/nAQsA==") - ] - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def random_password(self): - # NOTE: this hash currently rejects the empty string. - while True: - pwd = super(ldap_plaintext_test.FuzzHashGenerator, self).random_password() - if pwd: - return pwd - -class _ldap_md5_crypt_test(HandlerCase): - # NOTE: since the ldap_{crypt} handlers are all wrappers, don't need - # separate test; this is just to test the codebase end-to-end - handler = hash.ldap_md5_crypt - - known_correct_hashes = [ - # - # custom - # - ('', '{CRYPT}$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o.'), - (' ', '{CRYPT}$1$m/5ee7ol$bZn0kIBFipq39e.KDXX8I0'), - ('test', '{CRYPT}$1$ec6XvcoW$ghEtNK2U1MC5l.Dwgi3020'), - ('Compl3X AlphaNu3meric', '{CRYPT}$1$nX1e7EeI$ljQn72ZUgt6Wxd9hfvHdV0'), - ('4lpHa N|_|M3r1K W/ Cur5Es: #$%(*)(*%#', '{CRYPT}$1$jQS7o98J$V6iTcr71CGgwW2laf17pi1'), - ('test', '{CRYPT}$1$SuMrG47N$ymvzYjr7QcEQjaK5m1PGx1'), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '{CRYPT}$1$d6/Ky1lU$/xpf8m7ftmWLF.TjHCqel0'), - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash - '{CRYPT}$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o!', - ] - -# create test cases for specific backends -ldap_md5_crypt_os_crypt_test =_ldap_md5_crypt_test.create_backend_case("os_crypt") -ldap_md5_crypt_builtin_test =_ldap_md5_crypt_test.create_backend_case("builtin") - -class _ldap_sha1_crypt_test(HandlerCase): - # NOTE: this isn't for testing the hash (see ldap_md5_crypt note) - # but as a self-test of the os_crypt patching code in HandlerCase. - handler = hash.ldap_sha1_crypt - - known_correct_hashes = [ - ('password', '{CRYPT}$sha1$10$c.mcTzCw$gF8UeYst9yXX7WNZKc5Fjkq0.au7'), - (UPASS_TABLE, '{CRYPT}$sha1$10$rnqXlOsF$aGJf.cdRPewJAXo1Rn1BkbaYh0fP'), - ] - - def populate_settings(self, kwds): - kwds.setdefault("rounds", 10) - super(_ldap_sha1_crypt_test, self).populate_settings(kwds) - - def test_77_fuzz_input(self): - raise self.skipTest("unneeded") - -# create test cases for specific backends -ldap_sha1_crypt_os_crypt_test = _ldap_sha1_crypt_test.create_backend_case("os_crypt") - -#============================================================================= -# lanman -#============================================================================= -class lmhash_test(EncodingHandlerMixin, HandlerCase): - handler = hash.lmhash - secret_case_insensitive = True - - known_correct_hashes = [ - # - # http://msdn.microsoft.com/en-us/library/cc245828(v=prot.10).aspx - # - ("OLDPASSWORD", "c9b81d939d6fd80cd408e6b105741864"), - ("NEWPASSWORD", '09eeab5aa415d6e4d408e6b105741864'), - ("welcome", "c23413a8a1e7665faad3b435b51404ee"), - - # - # custom - # - ('', 'aad3b435b51404eeaad3b435b51404ee'), - ('zzZZZzz', 'a5e6066de61c3e35aad3b435b51404ee'), - ('passphrase', '855c3697d9979e78ac404c4ba2c66533'), - ('Yokohama', '5ecd9236d21095ce7584248b8d2c9f9e'), - - # ensures cp437 used for unicode - (u('ENCYCLOP\xC6DIA'), 'fed6416bffc9750d48462b9d7aaac065'), - (u('encyclop\xE6dia'), 'fed6416bffc9750d48462b9d7aaac065'), - - # test various encoding values - ((u("\xC6"), None), '25d8ab4a0659c97aaad3b435b51404ee'), - ((u("\xC6"), "cp437"), '25d8ab4a0659c97aaad3b435b51404ee'), - ((u("\xC6"), "latin-1"), '184eecbbe9991b44aad3b435b51404ee'), - ((u("\xC6"), "utf-8"), '00dd240fcfab20b8aad3b435b51404ee'), - ] - - known_unidentified_hashes = [ - # bad char in otherwise correct hash - '855c3697d9979e78ac404c4ba2c6653X', - ] - - def test_90_raw(self): - """test lmhash.raw() method""" - from binascii import unhexlify - from passlib.utils.compat import str_to_bascii - lmhash = self.handler - for secret, hash in self.known_correct_hashes: - kwds = {} - secret = self.populate_context(secret, kwds) - data = unhexlify(str_to_bascii(hash)) - self.assertEqual(lmhash.raw(secret, **kwds), data) - self.assertRaises(TypeError, lmhash.raw, 1) - -#============================================================================= -# md5 crypt -#============================================================================= -class _md5_crypt_test(HandlerCase): - handler = hash.md5_crypt - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('U*U*U*U*', '$1$dXc3I7Rw$ctlgjDdWJLMT.qwHsWhXR1'), - ('U*U***U', '$1$dXc3I7Rw$94JPyQc/eAgQ3MFMCoMF.0'), - ('U*U***U*', '$1$dXc3I7Rw$is1mVIAEtAhIzSdfn5JOO0'), - ('*U*U*U*U', '$1$eQT9Hwbt$XtuElNJD.eW5MN5UCWyTQ0'), - ('', '$1$Eu.GHtia$CFkL/nE1BYTlEPiVx1VWX0'), - - # - # custom - # - - # NOTE: would need to patch HandlerCase to coerce hashes - # to native str for this first one to work under py3. -## ('', b('$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o.')), - ('', '$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o.'), - (' ', '$1$m/5ee7ol$bZn0kIBFipq39e.KDXX8I0'), - ('test', '$1$ec6XvcoW$ghEtNK2U1MC5l.Dwgi3020'), - ('Compl3X AlphaNu3meric', '$1$nX1e7EeI$ljQn72ZUgt6Wxd9hfvHdV0'), - ('4lpHa N|_|M3r1K W/ Cur5Es: #$%(*)(*%#', '$1$jQS7o98J$V6iTcr71CGgwW2laf17pi1'), - ('test', '$1$SuMrG47N$ymvzYjr7QcEQjaK5m1PGx1'), - (b'test', '$1$SuMrG47N$ymvzYjr7QcEQjaK5m1PGx1'), - (u('s'), '$1$ssssssss$YgmLTApYTv12qgTwBoj8i/'), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '$1$d6/Ky1lU$/xpf8m7ftmWLF.TjHCqel0'), - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash \/ - '$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o!', - - # too many fields - '$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o.$', - ] - - platform_crypt_support = [ - ("freebsd|openbsd|netbsd|linux|solaris", True), - ("darwin", False), - ] - -# create test cases for specific backends -md5_crypt_os_crypt_test = _md5_crypt_test.create_backend_case("os_crypt") -md5_crypt_builtin_test = _md5_crypt_test.create_backend_case("builtin") - -#============================================================================= -# msdcc 1 & 2 -#============================================================================= -class msdcc_test(UserHandlerMixin, HandlerCase): - handler = hash.msdcc - user_case_insensitive = True - - known_correct_hashes = [ - - # - # http://www.jedge.com/wordpress/windows-password-cache/ - # - (("Asdf999", "sevans"), "b1176c2587478785ec1037e5abc916d0"), - - # - # http://infosecisland.com/blogview/12156-Cachedump-for-Meterpreter-in-Action.html - # - (("ASDqwe123", "jdoe"), "592cdfbc3f1ef77ae95c75f851e37166"), - - # - # http://comments.gmane.org/gmane.comp.security.openwall.john.user/1917 - # - (("test1", "test1"), "64cd29e36a8431a2b111378564a10631"), - (("test2", "test2"), "ab60bdb4493822b175486810ac2abe63"), - (("test3", "test3"), "14dd041848e12fc48c0aa7a416a4a00c"), - (("test4", "test4"), "b945d24866af4b01a6d89b9d932a153c"), - - # - # http://ciscoit.wordpress.com/2011/04/13/metasploit-hashdump-vs-cachedump/ - # - (("1234qwer!@#$", "Administrator"), "7b69d06ef494621e3f47b9802fe7776d"), - - # - # http://www.securiteam.com/tools/5JP0I2KFPA.html - # - (("password", "user"), "2d9f0b052932ad18b87f315641921cda"), - - # - # from JTR 1.7.9 - # - (("", "root"), "176a4c2bd45ac73687676c2f09045353"), - (("test1", "TEST1"), "64cd29e36a8431a2b111378564a10631"), - (("okolada", "nineteen_characters"), "290efa10307e36a79b3eebf2a6b29455"), - ((u("\u00FC"), u("\u00FC")), "48f84e6f73d6d5305f6558a33fa2c9bb"), - ((u("\u00FC\u00FC"), u("\u00FC\u00FC")), "593246a8335cf0261799bda2a2a9c623"), - ((u("\u20AC\u20AC"), "user"), "9121790702dda0fa5d353014c334c2ce"), - - # - # custom - # - - # ensures utf-8 used for unicode - ((UPASS_TABLE, 'bob'), 'fcb82eb4212865c7ac3503156ca3f349'), - ] - - known_alternate_hashes = [ - # check uppercase accepted. - ("B1176C2587478785EC1037E5ABC916D0", ("Asdf999", "sevans"), - "b1176c2587478785ec1037e5abc916d0"), - ] - -class msdcc2_test(UserHandlerMixin, HandlerCase): - handler = hash.msdcc2 - user_case_insensitive = True - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - (("test1", "test1"), "607bbe89611e37446e736f7856515bf8"), - (("qerwt", "Joe"), "e09b38f84ab0be586b730baf61781e30"), - (("12345", "Joe"), "6432f517a900b3fc34ffe57f0f346e16"), - (("", "bin"), "c0cbe0313a861062e29f92ede58f9b36"), - (("w00t", "nineteen_characters"), "87136ae0a18b2dafe4a41d555425b2ed"), - (("w00t", "eighteencharacters"), "fc5df74eca97afd7cd5abb0032496223"), - (("longpassword", "twentyXXX_characters"), "cfc6a1e33eb36c3d4f84e4c2606623d2"), - (("longpassword", "twentyoneX_characters"), "99ff74cea552799da8769d30b2684bee"), - (("longpassword", "twentytwoXX_characters"), "0a721bdc92f27d7fb23b87a445ec562f"), - (("test2", "TEST2"), "c6758e5be7fc943d00b97972a8a97620"), - (("test3", "test3"), "360e51304a2d383ea33467ab0b639cc4"), - (("test4", "test4"), "6f79ee93518306f071c47185998566ae"), - ((u("\u00FC"), "joe"), "bdb80f2c4656a8b8591bd27d39064a54"), - ((u("\u20AC\u20AC"), "joe"), "1e1e20f482ff748038e47d801d0d1bda"), - ((u("\u00FC\u00FC"), "admin"), "0839e4a07c00f18a8c65cf5b985b9e73"), - - # - # custom - # - - # custom unicode test - ((UPASS_TABLE, 'bob'), 'cad511dc9edefcf69201da72efb6bb55'), - ] - -#============================================================================= -# mssql 2000 & 2005 -#============================================================================= -class mssql2000_test(HandlerCase): - handler = hash.mssql2000 - secret_case_insensitive = "verify-only" - # FIXME: fix UT framework - this hash is sensitive to password case, but verify() is not - - known_correct_hashes = [ - # - # http://hkashfi.blogspot.com/2007/08/breaking-sql-server-2005-hashes.html - # - ('Test', '0x010034767D5C0CFA5FDCA28C4A56085E65E882E71CB0ED2503412FD54D6119FFF04129A1D72E7C3194F7284A7F3A'), - ('TEST', '0x010034767D5C2FD54D6119FFF04129A1D72E7C3194F7284A7F3A2FD54D6119FFF04129A1D72E7C3194F7284A7F3A'), - - # - # http://www.sqlmag.com/forums/aft/68438 - # - ('x', '0x010086489146C46DD7318D2514D1AC706457CBF6CD3DF8407F071DB4BBC213939D484BF7A766E974F03C96524794'), - - # - # http://stackoverflow.com/questions/173329/how-to-decrypt-a-password-from-sql-server - # - ('AAAA', '0x0100CF465B7B12625EF019E157120D58DD46569AC7BF4118455D12625EF019E157120D58DD46569AC7BF4118455D'), - - # - # http://msmvps.com/blogs/gladchenko/archive/2005/04/06/41083.aspx - # - ('123', '0x01002D60BA07FE612C8DE537DF3BFCFA49CD9968324481C1A8A8FE612C8DE537DF3BFCFA49CD9968324481C1A8A8'), - - # - # http://www.simple-talk.com/sql/t-sql-programming/temporarily-changing-an-unknown-password-of-the-sa-account-/ - # - ('12345', '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3'), - - # - # XXX: sample is incomplete, password unknown - # https://anthonystechblog.wordpress.com/2011/04/20/password-encryption-in-sql-server-how-to-tell-if-a-user-is-using-a-weak-password/ - # (????, '0x0100813F782D66EF15E40B1A3FDF7AB88B322F51401A87D8D3E3A8483C4351A3D96FC38499E6CDD2B6F?????????'), - # - - # - # from JTR 1.7.9 - # - ('foo', '0x0100A607BA7C54A24D17B565C59F1743776A10250F581D482DA8B6D6261460D3F53B279CC6913CE747006A2E3254'), - ('bar', '0x01000508513EADDF6DB7DDD270CCA288BF097F2FF69CC2DB74FBB9644D6901764F999BAB9ECB80DE578D92E3F80D'), - ('canard', '0x01008408C523CF06DCB237835D701C165E68F9460580132E28ED8BC558D22CEDF8801F4503468A80F9C52A12C0A3'), - ('lapin', '0x0100BF088517935FC9183FE39FDEC77539FD5CB52BA5F5761881E5B9638641A79DBF0F1501647EC941F3355440A2'), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_USD, '0x0100624C0961B28E39FEE13FD0C35F57B4523F0DA1861C11D5A5B28E39FEE13FD0C35F57B4523F0DA1861C11D5A5'), - (UPASS_TABLE, '0x010083104228FAD559BE52477F2131E538BE9734E5C4B0ADEFD7F6D784B03C98585DC634FE2B8CA3A6DFFEC729B4'), - - ] - - known_alternate_hashes = [ - # lower case hex - ('0x01005b20054332752e1bc2e7c5df0f9ebfe486e9bee063e8d3b332752e1bc2e7c5df0f9ebfe486e9bee063e8d3b3', - '12345', '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3'), - ] - - known_unidentified_hashes = [ - # malformed start - '0X01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3', - - # wrong magic value - '0x02005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3', - - # wrong size - '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3', - '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3AF', - - # mssql2005 - '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3', - ] - - known_malformed_hashes = [ - # non-hex char -----\/ - b'0x01005B200543327G2E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3', - u('0x01005B200543327G2E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3'), - ] - -class mssql2005_test(HandlerCase): - handler = hash.mssql2005 - - known_correct_hashes = [ - # - # http://hkashfi.blogspot.com/2007/08/breaking-sql-server-2005-hashes.html - # - ('TEST', '0x010034767D5C2FD54D6119FFF04129A1D72E7C3194F7284A7F3A'), - - # - # http://www.openwall.com/lists/john-users/2009/07/14/2 - # - ('toto', '0x01004086CEB6BF932BC4151A1AF1F13CD17301D70816A8886908'), - - # - # http://msmvps.com/blogs/gladchenko/archive/2005/04/06/41083.aspx - # - ('123', '0x01004A335DCEDB366D99F564D460B1965B146D6184E4E1025195'), - ('123', '0x0100E11D573F359629B344990DCD3D53DE82CF8AD6BBA7B638B6'), - - # - # XXX: password unknown - # http://www.simple-talk.com/sql/t-sql-programming/temporarily-changing-an-unknown-password-of-the-sa-account-/ - # (???, '0x01004086CEB6301EEC0A994E49E30DA235880057410264030797'), - # - - # - # http://therelentlessfrontend.com/2010/03/26/encrypting-and-decrypting-passwords-in-sql-server/ - # - ('AAAA', '0x010036D726AE86834E97F20B198ACD219D60B446AC5E48C54F30'), - - # - # from JTR 1.7.9 - # - ("toto", "0x01004086CEB6BF932BC4151A1AF1F13CD17301D70816A8886908"), - ("titi", "0x01004086CEB60ED526885801C23B366965586A43D3DEAC6DD3FD"), - ("foo", "0x0100A607BA7C54A24D17B565C59F1743776A10250F581D482DA8"), - ("bar", "0x01000508513EADDF6DB7DDD270CCA288BF097F2FF69CC2DB74FB"), - ("canard", "0x01008408C523CF06DCB237835D701C165E68F9460580132E28ED"), - ("lapin", "0x0100BF088517935FC9183FE39FDEC77539FD5CB52BA5F5761881"), - - # - # adapted from mssql2000.known_correct_hashes (above) - # - ('Test', '0x010034767D5C0CFA5FDCA28C4A56085E65E882E71CB0ED250341'), - ('Test', '0x0100993BF2315F36CC441485B35C4D84687DC02C78B0E680411F'), - ('x', '0x010086489146C46DD7318D2514D1AC706457CBF6CD3DF8407F07'), - ('AAAA', '0x0100CF465B7B12625EF019E157120D58DD46569AC7BF4118455D'), - ('123', '0x01002D60BA07FE612C8DE537DF3BFCFA49CD9968324481C1A8A8'), - ('12345', '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3'), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_USD, '0x0100624C0961B28E39FEE13FD0C35F57B4523F0DA1861C11D5A5'), - (UPASS_TABLE, '0x010083104228FAD559BE52477F2131E538BE9734E5C4B0ADEFD7'), - ] - - known_alternate_hashes = [ - # lower case hex - ('0x01005b20054332752e1bc2e7c5df0f9ebfe486e9bee063e8d3b3', - '12345', '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3'), - ] - - known_unidentified_hashes = [ - # malformed start - '0X010036D726AE86834E97F20B198ACD219D60B446AC5E48C54F30', - - # wrong magic value - '0x020036D726AE86834E97F20B198ACD219D60B446AC5E48C54F30', - - # wrong size - '0x010036D726AE86834E97F20B198ACD219D60B446AC5E48C54F', - '0x010036D726AE86834E97F20B198ACD219D60B446AC5E48C54F3012', - - # mssql2000 - '0x01005B20054332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B332752E1BC2E7C5DF0F9EBFE486E9BEE063E8D3B3', - ] - - known_malformed_hashes = [ - # non-hex char --\/ - '0x010036D726AE86G34E97F20B198ACD219D60B446AC5E48C54F30', - ] - -#============================================================================= -# mysql 323 & 41 -#============================================================================= -class mysql323_test(HandlerCase): - handler = hash.mysql323 - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('drew', '697a7de87c5390b2'), - ('password', "5d2e19393cc5ef67"), - - # - # custom - # - ('mypass', '6f8c114b58f2ce9e'), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '4ef327ca5491c8d7'), - ] - - known_unidentified_hashes = [ - # bad char in otherwise correct hash - '6z8c114b58f2ce9e', - ] - - def test_90_whitespace(self): - """check whitespace is ignored per spec""" - h = self.do_encrypt("mypass") - h2 = self.do_encrypt("my pass") - self.assertEqual(h, h2) - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def accept_password_pair(self, secret, other): - # override to handle whitespace - return secret.replace(" ","") != other.replace(" ","") - -class mysql41_test(HandlerCase): - handler = hash.mysql41 - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('verysecretpassword', '*2C905879F74F28F8570989947D06A8429FB943E6'), - ('12345678123456781234567812345678', '*F9F1470004E888963FB466A5452C9CBD9DF6239C'), - ("' OR 1 /*'", '*97CF7A3ACBE0CA58D5391AC8377B5D9AC11D46D9'), - - # - # custom - # - ('mypass', '*6C8989366EAF75BB670AD8EA7A7FC1176A95CEF4'), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '*E7AFE21A9CFA2FC9D15D942AE8FB5C240FE5837B'), - ] - known_unidentified_hashes = [ - # bad char in otherwise correct hash - '*6Z8989366EAF75BB670AD8EA7A7FC1176A95CEF4', - ] - -#============================================================================= -# NTHASH -#============================================================================= -class nthash_test(HandlerCase): - handler = hash.nthash - - known_correct_hashes = [ - # - # http://msdn.microsoft.com/en-us/library/cc245828(v=prot.10).aspx - # - ("OLDPASSWORD", u("6677b2c394311355b54f25eec5bfacf5")), - ("NEWPASSWORD", u("256781a62031289d3c2c98c14f1efc8c")), - - # - # from JTR 1.7.9 - # - - # ascii - ('', '31d6cfe0d16ae931b73c59d7e0c089c0'), - ('tigger', 'b7e0ea9fbffcf6dd83086e905089effd'), - - # utf-8 - (b'\xC3\xBC', '8bd6e4fb88e01009818749c5443ea712'), - (b'\xC3\xBC\xC3\xBC', 'cc1260adb6985ca749f150c7e0b22063'), - (b'\xE2\x82\xAC', '030926b781938db4365d46adc7cfbcb8'), - (b'\xE2\x82\xAC\xE2\x82\xAC','682467b963bb4e61943e170a04f7db46'), - - # - # custom - # - ('passphrase', '7f8fe03093cc84b267b109625f6bbf4b'), - ] - - known_unidentified_hashes = [ - # bad char in otherwise correct hash - '7f8fe03093cc84b267b109625f6bbfxb', - ] - -class bsd_nthash_test(HandlerCase): - handler = hash.bsd_nthash - - known_correct_hashes = [ - ('passphrase', '$3$$7f8fe03093cc84b267b109625f6bbf4b'), - (b'\xC3\xBC', '$3$$8bd6e4fb88e01009818749c5443ea712'), - ] - - known_unidentified_hashes = [ - # bad char in otherwise correct hash --\/ - '$3$$7f8fe03093cc84b267b109625f6bbfxb', - ] - -#============================================================================= -# oracle 10 & 11 -#============================================================================= -class oracle10_test(UserHandlerMixin, HandlerCase): - handler = hash.oracle10 - secret_case_insensitive = True - user_case_insensitive = True - - # TODO: get more test vectors (especially ones which properly test unicode) - known_correct_hashes = [ - # ((secret,user),hash) - - # - # http://www.petefinnigan.com/default/default_password_list.htm - # - (('tiger', 'scott'), 'F894844C34402B67'), - ((u('ttTiGGeR'), u('ScO')), '7AA1A84E31ED7771'), - (("d_syspw", "SYSTEM"), '1B9F1F9A5CB9EB31'), - (("strat_passwd", "strat_user"), 'AEBEDBB4EFB5225B'), - - # - # http://openwall.info/wiki/john/sample-hashes - # - (('#95LWEIGHTS', 'USER'), '000EA4D72A142E29'), - (('CIAO2010', 'ALFREDO'), 'EB026A76F0650F7B'), - - # - # from JTR 1.7.9 - # - (('GLOUGlou', 'Bob'), 'CDC6B483874B875B'), - (('GLOUGLOUTER', 'bOB'), 'EF1F9139DB2D5279'), - (('LONG_MOT_DE_PASSE_OUI', 'BOB'), 'EC8147ABB3373D53'), - - # - # custom - # - ((UPASS_TABLE, 'System'), 'B915A853F297B281'), - ] - - known_unidentified_hashes = [ - # bad char in hash --\ - 'F894844C34402B6Z', - ] - -class oracle11_test(HandlerCase): - handler = hash.oracle11 - # TODO: find more test vectors (especially ones which properly test unicode) - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ("abc123", "S:5FDAB69F543563582BA57894FE1C1361FB8ED57B903603F2C52ED1B4D642"), - ("SyStEm123!@#", "S:450F957ECBE075D2FA009BA822A9E28709FBC3DA82B44D284DDABEC14C42"), - ("oracle", "S:3437FF72BD69E3FB4D10C750B92B8FB90B155E26227B9AB62D94F54E5951"), - ("11g", "S:61CE616647A4F7980AFD7C7245261AF25E0AFE9C9763FCF0D54DA667D4E6"), - ("11g", "S:B9E7556F53500C8C78A58F50F24439D79962DE68117654B6700CE7CC71CF"), - - # - # source? - # - ("SHAlala", "S:2BFCFDF5895014EE9BB2B9BA067B01E0389BB5711B7B5F82B7235E9E182C"), - - # - # custom - # - (UPASS_TABLE, 'S:51586343E429A6DF024B8F242F2E9F8507B1096FACD422E29142AA4974B0'), - ] - -#============================================================================= -# PHPass Portable Crypt -#============================================================================= -class phpass_test(HandlerCase): - handler = hash.phpass - - known_correct_hashes = [ - # - # from official 0.3 implementation - # http://www.openwall.com/phpass/ - # - ('test12345', '$P$9IQRaTwmfeRo7ud9Fh4E2PdI0S3r.L0'), # from the source - - # - # from JTR 1.7.9 - # - ('test1', '$H$9aaaaaSXBjgypwqm.JsMssPLiS8YQ00'), - ('123456', '$H$9PE8jEklgZhgLmZl5.HYJAzfGCQtzi1'), - ('123456', '$H$9pdx7dbOW3Nnt32sikrjAxYFjX8XoK1'), - ('thisisalongertestPW', '$P$912345678LIjjb6PhecupozNBmDndU0'), - ('JohnRipper', '$P$612345678si5M0DDyPpmRCmcltU/YW/'), - ('JohnRipper', '$H$712345678WhEyvy1YWzT4647jzeOmo0'), - ('JohnRipper', '$P$B12345678L6Lpt4BxNotVIMILOa9u81'), - - # - # custom - # - ('', '$P$7JaFQsPzJSuenezefD/3jHgt5hVfNH0'), - ('compL3X!', '$P$FiS0N5L672xzQx1rt1vgdJQRYKnQM9/'), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '$P$7SMy8VxnfsIy2Sxm7fJxDSdil.h7TW.'), - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash - # ---\/ - '$P$9IQRaTwmfeRo7ud9Fh4E2PdI0S3r!L0', - ] - -#============================================================================= -# plaintext -#============================================================================= -class plaintext_test(HandlerCase): - # TODO: integrate EncodingHandlerMixin - handler = hash.plaintext - accepts_all_hashes = True - - known_correct_hashes = [ - ('',''), - ('password', 'password'), - - # ensure unicode uses utf-8 - (UPASS_TABLE, UPASS_TABLE if PY3 else PASS_TABLE_UTF8), - (PASS_TABLE_UTF8, UPASS_TABLE if PY3 else PASS_TABLE_UTF8), - ] - -#============================================================================= -# postgres_md5 -#============================================================================= -class postgres_md5_test(UserHandlerMixin, HandlerCase): - handler = hash.postgres_md5 - known_correct_hashes = [ - # ((secret,user),hash) - - # - # generated using postgres 8.1 - # - (('mypass', 'postgres'), 'md55fba2ea04fd36069d2574ea71c8efe9d'), - (('mypass', 'root'), 'md540c31989b20437833f697e485811254b'), - (("testpassword",'testuser'), 'md5d4fc5129cc2c25465a5370113ae9835f'), - - # - # custom - # - - # verify unicode->utf8 - ((UPASS_TABLE, 'postgres'), 'md5cb9f11283265811ce076db86d18a22d2'), - ] - known_unidentified_hashes = [ - # bad 'z' char in otherwise correct hash - 'md54zc31989b20437833f697e485811254b', - ] - -#============================================================================= -# (netbsd's) sha1 crypt -#============================================================================= -class _sha1_crypt_test(HandlerCase): - handler = hash.sha1_crypt - - known_correct_hashes = [ - # - # custom - # - ("password", "$sha1$19703$iVdJqfSE$v4qYKl1zqYThwpjJAoKX6UvlHq/a"), - ("password", "$sha1$21773$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH"), - (UPASS_TABLE, '$sha1$40000$uJ3Sp7LE$.VEmLO5xntyRFYihC7ggd3297T/D'), - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash - '$sha1$21773$u!7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH', - - # zero padded rounds - '$sha1$01773$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH', - - # too many fields - '$sha1$21773$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH$', - - # empty rounds field - '$sha1$$uV7PTeux$I9oHnvwPZHMO0Nq6/WgyGV/tDJIH$', - ] - - platform_crypt_support = [ - ("netbsd", True), - ("freebsd|openbsd|linux|solaris|darwin", False), - ] - -# create test cases for specific backends -sha1_crypt_os_crypt_test = _sha1_crypt_test.create_backend_case("os_crypt") -sha1_crypt_builtin_test = _sha1_crypt_test.create_backend_case("builtin") - -#============================================================================= -# roundup -#============================================================================= - -# NOTE: all roundup hashes use PrefixWrapper, -# so there's nothing natively to test. -# so we just have a few quick cases... - -class RoundupTest(TestCase): - - def _test_pair(self, h, secret, hash): - self.assertTrue(h.verify(secret, hash)) - self.assertFalse(h.verify('x'+secret, hash)) - - def test_pairs(self): - self._test_pair( - hash.ldap_hex_sha1, - "sekrit", - '{SHA}8d42e738c7adee551324955458b5e2c0b49ee655') - - self._test_pair( - hash.ldap_hex_md5, - "sekrit", - '{MD5}ccbc53f4464604e714f69dd11138d8b5') - - self._test_pair( - hash.ldap_des_crypt, - "sekrit", - '{CRYPT}nFia0rj2TT59A') - - self._test_pair( - hash.roundup_plaintext, - "sekrit", - '{plaintext}sekrit') - - self._test_pair( - hash.ldap_pbkdf2_sha1, - "sekrit", - '{PBKDF2}5000$7BvbBq.EZzz/O0HuwX3iP.nAG3s$g3oPnFFaga2BJaX5PoPRljl4XIE') - -#============================================================================= -# sha256-crypt -#============================================================================= -class _sha256_crypt_test(HandlerCase): - handler = hash.sha256_crypt - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('U*U*U*U*', '$5$LKO/Ute40T3FNF95$U0prpBQd4PloSGU0pnpM4z9wKn4vZ1.jsrzQfPqxph9'), - ('U*U***U', '$5$LKO/Ute40T3FNF95$fdgfoJEBoMajNxCv3Ru9LyQ0xZgv0OBMQoq80LQ/Qd.'), - ('U*U***U*', '$5$LKO/Ute40T3FNF95$8Ry82xGnnPI/6HtFYnvPBTYgOL23sdMXn8C29aO.x/A'), - ('*U*U*U*U', '$5$9mx1HkCz7G1xho50$O7V7YgleJKLUhcfk9pgzdh3RapEaWqMtEp9UUBAKIPA'), - ('', '$5$kc7lRD1fpYg0g.IP$d7CMTcEqJyTXyeq8hTdu/jB/I6DGkoo62NXbHIR7S43'), - - # - # custom tests - # - ('', '$5$rounds=10428$uy/jIAhCetNCTtb0$YWvUOXbkqlqhyoPMpN8BMe.ZGsGx2aBvxTvDFI613c3'), - (' ', '$5$rounds=10376$I5lNtXtRmf.OoMd8$Ko3AI1VvTANdyKhBPavaRjJzNpSatKU6QVN9uwS9MH.'), - ('test', '$5$rounds=11858$WH1ABM5sKhxbkgCK$aTQsjPkz0rBsH3lQlJxw9HDTDXPKBxC0LlVeV69P.t1'), - ('Compl3X AlphaNu3meric', '$5$rounds=10350$o.pwkySLCzwTdmQX$nCMVsnF3TXWcBPOympBUUSQi6LGGloZoOsVJMGJ09UB'), - ('4lpHa N|_|M3r1K W/ Cur5Es: #$%(*)(*%#', '$5$rounds=11944$9dhlu07dQMRWvTId$LyUI5VWkGFwASlzntk1RLurxX54LUhgAcJZIt0pYGT7'), - (u('with unic\u00D6de'), '$5$rounds=1000$IbG0EuGQXw5EkMdP$LQ5AfPf13KufFsKtmazqnzSGZ4pxtUNw3woQ.ELRDF4'), - ] - - if TEST_MODE("full"): - # builtin alg was changed in 1.6, and had possibility of fencepost - # errors near rounds that are multiples of 42. these hashes test rounds - # 1004..1012 (42*24=1008 +/- 4) to ensure no mistakes were made. - # (also relying on fuzz testing against os_crypt backend). - known_correct_hashes.extend([ - ("secret", '$5$rounds=1004$nacl$oiWPbm.kQ7.jTCZoOtdv7/tO5mWv/vxw5yTqlBagVR7'), - ("secret", '$5$rounds=1005$nacl$6Mo/TmGDrXxg.bMK9isRzyWH3a..6HnSVVsJMEX7ud/'), - ("secret", '$5$rounds=1006$nacl$I46VwuAiUBwmVkfPFakCtjVxYYaOJscsuIeuZLbfKID'), - ("secret", '$5$rounds=1007$nacl$9fY4j1AV3N/dV/YMUn1enRHKH.7nEL4xf1wWB6wfDD4'), - ("secret", '$5$rounds=1008$nacl$CiFWCfn8ODmWs0I1xAdXFo09tM8jr075CyP64bu3by9'), - ("secret", '$5$rounds=1009$nacl$QtpFX.CJHgVQ9oAjVYStxAeiU38OmFILWm684c6FyED'), - ("secret", '$5$rounds=1010$nacl$ktAwXuT5WbjBW/0ZU1eNMpqIWY1Sm4twfRE1zbZyo.B'), - ("secret", '$5$rounds=1011$nacl$QJWLBEhO9qQHyMx4IJojSN9sS41P1Yuz9REddxdO721'), - ("secret", '$5$rounds=1012$nacl$mmf/k2PkbBF4VCtERgky3bEVavmLZKFwAcvxD1p3kV2'), - ]) - - known_malformed_hashes = [ - # bad char in otherwise correct hash - '$5$rounds=10428$uy/:jIAhCetNCTtb0$YWvUOXbkqlqhyoPMpN8BMeZGsGx2aBvxTvDFI613c3', - - # zero-padded rounds - '$5$rounds=010428$uy/jIAhCetNCTtb0$YWvUOXbkqlqhyoPMpN8BMe.ZGsGx2aBvxTvDFI613c3', - - # extra "$" - '$5$rounds=10428$uy/jIAhCetNCTtb0$YWvUOXbkqlqhyoPMpN8BMe.ZGsGx2aBvxTvDFI613c3$', - ] - - known_correct_configs = [ - # config, secret, result - - # - # taken from official specification at http://www.akkadia.org/drepper/SHA-crypt.txt - # - ( "$5$saltstring", "Hello world!", - "$5$saltstring$5B8vYYiY.CVt1RlTTf8KbXBH3hsxY/GNooZaBBGWEc5" ), - ( "$5$rounds=10000$saltstringsaltstring", "Hello world!", - "$5$rounds=10000$saltstringsaltst$3xv.VbSHBb41AL9AvLeujZkZRBAwqFMz2." - "opqey6IcA" ), - ( "$5$rounds=5000$toolongsaltstring", "This is just a test", - "$5$rounds=5000$toolongsaltstrin$Un/5jzAHMgOGZ5.mWJpuVolil07guHPvOW8" - "mGRcvxa5" ), - ( "$5$rounds=1400$anotherlongsaltstring", - "a very much longer text to encrypt. This one even stretches over more" - "than one line.", - "$5$rounds=1400$anotherlongsalts$Rx.j8H.h8HjEDGomFU8bDkXm3XIUnzyxf12" - "oP84Bnq1" ), - ( "$5$rounds=77777$short", - "we have a short salt string but not a short password", - "$5$rounds=77777$short$JiO1O3ZpDAxGJeaDIuqCoEFysAe1mZNJRs3pw0KQRd/" ), - ( "$5$rounds=123456$asaltof16chars..", "a short string", - "$5$rounds=123456$asaltof16chars..$gP3VQ/6X7UUEW3HkBn2w1/Ptq2jxPyzV/" - "cZKmF/wJvD" ), - ( "$5$rounds=10$roundstoolow", "the minimum number is still observed", - "$5$rounds=1000$roundstoolow$yfvwcWrQ8l/K0DAWyuPMDNHpIVlTQebY9l/gL97" - "2bIC" ), - ] - - filter_config_warnings = True # rounds too low, salt too small - - platform_crypt_support = [ - ("freebsd(9|1\d)|linux", True), - ("freebsd8", None), # added in freebsd 8.3 - ("freebsd|openbsd|netbsd|darwin", False), - # solaris - depends on policy - ] - -# create test cases for specific backends -sha256_crypt_os_crypt_test = _sha256_crypt_test.create_backend_case("os_crypt") -sha256_crypt_builtin_test = _sha256_crypt_test.create_backend_case("builtin") - -#============================================================================= -# test sha512-crypt -#============================================================================= -class _sha512_crypt_test(HandlerCase): - handler = hash.sha512_crypt - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('U*U*U*U*', "$6$LKO/Ute40T3FNF95$6S/6T2YuOIHY0N3XpLKABJ3soYcXD9mB7uVbtEZDj/LNscVhZoZ9DEH.sBciDrMsHOWOoASbNLTypH/5X26gN0"), - ('U*U***U', "$6$LKO/Ute40T3FNF95$wK80cNqkiAUzFuVGxW6eFe8J.fSVI65MD5yEm8EjYMaJuDrhwe5XXpHDJpwF/kY.afsUs1LlgQAaOapVNbggZ1"), - ('U*U***U*', "$6$LKO/Ute40T3FNF95$YS81pp1uhOHTgKLhSMtQCr2cDiUiN03Ud3gyD4ameviK1Zqz.w3oXsMgO6LrqmIEcG3hiqaUqHi/WEE2zrZqa/"), - ('*U*U*U*U', "$6$OmBOuxFYBZCYAadG$WCckkSZok9xhp4U1shIZEV7CCVwQUwMVea7L3A77th6SaE9jOPupEMJB.z0vIWCDiN9WLh2m9Oszrj5G.gt330"), - ('', "$6$ojWH1AiTee9x1peC$QVEnTvRVlPRhcLQCk/HnHaZmlGAAjCfrAN0FtOsOnUk5K5Bn/9eLHHiRzrTzaIKjW9NTLNIBUCtNVOowWS2mN."), - - # - # custom tests - # - ('', '$6$rounds=11021$KsvQipYPWpr93wWP$v7xjI4X6vyVptJjB1Y02vZC5SaSijBkGmq1uJhPr3cvqvvkd42Xvo48yLVPFt8dvhCsnlUgpX.//Cxn91H4qy1'), - (' ', '$6$rounds=11104$ED9SA4qGmd57Fq2m$q/.PqACDM/JpAHKmr86nkPzzuR5.YpYa8ZJJvI8Zd89ZPUYTJExsFEIuTYbM7gAGcQtTkCEhBKmp1S1QZwaXx0'), - ('test', '$6$rounds=11531$G/gkPn17kHYo0gTF$Kq.uZBHlSBXyzsOJXtxJruOOH4yc0Is13uY7yK0PvAvXxbvc1w8DO1RzREMhKsc82K/Jh8OquV8FZUlreYPJk1'), - ('Compl3X AlphaNu3meric', '$6$rounds=10787$wakX8nGKEzgJ4Scy$X78uqaX1wYXcSCtS4BVYw2trWkvpa8p7lkAtS9O/6045fK4UB2/Jia0Uy/KzCpODlfVxVNZzCCoV9s2hoLfDs/'), - ('4lpHa N|_|M3r1K W/ Cur5Es: #$%(*)(*%#', '$6$rounds=11065$5KXQoE1bztkY5IZr$Jf6krQSUKKOlKca4hSW07MSerFFzVIZt/N3rOTsUgKqp7cUdHrwV8MoIVNCk9q9WL3ZRMsdbwNXpVk0gVxKtz1'), - - # ensures utf-8 used for unicode - (UPASS_TABLE, '$6$rounds=40000$PEZTJDiyzV28M3.m$GTlnzfzGB44DGd1XqlmC4erAJKCP.rhvLvrYxiT38htrNzVGBnplFOHjejUGVrCfusGWxLQCc3pFO0A/1jYYr0'), - ] - - known_malformed_hashes = [ - # zero-padded rounds - '$6$rounds=011021$KsvQipYPWpr93wWP$v7xjI4X6vyVptJjB1Y02vZC5SaSijBkGmq1uJhPr3cvqvvkd42Xvo48yLVPFt8dvhCsnlUgpX.//Cxn91H4qy1', - # bad char in otherwise correct hash - '$6$rounds=11021$KsvQipYPWpr9:wWP$v7xjI4X6vyVptJjB1Y02vZC5SaSijBkGmq1uJhPr3cvqvvkd42Xvo48yLVPFt8dvhCsnlUgpX.//Cxn91H4qy1', - ] - - known_correct_configs = [ - # config, secret, result - - # - # taken from official specification at http://www.akkadia.org/drepper/SHA-crypt.txt - # - ("$6$saltstring", "Hello world!", - "$6$saltstring$svn8UoSVapNtMuq1ukKS4tPQd8iKwSMHWjl/O817G3uBnIFNjnQJu" - "esI68u4OTLiBFdcbYEdFCoEOfaS35inz1" ), - - ( "$6$rounds=10000$saltstringsaltstring", "Hello world!", - "$6$rounds=10000$saltstringsaltst$OW1/O6BYHV6BcXZu8QVeXbDWra3Oeqh0sb" - "HbbMCVNSnCM/UrjmM0Dp8vOuZeHBy/YTBmSK6H9qs/y3RnOaw5v." ), - - ( "$6$rounds=5000$toolongsaltstring", "This is just a test", - "$6$rounds=5000$toolongsaltstrin$lQ8jolhgVRVhY4b5pZKaysCLi0QBxGoNeKQ" - "zQ3glMhwllF7oGDZxUhx1yxdYcz/e1JSbq3y6JMxxl8audkUEm0" ), - - ( "$6$rounds=1400$anotherlongsaltstring", - "a very much longer text to encrypt. This one even stretches over more" - "than one line.", - "$6$rounds=1400$anotherlongsalts$POfYwTEok97VWcjxIiSOjiykti.o/pQs.wP" - "vMxQ6Fm7I6IoYN3CmLs66x9t0oSwbtEW7o7UmJEiDwGqd8p4ur1" ), - - ( "$6$rounds=77777$short", - "we have a short salt string but not a short password", - "$6$rounds=77777$short$WuQyW2YR.hBNpjjRhpYD/ifIw05xdfeEyQoMxIXbkvr0g" - "ge1a1x3yRULJ5CCaUeOxFmtlcGZelFl5CxtgfiAc0" ), - - ( "$6$rounds=123456$asaltof16chars..", "a short string", - "$6$rounds=123456$asaltof16chars..$BtCwjqMJGx5hrJhZywWvt0RLE8uZ4oPwc" - "elCjmw2kSYu.Ec6ycULevoBK25fs2xXgMNrCzIMVcgEJAstJeonj1" ), - - ( "$6$rounds=10$roundstoolow", "the minimum number is still observed", - "$6$rounds=1000$roundstoolow$kUMsbe306n21p9R.FRkW3IGn.S9NPN0x50YhH1x" - "hLsPuWGsUSklZt58jaTfF4ZEQpyUNGc0dqbpBYYBaHHrsX." ), - ] - - filter_config_warnings = True # rounds too low, salt too small - - platform_crypt_support = _sha256_crypt_test.platform_crypt_support - -# create test cases for specific backends -sha512_crypt_os_crypt_test = _sha512_crypt_test.create_backend_case("os_crypt") -sha512_crypt_builtin_test = _sha512_crypt_test.create_backend_case("builtin") - -#============================================================================= -# sun md5 crypt -#============================================================================= -class sun_md5_crypt_test(HandlerCase): - handler = hash.sun_md5_crypt - - # TODO: this scheme needs some real test vectors, especially due to - # the "bare salt" issue which plagued the official parser. - known_correct_hashes = [ - # - # http://forums.halcyoninc.com/showthread.php?t=258 - # - ("Gpcs3_adm", "$md5$zrdhpMlZ$$wBvMOEqbSjU.hu5T2VEP01"), - - # - # http://www.c0t0d0s0.org/archives/4453-Less-known-Solaris-features-On-passwords-Part-2-Using-stronger-password-hashing.html - # - ("aa12345678", "$md5$vyy8.OVF$$FY4TWzuauRl4.VQNobqMY."), - - # - # http://www.cuddletech.com/blog/pivot/entry.php?id=778 - # - ("this", "$md5$3UqYqndY$$6P.aaWOoucxxq.l00SS9k0"), - - # - # http://compgroups.net/comp.unix.solaris/password-file-in-linux-and-solaris-8-9 - # - ("passwd", "$md5$RPgLF6IJ$WTvAlUJ7MqH5xak2FMEwS/"), - - # - # source: http://solaris-training.com/301_HTML/docs/deepdiv.pdf page 27 - # FIXME: password unknown - # "$md5,rounds=8000$kS9FT1JC$$mnUrRO618lLah5iazwJ9m1" - - # - # source: http://www.visualexams.com/310-303.htm - # XXX: this has 9 salt chars unlike all other hashes. is that valid? - # FIXME: password unknown - # "$md5,rounds=2006$2amXesSj5$$kCF48vfPsHDjlKNXeEw7V." - # - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, '$md5,rounds=5000$10VYDzAA$$1arAVtMA3trgE1qJ2V0Ez1'), - ] - - known_correct_configs = [ - # (config, secret, hash) - - #--------------------------- - # test salt string handling - # - # these tests attempt to verify that passlib is handling - # the "bare salt" issue (see sun md5 crypt docs) - # in a sane manner - #--------------------------- - - # config with "$" suffix, hash strings with "$$" suffix, - # should all be treated the same, with one "$" added to salt digest. - ("$md5$3UqYqndY$", - "this", "$md5$3UqYqndY$$6P.aaWOoucxxq.l00SS9k0"), - ("$md5$3UqYqndY$$.................DUMMY", - "this", "$md5$3UqYqndY$$6P.aaWOoucxxq.l00SS9k0"), - - # config with no suffix, hash strings with "$" suffix, - # should all be treated the same, and no suffix added to salt digest. - # NOTE: this is just a guess re: config w/ no suffix, - # but otherwise there's no sane way to encode bare_salt=False - # within config string. - ("$md5$3UqYqndY", - "this", "$md5$3UqYqndY$HIZVnfJNGCPbDZ9nIRSgP1"), - ("$md5$3UqYqndY$.................DUMMY", - "this", "$md5$3UqYqndY$HIZVnfJNGCPbDZ9nIRSgP1"), - ] - - known_malformed_hashes = [ - # unexpected end of hash - "$md5,rounds=5000", - - # bad rounds - "$md5,rounds=500A$xxxx", - "$md5,rounds=0500$xxxx", - "$md5,rounds=0$xxxx", - - # bad char in otherwise correct hash - "$md5$RPgL!6IJ$WTvAlUJ7MqH5xak2FMEwS/", - - # digest too short - "$md5$RPgLa6IJ$WTvAlUJ7MqH5xak2FMEwS", - - # digest too long - "$md5$RPgLa6IJ$WTvAlUJ7MqH5xak2FMEwS/.", - - # 2+ "$" at end of salt in config - # NOTE: not sure what correct behavior is, so forbidding format for now. - "$md5$3UqYqndY$$", - - # 3+ "$" at end of salt in hash - # NOTE: not sure what correct behavior is, so forbidding format for now. - "$md5$RPgLa6IJ$$$WTvAlUJ7MqH5xak2FMEwS/", - - ] - - platform_crypt_support = [ - ("solaris", True), - ("freebsd|openbsd|netbsd|linux|darwin", False), - ] - def do_verify(self, secret, hash): - # Override to fake error for "$..." hash string listed in known_correct_configs (above) - # These have to be hash strings, in order to test bare salt issue. - if isinstance(hash, str) and hash.endswith("$.................DUMMY"): - raise ValueError("pretending '$...' stub hash is config string") - return self.handler.verify(secret, hash) - -#============================================================================= -# unix disabled / fallback -#============================================================================= -class unix_disabled_test(HandlerCase): - handler = hash.unix_disabled -# accepts_all_hashes = True # TODO: turn this off. - - known_correct_hashes = [ - # everything should hash to "!" (or "*" on BSD), - # and nothing should verify against either string - ("password", "!"), - (UPASS_TABLE, "*"), - ] - - known_unidentified_hashes = [ - # should never identify anything crypt() could return... - "$1$xxx", - "abc", - "./az", - "{SHA}xxx", - ] - - def test_76_hash_border(self): - # so empty strings pass - self.accepts_all_hashes = True - super(unix_disabled_test, self).test_76_hash_border() - - def test_90_special(self): - """test marker option & special behavior""" - warnings.filterwarnings("ignore", "passing settings to .*.hash\(\) is deprecated") - handler = self.handler - - # preserve hash if provided - self.assertEqual(handler.genhash("stub", "!asd"), "!asd") - - # use marker if no hash - self.assertEqual(handler.genhash("stub", ""), handler.default_marker) - self.assertEqual(handler.hash("stub"), handler.default_marker) - self.assertEqual(handler.using().default_marker, handler.default_marker) - - # custom marker - self.assertEqual(handler.genhash("stub", "", marker="*xxx"), "*xxx") - self.assertEqual(handler.hash("stub", marker="*xxx"), "*xxx") - self.assertEqual(handler.using(marker="*xxx").hash("stub"), "*xxx") - - # reject invalid marker - self.assertRaises(ValueError, handler.genhash, 'stub', "", marker='abc') - self.assertRaises(ValueError, handler.hash, 'stub', marker='abc') - self.assertRaises(ValueError, handler.using, marker='abc') - -class unix_fallback_test(HandlerCase): - handler = hash.unix_fallback - accepts_all_hashes = True - - known_correct_hashes = [ - # *everything* should hash to "!", and nothing should verify - ("password", "!"), - (UPASS_TABLE, "!"), - ] - - # silence annoying deprecation warning - def setUp(self): - super(unix_fallback_test, self).setUp() - warnings.filterwarnings("ignore", "'unix_fallback' is deprecated") - - def test_90_wildcard(self): - """test enable_wildcard flag""" - h = self.handler - self.assertTrue(h.verify('password','', enable_wildcard=True)) - self.assertFalse(h.verify('password','')) - for c in "!*x": - self.assertFalse(h.verify('password',c, enable_wildcard=True)) - self.assertFalse(h.verify('password',c)) - - def test_91_preserves_existing(self): - """test preserves existing disabled hash""" - handler = self.handler - - # use marker if no hash - self.assertEqual(handler.genhash("stub", ""), "!") - self.assertEqual(handler.hash("stub"), "!") - - # use hash if provided and valid - self.assertEqual(handler.genhash("stub", "!asd"), "!asd") - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers_argon2.py b/src/passlib/tests/test_handlers_argon2.py deleted file mode 100644 index 5e9af435..00000000 --- a/src/passlib/tests/test_handlers_argon2.py +++ /dev/null @@ -1,366 +0,0 @@ -"""passlib.tests.test_handlers_argon2 - tests for passlib hash algorithms""" -#============================================================================= -# imports -#============================================================================= -# core -import logging -log = logging.getLogger(__name__) -import warnings -# site -# pkg -from passlib import hash -from passlib.tests.utils import HandlerCase, TEST_MODE -from passlib.tests.test_handlers import UPASS_TABLE, PASS_TABLE_UTF8 -# module - -#============================================================================= -# a bunch of tests lifted nearlky verbatim from official argon2 UTs... -# https://github.com/P-H-C/phc-winner-argon2/blob/master/src/test.c -#============================================================================= -def hashtest(version, t, logM, p, secret, salt, hex_digest, hash): - return dict(version=version, rounds=t, logM=logM, memory_cost=1< max uint32 - "$argon2i$v=19$m=65536,t=8589934592,p=4$c29tZXNhbHQAAAAAAAAAAA$QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY", - - # unexpected param - "$argon2i$v=19$m=65536,t=2,p=4,q=5$c29tZXNhbHQAAAAAAAAAAA$QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY", - - # wrong param order - "$argon2i$v=19$t=2,m=65536,p=4,q=5$c29tZXNhbHQAAAAAAAAAAA$QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY", - - # constraint violation: m < 8 * p - "$argon2i$v=19$m=127,t=2,p=16$c29tZXNhbHQ$IMit9qkFULCMA/ViizL57cnTLOa5DiVM9eMwpAvPwr4", - ] - - def setUpWarnings(self): - super(_base_argon2_test, self).setUpWarnings() - warnings.filterwarnings("ignore", ".*Using argon2pure backend.*") - - def do_stub_encrypt(self, handler=None, **settings): - if self.backend == "argon2_cffi": - # overriding default since no way to get stub config from argon2._calc_hash() - # (otherwise test_21b_max_rounds blocks trying to do max rounds) - handler = (handler or self.handler).using(**settings) - self = handler(use_defaults=True) - self.checksum = self._stub_checksum - assert self.checksum - return self.to_string() - else: - return super(_base_argon2_test, self).do_stub_encrypt(handler, **settings) - - def test_03_legacy_hash_workflow(self): - # override base method - raise self.skipTest("legacy 1.6 workflow not supported") - - def test_keyid_parameter(self): - # NOTE: keyid parameter currently not supported by official argon2 hash parser, - # even though it's mentioned in the format spec. - # we're trying to be consistent w/ this, so hashes w/ keyid should - # always through a NotImplementedError. - self.assertRaises(NotImplementedError, self.handler.verify, 'password', - "$argon2i$v=19$m=65536,t=2,p=4,keyid=ABCD$c29tZXNhbHQ$" - "IMit9qkFULCMA/ViizL57cnTLOa5DiVM9eMwpAvPwr4") - - def test_data_parameter(self): - # NOTE: argon2 c library doesn't support passing in a data parameter to argon2_hash(); - # but argon2_verify() appears to parse that info... but then discards it (!?). - # not sure what proper behavior is, filed issue -- https://github.com/P-H-C/phc-winner-argon2/issues/143 - # For now, replicating behavior we have for the two backends, to detect when things change. - handler = self.handler - - # ref hash of 'password' when 'data' is correctly passed into argon2() - sample1 = '$argon2i$v=19$m=512,t=2,p=2,data=c29tZWRhdGE$c29tZXNhbHQ$KgHyCesFyyjkVkihZ5VNFw' - - # ref hash of 'password' when 'data' is silently discarded (same digest as w/o data) - sample2 = '$argon2i$v=19$m=512,t=2,p=2,data=c29tZWRhdGE$c29tZXNhbHQ$uEeXt1dxN1iFKGhklseW4w' - - # hash of 'password' w/o the data field - sample3 = '$argon2i$v=19$m=512,t=2,p=2$c29tZXNhbHQ$uEeXt1dxN1iFKGhklseW4w' - - # - # test sample 1 - # - - if self.backend == "argon2_cffi": - # argon2_cffi v16.1 would incorrectly return False here. - # but v16.2 patches so it throws error on data parameter. - # our code should detect that, and adapt it into a NotImplementedError - self.assertRaises(NotImplementedError, handler.verify, "password", sample1) - - # incorrectly returns sample3, dropping data parameter - self.assertEqual(handler.genhash("password", sample1), sample3) - - else: - assert self.backend == "argon2pure" - # should parse and verify - self.assertTrue(handler.verify("password", sample1)) - - # should preserve sample1 - self.assertEqual(handler.genhash("password", sample1), sample1) - - # - # test sample 2 - # - - if self.backend == "argon2_cffi": - # argon2_cffi v16.1 would incorrectly return True here. - # but v16.2 patches so it throws error on data parameter. - # our code should detect that, and adapt it into a NotImplementedError - self.assertRaises(NotImplementedError, handler.verify,"password", sample2) - - # incorrectly returns sample3, dropping data parameter - self.assertEqual(handler.genhash("password", sample1), sample3) - - else: - assert self.backend == "argon2pure" - # should parse, but fail to verify - self.assertFalse(self.handler.verify("password", sample2)) - - # should return sample1 (corrected digest) - self.assertEqual(handler.genhash("password", sample2), sample1) - - def test_keyid_and_data_parameters(self): - # test combination of the two, just in case - self.assertRaises(NotImplementedError, self.handler.verify, 'stub', - "$argon2i$v=19$m=65536,t=2,p=4,keyid=ABCD,data=EFGH$c29tZXNhbHQ$" - "IMit9qkFULCMA/ViizL57cnTLOa5DiVM9eMwpAvPwr4") - - def test_needs_update_w_type(self): - handler = self.handler - - hash = handler.hash("stub") - self.assertFalse(handler.needs_update(hash)) - - hash2 = hash.replace("$argon2i$", "$argon2d$") - self.assertTrue(handler.needs_update(hash2)) - - def test_needs_update_w_version(self): - handler = self.handler.using(memory_cost=65536, time_cost=2, parallelism=4, - digest_size=32) - hash = ("$argon2i$m=65536,t=2,p=4$c29tZXNhbHQAAAAAAAAAAA$" - "QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY") - if handler.max_version == 0x10: - self.assertFalse(handler.needs_update(hash)) - else: - self.assertTrue(handler.needs_update(hash)) - - def test_argon_byte_encoding(self): - """verify we're using right base64 encoding for argon2""" - handler = self.handler - if handler.version != 0x13: - # TODO: make this fatal, and add refs for other version. - raise self.skipTest("handler uses wrong version for sample hashes") - - # 8 byte salt - salt = b'somesalt' - temp = handler.using(memory_cost=256, time_cost=2, parallelism=2, salt=salt, - checksum_size=32) - hash = temp.hash("password") - self.assertEqual(hash, "$argon2i$v=19$m=256,t=2,p=2" - "$c29tZXNhbHQ" - "$T/XOJ2mh1/TIpJHfCdQan76Q5esCFVoT5MAeIM1Oq2E") - - # 16 byte salt - salt = b'somesalt\x00\x00\x00\x00\x00\x00\x00\x00' - temp = handler.using(memory_cost=256, time_cost=2, parallelism=2, salt=salt, - checksum_size=32) - hash = temp.hash("password") - self.assertEqual(hash, "$argon2i$v=19$m=256,t=2,p=2" - "$c29tZXNhbHQAAAAAAAAAAA" - "$rqnbEp1/jFDUEKZZmw+z14amDsFqMDC53dIe57ZHD38") - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - settings_map = HandlerCase.FuzzHashGenerator.settings_map.copy() - settings_map.update(memory_cost="random_memory_cost") - - def random_memory_cost(self): - if self.test.backend == "argon2pure": - return self.randintgauss(128, 384, 256, 128) - else: - return self.randintgauss(128, 32767, 16384, 4096) - - # TODO: fuzz parallelism, digest_size - -#----------------------------------------- -# test suites for specific backends -#----------------------------------------- - -class argon2_argon2_cffi_test(_base_argon2_test.create_backend_case("argon2_cffi")): - - # add some more test vectors that take too long under argon2pure - known_correct_hashes = _base_argon2_test.known_correct_hashes + [ - # - # sample hashes from argon2 cffi package's unittests, - # which in turn were generated by official argon2 cmdline tool. - # - - # v1.2, type I, w/o a version tag - ('password', "$argon2i$m=65536,t=2,p=4$c29tZXNhbHQAAAAAAAAAAA$" - "QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY"), - - # v1.3, type I - ('password', "$argon2i$v=19$m=65536,t=2,p=4$c29tZXNhbHQ$" - "IMit9qkFULCMA/ViizL57cnTLOa5DiVM9eMwpAvPwr4"), - - # v1.3, type D - ('password', "$argon2d$v=19$m=65536,t=2,p=4$c29tZXNhbHQ$" - "cZn5d+rFh+ZfuRhm2iGUGgcrW5YLeM6q7L3vBsdmFA0"), - - # - # custom - # - - # ensure trailing null bytes handled correctly - ('password\x00', "$argon2i$v=19$m=65536,t=2,p=4$c29tZXNhbHQ$" - "Vpzuc0v0SrP88LcVvmg+z5RoOYpMDKH/lt6O+CZabIQ"), - - ] - - # add reference hashes from argon2 clib tests - known_correct_hashes.extend( - (info['secret'], info['hash']) for info in reference_data - if info['logM'] <= (18 if TEST_MODE("full") else 16) - ) - -class argon2_argon2pure_test(_base_argon2_test.create_backend_case("argon2pure")): - - # XXX: setting max_threads at 1 to prevent argon2pure from using multiprocessing, - # which causes big problems when testing under pypy. - # would like a "pure_use_threads" option instead, to make it use multiprocessing.dummy instead. - handler = hash.argon2.using(memory_cost=32, parallelism=2) - - # don't use multiprocessing for unittests, makes it a lot harder to ctrl-c - # XXX: make this controlled by env var? - handler.pure_use_threads = True - - # add reference hashes from argon2 clib tests - known_correct_hashes = _base_argon2_test.known_correct_hashes[:] - known_correct_hashes.extend( - (info['secret'], info['hash']) for info in reference_data - if info['logM'] < 16 - ) - - class FuzzHashGenerator(_base_argon2_test.FuzzHashGenerator): - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(1, 3, 2, 1) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers_bcrypt.py b/src/passlib/tests/test_handlers_bcrypt.py deleted file mode 100644 index 978b68ba..00000000 --- a/src/passlib/tests/test_handlers_bcrypt.py +++ /dev/null @@ -1,544 +0,0 @@ -"""passlib.tests.test_handlers - tests for passlib hash algorithms""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -import os -import warnings -# site -# pkg -from passlib import hash -from passlib.handlers.bcrypt import IDENT_2, IDENT_2X -from passlib.utils import repeat_string, to_bytes -from passlib.utils.compat import irange -from passlib.tests.utils import HandlerCase, TEST_MODE -from passlib.tests.test_handlers import UPASS_TABLE -# module - -#============================================================================= -# bcrypt -#============================================================================= -class _bcrypt_test(HandlerCase): - """base for BCrypt test cases""" - handler = hash.bcrypt - reduce_default_rounds = True - fuzz_salts_need_bcrypt_repair = True - has_os_crypt_fallback = False - - known_correct_hashes = [ - # - # from JTR 1.7.9 - # - ('U*U*U*U*', '$2a$05$c92SVSfjeiCD6F2nAD6y0uBpJDjdRkt0EgeC4/31Rf2LUZbDRDE.O'), - ('U*U***U', '$2a$05$WY62Xk2TXZ7EvVDQ5fmjNu7b0GEzSzUXUh2cllxJwhtOeMtWV3Ujq'), - ('U*U***U*', '$2a$05$Fa0iKV3E2SYVUlMknirWU.CFYGvJ67UwVKI1E2FP6XeLiZGcH3MJi'), - ('*U*U*U*U', '$2a$05$.WRrXibc1zPgIdRXYfv.4uu6TD1KWf0VnHzq/0imhUhuxSxCyeBs2'), - ('', '$2a$05$Otz9agnajgrAe0.kFVF9V.tzaStZ2s1s4ZWi/LY4sw2k/MTVFj/IO'), - - # - # test vectors from http://www.openwall.com/crypt v1.2 - # note that this omits any hashes that depend on crypt_blowfish's - # various CVE-2011-2483 workarounds (hash 2a and \xff\xff in password, - # and any 2x hashes); and only contain hashes which are correct - # under both crypt_blowfish 1.2 AND OpenBSD. - # - ('U*U', '$2a$05$CCCCCCCCCCCCCCCCCCCCC.E5YPO9kmyuRGyh0XouQYb4YMJKvyOeW'), - ('U*U*', '$2a$05$CCCCCCCCCCCCCCCCCCCCC.VGOzA784oUp/Z0DY336zx7pLYAy0lwK'), - ('U*U*U', '$2a$05$XXXXXXXXXXXXXXXXXXXXXOAcXxm9kjPGEMsLznoKqmqw7tc8WCx4a'), - ('', '$2a$05$CCCCCCCCCCCCCCCCCCCCC.7uG0VCzI2bS7j6ymqJi9CdcdxiRTWNy'), - ('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - '0123456789chars after 72 are ignored', - '$2a$05$abcdefghijklmnopqrstuu5s2v8.iXieOjg/.AySBTTZIIVFJeBui'), - (b'\xa3', - '$2a$05$/OK.fbVrR/bpIqNJ5ianF.Sa7shbm4.OzKpvFnX1pQLmQW96oUlCq'), - (b'\xff\xa3345', - '$2a$05$/OK.fbVrR/bpIqNJ5ianF.nRht2l/HRhr6zmCp9vYUvvsqynflf9e'), - (b'\xa3ab', - '$2a$05$/OK.fbVrR/bpIqNJ5ianF.6IflQkJytoRVc1yuaNtHfiuq.FRlSIS'), - (b'\xaa'*72 + b'chars after 72 are ignored as usual', - '$2a$05$/OK.fbVrR/bpIqNJ5ianF.swQOIzjOiJ9GHEPuhEkvqrUyvWhEMx6'), - (b'\xaa\x55'*36, - '$2a$05$/OK.fbVrR/bpIqNJ5ianF.R9xrDjiycxMbQE2bp.vgqlYpW5wx2yy'), - (b'\x55\xaa\xff'*24, - '$2a$05$/OK.fbVrR/bpIqNJ5ianF.9tQZzcJfm3uj2NvJ/n5xkhpqLrMpWCe'), - - # keeping one of their 2y tests, because we are supporting that. - (b'\xa3', - '$2y$05$/OK.fbVrR/bpIqNJ5ianF.Sa7shbm4.OzKpvFnX1pQLmQW96oUlCq'), - - # - # bsd wraparound bug (fixed in 2b) - # - - # NOTE: if backend is vulnerable, password will hash the same as '0'*72 - # ("$2a$04$R1lJ2gkNaoPGdafE.H.16.nVyh2niHsGJhayOHLMiXlI45o8/DU.6"), - # rather than same as ("0123456789"*8)[:72] - # 255 should be sufficient, but checking - (("0123456789"*26)[:254], '$2a$04$R1lJ2gkNaoPGdafE.H.16.1MKHPvmKwryeulRe225LKProWYwt9Oi'), - (("0123456789"*26)[:255], '$2a$04$R1lJ2gkNaoPGdafE.H.16.1MKHPvmKwryeulRe225LKProWYwt9Oi'), - (("0123456789"*26)[:256], '$2a$04$R1lJ2gkNaoPGdafE.H.16.1MKHPvmKwryeulRe225LKProWYwt9Oi'), - (("0123456789"*26)[:257], '$2a$04$R1lJ2gkNaoPGdafE.H.16.1MKHPvmKwryeulRe225LKProWYwt9Oi'), - - - # - # from py-bcrypt tests - # - ('', '$2a$06$DCq7YPn5Rq63x1Lad4cll.TV4S6ytwfsfvkgY8jIucDrjc8deX1s.'), - ('a', '$2a$10$k87L/MF28Q673VKh8/cPi.SUl7MU/rWuSiIDDFayrKk/1tBsSQu4u'), - ('abc', '$2a$10$WvvTPHKwdBJ3uk0Z37EMR.hLA2W6N9AEBhEgrAOljy2Ae5MtaSIUi'), - ('abcdefghijklmnopqrstuvwxyz', - '$2a$10$fVH8e28OQRj9tqiDXs1e1uxpsjN0c7II7YPKXua2NAKYvM6iQk7dq'), - ('~!@#$%^&*() ~!@#$%^&*()PNBFRD', - '$2a$10$LgfYWkbzEvQ4JakH7rOvHe0y8pHKF9OaFgwUZ2q7W2FFZmZzJYlfS'), - - # - # custom test vectors - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, - '$2a$05$Z17AXnnlpzddNUvnC6cZNOSwMA/8oNiKnHTHTwLlBijfucQQlHjaG'), - - # ensure 2b support - (UPASS_TABLE, - '$2b$05$Z17AXnnlpzddNUvnC6cZNOSwMA/8oNiKnHTHTwLlBijfucQQlHjaG'), - - ] - - if TEST_MODE("full"): - # - # add some extra tests related to 2/2a - # - CONFIG_2 = '$2$05$' + '.'*22 - CONFIG_A = '$2a$05$' + '.'*22 - known_correct_hashes.extend([ - ("", CONFIG_2 + 'J2ihDv8vVf7QZ9BsaRrKyqs2tkn55Yq'), - ("", CONFIG_A + 'J2ihDv8vVf7QZ9BsaRrKyqs2tkn55Yq'), - ("abc", CONFIG_2 + 'XuQjdH.wPVNUZ/bOfstdW/FqB8QSjte'), - ("abc", CONFIG_A + 'ev6gDwpVye3oMCUpLY85aTpfBNHD0Ga'), - ("abc"*23, CONFIG_2 + 'XuQjdH.wPVNUZ/bOfstdW/FqB8QSjte'), - ("abc"*23, CONFIG_A + '2kIdfSj/4/R/Q6n847VTvc68BXiRYZC'), - ("abc"*24, CONFIG_2 + 'XuQjdH.wPVNUZ/bOfstdW/FqB8QSjte'), - ("abc"*24, CONFIG_A + 'XuQjdH.wPVNUZ/bOfstdW/FqB8QSjte'), - ("abc"*24+'x', CONFIG_2 + 'XuQjdH.wPVNUZ/bOfstdW/FqB8QSjte'), - ("abc"*24+'x', CONFIG_A + 'XuQjdH.wPVNUZ/bOfstdW/FqB8QSjte'), - ]) - - known_correct_configs = [ - ('$2a$04$uM6csdM8R9SXTex/gbTaye', UPASS_TABLE, - '$2a$04$uM6csdM8R9SXTex/gbTayezuvzFEufYGd2uB6of7qScLjQ4GwcD4G'), - ] - - known_unidentified_hashes = [ - # invalid minor version - "$2f$12$EXRkfkdmXnagzds2SSitu.MW9.gAVqa9eLS1//RYtYCmB1eLHg.9q", - "$2`$12$EXRkfkdmXnagzds2SSitu.MW9.gAVqa9eLS1//RYtYCmB1eLHg.9q", - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash - # \/ - "$2a$12$EXRkfkdmXn!gzds2SSitu.MW9.gAVqa9eLS1//RYtYCmB1eLHg.9q", - - # unsupported (but recognized) minor version - "$2x$12$EXRkfkdmXnagzds2SSitu.MW9.gAVqa9eLS1//RYtYCmB1eLHg.9q", - - # rounds not zero-padded (py-bcrypt rejects this, therefore so do we) - '$2a$6$DCq7YPn5Rq63x1Lad4cll.TV4S6ytwfsfvkgY8jIucDrjc8deX1s.' - - # NOTE: salts with padding bits set are technically malformed, - # but we can reliably correct & issue a warning for that. - ] - - platform_crypt_support = [ - ("freedbsd|openbsd|netbsd", True), - ("darwin", False), - # linux - may be present via addon, e.g. debian's libpam-unix2 - # solaris - depends on policy - ] - - #=================================================================== - # override some methods - #=================================================================== - def setUp(self): - # ensure builtin is enabled for duration of test. - if TEST_MODE("full") and self.backend == "builtin": - key = "PASSLIB_BUILTIN_BCRYPT" - orig = os.environ.get(key) - if orig: - self.addCleanup(os.environ.__setitem__, key, orig) - else: - self.addCleanup(os.environ.__delitem__, key) - os.environ[key] = "true" - super(_bcrypt_test, self).setUp() - warnings.filterwarnings("ignore", ".*backend is vulnerable to the bsd wraparound bug.*") - - def populate_settings(self, kwds): - # builtin is still just way too slow. - if self.backend == "builtin": - kwds.setdefault("rounds", 4) - super(_bcrypt_test, self).populate_settings(kwds) - - #=================================================================== - # fuzz testing - #=================================================================== - def crypt_supports_variant(self, hash): - """check if OS crypt is expected to support given ident""" - from passlib.handlers.bcrypt import bcrypt, IDENT_2X, IDENT_2Y - from passlib.utils import safe_crypt - ident = bcrypt.from_string(hash) - return (safe_crypt("test", ident + "04$5BJqKfqMQvV7nS.yUguNcu") or "").startswith(ident) - - fuzz_verifiers = HandlerCase.fuzz_verifiers + ( - "fuzz_verifier_bcrypt", - "fuzz_verifier_pybcrypt", - "fuzz_verifier_bcryptor", - ) - - def fuzz_verifier_bcrypt(self): - # test against bcrypt, if available - from passlib.handlers.bcrypt import IDENT_2, IDENT_2A, IDENT_2B, IDENT_2X, IDENT_2Y, _detect_pybcrypt - from passlib.utils import to_native_str, to_bytes - try: - import bcrypt - except ImportError: - return - if _detect_pybcrypt(): - return - def check_bcrypt(secret, hash): - """bcrypt""" - secret = to_bytes(secret, self.FuzzHashGenerator.password_encoding) - if hash.startswith(IDENT_2B): - # bcrypt <1.1 lacks 2B support - hash = IDENT_2A + hash[4:] - elif hash.startswith(IDENT_2): - # bcrypt doesn't support $2$ hashes; but we can fake it - # using the $2a$ algorithm, by repeating the password until - # it's 72 chars in length. - hash = IDENT_2A + hash[3:] - if secret: - secret = repeat_string(secret, 72) - elif hash.startswith(IDENT_2Y) and bcrypt.__version__ == "3.0.0": - hash = IDENT_2B + hash[4:] - hash = to_bytes(hash) - try: - return bcrypt.hashpw(secret, hash) == hash - except ValueError: - raise ValueError("bcrypt rejected hash: %r (secret=%r)" % (hash, secret)) - return check_bcrypt - - def fuzz_verifier_pybcrypt(self): - # test against py-bcrypt, if available - from passlib.handlers.bcrypt import ( - IDENT_2, IDENT_2A, IDENT_2B, IDENT_2X, IDENT_2Y, - _PyBcryptBackend, - ) - from passlib.utils import to_native_str - - loaded = _PyBcryptBackend._load_backend_mixin("pybcrypt", False) - if not loaded: - return - - from passlib.handlers.bcrypt import _pybcrypt as bcrypt_mod - - lock = _PyBcryptBackend._calc_lock # reuse threadlock workaround for pybcrypt 0.2 - - def check_pybcrypt(secret, hash): - """pybcrypt""" - secret = to_native_str(secret, self.FuzzHashGenerator.password_encoding) - if len(secret) > 200: # vulnerable to wraparound bug - secret = secret[:200] - if hash.startswith((IDENT_2B, IDENT_2Y)): - hash = IDENT_2A + hash[4:] - try: - if lock: - with lock: - return bcrypt_mod.hashpw(secret, hash) == hash - else: - return bcrypt_mod.hashpw(secret, hash) == hash - except ValueError: - raise ValueError("py-bcrypt rejected hash: %r" % (hash,)) - return check_pybcrypt - - def fuzz_verifier_bcryptor(self): - # test against bcryptor if available - from passlib.handlers.bcrypt import IDENT_2, IDENT_2A, IDENT_2Y, IDENT_2B - from passlib.utils import to_native_str - try: - from bcryptor.engine import Engine - except ImportError: - return - def check_bcryptor(secret, hash): - """bcryptor""" - secret = to_native_str(secret, self.FuzzHashGenerator.password_encoding) - if hash.startswith((IDENT_2B, IDENT_2Y)): - hash = IDENT_2A + hash[4:] - elif hash.startswith(IDENT_2): - # bcryptor doesn't support $2$ hashes; but we can fake it - # using the $2a$ algorithm, by repeating the password until - # it's 72 chars in length. - hash = IDENT_2A + hash[3:] - if secret: - secret = repeat_string(secret, 72) - return Engine(False).hash_key(secret, hash) == hash - return check_bcryptor - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def generate(self): - opts = super(_bcrypt_test.FuzzHashGenerator, self).generate() - - secret = opts['secret'] - other = opts['other'] - settings = opts['settings'] - ident = settings.get('ident') - - if ident == IDENT_2X: - # 2x is just recognized, not supported. don't test with it. - del settings['ident'] - - elif ident == IDENT_2 and other and repeat_string(to_bytes(other), len(to_bytes(secret))) == to_bytes(secret): - # avoid false failure due to flaw in 0-revision bcrypt: - # repeated strings like 'abc' and 'abcabc' hash identically. - opts['secret'], opts['other'] = self.random_password_pair() - - return opts - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(5, 8, 6, 1) - - #=================================================================== - # custom tests - #=================================================================== - known_incorrect_padding = [ - # password, bad hash, good hash - - # 2 bits of salt padding set -# ("loppux", # \/ -# "$2a$12$oaQbBqq8JnSM1NHRPQGXORm4GCUMqp7meTnkft4zgSnrbhoKdDV0C", -# "$2a$12$oaQbBqq8JnSM1NHRPQGXOOm4GCUMqp7meTnkft4zgSnrbhoKdDV0C"), - ("test", # \/ - '$2a$04$oaQbBqq8JnSM1NHRPQGXORY4Vw3bdHKLIXTecPDRAcJ98cz1ilveO', - '$2a$04$oaQbBqq8JnSM1NHRPQGXOOY4Vw3bdHKLIXTecPDRAcJ98cz1ilveO'), - - # all 4 bits of salt padding set -# ("Passlib11", # \/ -# "$2a$12$M8mKpW9a2vZ7PYhq/8eJVcUtKxpo6j0zAezu0G/HAMYgMkhPu4fLK", -# "$2a$12$M8mKpW9a2vZ7PYhq/8eJVOUtKxpo6j0zAezu0G/HAMYgMkhPu4fLK"), - ("test", # \/ - "$2a$04$yjDgE74RJkeqC0/1NheSScrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS", - "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS"), - - # bad checksum padding - ("test", # \/ - "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIV", - "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS"), - ] - - def test_90_bcrypt_padding(self): - """test passlib correctly handles bcrypt padding bits""" - self.require_TEST_MODE("full") - # - # prevents reccurrence of issue 25 (https://code.google.com/p/passlib/issues/detail?id=25) - # were some unused bits were incorrectly set in bcrypt salt strings. - # (fixed since 1.5.3) - # - bcrypt = self.handler - corr_desc = ".*incorrectly set padding bits" - - # - # test hash() / genconfig() don't generate invalid salts anymore - # - def check_padding(hash): - assert hash.startswith(("$2a$", "$2b$")) and len(hash) >= 28, \ - "unexpectedly malformed hash: %r" % (hash,) - self.assertTrue(hash[28] in '.Oeu', - "unused bits incorrectly set in hash: %r" % (hash,)) - for i in irange(6): - check_padding(bcrypt.genconfig()) - for i in irange(3): - check_padding(bcrypt.using(rounds=bcrypt.min_rounds).hash("bob")) - - # - # test genconfig() corrects invalid salts & issues warning. - # - with self.assertWarningList(["salt too large", corr_desc]): - hash = bcrypt.genconfig(salt="."*21 + "A.", rounds=5, relaxed=True) - self.assertEqual(hash, "$2b$05$" + "." * (22 + 31)) - - # - # test public methods against good & bad hashes - # - samples = self.known_incorrect_padding - for pwd, bad, good in samples: - - # make sure genhash() corrects bad configs, leaves good unchanged - with self.assertWarningList([corr_desc]): - self.assertEqual(bcrypt.genhash(pwd, bad), good) - with self.assertWarningList([]): - self.assertEqual(bcrypt.genhash(pwd, good), good) - - # make sure verify() works correctly with good & bad hashes - with self.assertWarningList([corr_desc]): - self.assertTrue(bcrypt.verify(pwd, bad)) - with self.assertWarningList([]): - self.assertTrue(bcrypt.verify(pwd, good)) - - # make sure normhash() corrects bad hashes, leaves good unchanged - with self.assertWarningList([corr_desc]): - self.assertEqual(bcrypt.normhash(bad), good) - with self.assertWarningList([]): - self.assertEqual(bcrypt.normhash(good), good) - - # make sure normhash() leaves non-bcrypt hashes alone - self.assertEqual(bcrypt.normhash("$md5$abc"), "$md5$abc") - - def test_needs_update_w_padding(self): - """needs_update corrects bcrypt padding""" - # NOTE: see padding test above for details about issue this detects - bcrypt = self.handler.using(rounds=4) - - # PASS1 = "test" - BAD1 = "$2a$04$yjDgE74RJkeqC0/1NheSScrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS" - GOOD1 = "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS" - - self.assertTrue(bcrypt.needs_update(BAD1)) - self.assertFalse(bcrypt.needs_update(GOOD1)) - - #=================================================================== - # eoc - #=================================================================== - -# create test cases for specific backends -bcrypt_bcrypt_test = _bcrypt_test.create_backend_case("bcrypt") -bcrypt_pybcrypt_test = _bcrypt_test.create_backend_case("pybcrypt") -bcrypt_bcryptor_test = _bcrypt_test.create_backend_case("bcryptor") -bcrypt_os_crypt_test = _bcrypt_test.create_backend_case("os_crypt") -bcrypt_builtin_test = _bcrypt_test.create_backend_case("builtin") - -#============================================================================= -# bcrypt -#============================================================================= -class _bcrypt_sha256_test(HandlerCase): - "base for BCrypt-SHA256 test cases" - handler = hash.bcrypt_sha256 - reduce_default_rounds = True - forbidden_characters = None - fuzz_salts_need_bcrypt_repair = True - alt_safe_crypt_handler = hash.bcrypt - has_os_crypt_fallback = True - - known_correct_hashes = [ - # - # custom test vectors - # - - # empty - ("", - '$bcrypt-sha256$2a,5$E/e/2AOhqM5W/KJTFQzLce$F6dYSxOdAEoJZO2eoHUZWZljW/e0TXO'), - - # ascii - ("password", - '$bcrypt-sha256$2a,5$5Hg1DKFqPE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu'), - - # unicode / utf8 - (UPASS_TABLE, - '$bcrypt-sha256$2a,5$.US1fQ4TQS.ZTz/uJ5Kyn.$QNdPDOTKKT5/sovNz1iWg26quOU4Pje'), - (UPASS_TABLE.encode("utf-8"), - '$bcrypt-sha256$2a,5$.US1fQ4TQS.ZTz/uJ5Kyn.$QNdPDOTKKT5/sovNz1iWg26quOU4Pje'), - - # ensure 2b support - ("password", - '$bcrypt-sha256$2b,5$5Hg1DKFqPE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu'), - (UPASS_TABLE, - '$bcrypt-sha256$2b,5$.US1fQ4TQS.ZTz/uJ5Kyn.$QNdPDOTKKT5/sovNz1iWg26quOU4Pje'), - - # test >72 chars is hashed correctly -- under bcrypt these hash the same. - # NOTE: test_60_truncate_size() handles this already, this is just for overkill :) - (repeat_string("abc123",72), - '$bcrypt-sha256$2b,5$X1g1nh3g0v4h6970O68cxe$r/hyEtqJ0teqPEmfTLoZ83ciAI1Q74.'), - (repeat_string("abc123",72)+"qwr", - '$bcrypt-sha256$2b,5$X1g1nh3g0v4h6970O68cxe$021KLEif6epjot5yoxk0m8I0929ohEa'), - (repeat_string("abc123",72)+"xyz", - '$bcrypt-sha256$2b,5$X1g1nh3g0v4h6970O68cxe$7.1kgpHduMGEjvM3fX6e/QCvfn6OKja'), - ] - - known_correct_configs =[ - ('$bcrypt-sha256$2a,5$5Hg1DKFqPE8C2aflZ5vVoe', - "password", '$bcrypt-sha256$2a,5$5Hg1DKFqPE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu'), - ] - - known_malformed_hashes = [ - # bad char in otherwise correct hash - # \/ - '$bcrypt-sha256$2a,5$5Hg1DKF!PE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu', - - # unrecognized bcrypt variant - '$bcrypt-sha256$2c,5$5Hg1DKFqPE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu', - - # unsupported bcrypt variant - '$bcrypt-sha256$2x,5$5Hg1DKFqPE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu', - - # rounds zero-padded - '$bcrypt-sha256$2a,05$5Hg1DKFqPE8C2aflZ5vVoe$12BjNE0p7axMg55.Y/mHsYiVuFBDQyu', - - # config string w/ $ added - '$bcrypt-sha256$2a,5$5Hg1DKFqPE8C2aflZ5vVoe$', - ] - - #=================================================================== - # override some methods -- cloned from bcrypt - #=================================================================== - def setUp(self): - # ensure builtin is enabled for duration of test. - if TEST_MODE("full") and self.backend == "builtin": - key = "PASSLIB_BUILTIN_BCRYPT" - orig = os.environ.get(key) - if orig: - self.addCleanup(os.environ.__setitem__, key, orig) - else: - self.addCleanup(os.environ.__delitem__, key) - os.environ[key] = "enabled" - super(_bcrypt_sha256_test, self).setUp() - warnings.filterwarnings("ignore", ".*backend is vulnerable to the bsd wraparound bug.*") - - def populate_settings(self, kwds): - # builtin is still just way too slow. - if self.backend == "builtin": - kwds.setdefault("rounds", 4) - super(_bcrypt_sha256_test, self).populate_settings(kwds) - - #=================================================================== - # override ident tests for now - #=================================================================== - def test_30_HasManyIdents(self): - raise self.skipTest("multiple idents not supported") - - def test_30_HasOneIdent(self): - # forbidding ident keyword, we only support "2a" for now - handler = self.handler - handler(use_defaults=True) - self.assertRaises(ValueError, handler, ident="$2y$", use_defaults=True) - - #=================================================================== - # fuzz testing -- cloned from bcrypt - #=================================================================== - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(5, 8, 6, 1) - -# create test cases for specific backends -bcrypt_sha256_bcrypt_test = _bcrypt_sha256_test.create_backend_case("bcrypt") -bcrypt_sha256_pybcrypt_test = _bcrypt_sha256_test.create_backend_case("pybcrypt") -bcrypt_sha256_bcryptor_test = _bcrypt_sha256_test.create_backend_case("bcryptor") -bcrypt_sha256_os_crypt_test = _bcrypt_sha256_test.create_backend_case("os_crypt") -bcrypt_sha256_builtin_test = _bcrypt_sha256_test.create_backend_case("builtin") - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers_cisco.py b/src/passlib/tests/test_handlers_cisco.py deleted file mode 100644 index ea6594bf..00000000 --- a/src/passlib/tests/test_handlers_cisco.py +++ /dev/null @@ -1,457 +0,0 @@ -""" -passlib.tests.test_handlers_cisco - tests for Cisco-specific algorithms -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import absolute_import, division, print_function -# core -import logging -log = logging.getLogger(__name__) -# site -# pkg -from passlib import hash, exc -from passlib.utils.compat import u -from .utils import UserHandlerMixin, HandlerCase, repeat_string -from .test_handlers import UPASS_TABLE -# module -__all__ = [ - "cisco_pix_test", - "cisco_asa_test", - "cisco_type7_test", -] -#============================================================================= -# shared code for cisco PIX & ASA -#============================================================================= - -class _PixAsaSharedTest(UserHandlerMixin, HandlerCase): - """ - class w/ shared info for PIX & ASA tests. - """ - __unittest_skip = True # for TestCase - requires_user = False # for UserHandlerMixin - - #: shared list of hashes which should be identical under pix & asa7 - #: (i.e. combined secret + user < 17 bytes) - pix_asa_shared_hashes = [ - # - # http://www.perlmonks.org/index.pl?node_id=797623 - # - (("cisco", ""), "2KFQnbNIdI.2KYOU"), # confirmed ASA 9.6 - - # - # http://www.hsc.fr/ressources/breves/pix_crack.html.en - # - (("hsc", ""), "YtT8/k6Np8F1yz2c"), # confirmed ASA 9.6 - - # - # www.freerainbowtables.com/phpBB3/viewtopic.php?f=2&t=1441 - # - (("", ""), "8Ry2YjIyt7RRXU24"), # confirmed ASA 9.6 - (("cisco", "john"), "hN7LzeyYjw12FSIU"), - (("cisco", "jack"), "7DrfeZ7cyOj/PslD"), - - # - # http://comments.gmane.org/gmane.comp.security.openwall.john.user/2529 - # - (("ripper", "alex"), "h3mJrcH0901pqX/m"), - (("cisco", "cisco"), "3USUcOPFUiMCO4Jk"), - (("cisco", "cisco1"), "3USUcOPFUiMCO4Jk"), - (("CscFw-ITC!", "admcom"), "lZt7HSIXw3.QP7.R"), - ("cangetin", "TynyB./ftknE77QP"), - (("cangetin", "rramsey"), "jgBZqYtsWfGcUKDi"), - - # - # http://openwall.info/wiki/john/sample-hashes - # - (("phonehome", "rharris"), "zyIIMSYjiPm0L7a6"), - - # - # http://www.openwall.com/lists/john-users/2010/08/08/3 - # - (("cangetin", ""), "TynyB./ftknE77QP"), - (("cangetin", "rramsey"), "jgBZqYtsWfGcUKDi"), - - # - # from JTR 1.7.9 - # - ("test1", "TRPEas6f/aa6JSPL"), - ("test2", "OMT6mXmAvGyzrCtp"), - ("test3", "gTC7RIy1XJzagmLm"), - ("test4", "oWC1WRwqlBlbpf/O"), - ("password", "NuLKvvWGg.x9HEKO"), - ("0123456789abcdef", ".7nfVBEIEu4KbF/1"), - - # - # http://www.cisco.com/en/US/docs/security/pix/pix50/configuration/guide/commands.html#wp5472 - # - (("1234567890123456", ""), "feCkwUGktTCAgIbD"), # canonical source - (("watag00s1am", ""), "jMorNbK0514fadBh"), # canonical source - - # - # custom - # - (("cisco1", "cisco1"), "jmINXNH6p1BxUppp"), - - # ensures utf-8 used for unicode - (UPASS_TABLE, 'CaiIvkLMu2TOHXGT'), - - # - # passlib reference vectors - # - # Some of these have been confirmed on various ASA firewalls, - # and the exact version is noted next to each hash. - # Would like to verify these under more PIX & ASA versions. - # - # Those without a note are generally an extrapolation, - # to ensure the code stays consistent, but for various reasons, - # hasn't been verified. - # - # * One such case is usernames w/ 1 & 2 digits -- - # ASA (9.6 at least) requires 3+ digits in username. - # - # The following hashes (below 13 chars) should be identical for PIX/ASA. - # Ones which differ are listed separately in the known_correct_hashes - # list for the two test classes. - # - - # 4 char password - (('1234', ''), 'RLPMUQ26KL4blgFN'), # confirmed ASA 9.6 - - # 8 char password - (('01234567', ''), '0T52THgnYdV1tlOF'), # confirmed ASA 9.6 - (('01234567', '3'), '.z0dT9Alkdc7EIGS'), - (('01234567', '36'), 'CC3Lam53t/mHhoE7'), - (('01234567', '365'), '8xPrWpNnBdD2DzdZ'), # confirmed ASA 9.6 - (('01234567', '3333'), '.z0dT9Alkdc7EIGS'), # confirmed ASA 9.6 - (('01234567', '3636'), 'CC3Lam53t/mHhoE7'), # confirmed ASA 9.6 - (('01234567', '3653'), '8xPrWpNnBdD2DzdZ'), # confirmed ASA 9.6 - (('01234567', 'adm'), 'dfWs2qiao6KD/P2L'), # confirmed ASA 9.6 - (('01234567', 'adma'), 'dfWs2qiao6KD/P2L'), # confirmed ASA 9.6 - (('01234567', 'admad'), 'dfWs2qiao6KD/P2L'), # confirmed ASA 9.6 - (('01234567', 'user'), 'PNZ4ycbbZ0jp1.j1'), # confirmed ASA 9.6 - (('01234567', 'user1234'), 'PNZ4ycbbZ0jp1.j1'), # confirmed ASA 9.6 - - # 12 char password - (('0123456789ab', ''), 'S31BxZOGlAigndcJ'), # confirmed ASA 9.6 - (('0123456789ab', '36'), 'wFqSX91X5.YaRKsi'), - (('0123456789ab', '365'), 'qjgo3kNgTVxExbno'), # confirmed ASA 9.6 - (('0123456789ab', '3333'), 'mcXPL/vIZcIxLUQs'), # confirmed ASA 9.6 - (('0123456789ab', '3636'), 'wFqSX91X5.YaRKsi'), # confirmed ASA 9.6 - (('0123456789ab', '3653'), 'qjgo3kNgTVxExbno'), # confirmed ASA 9.6 - (('0123456789ab', 'user'), 'f.T4BKdzdNkjxQl7'), # confirmed ASA 9.6 - (('0123456789ab', 'user1234'), 'f.T4BKdzdNkjxQl7'), # confirmed ASA 9.6 - - # NOTE: remaining reference vectors for 13+ char passwords - # are split up between cisco_pix & cisco_asa tests. - - # unicode passwords - # ASA supposedly uses utf-8 encoding, but entering non-ascii - # chars is error-prone, and while UTF-8 appears to be intended, - # observed behaviors include: - # * ssh cli stripping non-ascii chars entirely - # * ASDM web iface double-encoding utf-8 strings - ((u("t\xe1ble").encode("utf-8"), 'user'), 'Og8fB4NyF0m5Ed9c'), - ((u("t\xe1ble").encode("utf-8").decode("latin-1").encode("utf-8"), - 'user'), 'cMvFC2XVBmK/68yB'), # confirmed ASA 9.6 when typed into ASDM - ] - - def test_calc_digest_spoiler(self): - """ - _calc_checksum() -- spoil oversize passwords during verify - - for details, see 'spoil_digest' flag instead that function. - this helps cisco_pix/cisco_asa implement their policy of - ``.truncate_verify_reject=True``. - """ - def calc(secret, for_hash=False): - return self.handler(use_defaults=for_hash)._calc_checksum(secret) - - # short (non-truncated) password - short_secret = repeat_string("1234", self.handler.truncate_size) - short_hash = calc(short_secret) - - # longer password should have totally different hash, - # to prevent verify from matching (i.e. "spoiled"). - long_secret = short_secret + "X" - long_hash = calc(long_secret) - self.assertNotEqual(long_hash, short_hash) - - # spoiled hash should depend on whole secret, - # so that output isn't predictable - alt_long_secret = short_secret + "Y" - alt_long_hash = calc(alt_long_secret) - self.assertNotEqual(alt_long_hash, short_hash) - self.assertNotEqual(alt_long_hash, long_hash) - - # for hash(), should throw error if password too large - calc(short_secret, for_hash=True) - self.assertRaises(exc.PasswordSizeError, calc, long_secret, for_hash=True) - self.assertRaises(exc.PasswordSizeError, calc, alt_long_secret, for_hash=True) - -#============================================================================= -# cisco pix -#============================================================================= -class cisco_pix_test(_PixAsaSharedTest): - handler = hash.cisco_pix - - #: known correct pix hashes - known_correct_hashes = _PixAsaSharedTest.pix_asa_shared_hashes + [ - # - # passlib reference vectors (PIX-specific) - # - # NOTE: See 'pix_asa_shared_hashes' for general PIX+ASA vectors, - # and general notes about the 'passlib reference vectors' test set. - # - # All of the following are PIX-specific, as ASA starts - # to use a different padding size at 13 characters. - # - # TODO: these need confirming w/ an actual PIX system. - # - - # 13 char password - (('0123456789abc', ''), 'eacOpB7vE7ZDukSF'), - (('0123456789abc', '3'), 'ylJTd/qei66WZe3w'), - (('0123456789abc', '36'), 'hDx8QRlUhwd6bU8N'), - (('0123456789abc', '365'), 'vYOOtnkh1HXcMrM7'), - (('0123456789abc', '3333'), 'ylJTd/qei66WZe3w'), - (('0123456789abc', '3636'), 'hDx8QRlUhwd6bU8N'), - (('0123456789abc', '3653'), 'vYOOtnkh1HXcMrM7'), - (('0123456789abc', 'user'), 'f4/.SALxqDo59mfV'), - (('0123456789abc', 'user1234'), 'f4/.SALxqDo59mfV'), - - # 14 char password - (('0123456789abcd', ''), '6r8888iMxEoPdLp4'), - (('0123456789abcd', '3'), 'f5lvmqWYj9gJqkIH'), - (('0123456789abcd', '36'), 'OJJ1Khg5HeAYBH1c'), - (('0123456789abcd', '365'), 'OJJ1Khg5HeAYBH1c'), - (('0123456789abcd', '3333'), 'f5lvmqWYj9gJqkIH'), - (('0123456789abcd', '3636'), 'OJJ1Khg5HeAYBH1c'), - (('0123456789abcd', '3653'), 'OJJ1Khg5HeAYBH1c'), - (('0123456789abcd', 'adm'), 'DbPLCFIkHc2SiyDk'), - (('0123456789abcd', 'adma'), 'DbPLCFIkHc2SiyDk'), - (('0123456789abcd', 'user'), 'WfO2UiTapPkF/FSn'), - (('0123456789abcd', 'user1234'), 'WfO2UiTapPkF/FSn'), - - # 15 char password - (('0123456789abcde', ''), 'al1e0XFIugTYLai3'), - (('0123456789abcde', '3'), 'lYbwBu.f82OIApQB'), - (('0123456789abcde', '36'), 'lYbwBu.f82OIApQB'), - (('0123456789abcde', '365'), 'lYbwBu.f82OIApQB'), - (('0123456789abcde', '3333'), 'lYbwBu.f82OIApQB'), - (('0123456789abcde', '3636'), 'lYbwBu.f82OIApQB'), - (('0123456789abcde', '3653'), 'lYbwBu.f82OIApQB'), - (('0123456789abcde', 'adm'), 'KgKx1UQvdR/09i9u'), - (('0123456789abcde', 'adma'), 'KgKx1UQvdR/09i9u'), - (('0123456789abcde', 'user'), 'qLopkenJ4WBqxaZN'), - (('0123456789abcde', 'user1234'), 'qLopkenJ4WBqxaZN'), - - # 16 char password - (('0123456789abcdef', ''), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', '36'), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', '365'), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', '3333'), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', '3636'), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', '3653'), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', 'user'), '.7nfVBEIEu4KbF/1'), - (('0123456789abcdef', 'user1234'), '.7nfVBEIEu4KbF/1'), - ] - - -#============================================================================= -# cisco asa -#============================================================================= -class cisco_asa_test(_PixAsaSharedTest): - handler = hash.cisco_asa - - known_correct_hashes = _PixAsaSharedTest.pix_asa_shared_hashes + [ - # - # passlib reference vectors (ASA-specific) - # - # NOTE: See 'pix_asa_shared_hashes' for general PIX+ASA vectors, - # and general notes about the 'passlib reference vectors' test set. - # - - # 13 char password - # NOTE: past this point, ASA pads to 32 bytes instead of 16 - # for all cases where user is set (secret + 4 bytes > 16), - # but still uses 16 bytes for enable pwds (secret <= 16). - # hashes w/ user WON'T match PIX, but "enable" passwords will. - (('0123456789abc', ''), 'eacOpB7vE7ZDukSF'), # confirmed ASA 9.6 - (('0123456789abc', '36'), 'FRV9JG18UBEgX0.O'), - (('0123456789abc', '365'), 'NIwkusG9hmmMy6ZQ'), # confirmed ASA 9.6 - (('0123456789abc', '3333'), 'NmrkP98nT7RAeKZz'), # confirmed ASA 9.6 - (('0123456789abc', '3636'), 'FRV9JG18UBEgX0.O'), # confirmed ASA 9.6 - (('0123456789abc', '3653'), 'NIwkusG9hmmMy6ZQ'), # confirmed ASA 9.6 - (('0123456789abc', 'user'), '8Q/FZeam5ai1A47p'), # confirmed ASA 9.6 - (('0123456789abc', 'user1234'), '8Q/FZeam5ai1A47p'), # confirmed ASA 9.6 - - # 14 char password - (('0123456789abcd', ''), '6r8888iMxEoPdLp4'), # confirmed ASA 9.6 - (('0123456789abcd', '3'), 'yxGoujXKPduTVaYB'), - (('0123456789abcd', '36'), 'W0jckhnhjnr/DiT/'), - (('0123456789abcd', '365'), 'HuVOxfMQNahaoF8u'), # confirmed ASA 9.6 - (('0123456789abcd', '3333'), 'yxGoujXKPduTVaYB'), # confirmed ASA 9.6 - (('0123456789abcd', '3636'), 'W0jckhnhjnr/DiT/'), # confirmed ASA 9.6 - (('0123456789abcd', '3653'), 'HuVOxfMQNahaoF8u'), # confirmed ASA 9.6 - (('0123456789abcd', 'adm'), 'RtOmSeoCs4AUdZqZ'), # confirmed ASA 9.6 - (('0123456789abcd', 'adma'), 'RtOmSeoCs4AUdZqZ'), # confirmed ASA 9.6 - (('0123456789abcd', 'user'), 'rrucwrcM0h25pr.m'), # confirmed ASA 9.6 - (('0123456789abcd', 'user1234'), 'rrucwrcM0h25pr.m'), # confirmed ASA 9.6 - - # 15 char password - (('0123456789abcde', ''), 'al1e0XFIugTYLai3'), # confirmed ASA 9.6 - (('0123456789abcde', '3'), 'nAZrQoHaL.fgrIqt'), - (('0123456789abcde', '36'), '2GxIQ6ICE795587X'), - (('0123456789abcde', '365'), 'QmDsGwCRBbtGEKqM'), # confirmed ASA 9.6 - (('0123456789abcde', '3333'), 'nAZrQoHaL.fgrIqt'), # confirmed ASA 9.6 - (('0123456789abcde', '3636'), '2GxIQ6ICE795587X'), # confirmed ASA 9.6 - (('0123456789abcde', '3653'), 'QmDsGwCRBbtGEKqM'), # confirmed ASA 9.6 - (('0123456789abcde', 'adm'), 'Aj2aP0d.nk62wl4m'), # confirmed ASA 9.6 - (('0123456789abcde', 'adma'), 'Aj2aP0d.nk62wl4m'), # confirmed ASA 9.6 - (('0123456789abcde', 'user'), 'etxiXfo.bINJcXI7'), # confirmed ASA 9.6 - (('0123456789abcde', 'user1234'), 'etxiXfo.bINJcXI7'), # confirmed ASA 9.6 - - # 16 char password - (('0123456789abcdef', ''), '.7nfVBEIEu4KbF/1'), # confirmed ASA 9.6 - (('0123456789abcdef', '36'), 'GhI8.yFSC5lwoafg'), - (('0123456789abcdef', '365'), 'KFBI6cNQauyY6h/G'), # confirmed ASA 9.6 - (('0123456789abcdef', '3333'), 'Ghdi1IlsswgYzzMH'), # confirmed ASA 9.6 - (('0123456789abcdef', '3636'), 'GhI8.yFSC5lwoafg'), # confirmed ASA 9.6 - (('0123456789abcdef', '3653'), 'KFBI6cNQauyY6h/G'), # confirmed ASA 9.6 - (('0123456789abcdef', 'user'), 'IneB.wc9sfRzLPoh'), # confirmed ASA 9.6 - (('0123456789abcdef', 'user1234'), 'IneB.wc9sfRzLPoh'), # confirmed ASA 9.6 - - # 17 char password - # NOTE: past this point, ASA pads to 32 bytes instead of 16 - # for ALL cases, since secret > 16 bytes even for enable pwds; - # and so none of these rest here should match PIX. - (('0123456789abcdefq', ''), 'bKshl.EN.X3CVFRQ'), # confirmed ASA 9.6 - (('0123456789abcdefq', '36'), 'JAeTXHs0n30svlaG'), - (('0123456789abcdefq', '365'), '4fKSSUBHT1ChGqHp'), # confirmed ASA 9.6 - (('0123456789abcdefq', '3333'), 'USEJbxI6.VY4ecBP'), # confirmed ASA 9.6 - (('0123456789abcdefq', '3636'), 'JAeTXHs0n30svlaG'), # confirmed ASA 9.6 - (('0123456789abcdefq', '3653'), '4fKSSUBHT1ChGqHp'), # confirmed ASA 9.6 - (('0123456789abcdefq', 'user'), '/dwqyD7nGdwSrDwk'), # confirmed ASA 9.6 - (('0123456789abcdefq', 'user1234'), '/dwqyD7nGdwSrDwk'), # confirmed ASA 9.6 - - # 27 char password - (('0123456789abcdefqwertyuiopa', ''), '4wp19zS3OCe.2jt5'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopa', '36'), 'PjUoGqWBKPyV9qOe'), - (('0123456789abcdefqwertyuiopa', '365'), 'bfCy6xFAe5O/gzvM'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopa', '3333'), 'rd/ZMuGTJFIb2BNG'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopa', '3636'), 'PjUoGqWBKPyV9qOe'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopa', '3653'), 'bfCy6xFAe5O/gzvM'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopa', 'user'), 'zynfWw3UtszxLMgL'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopa', 'user1234'), 'zynfWw3UtszxLMgL'), # confirmed ASA 9.6 - - # 28 char password - # NOTE: past this point, ASA stops appending the username AT ALL, - # even though there's still room for the first few chars. - (('0123456789abcdefqwertyuiopas', ''), 'W6nbOddI0SutTK7m'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopas', '36'), 'W6nbOddI0SutTK7m'), - (('0123456789abcdefqwertyuiopas', '365'), 'W6nbOddI0SutTK7m'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopas', 'user'), 'W6nbOddI0SutTK7m'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopas', 'user1234'), 'W6nbOddI0SutTK7m'), # confirmed ASA 9.6 - - # 32 char password - # NOTE: this is max size that ASA allows, and throws error for larger - (('0123456789abcdefqwertyuiopasdfgh', ''), '5hPT/iC6DnoBxo6a'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopasdfgh', '36'), '5hPT/iC6DnoBxo6a'), - (('0123456789abcdefqwertyuiopasdfgh', '365'), '5hPT/iC6DnoBxo6a'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopasdfgh', 'user'), '5hPT/iC6DnoBxo6a'), # confirmed ASA 9.6 - (('0123456789abcdefqwertyuiopasdfgh', 'user1234'), '5hPT/iC6DnoBxo6a'), # confirmed ASA 9.6 - ] - - -#============================================================================= -# cisco type 7 -#============================================================================= -class cisco_type7_test(HandlerCase): - handler = hash.cisco_type7 - salt_bits = 4 - salt_type = int - - known_correct_hashes = [ - # - # http://mccltd.net/blog/?p=1034 - # - ("secure ", "04480E051A33490E"), - - # - # http://insecure.org/sploits/cisco.passwords.html - # - ("Its time to go to lunch!", - "153B1F1F443E22292D73212D5300194315591954465A0D0B59"), - - # - # http://blog.ioshints.info/2007/11/type-7-decryption-in-cisco-ios.html - # - ("t35t:pa55w0rd", "08351F1B1D431516475E1B54382F"), - - # - # http://www.m00nie.com/2011/09/cisco-type-7-password-decryption-and-encryption-with-perl/ - # - ("hiImTesting:)", "020E0D7206320A325847071E5F5E"), - - # - # http://packetlife.net/forums/thread/54/ - # - ("cisco123", "060506324F41584B56"), - ("cisco123", "1511021F07257A767B"), - - # - # source ? - # - ('Supe&8ZUbeRp4SS', "06351A3149085123301517391C501918"), - - # - # custom - # - - # ensures utf-8 used for unicode - (UPASS_TABLE, '0958EDC8A9F495F6F8A5FD'), - ] - - known_unidentified_hashes = [ - # salt with hex value - "0A480E051A33490E", - - # salt value > 52. this may in fact be valid, but we reject it for now - # (see docs for more). - '99400E4812', - ] - - def test_90_decode(self): - """test cisco_type7.decode()""" - from passlib.utils import to_unicode, to_bytes - - handler = self.handler - for secret, hash in self.known_correct_hashes: - usecret = to_unicode(secret) - bsecret = to_bytes(secret) - self.assertEqual(handler.decode(hash), usecret) - self.assertEqual(handler.decode(hash, None), bsecret) - - self.assertRaises(UnicodeDecodeError, handler.decode, - '0958EDC8A9F495F6F8A5FD', 'ascii') - - def test_91_salt(self): - """test salt value border cases""" - handler = self.handler - self.assertRaises(TypeError, handler, salt=None) - handler(salt=None, use_defaults=True) - self.assertRaises(TypeError, handler, salt='abc') - self.assertRaises(ValueError, handler, salt=-10) - self.assertRaises(ValueError, handler, salt=100) - - self.assertRaises(TypeError, handler.using, salt='abc') - self.assertRaises(ValueError, handler.using, salt=-10) - self.assertRaises(ValueError, handler.using, salt=100) - with self.assertWarningList("salt/offset must be.*"): - subcls = handler.using(salt=100, relaxed=True) - self.assertEqual(subcls(use_defaults=True).salt, 52) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers_django.py b/src/passlib/tests/test_handlers_django.py deleted file mode 100644 index 72e42a49..00000000 --- a/src/passlib/tests/test_handlers_django.py +++ /dev/null @@ -1,401 +0,0 @@ -"""passlib.tests.test_handlers_django - tests for passlib hash algorithms""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -import warnings -# site -# pkg -from passlib import hash -from passlib.utils import repeat_string -from passlib.utils.compat import u -from passlib.tests.utils import TestCase, HandlerCase, skipUnless, SkipTest -from passlib.tests.test_handlers import UPASS_USD, UPASS_TABLE -from passlib.tests.test_ext_django import DJANGO_VERSION, MIN_DJANGO_VERSION -# module - -#============================================================================= -# django -#============================================================================= - -# standard string django uses -UPASS_LETMEIN = u('l\xe8tmein') - -def vstr(version): - return ".".join(str(e) for e in version) - -class _DjangoHelper(TestCase): - __unittest_skip = True - - #: minimum django version where hash alg is present / that we support testing against - min_django_version = MIN_DJANGO_VERSION - - #: max django version where hash alg is present - max_django_version = None - - def _require_django_support(self): - if DJANGO_VERSION < self.min_django_version: - raise self.skipTest("Django >= %s not installed" % vstr(self.min_django_version)) - if self.max_django_version and DJANGO_VERSION > self.max_django_version: - raise self.skipTest("Django <= %s not installed" % vstr(self.max_django_version)) - return True - - extra_fuzz_verifiers = HandlerCase.fuzz_verifiers + ( - "fuzz_verifier_django", - ) - - def fuzz_verifier_django(self): - try: - self._require_django_support() - except SkipTest: - return None - from django.contrib.auth.hashers import check_password - - def verify_django(secret, hash): - """django/check_password""" - if self.handler.name == "django_bcrypt" and hash.startswith("bcrypt$$2y$"): - hash = hash.replace("$$2y$", "$$2a$") - if self.django_has_encoding_glitch and isinstance(secret, bytes): - # e.g. unsalted_md5 on 1.5 and higher try to combine - # salt + password before encoding to bytes, leading to ascii error. - # this works around that issue. - secret = secret.decode("utf-8") - return check_password(secret, hash) - return verify_django - - def test_90_django_reference(self): - """run known correct hashes through Django's check_password()""" - self._require_django_support() - # XXX: esp. when it's no longer supported by django, - # should verify it's *NOT* recognized - from django.contrib.auth.hashers import check_password - assert self.known_correct_hashes - for secret, hash in self.iter_known_hashes(): - self.assertTrue(check_password(secret, hash), - "secret=%r hash=%r failed to verify" % - (secret, hash)) - self.assertFalse(check_password('x' + secret, hash), - "mangled secret=%r hash=%r incorrect verified" % - (secret, hash)) - - django_has_encoding_glitch = False - - def test_91_django_generation(self): - """test against output of Django's make_password()""" - self._require_django_support() - # XXX: esp. when it's no longer supported by django, - # should verify it's *NOT* recognized - from passlib.utils import tick - from django.contrib.auth.hashers import make_password - name = self.handler.django_name # set for all the django_* handlers - end = tick() + self.max_fuzz_time/2 - generator = self.FuzzHashGenerator(self, self.getRandom()) - while tick() < end: - secret, other = generator.random_password_pair() - if not secret: # django rejects empty passwords. - continue - if self.django_has_encoding_glitch and isinstance(secret, bytes): - # e.g. unsalted_md5 tried to combine salt + password before encoding to bytes, - # leading to ascii error. this works around that issue. - secret = secret.decode("utf-8") - hash = make_password(secret, hasher=name) - self.assertTrue(self.do_identify(hash)) - self.assertTrue(self.do_verify(secret, hash)) - self.assertFalse(self.do_verify(other, hash)) - -class django_disabled_test(HandlerCase): - """test django_disabled""" - handler = hash.django_disabled - disabled_contains_salt = True - - known_correct_hashes = [ - # *everything* should hash to "!", and nothing should verify - ("password", "!"), - ("", "!"), - (UPASS_TABLE, "!"), - ] - - known_alternate_hashes = [ - # django 1.6 appends random alpnum string - ("!9wa845vn7098ythaehasldkfj", "password", "!"), - ] - -class django_des_crypt_test(HandlerCase, _DjangoHelper): - """test django_des_crypt""" - handler = hash.django_des_crypt - max_django_version = (1,9) - - known_correct_hashes = [ - # ensures only first two digits of salt count. - ("password", 'crypt$c2$c2M87q...WWcU'), - ("password", 'crypt$c2e86$c2M87q...WWcU'), - ("passwordignoreme", 'crypt$c2.AZ$c2M87q...WWcU'), - - # ensures utf-8 used for unicode - (UPASS_USD, 'crypt$c2e86$c2hN1Bxd6ZiWs'), - (UPASS_TABLE, 'crypt$0.aQs$0.wB.TT0Czvlo'), - (u("hell\u00D6"), "crypt$sa$saykDgk3BPZ9E"), - - # prevent regression of issue 22 - ("foo", 'crypt$MNVY.9ajgdvDQ$MNVY.9ajgdvDQ'), - ] - - known_alternate_hashes = [ - # ensure django 1.4 empty salt field is accepted; - # but that salt field is re-filled (for django 1.0 compatibility) - ('crypt$$c2M87q...WWcU', "password", 'crypt$c2$c2M87q...WWcU'), - ] - - known_unidentified_hashes = [ - 'sha1$aa$bb', - ] - - known_malformed_hashes = [ - # checksum too short - 'crypt$c2$c2M87q', - - # salt must be >2 - 'crypt$f$c2M87q...WWcU', - - # make sure first 2 chars of salt & chk field agree. - 'crypt$ffe86$c2M87q...WWcU', - ] - -class django_salted_md5_test(HandlerCase, _DjangoHelper): - """test django_salted_md5""" - handler = hash.django_salted_md5 - max_django_version = (1,9) - - django_has_encoding_glitch = True - - known_correct_hashes = [ - # test extra large salt - ("password", 'md5$123abcdef$c8272612932975ee80e8a35995708e80'), - - # test django 1.4 alphanumeric salt - ("test", 'md5$3OpqnFAHW5CT$54b29300675271049a1ebae07b395e20'), - - # ensures utf-8 used for unicode - (UPASS_USD, 'md5$c2e86$92105508419a81a6babfaecf876a2fa0'), - (UPASS_TABLE, 'md5$d9eb8$01495b32852bffb27cf5d4394fe7a54c'), - ] - - known_unidentified_hashes = [ - 'sha1$aa$bb', - ] - - known_malformed_hashes = [ - # checksum too short - 'md5$aa$bb', - ] - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def random_salt_size(self): - # workaround for django14 regression -- - # 1.4 won't accept hashes with empty salt strings, unlike 1.3 and earlier. - # looks to be fixed in a future release -- https://code.djangoproject.com/ticket/18144 - # for now, we avoid salt_size==0 under 1.4 - handler = self.handler - default = handler.default_salt_size - assert handler.min_salt_size == 0 - lower = 1 - upper = handler.max_salt_size or default*4 - return self.randintgauss(lower, upper, default, default*.5) - -class django_salted_sha1_test(HandlerCase, _DjangoHelper): - """test django_salted_sha1""" - handler = hash.django_salted_sha1 - max_django_version = (1,9) - - django_has_encoding_glitch = True - - known_correct_hashes = [ - # test extra large salt - ("password",'sha1$123abcdef$e4a1877b0e35c47329e7ed7e58014276168a37ba'), - - # test django 1.4 alphanumeric salt - ("test", 'sha1$bcwHF9Hy8lxS$6b4cfa0651b43161c6f1471ce9523acf1f751ba3'), - - # ensures utf-8 used for unicode - (UPASS_USD, 'sha1$c2e86$0f75c5d7fbd100d587c127ef0b693cde611b4ada'), - (UPASS_TABLE, 'sha1$6d853$ef13a4d8fb57aed0cb573fe9c82e28dc7fd372d4'), - - # generic password - ("MyPassword", 'sha1$54123$893cf12e134c3c215f3a76bd50d13f92404a54d3'), - ] - - known_unidentified_hashes = [ - 'md5$aa$bb', - ] - - known_malformed_hashes = [ - # checksum too short - 'sha1$c2e86$0f75', - ] - - # reuse custom random_salt_size() helper... - FuzzHashGenerator = django_salted_md5_test.FuzzHashGenerator - -class django_pbkdf2_sha256_test(HandlerCase, _DjangoHelper): - """test django_pbkdf2_sha256""" - handler = hash.django_pbkdf2_sha256 - - known_correct_hashes = [ - # - # custom - generated via django 1.4 hasher - # - ('not a password', - 'pbkdf2_sha256$10000$kjVJaVz6qsnJ$5yPHw3rwJGECpUf70daLGhOrQ5+AMxIJdz1c3bqK1Rs='), - (UPASS_TABLE, - 'pbkdf2_sha256$10000$bEwAfNrH1TlQ$OgYUblFNUX1B8GfMqaCYUK/iHyO0pa7STTDdaEJBuY0='), - ] - -class django_pbkdf2_sha1_test(HandlerCase, _DjangoHelper): - """test django_pbkdf2_sha1""" - handler = hash.django_pbkdf2_sha1 - - known_correct_hashes = [ - # - # custom - generated via django 1.4 hashers - # - ('not a password', - 'pbkdf2_sha1$10000$wz5B6WkasRoF$atJmJ1o+XfJxKq1+Nu1f1i57Z5I='), - (UPASS_TABLE, - 'pbkdf2_sha1$10000$KZKWwvqb8BfL$rw5pWsxJEU4JrZAQhHTCO+u0f5Y='), - ] - -@skipUnless(hash.bcrypt.has_backend(), "no bcrypt backends available") -class django_bcrypt_test(HandlerCase, _DjangoHelper): - """test django_bcrypt""" - handler = hash.django_bcrypt - fuzz_salts_need_bcrypt_repair = True - - known_correct_hashes = [ - # - # just copied and adapted a few test vectors from bcrypt (above), - # since django_bcrypt is just a wrapper for the real bcrypt class. - # - ('', 'bcrypt$$2a$06$DCq7YPn5Rq63x1Lad4cll.TV4S6ytwfsfvkgY8jIucDrjc8deX1s.'), - ('abcdefghijklmnopqrstuvwxyz', - 'bcrypt$$2a$10$fVH8e28OQRj9tqiDXs1e1uxpsjN0c7II7YPKXua2NAKYvM6iQk7dq'), - (UPASS_TABLE, - 'bcrypt$$2a$05$Z17AXnnlpzddNUvnC6cZNOSwMA/8oNiKnHTHTwLlBijfucQQlHjaG'), - ] - - # NOTE: the following have been cloned from _bcrypt_test() - - def populate_settings(self, kwds): - # speed up test w/ lower rounds - kwds.setdefault("rounds", 4) - super(django_bcrypt_test, self).populate_settings(kwds) - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(5, 8, 6, 1) - - def random_ident(self): - # omit multi-ident tests, only $2a$ counts for this class - # XXX: enable this to check 2a / 2b? - return None - -@skipUnless(hash.bcrypt.has_backend(), "no bcrypt backends available") -class django_bcrypt_sha256_test(HandlerCase, _DjangoHelper): - """test django_bcrypt_sha256""" - handler = hash.django_bcrypt_sha256 - forbidden_characters = None - fuzz_salts_need_bcrypt_repair = True - - known_correct_hashes = [ - # - # custom - generated via django 1.6 hasher - # - ('', - 'bcrypt_sha256$$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu'), - (UPASS_LETMEIN, - 'bcrypt_sha256$$2a$08$NDjSAIcas.EcoxCRiArvT.MkNiPYVhrsrnJsRkLueZOoV1bsQqlmC'), - (UPASS_TABLE, - 'bcrypt_sha256$$2a$06$kCXUnRFQptGg491siDKNTu8RxjBGSjALHRuvhPYNFsa4Ea5d9M48u'), - - # test >72 chars is hashed correctly -- under bcrypt these hash the same. - (repeat_string("abc123",72), - 'bcrypt_sha256$$2a$06$Tg/oYyZTyAf.Nb3qSgN61OySmyXA8FoY4PjGizjE1QSDfuL5MXNni'), - (repeat_string("abc123",72)+"qwr", - 'bcrypt_sha256$$2a$06$Tg/oYyZTyAf.Nb3qSgN61Ocy0BEz1RK6xslSNi8PlaLX2pe7x/KQG'), - (repeat_string("abc123",72)+"xyz", - 'bcrypt_sha256$$2a$06$Tg/oYyZTyAf.Nb3qSgN61OvY2zoRVUa2Pugv2ExVOUT2YmhvxUFUa'), - ] - - known_malformed_hashers = [ - # data in django salt field - 'bcrypt_sha256$xyz$2a$06$/3OeRpbOf8/l6nPPRdZPp.nRiyYqPobEZGdNRBWihQhiFDh1ws1tu', - ] - - # NOTE: the following have been cloned from _bcrypt_test() - - def populate_settings(self, kwds): - # speed up test w/ lower rounds - kwds.setdefault("rounds", 4) - super(django_bcrypt_sha256_test, self).populate_settings(kwds) - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(5, 8, 6, 1) - - def random_ident(self): - # omit multi-ident tests, only $2a$ counts for this class - # XXX: enable this to check 2a / 2b? - return None - -from passlib.tests.test_handlers_argon2 import _base_argon2_test - -@skipUnless(hash.argon2.has_backend(), "no argon2 backends available") -class django_argon2_test(HandlerCase, _DjangoHelper): - """test django_bcrypt""" - handler = hash.django_argon2 - - # NOTE: most of this adapted from _base_argon2_test & argon2pure test - - known_correct_hashes = [ - # sample test - ("password", 'argon2$argon2i$v=19$m=256,t=1,p=1$c29tZXNhbHQ$AJFIsNZTMKTAewB4+ETN1A'), - - # sample w/ all parameters different - ("password", 'argon2$argon2i$v=19$m=380,t=2,p=2$c29tZXNhbHQ$SrssP8n7m/12VWPM8dvNrw'), - - # generated from django 1.10.3 - (UPASS_LETMEIN, 'argon2$argon2i$v=19$m=512,t=2,p=2$V25jN1l4UUJZWkR1$MxpA1BD2Gh7+D79gaAw6sQ'), - ] - - def setUpWarnings(self): - super(django_argon2_test, self).setUpWarnings() - warnings.filterwarnings("ignore", ".*Using argon2pure backend.*") - - def do_stub_encrypt(self, handler=None, **settings): - # overriding default since no way to get stub config from argon2._calc_hash() - # (otherwise test_21b_max_rounds blocks trying to do max rounds) - handler = (handler or self.handler).using(**settings) - self = handler.wrapped(use_defaults=True) - self.checksum = self._stub_checksum - assert self.checksum - return handler._wrap_hash(self.to_string()) - - def test_03_legacy_hash_workflow(self): - # override base method - raise self.skipTest("legacy 1.6 workflow not supported") - - class FuzzHashGenerator(_base_argon2_test.FuzzHashGenerator): - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(1, 3, 2, 1) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers_pbkdf2.py b/src/passlib/tests/test_handlers_pbkdf2.py deleted file mode 100644 index 4d2f048f..00000000 --- a/src/passlib/tests/test_handlers_pbkdf2.py +++ /dev/null @@ -1,480 +0,0 @@ -"""passlib.tests.test_handlers - tests for passlib hash algorithms""" -#============================================================================= -# imports -#============================================================================= -# core -import logging -log = logging.getLogger(__name__) -import warnings -# site -# pkg -from passlib import hash -from passlib.utils.compat import u -from passlib.tests.utils import TestCase, HandlerCase -from passlib.tests.test_handlers import UPASS_WAV -# module - -#============================================================================= -# ldap_pbkdf2_{digest} -#============================================================================= -# NOTE: since these are all wrappers for the pbkdf2_{digest} hasehs, -# they don't extensive separate testing. - -class ldap_pbkdf2_test(TestCase): - - def test_wrappers(self): - """test ldap pbkdf2 wrappers""" - - self.assertTrue( - hash.ldap_pbkdf2_sha1.verify( - "password", - '{PBKDF2}1212$OB.dtnSEXZK8U5cgxU/GYQ$y5LKPOplRmok7CZp/aqVDVg8zGI', - ) - ) - - self.assertTrue( - hash.ldap_pbkdf2_sha256.verify( - "password", - '{PBKDF2-SHA256}1212$4vjV83LKPjQzk31VI4E0Vw$hsYF68OiOUPdDZ1Fg' - '.fJPeq1h/gXXY7acBp9/6c.tmQ' - ) - ) - - self.assertTrue( - hash.ldap_pbkdf2_sha512.verify( - "password", - '{PBKDF2-SHA512}1212$RHY0Fr3IDMSVO/RSZyb5ow$eNLfBK.eVozomMr.1gYa1' - '7k9B7KIK25NOEshvhrSX.esqY3s.FvWZViXz4KoLlQI.BzY/YTNJOiKc5gBYFYGww' - ) - ) - -#============================================================================= -# pbkdf2 hashes -#============================================================================= -class atlassian_pbkdf2_sha1_test(HandlerCase): - handler = hash.atlassian_pbkdf2_sha1 - - known_correct_hashes = [ - # - # generated using Jira - # - ("admin", '{PKCS5S2}c4xaeTQM0lUieMS3V5voiexyX9XhqC2dBd5ecVy60IPksHChwoTAVYFrhsgoq8/p'), - (UPASS_WAV, - "{PKCS5S2}cE9Yq6Am5tQGdHSHhky2XLeOnURwzaLBG2sur7FHKpvy2u0qDn6GcVGRjlmJoIUy"), - ] - - known_malformed_hashes = [ - # bad char ---\/ - '{PKCS5S2}c4xaeTQM0lUieMS3V5voiexyX9XhqC2dBd5ecVy!0IPksHChwoTAVYFrhsgoq8/p' - - # bad size, missing padding - '{PKCS5S2}c4xaeTQM0lUieMS3V5voiexyX9XhqC2dBd5ecVy60IPksHChwoTAVYFrhsgoq8/' - - # bad size, with correct padding - '{PKCS5S2}c4xaeTQM0lUieMS3V5voiexyX9XhqC2dBd5ecVy60IPksHChwoTAVYFrhsgoq8/=' - ] - -class pbkdf2_sha1_test(HandlerCase): - handler = hash.pbkdf2_sha1 - known_correct_hashes = [ - ("password", '$pbkdf2$1212$OB.dtnSEXZK8U5cgxU/GYQ$y5LKPOplRmok7CZp/aqVDVg8zGI'), - (UPASS_WAV, - '$pbkdf2$1212$THDqatpidANpadlLeTeOEg$HV3oi1k5C5LQCgG1BMOL.BX4YZc'), - ] - - known_malformed_hashes = [ - # zero padded rounds field - '$pbkdf2$01212$THDqatpidANpadlLeTeOEg$HV3oi1k5C5LQCgG1BMOL.BX4YZc', - - # empty rounds field - '$pbkdf2$$THDqatpidANpadlLeTeOEg$HV3oi1k5C5LQCgG1BMOL.BX4YZc', - - # too many field - '$pbkdf2$1212$THDqatpidANpadlLeTeOEg$HV3oi1k5C5LQCgG1BMOL.BX4YZc$', - ] - -class pbkdf2_sha256_test(HandlerCase): - handler = hash.pbkdf2_sha256 - known_correct_hashes = [ - ("password", - '$pbkdf2-sha256$1212$4vjV83LKPjQzk31VI4E0Vw$hsYF68OiOUPdDZ1Fg.fJPeq1h/gXXY7acBp9/6c.tmQ' - ), - (UPASS_WAV, - '$pbkdf2-sha256$1212$3SABFJGDtyhrQMVt1uABPw$WyaUoqCLgvz97s523nF4iuOqZNbp5Nt8do/cuaa7AiI' - ), - ] - -class pbkdf2_sha512_test(HandlerCase): - handler = hash.pbkdf2_sha512 - known_correct_hashes = [ - ("password", - '$pbkdf2-sha512$1212$RHY0Fr3IDMSVO/RSZyb5ow$eNLfBK.eVozomMr.1gYa1' - '7k9B7KIK25NOEshvhrSX.esqY3s.FvWZViXz4KoLlQI.BzY/YTNJOiKc5gBYFYGww' - ), - (UPASS_WAV, - '$pbkdf2-sha512$1212$KkbvoKGsAIcF8IslDR6skQ$8be/PRmd88Ps8fmPowCJt' - 'tH9G3vgxpG.Krjt3KT.NP6cKJ0V4Prarqf.HBwz0dCkJ6xgWnSj2ynXSV7MlvMa8Q' - ), - ] - -class cta_pbkdf2_sha1_test(HandlerCase): - handler = hash.cta_pbkdf2_sha1 - known_correct_hashes = [ - # - # test vectors from original implementation - # - (u("hashy the \N{SNOWMAN}"), '$p5k2$1000$ZxK4ZBJCfQg=$jJZVscWtO--p1-xIZl6jhO2LKR0='), - - # - # custom - # - ("password", "$p5k2$1$$h1TDLGSw9ST8UMAPeIE13i0t12c="), - (UPASS_WAV, - "$p5k2$4321$OTg3NjU0MzIx$jINJrSvZ3LXeIbUdrJkRpN62_WQ="), - ] - -class dlitz_pbkdf2_sha1_test(HandlerCase): - handler = hash.dlitz_pbkdf2_sha1 - known_correct_hashes = [ - # - # test vectors from original implementation - # - ('cloadm', '$p5k2$$exec$r1EWMCMk7Rlv3L/RNcFXviDefYa0hlql'), - ('gnu', '$p5k2$c$u9HvcT4d$Sd1gwSVCLZYAuqZ25piRnbBEoAesaa/g'), - ('dcl', '$p5k2$d$tUsch7fU$nqDkaxMDOFBeJsTSfABsyn.PYUXilHwL'), - ('spam', '$p5k2$3e8$H0NX9mT/$wk/sE8vv6OMKuMaqazCJYDSUhWY9YB2J'), - (UPASS_WAV, - '$p5k2$$KosHgqNo$9mjN8gqjt02hDoP0c2J0ABtLIwtot8cQ'), - ] - -class grub_pbkdf2_sha512_test(HandlerCase): - handler = hash.grub_pbkdf2_sha512 - known_correct_hashes = [ - # - # test vectors generated from cmd line tool - # - - # salt=32 bytes - (UPASS_WAV, - 'grub.pbkdf2.sha512.10000.BCAC1CEC5E4341C8C511C529' - '7FA877BE91C2817B32A35A3ECF5CA6B8B257F751.6968526A' - '2A5B1AEEE0A29A9E057336B48D388FFB3F600233237223C21' - '04DE1752CEC35B0DD1ED49563398A282C0F471099C2803FBA' - '47C7919CABC43192C68F60'), - - # salt=64 bytes - ('toomanysecrets', - 'grub.pbkdf2.sha512.10000.9B436BB6978682363D5C449B' - 'BEAB322676946C632208BC1294D51F47174A9A3B04A7E4785' - '986CD4EA7470FAB8FE9F6BD522D1FC6C51109A8596FB7AD48' - '7C4493.0FE5EF169AFFCB67D86E2581B1E251D88C777B98BA' - '2D3256ECC9F765D84956FC5CA5C4B6FD711AA285F0A04DCF4' - '634083F9A20F4B6F339A52FBD6BED618E527B'), - - ] - -#============================================================================= -# scram hash -#============================================================================= -class scram_test(HandlerCase): - handler = hash.scram - - # TODO: need a bunch more reference vectors from some real - # SCRAM transactions. - known_correct_hashes = [ - # - # taken from example in SCRAM specification (rfc 5802) - # - ('pencil', '$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30'), - - # - # custom - # - - # same as 5802 example hash, but with sha-256 & sha-512 added. - ('pencil', '$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' - 'sha-256=qXUXrlcvnaxxWG00DdRgVioR2gnUpuX5r.3EZ1rdhVY,' - 'sha-512=lzgniLFcvglRLS0gt.C4gy.NurS3OIOVRAU1zZOV4P.qFiVFO2/' - 'edGQSu/kD1LwdX0SNV/KsPdHSwEl5qRTuZQ'), - - # test unicode passwords & saslprep (all the passwords below - # should normalize to the same value: 'IX \xE0') - (u('IX \xE0'), '$scram$6400$0BojBCBE6P2/N4bQ$' - 'sha-1=YniLes.b8WFMvBhtSACZyyvxeCc'), - (u('\u2168\u3000a\u0300'), '$scram$6400$0BojBCBE6P2/N4bQ$' - 'sha-1=YniLes.b8WFMvBhtSACZyyvxeCc'), - (u('\u00ADIX \xE0'), '$scram$6400$0BojBCBE6P2/N4bQ$' - 'sha-1=YniLes.b8WFMvBhtSACZyyvxeCc'), - ] - - known_malformed_hashes = [ - # zero-padding in rounds - '$scram$04096$QSXCR.Q6sek8bf92$sha-1=HZbuOlKbWl.eR8AfIposuKbhX30', - - # non-digit in rounds - '$scram$409A$QSXCR.Q6sek8bf92$sha-1=HZbuOlKbWl.eR8AfIposuKbhX30', - - # bad char in salt ---\/ - '$scram$4096$QSXCR.Q6sek8bf9-$sha-1=HZbuOlKbWl.eR8AfIposuKbhX30', - - # bad char in digest ---\/ - '$scram$4096$QSXCR.Q6sek8bf92$sha-1=HZbuOlKbWl.eR8AfIposuKbhX3-', - - # missing sections - '$scram$4096$QSXCR.Q6sek8bf92', - '$scram$4096$QSXCR.Q6sek8bf92$', - - # too many sections - '$scram$4096$QSXCR.Q6sek8bf92$sha-1=HZbuOlKbWl.eR8AfIposuKbhX30$', - - # missing separator - '$scram$4096$QSXCR.Q6sek8bf92$sha-1=HZbuOlKbWl.eR8AfIposuKbhX30' - 'sha-256=qXUXrlcvnaxxWG00DdRgVioR2gnUpuX5r.3EZ1rdhVY', - - # too many chars in alg name - '$scram$4096$QSXCR.Q6sek8bf92$sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' - 'shaxxx-190=HZbuOlKbWl.eR8AfIposuKbhX30', - - # missing sha-1 alg - '$scram$4096$QSXCR.Q6sek8bf92$sha-256=HZbuOlKbWl.eR8AfIposuKbhX30', - - # non-iana name - '$scram$4096$QSXCR.Q6sek8bf92$sha1=HZbuOlKbWl.eR8AfIposuKbhX30', - ] - - def setUp(self): - super(scram_test, self).setUp() - - # some platforms lack stringprep (e.g. Jython, IronPython) - self.require_stringprep() - - # silence norm_hash_name() warning - warnings.filterwarnings("ignore", r"norm_hash_name\(\): unknown hash") - - def test_90_algs(self): - """test parsing of 'algs' setting""" - defaults = dict(salt=b'A'*10, rounds=1000) - def parse(algs, **kwds): - for k in defaults: - kwds.setdefault(k, defaults[k]) - return self.handler(algs=algs, **kwds).algs - - # None -> default list - self.assertEqual(parse(None, use_defaults=True), hash.scram.default_algs) - self.assertRaises(TypeError, parse, None) - - # strings should be parsed - self.assertEqual(parse("sha1"), ["sha-1"]) - self.assertEqual(parse("sha1, sha256, md5"), ["md5","sha-1","sha-256"]) - - # lists should be normalized - self.assertEqual(parse(["sha-1","sha256"]), ["sha-1","sha-256"]) - - # sha-1 required - self.assertRaises(ValueError, parse, ["sha-256"]) - self.assertRaises(ValueError, parse, algs=[], use_defaults=True) - - # alg names must be < 10 chars - self.assertRaises(ValueError, parse, ["sha-1","shaxxx-190"]) - - # alg & checksum mutually exclusive. - self.assertRaises(RuntimeError, parse, ['sha-1'], - checksum={"sha-1": b"\x00"*20}) - - def test_90_checksums(self): - """test internal parsing of 'checksum' keyword""" - # check non-bytes checksum values are rejected - self.assertRaises(TypeError, self.handler, use_defaults=True, - checksum={'sha-1': u('X')*20}) - - # check sha-1 is required - self.assertRaises(ValueError, self.handler, use_defaults=True, - checksum={'sha-256': b'X'*32}) - - # XXX: anything else that's not tested by the other code already? - - def test_91_extract_digest_info(self): - """test scram.extract_digest_info()""" - edi = self.handler.extract_digest_info - - # return appropriate value or throw KeyError - h = "$scram$10$AAAAAA$sha-1=AQ,bbb=Ag,ccc=Aw" - s = b'\x00'*4 - self.assertEqual(edi(h,"SHA1"), (s,10, b'\x01')) - self.assertEqual(edi(h,"bbb"), (s,10, b'\x02')) - self.assertEqual(edi(h,"ccc"), (s,10, b'\x03')) - self.assertRaises(KeyError, edi, h, "ddd") - - # config strings should cause value error. - c = "$scram$10$....$sha-1,bbb,ccc" - self.assertRaises(ValueError, edi, c, "sha-1") - self.assertRaises(ValueError, edi, c, "bbb") - self.assertRaises(ValueError, edi, c, "ddd") - - def test_92_extract_digest_algs(self): - """test scram.extract_digest_algs()""" - eda = self.handler.extract_digest_algs - - self.assertEqual(eda('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30'), ["sha-1"]) - - self.assertEqual(eda('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30', format="hashlib"), - ["sha1"]) - - self.assertEqual(eda('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' - 'sha-256=qXUXrlcvnaxxWG00DdRgVioR2gnUpuX5r.3EZ1rdhVY,' - 'sha-512=lzgniLFcvglRLS0gt.C4gy.NurS3OIOVRAU1zZOV4P.qFiVFO2/' - 'edGQSu/kD1LwdX0SNV/KsPdHSwEl5qRTuZQ'), - ["sha-1","sha-256","sha-512"]) - - def test_93_derive_digest(self): - """test scram.derive_digest()""" - # NOTE: this just does a light test, since derive_digest - # is used by hash / verify, and is tested pretty well via those. - hash = self.handler.derive_digest - - # check various encodings of password work. - s1 = b'\x01\x02\x03' - d1 = b'\xb2\xfb\xab\x82[tNuPnI\x8aZZ\x19\x87\xcen\xe9\xd3' - self.assertEqual(hash(u("\u2168"), s1, 1000, 'sha-1'), d1) - self.assertEqual(hash(b"\xe2\x85\xa8", s1, 1000, 'SHA-1'), d1) - self.assertEqual(hash(u("IX"), s1, 1000, 'sha1'), d1) - self.assertEqual(hash(b"IX", s1, 1000, 'SHA1'), d1) - - # check algs - self.assertEqual(hash("IX", s1, 1000, 'md5'), - b'3\x19\x18\xc0\x1c/\xa8\xbf\xe4\xa3\xc2\x8eM\xe8od') - self.assertRaises(ValueError, hash, "IX", s1, 1000, 'sha-666') - - # check rounds - self.assertRaises(ValueError, hash, "IX", s1, 0, 'sha-1') - - # unicode salts accepted as of passlib 1.7 (previous caused TypeError) - self.assertEqual(hash(u("IX"), s1.decode("latin-1"), 1000, 'sha1'), d1) - - def test_94_saslprep(self): - """test hash/verify use saslprep""" - # NOTE: this just does a light test that saslprep() is being - # called in various places, relying in saslpreps()'s tests - # to verify full normalization behavior. - - # hash unnormalized - h = self.do_encrypt(u("I\u00ADX")) - self.assertTrue(self.do_verify(u("IX"), h)) - self.assertTrue(self.do_verify(u("\u2168"), h)) - - # hash normalized - h = self.do_encrypt(u("\xF3")) - self.assertTrue(self.do_verify(u("o\u0301"), h)) - self.assertTrue(self.do_verify(u("\u200Do\u0301"), h)) - - # throws error if forbidden char provided - self.assertRaises(ValueError, self.do_encrypt, u("\uFDD0")) - self.assertRaises(ValueError, self.do_verify, u("\uFDD0"), h) - - def test_94_using_w_default_algs(self, param="default_algs"): - """using() -- 'default_algs' parameter""" - # create subclass - handler = self.handler - orig = list(handler.default_algs) # in case it's modified in place - subcls = handler.using(**{param: "sha1,md5"}) - - # shouldn't have changed handler - self.assertEqual(handler.default_algs, orig) - - # should have own set - self.assertEqual(subcls.default_algs, ["md5", "sha-1"]) - - # test hash output - h1 = subcls.hash("dummy") - self.assertEqual(handler.extract_digest_algs(h1), ["md5", "sha-1"]) - - def test_94_using_w_algs(self): - """using() -- 'algs' parameter""" - self.test_94_using_w_default_algs(param="algs") - - def test_94_needs_update_algs(self): - """needs_update() -- algs setting""" - handler1 = self.handler.using(algs="sha1,md5") - - # shouldn't need update, has same algs - h1 = handler1.hash("dummy") - self.assertFalse(handler1.needs_update(h1)) - - # *currently* shouldn't need update, has superset of algs required by handler2 - # (may change this policy) - handler2 = handler1.using(algs="sha1") - self.assertFalse(handler2.needs_update(h1)) - - # should need update, doesn't have all algs required by handler3 - handler3 = handler1.using(algs="sha1,sha256") - self.assertTrue(handler3.needs_update(h1)) - - def test_95_context_algs(self): - """test handling of 'algs' in context object""" - handler = self.handler - from passlib.context import CryptContext - c1 = CryptContext(["scram"], scram__algs="sha1,md5") - - h = c1.hash("dummy") - self.assertEqual(handler.extract_digest_algs(h), ["md5", "sha-1"]) - self.assertFalse(c1.needs_update(h)) - - c2 = c1.copy(scram__algs="sha1") - self.assertFalse(c2.needs_update(h)) - - c2 = c1.copy(scram__algs="sha1,sha256") - self.assertTrue(c2.needs_update(h)) - - def test_96_full_verify(self): - """test verify(full=True) flag""" - def vpart(s, h): - return self.handler.verify(s, h) - def vfull(s, h): - return self.handler.verify(s, h, full=True) - - # reference - h = ('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' - 'sha-256=qXUXrlcvnaxxWG00DdRgVioR2gnUpuX5r.3EZ1rdhVY,' - 'sha-512=lzgniLFcvglRLS0gt.C4gy.NurS3OIOVRAU1zZOV4P.qFiVFO2/' - 'edGQSu/kD1LwdX0SNV/KsPdHSwEl5qRTuZQ') - self.assertTrue(vfull('pencil', h)) - self.assertFalse(vfull('tape', h)) - - # catch truncated digests. - h = ('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' - 'sha-256=qXUXrlcvnaxxWG00DdRgVioR2gnUpuX5r.3EZ1rdhV,' # -1 char - 'sha-512=lzgniLFcvglRLS0gt.C4gy.NurS3OIOVRAU1zZOV4P.qFiVFO2/' - 'edGQSu/kD1LwdX0SNV/KsPdHSwEl5qRTuZQ') - self.assertRaises(ValueError, vfull, 'pencil', h) - - # catch padded digests. - h = ('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' - 'sha-256=qXUXrlcvnaxxWG00DdRgVioR2gnUpuX5r.3EZ1rdhVYa,' # +1 char - 'sha-512=lzgniLFcvglRLS0gt.C4gy.NurS3OIOVRAU1zZOV4P.qFiVFO2/' - 'edGQSu/kD1LwdX0SNV/KsPdHSwEl5qRTuZQ') - self.assertRaises(ValueError, vfull, 'pencil', h) - - # catch hash containing digests belonging to diff passwords. - # proper behavior for quick-verify (the default) is undefined, - # but full-verify should throw error. - h = ('$scram$4096$QSXCR.Q6sek8bf92$' - 'sha-1=HZbuOlKbWl.eR8AfIposuKbhX30,' # 'pencil' - 'sha-256=R7RJDWIbeKRTFwhE9oxh04kab0CllrQ3kCcpZUcligc,' # 'tape' - 'sha-512=lzgniLFcvglRLS0gt.C4gy.NurS3OIOVRAU1zZOV4P.qFiVFO2/' # 'pencil' - 'edGQSu/kD1LwdX0SNV/KsPdHSwEl5qRTuZQ') - self.assertTrue(vpart('tape', h)) - self.assertFalse(vpart('pencil', h)) - self.assertRaises(ValueError, vfull, 'pencil', h) - self.assertRaises(ValueError, vfull, 'tape', h) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_handlers_scrypt.py b/src/passlib/tests/test_handlers_scrypt.py deleted file mode 100644 index bbd3cd70..00000000 --- a/src/passlib/tests/test_handlers_scrypt.py +++ /dev/null @@ -1,110 +0,0 @@ -"""passlib.tests.test_handlers - tests for passlib hash algorithms""" -#============================================================================= -# imports -#============================================================================= -# core -import logging; log = logging.getLogger(__name__) -import warnings -warnings.filterwarnings("ignore", ".*using builtin scrypt backend.*") -# site -# pkg -from passlib import hash -from passlib.tests.utils import HandlerCase, TEST_MODE -from passlib.tests.test_handlers import UPASS_TABLE, PASS_TABLE_UTF8 -# module - -#============================================================================= -# scrypt hash -#============================================================================= -class _scrypt_test(HandlerCase): - handler = hash.scrypt - - known_correct_hashes = [ - # - # excepted from test vectors from scrypt whitepaper - # (http://www.tarsnap.com/scrypt/scrypt.pdf, appendix b), - # and encoded using passlib's custom format - # - - # salt=b"" - ("", "$scrypt$ln=4,r=1,p=1$$d9ZXYjhleyA7GcpCwYoEl/FrSETjB0ro39/6P+3iFEI"), - - # salt=b"NaCl" - ("password", "$scrypt$ln=10,r=8,p=16$TmFDbA$/bq+HJ00cgB4VucZDQHp/nxq18vII3gw53N2Y0s3MWI"), - - # - # custom - # - - # simple test - ("test", '$scrypt$ln=8,r=8,p=1$wlhLyXmP8b53bm1NKYVQqg$mTpvG8lzuuDk+DWz8HZIB6Vum6erDuUm0As5yU+VxWA'), - - # different block value - ("password", '$scrypt$ln=8,r=2,p=1$dO6d0xoDoLT2PofQGoNQag$g/Wf2A0vhHhaJM+addK61QPBthSmYB6uVTtQzh8CM3o'), - - # different rounds - (UPASS_TABLE, '$scrypt$ln=7,r=8,p=1$jjGmtDamdA4BQAjBeA9BSA$OiWRHhQtpDx7M/793x6UXK14AD512jg/qNm/hkWZG4M'), - - # alt encoding - (PASS_TABLE_UTF8, '$scrypt$ln=7,r=8,p=1$jjGmtDamdA4BQAjBeA9BSA$OiWRHhQtpDx7M/793x6UXK14AD512jg/qNm/hkWZG4M'), - - # diff block & parallel counts as well - ("nacl", '$scrypt$ln=1,r=4,p=2$yhnD+J+Tci4lZCwFgHCuVQ$fAsEWmxSHuC0cHKMwKVFPzrQukgvK09Sj+NueTSxKds') - ] - - if TEST_MODE("full"): - # add some hashes with larger rounds value. - known_correct_hashes.extend([ - # - # from scrypt whitepaper - # - - # salt=b"SodiumChloride" - ("pleaseletmein", "$scrypt$ln=14,r=8,p=1$U29kaXVtQ2hsb3JpZGU" - "$cCO9yzr9c0hGHAbNgf046/2o+7qQT44+qbVD9lRdofI"), - - # - # openwall format (https://gitlab.com/jas/scrypt-unix-crypt/blob/master/unix-scrypt.txt) - # - ("pleaseletmein", - "$7$C6..../....SodiumChloride$kBGj9fHznVYFQMEn/qDCfrDevf9YDtcDdKvEqHJLV8D"), - - ]) - - known_malformed_hashes = [ - # missing 'p' value - '$scrypt$ln=10,r=1$wvif8/4fg1Cq9V7L2dv73w$bJcLia1lyfQ1X2x0xflehwVXPzWIUQWWdnlGwfVzBeQ', - - # rounds too low - '$scrypt$ln=0,r=1,p=1$wvif8/4fg1Cq9V7L2dv73w$bJcLia1lyfQ1X2x0xflehwVXPzWIUQWWdnlGwfVzBeQ', - - # invalid block size - '$scrypt$ln=10,r=A,p=1$wvif8/4fg1Cq9V7L2dv73w$bJcLia1lyfQ1X2x0xflehwVXPzWIUQWWdnlGwfVzBeQ', - - # r*p too large - '$scrypt$ln=10,r=134217728,p=8$wvif8/4fg1Cq9V7L2dv73w$bJcLia1lyfQ1X2x0xflehwVXPzWIUQWWdnlGwfVzBeQ', - ] - - def setUpWarnings(self): - super(_scrypt_test, self).setUpWarnings() - warnings.filterwarnings("ignore", ".*using builtin scrypt backend.*") - - def populate_settings(self, kwds): - # builtin is still just way too slow. - if self.backend == "builtin": - kwds.setdefault("rounds", 6) - super(_scrypt_test, self).populate_settings(kwds) - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - def random_rounds(self): - # decrease default rounds for fuzz testing to speed up volume. - return self.randintgauss(4, 10, 6, 1) - -# create test cases for specific backends -scrypt_scrypt_test = _scrypt_test.create_backend_case("scrypt") -scrypt_builtin_test = _scrypt_test.create_backend_case("builtin") - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_hosts.py b/src/passlib/tests/test_hosts.py deleted file mode 100644 index cbf93ab7..00000000 --- a/src/passlib/tests/test_hosts.py +++ /dev/null @@ -1,97 +0,0 @@ -"""test passlib.hosts""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib import hosts, hash as hashmod -from passlib.utils import unix_crypt_schemes -from passlib.tests.utils import TestCase -# module - -#============================================================================= -# test predefined app contexts -#============================================================================= -class HostsTest(TestCase): - """perform general tests to make sure contexts work""" - # NOTE: these tests are not really comprehensive, - # since they would do little but duplicate - # the presets in apps.py - # - # they mainly try to ensure no typos - # or dynamic behavior foul-ups. - - def check_unix_disabled(self, ctx): - for hash in [ - "", - "!", - "*", - "!$1$TXl/FX/U$BZge.lr.ux6ekjEjxmzwz0", - ]: - self.assertEqual(ctx.identify(hash), 'unix_disabled') - self.assertFalse(ctx.verify('test', hash)) - - def test_linux_context(self): - ctx = hosts.linux_context - for hash in [ - ('$6$rounds=41128$VoQLvDjkaZ6L6BIE$4pt.1Ll1XdDYduEwEYPCMOBiR6W6' - 'znsyUEoNlcVXpv2gKKIbQolgmTGe6uEEVJ7azUxuc8Tf7zV9SD2z7Ij751'), - ('$5$rounds=31817$iZGmlyBQ99JSB5n6$p4E.pdPBWx19OajgjLRiOW0itGny' - 'xDGgMlDcOsfaI17'), - '$1$TXl/FX/U$BZge.lr.ux6ekjEjxmzwz0', - 'kAJJz.Rwp0A/I', - ]: - self.assertTrue(ctx.verify("test", hash)) - self.check_unix_disabled(ctx) - - def test_bsd_contexts(self): - for ctx in [ - hosts.freebsd_context, - hosts.openbsd_context, - hosts.netbsd_context, - ]: - for hash in [ - '$1$TXl/FX/U$BZge.lr.ux6ekjEjxmzwz0', - 'kAJJz.Rwp0A/I', - ]: - self.assertTrue(ctx.verify("test", hash)) - h1 = "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS" - if hashmod.bcrypt.has_backend(): - self.assertTrue(ctx.verify("test", h1)) - else: - self.assertEqual(ctx.identify(h1), "bcrypt") - self.check_unix_disabled(ctx) - - def test_host_context(self): - ctx = getattr(hosts, "host_context", None) - if not ctx: - return self.skipTest("host_context not available on this platform") - - # validate schemes is non-empty, - # and contains unix_disabled + at least one real scheme - schemes = list(ctx.schemes()) - self.assertTrue(schemes, "appears to be unix system, but no known schemes supported by crypt") - self.assertTrue('unix_disabled' in schemes) - schemes.remove("unix_disabled") - self.assertTrue(schemes, "should have schemes beside fallback scheme") - self.assertTrue(set(unix_crypt_schemes).issuperset(schemes)) - - # check for hash support - self.check_unix_disabled(ctx) - for scheme, hash in [ - ("sha512_crypt", ('$6$rounds=41128$VoQLvDjkaZ6L6BIE$4pt.1Ll1XdDYduEwEYPCMOBiR6W6' - 'znsyUEoNlcVXpv2gKKIbQolgmTGe6uEEVJ7azUxuc8Tf7zV9SD2z7Ij751')), - ("sha256_crypt", ('$5$rounds=31817$iZGmlyBQ99JSB5n6$p4E.pdPBWx19OajgjLRiOW0itGny' - 'xDGgMlDcOsfaI17')), - ("md5_crypt", '$1$TXl/FX/U$BZge.lr.ux6ekjEjxmzwz0'), - ("des_crypt", 'kAJJz.Rwp0A/I'), - ]: - if scheme in schemes: - self.assertTrue(ctx.verify("test", hash)) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_pwd.py b/src/passlib/tests/test_pwd.py deleted file mode 100644 index 2c983cdf..00000000 --- a/src/passlib/tests/test_pwd.py +++ /dev/null @@ -1,205 +0,0 @@ -"""passlib.tests -- tests for passlib.pwd""" -#============================================================================= -# imports -#============================================================================= -# core -import itertools -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.tests.utils import TestCase -# local -__all__ = [ - "UtilsTest", - "GenerateTest", - "StrengthTest", -] - -#============================================================================= -# -#============================================================================= -class UtilsTest(TestCase): - """test internal utilities""" - descriptionPrefix = "passlib.pwd" - - def test_self_info_rate(self): - """_self_info_rate()""" - from passlib.pwd import _self_info_rate - - self.assertEqual(_self_info_rate(""), 0) - - self.assertEqual(_self_info_rate("a" * 8), 0) - - self.assertEqual(_self_info_rate("ab"), 1) - self.assertEqual(_self_info_rate("ab" * 8), 1) - - self.assertEqual(_self_info_rate("abcd"), 2) - self.assertEqual(_self_info_rate("abcd" * 8), 2) - self.assertAlmostEqual(_self_info_rate("abcdaaaa"), 1.5488, places=4) - - # def test_total_self_info(self): - # """_total_self_info()""" - # from passlib.pwd import _total_self_info - # - # self.assertEqual(_total_self_info(""), 0) - # - # self.assertEqual(_total_self_info("a" * 8), 0) - # - # self.assertEqual(_total_self_info("ab"), 2) - # self.assertEqual(_total_self_info("ab" * 8), 16) - # - # self.assertEqual(_total_self_info("abcd"), 8) - # self.assertEqual(_total_self_info("abcd" * 8), 64) - # self.assertAlmostEqual(_total_self_info("abcdaaaa"), 12.3904, places=4) - -#============================================================================= -# word generation -#============================================================================= - -# import subject -from passlib.pwd import genword, default_charsets -ascii_62 = default_charsets['ascii_62'] -hex = default_charsets['hex'] - -class WordGeneratorTest(TestCase): - """test generation routines""" - descriptionPrefix = "passlib.pwd.genword()" - - def setUp(self): - super(WordGeneratorTest, self).setUp() - - # patch some RNG references so they're reproducible. - from passlib.pwd import SequenceGenerator - self.patchAttr(SequenceGenerator, "rng", - self.getRandom("pwd generator")) - - def assertResultContents(self, results, count, chars, unique=True): - """check result list matches expected count & charset""" - self.assertEqual(len(results), count) - if unique: - if unique is True: - unique = count - self.assertEqual(len(set(results)), unique) - self.assertEqual(set("".join(results)), set(chars)) - - def test_general(self): - """general behavior""" - - # basic usage - result = genword() - self.assertEqual(len(result), 9) - - # malformed keyword should have useful error. - self.assertRaisesRegex(TypeError, "(?i)unexpected keyword.*badkwd", genword, badkwd=True) - - def test_returns(self): - """'returns' keyword""" - # returns=int option - results = genword(returns=5000) - self.assertResultContents(results, 5000, ascii_62) - - # returns=iter option - gen = genword(returns=iter) - results = [next(gen) for _ in range(5000)] - self.assertResultContents(results, 5000, ascii_62) - - # invalid returns option - self.assertRaises(TypeError, genword, returns='invalid-type') - - def test_charset(self): - """'charset' & 'chars' options""" - # charset option - results = genword(charset="hex", returns=5000) - self.assertResultContents(results, 5000, hex) - - # chars option - # there are 3**3=27 possible combinations - results = genword(length=3, chars="abc", returns=5000) - self.assertResultContents(results, 5000, "abc", unique=27) - - # chars + charset - self.assertRaises(TypeError, genword, chars='abc', charset='hex') - - # TODO: test rng option - -#============================================================================= -# phrase generation -#============================================================================= - -# import subject -from passlib.pwd import genphrase -simple_words = ["alpha", "beta", "gamma"] - -class PhraseGeneratorTest(TestCase): - """test generation routines""" - descriptionPrefix = "passlib.pwd.genphrase()" - - def assertResultContents(self, results, count, words, unique=True, sep=" "): - """check result list matches expected count & charset""" - self.assertEqual(len(results), count) - if unique: - if unique is True: - unique = count - self.assertEqual(len(set(results)), unique) - out = set(itertools.chain.from_iterable(elem.split(sep) for elem in results)) - self.assertEqual(out, set(words)) - - def test_general(self): - """general behavior""" - - # basic usage - result = genphrase() - self.assertEqual(len(result.split(" ")), 4) # 48 / log(7776, 2) ~= 3.7 -> 4 - - # malformed keyword should have useful error. - self.assertRaisesRegex(TypeError, "(?i)unexpected keyword.*badkwd", genphrase, badkwd=True) - - def test_entropy(self): - """'length' & 'entropy' keywords""" - - # custom entropy - result = genphrase(entropy=70) - self.assertEqual(len(result.split(" ")), 6) # 70 / log(7776, 2) ~= 5.4 -> 6 - - # custom length - result = genphrase(length=3) - self.assertEqual(len(result.split(" ")), 3) - - # custom length < entropy - result = genphrase(length=3, entropy=48) - self.assertEqual(len(result.split(" ")), 4) - - # custom length > entropy - result = genphrase(length=4, entropy=12) - self.assertEqual(len(result.split(" ")), 4) - - def test_returns(self): - """'returns' keyword""" - # returns=int option - results = genphrase(returns=1000, words=simple_words) - self.assertResultContents(results, 1000, simple_words) - - # returns=iter option - gen = genphrase(returns=iter, words=simple_words) - results = [next(gen) for _ in range(1000)] - self.assertResultContents(results, 1000, simple_words) - - # invalid returns option - self.assertRaises(TypeError, genphrase, returns='invalid-type') - - def test_wordset(self): - """'wordset' & 'words' options""" - # wordset option - results = genphrase(words=simple_words, returns=5000) - self.assertResultContents(results, 5000, simple_words) - - # words option - results = genphrase(length=3, words=simple_words, returns=5000) - self.assertResultContents(results, 5000, simple_words, unique=3**3) - - # words + wordset - self.assertRaises(TypeError, genphrase, words=simple_words, wordset='bip39') - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_registry.py b/src/passlib/tests/test_registry.py deleted file mode 100644 index 7540ee21..00000000 --- a/src/passlib/tests/test_registry.py +++ /dev/null @@ -1,229 +0,0 @@ -"""tests for passlib.hash -- (c) Assurance Technologies 2003-2009""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -from logging import getLogger -import warnings -import sys -# site -# pkg -from passlib import hash, registry, exc -from passlib.registry import register_crypt_handler, register_crypt_handler_path, \ - get_crypt_handler, list_crypt_handlers, _unload_handler_name as unload_handler_name -import passlib.utils.handlers as uh -from passlib.tests.utils import TestCase -# module -log = getLogger(__name__) - -#============================================================================= -# dummy handlers -# -# NOTE: these are defined outside of test case -# since they're used by test_register_crypt_handler_path(), -# which needs them to be available as module globals. -#============================================================================= -class dummy_0(uh.StaticHandler): - name = "dummy_0" - -class alt_dummy_0(uh.StaticHandler): - name = "dummy_0" - -dummy_x = 1 - -#============================================================================= -# test registry -#============================================================================= -class RegistryTest(TestCase): - - descriptionPrefix = "passlib.registry" - - def setUp(self): - super(RegistryTest, self).setUp() - - # backup registry state & restore it after test. - locations = dict(registry._locations) - handlers = dict(registry._handlers) - def restore(): - registry._locations.clear() - registry._locations.update(locations) - registry._handlers.clear() - registry._handlers.update(handlers) - self.addCleanup(restore) - - def test_hash_proxy(self): - """test passlib.hash proxy object""" - # check dir works - dir(hash) - - # check repr works - repr(hash) - - # check non-existent attrs raise error - self.assertRaises(AttributeError, getattr, hash, 'fooey') - - # GAE tries to set __loader__, - # make sure that doesn't call register_crypt_handler. - old = getattr(hash, "__loader__", None) - test = object() - hash.__loader__ = test - self.assertIs(hash.__loader__, test) - if old is None: - del hash.__loader__ - self.assertFalse(hasattr(hash, "__loader__")) - else: - hash.__loader__ = old - self.assertIs(hash.__loader__, old) - - # check storing attr calls register_crypt_handler - class dummy_1(uh.StaticHandler): - name = "dummy_1" - hash.dummy_1 = dummy_1 - self.assertIs(get_crypt_handler("dummy_1"), dummy_1) - - # check storing under wrong name results in error - self.assertRaises(ValueError, setattr, hash, "dummy_1x", dummy_1) - - def test_register_crypt_handler_path(self): - """test register_crypt_handler_path()""" - # NOTE: this messes w/ internals of registry, shouldn't be used publically. - paths = registry._locations - - # check namespace is clear - self.assertTrue('dummy_0' not in paths) - self.assertFalse(hasattr(hash, 'dummy_0')) - - # check invalid names are rejected - self.assertRaises(ValueError, register_crypt_handler_path, - "dummy_0", ".test_registry") - self.assertRaises(ValueError, register_crypt_handler_path, - "dummy_0", __name__ + ":dummy_0:xxx") - self.assertRaises(ValueError, register_crypt_handler_path, - "dummy_0", __name__ + ":dummy_0.xxx") - - # try lazy load - register_crypt_handler_path('dummy_0', __name__) - self.assertTrue('dummy_0' in list_crypt_handlers()) - self.assertTrue('dummy_0' not in list_crypt_handlers(loaded_only=True)) - self.assertIs(hash.dummy_0, dummy_0) - self.assertTrue('dummy_0' in list_crypt_handlers(loaded_only=True)) - unload_handler_name('dummy_0') - - # try lazy load w/ alt - register_crypt_handler_path('dummy_0', __name__ + ':alt_dummy_0') - self.assertIs(hash.dummy_0, alt_dummy_0) - unload_handler_name('dummy_0') - - # check lazy load w/ wrong type fails - register_crypt_handler_path('dummy_x', __name__) - self.assertRaises(TypeError, get_crypt_handler, 'dummy_x') - - # check lazy load w/ wrong name fails - register_crypt_handler_path('alt_dummy_0', __name__) - self.assertRaises(ValueError, get_crypt_handler, "alt_dummy_0") - unload_handler_name("alt_dummy_0") - - # TODO: check lazy load which calls register_crypt_handler (warning should be issued) - sys.modules.pop("passlib.tests._test_bad_register", None) - register_crypt_handler_path("dummy_bad", "passlib.tests._test_bad_register") - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "xxxxxxxxxx", DeprecationWarning) - h = get_crypt_handler("dummy_bad") - from passlib.tests import _test_bad_register as tbr - self.assertIs(h, tbr.alt_dummy_bad) - - def test_register_crypt_handler(self): - """test register_crypt_handler()""" - - self.assertRaises(TypeError, register_crypt_handler, {}) - - self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name=None))) - self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="AB_CD"))) - self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="ab-cd"))) - self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="ab__cd"))) - self.assertRaises(ValueError, register_crypt_handler, type('x', (uh.StaticHandler,), dict(name="default"))) - - class dummy_1(uh.StaticHandler): - name = "dummy_1" - - class dummy_1b(uh.StaticHandler): - name = "dummy_1" - - self.assertTrue('dummy_1' not in list_crypt_handlers()) - - register_crypt_handler(dummy_1) - register_crypt_handler(dummy_1) - self.assertIs(get_crypt_handler("dummy_1"), dummy_1) - - self.assertRaises(KeyError, register_crypt_handler, dummy_1b) - self.assertIs(get_crypt_handler("dummy_1"), dummy_1) - - register_crypt_handler(dummy_1b, force=True) - self.assertIs(get_crypt_handler("dummy_1"), dummy_1b) - - self.assertTrue('dummy_1' in list_crypt_handlers()) - - def test_get_crypt_handler(self): - """test get_crypt_handler()""" - - class dummy_1(uh.StaticHandler): - name = "dummy_1" - - # without available handler - self.assertRaises(KeyError, get_crypt_handler, "dummy_1") - self.assertIs(get_crypt_handler("dummy_1", None), None) - - # already loaded handler - register_crypt_handler(dummy_1) - self.assertIs(get_crypt_handler("dummy_1"), dummy_1) - - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "handler names should be lower-case, and use underscores instead of hyphens:.*", UserWarning) - - # already loaded handler, using incorrect name - self.assertIs(get_crypt_handler("DUMMY-1"), dummy_1) - - # lazy load of unloaded handler, using incorrect name - register_crypt_handler_path('dummy_0', __name__) - self.assertIs(get_crypt_handler("DUMMY-0"), dummy_0) - - # check system & private names aren't returned - import passlib.hash # ensure module imported, so py3.3 sets __package__ - passlib.hash.__dict__["_fake"] = "dummy" # so behavior seen under py2x also - for name in ["_fake", "__package__"]: - self.assertRaises(KeyError, get_crypt_handler, name) - self.assertIs(get_crypt_handler(name, None), None) - - def test_list_crypt_handlers(self): - """test list_crypt_handlers()""" - from passlib.registry import list_crypt_handlers - - # check system & private names aren't returned - import passlib.hash # ensure module imported, so py3.3 sets __package__ - passlib.hash.__dict__["_fake"] = "dummy" # so behavior seen under py2x also - for name in list_crypt_handlers(): - self.assertFalse(name.startswith("_"), "%r: " % name) - unload_handler_name("_fake") - - def test_handlers(self): - """verify we have tests for all builtin handlers""" - from passlib.registry import list_crypt_handlers - from passlib.tests.test_handlers import get_handler_case, conditionally_available_hashes - for name in list_crypt_handlers(): - # skip some wrappers that don't need independant testing - if name.startswith("ldap_") and name[5:] in list_crypt_handlers(): - continue - if name in ["roundup_plaintext"]: - continue - # check the remaining ones all have a handler - try: - self.assertTrue(get_handler_case(name)) - except exc.MissingBackendError: - if name in conditionally_available_hashes: # expected to fail on some setups - continue - raise - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_totp.py b/src/passlib/tests/test_totp.py deleted file mode 100644 index 1229da72..00000000 --- a/src/passlib/tests/test_totp.py +++ /dev/null @@ -1,1602 +0,0 @@ -"""passlib.tests -- test passlib.totp""" -#============================================================================= -# imports -#============================================================================= -# core -import datetime -from functools import partial -import logging; log = logging.getLogger(__name__) -import sys -import time as _time -# site -# pkg -from passlib import exc -from passlib.utils.compat import unicode, u -from passlib.tests.utils import TestCase, time_call -# subject -from passlib import totp as totp_module -from passlib.totp import TOTP, AppWallet, AES_SUPPORT -# local -__all__ = [ - "EngineTest", -] - -#============================================================================= -# helpers -#============================================================================= - -# XXX: python 3 changed what error base64.b16decode() throws, from TypeError to base64.Error(). -# it wasn't until 3.3 that base32decode() also got changed. -# really should normalize this in the code to a single BinaryDecodeError, -# predicting this cross-version is getting unmanagable. -Base32DecodeError = Base16DecodeError = TypeError -if sys.version_info >= (3,0): - from binascii import Error as Base16DecodeError -if sys.version_info >= (3,3): - from binascii import Error as Base32DecodeError - -PASS1 = "abcdef" -PASS2 = b"\x00\xFF" -KEY1 = '4AOGGDBBQSYHNTUZ' -KEY1_RAW = b'\xe0\x1cc\x0c!\x84\xb0v\xce\x99' -KEY2_RAW = b'\xee]\xcb9\x870\x06 D\xc8y/\xa54&\xe4\x9c\x13\xc2\x18' -KEY3 = 'S3JDVB7QD2R7JPXX' # used in docstrings -KEY4 = 'JBSWY3DPEHPK3PXP' # from google keyuri spec -KEY4_RAW = b'Hello!\xde\xad\xbe\xef' - -# NOTE: for randtime() below, -# * want at least 7 bits on fractional side, to test fractional times to at least 0.01s precision -# * want at least 32 bits on integer side, to test for 32-bit epoch issues. -# most systems *should* have 53 bit mantissa, leaving plenty of room on both ends, -# so using (1<<37) as scale, to allocate 16 bits on fractional side, but generate reasonable # of > 1<<32 times. -# sanity check that we're above 44 ensures minimum requirements (44 - 37 int = 7 frac) -assert sys.float_info.radix == 2, "unexpected float_info.radix" -assert sys.float_info.mant_dig >= 44, "double precision unexpectedly small" - -def _get_max_time_t(): - """ - helper to calc max_time_t constant (see below) - """ - value = 1 << 30 # even for 32 bit systems will handle this - year = 0 - while True: - next_value = value << 1 - try: - next_year = datetime.datetime.utcfromtimestamp(next_value-1).year - except (ValueError, OSError, OverflowError): - # utcfromtimestamp() may throw any of the following: - # - # * year out of range for datetime: - # py < 3.6 throws ValueError. - # (py 3.6.0 returns odd value instead, see workaround below) - # - # * int out of range for host's gmtime/localtime: - # py2 throws ValueError, py3 throws OSError. - # - # * int out of range for host's time_t: - # py2 throws ValueError, py3 throws OverflowError. - # - break - - # Workaround for python 3.6.0 issue -- - # Instead of throwing ValueError if year out of range for datetime, - # Python 3.6 will do some weird behavior that masks high bits - # e.g. (1<<40) -> year 36812, but (1<<41) -> year 6118. - # (Appears to be bug http://bugs.python.org/issue29100) - # This check stops at largest non-wrapping bit size. - if next_year < year: - break - - value = next_value - - # 'value-1' is maximum. - value -= 1 - - # check for crazy case where we're beyond what datetime supports - # (caused by bug 29100 again). compare to max value that datetime - # module supports -- datetime.datetime(9999, 12, 31, 23, 59, 59, 999999) - max_datetime_timestamp = 253402318800 - return min(value, max_datetime_timestamp) - -#: Rough approximation of max value acceptable by hosts's time_t. -#: This is frequently ~2**37 on 64 bit, and ~2**31 on 32 bit systems. -max_time_t = _get_max_time_t() - -def to_b32_size(raw_size): - return (raw_size * 8 + 4) // 5 - -#============================================================================= -# wallet -#============================================================================= -class AppWalletTest(TestCase): - descriptionPrefix = "passlib.totp.AppWallet" - - #============================================================================= - # constructor - #============================================================================= - - def test_secrets_types(self): - """constructor -- 'secrets' param -- input types""" - - # no secrets - wallet = AppWallet() - self.assertEqual(wallet._secrets, {}) - self.assertFalse(wallet.has_secrets) - - # dict - ref = {"1": b"aaa", "2": b"bbb"} - wallet = AppWallet(ref) - self.assertEqual(wallet._secrets, ref) - self.assertTrue(wallet.has_secrets) - - # # list - # wallet = AppWallet(list(ref.items())) - # self.assertEqual(wallet._secrets, ref) - - # # iter - # wallet = AppWallet(iter(ref.items())) - # self.assertEqual(wallet._secrets, ref) - - # "tag:value" string - wallet = AppWallet("\n 1: aaa\n# comment\n \n2: bbb ") - self.assertEqual(wallet._secrets, ref) - - # ensure ":" allowed in secret - wallet = AppWallet("1: aaa: bbb \n# comment\n \n2: bbb ") - self.assertEqual(wallet._secrets, {"1": b"aaa: bbb", "2": b"bbb"}) - - # json dict - wallet = AppWallet('{"1":"aaa","2":"bbb"}') - self.assertEqual(wallet._secrets, ref) - - # # json list - # wallet = AppWallet('[["1","aaa"],["2","bbb"]]') - # self.assertEqual(wallet._secrets, ref) - - # invalid type - self.assertRaises(TypeError, AppWallet, 123) - - # invalid json obj - self.assertRaises(TypeError, AppWallet, "[123]") - - # # invalid list items - # self.assertRaises(ValueError, AppWallet, ["1", b"aaa"]) - - # forbid empty secret - self.assertRaises(ValueError, AppWallet, {"1": "aaa", "2": ""}) - - def test_secrets_tags(self): - """constructor -- 'secrets' param -- tag/value normalization""" - - # test reference - ref = {"1": b"aaa", "02": b"bbb", "C": b"ccc"} - wallet = AppWallet(ref) - self.assertEqual(wallet._secrets, ref) - - # accept unicode - wallet = AppWallet({u("1"): b"aaa", u("02"): b"bbb", u("C"): b"ccc"}) - self.assertEqual(wallet._secrets, ref) - - # normalize int tags - wallet = AppWallet({1: b"aaa", "02": b"bbb", "C": b"ccc"}) - self.assertEqual(wallet._secrets, ref) - - # forbid non-str/int tags - self.assertRaises(TypeError, AppWallet, {(1,): "aaa"}) - - # accept valid tags - wallet = AppWallet({"1-2_3.4": b"aaa"}) - - # forbid invalid tags - self.assertRaises(ValueError, AppWallet, {"-abc": "aaa"}) - self.assertRaises(ValueError, AppWallet, {"ab*$": "aaa"}) - - # coerce value to bytes - wallet = AppWallet({"1": u("aaa"), "02": "bbb", "C": b"ccc"}) - self.assertEqual(wallet._secrets, ref) - - # forbid invalid value types - self.assertRaises(TypeError, AppWallet, {"1": 123}) - self.assertRaises(TypeError, AppWallet, {"1": None}) - self.assertRaises(TypeError, AppWallet, {"1": []}) - - # TODO: test secrets_path - - def test_default_tag(self): - """constructor -- 'default_tag' param""" - - # should sort numerically - wallet = AppWallet({"1": "one", "02": "two"}) - self.assertEqual(wallet.default_tag, "02") - self.assertEqual(wallet.get_secret(wallet.default_tag), b"two") - - # should sort alphabetically if non-digit present - wallet = AppWallet({"1": "one", "02": "two", "A": "aaa"}) - self.assertEqual(wallet.default_tag, "A") - self.assertEqual(wallet.get_secret(wallet.default_tag), b"aaa") - - # should use honor custom tag - wallet = AppWallet({"1": "one", "02": "two", "A": "aaa"}, default_tag="1") - self.assertEqual(wallet.default_tag, "1") - self.assertEqual(wallet.get_secret(wallet.default_tag), b"one") - - # throw error on unknown value - self.assertRaises(KeyError, AppWallet, {"1": "one", "02": "two", "A": "aaa"}, - default_tag="B") - - # should be empty - wallet = AppWallet() - self.assertEqual(wallet.default_tag, None) - self.assertRaises(KeyError, wallet.get_secret, None) - - # TODO: test 'cost' param - - #============================================================================= - # encrypt_key() & decrypt_key() helpers - #============================================================================= - def require_aes_support(self, canary=None): - if AES_SUPPORT: - canary and canary() - else: - canary and self.assertRaises(RuntimeError, canary) - raise self.skipTest("'cryptography' package not installed") - - def test_decrypt_key(self): - """.decrypt_key()""" - - wallet = AppWallet({"1": PASS1, "2": PASS2}) - - # check for support - CIPHER1 = dict(v=1, c=13, s='6D7N7W53O7HHS37NLUFQ', - k='MHCTEGSNPFN5CGBJ', t='1') - self.require_aes_support(canary=partial(wallet.decrypt_key, CIPHER1)) - - # reference key - self.assertEqual(wallet.decrypt_key(CIPHER1)[0], KEY1_RAW) - - # different salt used to encrypt same raw key - CIPHER2 = dict(v=1, c=13, s='SPZJ54Y6IPUD2BYA4C6A', - k='ZGDXXTVQOWYLC2AU', t='1') - self.assertEqual(wallet.decrypt_key(CIPHER2)[0], KEY1_RAW) - - # different sized key, password, and cost - CIPHER3 = dict(v=1, c=8, s='FCCTARTIJWE7CPQHUDKA', - k='D2DRS32YESGHHINWFFCELKN7Z6NAHM4M', t='2') - self.assertEqual(wallet.decrypt_key(CIPHER3)[0], KEY2_RAW) - - # wrong password should silently result in wrong key - temp = CIPHER1.copy() - temp.update(t='2') - self.assertEqual(wallet.decrypt_key(temp)[0], b'\xafD6.F7\xeb\x19\x05Q') - - # missing tag should throw error - temp = CIPHER1.copy() - temp.update(t='3') - self.assertRaises(KeyError, wallet.decrypt_key, temp) - - # unknown version should throw error - temp = CIPHER1.copy() - temp.update(v=999) - self.assertRaises(ValueError, wallet.decrypt_key, temp) - - def test_decrypt_key_needs_recrypt(self): - """.decrypt_key() -- needs_recrypt flag""" - self.require_aes_support() - - wallet = AppWallet({"1": PASS1, "2": PASS2}, encrypt_cost=13) - - # ref should be accepted - ref = dict(v=1, c=13, s='AAAA', k='AAAA', t='2') - self.assertFalse(wallet.decrypt_key(ref)[1]) - - # wrong cost - temp = ref.copy() - temp.update(c=8) - self.assertTrue(wallet.decrypt_key(temp)[1]) - - # wrong tag - temp = ref.copy() - temp.update(t="1") - self.assertTrue(wallet.decrypt_key(temp)[1]) - - # XXX: should this check salt_size? - - def assertSaneResult(self, result, wallet, key, tag="1", - needs_recrypt=False): - """check encrypt_key() result has expected format""" - - self.assertEqual(set(result), set(["v", "t", "c", "s", "k"])) - - self.assertEqual(result['v'], 1) - self.assertEqual(result['t'], tag) - self.assertEqual(result['c'], wallet.encrypt_cost) - - self.assertEqual(len(result['s']), to_b32_size(wallet.salt_size)) - self.assertEqual(len(result['k']), to_b32_size(len(key))) - - result_key, result_needs_recrypt = wallet.decrypt_key(result) - self.assertEqual(result_key, key) - self.assertEqual(result_needs_recrypt, needs_recrypt) - - def test_encrypt_key(self): - """.encrypt_key()""" - - # check for support - wallet = AppWallet({"1": PASS1}, encrypt_cost=5) - self.require_aes_support(canary=partial(wallet.encrypt_key, KEY1_RAW)) - - # basic behavior - result = wallet.encrypt_key(KEY1_RAW) - self.assertSaneResult(result, wallet, KEY1_RAW) - - # creates new salt each time - other = wallet.encrypt_key(KEY1_RAW) - self.assertSaneResult(result, wallet, KEY1_RAW) - self.assertNotEqual(other['s'], result['s']) - self.assertNotEqual(other['k'], result['k']) - - # honors custom cost - wallet2 = AppWallet({"1": PASS1}, encrypt_cost=6) - result = wallet2.encrypt_key(KEY1_RAW) - self.assertSaneResult(result, wallet2, KEY1_RAW) - - # honors default tag - wallet2 = AppWallet({"1": PASS1, "2": PASS2}) - result = wallet2.encrypt_key(KEY1_RAW) - self.assertSaneResult(result, wallet2, KEY1_RAW, tag="2") - - # honor salt size - wallet2 = AppWallet({"1": PASS1}) - wallet2.salt_size = 64 - result = wallet2.encrypt_key(KEY1_RAW) - self.assertSaneResult(result, wallet2, KEY1_RAW) - - # larger key - result = wallet.encrypt_key(KEY2_RAW) - self.assertSaneResult(result, wallet, KEY2_RAW) - - # border case: empty key - # XXX: might want to allow this, but documenting behavior for now - self.assertRaises(ValueError, wallet.encrypt_key, b"") - - def test_encrypt_cost_timing(self): - """verify cost parameter via timing""" - self.require_aes_support() - - # time default cost - wallet = AppWallet({"1": "aaa"}) - wallet.encrypt_cost -= 2 - delta, _ = time_call(partial(wallet.encrypt_key, KEY1_RAW), maxtime=0) - - # this should take (2**3=8) times as long - wallet.encrypt_cost += 3 - delta2, _ = time_call(partial(wallet.encrypt_key, KEY1_RAW), maxtime=0) - - self.assertAlmostEqual(delta2, delta*8, delta=(delta*8)*0.5) - - #============================================================================= - # eoc - #============================================================================= - -#============================================================================= -# common OTP code -#============================================================================= - -#: used as base value for RFC test vector keys -RFC_KEY_BYTES_20 = "12345678901234567890".encode("ascii") -RFC_KEY_BYTES_32 = (RFC_KEY_BYTES_20*2)[:32] -RFC_KEY_BYTES_64 = (RFC_KEY_BYTES_20*4)[:64] - -# TODO: this class is separate from TotpTest due to historical issue, -# when there was a base class, and a separate HOTP class. -# these test case classes should probably be combined. -class TotpTest(TestCase): - """ - common code shared by TotpTest & HotpTest - """ - #============================================================================= - # class attrs - #============================================================================= - - descriptionPrefix = "passlib.totp.TOTP" - - #============================================================================= - # setup - #============================================================================= - def setUp(self): - super(TotpTest, self).setUp() - - # clear norm_hash_name() cache so 'unknown hash' warnings get emitted each time - from passlib.crypto.digest import lookup_hash - lookup_hash.clear_cache() - - # monkeypatch module's rng to be deterministic - self.patchAttr(totp_module, "rng", self.getRandom()) - - #============================================================================= - # general helpers - #============================================================================= - def randtime(self): - """ - helper to generate random epoch time - :returns float: epoch time - """ - return self.getRandom().random() * max_time_t - - def randotp(self, cls=None, **kwds): - """ - helper which generates a random TOTP instance. - """ - rng = self.getRandom() - if "key" not in kwds: - kwds['new'] = True - kwds.setdefault("digits", rng.randint(6, 10)) - kwds.setdefault("alg", rng.choice(["sha1", "sha256", "sha512"])) - kwds.setdefault("period", rng.randint(10, 120)) - return (cls or TOTP)(**kwds) - - def test_randotp(self): - """ - internal test -- randotp() - """ - otp1 = self.randotp() - otp2 = self.randotp() - - self.assertNotEqual(otp1.key, otp2.key, "key not randomized:") - - # NOTE: has (1/5)**10 odds of failure - for _ in range(10): - if otp1.digits != otp2.digits: - break - otp2 = self.randotp() - else: - self.fail("digits not randomized") - - # NOTE: has (1/3)**10 odds of failure - for _ in range(10): - if otp1.alg != otp2.alg: - break - otp2 = self.randotp() - else: - self.fail("alg not randomized") - - #============================================================================= - # reference vector helpers - #============================================================================= - - #: default options used by test vectors (unless otherwise stated) - vector_defaults = dict(format="base32", alg="sha1", period=30, digits=8) - - #: various TOTP test vectors, - #: each element in list has format [options, (time, token <, int(expires)>), ...] - vectors = [ - - #------------------------------------------------------------------------- - # passlib test vectors - #------------------------------------------------------------------------- - - # 10 byte key, 6 digits - [dict(key="ACDEFGHJKL234567", digits=6), - # test fencepost to make sure we're rounding right - (1412873399, '221105'), # == 29 mod 30 - (1412873400, '178491'), # == 0 mod 30 - (1412873401, '178491'), # == 1 mod 30 - (1412873429, '178491'), # == 29 mod 30 - (1412873430, '915114'), # == 0 mod 30 - ], - - # 10 byte key, 8 digits - [dict(key="ACDEFGHJKL234567", digits=8), - # should be same as 6 digits (above), but w/ 2 more digits on left side of token. - (1412873399, '20221105'), # == 29 mod 30 - (1412873400, '86178491'), # == 0 mod 30 - (1412873401, '86178491'), # == 1 mod 30 - (1412873429, '86178491'), # == 29 mod 30 - (1412873430, '03915114'), # == 0 mod 30 - ], - - # sanity check on key used in docstrings - [dict(key="S3JD-VB7Q-D2R7-JPXX", digits=6), - (1419622709, '000492'), - (1419622739, '897212'), - ], - - #------------------------------------------------------------------------- - # reference vectors taken from http://tools.ietf.org/html/rfc6238, appendix B - # NOTE: while appendix B states same key used for all tests, the reference - # code in the appendix repeats the key up to the alg's block size, - # and uses *that* as the secret... so that's what we're doing here. - #------------------------------------------------------------------------- - - # sha1 test vectors - [dict(key=RFC_KEY_BYTES_20, format="raw", alg="sha1"), - (59, '94287082'), - (1111111109, '07081804'), - (1111111111, '14050471'), - (1234567890, '89005924'), - (2000000000, '69279037'), - (20000000000, '65353130'), - ], - - # sha256 test vectors - [dict(key=RFC_KEY_BYTES_32, format="raw", alg="sha256"), - (59, '46119246'), - (1111111109, '68084774'), - (1111111111, '67062674'), - (1234567890, '91819424'), - (2000000000, '90698825'), - (20000000000, '77737706'), - ], - - # sha512 test vectors - [dict(key=RFC_KEY_BYTES_64, format="raw", alg="sha512"), - (59, '90693936'), - (1111111109, '25091201'), - (1111111111, '99943326'), - (1234567890, '93441116'), - (2000000000, '38618901'), - (20000000000, '47863826'), - ], - - #------------------------------------------------------------------------- - # other test vectors - #------------------------------------------------------------------------- - - # generated at http://blog.tinisles.com/2011/10/google-authenticator-one-time-password-algorithm-in-javascript - [dict(key="JBSWY3DPEHPK3PXP", digits=6), (1409192430, '727248'), (1419890990, '122419')], - [dict(key="JBSWY3DPEHPK3PXP", digits=9, period=41), (1419891152, '662331049')], - - # found in https://github.com/eloquent/otis/blob/develop/test/suite/Totp/Value/TotpValueGeneratorTest.php, line 45 - [dict(key=RFC_KEY_BYTES_20, format="raw", period=60), (1111111111, '19360094')], - [dict(key=RFC_KEY_BYTES_32, format="raw", alg="sha256", period=60), (1111111111, '40857319')], - [dict(key=RFC_KEY_BYTES_64, format="raw", alg="sha512", period=60), (1111111111, '37023009')], - - ] - - def iter_test_vectors(self): - """ - helper to iterate over test vectors. - yields ``(totp, time, token, expires, prefix)`` tuples. - """ - from passlib.totp import TOTP - for row in self.vectors: - kwds = self.vector_defaults.copy() - kwds.update(row[0]) - for entry in row[1:]: - if len(entry) == 3: - time, token, expires = entry - else: - time, token = entry - expires = None - # NOTE: not re-using otp between calls so that stateful methods - # (like .match) don't have problems. - log.debug("test vector: %r time=%r token=%r expires=%r", kwds, time, token, expires) - otp = TOTP(**kwds) - prefix = "alg=%r time=%r token=%r: " % (otp.alg, time, token) - yield otp, time, token, expires, prefix - - #============================================================================= - # constructor tests - #============================================================================= - def test_ctor_w_new(self): - """constructor -- 'new' parameter""" - - # exactly one of 'key' or 'new' is required - self.assertRaises(TypeError, TOTP) - self.assertRaises(TypeError, TOTP, key='4aoggdbbqsyhntuz', new=True) - - # generates new key - otp = TOTP(new=True) - otp2 = TOTP(new=True) - self.assertNotEqual(otp.key, otp2.key) - - def test_ctor_w_size(self): - """constructor -- 'size' parameter""" - - # should default to digest size, per RFC - self.assertEqual(len(TOTP(new=True, alg="sha1").key), 20) - self.assertEqual(len(TOTP(new=True, alg="sha256").key), 32) - self.assertEqual(len(TOTP(new=True, alg="sha512").key), 64) - - # explicit key size - self.assertEqual(len(TOTP(new=True, size=10).key), 10) - self.assertEqual(len(TOTP(new=True, size=16).key), 16) - - # for new=True, maximum size enforced (based on alg) - self.assertRaises(ValueError, TOTP, new=True, size=21, alg="sha1") - - # for new=True, minimum size enforced - self.assertRaises(ValueError, TOTP, new=True, size=9) - - # for existing key, minimum size is only warned about - with self.assertWarningList([ - dict(category=exc.PasslibSecurityWarning, message_re=".*for security purposes, secret key must be.*") - ]): - _ = TOTP('0A'*9, 'hex') - - def test_ctor_w_key_and_format(self): - """constructor -- 'key' and 'format' parameters""" - - # handle base32 encoding (the default) - self.assertEqual(TOTP(KEY1).key, KEY1_RAW) - - # .. w/ lower case - self.assertEqual(TOTP(KEY1.lower()).key, KEY1_RAW) - - # .. w/ spaces (e.g. user-entered data) - self.assertEqual(TOTP(' 4aog gdbb qsyh ntuz ').key, KEY1_RAW) - - # .. w/ invalid char - self.assertRaises(Base32DecodeError, TOTP, 'ao!ggdbbqsyhntuz') - - # handle hex encoding - self.assertEqual(TOTP('e01c630c2184b076ce99', 'hex').key, KEY1_RAW) - - # .. w/ invalid char - self.assertRaises(Base16DecodeError, TOTP, 'X01c630c2184b076ce99', 'hex') - - # handle raw bytes - self.assertEqual(TOTP(KEY1_RAW, "raw").key, KEY1_RAW) - - def test_ctor_w_alg(self): - """constructor -- 'alg' parameter""" - - # normalize hash names - self.assertEqual(TOTP(KEY1, alg="SHA-256").alg, "sha256") - self.assertEqual(TOTP(KEY1, alg="SHA256").alg, "sha256") - - # invalid alg - self.assertRaises(ValueError, TOTP, KEY1, alg="SHA-333") - - def test_ctor_w_digits(self): - """constructor -- 'digits' parameter""" - self.assertRaises(ValueError, TOTP, KEY1, digits=5) - self.assertEqual(TOTP(KEY1, digits=6).digits, 6) # min value - self.assertEqual(TOTP(KEY1, digits=10).digits, 10) # max value - self.assertRaises(ValueError, TOTP, KEY1, digits=11) - - def test_ctor_w_period(self): - """constructor -- 'period' parameter""" - - # default - self.assertEqual(TOTP(KEY1).period, 30) - - # explicit value - self.assertEqual(TOTP(KEY1, period=63).period, 63) - - # reject wrong type - self.assertRaises(TypeError, TOTP, KEY1, period=1.5) - self.assertRaises(TypeError, TOTP, KEY1, period='abc') - - # reject non-positive values - self.assertRaises(ValueError, TOTP, KEY1, period=0) - self.assertRaises(ValueError, TOTP, KEY1, period=-1) - - def test_ctor_w_label(self): - """constructor -- 'label' parameter""" - self.assertEqual(TOTP(KEY1).label, None) - self.assertEqual(TOTP(KEY1, label="foo@bar").label, "foo@bar") - self.assertRaises(ValueError, TOTP, KEY1, label="foo:bar") - - def test_ctor_w_issuer(self): - """constructor -- 'issuer' parameter""" - self.assertEqual(TOTP(KEY1).issuer, None) - self.assertEqual(TOTP(KEY1, issuer="foo.com").issuer, "foo.com") - self.assertRaises(ValueError, TOTP, KEY1, issuer="foo.com:bar") - - #============================================================================= - # using() tests - #============================================================================= - - # TODO: test using() w/ 'digits', 'alg', 'issue', 'wallet', **wallet_kwds - - def test_using_w_period(self): - """using() -- 'period' parameter""" - - # default - self.assertEqual(TOTP(KEY1).period, 30) - - # explicit value - self.assertEqual(TOTP.using(period=63)(KEY1).period, 63) - - # reject wrong type - self.assertRaises(TypeError, TOTP.using, period=1.5) - self.assertRaises(TypeError, TOTP.using, period='abc') - - # reject non-positive values - self.assertRaises(ValueError, TOTP.using, period=0) - self.assertRaises(ValueError, TOTP.using, period=-1) - - def test_using_w_now(self): - """using -- 'now' parameter""" - - # NOTE: reading time w/ normalize_time() to make sure custom .now actually has effect. - - # default -- time.time - otp = self.randotp() - self.assertIs(otp.now, _time.time) - self.assertAlmostEqual(otp.normalize_time(None), int(_time.time())) - - # custom function - counter = [123.12] - def now(): - counter[0] += 1 - return counter[0] - otp = self.randotp(cls=TOTP.using(now=now)) - # NOTE: TOTP() constructor invokes this as part of test, using up counter values 124 & 125 - self.assertEqual(otp.normalize_time(None), 126) - self.assertEqual(otp.normalize_time(None), 127) - - # require callable - self.assertRaises(TypeError, TOTP.using, now=123) - - # require returns int/float - msg_re = r"now\(\) function must return non-negative" - self.assertRaisesRegex(AssertionError, msg_re, TOTP.using, now=lambda: 'abc') - - # require returns non-negative value - self.assertRaisesRegex(AssertionError, msg_re, TOTP.using, now=lambda: -1) - - #============================================================================= - # internal method tests - #============================================================================= - - def test_normalize_token_instance(self, otp=None): - """normalize_token() -- instance method""" - if otp is None: - otp = self.randotp(digits=7) - - # unicode & bytes - self.assertEqual(otp.normalize_token(u('1234567')), '1234567') - self.assertEqual(otp.normalize_token(b'1234567'), '1234567') - - # int - self.assertEqual(otp.normalize_token(1234567), '1234567') - - # int which needs 0 padding - self.assertEqual(otp.normalize_token(234567), '0234567') - - # reject wrong types (float, None) - self.assertRaises(TypeError, otp.normalize_token, 1234567.0) - self.assertRaises(TypeError, otp.normalize_token, None) - - # too few digits - self.assertRaises(exc.MalformedTokenError, otp.normalize_token, '123456') - - # too many digits - self.assertRaises(exc.MalformedTokenError, otp.normalize_token, '01234567') - self.assertRaises(exc.MalformedTokenError, otp.normalize_token, 12345678) - - def test_normalize_token_class(self): - """normalize_token() -- class method""" - self.test_normalize_token_instance(otp=TOTP.using(digits=7)) - - def test_normalize_time(self): - """normalize_time()""" - TotpFactory = TOTP.using() - otp = self.randotp(TotpFactory) - - for _ in range(10): - time = self.randtime() - tint = int(time) - - self.assertEqual(otp.normalize_time(time), tint) - self.assertEqual(otp.normalize_time(tint + 0.5), tint) - - self.assertEqual(otp.normalize_time(tint), tint) - - dt = datetime.datetime.utcfromtimestamp(time) - self.assertEqual(otp.normalize_time(dt), tint) - - orig = TotpFactory.now - try: - TotpFactory.now = staticmethod(lambda: time) - self.assertEqual(otp.normalize_time(None), tint) - finally: - TotpFactory.now = orig - - self.assertRaises(TypeError, otp.normalize_time, '1234') - - #============================================================================= - # key attr tests - #============================================================================= - - def test_key_attrs(self): - """pretty_key() and .key attributes""" - rng = self.getRandom() - - # test key attrs - otp = TOTP(KEY1_RAW, "raw") - self.assertEqual(otp.key, KEY1_RAW) - self.assertEqual(otp.hex_key, 'e01c630c2184b076ce99') - self.assertEqual(otp.base32_key, KEY1) - - # test pretty_key() - self.assertEqual(otp.pretty_key(), '4AOG-GDBB-QSYH-NTUZ') - self.assertEqual(otp.pretty_key(sep=" "), '4AOG GDBB QSYH NTUZ') - self.assertEqual(otp.pretty_key(sep=False), KEY1) - self.assertEqual(otp.pretty_key(format="hex"), 'e01c-630c-2184-b076-ce99') - - # quick fuzz test: make attr access works for random key & random size - otp = TOTP(new=True, size=rng.randint(10, 20)) - _ = otp.hex_key - _ = otp.base32_key - _ = otp.pretty_key() - - #============================================================================= - # generate() tests - #============================================================================= - def test_totp_token(self): - """generate() -- TotpToken() class""" - from passlib.totp import TOTP, TotpToken - - # test known set of values - otp = TOTP('s3jdvb7qd2r7jpxx') - result = otp.generate(1419622739) - self.assertIsInstance(result, TotpToken) - self.assertEqual(result.token, '897212') - self.assertEqual(result.counter, 47320757) - ##self.assertEqual(result.start_time, 1419622710) - self.assertEqual(result.expire_time, 1419622740) - self.assertEqual(result, ('897212', 1419622740)) - self.assertEqual(len(result), 2) - self.assertEqual(result[0], '897212') - self.assertEqual(result[1], 1419622740) - self.assertRaises(IndexError, result.__getitem__, -3) - self.assertRaises(IndexError, result.__getitem__, 2) - self.assertTrue(result) - - # time dependant bits... - otp.now = lambda : 1419622739.5 - self.assertEqual(result.remaining, 0.5) - self.assertTrue(result.valid) - - otp.now = lambda : 1419622741 - self.assertEqual(result.remaining, 0) - self.assertFalse(result.valid) - - # same time -- shouldn't return same object, but should be equal - result2 = otp.generate(1419622739) - self.assertIsNot(result2, result) - self.assertEqual(result2, result) - - # diff time in period -- shouldn't return same object, but should be equal - result3 = otp.generate(1419622711) - self.assertIsNot(result3, result) - self.assertEqual(result3, result) - - # shouldn't be equal - result4 = otp.generate(1419622999) - self.assertNotEqual(result4, result) - - def test_generate(self): - """generate()""" - from passlib.totp import TOTP - - # generate token - otp = TOTP(new=True) - time = self.randtime() - result = otp.generate(time) - token = result.token - self.assertIsInstance(token, unicode) - start_time = result.counter * 30 - - # should generate same token for next 29s - self.assertEqual(otp.generate(start_time + 29).token, token) - - # and new one at 30s - self.assertNotEqual(otp.generate(start_time + 30).token, token) - - # verify round-trip conversion of datetime - dt = datetime.datetime.utcfromtimestamp(time) - self.assertEqual(int(otp.normalize_time(dt)), int(time)) - - # handle datetime object - self.assertEqual(otp.generate(dt).token, token) - - # omitting value should use current time - otp2 = TOTP.using(now=lambda: time)(key=otp.base32_key) - self.assertEqual(otp2.generate().token, token) - - # reject invalid time - self.assertRaises(ValueError, otp.generate, -1) - - def test_generate_w_reference_vectors(self): - """generate() -- reference vectors""" - for otp, time, token, expires, prefix in self.iter_test_vectors(): - # should output correct token for specified time - result = otp.generate(time) - self.assertEqual(result.token, token, msg=prefix) - self.assertEqual(result.counter, time // otp.period, msg=prefix) - if expires: - self.assertEqual(result.expire_time, expires) - - #============================================================================= - # TotpMatch() tests - #============================================================================= - - def assertTotpMatch(self, match, time, skipped=0, period=30, window=30, msg=''): - from passlib.totp import TotpMatch - - # test type - self.assertIsInstance(match, TotpMatch) - - # totp sanity check - self.assertIsInstance(match.totp, TOTP) - self.assertEqual(match.totp.period, period) - - # test attrs - self.assertEqual(match.time, time, msg=msg + " matched time:") - expected = time // period - counter = expected + skipped - self.assertEqual(match.counter, counter, msg=msg + " matched counter:") - self.assertEqual(match.expected_counter, expected, msg=msg + " expected counter:") - self.assertEqual(match.skipped, skipped, msg=msg + " skipped:") - self.assertEqual(match.cache_seconds, period + window) - expire_time = (counter + 1) * period - self.assertEqual(match.expire_time, expire_time) - self.assertEqual(match.cache_time, expire_time + window) - - # test tuple - self.assertEqual(len(match), 2) - self.assertEqual(match, (counter, time)) - self.assertRaises(IndexError, match.__getitem__, -3) - self.assertEqual(match[0], counter) - self.assertEqual(match[1], time) - self.assertRaises(IndexError, match.__getitem__, 2) - - # test bool - self.assertTrue(match) - - def test_totp_match_w_valid_token(self): - """match() -- valid TotpMatch object""" - time = 141230981 - token = '781501' - otp = TOTP.using(now=lambda: time + 24 * 3600)(KEY3) - result = otp.match(token, time) - self.assertTotpMatch(result, time=time, skipped=0) - - def test_totp_match_w_older_token(self): - """match() -- valid TotpMatch object with future token""" - from passlib.totp import TotpMatch - - time = 141230981 - token = '781501' - otp = TOTP.using(now=lambda: time + 24 * 3600)(KEY3) - result = otp.match(token, time - 30) - self.assertTotpMatch(result, time=time - 30, skipped=1) - - def test_totp_match_w_new_token(self): - """match() -- valid TotpMatch object with past token""" - time = 141230981 - token = '781501' - otp = TOTP.using(now=lambda: time + 24 * 3600)(KEY3) - result = otp.match(token, time + 30) - self.assertTotpMatch(result, time=time + 30, skipped=-1) - - def test_totp_match_w_invalid_token(self): - """match() -- invalid TotpMatch object""" - time = 141230981 - token = '781501' - otp = TOTP.using(now=lambda: time + 24 * 3600)(KEY3) - self.assertRaises(exc.InvalidTokenError, otp.match, token, time + 60) - - #============================================================================= - # match() tests - #============================================================================= - - def assertVerifyMatches(self, expect_skipped, token, time, # * - otp, gen_time=None, **kwds): - """helper to test otp.match() output is correct""" - # NOTE: TotpMatch return type tested more throughly above ^^^ - msg = "key=%r alg=%r period=%r token=%r gen_time=%r time=%r:" % \ - (otp.base32_key, otp.alg, otp.period, token, gen_time, time) - result = otp.match(token, time, **kwds) - self.assertTotpMatch(result, - time=otp.normalize_time(time), - period=otp.period, - window=kwds.get("window", 30), - skipped=expect_skipped, - msg=msg) - - def assertVerifyRaises(self, exc_class, token, time, # * - otp, gen_time=None, - **kwds): - """helper to test otp.match() throws correct error""" - # NOTE: TotpMatch return type tested more throughly above ^^^ - msg = "key=%r alg=%r period=%r token=%r gen_time=%r time=%r:" % \ - (otp.base32_key, otp.alg, otp.period, token, gen_time, time) - return self.assertRaises(exc_class, otp.match, token, time, - __msg__=msg, **kwds) - - def test_match_w_window(self): - """match() -- 'time' and 'window' parameters""" - - # init generator & helper - otp = self.randotp() - period = otp.period - time = self.randtime() - token = otp.generate(time).token - common = dict(otp=otp, gen_time=time) - assertMatches = partial(self.assertVerifyMatches, **common) - assertRaises = partial(self.assertVerifyRaises, **common) - - #------------------------------- - # basic validation, and 'window' parameter - #------------------------------- - - # validate against previous counter (passes if window >= period) - assertRaises(exc.InvalidTokenError, token, time - period, window=0) - assertMatches(+1, token, time - period, window=period) - assertMatches(+1, token, time - period, window=2 * period) - - # validate against current counter - assertMatches(0, token, time, window=0) - - # validate against next counter (passes if window >= period) - assertRaises(exc.InvalidTokenError, token, time + period, window=0) - assertMatches(-1, token, time + period, window=period) - assertMatches(-1, token, time + period, window=2 * period) - - # validate against two time steps later (should never pass) - assertRaises(exc.InvalidTokenError, token, time + 2 * period, window=0) - assertRaises(exc.InvalidTokenError, token, time + 2 * period, window=period) - assertMatches(-2, token, time + 2 * period, window=2 * period) - - # TODO: test window values that aren't multiples of period - # (esp ensure counter rounding works correctly) - - #------------------------------- - # time normalization - #------------------------------- - - # handle datetimes - dt = datetime.datetime.utcfromtimestamp(time) - assertMatches(0, token, dt, window=0) - - # reject invalid time - assertRaises(ValueError, token, -1) - - def test_match_w_skew(self): - """match() -- 'skew' parameters""" - # init generator & helper - otp = self.randotp() - period = otp.period - time = self.randtime() - common = dict(otp=otp, gen_time=time) - assertMatches = partial(self.assertVerifyMatches, **common) - assertRaises = partial(self.assertVerifyRaises, **common) - - # assume client is running far behind server / has excessive transmission delay - skew = 3 * period - behind_token = otp.generate(time - skew).token - assertRaises(exc.InvalidTokenError, behind_token, time, window=0) - assertMatches(-3, behind_token, time, window=0, skew=-skew) - - # assume client is running far ahead of server - ahead_token = otp.generate(time + skew).token - assertRaises(exc.InvalidTokenError, ahead_token, time, window=0) - assertMatches(+3, ahead_token, time, window=0, skew=skew) - - # TODO: test skew + larger window - - def test_match_w_reuse(self): - """match() -- 'reuse' and 'last_counter' parameters""" - - # init generator & helper - otp = self.randotp() - period = otp.period - time = self.randtime() - tdata = otp.generate(time) - token = tdata.token - counter = tdata.counter - expire_time = tdata.expire_time - common = dict(otp=otp, gen_time=time) - assertMatches = partial(self.assertVerifyMatches, **common) - assertRaises = partial(self.assertVerifyRaises, **common) - - # last counter unset -- - # previous period's token should count as valid - assertMatches(-1, token, time + period, window=period) - - # last counter set 2 periods ago -- - # previous period's token should count as valid - assertMatches(-1, token, time + period, last_counter=counter-1, - window=period) - - # last counter set 2 periods ago -- - # 2 periods ago's token should NOT count as valid - assertRaises(exc.InvalidTokenError, token, time + 2 * period, - last_counter=counter, window=period) - - # last counter set 1 period ago -- - # previous period's token should now be rejected as 'used' - err = assertRaises(exc.UsedTokenError, token, time + period, - last_counter=counter, window=period) - self.assertEqual(err.expire_time, expire_time) - - # last counter set to current period -- - # current period's token should be rejected - err = assertRaises(exc.UsedTokenError, token, time, - last_counter=counter, window=0) - self.assertEqual(err.expire_time, expire_time) - - def test_match_w_token_normalization(self): - """match() -- token normalization""" - # setup test helper - otp = TOTP('otxl2f5cctbprpzx') - match = otp.match - time = 1412889861 - - # separators / spaces should be stripped (orig token '332136') - self.assertTrue(match(' 3 32-136 ', time)) - - # ascii bytes - self.assertTrue(match(b'332136', time)) - - # too few digits - self.assertRaises(exc.MalformedTokenError, match, '12345', time) - - # invalid char - self.assertRaises(exc.MalformedTokenError, match, '12345X', time) - - # leading zeros count towards size - self.assertRaises(exc.MalformedTokenError, match, '0123456', time) - - def test_match_w_reference_vectors(self): - """match() -- reference vectors""" - for otp, time, token, expires, msg in self.iter_test_vectors(): - # create wrapper - match = otp.match - - # token should match against time - result = match(token, time) - self.assertTrue(result) - self.assertEqual(result.counter, time // otp.period, msg=msg) - - # should NOT match against another time - self.assertRaises(exc.InvalidTokenError, match, token, time + 100, window=0) - - #============================================================================= - # verify() tests - #============================================================================= - def test_verify(self): - """verify()""" - # NOTE: since this is thin wrapper around .from_source() and .match(), - # just testing basic behavior here. - - from passlib.totp import TOTP - - time = 1412889861 - TotpFactory = TOTP.using(now=lambda: time) - - # successful match - source1 = dict(v=1, type="totp", key='otxl2f5cctbprpzx') - match = TotpFactory.verify('332136', source1) - self.assertTotpMatch(match, time=time) - - # failed match - source1 = dict(v=1, type="totp", key='otxl2f5cctbprpzx') - self.assertRaises(exc.InvalidTokenError, TotpFactory.verify, '332155', source1) - - # bad source - source1 = dict(v=1, type="totp") - self.assertRaises(ValueError, TotpFactory.verify, '332155', source1) - - # successful match -- json source - source1json = '{"v": 1, "type": "totp", "key": "otxl2f5cctbprpzx"}' - match = TotpFactory.verify('332136', source1json) - self.assertTotpMatch(match, time=time) - - # successful match -- URI - source1uri = 'otpauth://totp/Label?secret=otxl2f5cctbprpzx' - match = TotpFactory.verify('332136', source1uri) - self.assertTotpMatch(match, time=time) - - #============================================================================= - # serialization frontend tests - #============================================================================= - def test_from_source(self): - """from_source()""" - from passlib.totp import TOTP - from_source = TOTP.from_source - - # uri (unicode) - otp = from_source(u("otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "issuer=Example")) - self.assertEqual(otp.key, KEY4_RAW) - - # uri (bytes) - otp = from_source(b"otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&" - b"issuer=Example") - self.assertEqual(otp.key, KEY4_RAW) - - # dict - otp = from_source(dict(v=1, type="totp", key=KEY4)) - self.assertEqual(otp.key, KEY4_RAW) - - # json (unicode) - otp = from_source(u('{"v": 1, "type": "totp", "key": "JBSWY3DPEHPK3PXP"}')) - self.assertEqual(otp.key, KEY4_RAW) - - # json (bytes) - otp = from_source(b'{"v": 1, "type": "totp", "key": "JBSWY3DPEHPK3PXP"}') - self.assertEqual(otp.key, KEY4_RAW) - - # TOTP object -- return unchanged - self.assertIs(from_source(otp), otp) - - # TOTP object w/ different wallet -- return new one. - wallet1 = AppWallet() - otp1 = TOTP.using(wallet=wallet1).from_source(otp) - self.assertIsNot(otp1, otp) - self.assertEqual(otp1.to_dict(), otp.to_dict()) - - # TOTP object w/ same wallet -- return original - otp2 = TOTP.using(wallet=wallet1).from_source(otp1) - self.assertIs(otp2, otp1) - - # random string - self.assertRaises(ValueError, from_source, u("foo")) - self.assertRaises(ValueError, from_source, b"foo") - - #============================================================================= - # uri serialization tests - #============================================================================= - def test_from_uri(self): - """from_uri()""" - from passlib.totp import TOTP - from_uri = TOTP.from_uri - - # URIs from https://code.google.com/p/google-authenticator/wiki/KeyUriFormat - - #-------------------------------------------------------------------------------- - # canonical uri - #-------------------------------------------------------------------------------- - otp = from_uri("otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "issuer=Example") - self.assertIsInstance(otp, TOTP) - self.assertEqual(otp.key, KEY4_RAW) - self.assertEqual(otp.label, "alice@google.com") - self.assertEqual(otp.issuer, "Example") - self.assertEqual(otp.alg, "sha1") # default - self.assertEqual(otp.period, 30) # default - self.assertEqual(otp.digits, 6) # default - - #-------------------------------------------------------------------------------- - # secret param - #-------------------------------------------------------------------------------- - - # secret case insensitive - otp = from_uri("otpauth://totp/Example:alice@google.com?secret=jbswy3dpehpk3pxp&" - "issuer=Example") - self.assertEqual(otp.key, KEY4_RAW) - - # missing secret - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?digits=6") - - # undecodable secret - self.assertRaises(Base32DecodeError, from_uri, "otpauth://totp/Example:alice@google.com?" - "secret=JBSWY3DPEHP@3PXP") - - #-------------------------------------------------------------------------------- - # label param - #-------------------------------------------------------------------------------- - - # w/ encoded space - otp = from_uri("otpauth://totp/Provider1:Alice%20Smith?secret=JBSWY3DPEHPK3PXP&" - "issuer=Provider1") - self.assertEqual(otp.label, "Alice Smith") - self.assertEqual(otp.issuer, "Provider1") - - # w/ encoded space and colon - # (note url has leading space before 'alice') -- taken from KeyURI spec - otp = from_uri("otpauth://totp/Big%20Corporation%3A%20alice@bigco.com?" - "secret=JBSWY3DPEHPK3PXP") - self.assertEqual(otp.label, "alice@bigco.com") - self.assertEqual(otp.issuer, "Big Corporation") - - #-------------------------------------------------------------------------------- - # issuer param / prefix - #-------------------------------------------------------------------------------- - - # 'new style' issuer only - otp = from_uri("otpauth://totp/alice@bigco.com?secret=JBSWY3DPEHPK3PXP&issuer=Big%20Corporation") - self.assertEqual(otp.label, "alice@bigco.com") - self.assertEqual(otp.issuer, "Big Corporation") - - # new-vs-old issuer mismatch - self.assertRaises(ValueError, TOTP.from_uri, - "otpauth://totp/Provider1:alice?secret=JBSWY3DPEHPK3PXP&issuer=Provider2") - - #-------------------------------------------------------------------------------- - # algorithm param - #-------------------------------------------------------------------------------- - - # custom alg - otp = from_uri("otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&algorithm=SHA256") - self.assertEqual(otp.alg, "sha256") - - # unknown alg - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?" - "secret=JBSWY3DPEHPK3PXP&algorithm=SHA333") - - #-------------------------------------------------------------------------------- - # digit param - #-------------------------------------------------------------------------------- - - # custom digits - otp = from_uri("otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&digits=8") - self.assertEqual(otp.digits, 8) - - # digits out of range / invalid - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&digits=A") - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&digits=%20") - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&digits=15") - - #-------------------------------------------------------------------------------- - # period param - #-------------------------------------------------------------------------------- - - # custom period - otp = from_uri("otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&period=63") - self.assertEqual(otp.period, 63) - - # reject period < 1 - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?" - "secret=JBSWY3DPEHPK3PXP&period=0") - - self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?" - "secret=JBSWY3DPEHPK3PXP&period=-1") - - #-------------------------------------------------------------------------------- - # unrecognized param - #-------------------------------------------------------------------------------- - - # should issue warning, but otherwise ignore extra param - with self.assertWarningList([ - dict(category=exc.PasslibRuntimeWarning, message_re="unexpected parameters encountered") - ]): - otp = from_uri("otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "foo=bar&period=63") - self.assertEqual(otp.base32_key, KEY4) - self.assertEqual(otp.period, 63) - - def test_to_uri(self): - """to_uri()""" - - #------------------------------------------------------------------------- - # label & issuer parameters - #------------------------------------------------------------------------- - - # with label & issuer - otp = TOTP(KEY4, alg="sha1", digits=6, period=30) - self.assertEqual(otp.to_uri("alice@google.com", "Example Org"), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "issuer=Example%20Org") - - # label is required - self.assertRaises(ValueError, otp.to_uri, None, "Example Org") - - # with label only - self.assertEqual(otp.to_uri("alice@google.com"), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP") - - # with default label from constructor - otp.label = "alice@google.com" - self.assertEqual(otp.to_uri(), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP") - - # with default label & default issuer from constructor - otp.issuer = "Example Org" - self.assertEqual(otp.to_uri(), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP" - "&issuer=Example%20Org") - - # reject invalid label - self.assertRaises(ValueError, otp.to_uri, "label:with:semicolons") - - # reject invalid issue - self.assertRaises(ValueError, otp.to_uri, "alice@google.com", "issuer:with:semicolons") - - #------------------------------------------------------------------------- - # algorithm parameter - #------------------------------------------------------------------------- - self.assertEqual(TOTP(KEY4, alg="sha256").to_uri("alice@google.com"), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "algorithm=SHA256") - - #------------------------------------------------------------------------- - # digits parameter - #------------------------------------------------------------------------- - self.assertEqual(TOTP(KEY4, digits=8).to_uri("alice@google.com"), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "digits=8") - - #------------------------------------------------------------------------- - # period parameter - #------------------------------------------------------------------------- - self.assertEqual(TOTP(KEY4, period=63).to_uri("alice@google.com"), - "otpauth://totp/alice@google.com?secret=JBSWY3DPEHPK3PXP&" - "period=63") - - #============================================================================= - # dict serialization tests - #============================================================================= - def test_from_dict(self): - """from_dict()""" - from passlib.totp import TOTP - from_dict = TOTP.from_dict - - #-------------------------------------------------------------------------------- - # canonical simple example - #-------------------------------------------------------------------------------- - otp = from_dict(dict(v=1, type="totp", key=KEY4, label="alice@google.com", issuer="Example")) - self.assertIsInstance(otp, TOTP) - self.assertEqual(otp.key, KEY4_RAW) - self.assertEqual(otp.label, "alice@google.com") - self.assertEqual(otp.issuer, "Example") - self.assertEqual(otp.alg, "sha1") # default - self.assertEqual(otp.period, 30) # default - self.assertEqual(otp.digits, 6) # default - - #-------------------------------------------------------------------------------- - # metadata - #-------------------------------------------------------------------------------- - - # missing version - self.assertRaises(ValueError, from_dict, dict(type="totp", key=KEY4)) - - # invalid version - self.assertRaises(ValueError, from_dict, dict(v=0, type="totp", key=KEY4)) - self.assertRaises(ValueError, from_dict, dict(v=999, type="totp", key=KEY4)) - - # missing type - self.assertRaises(ValueError, from_dict, dict(v=1, key=KEY4)) - - #-------------------------------------------------------------------------------- - # secret param - #-------------------------------------------------------------------------------- - - # secret case insensitive - otp = from_dict(dict(v=1, type="totp", key=KEY4.lower(), label="alice@google.com", issuer="Example")) - self.assertEqual(otp.key, KEY4_RAW) - - # missing secret - self.assertRaises(ValueError, from_dict, dict(v=1, type="totp")) - - # undecodable secret - self.assertRaises(Base32DecodeError, from_dict, - dict(v=1, type="totp", key="JBSWY3DPEHP@3PXP")) - - #-------------------------------------------------------------------------------- - # label & issuer params - #-------------------------------------------------------------------------------- - - otp = from_dict(dict(v=1, type="totp", key=KEY4, label="Alice Smith", issuer="Provider1")) - self.assertEqual(otp.label, "Alice Smith") - self.assertEqual(otp.issuer, "Provider1") - - #-------------------------------------------------------------------------------- - # algorithm param - #-------------------------------------------------------------------------------- - - # custom alg - otp = from_dict(dict(v=1, type="totp", key=KEY4, alg="sha256")) - self.assertEqual(otp.alg, "sha256") - - # unknown alg - self.assertRaises(ValueError, from_dict, dict(v=1, type="totp", key=KEY4, alg="sha333")) - - #-------------------------------------------------------------------------------- - # digit param - #-------------------------------------------------------------------------------- - - # custom digits - otp = from_dict(dict(v=1, type="totp", key=KEY4, digits=8)) - self.assertEqual(otp.digits, 8) - - # digits out of range / invalid - self.assertRaises(TypeError, from_dict, dict(v=1, type="totp", key=KEY4, digits="A")) - self.assertRaises(ValueError, from_dict, dict(v=1, type="totp", key=KEY4, digits=15)) - - #-------------------------------------------------------------------------------- - # period param - #-------------------------------------------------------------------------------- - - # custom period - otp = from_dict(dict(v=1, type="totp", key=KEY4, period=63)) - self.assertEqual(otp.period, 63) - - # reject period < 1 - self.assertRaises(ValueError, from_dict, dict(v=1, type="totp", key=KEY4, period=0)) - self.assertRaises(ValueError, from_dict, dict(v=1, type="totp", key=KEY4, period=-1)) - - #-------------------------------------------------------------------------------- - # unrecognized param - #-------------------------------------------------------------------------------- - self.assertRaises(TypeError, from_dict, dict(v=1, type="totp", key=KEY4, INVALID=123)) - - def test_to_dict(self): - """to_dict()""" - - #------------------------------------------------------------------------- - # label & issuer parameters - #------------------------------------------------------------------------- - - # without label or issuer - otp = TOTP(KEY4, alg="sha1", digits=6, period=30) - self.assertEqual(otp.to_dict(), dict(v=1, type="totp", key=KEY4)) - - # with label & issuer from constructor - otp = TOTP(KEY4, alg="sha1", digits=6, period=30, - label="alice@google.com", issuer="Example Org") - self.assertEqual(otp.to_dict(), - dict(v=1, type="totp", key=KEY4, - label="alice@google.com", issuer="Example Org")) - - # with label only - otp = TOTP(KEY4, alg="sha1", digits=6, period=30, - label="alice@google.com") - self.assertEqual(otp.to_dict(), - dict(v=1, type="totp", key=KEY4, - label="alice@google.com")) - - # with issuer only - otp = TOTP(KEY4, alg="sha1", digits=6, period=30, - issuer="Example Org") - self.assertEqual(otp.to_dict(), - dict(v=1, type="totp", key=KEY4, - issuer="Example Org")) - - # don't serialize default issuer - TotpFactory = TOTP.using(issuer="Example Org") - otp = TotpFactory(KEY4) - self.assertEqual(otp.to_dict(), dict(v=1, type="totp", key=KEY4)) - - # don't serialize default issuer *even if explicitly set* - otp = TotpFactory(KEY4, issuer="Example Org") - self.assertEqual(otp.to_dict(), dict(v=1, type="totp", key=KEY4)) - - #------------------------------------------------------------------------- - # algorithm parameter - #------------------------------------------------------------------------- - self.assertEqual(TOTP(KEY4, alg="sha256").to_dict(), - dict(v=1, type="totp", key=KEY4, alg="sha256")) - - #------------------------------------------------------------------------- - # digits parameter - #------------------------------------------------------------------------- - self.assertEqual(TOTP(KEY4, digits=8).to_dict(), - dict(v=1, type="totp", key=KEY4, digits=8)) - - #------------------------------------------------------------------------- - # period parameter - #------------------------------------------------------------------------- - self.assertEqual(TOTP(KEY4, period=63).to_dict(), - dict(v=1, type="totp", key=KEY4, period=63)) - - # TODO: to_dict() - # with encrypt=False - # with encrypt="auto" + wallet + secrets - # with encrypt="auto" + wallet + no secrets - # with encrypt="auto" + no wallet - # with encrypt=True + wallet + secrets - # with encrypt=True + wallet + no secrets - # with encrypt=True + no wallet - # that 'changed' is set for old versions, and old encryption tags. - - #============================================================================= - # json serialization tests - #============================================================================= - - # TODO: from_json() / to_json(). - # (skipped for right now cause just wrapper for from_dict/to_dict) - - #============================================================================= - # eoc - #============================================================================= - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_utils.py b/src/passlib/tests/test_utils.py deleted file mode 100644 index 3fd00015..00000000 --- a/src/passlib/tests/test_utils.py +++ /dev/null @@ -1,1023 +0,0 @@ -"""tests for passlib.util""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -from functools import partial -import warnings -# site -# pkg -# module -from passlib.utils import is_ascii_safe -from passlib.utils.compat import irange, PY2, PY3, u, unicode, join_bytes, PYPY -from passlib.tests.utils import TestCase, hb, run_with_fixed_seeds - -#============================================================================= -# byte funcs -#============================================================================= -class MiscTest(TestCase): - """tests various parts of utils module""" - - # NOTE: could test xor_bytes(), but it's exercised well enough by pbkdf2 test - - def test_compat(self): - """test compat's lazymodule""" - from passlib.utils import compat - # "" - self.assertRegex(repr(compat), - r"^$") - - # test synthentic dir() - dir(compat) - self.assertTrue('UnicodeIO' in dir(compat)) - self.assertTrue('irange' in dir(compat)) - - def test_classproperty(self): - from passlib.utils.decor import classproperty - - class test(object): - xvar = 1 - @classproperty - def xprop(cls): - return cls.xvar - - self.assertEqual(test.xprop, 1) - prop = test.__dict__['xprop'] - self.assertIs(prop.im_func, prop.__func__) - - def test_deprecated_function(self): - from passlib.utils.decor import deprecated_function - # NOTE: not comprehensive, just tests the basic behavior - - @deprecated_function(deprecated="1.6", removed="1.8") - def test_func(*args): - """test docstring""" - return args - - self.assertTrue(".. deprecated::" in test_func.__doc__) - - with self.assertWarningList(dict(category=DeprecationWarning, - message="the function passlib.tests.test_utils.test_func() " - "is deprecated as of Passlib 1.6, and will be " - "removed in Passlib 1.8." - )): - self.assertEqual(test_func(1,2), (1,2)) - - def test_memoized_property(self): - from passlib.utils.decor import memoized_property - - class dummy(object): - counter = 0 - - @memoized_property - def value(self): - value = self.counter - self.counter = value+1 - return value - - d = dummy() - self.assertEqual(d.value, 0) - self.assertEqual(d.value, 0) - self.assertEqual(d.counter, 1) - - prop = dummy.value - if not PY3: - self.assertIs(prop.im_func, prop.__func__) - - def test_getrandbytes(self): - """getrandbytes()""" - from passlib.utils import getrandbytes - wrapper = partial(getrandbytes, self.getRandom()) - self.assertEqual(len(wrapper(0)), 0) - a = wrapper(10) - b = wrapper(10) - self.assertIsInstance(a, bytes) - self.assertEqual(len(a), 10) - self.assertEqual(len(b), 10) - self.assertNotEqual(a, b) - - @run_with_fixed_seeds(count=1024) - def test_getrandstr(self, seed): - """getrandstr()""" - from passlib.utils import getrandstr - - wrapper = partial(getrandstr, self.getRandom(seed=seed)) - - # count 0 - self.assertEqual(wrapper('abc',0), '') - - # count <0 - self.assertRaises(ValueError, wrapper, 'abc', -1) - - # letters 0 - self.assertRaises(ValueError, wrapper, '', 0) - - # letters 1 - self.assertEqual(wrapper('a', 5), 'aaaaa') - - # NOTE: the following parts are non-deterministic, - # with a small chance of failure (outside chance it may pick - # a string w/o one char, even more remote chance of picking - # same string). to combat this, we run it against multiple - # fixed seeds (using run_with_fixed_seeds decorator), - # and hope that they're sufficient to test the range of behavior. - - # letters - x = wrapper(u('abc'), 32) - y = wrapper(u('abc'), 32) - self.assertIsInstance(x, unicode) - self.assertNotEqual(x,y) - self.assertEqual(sorted(set(x)), [u('a'),u('b'),u('c')]) - - # bytes - x = wrapper(b'abc', 32) - y = wrapper(b'abc', 32) - self.assertIsInstance(x, bytes) - self.assertNotEqual(x,y) - # NOTE: decoding this due to py3 bytes - self.assertEqual(sorted(set(x.decode("ascii"))), [u('a'),u('b'),u('c')]) - - def test_generate_password(self): - """generate_password()""" - from passlib.utils import generate_password - warnings.filterwarnings("ignore", "The function.*generate_password\(\) is deprecated") - self.assertEqual(len(generate_password(15)), 15) - - def test_is_crypt_context(self): - """test is_crypt_context()""" - from passlib.utils import is_crypt_context - from passlib.context import CryptContext - cc = CryptContext(["des_crypt"]) - self.assertTrue(is_crypt_context(cc)) - self.assertFalse(not is_crypt_context(cc)) - - def test_genseed(self): - """test genseed()""" - import random - from passlib.utils import genseed - rng = random.Random(genseed()) - a = rng.randint(0, 10**10) - - rng = random.Random(genseed()) - b = rng.randint(0, 10**10) - - self.assertNotEqual(a,b) - - rng.seed(genseed(rng)) - - def test_crypt(self): - """test crypt.crypt() wrappers""" - from passlib.utils import has_crypt, safe_crypt, test_crypt - - # test everything is disabled - if not has_crypt: - self.assertEqual(safe_crypt("test", "aa"), None) - self.assertFalse(test_crypt("test", "aaqPiZY5xR5l.")) - raise self.skipTest("crypt.crypt() not available") - - # XXX: this assumes *every* crypt() implementation supports des_crypt. - # if this fails for some platform, this test will need modifying. - - # test return type - self.assertIsInstance(safe_crypt(u("test"), u("aa")), unicode) - - # test ascii password - h1 = u('aaqPiZY5xR5l.') - self.assertEqual(safe_crypt(u('test'), u('aa')), h1) - self.assertEqual(safe_crypt(b'test', b'aa'), h1) - - # test utf-8 / unicode password - h2 = u('aahWwbrUsKZk.') - self.assertEqual(safe_crypt(u('test\u1234'), 'aa'), h2) - self.assertEqual(safe_crypt(b'test\xe1\x88\xb4', 'aa'), h2) - - # test latin-1 password - hash = safe_crypt(b'test\xff', 'aa') - if PY3: # py3 supports utf-8 bytes only. - self.assertEqual(hash, None) - else: # but py2 is fine. - self.assertEqual(hash, u('aaOx.5nbTU/.M')) - - # test rejects null chars in password - self.assertRaises(ValueError, safe_crypt, '\x00', 'aa') - - # check test_crypt() - h1x = h1[:-1] + 'x' - self.assertTrue(test_crypt("test", h1)) - self.assertFalse(test_crypt("test", h1x)) - - # check crypt returning variant error indicators - # some platforms return None on errors, others empty string, - # The BSDs in some cases return ":" - import passlib.utils as mod - orig = mod._crypt - try: - fake = None - mod._crypt = lambda secret, hash: fake - for fake in [None, "", ":", ":0", "*0"]: - self.assertEqual(safe_crypt("test", "aa"), None) - self.assertFalse(test_crypt("test", h1)) - fake = 'xxx' - self.assertEqual(safe_crypt("test", "aa"), "xxx") - finally: - mod._crypt = orig - - def test_consteq(self): - """test consteq()""" - # NOTE: this test is kind of over the top, but that's only because - # this is used for the critical task of comparing hashes for equality. - from passlib.utils import consteq, str_consteq - - # ensure error raises for wrong types - self.assertRaises(TypeError, consteq, u(''), b'') - self.assertRaises(TypeError, consteq, u(''), 1) - self.assertRaises(TypeError, consteq, u(''), None) - - self.assertRaises(TypeError, consteq, b'', u('')) - self.assertRaises(TypeError, consteq, b'', 1) - self.assertRaises(TypeError, consteq, b'', None) - - self.assertRaises(TypeError, consteq, None, u('')) - self.assertRaises(TypeError, consteq, None, b'') - self.assertRaises(TypeError, consteq, 1, u('')) - self.assertRaises(TypeError, consteq, 1, b'') - - def consteq_supports_string(value): - # under PY2, it supports all unicode strings (when present at all), - # under PY3, compare_digest() only supports ascii unicode strings. - # confirmed for: cpython 2.7.9, cpython 3.4, pypy, pypy3, pyston - return (consteq is str_consteq or PY2 or is_ascii_safe(value)) - - # check equal inputs compare correctly - for value in [ - u("a"), - u("abc"), - u("\xff\xa2\x12\x00")*10, - ]: - if consteq_supports_string(value): - self.assertTrue(consteq(value, value), "value %r:" % (value,)) - else: - self.assertRaises(TypeError, consteq, value, value) - self.assertTrue(str_consteq(value, value), "value %r:" % (value,)) - - value = value.encode("latin-1") - self.assertTrue(consteq(value, value), "value %r:" % (value,)) - - # check non-equal inputs compare correctly - for l,r in [ - # check same-size comparisons with differing contents fail. - (u("a"), u("c")), - (u("abcabc"), u("zbaabc")), - (u("abcabc"), u("abzabc")), - (u("abcabc"), u("abcabz")), - ((u("\xff\xa2\x12\x00")*10)[:-1] + u("\x01"), - u("\xff\xa2\x12\x00")*10), - - # check different-size comparisons fail. - (u(""), u("a")), - (u("abc"), u("abcdef")), - (u("abc"), u("defabc")), - (u("qwertyuiopasdfghjklzxcvbnm"), u("abc")), - ]: - if consteq_supports_string(l) and consteq_supports_string(r): - self.assertFalse(consteq(l, r), "values %r %r:" % (l,r)) - self.assertFalse(consteq(r, l), "values %r %r:" % (r,l)) - else: - self.assertRaises(TypeError, consteq, l, r) - self.assertRaises(TypeError, consteq, r, l) - self.assertFalse(str_consteq(l, r), "values %r %r:" % (l,r)) - self.assertFalse(str_consteq(r, l), "values %r %r:" % (r,l)) - - l = l.encode("latin-1") - r = r.encode("latin-1") - self.assertFalse(consteq(l, r), "values %r %r:" % (l,r)) - self.assertFalse(consteq(r, l), "values %r %r:" % (r,l)) - - # TODO: add some tests to ensure we take THETA(strlen) time. - # this might be hard to do reproducably. - # NOTE: below code was used to generate stats for analysis - ##from math import log as logb - ##import timeit - ##multipliers = [ 1< encode() -> decode() -> raw - # - - # generate some random bytes - size = rng.randint(1 if saw_zero else 0, 12) - if not size: - saw_zero = True - enc_size = (4*size+2)//3 - raw = getrandbytes(rng, size) - - # encode them, check invariants - encoded = engine.encode_bytes(raw) - self.assertEqual(len(encoded), enc_size) - - # make sure decode returns original - result = engine.decode_bytes(encoded) - self.assertEqual(result, raw) - - # - # test encoded -> decode() -> encode() -> encoded - # - - # generate some random encoded data - if size % 4 == 1: - size += rng.choice([-1,1,2]) - raw_size = 3*size//4 - encoded = getrandstr(rng, engine.bytemap, size) - - # decode them, check invariants - raw = engine.decode_bytes(encoded) - self.assertEqual(len(raw), raw_size, "encoded %d:" % size) - - # make sure encode returns original (barring padding bits) - result = engine.encode_bytes(raw) - if size % 4: - self.assertEqual(result[:-1], encoded[:-1]) - else: - self.assertEqual(result, encoded) - - def test_repair_unused(self): - """test repair_unused()""" - # NOTE: this test relies on encode_bytes() always returning clear - # padding bits - which should be ensured by test vectors. - from passlib.utils import getrandstr - rng = self.getRandom() - engine = self.engine - check_repair_unused = self.engine.check_repair_unused - i = 0 - while i < 300: - size = rng.randint(0,23) - cdata = getrandstr(rng, engine.charmap, size).encode("ascii") - if size & 3 == 1: - # should throw error - self.assertRaises(ValueError, check_repair_unused, cdata) - continue - rdata = engine.encode_bytes(engine.decode_bytes(cdata)) - if rng.random() < .5: - cdata = cdata.decode("ascii") - rdata = rdata.decode("ascii") - if cdata == rdata: - # should leave unchanged - ok, result = check_repair_unused(cdata) - self.assertFalse(ok) - self.assertEqual(result, rdata) - else: - # should repair bits - self.assertNotEqual(size % 4, 0) - ok, result = check_repair_unused(cdata) - self.assertTrue(ok) - self.assertEqual(result, rdata) - i += 1 - - #=================================================================== - # test transposed encode/decode - encoding independant - #=================================================================== - # NOTE: these tests assume normal encode/decode has been tested elsewhere. - - transposed = [ - # orig, result, transpose map - (b"\x33\x22\x11", b"\x11\x22\x33",[2,1,0]), - (b"\x22\x33\x11", b"\x11\x22\x33",[1,2,0]), - ] - - transposed_dups = [ - # orig, result, transpose projection - (b"\x11\x11\x22", b"\x11\x22\x33",[0,0,1]), - ] - - def test_encode_transposed_bytes(self): - """test encode_transposed_bytes()""" - engine = self.engine - for result, input, offsets in self.transposed + self.transposed_dups: - tmp = engine.encode_transposed_bytes(input, offsets) - out = engine.decode_bytes(tmp) - self.assertEqual(out, result) - - self.assertRaises(TypeError, engine.encode_transposed_bytes, u("a"), []) - - def test_decode_transposed_bytes(self): - """test decode_transposed_bytes()""" - engine = self.engine - for input, result, offsets in self.transposed: - tmp = engine.encode_bytes(input) - out = engine.decode_transposed_bytes(tmp, offsets) - self.assertEqual(out, result) - - def test_decode_transposed_bytes_bad(self): - """test decode_transposed_bytes() fails if map is a one-way""" - engine = self.engine - for input, _, offsets in self.transposed_dups: - tmp = engine.encode_bytes(input) - self.assertRaises(TypeError, engine.decode_transposed_bytes, tmp, - offsets) - - #=================================================================== - # test 6bit handling - #=================================================================== - def check_int_pair(self, bits, encoded_pairs): - """helper to check encode_intXX & decode_intXX functions""" - rng = self.getRandom() - engine = self.engine - encode = getattr(engine, "encode_int%s" % bits) - decode = getattr(engine, "decode_int%s" % bits) - pad = -bits % 6 - chars = (bits+pad)//6 - upper = 1< hex digest - # test vectors from http://www.faqs.org/rfcs/rfc1320.html - A.5 - (b(""), "31d6cfe0d16ae931b73c59d7e0c089c0"), - (b("a"), "bde52cb31de33e46245e05fbdbd6fb24"), - (b("abc"), "a448017aaf21d8525fc10ae87aa6729d"), - (b("message digest"), "d9130a8164549fe818874806e1c7014b"), - (b("abcdefghijklmnopqrstuvwxyz"), "d79e1c308aa5bbcdeea8ed63df412da9"), - (b("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), "043f8582f241db351ce627e153e7f0e4"), - (b("12345678901234567890123456789012345678901234567890123456789012345678901234567890"), "e33b4ddc9c38f2199c3e7b164fcc0536"), - ] - - def test_md4_update(self): - """test md4 update""" - from passlib.utils.md4 import md4 - h = md4(b('')) - self.assertEqual(h.hexdigest(), "31d6cfe0d16ae931b73c59d7e0c089c0") - - # NOTE: under py2, hashlib methods try to encode to ascii, - # though shouldn't rely on that. - if PY3 or self._disable_native: - self.assertRaises(TypeError, h.update, u('x')) - - h.update(b('a')) - self.assertEqual(h.hexdigest(), "bde52cb31de33e46245e05fbdbd6fb24") - - h.update(b('bcdefghijklmnopqrstuvwxyz')) - self.assertEqual(h.hexdigest(), "d79e1c308aa5bbcdeea8ed63df412da9") - - def test_md4_hexdigest(self): - """test md4 hexdigest()""" - from passlib.utils.md4 import md4 - for input, hex in self.vectors: - out = md4(input).hexdigest() - self.assertEqual(out, hex) - - def test_md4_digest(self): - """test md4 digest()""" - from passlib.utils.md4 import md4 - for input, hex in self.vectors: - out = bascii_to_str(hexlify(md4(input).digest())) - self.assertEqual(out, hex) - - def test_md4_copy(self): - """test md4 copy()""" - from passlib.utils.md4 import md4 - h = md4(b('abc')) - - h2 = h.copy() - h2.update(b('def')) - self.assertEqual(h2.hexdigest(), '804e7f1c2586e50b49ac65db5b645131') - - h.update(b('ghi')) - self.assertEqual(h.hexdigest(), 'c5225580bfe176f6deeee33dee98732c') - -# create subclasses to test with and without native backend -class MD4_SSL_Test(_MD4_Test): - descriptionPrefix = "MD4 (ssl version)" -MD4_SSL_TEST = skipUnless(has_native_md4, "hashlib lacks ssl support")(MD4_SSL_Test) - -class MD4_Builtin_Test(_MD4_Test): - descriptionPrefix = "MD4 (builtin version)" - _disable_native = True -MD4_Builtin_Test = skipUnless(TEST_MODE("full") or not has_native_md4, - "skipped under current test mode")(MD4_Builtin_Test) - -#============================================================================= -# test PBKDF1 support -#============================================================================= -class Pbkdf1_Test(TestCase): - """test kdf helpers""" - descriptionPrefix = "pbkdf1" - - pbkdf1_tests = [ - # (password, salt, rounds, keylen, hash, result) - - # - # from http://www.di-mgt.com.au/cryptoKDFs.html - # - (b('password'), hb('78578E5A5D63CB06'), 1000, 16, 'sha1', hb('dc19847e05c64d2faf10ebfb4a3d2a20')), - - # - # custom - # - (b('password'), b('salt'), 1000, 0, 'md5', b('')), - (b('password'), b('salt'), 1000, 1, 'md5', hb('84')), - (b('password'), b('salt'), 1000, 8, 'md5', hb('8475c6a8531a5d27')), - (b('password'), b('salt'), 1000, 16, 'md5', hb('8475c6a8531a5d27e386cd496457812c')), - (b('password'), b('salt'), 1000, None, 'md5', hb('8475c6a8531a5d27e386cd496457812c')), - (b('password'), b('salt'), 1000, None, 'sha1', hb('4a8fd48e426ed081b535be5769892fa396293efb')), - ] - if not (PYPY or JYTHON): - pbkdf1_tests.append( - (b('password'), b('salt'), 1000, None, 'md4', hb('f7f2e91100a8f96190f2dd177cb26453')) - ) - - def test_known(self): - """test reference vectors""" - from passlib.utils.pbkdf2 import pbkdf1 - for secret, salt, rounds, keylen, digest, correct in self.pbkdf1_tests: - result = pbkdf1(secret, salt, rounds, keylen, digest) - self.assertEqual(result, correct) - - def test_border(self): - """test border cases""" - from passlib.utils.pbkdf2 import pbkdf1 - def helper(secret=b('secret'), salt=b('salt'), rounds=1, keylen=1, hash='md5'): - return pbkdf1(secret, salt, rounds, keylen, hash) - helper() - - # salt/secret wrong type - self.assertRaises(TypeError, helper, secret=1) - self.assertRaises(TypeError, helper, salt=1) - - # non-existent hashes - self.assertRaises(ValueError, helper, hash='missing') - - # rounds < 1 and wrong type - self.assertRaises(ValueError, helper, rounds=0) - self.assertRaises(TypeError, helper, rounds='1') - - # keylen < 0, keylen > block_size, and wrong type - self.assertRaises(ValueError, helper, keylen=-1) - self.assertRaises(ValueError, helper, keylen=17, hash='md5') - self.assertRaises(TypeError, helper, keylen='1') - -#============================================================================= -# test PBKDF2 support -#============================================================================= -class _Pbkdf2_Test(TestCase): - """test pbkdf2() support""" - _disable_m2crypto = False - - def setUp(self): - super(_Pbkdf2_Test, self).setUp() - import passlib.utils.pbkdf2 as mod - - # disable m2crypto support, and use software backend - if M2Crypto and self._disable_m2crypto: - self.addCleanup(setattr, mod, "_EVP", mod._EVP) - mod._EVP = None - - # flush cached prf functions, since we're screwing with their backend. - mod._clear_prf_cache() - self.addCleanup(mod._clear_prf_cache) - - pbkdf2_test_vectors = [ - # (result, secret, salt, rounds, keylen, prf="sha1") - - # - # from rfc 3962 - # - - # test case 1 / 128 bit - ( - hb("cdedb5281bb2f801565a1122b2563515"), - b("password"), b("ATHENA.MIT.EDUraeburn"), 1, 16 - ), - - # test case 2 / 128 bit - ( - hb("01dbee7f4a9e243e988b62c73cda935d"), - b("password"), b("ATHENA.MIT.EDUraeburn"), 2, 16 - ), - - # test case 2 / 256 bit - ( - hb("01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86"), - b("password"), b("ATHENA.MIT.EDUraeburn"), 2, 32 - ), - - # test case 3 / 256 bit - ( - hb("5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13"), - b("password"), b("ATHENA.MIT.EDUraeburn"), 1200, 32 - ), - - # test case 4 / 256 bit - ( - hb("d1daa78615f287e6a1c8b120d7062a493f98d203e6be49a6adf4fa574b6e64ee"), - b("password"), b('\x12\x34\x56\x78\x78\x56\x34\x12'), 5, 32 - ), - - # test case 5 / 256 bit - ( - hb("139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1"), - b("X"*64), b("pass phrase equals block size"), 1200, 32 - ), - - # test case 6 / 256 bit - ( - hb("9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a"), - b("X"*65), b("pass phrase exceeds block size"), 1200, 32 - ), - - # - # from rfc 6070 - # - ( - hb("0c60c80f961f0e71f3a9b524af6012062fe037a6"), - b("password"), b("salt"), 1, 20, - ), - - ( - hb("ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"), - b("password"), b("salt"), 2, 20, - ), - - ( - hb("4b007901b765489abead49d926f721d065a429c1"), - b("password"), b("salt"), 4096, 20, - ), - - # just runs too long - could enable if ALL option is set - ##( - ## - ## unhexlify("eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"), - ## "password", "salt", 16777216, 20, - ##), - - ( - hb("3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038"), - b("passwordPASSWORDpassword"), - b("saltSALTsaltSALTsaltSALTsaltSALTsalt"), - 4096, 25, - ), - - ( - hb("56fa6aa75548099dcc37d7f03425e0c3"), - b("pass\00word"), b("sa\00lt"), 4096, 16, - ), - - # - # from example in http://grub.enbug.org/Authentication - # - ( - hb("887CFF169EA8335235D8004242AA7D6187A41E3187DF0CE14E256D85ED" - "97A97357AAA8FF0A3871AB9EEFF458392F462F495487387F685B7472FC" - "6C29E293F0A0"), - b("hello"), - hb("9290F727ED06C38BA4549EF7DE25CF5642659211B7FC076F2D28FEFD71" - "784BB8D8F6FB244A8CC5C06240631B97008565A120764C0EE9C2CB0073" - "994D79080136"), - 10000, 64, "hmac-sha512" - ), - - # - # custom - # - ( - hb('e248fb6b13365146f8ac6307cc222812'), - b("secret"), b("salt"), 10, 16, "hmac-sha1", - ), - ( - hb('e248fb6b13365146f8ac6307cc2228127872da6d'), - b("secret"), b("salt"), 10, None, "hmac-sha1", - ), - - ] - - def test_known(self): - """test reference vectors""" - from passlib.utils.pbkdf2 import pbkdf2 - for row in self.pbkdf2_test_vectors: - correct, secret, salt, rounds, keylen = row[:5] - prf = row[5] if len(row) == 6 else "hmac-sha1" - result = pbkdf2(secret, salt, rounds, keylen, prf) - self.assertEqual(result, correct) - - def test_border(self): - """test border cases""" - from passlib.utils.pbkdf2 import pbkdf2 - def helper(secret=b('password'), salt=b('salt'), rounds=1, keylen=None, prf="hmac-sha1"): - return pbkdf2(secret, salt, rounds, keylen, prf) - helper() - - # invalid rounds - self.assertRaises(ValueError, helper, rounds=0) - self.assertRaises(TypeError, helper, rounds='x') - - # invalid keylen - helper(keylen=0) - self.assertRaises(ValueError, helper, keylen=-1) - self.assertRaises(ValueError, helper, keylen=20*(2**32-1)+1) - self.assertRaises(TypeError, helper, keylen='x') - - # invalid secret/salt type - self.assertRaises(TypeError, helper, salt=5) - self.assertRaises(TypeError, helper, secret=5) - - # invalid hash - self.assertRaises(ValueError, helper, prf='hmac-foo') - self.assertRaises(ValueError, helper, prf='foo') - self.assertRaises(TypeError, helper, prf=5) - - def test_default_keylen(self): - """test keylen==None""" - from passlib.utils.pbkdf2 import pbkdf2 - def helper(secret=b('password'), salt=b('salt'), rounds=1, keylen=None, prf="hmac-sha1"): - return pbkdf2(secret, salt, rounds, keylen, prf) - self.assertEqual(len(helper(prf='hmac-sha1')), 20) - self.assertEqual(len(helper(prf='hmac-sha256')), 32) - - def test_custom_prf(self): - """test custom prf function""" - from passlib.utils.pbkdf2 import pbkdf2 - def prf(key, msg): - return hashlib.md5(key+msg+b('fooey')).digest() - result = pbkdf2(b('secret'), b('salt'), 1000, 20, prf) - self.assertEqual(result, hb('5fe7ce9f7e379d3f65cbc66ba8aa6440474a6849')) - -# create subclasses to test with and without m2crypto -class Pbkdf2_M2Crypto_Test(_Pbkdf2_Test): - descriptionPrefix = "pbkdf2 (m2crypto backend)" -Pbkdf2_M2Crypto_Test = skipUnless(M2Crypto, "M2Crypto not found")(Pbkdf2_M2Crypto_Test) - -class Pbkdf2_Builtin_Test(_Pbkdf2_Test): - descriptionPrefix = "pbkdf2 (builtin backend)" - _disable_m2crypto = True -Pbkdf2_Builtin_Test = skipUnless(TEST_MODE("full") or not M2Crypto, - "skipped under current test mode")(Pbkdf2_Builtin_Test) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_utils_handlers.py b/src/passlib/tests/test_utils_handlers.py deleted file mode 100644 index 19cd4ca9..00000000 --- a/src/passlib/tests/test_utils_handlers.py +++ /dev/null @@ -1,870 +0,0 @@ -"""tests for passlib.hash -- (c) Assurance Technologies 2003-2009""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import re -import hashlib -from logging import getLogger -import warnings -# site -# pkg -from passlib.hash import ldap_md5, sha256_crypt -from passlib.exc import MissingBackendError, PasslibHashWarning -from passlib.utils.compat import str_to_uascii, \ - uascii_to_str, unicode -import passlib.utils.handlers as uh -from passlib.tests.utils import HandlerCase, TestCase -from passlib.utils.compat import u -# module -log = getLogger(__name__) - -#============================================================================= -# utils -#============================================================================= -def _makelang(alphabet, size): - """generate all strings of given size using alphabet""" - def helper(size): - if size < 2: - for char in alphabet: - yield char - else: - for char in alphabet: - for tail in helper(size-1): - yield char+tail - return set(helper(size)) - -#============================================================================= -# test GenericHandler & associates mixin classes -#============================================================================= -class SkeletonTest(TestCase): - """test hash support classes""" - - #=================================================================== - # StaticHandler - #=================================================================== - def test_00_static_handler(self): - """test StaticHandler class""" - - class d1(uh.StaticHandler): - name = "d1" - context_kwds = ("flag",) - _hash_prefix = u("_") - checksum_chars = u("ab") - checksum_size = 1 - - def __init__(self, flag=False, **kwds): - super(d1, self).__init__(**kwds) - self.flag = flag - - def _calc_checksum(self, secret): - return u('b') if self.flag else u('a') - - # check default identify method - self.assertTrue(d1.identify(u('_a'))) - self.assertTrue(d1.identify(b'_a')) - self.assertTrue(d1.identify(u('_b'))) - - self.assertFalse(d1.identify(u('_c'))) - self.assertFalse(d1.identify(b'_c')) - self.assertFalse(d1.identify(u('a'))) - self.assertFalse(d1.identify(u('b'))) - self.assertFalse(d1.identify(u('c'))) - self.assertRaises(TypeError, d1.identify, None) - self.assertRaises(TypeError, d1.identify, 1) - - # check default genconfig method - self.assertEqual(d1.genconfig(), d1.hash("")) - - # check default verify method - self.assertTrue(d1.verify('s', b'_a')) - self.assertTrue(d1.verify('s',u('_a'))) - self.assertFalse(d1.verify('s', b'_b')) - self.assertFalse(d1.verify('s',u('_b'))) - self.assertTrue(d1.verify('s', b'_b', flag=True)) - self.assertRaises(ValueError, d1.verify, 's', b'_c') - self.assertRaises(ValueError, d1.verify, 's', u('_c')) - - # check default hash method - self.assertEqual(d1.hash('s'), '_a') - self.assertEqual(d1.hash('s', flag=True), '_b') - - def test_01_calc_checksum_hack(self): - """test StaticHandler legacy attr""" - # release 1.5 StaticHandler required genhash(), - # not _calc_checksum, be implemented. we have backward compat wrapper, - # this tests that it works. - - class d1(uh.StaticHandler): - name = "d1" - - @classmethod - def identify(cls, hash): - if not hash or len(hash) != 40: - return False - try: - int(hash, 16) - except ValueError: - return False - return True - - @classmethod - def genhash(cls, secret, hash): - if secret is None: - raise TypeError("no secret provided") - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - # NOTE: have to support hash=None since this is test of legacy 1.5 api - if hash is not None and not cls.identify(hash): - raise ValueError("invalid hash") - return hashlib.sha1(b"xyz" + secret).hexdigest() - - @classmethod - def verify(cls, secret, hash): - if hash is None: - raise ValueError("no hash specified") - return cls.genhash(secret, hash) == hash.lower() - - # hash should issue api warnings, but everything else should be fine. - with self.assertWarningList("d1.*should be updated.*_calc_checksum"): - hash = d1.hash("test") - self.assertEqual(hash, '7c622762588a0e5cc786ad0a143156f9fd38eea3') - - self.assertTrue(d1.verify("test", hash)) - self.assertFalse(d1.verify("xtest", hash)) - - # not defining genhash either, however, should cause NotImplementedError - del d1.genhash - self.assertRaises(NotImplementedError, d1.hash, 'test') - - #=================================================================== - # GenericHandler & mixins - #=================================================================== - def test_10_identify(self): - """test GenericHandler.identify()""" - class d1(uh.GenericHandler): - @classmethod - def from_string(cls, hash): - if isinstance(hash, bytes): - hash = hash.decode("ascii") - if hash == u('a'): - return cls(checksum=hash) - else: - raise ValueError - - # check fallback - self.assertRaises(TypeError, d1.identify, None) - self.assertRaises(TypeError, d1.identify, 1) - self.assertFalse(d1.identify('')) - self.assertTrue(d1.identify('a')) - self.assertFalse(d1.identify('b')) - - # check regexp - d1._hash_regex = re.compile(u('@.')) - self.assertRaises(TypeError, d1.identify, None) - self.assertRaises(TypeError, d1.identify, 1) - self.assertTrue(d1.identify('@a')) - self.assertFalse(d1.identify('a')) - del d1._hash_regex - - # check ident-based - d1.ident = u('!') - self.assertRaises(TypeError, d1.identify, None) - self.assertRaises(TypeError, d1.identify, 1) - self.assertTrue(d1.identify('!a')) - self.assertFalse(d1.identify('a')) - del d1.ident - - def test_11_norm_checksum(self): - """test GenericHandler checksum handling""" - # setup helpers - class d1(uh.GenericHandler): - name = 'd1' - checksum_size = 4 - checksum_chars = u('xz') - - def norm_checksum(checksum=None, **k): - return d1(checksum=checksum, **k).checksum - - # too small - self.assertRaises(ValueError, norm_checksum, u('xxx')) - - # right size - self.assertEqual(norm_checksum(u('xxxx')), u('xxxx')) - self.assertEqual(norm_checksum(u('xzxz')), u('xzxz')) - - # too large - self.assertRaises(ValueError, norm_checksum, u('xxxxx')) - - # wrong chars - self.assertRaises(ValueError, norm_checksum, u('xxyx')) - - # wrong type - self.assertRaises(TypeError, norm_checksum, b'xxyx') - - # relaxed - # NOTE: this could be turned back on if we test _norm_checksum() directly... - #with self.assertWarningList("checksum should be unicode"): - # self.assertEqual(norm_checksum(b'xxzx', relaxed=True), u('xxzx')) - #self.assertRaises(TypeError, norm_checksum, 1, relaxed=True) - - # test _stub_checksum behavior - self.assertEqual(d1()._stub_checksum, u('xxxx')) - - def test_12_norm_checksum_raw(self): - """test GenericHandler + HasRawChecksum mixin""" - class d1(uh.HasRawChecksum, uh.GenericHandler): - name = 'd1' - checksum_size = 4 - - def norm_checksum(*a, **k): - return d1(*a, **k).checksum - - # test bytes - self.assertEqual(norm_checksum(b'1234'), b'1234') - - # test unicode - self.assertRaises(TypeError, norm_checksum, u('xxyx')) - - # NOTE: this could be turned back on if we test _norm_checksum() directly... - # self.assertRaises(TypeError, norm_checksum, u('xxyx'), relaxed=True) - - # test _stub_checksum behavior - self.assertEqual(d1()._stub_checksum, b'\x00'*4) - - def test_20_norm_salt(self): - """test GenericHandler + HasSalt mixin""" - # setup helpers - class d1(uh.HasSalt, uh.GenericHandler): - name = 'd1' - setting_kwds = ('salt',) - min_salt_size = 2 - max_salt_size = 4 - default_salt_size = 3 - salt_chars = 'ab' - - def norm_salt(**k): - return d1(**k).salt - - def gen_salt(sz, **k): - return d1.using(salt_size=sz, **k)(use_defaults=True).salt - - salts2 = _makelang('ab', 2) - salts3 = _makelang('ab', 3) - salts4 = _makelang('ab', 4) - - # check salt=None - self.assertRaises(TypeError, norm_salt) - self.assertRaises(TypeError, norm_salt, salt=None) - self.assertIn(norm_salt(use_defaults=True), salts3) - - # check explicit salts - with warnings.catch_warnings(record=True) as wlog: - - # check too-small salts - self.assertRaises(ValueError, norm_salt, salt='') - self.assertRaises(ValueError, norm_salt, salt='a') - self.consumeWarningList(wlog) - - # check correct salts - self.assertEqual(norm_salt(salt='ab'), 'ab') - self.assertEqual(norm_salt(salt='aba'), 'aba') - self.assertEqual(norm_salt(salt='abba'), 'abba') - self.consumeWarningList(wlog) - - # check too-large salts - self.assertRaises(ValueError, norm_salt, salt='aaaabb') - self.consumeWarningList(wlog) - - # check generated salts - with warnings.catch_warnings(record=True) as wlog: - - # check too-small salt size - self.assertRaises(ValueError, gen_salt, 0) - self.assertRaises(ValueError, gen_salt, 1) - self.consumeWarningList(wlog) - - # check correct salt size - self.assertIn(gen_salt(2), salts2) - self.assertIn(gen_salt(3), salts3) - self.assertIn(gen_salt(4), salts4) - self.consumeWarningList(wlog) - - # check too-large salt size - self.assertRaises(ValueError, gen_salt, 5) - self.consumeWarningList(wlog) - - self.assertIn(gen_salt(5, relaxed=True), salts4) - self.consumeWarningList(wlog, ["salt_size.*above max_salt_size"]) - - # test with max_salt_size=None - del d1.max_salt_size - with self.assertWarningList([]): - self.assertEqual(len(gen_salt(None)), 3) - self.assertEqual(len(gen_salt(5)), 5) - - # TODO: test HasRawSalt mixin - - def test_30_init_rounds(self): - """test GenericHandler + HasRounds mixin""" - # setup helpers - class d1(uh.HasRounds, uh.GenericHandler): - name = 'd1' - setting_kwds = ('rounds',) - min_rounds = 1 - max_rounds = 3 - default_rounds = 2 - - # NOTE: really is testing _init_rounds(), could dup to test _norm_rounds() via .replace - def norm_rounds(**k): - return d1(**k).rounds - - # check rounds=None - self.assertRaises(TypeError, norm_rounds) - self.assertRaises(TypeError, norm_rounds, rounds=None) - self.assertEqual(norm_rounds(use_defaults=True), 2) - - # check rounds=non int - self.assertRaises(TypeError, norm_rounds, rounds=1.5) - - # check explicit rounds - with warnings.catch_warnings(record=True) as wlog: - # too small - self.assertRaises(ValueError, norm_rounds, rounds=0) - self.consumeWarningList(wlog) - - # just right - self.assertEqual(norm_rounds(rounds=1), 1) - self.assertEqual(norm_rounds(rounds=2), 2) - self.assertEqual(norm_rounds(rounds=3), 3) - self.consumeWarningList(wlog) - - # too large - self.assertRaises(ValueError, norm_rounds, rounds=4) - self.consumeWarningList(wlog) - - # check no default rounds - d1.default_rounds = None - self.assertRaises(TypeError, norm_rounds, use_defaults=True) - - def test_40_backends(self): - """test GenericHandler + HasManyBackends mixin""" - class d1(uh.HasManyBackends, uh.GenericHandler): - name = 'd1' - setting_kwds = () - - backends = ("a", "b") - - _enable_a = False - _enable_b = False - - @classmethod - def _load_backend_a(cls): - if cls._enable_a: - cls._set_calc_checksum_backend(cls._calc_checksum_a) - return True - else: - return False - - @classmethod - def _load_backend_b(cls): - if cls._enable_b: - cls._set_calc_checksum_backend(cls._calc_checksum_b) - return True - else: - return False - - def _calc_checksum_a(self, secret): - return 'a' - - def _calc_checksum_b(self, secret): - return 'b' - - # test no backends - self.assertRaises(MissingBackendError, d1.get_backend) - self.assertRaises(MissingBackendError, d1.set_backend) - self.assertRaises(MissingBackendError, d1.set_backend, 'any') - self.assertRaises(MissingBackendError, d1.set_backend, 'default') - self.assertFalse(d1.has_backend()) - - # enable 'b' backend - d1._enable_b = True - - # test lazy load - obj = d1() - self.assertEqual(obj._calc_checksum('s'), 'b') - - # test repeat load - d1.set_backend('b') - d1.set_backend('any') - self.assertEqual(obj._calc_checksum('s'), 'b') - - # test unavailable - self.assertRaises(MissingBackendError, d1.set_backend, 'a') - self.assertTrue(d1.has_backend('b')) - self.assertFalse(d1.has_backend('a')) - - # enable 'a' backend also - d1._enable_a = True - - # test explicit - self.assertTrue(d1.has_backend()) - d1.set_backend('a') - self.assertEqual(obj._calc_checksum('s'), 'a') - - # test unknown backend - self.assertRaises(ValueError, d1.set_backend, 'c') - self.assertRaises(ValueError, d1.has_backend, 'c') - - # test error thrown if _has & _load are mixed - d1.set_backend("b") # switch away from 'a' so next call actually checks loader - class d2(d1): - _has_backend_a = True - self.assertRaises(AssertionError, d2.has_backend, "a") - - def test_41_backends(self): - """test GenericHandler + HasManyBackends mixin (deprecated api)""" - warnings.filterwarnings("ignore", - category=DeprecationWarning, - message=r".* support for \._has_backend_.* is deprecated.*", - ) - - class d1(uh.HasManyBackends, uh.GenericHandler): - name = 'd1' - setting_kwds = () - - backends = ("a", "b") - - _has_backend_a = False - _has_backend_b = False - - def _calc_checksum_a(self, secret): - return 'a' - - def _calc_checksum_b(self, secret): - return 'b' - - # test no backends - self.assertRaises(MissingBackendError, d1.get_backend) - self.assertRaises(MissingBackendError, d1.set_backend) - self.assertRaises(MissingBackendError, d1.set_backend, 'any') - self.assertRaises(MissingBackendError, d1.set_backend, 'default') - self.assertFalse(d1.has_backend()) - - # enable 'b' backend - d1._has_backend_b = True - - # test lazy load - obj = d1() - self.assertEqual(obj._calc_checksum('s'), 'b') - - # test repeat load - d1.set_backend('b') - d1.set_backend('any') - self.assertEqual(obj._calc_checksum('s'), 'b') - - # test unavailable - self.assertRaises(MissingBackendError, d1.set_backend, 'a') - self.assertTrue(d1.has_backend('b')) - self.assertFalse(d1.has_backend('a')) - - # enable 'a' backend also - d1._has_backend_a = True - - # test explicit - self.assertTrue(d1.has_backend()) - d1.set_backend('a') - self.assertEqual(obj._calc_checksum('s'), 'a') - - # test unknown backend - self.assertRaises(ValueError, d1.set_backend, 'c') - self.assertRaises(ValueError, d1.has_backend, 'c') - - def test_50_norm_ident(self): - """test GenericHandler + HasManyIdents""" - # setup helpers - class d1(uh.HasManyIdents, uh.GenericHandler): - name = 'd1' - setting_kwds = ('ident',) - default_ident = u("!A") - ident_values = (u("!A"), u("!B")) - ident_aliases = { u("A"): u("!A")} - - def norm_ident(**k): - return d1(**k).ident - - # check ident=None - self.assertRaises(TypeError, norm_ident) - self.assertRaises(TypeError, norm_ident, ident=None) - self.assertEqual(norm_ident(use_defaults=True), u('!A')) - - # check valid idents - self.assertEqual(norm_ident(ident=u('!A')), u('!A')) - self.assertEqual(norm_ident(ident=u('!B')), u('!B')) - self.assertRaises(ValueError, norm_ident, ident=u('!C')) - - # check aliases - self.assertEqual(norm_ident(ident=u('A')), u('!A')) - - # check invalid idents - self.assertRaises(ValueError, norm_ident, ident=u('B')) - - # check identify is honoring ident system - self.assertTrue(d1.identify(u("!Axxx"))) - self.assertTrue(d1.identify(u("!Bxxx"))) - self.assertFalse(d1.identify(u("!Cxxx"))) - self.assertFalse(d1.identify(u("A"))) - self.assertFalse(d1.identify(u(""))) - self.assertRaises(TypeError, d1.identify, None) - self.assertRaises(TypeError, d1.identify, 1) - - # check default_ident missing is detected. - d1.default_ident = None - self.assertRaises(AssertionError, norm_ident, use_defaults=True) - - #=================================================================== - # experimental - the following methods are not finished or tested, - # but way work correctly for some hashes - #=================================================================== - def test_91_parsehash(self): - """test parsehash()""" - # NOTE: this just tests some existing GenericHandler classes - from passlib import hash - - # - # parsehash() - # - - # simple hash w/ salt - result = hash.des_crypt.parsehash("OgAwTx2l6NADI") - self.assertEqual(result, {'checksum': u('AwTx2l6NADI'), 'salt': u('Og')}) - - # parse rounds and extra implicit_rounds flag - h = '$5$LKO/Ute40T3FNF95$U0prpBQd4PloSGU0pnpM4z9wKn4vZ1.jsrzQfPqxph9' - s = u('LKO/Ute40T3FNF95') - c = u('U0prpBQd4PloSGU0pnpM4z9wKn4vZ1.jsrzQfPqxph9') - result = hash.sha256_crypt.parsehash(h) - self.assertEqual(result, dict(salt=s, rounds=5000, - implicit_rounds=True, checksum=c)) - - # omit checksum - result = hash.sha256_crypt.parsehash(h, checksum=False) - self.assertEqual(result, dict(salt=s, rounds=5000, implicit_rounds=True)) - - # sanitize - result = hash.sha256_crypt.parsehash(h, sanitize=True) - self.assertEqual(result, dict(rounds=5000, implicit_rounds=True, - salt=u('LK**************'), - checksum=u('U0pr***************************************'))) - - # parse w/o implicit rounds flag - result = hash.sha256_crypt.parsehash('$5$rounds=10428$uy/jIAhCetNCTtb0$YWvUOXbkqlqhyoPMpN8BMe.ZGsGx2aBvxTvDFI613c3') - self.assertEqual(result, dict( - checksum=u('YWvUOXbkqlqhyoPMpN8BMe.ZGsGx2aBvxTvDFI613c3'), - salt=u('uy/jIAhCetNCTtb0'), - rounds=10428, - )) - - # parsing of raw checksums & salts - h1 = '$pbkdf2$60000$DoEwpvQeA8B4T.k951yLUQ$O26Y3/NJEiLCVaOVPxGXshyjW8k' - result = hash.pbkdf2_sha1.parsehash(h1) - self.assertEqual(result, dict( - checksum=b';n\x98\xdf\xf3I\x12"\xc2U\xa3\x95?\x11\x97\xb2\x1c\xa3[\xc9', - rounds=60000, - salt=b'\x0e\x810\xa6\xf4\x1e\x03\xc0xO\xe9=\xe7\\\x8bQ', - )) - - # sanitizing of raw checksums & salts - result = hash.pbkdf2_sha1.parsehash(h1, sanitize=True) - self.assertEqual(result, dict( - checksum=u('O26************************'), - rounds=60000, - salt=u('Do********************'), - )) - - def test_92_bitsize(self): - """test bitsize()""" - # NOTE: this just tests some existing GenericHandler classes - from passlib import hash - - # no rounds - self.assertEqual(hash.des_crypt.bitsize(), - {'checksum': 66, 'salt': 12}) - - # log2 rounds - self.assertEqual(hash.bcrypt.bitsize(), - {'checksum': 186, 'salt': 132}) - - # linear rounds - # NOTE: +3 comes from int(math.log(.1,2)), - # where 0.1 = 10% = default allowed variation in rounds - self.patchAttr(hash.sha256_crypt, "default_rounds", 1 << (14 + 3)) - self.assertEqual(hash.sha256_crypt.bitsize(), - {'checksum': 258, 'rounds': 14, 'salt': 96}) - - # raw checksum - self.patchAttr(hash.pbkdf2_sha1, "default_rounds", 1 << (13 + 3)) - self.assertEqual(hash.pbkdf2_sha1.bitsize(), - {'checksum': 160, 'rounds': 13, 'salt': 128}) - - # TODO: handle fshp correctly, and other glitches noted in code. - ##self.assertEqual(hash.fshp.bitsize(variant=1), - ## {'checksum': 256, 'rounds': 13, 'salt': 128}) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# PrefixWrapper -#============================================================================= -class dummy_handler_in_registry(object): - """context manager that inserts dummy handler in registry""" - def __init__(self, name): - self.name = name - self.dummy = type('dummy_' + name, (uh.GenericHandler,), dict( - name=name, - setting_kwds=(), - )) - - def __enter__(self): - from passlib import registry - registry._unload_handler_name(self.name, locations=False) - registry.register_crypt_handler(self.dummy) - assert registry.get_crypt_handler(self.name) is self.dummy - return self.dummy - - def __exit__(self, *exc_info): - from passlib import registry - registry._unload_handler_name(self.name, locations=False) - -class PrefixWrapperTest(TestCase): - """test PrefixWrapper class""" - - def test_00_lazy_loading(self): - """test PrefixWrapper lazy loading of handler""" - d1 = uh.PrefixWrapper("d1", "ldap_md5", "{XXX}", "{MD5}", lazy=True) - - # check base state - self.assertEqual(d1._wrapped_name, "ldap_md5") - self.assertIs(d1._wrapped_handler, None) - - # check loading works - self.assertIs(d1.wrapped, ldap_md5) - self.assertIs(d1._wrapped_handler, ldap_md5) - - # replace w/ wrong handler, make sure doesn't reload w/ dummy - with dummy_handler_in_registry("ldap_md5") as dummy: - self.assertIs(d1.wrapped, ldap_md5) - - def test_01_active_loading(self): - """test PrefixWrapper active loading of handler""" - d1 = uh.PrefixWrapper("d1", "ldap_md5", "{XXX}", "{MD5}") - - # check base state - self.assertEqual(d1._wrapped_name, "ldap_md5") - self.assertIs(d1._wrapped_handler, ldap_md5) - self.assertIs(d1.wrapped, ldap_md5) - - # replace w/ wrong handler, make sure doesn't reload w/ dummy - with dummy_handler_in_registry("ldap_md5") as dummy: - self.assertIs(d1.wrapped, ldap_md5) - - def test_02_explicit(self): - """test PrefixWrapper with explicitly specified handler""" - - d1 = uh.PrefixWrapper("d1", ldap_md5, "{XXX}", "{MD5}") - - # check base state - self.assertEqual(d1._wrapped_name, None) - self.assertIs(d1._wrapped_handler, ldap_md5) - self.assertIs(d1.wrapped, ldap_md5) - - # replace w/ wrong handler, make sure doesn't reload w/ dummy - with dummy_handler_in_registry("ldap_md5") as dummy: - self.assertIs(d1.wrapped, ldap_md5) - - def test_10_wrapped_attributes(self): - d1 = uh.PrefixWrapper("d1", "ldap_md5", "{XXX}", "{MD5}") - self.assertEqual(d1.name, "d1") - self.assertIs(d1.setting_kwds, ldap_md5.setting_kwds) - self.assertFalse('max_rounds' in dir(d1)) - - d2 = uh.PrefixWrapper("d2", "sha256_crypt", "{XXX}") - self.assertIs(d2.setting_kwds, sha256_crypt.setting_kwds) - self.assertTrue('max_rounds' in dir(d2)) - - def test_11_wrapped_methods(self): - d1 = uh.PrefixWrapper("d1", "ldap_md5", "{XXX}", "{MD5}") - dph = "{XXX}X03MO1qnZdYdgyfeuILPmQ==" - lph = "{MD5}X03MO1qnZdYdgyfeuILPmQ==" - - # genconfig - self.assertEqual(d1.genconfig(), '{XXX}1B2M2Y8AsgTpgAmY7PhCfg==') - - # genhash - self.assertRaises(TypeError, d1.genhash, "password", None) - self.assertEqual(d1.genhash("password", dph), dph) - self.assertRaises(ValueError, d1.genhash, "password", lph) - - # hash - self.assertEqual(d1.hash("password"), dph) - - # identify - self.assertTrue(d1.identify(dph)) - self.assertFalse(d1.identify(lph)) - - # verify - self.assertRaises(ValueError, d1.verify, "password", lph) - self.assertTrue(d1.verify("password", dph)) - - def test_12_ident(self): - # test ident is proxied - h = uh.PrefixWrapper("h2", "ldap_md5", "{XXX}") - self.assertEqual(h.ident, u("{XXX}{MD5}")) - self.assertIs(h.ident_values, None) - - # test lack of ident means no proxy - h = uh.PrefixWrapper("h2", "des_crypt", "{XXX}") - self.assertIs(h.ident, None) - self.assertIs(h.ident_values, None) - - # test orig_prefix disabled ident proxy - h = uh.PrefixWrapper("h1", "ldap_md5", "{XXX}", "{MD5}") - self.assertIs(h.ident, None) - self.assertIs(h.ident_values, None) - - # test custom ident overrides default - h = uh.PrefixWrapper("h3", "ldap_md5", "{XXX}", ident="{X") - self.assertEqual(h.ident, u("{X")) - self.assertIs(h.ident_values, None) - - # test custom ident must match - h = uh.PrefixWrapper("h3", "ldap_md5", "{XXX}", ident="{XXX}A") - self.assertRaises(ValueError, uh.PrefixWrapper, "h3", "ldap_md5", - "{XXX}", ident="{XY") - self.assertRaises(ValueError, uh.PrefixWrapper, "h3", "ldap_md5", - "{XXX}", ident="{XXXX") - - # test ident_values is proxied - h = uh.PrefixWrapper("h4", "phpass", "{XXX}") - self.assertIs(h.ident, None) - self.assertEqual(h.ident_values, (u("{XXX}$P$"), u("{XXX}$H$"))) - - # test ident=True means use prefix even if hash has no ident. - h = uh.PrefixWrapper("h5", "des_crypt", "{XXX}", ident=True) - self.assertEqual(h.ident, u("{XXX}")) - self.assertIs(h.ident_values, None) - - # ... but requires prefix - self.assertRaises(ValueError, uh.PrefixWrapper, "h6", "des_crypt", ident=True) - - # orig_prefix + HasManyIdent - warning - with self.assertWarningList("orig_prefix.*may not work correctly"): - h = uh.PrefixWrapper("h7", "phpass", orig_prefix="$", prefix="?") - self.assertEqual(h.ident_values, None) # TODO: should output (u("?P$"), u("?H$"))) - self.assertEqual(h.ident, None) - - def test_13_repr(self): - """test repr()""" - h = uh.PrefixWrapper("h2", "md5_crypt", "{XXX}", orig_prefix="$1$") - self.assertRegex(repr(h), - r"""(?x)^PrefixWrapper\( - ['"]h2['"],\s+ - ['"]md5_crypt['"],\s+ - prefix=u?["']{XXX}['"],\s+ - orig_prefix=u?["']\$1\$['"] - \)$""") - - def test_14_bad_hash(self): - """test orig_prefix sanity check""" - # shoudl throw InvalidHashError if wrapped hash doesn't begin - # with orig_prefix. - h = uh.PrefixWrapper("h2", "md5_crypt", orig_prefix="$6$") - self.assertRaises(ValueError, h.hash, 'test') - -#============================================================================= -# sample algorithms - these serve as known quantities -# to test the unittests themselves, as well as other -# parts of passlib. they shouldn't be used as actual password schemes. -#============================================================================= -class UnsaltedHash(uh.StaticHandler): - """test algorithm which lacks a salt""" - name = "unsalted_test_hash" - checksum_chars = uh.LOWER_HEX_CHARS - checksum_size = 40 - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - data = b"boblious" + secret - return str_to_uascii(hashlib.sha1(data).hexdigest()) - -class SaltedHash(uh.HasSalt, uh.GenericHandler): - """test algorithm with a salt""" - name = "salted_test_hash" - setting_kwds = ("salt",) - - min_salt_size = 2 - max_salt_size = 4 - checksum_size = 40 - salt_chars = checksum_chars = uh.LOWER_HEX_CHARS - - _hash_regex = re.compile(u("^@salt[0-9a-f]{42,44}$")) - - @classmethod - def from_string(cls, hash): - if not cls.identify(hash): - raise uh.exc.InvalidHashError(cls) - if isinstance(hash, bytes): - hash = hash.decode("ascii") - return cls(salt=hash[5:-40], checksum=hash[-40:]) - - def to_string(self): - hash = u("@salt%s%s") % (self.salt, self.checksum) - return uascii_to_str(hash) - - def _calc_checksum(self, secret): - if isinstance(secret, unicode): - secret = secret.encode("utf-8") - data = self.salt.encode("ascii") + secret + self.salt.encode("ascii") - return str_to_uascii(hashlib.sha1(data).hexdigest()) - -#============================================================================= -# test sample algorithms - really a self-test of HandlerCase -#============================================================================= - -# TODO: provide data samples for algorithms -# (positive knowns, negative knowns, invalid identify) - -UPASS_TEMP = u('\u0399\u03c9\u03b1\u03bd\u03bd\u03b7\u03c2') - -class UnsaltedHashTest(HandlerCase): - handler = UnsaltedHash - - known_correct_hashes = [ - ("password", "61cfd32684c47de231f1f982c214e884133762c0"), - (UPASS_TEMP, '96b329d120b97ff81ada770042e44ba87343ad2b'), - ] - - def test_bad_kwds(self): - self.assertRaises(TypeError, UnsaltedHash, salt='x') - self.assertRaises(TypeError, UnsaltedHash.genconfig, rounds=1) - -class SaltedHashTest(HandlerCase): - handler = SaltedHash - - known_correct_hashes = [ - ("password", '@salt77d71f8fe74f314dac946766c1ac4a2a58365482c0'), - (UPASS_TEMP, '@salt9f978a9bfe360d069b0c13f2afecd570447407fa7e48'), - ] - - def test_bad_kwds(self): - stub = SaltedHash(use_defaults=True)._stub_checksum - self.assertRaises(TypeError, SaltedHash, checksum=stub, salt=None) - self.assertRaises(ValueError, SaltedHash, checksum=stub, salt='xxx') - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_utils_md4.py b/src/passlib/tests/test_utils_md4.py deleted file mode 100644 index 5d824a19..00000000 --- a/src/passlib/tests/test_utils_md4.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -passlib.tests -- tests for passlib.utils.md4 - -.. warning:: - - This module & it's functions have been deprecated, and superceded - by the functions in passlib.crypto. This file is being maintained - until the deprecated functions are removed, and is only present prevent - historical regressions up to that point. New and more thorough testing - is being done by the replacement tests in ``test_utils_crypto_builtin_md4``. -""" -#============================================================================= -# imports -#============================================================================= -# core -import warnings -# site -# pkg -# module -from passlib.tests.test_crypto_builtin_md4 import _Common_MD4_Test -# local -__all__ = [ - "Legacy_MD4_Test", -] -#============================================================================= -# test pure-python MD4 implementation -#============================================================================= -class Legacy_MD4_Test(_Common_MD4_Test): - descriptionPrefix = "passlib.utils.md4.md4()" - - def setUp(self): - super(Legacy_MD4_Test, self).setUp() - warnings.filterwarnings("ignore", ".*passlib.utils.md4.*deprecated", DeprecationWarning) - - def get_md4_const(self): - from passlib.utils.md4 import md4 - return md4 - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_utils_pbkdf2.py b/src/passlib/tests/test_utils_pbkdf2.py deleted file mode 100644 index 3b2bd09f..00000000 --- a/src/passlib/tests/test_utils_pbkdf2.py +++ /dev/null @@ -1,322 +0,0 @@ -""" -passlib.tests -- tests for passlib.utils.pbkdf2 - -.. warning:: - - This module & it's functions have been deprecated, and superceded - by the functions in passlib.crypto. This file is being maintained - until the deprecated functions are removed, and is only present prevent - historical regressions up to that point. New and more thorough testing - is being done by the replacement tests in ``test_utils_crypto.py``. -""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -import hashlib -import warnings -# site -# pkg -# module -from passlib.utils.compat import u, JYTHON -from passlib.tests.utils import TestCase, hb - -#============================================================================= -# test assorted crypto helpers -#============================================================================= -class UtilsTest(TestCase): - """test various utils functions""" - descriptionPrefix = "passlib.utils.pbkdf2" - - ndn_formats = ["hashlib", "iana"] - ndn_values = [ - # (iana name, hashlib name, ... other unnormalized names) - ("md5", "md5", "SCRAM-MD5-PLUS", "MD-5"), - ("sha1", "sha-1", "SCRAM-SHA-1", "SHA1"), - ("sha256", "sha-256", "SHA_256", "sha2-256"), - ("ripemd", "ripemd", "SCRAM-RIPEMD", "RIPEMD"), - ("ripemd160", "ripemd-160", - "SCRAM-RIPEMD-160", "RIPEmd160"), - ("test128", "test-128", "TEST128"), - ("test2", "test2", "TEST-2"), - ("test3_128", "test3-128", "TEST-3-128"), - ] - - def setUp(self): - super(UtilsTest, self).setUp() - warnings.filterwarnings("ignore", ".*passlib.utils.pbkdf2.*deprecated", DeprecationWarning) - - def test_norm_hash_name(self): - """norm_hash_name()""" - from itertools import chain - from passlib.utils.pbkdf2 import norm_hash_name - from passlib.crypto.digest import _known_hash_names - - # test formats - for format in self.ndn_formats: - norm_hash_name("md4", format) - self.assertRaises(ValueError, norm_hash_name, "md4", None) - self.assertRaises(ValueError, norm_hash_name, "md4", "fake") - - # test types - self.assertEqual(norm_hash_name(u("MD4")), "md4") - self.assertEqual(norm_hash_name(b"MD4"), "md4") - self.assertRaises(TypeError, norm_hash_name, None) - - # test selected results - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", '.*unknown hash') - for row in chain(_known_hash_names, self.ndn_values): - for idx, format in enumerate(self.ndn_formats): - correct = row[idx] - for value in row: - result = norm_hash_name(value, format) - self.assertEqual(result, correct, - "name=%r, format=%r:" % (value, - format)) - -#============================================================================= -# test PBKDF1 support -#============================================================================= -class Pbkdf1_Test(TestCase): - """test kdf helpers""" - descriptionPrefix = "passlib.utils.pbkdf2.pbkdf1()" - - pbkdf1_tests = [ - # (password, salt, rounds, keylen, hash, result) - - # - # from http://www.di-mgt.com.au/cryptoKDFs.html - # - (b'password', hb('78578E5A5D63CB06'), 1000, 16, 'sha1', hb('dc19847e05c64d2faf10ebfb4a3d2a20')), - - # - # custom - # - (b'password', b'salt', 1000, 0, 'md5', b''), - (b'password', b'salt', 1000, 1, 'md5', hb('84')), - (b'password', b'salt', 1000, 8, 'md5', hb('8475c6a8531a5d27')), - (b'password', b'salt', 1000, 16, 'md5', hb('8475c6a8531a5d27e386cd496457812c')), - (b'password', b'salt', 1000, None, 'md5', hb('8475c6a8531a5d27e386cd496457812c')), - (b'password', b'salt', 1000, None, 'sha1', hb('4a8fd48e426ed081b535be5769892fa396293efb')), - ] - if not JYTHON: - pbkdf1_tests.append( - (b'password', b'salt', 1000, None, 'md4', hb('f7f2e91100a8f96190f2dd177cb26453')) - ) - - def setUp(self): - super(Pbkdf1_Test, self).setUp() - warnings.filterwarnings("ignore", ".*passlib.utils.pbkdf2.*deprecated", DeprecationWarning) - - def test_known(self): - """test reference vectors""" - from passlib.utils.pbkdf2 import pbkdf1 - for secret, salt, rounds, keylen, digest, correct in self.pbkdf1_tests: - result = pbkdf1(secret, salt, rounds, keylen, digest) - self.assertEqual(result, correct) - - def test_border(self): - """test border cases""" - from passlib.utils.pbkdf2 import pbkdf1 - def helper(secret=b'secret', salt=b'salt', rounds=1, keylen=1, hash='md5'): - return pbkdf1(secret, salt, rounds, keylen, hash) - helper() - - # salt/secret wrong type - self.assertRaises(TypeError, helper, secret=1) - self.assertRaises(TypeError, helper, salt=1) - - # non-existent hashes - self.assertRaises(ValueError, helper, hash='missing') - - # rounds < 1 and wrong type - self.assertRaises(ValueError, helper, rounds=0) - self.assertRaises(TypeError, helper, rounds='1') - - # keylen < 0, keylen > block_size, and wrong type - self.assertRaises(ValueError, helper, keylen=-1) - self.assertRaises(ValueError, helper, keylen=17, hash='md5') - self.assertRaises(TypeError, helper, keylen='1') - -#============================================================================= -# test PBKDF2 support -#============================================================================= -class Pbkdf2_Test(TestCase): - """test pbkdf2() support""" - descriptionPrefix = "passlib.utils.pbkdf2.pbkdf2()" - - pbkdf2_test_vectors = [ - # (result, secret, salt, rounds, keylen, prf="sha1") - - # - # from rfc 3962 - # - - # test case 1 / 128 bit - ( - hb("cdedb5281bb2f801565a1122b2563515"), - b"password", b"ATHENA.MIT.EDUraeburn", 1, 16 - ), - - # test case 2 / 128 bit - ( - hb("01dbee7f4a9e243e988b62c73cda935d"), - b"password", b"ATHENA.MIT.EDUraeburn", 2, 16 - ), - - # test case 2 / 256 bit - ( - hb("01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86"), - b"password", b"ATHENA.MIT.EDUraeburn", 2, 32 - ), - - # test case 3 / 256 bit - ( - hb("5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13"), - b"password", b"ATHENA.MIT.EDUraeburn", 1200, 32 - ), - - # test case 4 / 256 bit - ( - hb("d1daa78615f287e6a1c8b120d7062a493f98d203e6be49a6adf4fa574b6e64ee"), - b"password", b'\x12\x34\x56\x78\x78\x56\x34\x12', 5, 32 - ), - - # test case 5 / 256 bit - ( - hb("139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1"), - b"X"*64, b"pass phrase equals block size", 1200, 32 - ), - - # test case 6 / 256 bit - ( - hb("9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a"), - b"X"*65, b"pass phrase exceeds block size", 1200, 32 - ), - - # - # from rfc 6070 - # - ( - hb("0c60c80f961f0e71f3a9b524af6012062fe037a6"), - b"password", b"salt", 1, 20, - ), - - ( - hb("ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"), - b"password", b"salt", 2, 20, - ), - - ( - hb("4b007901b765489abead49d926f721d065a429c1"), - b"password", b"salt", 4096, 20, - ), - - # just runs too long - could enable if ALL option is set - ##( - ## - ## unhexlify("eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"), - ## "password", "salt", 16777216, 20, - ##), - - ( - hb("3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038"), - b"passwordPASSWORDpassword", - b"saltSALTsaltSALTsaltSALTsaltSALTsalt", - 4096, 25, - ), - - ( - hb("56fa6aa75548099dcc37d7f03425e0c3"), - b"pass\00word", b"sa\00lt", 4096, 16, - ), - - # - # from example in http://grub.enbug.org/Authentication - # - ( - hb("887CFF169EA8335235D8004242AA7D6187A41E3187DF0CE14E256D85ED" - "97A97357AAA8FF0A3871AB9EEFF458392F462F495487387F685B7472FC" - "6C29E293F0A0"), - b"hello", - hb("9290F727ED06C38BA4549EF7DE25CF5642659211B7FC076F2D28FEFD71" - "784BB8D8F6FB244A8CC5C06240631B97008565A120764C0EE9C2CB0073" - "994D79080136"), - 10000, 64, "hmac-sha512" - ), - - # - # custom - # - ( - hb('e248fb6b13365146f8ac6307cc222812'), - b"secret", b"salt", 10, 16, "hmac-sha1", - ), - ( - hb('e248fb6b13365146f8ac6307cc2228127872da6d'), - b"secret", b"salt", 10, None, "hmac-sha1", - ), - - ] - - def setUp(self): - super(Pbkdf2_Test, self).setUp() - warnings.filterwarnings("ignore", ".*passlib.utils.pbkdf2.*deprecated", DeprecationWarning) - - def test_known(self): - """test reference vectors""" - from passlib.utils.pbkdf2 import pbkdf2 - for row in self.pbkdf2_test_vectors: - correct, secret, salt, rounds, keylen = row[:5] - prf = row[5] if len(row) == 6 else "hmac-sha1" - result = pbkdf2(secret, salt, rounds, keylen, prf) - self.assertEqual(result, correct) - - def test_border(self): - """test border cases""" - from passlib.utils.pbkdf2 import pbkdf2 - def helper(secret=b'password', salt=b'salt', rounds=1, keylen=None, prf="hmac-sha1"): - return pbkdf2(secret, salt, rounds, keylen, prf) - helper() - - # invalid rounds - self.assertRaises(ValueError, helper, rounds=-1) - self.assertRaises(ValueError, helper, rounds=0) - self.assertRaises(TypeError, helper, rounds='x') - - # invalid keylen - self.assertRaises(ValueError, helper, keylen=-1) - self.assertRaises(ValueError, helper, keylen=0) - helper(keylen=1) - self.assertRaises(OverflowError, helper, keylen=20*(2**32-1)+1) - self.assertRaises(TypeError, helper, keylen='x') - - # invalid secret/salt type - self.assertRaises(TypeError, helper, salt=5) - self.assertRaises(TypeError, helper, secret=5) - - # invalid hash - self.assertRaises(ValueError, helper, prf='hmac-foo') - self.assertRaises(NotImplementedError, helper, prf='foo') - self.assertRaises(TypeError, helper, prf=5) - - def test_default_keylen(self): - """test keylen==None""" - from passlib.utils.pbkdf2 import pbkdf2 - def helper(secret=b'password', salt=b'salt', rounds=1, keylen=None, prf="hmac-sha1"): - return pbkdf2(secret, salt, rounds, keylen, prf) - self.assertEqual(len(helper(prf='hmac-sha1')), 20) - self.assertEqual(len(helper(prf='hmac-sha256')), 32) - - def test_custom_prf(self): - """test custom prf function""" - from passlib.utils.pbkdf2 import pbkdf2 - def prf(key, msg): - return hashlib.md5(key+msg+b'fooey').digest() - self.assertRaises(NotImplementedError, pbkdf2, b'secret', b'salt', 1000, 20, prf) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/test_win32.py b/src/passlib/tests/test_win32.py deleted file mode 100644 index e818b62b..00000000 --- a/src/passlib/tests/test_win32.py +++ /dev/null @@ -1,50 +0,0 @@ -"""tests for passlib.win32 -- (c) Assurance Technologies 2003-2009""" -#============================================================================= -# imports -#============================================================================= -# core -import warnings -# site -# pkg -from passlib.tests.utils import TestCase -# module -from passlib.utils.compat import u - -#============================================================================= -# -#============================================================================= -class UtilTest(TestCase): - """test util funcs in passlib.win32""" - - ##test hashes from http://msdn.microsoft.com/en-us/library/cc245828(v=prot.10).aspx - ## among other places - - def setUp(self): - super(UtilTest, self).setUp() - warnings.filterwarnings("ignore", - "the 'passlib.win32' module is deprecated") - - def test_lmhash(self): - from passlib.win32 import raw_lmhash - for secret, hash in [ - ("OLDPASSWORD", u("c9b81d939d6fd80cd408e6b105741864")), - ("NEWPASSWORD", u('09eeab5aa415d6e4d408e6b105741864')), - ("welcome", u("c23413a8a1e7665faad3b435b51404ee")), - ]: - result = raw_lmhash(secret, hex=True) - self.assertEqual(result, hash) - - def test_nthash(self): - warnings.filterwarnings("ignore", - r"nthash\.raw_nthash\(\) is deprecated") - from passlib.win32 import raw_nthash - for secret, hash in [ - ("OLDPASSWORD", u("6677b2c394311355b54f25eec5bfacf5")), - ("NEWPASSWORD", u("256781a62031289d3c2c98c14f1efc8c")), - ]: - result = raw_nthash(secret, hex=True) - self.assertEqual(result, hash) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/tox_support.py b/src/passlib/tests/tox_support.py deleted file mode 100644 index 43170bc4..00000000 --- a/src/passlib/tests/tox_support.py +++ /dev/null @@ -1,83 +0,0 @@ -"""passlib.tests.tox_support - helper script for tox tests""" -#============================================================================= -# init script env -#============================================================================= -import os, sys -root_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) -sys.path.insert(0, root_dir) - -#============================================================================= -# imports -#============================================================================= -# core -import re -import logging; log = logging.getLogger(__name__) -# site -# pkg -from passlib.utils.compat import print_ -# local -__all__ = [ -] - -#============================================================================= -# main -#============================================================================= -TH_PATH = "passlib.tests.test_handlers" - -def do_hash_tests(*args): - """return list of hash algorithm tests that match regexes""" - if not args: - print(TH_PATH) - return - suffix = '' - args = list(args) - while True: - if args[0] == "--method": - suffix = '.' + args[1] - del args[:2] - else: - break - from passlib.tests import test_handlers - names = [TH_PATH + ":" + name + suffix for name in dir(test_handlers) - if not name.startswith("_") and any(re.match(arg,name) for arg in args)] - print_("\n".join(names)) - return not names - -def do_preset_tests(name): - """return list of preset test names""" - if name == "django" or name == "django-hashes": - do_hash_tests("django_.*_test", "hex_md5_test") - if name == "django": - print_("passlib.tests.test_ext_django") - else: - raise ValueError("unknown name: %r" % name) - -def do_setup_gae(path, runtime): - """write fake GAE ``app.yaml`` to current directory so nosegae will work""" - from passlib.tests.utils import set_file - set_file(os.path.join(path, "app.yaml"), """\ -application: fake-app -version: 2 -runtime: %s -api_version: 1 -threadsafe: no - -handlers: -- url: /.* - script: dummy.py - -libraries: -- name: django - version: "latest" -""" % runtime) - -def main(cmd, *args): - return globals()["do_" + cmd](*args) - -if __name__ == "__main__": - import sys - sys.exit(main(*sys.argv[1:]) or 0) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/tests/utils.py b/src/passlib/tests/utils.py deleted file mode 100644 index 1ac52fdd..00000000 --- a/src/passlib/tests/utils.py +++ /dev/null @@ -1,3401 +0,0 @@ -"""helpers for passlib unittests""" -#============================================================================= -# imports -#============================================================================= -from __future__ import with_statement -# core -from binascii import unhexlify -import contextlib -from functools import wraps, partial -import hashlib -import logging; log = logging.getLogger(__name__) -import random -import re -import os -import sys -import tempfile -import threading -import time -from passlib.exc import PasslibHashWarning, PasslibConfigWarning -from passlib.utils.compat import PY3, JYTHON -import warnings -from warnings import warn -# site -# pkg -from passlib import exc -from passlib.exc import MissingBackendError -import passlib.registry as registry -from passlib.tests.backports import TestCase as _TestCase, skip, skipIf, skipUnless, SkipTest -from passlib.utils import has_rounds_info, has_salt_info, rounds_cost_values, \ - rng as sys_rng, getrandstr, is_ascii_safe, to_native_str, \ - repeat_string, tick, batch -from passlib.utils.compat import iteritems, irange, u, unicode, PY2 -from passlib.utils.decor import classproperty -import passlib.utils.handlers as uh -# local -__all__ = [ - # util funcs - 'TEST_MODE', - 'set_file', 'get_file', - - # unit testing - 'TestCase', - 'HandlerCase', -] - -#============================================================================= -# environment detection -#============================================================================= -# figure out if we're running under GAE; -# some tests (e.g. FS writing) should be skipped. -# XXX: is there better way to do this? -try: - import google.appengine -except ImportError: - GAE = False -else: - GAE = True - -def ensure_mtime_changed(path): - """ensure file's mtime has changed""" - # NOTE: this is hack to deal w/ filesystems whose mtime resolution is >= 1s, - # when a test needs to be sure the mtime changed after writing to the file. - last = os.path.getmtime(path) - while os.path.getmtime(path) == last: - time.sleep(0.1) - os.utime(path, None) - -def _get_timer_resolution(timer): - def sample(): - start = cur = timer() - while start == cur: - cur = timer() - return cur-start - return min(sample() for _ in range(3)) -TICK_RESOLUTION = _get_timer_resolution(tick) - -#============================================================================= -# test mode -#============================================================================= -_TEST_MODES = ["quick", "default", "full"] -_test_mode = _TEST_MODES.index(os.environ.get("PASSLIB_TEST_MODE", - "default").strip().lower()) - -def TEST_MODE(min=None, max=None): - """check if test for specified mode should be enabled. - - ``"quick"`` - run the bare minimum tests to ensure functionality. - variable-cost hashes are tested at their lowest setting. - hash algorithms are only tested against the backend that will - be used on the current host. no fuzz testing is done. - - ``"default"`` - same as ``"quick"``, except: hash algorithms are tested - at default levels, and a brief round of fuzz testing is done - for each hash. - - ``"full"`` - extra regression and internal tests are enabled, hash algorithms are tested - against all available backends, unavailable ones are mocked whre possible, - additional time is devoted to fuzz testing. - """ - if min and _test_mode < _TEST_MODES.index(min): - return False - if max and _test_mode > _TEST_MODES.index(max): - return False - return True - -#============================================================================= -# hash object inspection -#============================================================================= -def has_relaxed_setting(handler): - """check if handler supports 'relaxed' kwd""" - # FIXME: I've been lazy, should probably just add 'relaxed' kwd - # to all handlers that derive from GenericHandler - - # ignore wrapper classes for now.. though could introspec. - if hasattr(handler, "orig_prefix"): - return False - - return 'relaxed' in handler.setting_kwds or issubclass(handler, - uh.GenericHandler) - -def get_effective_rounds(handler, rounds=None): - """get effective rounds value from handler""" - handler = unwrap_handler(handler) - return handler(rounds=rounds, use_defaults=True).rounds - -def is_default_backend(handler, backend): - """check if backend is the default for source""" - try: - orig = handler.get_backend() - except MissingBackendError: - return False - try: - handler.set_backend("default") - return handler.get_backend() == backend - finally: - handler.set_backend(orig) - -def iter_alt_backends(handler, current=None, fallback=False): - """ - iterate over alternate backends available to handler. - - .. warning:: - not thread-safe due to has_backend() call - """ - if current is None: - current = handler.get_backend() - backends = handler.backends - idx = backends.index(current)+1 if fallback else 0 - for backend in backends[idx:]: - if backend != current and handler.has_backend(backend): - yield backend - -def get_alt_backend(*args, **kwds): - for backend in iter_alt_backends(*args, **kwds): - return backend - return None - -def unwrap_handler(handler): - """return original handler, removing any wrapper objects""" - while hasattr(handler, "wrapped"): - handler = handler.wrapped - return handler - -def handler_derived_from(handler, base): - """ - test if was derived from via . - """ - # XXX: need way to do this more formally via ifc, - # for now just hacking in the cases we encounter in testing. - if handler == base: - return True - elif isinstance(handler, uh.PrefixWrapper): - while handler: - if handler == base: - return True - # helper set by PrefixWrapper().using() just for this case... - handler = handler._derived_from - return False - elif isinstance(handler, type) and issubclass(handler, uh.MinimalHandler): - return issubclass(handler, base) - else: - raise NotImplementedError("don't know how to inspect handler: %r" % (handler,)) - -@contextlib.contextmanager -def patch_calc_min_rounds(handler): - """ - internal helper for do_config_encrypt() -- - context manager which temporarily replaces handler's _calc_checksum() - with one that uses min_rounds; useful when trying to generate config - with high rounds value, but don't care if output is correct. - """ - if isinstance(handler, type) and issubclass(handler, uh.HasRounds): - # XXX: also require GenericHandler for this branch? - wrapped = handler._calc_checksum - def wrapper(self, *args, **kwds): - rounds = self.rounds - try: - self.rounds = self.min_rounds - return wrapped(self, *args, **kwds) - finally: - self.rounds = rounds - handler._calc_checksum = wrapper - try: - yield - finally: - handler._calc_checksum = wrapped - elif isinstance(handler, uh.PrefixWrapper): - with patch_calc_min_rounds(handler.wrapped): - yield - else: - yield - return - -#============================================================================= -# misc helpers -#============================================================================= -def set_file(path, content): - """set file to specified bytes""" - if isinstance(content, unicode): - content = content.encode("utf-8") - with open(path, "wb") as fh: - fh.write(content) - -def get_file(path): - """read file as bytes""" - with open(path, "rb") as fh: - return fh.read() - -def tonn(source): - """convert native string to non-native string""" - if not isinstance(source, str): - return source - elif PY3: - return source.encode("utf-8") - else: - try: - return source.decode("utf-8") - except UnicodeDecodeError: - return source.decode("latin-1") - -def hb(source): - """ - helper for represent byte strings in hex. - - usage: ``hb("deadbeef23")`` - """ - return unhexlify(re.sub(r"\s", "", source)) - -def limit(value, lower, upper): - if value < lower: - return lower - elif value > upper: - return upper - return value - -def quicksleep(delay): - """because time.sleep() doesn't even have 10ms accuracy on some OSes""" - start = tick() - while tick()-start < delay: - pass - -def time_call(func, setup=None, maxtime=1, bestof=10): - """ - timeit() wrapper which tries to get as accurate a measurement as possible w/in maxtime seconds. - - :returns: - ``(avg_seconds_per_call, log10_number_of_repetitions)`` - """ - from timeit import Timer - from math import log - timer = Timer(func, setup=setup or '') - number = 1 - end = tick() + maxtime - while True: - delta = min(timer.repeat(bestof, number)) - if tick() >= end: - return delta/number, int(log(number, 10)) - number *= 10 - -def run_with_fixed_seeds(count=128, master_seed=0x243F6A8885A308D3): - """ - decorator run test method w/ multiple fixed seeds. - """ - def builder(func): - @wraps(func) - def wrapper(*args, **kwds): - rng = random.Random(master_seed) - for _ in irange(count): - kwds['seed'] = rng.getrandbits(32) - func(*args, **kwds) - return wrapper - return builder - -#============================================================================= -# custom test harness -#============================================================================= - -class TestCase(_TestCase): - """passlib-specific test case class - - this class adds a number of features to the standard TestCase... - * common prefix for all test descriptions - * resets warnings filter & registry for every test - * tweaks to message formatting - * __msg__ kwd added to assertRaises() - * suite of methods for matching against warnings - """ - #=================================================================== - # add various custom features - #=================================================================== - - #--------------------------------------------------------------- - # make it easy for test cases to add common prefix to shortDescription - #--------------------------------------------------------------- - - # string prepended to all tests in TestCase - descriptionPrefix = None - - def shortDescription(self): - """wrap shortDescription() method to prepend descriptionPrefix""" - desc = super(TestCase, self).shortDescription() - prefix = self.descriptionPrefix - if prefix: - desc = "%s: %s" % (prefix, desc or str(self)) - return desc - - #--------------------------------------------------------------- - # hack things so nose and ut2 both skip subclasses who have - # "__unittest_skip=True" set, or whose names start with "_" - #--------------------------------------------------------------- - @classproperty - def __unittest_skip__(cls): - # NOTE: this attr is technically a unittest2 internal detail. - name = cls.__name__ - return name.startswith("_") or \ - getattr(cls, "_%s__unittest_skip" % name, False) - - @classproperty - def __test__(cls): - # make nose just proxy __unittest_skip__ - return not cls.__unittest_skip__ - - # flag to skip *this* class - __unittest_skip = True - - #--------------------------------------------------------------- - # reset warning filters & registry before each test - #--------------------------------------------------------------- - - # flag to reset all warning filters & ignore state - resetWarningState = True - - def setUp(self): - super(TestCase, self).setUp() - self.setUpWarnings() - - def setUpWarnings(self): - """helper to init warning filters before subclass setUp()""" - if self.resetWarningState: - ctx = reset_warnings() - ctx.__enter__() - self.addCleanup(ctx.__exit__) - - # ignore warnings about PasswordHash features deprecated in 1.7 - # TODO: should be cleaned in 2.0, when support will be dropped. - # should be kept until then, so we test the legacy paths. - warnings.filterwarnings("ignore", r"the method .*\.(encrypt|genconfig|genhash)\(\) is deprecated") - warnings.filterwarnings("ignore", r"the 'vary_rounds' option is deprecated") - - #--------------------------------------------------------------- - # tweak message formatting so longMessage mode is only enabled - # if msg ends with ":", and turn on longMessage by default. - #--------------------------------------------------------------- - longMessage = True - - def _formatMessage(self, msg, std): - if self.longMessage and msg and msg.rstrip().endswith(":"): - return '%s %s' % (msg.rstrip(), std) - else: - return msg or std - - #--------------------------------------------------------------- - # override assertRaises() to support '__msg__' keyword, - # and to return the caught exception for further examination - #--------------------------------------------------------------- - def assertRaises(self, _exc_type, _callable=None, *args, **kwds): - msg = kwds.pop("__msg__", None) - if _callable is None: - # FIXME: this ignores 'msg' - return super(TestCase, self).assertRaises(_exc_type, None, - *args, **kwds) - try: - result = _callable(*args, **kwds) - except _exc_type as err: - return err - std = "function returned %r, expected it to raise %r" % (result, - _exc_type) - raise self.failureException(self._formatMessage(msg, std)) - - #--------------------------------------------------------------- - # forbid a bunch of deprecated aliases so I stop using them - #--------------------------------------------------------------- - def assertEquals(self, *a, **k): - raise AssertionError("this alias is deprecated by unittest2") - assertNotEquals = assertRegexMatches = assertEquals - - #=================================================================== - # custom methods for matching warnings - #=================================================================== - def assertWarning(self, warning, - message_re=None, message=None, - category=None, - filename_re=None, filename=None, - lineno=None, - msg=None, - ): - """check if warning matches specified parameters. - 'warning' is the instance of Warning to match against; - can also be instance of WarningMessage (as returned by catch_warnings). - """ - # check input type - if hasattr(warning, "category"): - # resolve WarningMessage -> Warning, but preserve original - wmsg = warning - warning = warning.message - else: - # no original WarningMessage, passed raw Warning - wmsg = None - - # tests that can use a warning instance or WarningMessage object - if message: - self.assertEqual(str(warning), message, msg) - if message_re: - self.assertRegex(str(warning), message_re, msg) - if category: - self.assertIsInstance(warning, category, msg) - - # tests that require a WarningMessage object - if filename or filename_re: - if not wmsg: - raise TypeError("matching on filename requires a " - "WarningMessage instance") - real = wmsg.filename - if real.endswith(".pyc") or real.endswith(".pyo"): - # FIXME: should use a stdlib call to resolve this back - # to module's original filename. - real = real[:-1] - if filename: - self.assertEqual(real, filename, msg) - if filename_re: - self.assertRegex(real, filename_re, msg) - if lineno: - if not wmsg: - raise TypeError("matching on lineno requires a " - "WarningMessage instance") - self.assertEqual(wmsg.lineno, lineno, msg) - - class _AssertWarningList(warnings.catch_warnings): - """context manager for assertWarningList()""" - def __init__(self, case, **kwds): - self.case = case - self.kwds = kwds - self.__super = super(TestCase._AssertWarningList, self) - self.__super.__init__(record=True) - - def __enter__(self): - self.log = self.__super.__enter__() - - def __exit__(self, *exc_info): - self.__super.__exit__(*exc_info) - if exc_info[0] is None: - self.case.assertWarningList(self.log, **self.kwds) - - def assertWarningList(self, wlist=None, desc=None, msg=None): - """check that warning list (e.g. from catch_warnings) matches pattern""" - if desc is None: - assert wlist is not None - return self._AssertWarningList(self, desc=wlist, msg=msg) - # TODO: make this display better diff of *which* warnings did not match - assert desc is not None - if not isinstance(desc, (list,tuple)): - desc = [desc] - for idx, entry in enumerate(desc): - if isinstance(entry, str): - entry = dict(message_re=entry) - elif isinstance(entry, type) and issubclass(entry, Warning): - entry = dict(category=entry) - elif not isinstance(entry, dict): - raise TypeError("entry must be str, warning, or dict") - try: - data = wlist[idx] - except IndexError: - break - self.assertWarning(data, msg=msg, **entry) - else: - if len(wlist) == len(desc): - return - std = "expected %d warnings, found %d: wlist=%s desc=%r" % \ - (len(desc), len(wlist), self._formatWarningList(wlist), desc) - raise self.failureException(self._formatMessage(msg, std)) - - def consumeWarningList(self, wlist, desc=None, *args, **kwds): - """[deprecated] assertWarningList() variant that clears list afterwards""" - if desc is None: - desc = [] - self.assertWarningList(wlist, desc, *args, **kwds) - del wlist[:] - - def _formatWarning(self, entry): - tail = "" - if hasattr(entry, "message"): - # WarningMessage instance. - tail = " filename=%r lineno=%r" % (entry.filename, entry.lineno) - if entry.line: - tail += " line=%r" % (entry.line,) - entry = entry.message - cls = type(entry) - return "<%s.%s message=%r%s>" % (cls.__module__, cls.__name__, - str(entry), tail) - - def _formatWarningList(self, wlist): - return "[%s]" % ", ".join(self._formatWarning(entry) for entry in wlist) - - #=================================================================== - # capability tests - #=================================================================== - def require_stringprep(self): - """helper to skip test if stringprep is missing""" - from passlib.utils import stringprep - if not stringprep: - from passlib.utils import _stringprep_missing_reason - raise self.skipTest("not available - stringprep module is " + - _stringprep_missing_reason) - - def require_TEST_MODE(self, level): - """skip test for all PASSLIB_TEST_MODE values below """ - if not TEST_MODE(level): - raise self.skipTest("requires >= %r test mode" % level) - - def require_writeable_filesystem(self): - """skip test if writeable FS not available""" - if GAE: - return self.skipTest("GAE doesn't offer read/write filesystem access") - - #=================================================================== - # reproducible random helpers - #=================================================================== - - #: global thread lock for random state - #: XXX: could split into global & per-instance locks if need be - _random_global_lock = threading.Lock() - - #: cache of global seed value, initialized on first call to getRandom() - _random_global_seed = None - - #: per-instance cache of name -> RNG - _random_cache = None - - def getRandom(self, name="default", seed=None): - """ - Return a :class:`random.Random` object for current test method to use. - Within an instance, multiple calls with the same name will return - the same object. - - When first created, each RNG will be seeded with value derived from - a global seed, the test class module & name, the current test method name, - and the **name** parameter. - - The global seed taken from the $RANDOM_TEST_SEED env var, - the $PYTHONHASHSEED env var, or a randomly generated the - first time this method is called. In all cases, the value - is logged for reproducibility. - - :param name: - name to uniquely identify separate RNGs w/in a test - (e.g. for threaded tests). - - :param seed: - override global seed when initialzing rng. - - :rtype: random.Random - """ - # check cache - cache = self._random_cache - if cache and name in cache: - return cache[name] - - with self._random_global_lock: - - # check cache again, and initialize it - cache = self._random_cache - if cache and name in cache: - return cache[name] - elif not cache: - cache = self._random_cache = {} - - # init global seed - global_seed = seed or TestCase._random_global_seed - if global_seed is None: - # NOTE: checking PYTHONHASHSEED, because if that's set, - # the test runner wants something reproducible. - global_seed = TestCase._random_global_seed = \ - int(os.environ.get("RANDOM_TEST_SEED") or - os.environ.get("PYTHONHASHSEED") or - sys_rng.getrandbits(32)) - # XXX: would it be better to print() this? - log.info("using RANDOM_TEST_SEED=%d", global_seed) - - # create seed - cls = type(self) - source = "\n".join([str(global_seed), cls.__module__, cls.__name__, - self._testMethodName, name]) - digest = hashlib.sha256(source.encode("utf-8")).hexdigest() - seed = int(digest[:16], 16) - - # create rng - value = cache[name] = random.Random(seed) - return value - - #=================================================================== - # other - #=================================================================== - _mktemp_queue = None - - def mktemp(self, *args, **kwds): - """create temp file that's cleaned up at end of test""" - self.require_writeable_filesystem() - fd, path = tempfile.mkstemp(*args, **kwds) - os.close(fd) - queue = self._mktemp_queue - if queue is None: - queue = self._mktemp_queue = [] - def cleaner(): - for path in queue: - if os.path.exists(path): - os.remove(path) - del queue[:] - self.addCleanup(cleaner) - queue.append(path) - return path - - def patchAttr(self, obj, attr, value, require_existing=True, wrap=False): - """monkeypatch object value, restoring original value on cleanup""" - try: - orig = getattr(obj, attr) - except AttributeError: - if require_existing: - raise - def cleanup(): - try: - delattr(obj, attr) - except AttributeError: - pass - self.addCleanup(cleanup) - else: - self.addCleanup(setattr, obj, attr, orig) - if wrap: - value = partial(value, orig) - wraps(orig)(value) - setattr(obj, attr, value) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# other unittest helpers -#============================================================================= -RESERVED_BACKEND_NAMES = ["any", "default"] - -class HandlerCase(TestCase): - """base class for testing password hash handlers (esp passlib.utils.handlers subclasses) - - In order to use this to test a handler, - create a subclass will all the appropriate attributes - filled as listed in the example below, - and run the subclass via unittest. - - .. todo:: - - Document all of the options HandlerCase offers. - - .. note:: - - This is subclass of :class:`unittest.TestCase` - (or :class:`unittest2.TestCase` if available). - """ - #=================================================================== - # class attrs - should be filled in by subclass - #=================================================================== - - #--------------------------------------------------------------- - # handler setup - #--------------------------------------------------------------- - - # handler class to test [required] - handler = None - - # if set, run tests against specified backend - backend = None - - #--------------------------------------------------------------- - # test vectors - #--------------------------------------------------------------- - - # list of (secret, hash) tuples which are known to be correct - known_correct_hashes = [] - - # list of (config, secret, hash) tuples are known to be correct - known_correct_configs = [] - - # list of (alt_hash, secret, hash) tuples, where alt_hash is a hash - # using an alternate representation that should be recognized and verify - # correctly, but should be corrected to match hash when passed through - # genhash() - known_alternate_hashes = [] - - # hashes so malformed they aren't even identified properly - known_unidentified_hashes = [] - - # hashes which are identifiabled but malformed - they should identify() - # as True, but cause an error when passed to genhash/verify. - known_malformed_hashes = [] - - # list of (handler name, hash) pairs for other algorithm's hashes that - # handler shouldn't identify as belonging to it this list should generally - # be sufficient (if handler name in list, that entry will be skipped) - known_other_hashes = [ - ('des_crypt', '6f8c114b58f2c'), - ('md5_crypt', '$1$dOHYPKoP$tnxS1T8Q6VVn3kpV8cN6o.'), - ('sha512_crypt', "$6$rounds=123456$asaltof16chars..$BtCwjqMJGx5hrJhZywW" - "vt0RLE8uZ4oPwcelCjmw2kSYu.Ec6ycULevoBK25fs2xXgMNrCzIMVcgEJAstJeonj1"), - ] - - # passwords used to test basic hash behavior - generally - # don't need to be overidden. - stock_passwords = [ - u("test"), - u("\u20AC\u00A5$"), - b'\xe2\x82\xac\xc2\xa5$' - ] - - #--------------------------------------------------------------- - # option flags - #--------------------------------------------------------------- - - # whether hash is case insensitive - # True, False, or special value "verify-only" (which indicates - # hash contains case-sensitive portion, but verifies is case-insensitive) - secret_case_insensitive = False - - # flag if scheme accepts ALL hash strings (e.g. plaintext) - accepts_all_hashes = False - - # flag if scheme has "is_disabled" set, and contains 'salted' data - disabled_contains_salt = False - - # flag/hack to filter PasslibHashWarning issued by test_72_configs() - filter_config_warnings = False - - # forbid certain characters in passwords - @classproperty - def forbidden_characters(cls): - # anything that supports crypt() interface should forbid null chars, - # since crypt() uses null-terminated strings. - if 'os_crypt' in getattr(cls.handler, "backends", ()): - return b"\x00" - return None - - #=================================================================== - # internal class attrs - #=================================================================== - __unittest_skip = True - - @property - def descriptionPrefix(self): - handler = self.handler - name = handler.name - if hasattr(handler, "get_backend"): - name += " (%s backend)" % (handler.get_backend(),) - return name - - #=================================================================== - # support methods - #=================================================================== - - #--------------------------------------------------------------- - # configuration helpers - #--------------------------------------------------------------- - @classmethod - def iter_known_hashes(cls): - """iterate through known (secret, hash) pairs""" - for secret, hash in cls.known_correct_hashes: - yield secret, hash - for config, secret, hash in cls.known_correct_configs: - yield secret, hash - for alt, secret, hash in cls.known_alternate_hashes: - yield secret, hash - - def get_sample_hash(self): - """test random sample secret/hash pair""" - known = list(self.iter_known_hashes()) - return self.getRandom().choice(known) - - #--------------------------------------------------------------- - # test helpers - #--------------------------------------------------------------- - def check_verify(self, secret, hash, msg=None, negate=False): - """helper to check verify() outcome, honoring is_disabled_handler""" - result = self.do_verify(secret, hash) - self.assertTrue(result is True or result is False, - "verify() returned non-boolean value: %r" % (result,)) - if self.handler.is_disabled or negate: - if not result: - return - if not msg: - msg = ("verify incorrectly returned True: secret=%r, hash=%r" % - (secret, hash)) - raise self.failureException(msg) - else: - if result: - return - if not msg: - msg = "verify failed: secret=%r, hash=%r" % (secret, hash) - raise self.failureException(msg) - - def check_returned_native_str(self, result, func_name): - self.assertIsInstance(result, str, - "%s() failed to return native string: %r" % (func_name, result,)) - - #--------------------------------------------------------------- - # PasswordHash helpers - wraps all calls to PasswordHash api, - # so that subclasses can fill in defaults and account for other specialized behavior - #--------------------------------------------------------------- - def populate_settings(self, kwds): - """subclassable method to populate default settings""" - # use lower rounds settings for certain test modes - handler = self.handler - if 'rounds' in handler.setting_kwds and 'rounds' not in kwds: - mn = handler.min_rounds - df = handler.default_rounds - if TEST_MODE(max="quick"): - # use minimum rounds for quick mode - kwds['rounds'] = max(3, mn) - else: - # use default/16 otherwise - factor = 3 - if getattr(handler, "rounds_cost", None) == "log2": - df -= factor - else: - df //= (1<= 1") - - # check min_salt_size - if cls.min_salt_size < 0: - raise AssertionError("min_salt_chars must be >= 0") - if mx_set and cls.min_salt_size > cls.max_salt_size: - raise AssertionError("min_salt_chars must be <= max_salt_chars") - - # check default_salt_size - if cls.default_salt_size < cls.min_salt_size: - raise AssertionError("default_salt_size must be >= min_salt_size") - if mx_set and cls.default_salt_size > cls.max_salt_size: - raise AssertionError("default_salt_size must be <= max_salt_size") - - # check for 'salt_size' keyword - # NOTE: skipping warning if default salt size is already maxed out - # (might change that in future) - if 'salt_size' not in cls.setting_kwds and (not mx_set or cls.default_salt_size < cls.max_salt_size): - warn('%s: hash handler supports range of salt sizes, ' - 'but doesn\'t offer \'salt_size\' setting' % (cls.name,)) - - # check salt_chars & default_salt_chars - if cls.salt_chars: - if not cls.default_salt_chars: - raise AssertionError("default_salt_chars must not be empty") - for c in cls.default_salt_chars: - if c not in cls.salt_chars: - raise AssertionError("default_salt_chars must be subset of salt_chars: %r not in salt_chars" % (c,)) - else: - if not cls.default_salt_chars: - raise AssertionError("default_salt_chars MUST be specified if salt_chars is empty") - - @property - def salt_bits(self): - """calculate number of salt bits in hash""" - # XXX: replace this with bitsize() method? - handler = self.handler - assert has_salt_info(handler), "need explicit bit-size for " + handler.name - from math import log - # FIXME: this may be off for case-insensitive hashes, but that accounts - # for ~1 bit difference, which is good enough for test_11() - return int(handler.default_salt_size * - log(len(handler.default_salt_chars), 2)) - - def test_11_unique_salt(self): - """test hash() / genconfig() creates new salt each time""" - self.require_salt() - # odds of picking 'n' identical salts at random is '(.5**salt_bits)**n'. - # we want to pick the smallest N needed s.t. odds are <1/10**d, just - # to eliminate false-positives. which works out to n>3.33+d-salt_bits. - # for 1/1e12 odds, n=1 is sufficient for most hashes, but a few border cases (e.g. - # cisco_type7) have < 16 bits of salt, requiring more. - samples = max(1, 4 + 12 - self.salt_bits) - - def sampler(func): - value1 = func() - for _ in irange(samples): - value2 = func() - if value1 != value2: - return - raise self.failureException("failed to find different salt after " - "%d samples" % (samples,)) - sampler(self.do_genconfig) - sampler(lambda: self.do_encrypt("stub")) - - def test_12_min_salt_size(self): - """test hash() / genconfig() honors min_salt_size""" - self.require_salt_info() - - handler = self.handler - salt_char = handler.salt_chars[0:1] - min_size = handler.min_salt_size - - # - # check min is accepted - # - s1 = salt_char * min_size - self.do_genconfig(salt=s1) - - self.do_encrypt('stub', salt_size=min_size) - - # - # check min-1 is rejected - # - if min_size > 0: - self.assertRaises(ValueError, self.do_genconfig, - salt=s1[:-1]) - - self.assertRaises(ValueError, self.do_encrypt, 'stub', - salt_size=min_size-1) - - def test_13_max_salt_size(self): - """test hash() / genconfig() honors max_salt_size""" - self.require_salt_info() - - handler = self.handler - max_size = handler.max_salt_size - salt_char = handler.salt_chars[0:1] - - # NOTE: skipping this for hashes like argon2 since max_salt_size takes WAY too much memory - if max_size is None or max_size > (1 << 20): - # - # if it's not set, salt should never be truncated; so test it - # with an unreasonably large salt. - # - s1 = salt_char * 1024 - c1 = self.do_stub_encrypt(salt=s1) - c2 = self.do_stub_encrypt(salt=s1 + salt_char) - self.assertNotEqual(c1, c2) - - self.do_stub_encrypt(salt_size=1024) - - else: - # - # check max size is accepted - # - s1 = salt_char * max_size - c1 = self.do_stub_encrypt(salt=s1) - - self.do_stub_encrypt(salt_size=max_size) - - # - # check max size + 1 is rejected - # - s2 = s1 + salt_char - self.assertRaises(ValueError, self.do_stub_encrypt, salt=s2) - - self.assertRaises(ValueError, self.do_stub_encrypt, salt_size=max_size + 1) - - # - # should accept too-large salt in relaxed mode - # - if has_relaxed_setting(handler): - with warnings.catch_warnings(record=True): # issues passlibhandlerwarning - c2 = self.do_stub_encrypt(salt=s2, relaxed=True) - self.assertEqual(c2, c1) - - # - # if min_salt supports it, check smaller than mx is NOT truncated - # - if handler.min_salt_size < max_size: - c3 = self.do_stub_encrypt(salt=s1[:-1]) - self.assertNotEqual(c3, c1) - - # whether salt should be passed through bcrypt repair function - fuzz_salts_need_bcrypt_repair = False - - def prepare_salt(self, salt): - """prepare generated salt""" - if self.fuzz_salts_need_bcrypt_repair: - from passlib.utils.binary import bcrypt64 - salt = bcrypt64.repair_unused(salt) - return salt - - def test_14_salt_chars(self): - """test hash() honors salt_chars""" - self.require_salt_info() - - handler = self.handler - mx = handler.max_salt_size - mn = handler.min_salt_size - cs = handler.salt_chars - raw = isinstance(cs, bytes) - - # make sure all listed chars are accepted - for salt in batch(cs, mx or 32): - if len(salt) < mn: - salt = repeat_string(salt, mn) - salt = self.prepare_salt(salt) - self.do_stub_encrypt(salt=salt) - - # check some invalid salt chars, make sure they're rejected - source = u('\x00\xff') - if raw: - source = source.encode("latin-1") - chunk = max(mn, 1) - for c in source: - if c not in cs: - self.assertRaises(ValueError, self.do_stub_encrypt, salt=c*chunk, - __msg__="invalid salt char %r:" % (c,)) - - @property - def salt_type(self): - """hack to determine salt keyword's datatype""" - # NOTE: cisco_type7 uses 'int' - if getattr(self.handler, "_salt_is_bytes", False): - return bytes - else: - return unicode - - def test_15_salt_type(self): - """test non-string salt values""" - self.require_salt() - salt_type = self.salt_type - salt_size = getattr(self.handler, "min_salt_size", 0) or 8 - - # should always throw error for random class. - class fake(object): - pass - self.assertRaises(TypeError, self.do_encrypt, 'stub', salt=fake()) - - # unicode should be accepted only if salt_type is unicode. - if salt_type is not unicode: - self.assertRaises(TypeError, self.do_encrypt, 'stub', salt=u('x') * salt_size) - - # bytes should be accepted only if salt_type is bytes, - # OR if salt type is unicode and running PY2 - to allow native strings. - if not (salt_type is bytes or (PY2 and salt_type is unicode)): - self.assertRaises(TypeError, self.do_encrypt, 'stub', salt=b'x' * salt_size) - - def test_using_salt_size(self): - """Handler.using() -- default_salt_size""" - self.require_salt_info() - - handler = self.handler - mn = handler.min_salt_size - mx = handler.max_salt_size - df = handler.default_salt_size - - # should prevent setting below handler limit - self.assertRaises(ValueError, handler.using, default_salt_size=-1) - with self.assertWarningList([PasslibHashWarning]): - temp = handler.using(default_salt_size=-1, relaxed=True) - self.assertEqual(temp.default_salt_size, mn) - - # should prevent setting above handler limit - if mx: - self.assertRaises(ValueError, handler.using, default_salt_size=mx+1) - with self.assertWarningList([PasslibHashWarning]): - temp = handler.using(default_salt_size=mx+1, relaxed=True) - self.assertEqual(temp.default_salt_size, mx) - - # try setting to explicit value - if mn != mx: - temp = handler.using(default_salt_size=mn+1) - self.assertEqual(temp.default_salt_size, mn+1) - self.assertEqual(handler.default_salt_size, df) - - temp = handler.using(default_salt_size=mn+2) - self.assertEqual(temp.default_salt_size, mn+2) - self.assertEqual(handler.default_salt_size, df) - - # accept strings - if mn == mx: - ref = mn - else: - ref = mn + 1 - temp = handler.using(default_salt_size=str(ref)) - self.assertEqual(temp.default_salt_size, ref) - - # reject invalid strings - self.assertRaises(ValueError, handler.using, default_salt_size=str(ref) + "xxx") - - # honor 'salt_size' alias - temp = handler.using(salt_size=ref) - self.assertEqual(temp.default_salt_size, ref) - - #=================================================================== - # rounds - #=================================================================== - def require_rounds_info(self): - if not has_rounds_info(self.handler): - raise self.skipTest("handler lacks rounds attributes") - - def test_20_optional_rounds_attributes(self): - """validate optional rounds attributes""" - self.require_rounds_info() - - cls = self.handler - AssertionError = self.failureException - - # check max_rounds - if cls.max_rounds is None: - raise AssertionError("max_rounds not specified") - if cls.max_rounds < 1: - raise AssertionError("max_rounds must be >= 1") - - # check min_rounds - if cls.min_rounds < 0: - raise AssertionError("min_rounds must be >= 0") - if cls.min_rounds > cls.max_rounds: - raise AssertionError("min_rounds must be <= max_rounds") - - # check default_rounds - if cls.default_rounds is not None: - if cls.default_rounds < cls.min_rounds: - raise AssertionError("default_rounds must be >= min_rounds") - if cls.default_rounds > cls.max_rounds: - raise AssertionError("default_rounds must be <= max_rounds") - - # check rounds_cost - if cls.rounds_cost not in rounds_cost_values: - raise AssertionError("unknown rounds cost constant: %r" % (cls.rounds_cost,)) - - def test_21_min_rounds(self): - """test hash() / genconfig() honors min_rounds""" - self.require_rounds_info() - handler = self.handler - min_rounds = handler.min_rounds - - # check min is accepted - self.do_genconfig(rounds=min_rounds) - self.do_encrypt('stub', rounds=min_rounds) - - # check min-1 is rejected - self.assertRaises(ValueError, self.do_genconfig, rounds=min_rounds-1) - self.assertRaises(ValueError, self.do_encrypt, 'stub', rounds=min_rounds-1) - - # TODO: check relaxed mode clips min-1 - - def test_21b_max_rounds(self): - """test hash() / genconfig() honors max_rounds""" - self.require_rounds_info() - handler = self.handler - max_rounds = handler.max_rounds - - if max_rounds is not None: - # check max+1 is rejected - self.assertRaises(ValueError, self.do_genconfig, rounds=max_rounds+1) - self.assertRaises(ValueError, self.do_encrypt, 'stub', rounds=max_rounds+1) - - # handle max rounds - if max_rounds is None: - self.do_stub_encrypt(rounds=(1 << 31) - 1) - else: - self.do_stub_encrypt(rounds=max_rounds) - - # TODO: check relaxed mode clips max+1 - - #-------------------------------------------------------------------------------------- - # HasRounds.using() / .needs_update() -- desired rounds limits - #-------------------------------------------------------------------------------------- - def _create_using_rounds_helper(self): - """ - setup test helpers for testing handler.using()'s rounds parameters. - """ - self.require_rounds_info() - handler = self.handler - - if handler.name == "bsdi_crypt": - # hack to bypass bsdi-crypt's "odd rounds only" behavior, messes up this test - orig_handler = handler - handler = handler.using() - handler._generate_rounds = classmethod(lambda cls: super(orig_handler, cls)._generate_rounds()) - - # create some fake values to test with - orig_min_rounds = handler.min_rounds - orig_max_rounds = handler.max_rounds - orig_default_rounds = handler.default_rounds - medium = ((orig_max_rounds or 9999) + orig_min_rounds) // 2 - if medium == orig_default_rounds: - medium += 1 - small = (orig_min_rounds + medium) // 2 - large = ((orig_max_rounds or 9999) + medium) // 2 - - if handler.name == "bsdi_crypt": - # hack to avoid even numbered rounds - small |= 1 - medium |= 1 - large |= 1 - adj = 2 - else: - adj = 1 - - # create a subclass with small/medium/large as new default desired values - with self.assertWarningList([]): - subcls = handler.using( - min_desired_rounds=small, - max_desired_rounds=large, - default_rounds=medium, - ) - - # return helpers - return handler, subcls, small, medium, large, adj - - def test_has_rounds_using_harness(self): - """ - HasRounds.using() -- sanity check test harness - """ - # setup helpers - self.require_rounds_info() - handler = self.handler - orig_min_rounds = handler.min_rounds - orig_max_rounds = handler.max_rounds - orig_default_rounds = handler.default_rounds - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - - # shouldn't affect original handler at all - self.assertEqual(handler.min_rounds, orig_min_rounds) - self.assertEqual(handler.max_rounds, orig_max_rounds) - self.assertEqual(handler.min_desired_rounds, None) - self.assertEqual(handler.max_desired_rounds, None) - self.assertEqual(handler.default_rounds, orig_default_rounds) - - # should affect subcls' desired value, but not hard min/max - self.assertEqual(subcls.min_rounds, orig_min_rounds) - self.assertEqual(subcls.max_rounds, orig_max_rounds) - self.assertEqual(subcls.default_rounds, medium) - self.assertEqual(subcls.min_desired_rounds, small) - self.assertEqual(subcls.max_desired_rounds, large) - - def test_has_rounds_using_w_min_rounds(self): - """ - HasRounds.using() -- min_rounds / min_desired_rounds - """ - # setup helpers - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - orig_min_rounds = handler.min_rounds - orig_max_rounds = handler.max_rounds - orig_default_rounds = handler.default_rounds - - # .using() should clip values below valid minimum, w/ warning - if orig_min_rounds > 0: - self.assertRaises(ValueError, handler.using, min_desired_rounds=orig_min_rounds - adj) - with self.assertWarningList([PasslibHashWarning]): - temp = handler.using(min_desired_rounds=orig_min_rounds - adj, relaxed=True) - self.assertEqual(temp.min_desired_rounds, orig_min_rounds) - - # .using() should clip values above valid maximum, w/ warning - if orig_max_rounds: - self.assertRaises(ValueError, handler.using, min_desired_rounds=orig_max_rounds + adj) - with self.assertWarningList([PasslibHashWarning]): - temp = handler.using(min_desired_rounds=orig_max_rounds + adj, relaxed=True) - self.assertEqual(temp.min_desired_rounds, orig_max_rounds) - - # .using() should allow values below previous desired minimum, w/o warning - with self.assertWarningList([]): - temp = subcls.using(min_desired_rounds=small - adj) - self.assertEqual(temp.min_desired_rounds, small - adj) - - # .using() should allow values w/in previous range - temp = subcls.using(min_desired_rounds=small + 2 * adj) - self.assertEqual(temp.min_desired_rounds, small + 2 * adj) - - # .using() should allow values above previous desired maximum, w/o warning - with self.assertWarningList([]): - temp = subcls.using(min_desired_rounds=large + adj) - self.assertEqual(temp.min_desired_rounds, large + adj) - - # hash() etc should allow explicit values below desired minimum - # NOTE: formerly issued a warning in passlib 1.6, now just a wrapper for .using() - self.assertEqual(get_effective_rounds(subcls, small + adj), small + adj) - self.assertEqual(get_effective_rounds(subcls, small), small) - with self.assertWarningList([]): - self.assertEqual(get_effective_rounds(subcls, small - adj), small - adj) - - # 'min_rounds' should be treated as alias for 'min_desired_rounds' - temp = handler.using(min_rounds=small) - self.assertEqual(temp.min_desired_rounds, small) - - # should be able to specify strings - temp = handler.using(min_rounds=str(small)) - self.assertEqual(temp.min_desired_rounds, small) - - # invalid strings should cause error - self.assertRaises(ValueError, handler.using, min_rounds=str(small) + "xxx") - - def test_has_rounds_replace_w_max_rounds(self): - """ - HasRounds.using() -- max_rounds / max_desired_rounds - """ - # setup helpers - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - orig_min_rounds = handler.min_rounds - orig_max_rounds = handler.max_rounds - - # .using() should clip values below valid minimum w/ warning - if orig_min_rounds > 0: - self.assertRaises(ValueError, handler.using, max_desired_rounds=orig_min_rounds - adj) - with self.assertWarningList([PasslibHashWarning]): - temp = handler.using(max_desired_rounds=orig_min_rounds - adj, relaxed=True) - self.assertEqual(temp.max_desired_rounds, orig_min_rounds) - - # .using() should clip values above valid maximum, w/ warning - if orig_max_rounds: - self.assertRaises(ValueError, handler.using, max_desired_rounds=orig_max_rounds + adj) - with self.assertWarningList([PasslibHashWarning]): - temp = handler.using(max_desired_rounds=orig_max_rounds + adj, relaxed=True) - self.assertEqual(temp.max_desired_rounds, orig_max_rounds) - - # .using() should clip values below previous minimum, w/ warning - with self.assertWarningList([PasslibConfigWarning]): - temp = subcls.using(max_desired_rounds=small - adj) - self.assertEqual(temp.max_desired_rounds, small) - - # .using() should reject explicit min > max - self.assertRaises(ValueError, subcls.using, - min_desired_rounds=medium+adj, - max_desired_rounds=medium-adj) - - # .using() should allow values w/in previous range - temp = subcls.using(min_desired_rounds=large - 2 * adj) - self.assertEqual(temp.min_desired_rounds, large - 2 * adj) - - # .using() should allow values above previous desired maximum, w/o warning - with self.assertWarningList([]): - temp = subcls.using(max_desired_rounds=large + adj) - self.assertEqual(temp.max_desired_rounds, large + adj) - - # hash() etc should allow explicit values above desired minimum, w/o warning - # NOTE: formerly issued a warning in passlib 1.6, now just a wrapper for .using() - self.assertEqual(get_effective_rounds(subcls, large - adj), large - adj) - self.assertEqual(get_effective_rounds(subcls, large), large) - with self.assertWarningList([]): - self.assertEqual(get_effective_rounds(subcls, large + adj), large + adj) - - # 'max_rounds' should be treated as alias for 'max_desired_rounds' - temp = handler.using(max_rounds=large) - self.assertEqual(temp.max_desired_rounds, large) - - # should be able to specify strings - temp = handler.using(max_desired_rounds=str(large)) - self.assertEqual(temp.max_desired_rounds, large) - - # invalid strings should cause error - self.assertRaises(ValueError, handler.using, max_desired_rounds=str(large) + "xxx") - - def test_has_rounds_using_w_default_rounds(self): - """ - HasRounds.using() -- default_rounds - """ - # setup helpers - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - orig_max_rounds = handler.max_rounds - - # XXX: are there any other cases that need testing? - - # implicit default rounds -- increase to min_rounds - temp = subcls.using(min_rounds=medium+adj) - self.assertEqual(temp.default_rounds, medium+adj) - - # implicit default rounds -- decrease to max_rounds - temp = subcls.using(max_rounds=medium-adj) - self.assertEqual(temp.default_rounds, medium-adj) - - # explicit default rounds below desired minimum - # XXX: make this a warning if min is implicit? - self.assertRaises(ValueError, subcls.using, default_rounds=small-adj) - - # explicit default rounds above desired maximum - # XXX: make this a warning if max is implicit? - if orig_max_rounds: - self.assertRaises(ValueError, subcls.using, default_rounds=large+adj) - - # hash() etc should implicit default rounds, but get overridden - self.assertEqual(get_effective_rounds(subcls), medium) - self.assertEqual(get_effective_rounds(subcls, medium+adj), medium+adj) - - # should be able to specify strings - temp = handler.using(default_rounds=str(medium)) - self.assertEqual(temp.default_rounds, medium) - - # invalid strings should cause error - self.assertRaises(ValueError, handler.using, default_rounds=str(medium) + "xxx") - - def test_has_rounds_using_w_rounds(self): - """ - HasRounds.using() -- rounds - """ - # setup helpers - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - orig_max_rounds = handler.max_rounds - - # 'rounds' should be treated as fallback for min, max, and default - temp = subcls.using(rounds=medium+adj) - self.assertEqual(temp.min_desired_rounds, medium+adj) - self.assertEqual(temp.default_rounds, medium+adj) - self.assertEqual(temp.max_desired_rounds, medium+adj) - - # 'rounds' should be treated as fallback for min, max, and default - temp = subcls.using(rounds=medium+1, min_rounds=small+adj, - default_rounds=medium, max_rounds=large-adj) - self.assertEqual(temp.min_desired_rounds, small+adj) - self.assertEqual(temp.default_rounds, medium) - self.assertEqual(temp.max_desired_rounds, large-adj) - - def test_has_rounds_using_w_vary_rounds_parsing(self): - """ - HasRounds.using() -- vary_rounds parsing - """ - # setup helpers - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - - def parse(value): - return subcls.using(vary_rounds=value).vary_rounds - - # floats should be preserved - self.assertEqual(parse(0.1), 0.1) - self.assertEqual(parse('0.1'), 0.1) - - # 'xx%' should be converted to float - self.assertEqual(parse('10%'), 0.1) - - # ints should be preserved - self.assertEqual(parse(1000), 1000) - self.assertEqual(parse('1000'), 1000) - - # float bounds should be enforced - self.assertRaises(ValueError, parse, -0.1) - self.assertRaises(ValueError, parse, 1.1) - - def test_has_rounds_using_w_vary_rounds_generation(self): - """ - HasRounds.using() -- vary_rounds generation - """ - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - - def get_effective_range(cls): - seen = set(get_effective_rounds(cls) for _ in irange(1000)) - return min(seen), max(seen) - - def assert_rounds_range(vary_rounds, lower, upper): - temp = subcls.using(vary_rounds=vary_rounds) - seen_lower, seen_upper = get_effective_range(temp) - self.assertEqual(seen_lower, lower, "vary_rounds had wrong lower limit:") - self.assertEqual(seen_upper, upper, "vary_rounds had wrong upper limit:") - - # test static - assert_rounds_range(0, medium, medium) - assert_rounds_range("0%", medium, medium) - - # test absolute - assert_rounds_range(adj, medium - adj, medium + adj) - assert_rounds_range(50, max(small, medium - 50), min(large, medium + 50)) - - # test relative - should shift over at 50% mark - if handler.rounds_cost == "log2": - # log rounds "50%" variance should only increase/decrease by 1 cost value - assert_rounds_range("1%", medium, medium) - assert_rounds_range("49%", medium, medium) - assert_rounds_range("50%", medium - adj, medium) - else: - # for linear rounds, range is frequently so huge, won't ever see ends. - # so we just check it's within an expected range. - lower, upper = get_effective_range(subcls.using(vary_rounds="50%")) - - self.assertGreaterEqual(lower, max(small, medium * 0.5)) - self.assertLessEqual(lower, max(small, medium * 0.8)) - - self.assertGreaterEqual(upper, min(large, medium * 1.2)) - self.assertLessEqual(upper, min(large, medium * 1.5)) - - def test_has_rounds_using_and_needs_update(self): - """ - HasRounds.using() -- desired_rounds + needs_update() - """ - handler, subcls, small, medium, large, adj = self._create_using_rounds_helper() - - temp = subcls.using(min_desired_rounds=small+2, max_desired_rounds=large-2) - - # generate some sample hashes - small_hash = self.do_stub_encrypt(subcls, rounds=small) - medium_hash = self.do_stub_encrypt(subcls, rounds=medium) - large_hash = self.do_stub_encrypt(subcls, rounds=large) - - # everything should be w/in bounds for original handler - self.assertFalse(subcls.needs_update(small_hash)) - self.assertFalse(subcls.needs_update(medium_hash)) - self.assertFalse(subcls.needs_update(large_hash)) - - # small & large should require update for temp handler - self.assertTrue(temp.needs_update(small_hash)) - self.assertFalse(temp.needs_update(medium_hash)) - self.assertTrue(temp.needs_update(large_hash)) - - #=================================================================== - # idents - #=================================================================== - def require_many_idents(self): - handler = self.handler - if not isinstance(handler, type) or not issubclass(handler, uh.HasManyIdents): - raise self.skipTest("handler doesn't derive from HasManyIdents") - - def test_30_HasManyIdents(self): - """validate HasManyIdents configuration""" - cls = self.handler - self.require_many_idents() - - # check settings - self.assertTrue('ident' in cls.setting_kwds) - - # check ident_values list - for value in cls.ident_values: - self.assertIsInstance(value, unicode, - "cls.ident_values must be unicode:") - self.assertTrue(len(cls.ident_values)>1, - "cls.ident_values must have 2+ elements:") - - # check default_ident value - self.assertIsInstance(cls.default_ident, unicode, - "cls.default_ident must be unicode:") - self.assertTrue(cls.default_ident in cls.ident_values, - "cls.default_ident must specify member of cls.ident_values") - - # check optional aliases list - if cls.ident_aliases: - for alias, ident in iteritems(cls.ident_aliases): - self.assertIsInstance(alias, unicode, - "cls.ident_aliases keys must be unicode:") # XXX: allow ints? - self.assertIsInstance(ident, unicode, - "cls.ident_aliases values must be unicode:") - self.assertTrue(ident in cls.ident_values, - "cls.ident_aliases must map to cls.ident_values members: %r" % (ident,)) - - # check constructor validates ident correctly. - handler = cls - hash = self.get_sample_hash()[1] - kwds = handler.parsehash(hash) - del kwds['ident'] - - # ... accepts good ident - handler(ident=cls.default_ident, **kwds) - - # ... requires ident w/o defaults - self.assertRaises(TypeError, handler, **kwds) - - # ... supplies default ident - handler(use_defaults=True, **kwds) - - # ... rejects bad ident - self.assertRaises(ValueError, handler, ident='xXx', **kwds) - - # TODO: check various supported idents - - def test_has_many_idents_using(self): - """HasManyIdents.using() -- 'default_ident' and 'ident' keywords""" - self.require_many_idents() - - # pick alt ident to test with - handler = self.handler - orig_ident = handler.default_ident - for alt_ident in handler.ident_values: - if alt_ident != orig_ident: - break - else: - raise AssertionError("expected to find alternate ident: default=%r values=%r" % - (orig_ident, handler.ident_values)) - - def effective_ident(cls): - cls = unwrap_handler(cls) - return cls(use_defaults=True).ident - - # keep default if nothing else specified - subcls = handler.using() - self.assertEqual(subcls.default_ident, orig_ident) - - # accepts alt ident - subcls = handler.using(default_ident=alt_ident) - self.assertEqual(subcls.default_ident, alt_ident) - self.assertEqual(handler.default_ident, orig_ident) - - # check subcls actually *generates* default ident, - # and that we didn't affect orig handler - self.assertEqual(effective_ident(subcls), alt_ident) - self.assertEqual(effective_ident(handler), orig_ident) - - # rejects bad ident - self.assertRaises(ValueError, handler.using, default_ident='xXx') - - # honor 'ident' alias - subcls = handler.using(ident=alt_ident) - self.assertEqual(subcls.default_ident, alt_ident) - self.assertEqual(handler.default_ident, orig_ident) - - # forbid both at same time - self.assertRaises(TypeError, handler.using, default_ident=alt_ident, ident=alt_ident) - - # check ident aliases are being honored - if handler.ident_aliases: - for alias, ident in handler.ident_aliases.items(): - subcls = handler.using(ident=alias) - self.assertEqual(subcls.default_ident, ident, msg="alias %r:" % alias) - - #=================================================================== - # password size limits - #=================================================================== - def test_truncate_error_setting(self): - """ - validate 'truncate_error' setting & related attributes - """ - # If it doesn't have truncate_size set, - # it shouldn't support truncate_error - hasher = self.handler - if hasher.truncate_size is None: - self.assertNotIn("truncate_error", hasher.setting_kwds) - return - - # if hasher defaults to silently truncating, - # it MUST NOT use .truncate_verify_reject, - # because resulting hashes wouldn't verify! - if not hasher.truncate_error: - self.assertFalse(hasher.truncate_verify_reject) - - # if hasher doesn't have configurable policy, - # it must throw error by default - if "truncate_error" not in hasher.setting_kwds: - self.assertTrue(hasher.truncate_error) - return - - # test value parsing - def parse_value(value): - return hasher.using(truncate_error=value).truncate_error - self.assertEqual(parse_value(None), hasher.truncate_error) - self.assertEqual(parse_value(True), True) - self.assertEqual(parse_value("true"), True) - self.assertEqual(parse_value(False), False) - self.assertEqual(parse_value("false"), False) - self.assertRaises(ValueError, parse_value, "xxx") - - def test_secret_wo_truncate_size(self): - """ - test no password size limits enforced (if truncate_size=None) - """ - # skip if hasher has a maximum password size - hasher = self.handler - if hasher.truncate_size is not None: - self.assertGreaterEqual(hasher.truncate_size, 1) - raise self.skipTest("truncate_size is set") - - # NOTE: this doesn't do an exhaustive search to verify algorithm - # doesn't have some cutoff point, it just tries - # 1024-character string, and alters the last char. - # as long as algorithm doesn't clip secret at point <1024, - # the new secret shouldn't verify. - - # hash a 1024-byte secret - secret = "too many secrets" * 16 - alt = "x" - hash = self.do_encrypt(secret) - - # check that verify doesn't silently reject secret - # (i.e. hasher mistakenly honors .truncate_verify_reject) - verify_success = not hasher.is_disabled - self.assertEqual(self.do_verify(secret, hash), verify_success, - msg="verify rejected correct secret") - - # alter last byte, should get different hash, which won't verify - alt_secret = secret[:-1] + alt - self.assertFalse(self.do_verify(alt_secret, hash), - "full password not used in digest") - - def test_secret_w_truncate_size(self): - """ - test password size limits raise truncate_error (if appropriate) - """ - #-------------------------------------------------- - # check if test is applicable - #-------------------------------------------------- - handler = self.handler - truncate_size = handler.truncate_size - if not truncate_size: - raise self.skipTest("truncate_size not set") - - #-------------------------------------------------- - # setup vars - #-------------------------------------------------- - # try to get versions w/ and w/o truncate_error set. - # set to None if policy isn't configruable - size_error_type = exc.PasswordSizeError - if "truncate_error" in handler.setting_kwds: - without_error = handler.using(truncate_error=False) - with_error = handler.using(truncate_error=True) - size_error_type = exc.PasswordTruncateError - elif handler.truncate_error: - without_error = None - with_error = handler - else: - # NOTE: this mode is currently an error in test_truncate_error_setting() - without_error = handler - with_error = None - - # create some test secrets - base = "too many secrets" - alt = "x" # char that's not in base, used to mutate test secrets - long_secret = repeat_string(base, truncate_size+1) - short_secret = long_secret[:-1] - alt_long_secret = long_secret[:-1] + alt - alt_short_secret = short_secret[:-1] + alt - - # init flags - short_verify_success = not handler.is_disabled - long_verify_success = short_verify_success and \ - not handler.truncate_verify_reject - - #-------------------------------------------------- - # do tests on length secret, and resulting hash. - # should pass regardless of truncate_error policy. - #-------------------------------------------------- - assert without_error or with_error - for cand_hasher in [without_error, with_error]: - - # create & hash string that's exactly chars. - short_hash = self.do_encrypt(short_secret, handler=cand_hasher) - - # check hash verifies, regardless of .truncate_verify_reject - self.assertEqual(self.do_verify(short_secret, short_hash, - handler=cand_hasher), - short_verify_success) - - # changing 'th char should invalidate hash - # if this fails, means (reported) truncate_size is too large. - self.assertFalse(self.do_verify(alt_short_secret, short_hash, - handler=with_error), - "truncate_size value is too large") - - # verify should truncate long secret before comparing - # (unless truncate_verify_reject is set) - self.assertEqual(self.do_verify(long_secret, short_hash, - handler=cand_hasher), - long_verify_success) - - #-------------------------------------------------- - # do tests on length secret, - # w/ truncate error disabled (should silently truncate) - #-------------------------------------------------- - if without_error: - - # create & hash string that's exactly truncate_size+1 chars - long_hash = self.do_encrypt(long_secret, handler=without_error) - - # check verifies against secret (unless truncate_verify_reject=True) - self.assertEqual(self.do_verify(long_secret, long_hash, - handler=without_error), - short_verify_success) - - # check mutating last char doesn't change outcome. - # if this fails, means (reported) truncate_size is too small. - self.assertEqual(self.do_verify(alt_long_secret, long_hash, - handler=without_error), - short_verify_success) - - # check short_secret verifies against this hash - # if this fails, means (reported) truncate_size is too large. - self.assertTrue(self.do_verify(short_secret, long_hash, - handler=without_error)) - - #-------------------------------------------------- - # do tests on length secret, - # w/ truncate error - #-------------------------------------------------- - if with_error: - - # with errors enabled, should forbid truncation. - err = self.assertRaises(size_error_type, self.do_encrypt, - long_secret, handler=with_error) - self.assertEqual(err.max_size, truncate_size) - - #=================================================================== - # password contents - #=================================================================== - def test_61_secret_case_sensitive(self): - """test password case sensitivity""" - hash_insensitive = self.secret_case_insensitive is True - verify_insensitive = self.secret_case_insensitive in [True, - "verify-only"] - - # test hashing lower-case verifies against lower & upper - lower = 'test' - upper = 'TEST' - h1 = self.do_encrypt(lower) - if verify_insensitive and not self.handler.is_disabled: - self.assertTrue(self.do_verify(upper, h1), - "verify() should not be case sensitive") - else: - self.assertFalse(self.do_verify(upper, h1), - "verify() should be case sensitive") - - # test hashing upper-case verifies against lower & upper - h2 = self.do_encrypt(upper) - if verify_insensitive and not self.handler.is_disabled: - self.assertTrue(self.do_verify(lower, h2), - "verify() should not be case sensitive") - else: - self.assertFalse(self.do_verify(lower, h2), - "verify() should be case sensitive") - - # test genhash - # XXX: 2.0: what about 'verify-only' hashes once genhash() is removed? - # won't have easy way to recreate w/ same config to see if hash differs. - # (though only hash this applies to is mssql2000) - h2 = self.do_genhash(upper, h1) - if hash_insensitive or (self.handler.is_disabled and not self.disabled_contains_salt): - self.assertEqual(h2, h1, - "genhash() should not be case sensitive") - else: - self.assertNotEqual(h2, h1, - "genhash() should be case sensitive") - - def test_62_secret_border(self): - """test non-string passwords are rejected""" - hash = self.get_sample_hash()[1] - - # secret=None - self.assertRaises(TypeError, self.do_encrypt, None) - self.assertRaises(TypeError, self.do_genhash, None, hash) - self.assertRaises(TypeError, self.do_verify, None, hash) - - # secret=int (picked as example of entirely wrong class) - self.assertRaises(TypeError, self.do_encrypt, 1) - self.assertRaises(TypeError, self.do_genhash, 1, hash) - self.assertRaises(TypeError, self.do_verify, 1, hash) - - # xxx: move to password size limits section, above? - def test_63_large_secret(self): - """test MAX_PASSWORD_SIZE is enforced""" - from passlib.exc import PasswordSizeError - from passlib.utils import MAX_PASSWORD_SIZE - secret = '.' * (1+MAX_PASSWORD_SIZE) - hash = self.get_sample_hash()[1] - err = self.assertRaises(PasswordSizeError, self.do_genhash, secret, hash) - self.assertEqual(err.max_size, MAX_PASSWORD_SIZE) - self.assertRaises(PasswordSizeError, self.do_encrypt, secret) - self.assertRaises(PasswordSizeError, self.do_verify, secret, hash) - - def test_64_forbidden_chars(self): - """test forbidden characters not allowed in password""" - chars = self.forbidden_characters - if not chars: - raise self.skipTest("none listed") - base = u('stub') - if isinstance(chars, bytes): - from passlib.utils.compat import iter_byte_chars - chars = iter_byte_chars(chars) - base = base.encode("ascii") - for c in chars: - self.assertRaises(ValueError, self.do_encrypt, base + c + base) - - #=================================================================== - # check identify(), verify(), genhash() against test vectors - #=================================================================== - def is_secret_8bit(self, secret): - secret = self.populate_context(secret, {}) - return not is_ascii_safe(secret) - - def expect_os_crypt_failure(self, secret): - """ - check if we're expecting potential verify failure due to crypt.crypt() encoding limitation - """ - if PY3 and self.backend == "os_crypt" and isinstance(secret, bytes): - try: - secret.decode("utf-8") - except UnicodeDecodeError: - return True - return False - - def test_70_hashes(self): - """test known hashes""" - - # sanity check - self.assertTrue(self.known_correct_hashes or self.known_correct_configs, - "test must set at least one of 'known_correct_hashes' " - "or 'known_correct_configs'") - - # run through known secret/hash pairs - saw8bit = False - for secret, hash in self.iter_known_hashes(): - if self.is_secret_8bit(secret): - saw8bit = True - - # hash should be positively identified by handler - self.assertTrue(self.do_identify(hash), - "identify() failed to identify hash: %r" % (hash,)) - - # check if what we're about to do is expected to fail due to crypt.crypt() limitation. - expect_os_crypt_failure = self.expect_os_crypt_failure(secret) - try: - - # secret should verify successfully against hash - self.check_verify(secret, hash, "verify() of known hash failed: " - "secret=%r, hash=%r" % (secret, hash)) - - # genhash() should reproduce same hash - result = self.do_genhash(secret, hash) - self.assertIsInstance(result, str, - "genhash() failed to return native string: %r" % (result,)) - if self.handler.is_disabled and self.disabled_contains_salt: - continue - self.assertEqual(result, hash, "genhash() failed to reproduce " - "known hash: secret=%r, hash=%r: result=%r" % - (secret, hash, result)) - - except MissingBackendError: - if not expect_os_crypt_failure: - raise - - # would really like all handlers to have at least one 8-bit test vector - if not saw8bit: - warn("%s: no 8-bit secrets tested" % self.__class__) - - def test_71_alternates(self): - """test known alternate hashes""" - if not self.known_alternate_hashes: - raise self.skipTest("no alternate hashes provided") - for alt, secret, hash in self.known_alternate_hashes: - - # hash should be positively identified by handler - self.assertTrue(self.do_identify(hash), - "identify() failed to identify alternate hash: %r" % - (hash,)) - - # secret should verify successfully against hash - self.check_verify(secret, alt, "verify() of known alternate hash " - "failed: secret=%r, hash=%r" % (secret, alt)) - - # genhash() should reproduce canonical hash - result = self.do_genhash(secret, alt) - self.assertIsInstance(result, str, - "genhash() failed to return native string: %r" % (result,)) - if self.handler.is_disabled and self.disabled_contains_salt: - continue - self.assertEqual(result, hash, "genhash() failed to normalize " - "known alternate hash: secret=%r, alt=%r, hash=%r: " - "result=%r" % (secret, alt, hash, result)) - - def test_72_configs(self): - """test known config strings""" - # special-case handlers without settings - if not self.handler.setting_kwds: - self.assertFalse(self.known_correct_configs, - "handler should not have config strings") - raise self.skipTest("hash has no settings") - - if not self.known_correct_configs: - # XXX: make this a requirement? - raise self.skipTest("no config strings provided") - - # make sure config strings work (hashes in list tested in test_70) - if self.filter_config_warnings: - warnings.filterwarnings("ignore", category=PasslibHashWarning) - for config, secret, hash in self.known_correct_configs: - - # config should be positively identified by handler - self.assertTrue(self.do_identify(config), - "identify() failed to identify known config string: %r" % - (config,)) - - # verify() should throw error for config strings. - self.assertRaises(ValueError, self.do_verify, secret, config, - __msg__="verify() failed to reject config string: %r" % - (config,)) - - # genhash() should reproduce hash from config. - result = self.do_genhash(secret, config) - self.assertIsInstance(result, str, - "genhash() failed to return native string: %r" % (result,)) - self.assertEqual(result, hash, "genhash() failed to reproduce " - "known hash from config: secret=%r, config=%r, hash=%r: " - "result=%r" % (secret, config, hash, result)) - - def test_73_unidentified(self): - """test known unidentifiably-mangled strings""" - if not self.known_unidentified_hashes: - raise self.skipTest("no unidentified hashes provided") - for hash in self.known_unidentified_hashes: - - # identify() should reject these - self.assertFalse(self.do_identify(hash), - "identify() incorrectly identified known unidentifiable " - "hash: %r" % (hash,)) - - # verify() should throw error - self.assertRaises(ValueError, self.do_verify, 'stub', hash, - __msg__= "verify() failed to throw error for unidentifiable " - "hash: %r" % (hash,)) - - # genhash() should throw error - self.assertRaises(ValueError, self.do_genhash, 'stub', hash, - __msg__= "genhash() failed to throw error for unidentifiable " - "hash: %r" % (hash,)) - - def test_74_malformed(self): - """test known identifiable-but-malformed strings""" - if not self.known_malformed_hashes: - raise self.skipTest("no malformed hashes provided") - for hash in self.known_malformed_hashes: - - # identify() should accept these - self.assertTrue(self.do_identify(hash), - "identify() failed to identify known malformed " - "hash: %r" % (hash,)) - - # verify() should throw error - self.assertRaises(ValueError, self.do_verify, 'stub', hash, - __msg__= "verify() failed to throw error for malformed " - "hash: %r" % (hash,)) - - # genhash() should throw error - self.assertRaises(ValueError, self.do_genhash, 'stub', hash, - __msg__= "genhash() failed to throw error for malformed " - "hash: %r" % (hash,)) - - def test_75_foreign(self): - """test known foreign hashes""" - if self.accepts_all_hashes: - raise self.skipTest("not applicable") - if not self.known_other_hashes: - raise self.skipTest("no foreign hashes provided") - for name, hash in self.known_other_hashes: - # NOTE: most tests use default list of foreign hashes, - # so they may include ones belonging to that hash... - # hence the 'own' logic. - - if name == self.handler.name: - # identify should accept these - self.assertTrue(self.do_identify(hash), - "identify() failed to identify known hash: %r" % (hash,)) - - # verify & genhash should NOT throw error - self.do_verify('stub', hash) - result = self.do_genhash('stub', hash) - self.assertIsInstance(result, str, - "genhash() failed to return native string: %r" % (result,)) - - else: - # identify should reject these - self.assertFalse(self.do_identify(hash), - "identify() incorrectly identified hash belonging to " - "%s: %r" % (name, hash)) - - # verify should throw error - self.assertRaises(ValueError, self.do_verify, 'stub', hash, - __msg__= "verify() failed to throw error for hash " - "belonging to %s: %r" % (name, hash,)) - - # genhash() should throw error - self.assertRaises(ValueError, self.do_genhash, 'stub', hash, - __msg__= "genhash() failed to throw error for hash " - "belonging to %s: %r" % (name, hash)) - - def test_76_hash_border(self): - """test non-string hashes are rejected""" - # - # test hash=None is handled correctly - # - self.assertRaises(TypeError, self.do_identify, None) - self.assertRaises(TypeError, self.do_verify, 'stub', None) - - # NOTE: changed in 1.7 -- previously 'None' would be accepted when config strings not supported. - self.assertRaises(TypeError, self.do_genhash, 'stub', None) - - # - # test hash=int is rejected (picked as example of entirely wrong type) - # - self.assertRaises(TypeError, self.do_identify, 1) - self.assertRaises(TypeError, self.do_verify, 'stub', 1) - self.assertRaises(TypeError, self.do_genhash, 'stub', 1) - - # - # test hash='' is rejected for all but the plaintext hashes - # - for hash in [u(''), b'']: - if self.accepts_all_hashes: - # then it accepts empty string as well. - self.assertTrue(self.do_identify(hash)) - self.do_verify('stub', hash) - result = self.do_genhash('stub', hash) - self.check_returned_native_str(result, "genhash") - else: - # otherwise it should reject them - self.assertFalse(self.do_identify(hash), - "identify() incorrectly identified empty hash") - self.assertRaises(ValueError, self.do_verify, 'stub', hash, - __msg__="verify() failed to reject empty hash") - self.assertRaises(ValueError, self.do_genhash, 'stub', hash, - __msg__="genhash() failed to reject empty hash") - - # - # test identify doesn't throw decoding errors on 8-bit input - # - self.do_identify('\xe2\x82\xac\xc2\xa5$') # utf-8 - self.do_identify('abc\x91\x00') # non-utf8 - - #=================================================================== - # fuzz testing - #=================================================================== - def test_77_fuzz_input(self, threaded=False): - """fuzz testing -- random passwords and options - - This test attempts to perform some basic fuzz testing of the hash, - based on whatever information can be found about it. - It does as much as it can within a fixed amount of time - (defaults to 1 second, but can be overridden via $PASSLIB_TEST_FUZZ_TIME). - It tests the following: - - * randomly generated passwords including extended unicode chars - * randomly selected rounds values (if rounds supported) - * randomly selected salt sizes (if salts supported) - * randomly selected identifiers (if multiple found) - * runs output of selected backend against other available backends - (if any) to detect errors occurring between different backends. - * runs output against other "external" verifiers such as OS crypt() - - :param report_thread_state: - if true, writes state of loop to current_thread().passlib_fuzz_state. - used to help debug multi-threaded fuzz test issues (below) - """ - if self.handler.is_disabled: - raise self.skipTest("not applicable") - - # gather info - from passlib.utils import tick - max_time = self.max_fuzz_time - if max_time <= 0: - raise self.skipTest("disabled by test mode") - verifiers = self.get_fuzz_verifiers(threaded=threaded) - def vname(v): - return (v.__doc__ or v.__name__).splitlines()[0] - - # init rng -- using separate one for each thread - # so things are predictable for given RANDOM_TEST_SEED - # (relies on test_78_fuzz_threading() to give threads unique names) - if threaded: - thread_name = threading.current_thread().name - else: - thread_name = "fuzz test" - rng = self.getRandom(name=thread_name) - generator = self.FuzzHashGenerator(self, rng) - - # do as many tests as possible for max_time seconds - log.debug("%s: %s: started; max_time=%r verifiers=%d (%s)", - self.descriptionPrefix, thread_name, max_time, len(verifiers), - ", ".join(vname(v) for v in verifiers)) - start = tick() - stop = start + max_time - count = 0 - while tick() <= stop: - # generate random password & options - opts = generator.generate() - secret = opts['secret'] - other = opts['other'] - settings = opts['settings'] - ctx = opts['context'] - if ctx: - settings['context'] = ctx - - # create new hash - hash = self.do_encrypt(secret, **settings) - ##log.debug("fuzz test: hash=%r secret=%r other=%r", - ## hash, secret, other) - - # run through all verifiers we found. - for verify in verifiers: - name = vname(verify) - result = verify(secret, hash, **ctx) - if result == "skip": # let verifiers signal lack of support - continue - assert result is True or result is False - if not result: - raise self.failureException("failed to verify against %r verifier: " - "secret=%r config=%r hash=%r" % - (name, secret, settings, hash)) - # occasionally check that some other secrets WON'T verify - # against this hash. - if rng.random() < .1: - result = verify(other, hash, **ctx) - if result and result != "skip": - raise self.failureException("was able to verify wrong " - "password using %s: wrong_secret=%r real_secret=%r " - "config=%r hash=%r" % (name, other, secret, settings, hash)) - count += 1 - - log.debug("%s: %s: done; elapsed=%r count=%r", - self.descriptionPrefix, thread_name, tick() - start, count) - - def test_78_fuzz_threading(self): - """multithreaded fuzz testing -- random password & options using multiple threads - - run test_77 simultaneously in multiple threads - in an attempt to detect any concurrency issues - (e.g. the bug fixed by pybcrypt 0.3) - """ - self.require_TEST_MODE("full") - import threading - - # check if this test should run - if self.handler.is_disabled: - raise self.skipTest("not applicable") - thread_count = self.fuzz_thread_count - if thread_count < 1 or self.max_fuzz_time <= 0: - raise self.skipTest("disabled by test mode") - - # buffer to hold errors thrown by threads - failed_lock = threading.Lock() - failed = [0] - - # launch threads, all of which run - # test_77_fuzz_input(), and see if any errors get thrown. - # if hash has concurrency issues, this should reveal it. - def wrapper(): - try: - self.test_77_fuzz_input(threaded=True) - except SkipTest: - pass - except: - with failed_lock: - failed[0] += 1 - raise - def launch(n): - name = "Fuzz-Thread-%d" % (n,) - thread = threading.Thread(target=wrapper, name=name) - thread.setDaemon(True) - thread.start() - return thread - threads = [launch(n) for n in irange(thread_count)] - - # wait until all threads exit - timeout = self.max_fuzz_time * thread_count * 4 - stalled = 0 - for thread in threads: - thread.join(timeout) - if not thread.is_alive(): - continue - # XXX: not sure why this is happening, main one seems 1/4 times for sun_md5_crypt - log.error("%s timed out after %f seconds", thread.name, timeout) - stalled += 1 - - # if any thread threw an error, raise one ourselves. - if failed[0]: - raise self.fail("%d/%d threads failed concurrent fuzz testing " - "(see error log for details)" % (failed[0], thread_count)) - if stalled: - raise self.fail("%d/%d threads stalled during concurrent fuzz testing " - "(see error log for details)" % (stalled, thread_count)) - - #--------------------------------------------------------------- - # fuzz constants & helpers - #--------------------------------------------------------------- - - @property - def max_fuzz_time(self): - """amount of time to spend on fuzz testing""" - value = float(os.environ.get("PASSLIB_TEST_FUZZ_TIME") or 0) - if value: - return value - elif TEST_MODE(max="quick"): - return 0 - elif TEST_MODE(max="default"): - return 1 - else: - return 5 - - @property - def fuzz_thread_count(self): - """number of threads for threaded fuzz testing""" - value = int(os.environ.get("PASSLIB_TEST_FUZZ_THREADS") or 0) - if value: - return value - elif TEST_MODE(max="quick"): - return 0 - else: - return 10 - - #--------------------------------------------------------------- - # fuzz verifiers - #--------------------------------------------------------------- - - #: list of custom fuzz-test verifiers (in addition to hasher itself, - #: and backend-specific wrappers of hasher). each element is - #: name of method that will return None / a verifier callable. - fuzz_verifiers = ("fuzz_verifier_default",) - - def get_fuzz_verifiers(self, threaded=False): - """return list of password verifiers (including external libs) - - used by fuzz testing. - verifiers should be callable with signature - ``func(password: unicode, hash: ascii str) -> ok: bool``. - """ - handler = self.handler - verifiers = [] - - # call all methods starting with prefix in order to create - for method_name in self.fuzz_verifiers: - func = getattr(self, method_name)() - if func is not None: - verifiers.append(func) - - # create verifiers for any other available backends - # NOTE: skipping this under threading test, - # since backend switching isn't threadsafe (yet) - if hasattr(handler, "backends") and TEST_MODE("full") and not threaded: - def maker(backend): - def func(secret, hash): - orig_backend = handler.get_backend() - try: - handler.set_backend(backend) - return handler.verify(secret, hash) - finally: - handler.set_backend(orig_backend) - func.__name__ = "check_" + backend + "_backend" - func.__doc__ = backend + "-backend" - return func - for backend in iter_alt_backends(handler): - verifiers.append(maker(backend)) - - return verifiers - - def fuzz_verifier_default(self): - # test against self - def check_default(secret, hash, **ctx): - return self.do_verify(secret, hash, **ctx) - if self.backend: - check_default.__doc__ = self.backend + "-backend" - else: - check_default.__doc__ = "self" - return check_default - - #--------------------------------------------------------------- - # fuzz settings generation - #--------------------------------------------------------------- - class FuzzHashGenerator(object): - """ - helper which takes care of generating random - passwords & configuration options to test hash with. - separate from test class so we can create one per thread. - """ - #========================================================== - # class attrs - #========================================================== - - # alphabet for randomly generated passwords - password_alphabet = u('qwertyASDF1234<>.@*#! \u00E1\u0259\u0411\u2113') - - # encoding when testing bytes - password_encoding = "utf-8" - - # map of setting kwd -> method name. - # will ignore setting if method returns None. - # subclasses should make copy of dict. - settings_map = dict(rounds="random_rounds", - salt_size="random_salt_size", - ident="random_ident") - - # map of context kwd -> method name. - context_map = {} - - #========================================================== - # init / generation - #========================================================== - - def __init__(self, test, rng): - self.test = test - self.handler = test.handler - self.rng = rng - - def generate(self): - """ - generate random password and options for fuzz testing. - :returns: - `(secret, other_secret, settings_kwds, context_kwds)` - """ - def gendict(map): - out = {} - for key, meth in map.items(): - func = getattr(self, meth) - value = getattr(self, meth)() - if value is not None: - out[key] = value - return out - secret, other = self.random_password_pair() - return dict(secret=secret, - other=other, - settings=gendict(self.settings_map), - context=gendict(self.context_map), - ) - - #========================================================== - # helpers - #========================================================== - def randintgauss(self, lower, upper, mu, sigma): - """generate random int w/ gauss distirbution""" - value = self.rng.normalvariate(mu, sigma) - return int(limit(value, lower, upper)) - - #========================================================== - # settings generation - #========================================================== - - def random_rounds(self): - handler = self.handler - if not has_rounds_info(handler): - return None - default = handler.default_rounds or handler.min_rounds - lower = handler.min_rounds - if handler.rounds_cost == "log2": - upper = default - else: - upper = min(default*2, handler.max_rounds) - return self.randintgauss(lower, upper, default, default*.5) - - def random_salt_size(self): - handler = self.handler - if not (has_salt_info(handler) and 'salt_size' in handler.setting_kwds): - return None - default = handler.default_salt_size - lower = handler.min_salt_size - upper = handler.max_salt_size or default*4 - return self.randintgauss(lower, upper, default, default*.5) - - def random_ident(self): - rng = self.rng - handler = self.handler - if 'ident' not in handler.setting_kwds or not hasattr(handler, "ident_values"): - return None - if rng.random() < .5: - return None - # resolve wrappers before reading values - handler = getattr(handler, "wrapped", handler) - return rng.choice(handler.ident_values) - - #========================================================== - # fuzz password generation - #========================================================== - def random_password_pair(self): - """generate random password, and non-matching alternate password""" - secret = self.random_password() - while True: - other = self.random_password() - if self.accept_password_pair(secret, other): - break - rng = self.rng - if rng.randint(0,1): - secret = secret.encode(self.password_encoding) - if rng.randint(0,1): - other = other.encode(self.password_encoding) - return secret, other - - def random_password(self): - """generate random passwords for fuzz testing""" - # occasionally try an empty password - rng = self.rng - if rng.random() < .0001: - return u('') - - # check if truncate size needs to be considered - handler = self.handler - truncate_size = handler.truncate_error and handler.truncate_size - max_size = truncate_size or 999999 - - # pick endpoint - if max_size < 50 or rng.random() < .5: - # chance of small password (~15 chars) - size = self.randintgauss(1, min(max_size, 50), 15, 15) - else: - # otherwise large password (~70 chars) - size = self.randintgauss(50, min(max_size, 99), 70, 20) - - # generate random password - result = getrandstr(rng, self.password_alphabet, size) - - # trim ones that encode past truncate point. - if truncate_size and isinstance(result, unicode): - while len(result.encode("utf-8")) > truncate_size: - result = result[:-1] - - return result - - def accept_password_pair(self, secret, other): - """verify fuzz pair contains different passwords""" - return secret != other - - #========================================================== - # eoc FuzzGenerator - #========================================================== - - #=================================================================== - # "disabled hasher" api - #=================================================================== - - def test_disable_and_enable(self): - """.disable() / .enable() methods""" - # - # setup - # - handler = self.handler - if not handler.is_disabled: - self.assertFalse(hasattr(handler, "disable")) - self.assertFalse(hasattr(handler, "enable")) - self.assertFalse(self.disabled_contains_salt) - raise self.skipTest("not applicable") - - # - # disable() - # - - # w/o existing hash - disabled_default = handler.disable() - self.assertIsInstance(disabled_default, str, - msg="disable() must return native string") - self.assertTrue(handler.identify(disabled_default), - msg="identify() didn't recognize disable() result: %r" % (disabled_default)) - - # w/ existing hash - stub = self.getRandom().choice(self.known_other_hashes)[1] - disabled_stub = handler.disable(stub) - self.assertIsInstance(disabled_stub, str, - msg="disable() must return native string") - self.assertTrue(handler.identify(disabled_stub), - msg="identify() didn't recognize disable() result: %r" % (disabled_stub)) - - # - # enable() - # - - # w/o original hash - self.assertRaisesRegex(ValueError, "cannot restore original hash", - handler.enable, disabled_default) - - # w/ original hash - try: - result = handler.enable(disabled_stub) - error = None - except ValueError as e: - result = None - error = e - - if error is None: - # if supports recovery, should have returned stub (e.g. unix_disabled); - self.assertIsInstance(result, str, - msg="enable() must return native string") - self.assertEqual(result, stub) - else: - # if doesn't, should have thrown appropriate error - self.assertIsInstance(error, ValueError) - self.assertRegex("cannot restore original hash", str(error)) - - # - # test repeating disable() & salting state - # - - # repeating disabled - disabled_default2 = handler.disable() - if self.disabled_contains_salt: - # should return new salt for each call (e.g. django_disabled) - self.assertNotEqual(disabled_default2, disabled_default) - elif error is None: - # should return same result for each hash, but unique across hashes - self.assertEqual(disabled_default2, disabled_default) - - # repeating same hash ... - disabled_stub2 = handler.disable(stub) - if self.disabled_contains_salt: - # ... should return different string (if salted) - self.assertNotEqual(disabled_stub2, disabled_stub) - else: - # ... should return same string - self.assertEqual(disabled_stub2, disabled_stub) - - # using different hash ... - disabled_other = handler.disable(stub + 'xxx') - if self.disabled_contains_salt or error is None: - # ... should return different string (if salted or hash encoded) - self.assertNotEqual(disabled_other, disabled_stub) - else: - # ... should return same string - self.assertEqual(disabled_other, disabled_stub) - - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# HandlerCase mixins providing additional tests for certain hashes -#============================================================================= -class OsCryptMixin(HandlerCase): - """helper used by create_backend_case() which adds additional features - to test the os_crypt backend. - - * if crypt support is missing, inserts fake crypt support to simulate - a working safe_crypt, to test passlib's codepath as fully as possible. - - * extra tests to verify non-conformant crypt implementations are handled - correctly. - - * check that native crypt support is detected correctly for known platforms. - """ - #=================================================================== - # class attrs - #=================================================================== - - # platforms that are known to support / not support this hash natively. - # list of (platform_regex, True|False|None) entries. - platform_crypt_support = [] - - #: flag indicating backend provides a fallback when safe_crypt() can't handle password - has_os_crypt_fallback = True - - #: alternate handler to use when searching for backend to fake safe_crypt() support. - alt_safe_crypt_handler = None - - #=================================================================== - # instance attrs - #=================================================================== - __unittest_skip = True - - # force this backend - backend = "os_crypt" - - # flag read by HandlerCase to detect if fake os crypt is enabled. - using_patched_crypt = False - - #=================================================================== - # setup - #=================================================================== - def setUp(self): - assert self.backend == "os_crypt" - if not self.handler.has_backend("os_crypt"): - self._patch_safe_crypt() - super(OsCryptMixin, self).setUp() - - @classmethod - def _get_safe_crypt_handler_backend(cls): - """ - return (handler, backend) pair to use for faking crypt.crypt() support for hash. - backend will be None if none availabe. - """ - # find handler that generates safe_crypt() compatible hash - handler = cls.alt_safe_crypt_handler - if not handler: - handler = unwrap_handler(cls.handler) - - # hack to prevent recursion issue when .has_backend() is called - handler.get_backend() - - # find backend which isn't os_crypt - alt_backend = get_alt_backend(handler, "os_crypt") - return handler, alt_backend - - def _patch_safe_crypt(self): - """if crypt() doesn't support current hash alg, this patches - safe_crypt() so that it transparently uses another one of the handler's - backends, so that we can go ahead and test as much of code path - as possible. - """ - # find handler & backend - handler, alt_backend = self._get_safe_crypt_handler_backend() - if not alt_backend: - raise AssertionError("handler has no available alternate backends!") - - # create subclass of handler, which we swap to an alternate backend - alt_handler = handler.using() - alt_handler.set_backend(alt_backend) - - def crypt_stub(secret, hash): - hash = alt_handler.genhash(secret, hash) - assert isinstance(hash, str) - return hash - - import passlib.utils as mod - self.patchAttr(mod, "_crypt", crypt_stub) - self.using_patched_crypt = True - - @classmethod - def _get_skip_backend_reason(cls, backend): - """ - make sure os_crypt backend is tested - when it's known os_crypt will be faked by _patch_safe_crypt() - """ - assert backend == "os_crypt" - reason = super(OsCryptMixin, cls)._get_skip_backend_reason(backend) - - from passlib.utils import has_crypt - if reason == cls.BACKEND_NOT_AVAILABLE and has_crypt: - if TEST_MODE("full") and cls._get_safe_crypt_handler_backend()[1]: - # in this case, _patch_safe_crypt() will monkeypatch os_crypt - # to use another backend, just so we can test os_crypt fully. - return None - else: - return "hash not supported by os crypt()" - - return reason - - #=================================================================== - # custom tests - #=================================================================== - - # TODO: turn into decorator, and use mock library. - def _use_mock_crypt(self): - """ - patch passlib.utils.safe_crypt() so it returns mock value for duration of test. - returns function whose .return_value controls what's returned. - this defaults to None. - """ - import passlib.utils as mod - - def mock_crypt(secret, config): - # let 'test' string through so _load_os_crypt_backend() will still work - if secret == "test": - return mock_crypt.__wrapped__(secret, config) - else: - return mock_crypt.return_value - - mock_crypt.__wrapped__ = mod._crypt - mock_crypt.return_value = None - - self.patchAttr(mod, "_crypt", mock_crypt) - - return mock_crypt - - def test_80_faulty_crypt(self): - """test with faulty crypt()""" - hash = self.get_sample_hash()[1] - exc_types = (AssertionError,) - mock_crypt = self._use_mock_crypt() - - def test(value): - # set safe_crypt() to return specified value, and - # make sure assertion error is raised by handler. - mock_crypt.return_value = value - self.assertRaises(exc_types, self.do_genhash, "stub", hash) - self.assertRaises(exc_types, self.do_encrypt, "stub") - self.assertRaises(exc_types, self.do_verify, "stub", hash) - - test('$x' + hash[2:]) # detect wrong prefix - test(hash[:-1]) # detect too short - test(hash + 'x') # detect too long - - def test_81_crypt_fallback(self): - """test per-call crypt() fallback""" - - # mock up safe_crypt to return None - mock_crypt = self._use_mock_crypt() - mock_crypt.return_value = None - - if self.has_os_crypt_fallback: - # handler should have a fallback to use when os_crypt backend refuses to handle secret. - h1 = self.do_encrypt("stub") - h2 = self.do_genhash("stub", h1) - self.assertEqual(h2, h1) - self.assertTrue(self.do_verify("stub", h1)) - else: - # handler should give up - from passlib.exc import MissingBackendError - hash = self.get_sample_hash()[1] - self.assertRaises(MissingBackendError, self.do_encrypt, 'stub') - self.assertRaises(MissingBackendError, self.do_genhash, 'stub', hash) - self.assertRaises(MissingBackendError, self.do_verify, 'stub', hash) - - def test_82_crypt_support(self): - """test platform-specific crypt() support detection""" - # NOTE: this is mainly just a sanity check to ensure the runtime - # detection is functioning correctly on some known platforms, - # so that I can feel more confident it'll work right on unknown ones. - if hasattr(self.handler, "orig_prefix"): - raise self.skipTest("not applicable to wrappers") - platform = sys.platform - for pattern, state in self.platform_crypt_support: - if re.match(pattern, platform): - break - else: - raise self.skipTest("no data for %r platform" % platform) - if state is None: - # e.g. platform='freebsd8' ... sha256_crypt not added until 8.3 - raise self.skipTest("varied support on %r platform" % platform) - elif state != self.using_patched_crypt: - return - elif state: - self.fail("expected %r platform would have native support " - "for %r" % (platform, self.handler.name)) - else: - self.fail("did not expect %r platform would have native support " - "for %r" % (platform, self.handler.name)) - - #=================================================================== - # fuzzy verified support -- add new verified that uses os crypt() - #=================================================================== - def fuzz_verifier_crypt(self): - """test results against OS crypt()""" - - # don't use this if we're faking safe_crypt (pointless test), - # or if handler is a wrapper (only original handler will be supported by os) - handler = self.handler - if self.using_patched_crypt or hasattr(handler, "wrapped"): - return None - - # create a wrapper for fuzzy verified to use - from crypt import crypt - encoding = self.FuzzHashGenerator.password_encoding - - def check_crypt(secret, hash): - """stdlib-crypt""" - if not self.crypt_supports_variant(hash): - return "skip" - secret = to_native_str(secret, encoding) - return crypt(secret, hash) == hash - - return check_crypt - - def crypt_supports_variant(self, hash): - """ - fuzzy_verified_crypt() helper -- - used to determine if os crypt() supports a particular hash variant. - """ - return True - - #=================================================================== - # eoc - #=================================================================== - -class UserHandlerMixin(HandlerCase): - """helper for handlers w/ 'user' context kwd; mixin for HandlerCase - - this overrides the HandlerCase test harness methods - so that a username is automatically inserted to hash/verify - calls. as well, passing in a pair of strings as the password - will be interpreted as (secret,user) - """ - #=================================================================== - # option flags - #=================================================================== - default_user = "user" - requires_user = True - user_case_insensitive = False - - #=================================================================== - # instance attrs - #=================================================================== - __unittest_skip = True - - #=================================================================== - # custom tests - #=================================================================== - def test_80_user(self): - """test user context keyword""" - handler = self.handler - password = 'stub' - hash = handler.hash(password, user=self.default_user) - - if self.requires_user: - self.assertRaises(TypeError, handler.hash, password) - self.assertRaises(TypeError, handler.genhash, password, hash) - self.assertRaises(TypeError, handler.verify, password, hash) - else: - # e.g. cisco_pix works with or without one. - handler.hash(password) - handler.genhash(password, hash) - handler.verify(password, hash) - - def test_81_user_case(self): - """test user case sensitivity""" - lower = self.default_user.lower() - upper = lower.upper() - hash = self.do_encrypt('stub', context=dict(user=lower)) - if self.user_case_insensitive: - self.assertTrue(self.do_verify('stub', hash, user=upper), - "user should not be case sensitive") - else: - self.assertFalse(self.do_verify('stub', hash, user=upper), - "user should be case sensitive") - - def test_82_user_salt(self): - """test user used as salt""" - config = self.do_stub_encrypt() - h1 = self.do_genhash('stub', config, user='admin') - h2 = self.do_genhash('stub', config, user='admin') - self.assertEqual(h2, h1) - h3 = self.do_genhash('stub', config, user='root') - self.assertNotEqual(h3, h1) - - # TODO: user size? kinda dicey, depends on algorithm. - - #=================================================================== - # override test helpers - #=================================================================== - def populate_context(self, secret, kwds): - """insert username into kwds""" - if isinstance(secret, tuple): - secret, user = secret - elif not self.requires_user: - return secret - else: - user = self.default_user - if 'user' not in kwds: - kwds['user'] = user - return secret - - #=================================================================== - # modify fuzz testing - #=================================================================== - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - context_map = HandlerCase.FuzzHashGenerator.context_map.copy() - context_map.update(user="random_user") - - user_alphabet = u("asdQWE123") - - def random_user(self): - rng = self.rng - if not self.test.requires_user and rng.random() < .1: - return None - return getrandstr(rng, self.user_alphabet, rng.randint(2,10)) - - #=================================================================== - # eoc - #=================================================================== - -class EncodingHandlerMixin(HandlerCase): - """helper for handlers w/ 'encoding' context kwd; mixin for HandlerCase - - this overrides the HandlerCase test harness methods - so that an encoding can be inserted to hash/verify - calls by passing in a pair of strings as the password - will be interpreted as (secret,encoding) - """ - #=================================================================== - # instance attrs - #=================================================================== - __unittest_skip = True - - # restrict stock passwords & fuzz alphabet to latin-1, - # so different encodings can be tested safely. - stock_passwords = [ - u("test"), - b"test", - u("\u00AC\u00BA"), - ] - - class FuzzHashGenerator(HandlerCase.FuzzHashGenerator): - - password_alphabet = u('qwerty1234<>.@*#! \u00AC') - - def populate_context(self, secret, kwds): - """insert encoding into kwds""" - if isinstance(secret, tuple): - secret, encoding = secret - kwds.setdefault('encoding', encoding) - return secret - #=================================================================== - # eoc - #=================================================================== - -#============================================================================= -# warnings helpers -#============================================================================= -class reset_warnings(warnings.catch_warnings): - """catch_warnings() wrapper which clears warning registry & filters""" - - def __init__(self, reset_filter="always", reset_registry=".*", **kwds): - super(reset_warnings, self).__init__(**kwds) - self._reset_filter = reset_filter - self._reset_registry = re.compile(reset_registry) if reset_registry else None - - def __enter__(self): - # let parent class archive filter state - ret = super(reset_warnings, self).__enter__() - - # reset the filter to list everything - if self._reset_filter: - warnings.resetwarnings() - warnings.simplefilter(self._reset_filter) - - # archive and clear the __warningregistry__ key for all modules - # that match the 'reset' pattern. - pattern = self._reset_registry - if pattern: - backup = self._orig_registry = {} - for name, mod in list(sys.modules.items()): - if mod is None or not pattern.match(name): - continue - reg = getattr(mod, "__warningregistry__", None) - if reg: - backup[name] = reg.copy() - reg.clear() - return ret - - def __exit__(self, *exc_info): - # restore warning registry for all modules - pattern = self._reset_registry - if pattern: - # restore registry backup, clearing all registry entries that we didn't archive - backup = self._orig_registry - for name, mod in list(sys.modules.items()): - if mod is None or not pattern.match(name): - continue - reg = getattr(mod, "__warningregistry__", None) - if reg: - reg.clear() - orig = backup.get(name) - if orig: - if reg is None: - setattr(mod, "__warningregistry__", orig) - else: - reg.update(orig) - super(reset_warnings, self).__exit__(*exc_info) - -#============================================================================= -# eof -#============================================================================= diff --git a/src/passlib/totp.py b/src/passlib/totp.py deleted file mode 100644 index c2e88917..00000000 --- a/src/passlib/totp.py +++ /dev/null @@ -1,1900 +0,0 @@ -"""passlib.totp -- TOTP / RFC6238 / Google Authenticator utilities.""" -#============================================================================= -# imports -#============================================================================= -from __future__ import absolute_import, division, print_function -from passlib.utils.compat import PY3 -# core -import base64 -import collections -import calendar -import json -import logging; log = logging.getLogger(__name__) -import math -import struct -import sys -import time as _time -import re -if PY3: - from urllib.parse import urlparse, parse_qsl, quote, unquote -else: - from urllib import quote, unquote - from urlparse import urlparse, parse_qsl -from warnings import warn -# site -try: - # TOTP encrypted keys only supported if cryptography (https://cryptography.io) is installed - from cryptography.hazmat.backends import default_backend as _cg_default_backend - import cryptography.hazmat.primitives.ciphers.algorithms - import cryptography.hazmat.primitives.ciphers.modes - from cryptography.hazmat.primitives import ciphers as _cg_ciphers - del cryptography -except ImportError: - log.debug("can't import 'cryptography' package, totp encryption disabled") - _cg_ciphers = _cg_default_backend = None -# pkg -from passlib import exc -from passlib.exc import TokenError, MalformedTokenError, InvalidTokenError, UsedTokenError -from passlib.utils import (to_unicode, to_bytes, consteq, - getrandbytes, rng, SequenceMixin, xor_bytes, getrandstr) -from passlib.utils.binary import BASE64_CHARS, b32encode, b32decode -from passlib.utils.compat import (u, unicode, native_string_types, bascii_to_str, int_types, num_types, - irange, byte_elem_value, UnicodeIO, suppress_cause) -from passlib.utils.decor import hybrid_method, memoized_property -from passlib.crypto.digest import lookup_hash, compile_hmac, pbkdf2_hmac -from passlib.hash import pbkdf2_sha256 -# local -__all__ = [ - # frontend classes - "AppWallet", - "TOTP", - - # errors (defined in passlib.exc, but exposed here for convenience) - "TokenError", - "MalformedTokenError", - "InvalidTokenError", - "UsedTokenError", - - # internal helper classes - "TotpToken", - "TotpMatch", -] - -#============================================================================= -# HACK: python < 2.7.4's urlparse() won't parse query strings unless the url scheme -# is one of the schemes in the urlparse.uses_query list. 2.7 abandoned -# this, and parses query if present, regardless of the scheme. -# as a workaround for older versions, we add "otpauth" to the known list. -# this was fixed by https://bugs.python.org/issue9374, in 2.7.4 release. -#============================================================================= -if sys.version_info < (2,7,4): - from urlparse import uses_query - if "otpauth" not in uses_query: - uses_query.append("otpauth") - log.debug("registered 'otpauth' scheme with urlparse.uses_query") - del uses_query - -#============================================================================= -# internal helpers -#============================================================================= - -#----------------------------------------------------------------------------- -# token parsing / rendering helpers -#----------------------------------------------------------------------------- - -#: regex used to clean whitespace from tokens & keys -_clean_re = re.compile(u(r"\s|[-=]"), re.U) - -_chunk_sizes = [4,6,5] - -def _get_group_size(klen): - """ - helper for group_string() -- - calculates optimal size of group for given string size. - """ - # look for exact divisor - for size in _chunk_sizes: - if not klen % size: - return size - # fallback to divisor with largest remainder - # (so chunks are as close to even as possible) - best = _chunk_sizes[0] - rem = 0 - for size in _chunk_sizes: - if klen % size > rem: - best = size - rem = klen % size - return best - -def group_string(value, sep="-"): - """ - reformat string into (roughly) evenly-sized groups, separated by **sep**. - useful for making tokens & keys easier to read by humans. - """ - klen = len(value) - size = _get_group_size(klen) - return sep.join(value[o:o+size] for o in irange(0, klen, size)) - -#----------------------------------------------------------------------------- -# encoding helpers -#----------------------------------------------------------------------------- - -def _decode_bytes(key, format): - """ - internal TOTP() helper -- - decodes key according to specified format. - """ - if format == "raw": - if not isinstance(key, bytes): - raise exc.ExpectedTypeError(key, "bytes", "key") - return key - # for encoded data, key must be either unicode or ascii-encoded bytes, - # and must contain a hex or base32 string. - key = to_unicode(key, param="key") - key = _clean_re.sub("", key).encode("utf-8") # strip whitespace & hypens - if format == "hex" or format == "base16": - return base64.b16decode(key.upper()) - elif format == "base32": - return b32decode(key) - # XXX: add base64 support? - else: - raise ValueError("unknown byte-encoding format: %r" % (format,)) - -#============================================================================= -# OTP management -#============================================================================= - -#: flag for detecting if encrypted totp support is present -AES_SUPPORT = bool(_cg_ciphers) - -#: regex for validating secret tags -_tag_re = re.compile("(?i)^[a-z0-9][a-z0-9_.-]*$") - -class AppWallet(object): - """ - This class stores application-wide secrets that can be used - to encrypt & decrypt TOTP keys for storage. - It's mostly an internal detail, applications usually just need - to pass ``secrets`` or ``secrets_path`` to :meth:`TOTP.using`. - - .. seealso:: - - :ref:`totp-storing-instances` for more details on this workflow. - - Arguments - ========= - :param secrets: - Dict of application secrets to use when encrypting/decrypting - stored TOTP keys. This should include a secret to use when encrypting - new keys, but may contain additional older secrets to decrypt - existing stored keys. - - The dict should map tags -> secrets, so that each secret is identified - by a unique tag. This tag will be stored along with the encrypted - key in order to determine which secret should be used for decryption. - Tag should be string that starts with regex range ``[a-z0-9]``, - and the remaining characters must be in ``[a-z0-9_.-]``. - - It is recommended to use something like a incremental counter - ("1", "2", ...), an ISO date ("2016-01-01", "2016-05-16", ...), - or a timestamp ("19803495", "19813495", ...) when assigning tags. - - This mapping be provided in three formats: - - * A python dict mapping tag -> secret - * A JSON-formatted string containing the dict - * A multiline string with the format ``"tag: value\\ntag: value\\n..."`` - - (This last format is mainly useful when loading from a text file via **secrets_path**) - - .. seealso:: :func:`generate_secret` to create a secret with sufficient entropy - - :param secrets_path: - Alternately, callers can specify a separate file where the - application-wide secrets are stored, using either of the string - formats described in **secrets**. - - :param default_tag: - Specifies which tag in **secrets** should be used as the default - for encrypting new keys. If omitted, the tags will be sorted, - and the largest tag used as the default. - - if all tags are numeric, they will be sorted numerically; - otherwise they will be sorted alphabetically. - this permits tags to be assigned numerically, - or e.g. using ``YYYY-MM-DD`` dates. - - :param encrypt_cost: - Optional time-cost factor for key encryption. - This value corresponds to log2() of the number of PBKDF2 - rounds used. - - .. warning:: - - The application secret(s) should be stored in a secure location by - your application, and each secret should contain a large amount - of entropy (to prevent brute-force attacks if the encrypted keys - are leaked). - - :func:`generate_secret` is provided as a convenience helper - to generate a new application secret of suitable size. - - Best practice is to load these values from a file via **secrets_path**, - and then have your application give up permission to read this file - once it's running. - - Public Methods - ============== - .. autoattribute:: has_secrets - .. autoattribute:: default_tag - - Semi-Private Methods - ==================== - The following methods are used internally by the :class:`TOTP` - class in order to encrypt & decrypt keys using the provided application - secrets. They will generally not be publically useful, and may have their - API changed periodically. - - .. automethod:: get_secret - .. automethod:: encrypt_key - .. automethod:: decrypt_key - """ - #======================================================================== - # instance attrs - #======================================================================== - - #: default salt size for encrypt_key() output - salt_size = 12 - - #: default cost (log2 of pbkdf2 rounds) for encrypt_key() output - #: NOTE: this is relatively low, since the majority of the security - #: relies on a high entropy secret to pass to AES. - encrypt_cost = 14 - - #: map of secret tag -> secret bytes - _secrets = None - - #: tag for default secret - default_tag = None - - #======================================================================== - # init - #======================================================================== - def __init__(self, secrets=None, default_tag=None, encrypt_cost=None, - secrets_path=None): - - # TODO: allow a lot more things to be customized from here, - # e.g. setting default TOTP constructor options. - - # - # init cost - # - if encrypt_cost is not None: - if isinstance(encrypt_cost, native_string_types): - encrypt_cost = int(encrypt_cost) - assert encrypt_cost >= 0 - self.encrypt_cost = encrypt_cost - - # - # init secrets map - # - - # load secrets from file (if needed) - if secrets_path is not None: - if secrets is not None: - raise TypeError("'secrets' and 'secrets_path' are mutually exclusive") - secrets = open(secrets_path, "rt").read() - - # parse & store secrets - secrets = self._secrets = self._parse_secrets(secrets) - - # - # init default tag/secret - # - if secrets: - if default_tag is not None: - # verify that tag is present in map - self.get_secret(default_tag) - elif all(tag.isdigit() for tag in secrets): - default_tag = max(secrets, key=int) - else: - default_tag = max(secrets) - self.default_tag = default_tag - - def _parse_secrets(self, source): - """ - parse 'secrets' parameter - - :returns: - Dict[tag:str, secret:bytes] - """ - # parse string formats - # to make this easy to pass in configuration from a separate file, - # 'secrets' can be string using two formats -- json & "tag:value\n" - check_type = True - if isinstance(source, native_string_types): - if source.lstrip().startswith(("[", "{")): - # json list / dict - source = json.loads(source) - elif "\n" in source and ":" in source: - # multiline string containing series of "tag: value\n" rows; - # empty and "#\n" rows are ignored - def iter_pairs(source): - for line in source.splitlines(): - line = line.strip() - if line and not line.startswith("#"): - tag, secret = line.split(":", 1) - yield tag.strip(), secret.strip() - source = iter_pairs(source) - check_type = False - else: - raise ValueError("unrecognized secrets string format") - - # ensure we have iterable of (tag, value) pairs - # XXX: could support lists/iterable, but not yet needed... - # if isinstance(source, list) or isinstance(source, collections.Iterator): - # pass - if source is None: - return {} - elif isinstance(source, dict): - source = source.items() - elif check_type: - raise TypeError("'secrets' must be mapping, or list of items") - - # parse into final dict, normalizing contents - return dict(self._parse_secret_pair(tag, value) - for tag, value in source) - - def _parse_secret_pair(self, tag, value): - if isinstance(tag, native_string_types): - pass - elif isinstance(tag, int): - tag = str(tag) - else: - raise TypeError("tag must be unicode/string: %r" % (tag,)) - if not _tag_re.match(tag): - raise ValueError("tag contains invalid characters: %r" % (tag,)) - if not isinstance(value, bytes): - value = to_bytes(value, param="secret %r" % (tag,)) - if not value: - raise ValueError("tag contains empty secret: %r" % (tag,)) - return tag, value - - #======================================================================== - # accessing secrets - #======================================================================== - - @property - def has_secrets(self): - """whether at least one application secret is present""" - return self.default_tag is not None - - def get_secret(self, tag): - """ - resolve a secret tag to the secret (as bytes). - throws a KeyError if not found. - """ - secrets = self._secrets - if not secrets: - raise KeyError("no application secrets configured") - try: - return secrets[tag] - except KeyError: - raise suppress_cause(KeyError("unknown secret tag: %r" % (tag,))) - - #======================================================================== - # encrypted key helpers -- used internally by TOTP - #======================================================================== - - @staticmethod - def _cipher_aes_key(value, secret, salt, cost, decrypt=False): - """ - Internal helper for :meth:`encrypt_key` -- - handles lowlevel encryption/decryption. - - Algorithm details: - - This function uses PBKDF2-HMAC-SHA256 to generate a 32-byte AES key - and a 16-byte IV from the application secret & random salt. - It then uses AES-256-CTR to encrypt/decrypt the TOTP key. - - CTR mode was chosen over CBC because the main attack scenario here - is that the attacker has stolen the database, and is trying to decrypt a TOTP key - (the plaintext value here). To make it hard for them, we want every password - to decrypt to a potentially valid key -- thus need to avoid any authentication - or padding oracle attacks. While some random padding construction could be devised - to make this work for CBC mode, a stream cipher mode is just plain simpler. - OFB/CFB modes would also work here, but seeing as they have malleability - and cyclic issues (though remote and barely relevant here), - CTR was picked as the best overall choice. - """ - # make sure backend AES support is available - if _cg_ciphers is None: - raise RuntimeError("TOTP encryption requires 'cryptography' package " - "(https://cryptography.io)") - - # use pbkdf2 to derive both key (32 bytes) & iv (16 bytes) - # NOTE: this requires 2 sha256 blocks to be calculated. - keyiv = pbkdf2_hmac("sha256", secret, salt=salt, rounds=(1 << cost), keylen=48) - - # use AES-256-CTR to encrypt/decrypt input value - cipher = _cg_ciphers.Cipher(_cg_ciphers.algorithms.AES(keyiv[:32]), - _cg_ciphers.modes.CTR(keyiv[32:]), - _cg_default_backend()) - ctx = cipher.decryptor() if decrypt else cipher.encryptor() - return ctx.update(value) + ctx.finalize() - - def encrypt_key(self, key): - """ - Helper used to encrypt TOTP keys for storage. - - :param key: - TOTP key to encrypt, as raw bytes. - - :returns: - dict containing encrypted TOTP key & configuration parameters. - this format should be treated as opaque, and potentially subject - to change, though it is designed to be easily serialized/deserialized - (e.g. via JSON). - - .. note:: - - This function requires installation of the external - `cryptography `_ package. - - To give some algorithm details: This function uses AES-256-CTR to encrypt - the provided data. It takes the application secret and randomly generated salt, - and uses PBKDF2-HMAC-SHA256 to combine them and generate the AES key & IV. - """ - if not key: - raise ValueError("no key provided") - salt = getrandbytes(rng, self.salt_size) - cost = self.encrypt_cost - tag = self.default_tag - if not tag: - raise TypeError("no application secrets configured, can't encrypt OTP key") - ckey = self._cipher_aes_key(key, self.get_secret(tag), salt, cost) - # XXX: switch to base64? - return dict(v=1, c=cost, t=tag, s=b32encode(salt), k=b32encode(ckey)) - - def decrypt_key(self, enckey): - """ - Helper used to decrypt TOTP keys from storage format. - Consults configured secrets to decrypt key. - - :param source: - source object, as returned by :meth:`encrypt_key`. - - :returns: - ``(key, needs_recrypt)`` -- - - **key** will be the decrypted key, as bytes. - - **needs_recrypt** will be a boolean flag indicating - whether encryption cost or default tag is too old, - and henace that key needs re-encrypting before storing. - - .. note:: - - This function requires installation of the external - `cryptography `_ package. - """ - if not isinstance(enckey, dict): - raise TypeError("'enckey' must be dictionary") - version = enckey.get("v", None) - needs_recrypt = False - if version == 1: - _cipher_key = self._cipher_aes_key - else: - raise ValueError("missing / unrecognized 'enckey' version: %r" % (version,)) - tag = enckey['t'] - cost = enckey['c'] - key = _cipher_key( - value=b32decode(enckey['k']), - secret=self.get_secret(tag), - salt=b32decode(enckey['s']), - cost=cost, - ) - if cost != self.encrypt_cost or tag != self.default_tag: - needs_recrypt = True - return key, needs_recrypt - - #============================================================================= - # eoc - #============================================================================= - -#============================================================================= -# TOTP class -#============================================================================= - -#: helper to convert HOTP counter to bytes -_pack_uint64 = struct.Struct(">Q").pack - -#: helper to extract value from HOTP digest -_unpack_uint32 = struct.Struct(">I").unpack - -#: dummy bytes used as temp key for .using() method -_DUMMY_KEY = b"\x00" * 16 - -class TOTP(object): - """ - Helper for generating and verifying TOTP codes. - - Given a secret key and set of configuration options, this object - offers methods for token generation, token validation, and serialization. - It can also be used to track important persistent TOTP state, - such as the last counter used. - - This class accepts the following options - (only **key** and **format** may be specified as positional arguments). - - :arg str key: - The secret key to use. By default, should be encoded as - a base32 string (see **format** for other encodings). - - Exactly one of **key** or ``new=True`` must be specified. - - :arg str format: - The encoding used by the **key** parameter. May be one of: - ``"base32"`` (base32-encoded string), - ``"hex"`` (hexadecimal string), or ``"raw"`` (raw bytes). - Defaults to ``"base32"``. - - :param bool new: - If ``True``, a new key will be generated using :class:`random.SystemRandom`. - - Exactly one ``new=True`` or **key** must be specified. - - :param str label: - Label to associate with this token when generating a URI. - Displayed to user by most OTP client applications (e.g. Google Authenticator), - and typically has format such as ``"John Smith"`` or ``"jsmith@webservice.example.org"``. - Defaults to ``None``. - See :meth:`to_uri` for details. - - :param str issuer: - String identifying the token issuer (e.g. the domain name of your service). - Used internally by some OTP client applications (e.g. Google Authenticator) to distinguish entries - which otherwise have the same label. - Optional but strongly recommended if you're rendering to a URI. - Defaults to ``None``. - See :meth:`to_uri` for details. - - :param int size: - Number of bytes when generating new keys. Defaults to size of hash algorithm (e.g. 20 for SHA1). - - .. warning:: - - Overriding the default values for ``digits``, ``period``, or ``alg`` may - cause problems with some OTP client programs (such as Google Authenticator), - which may have these defaults hardcoded. - - :param int digits: - The number of digits in the generated / accepted tokens. Defaults to ``6``. - Must be in range [6 .. 10]. - - .. rst-class:: inline-title - .. caution:: - Due to a limitation of the HOTP algorithm, the 10th digit can only take on values 0 .. 2, - and thus offers very little extra security. - - :param str alg: - Name of hash algorithm to use. Defaults to ``"sha1"``. - ``"sha256"`` and ``"sha512"`` are also accepted, per :rfc:`6238`. - - :param int period: - The time-step period to use, in integer seconds. Defaults to ``30``. - - .. - See the passlib documentation for a full list of attributes & methods. - """ - #============================================================================= - # class attrs - #============================================================================= - - #: minimum number of bytes to allow in key, enforced by passlib. - # XXX: see if spec says anything relevant to this. - _min_key_size = 10 - - #: minimum & current serialization version (may be set independently by subclasses) - min_json_version = json_version = 1 - - #: AppWallet that this class will use for encrypting/decrypting keys. - #: (can be overwritten via the :meth:`TOTP.using()` constructor) - wallet = None - - #: function to get system time in seconds, as needed by :meth:`generate` and :meth:`verify`. - #: defaults to :func:`time.time`, but can be overridden on a per-instance basis. - now = _time.time - - #============================================================================= - # instance attrs - #============================================================================= - - #--------------------------------------------------------------------------- - # configuration attrs - #--------------------------------------------------------------------------- - - #: [private] secret key as raw :class:`!bytes` - #: see .key property for public access. - _key = None - - #: [private] cached copy of encrypted secret, - #: so .to_json() doesn't have to re-encrypt on each call. - _encrypted_key = None - - #: [private] cached copy of keyed HMAC function, - #: so ._generate() doesn't have to rebuild this each time - #: ._find_match() invokes it. - _keyed_hmac = None - - #: number of digits in the generated tokens. - digits = 6 - - #: name of hash algorithm in use (e.g. ``"sha1"``) - alg = "sha1" - - #: default label for :meth:`to_uri` - label = None - - #: default issuer for :meth:`to_uri` - issuer = None - - #: number of seconds per counter step. - #: *(TOTP uses an internal time-derived counter which - #: increments by 1 every* :attr:`!period` *seconds)*. - period = 30 - - #--------------------------------------------------------------------------- - # state attrs - #--------------------------------------------------------------------------- - - #: Flag set by deserialization methods to indicate the object needs to be re-serialized. - #: This can be for a number of reasons -- encoded using deprecated format, - #: or encrypted using a deprecated key or too few rounds. - changed = False - - #============================================================================= - # prototype construction - #============================================================================= - @classmethod - def using(cls, digits=None, alg=None, period=None, - issuer=None, wallet=None, now=None, **kwds): - """ - Dynamically create subtype of :class:`!TOTP` class - which has the specified defaults set. - - :parameters: **digits, alg, period, issuer**: - - All these options are the same as in the :class:`TOTP` constructor, - and the resulting class will use any values you specify here - as the default for all TOTP instances it creates. - - :param wallet: - Optional :class:`AppWallet` that will be used for encrypting/decrypting keys. - - :param secrets, secrets_path, encrypt_cost: - - If specified, these options will be passed to the :class:`AppWallet` constructor, - allowing you to directly specify the secret keys that should be used - to encrypt & decrypt stored keys. - - :returns: - subclass of :class:`!TOTP`. - - This method is useful for creating a TOTP class configured - to use your application's secrets for encrypting & decrypting - keys, as well as create new keys using it's desired configuration defaults. - - As an example:: - - >>> # your application can create a custom class when it initializes - >>> from passlib.totp import TOTP, generate_secret - >>> TotpFactory = TOTP.using(secrets={"1": generate_secret()}) - - >>> # subsequent TOTP objects created from this factory - >>> # will use the specified secrets to encrypt their keys... - >>> totp = TotpFactory.new() - >>> totp.to_dict() - {'enckey': {'c': 14, - 'k': 'H77SYXWORDPGVOQTFRR2HFUB3C45XXI7', - 's': 'G5DOQPIHIBUM2OOHHADQ', - 't': '1', - 'v': 1}, - 'type': 'totp', - 'v': 1} - - .. seealso:: :ref:`totp-creation` and :ref:`totp-storing-instances` tutorials for a usage example - """ - # XXX: could add support for setting default match 'window' and 'reuse' policy - - # :param now: - # Optional callable that should return current time for generator to use. - # Default to :func:`time.time`. This optional is generally not needed, - # and is mainly present for examples & unit-testing. - - subcls = type("TOTP", (cls,), {}) - - def norm_param(attr, value): - """ - helper which uses constructor to validate parameter value. - it returns corresponding attribute, so we use normalized value. - """ - # NOTE: this creates *subclass* instance, - # so normalization takes into account any custom params - # already stored. - kwds = dict(key=_DUMMY_KEY, format="raw") - kwds[attr] = value - obj = subcls(**kwds) - return getattr(obj, attr) - - if digits is not None: - subcls.digits = norm_param("digits", digits) - - if alg is not None: - subcls.alg = norm_param("alg", alg) - - if period is not None: - subcls.period = norm_param("period", period) - - # XXX: add default size as configurable parameter? - - if issuer is not None: - subcls.issuer = norm_param("issuer", issuer) - - if kwds: - subcls.wallet = AppWallet(**kwds) - if wallet: - raise TypeError("'wallet' and 'secrets' keywords are mutually exclusive") - elif wallet is not None: - if not isinstance(wallet, AppWallet): - raise exc.ExpectedTypeError(wallet, AppWallet, "wallet") - subcls.wallet = wallet - - if now is not None: - assert isinstance(now(), num_types) and now() >= 0, \ - "now() function must return non-negative int/float" - subcls.now = staticmethod(now) - - return subcls - - #============================================================================= - # init - #============================================================================= - - @classmethod - def new(cls, **kwds): - """ - convenience alias for creating new TOTP key, same as ``TOTP(new=True)`` - """ - return cls(new=True, **kwds) - - def __init__(self, key=None, format="base32", - # keyword only... - new=False, digits=None, alg=None, size=None, period=None, - label=None, issuer=None, changed=False, - **kwds): - super(TOTP, self).__init__(**kwds) - if changed: - self.changed = changed - - # validate & normalize alg - info = lookup_hash(alg or self.alg) - self.alg = info.name - digest_size = info.digest_size - if digest_size < 4: - raise RuntimeError("%r hash digest too small" % alg) - - # parse or generate new key - if new: - # generate new key - if key: - raise TypeError("'key' and 'new=True' are mutually exclusive") - if size is None: - # default to digest size, per RFC 6238 Section 5.1 - size = digest_size - elif size > digest_size: - # not forbidden by spec, but would just be wasted bytes. - # maybe just warn about this? - raise ValueError("'size' should be less than digest size " - "(%d)" % digest_size) - self.key = getrandbytes(rng, size) - elif not key: - raise TypeError("must specify either an existing 'key', or 'new=True'") - elif format == "encrypted": - # NOTE: this handles decrypting & setting '.key' - self.encrypted_key = key - elif key: - # use existing key, encoded using specified - self.key = _decode_bytes(key, format) - - # enforce min key size - if len(self.key) < self._min_key_size: - # only making this fatal for new=True, - # so that existing (but ridiculously small) keys can still be used. - msg = "for security purposes, secret key must be >= %d bytes" % self._min_key_size - if new: - raise ValueError(msg) - else: - warn(msg, exc.PasslibSecurityWarning, stacklevel=1) - - # validate digits - if digits is None: - digits = self.digits - if not isinstance(digits, int_types): - raise TypeError("digits must be an integer, not a %r" % type(digits)) - if digits < 6 or digits > 10: - raise ValueError("digits must in range(6,11)") - self.digits = digits - - # validate label - if label: - self._check_label(label) - self.label = label - - # validate issuer - if issuer: - self._check_issuer(issuer) - self.issuer = issuer - - # init period - if period is not None: - self._check_serial(period, "period", minval=1) - self.period = period - - #============================================================================= - # helpers to verify value types & ranges - #============================================================================= - - @staticmethod - def _check_serial(value, param, minval=0): - """ - check that serial value (e.g. 'counter') is non-negative integer - """ - if not isinstance(value, int_types): - raise exc.ExpectedTypeError(value, "int", param) - if value < minval: - raise ValueError("%s must be >= %d" % (param, minval)) - - @staticmethod - def _check_label(label): - """ - check that label doesn't contain chars forbidden by KeyURI spec - """ - if label and ":" in label: - raise ValueError("label may not contain ':'") - - @staticmethod - def _check_issuer(issuer): - """ - check that issuer doesn't contain chars forbidden by KeyURI spec - """ - if issuer and ":" in issuer: - raise ValueError("issuer may not contain ':'") - - #============================================================================= - # key attributes - #============================================================================= - - #------------------------------------------------------------------ - # raw key - #------------------------------------------------------------------ - @property - def key(self): - """ - secret key as raw bytes - """ - return self._key - - @key.setter - def key(self, value): - # set key - if not isinstance(value, bytes): - raise exc.ExpectedTypeError(value, bytes, "key") - self._key = value - - # clear cached properties derived from key - self._encrypted_key = self._keyed_hmac = None - - #------------------------------------------------------------------ - # encrypted key - #------------------------------------------------------------------ - @property - def encrypted_key(self): - """ - secret key, encrypted using application secret. - this match the output of :meth:`AppWallet.encrypt_key`, - and should be treated as an opaque json serializable object. - """ - enckey = self._encrypted_key - if enckey is None: - wallet = self.wallet - if not wallet: - raise TypeError("no application secrets present, can't encrypt TOTP key") - enckey = self._encrypted_key = wallet.encrypt_key(self.key) - return enckey - - @encrypted_key.setter - def encrypted_key(self, value): - wallet = self.wallet - if not wallet: - raise TypeError("no application secrets present, can't decrypt TOTP key") - self.key, needs_recrypt = wallet.decrypt_key(value) - if needs_recrypt: - # mark as changed so it gets re-encrypted & written to db - self.changed = True - else: - # cache encrypted key for re-use - self._encrypted_key = value - - #------------------------------------------------------------------ - # pretty-printed / encoded key helpers - #------------------------------------------------------------------ - - @property - def hex_key(self): - """ - secret key encoded as hexadecimal string - """ - return bascii_to_str(base64.b16encode(self.key)).lower() - - @property - def base32_key(self): - """ - secret key encoded as base32 string - """ - return b32encode(self.key) - - def pretty_key(self, format="base32", sep="-"): - """ - pretty-print the secret key. - - This is mainly useful for situations where the user cannot get the qrcode to work, - and must enter the key manually into their TOTP client. It tries to format - the key in a manner that is easier for humans to read. - - :param format: - format to output secret key. ``"hex"`` and ``"base32"`` are both accepted. - - :param sep: - separator to insert to break up key visually. - can be any of ``"-"`` (the default), ``" "``, or ``False`` (no separator). - - :return: - key as native string. - - Usage example:: - - >>> t = TOTP('s3jdvb7qd2r7jpxx') - >>> t.pretty_key() - 'S3JD-VB7Q-D2R7-JPXX' - """ - if format == "hex" or format == "base16": - key = self.hex_key - elif format == "base32": - key = self.base32_key - else: - raise ValueError("unknown byte-encoding format: %r" % (format,)) - if sep: - key = group_string(key, sep) - return key - - #============================================================================= - # time & token parsing - #============================================================================= - - @classmethod - def normalize_time(cls, time): - """ - Normalize time value to unix epoch seconds. - - :arg time: - Can be ``None``, :class:`!datetime`, - or unix epoch timestamp as :class:`!float` or :class:`!int`. - If ``None``, uses current system time. - Naive datetimes are treated as UTC. - - :returns: - unix epoch timestamp as :class:`int`. - """ - if isinstance(time, int_types): - return time - elif isinstance(time, float): - return int(time) - elif time is None: - return int(cls.now()) - elif hasattr(time, "utctimetuple"): - # coerce datetime to UTC timestamp - # NOTE: utctimetuple() assumes naive datetimes are in UTC - # NOTE: we explicitly *don't* want microseconds. - return calendar.timegm(time.utctimetuple()) - else: - raise exc.ExpectedTypeError(time, "int, float, or datetime", "time") - - def _time_to_counter(self, time): - """ - convert timestamp to HOTP counter using :attr:`period`. - """ - return time // self.period - - def _counter_to_time(self, counter): - """ - convert HOTP counter to timestamp using :attr:`period`. - """ - return counter * self.period - - @hybrid_method - def normalize_token(self_or_cls, token): - """ - Normalize OTP token representation: - strips whitespace, converts integers to a zero-padded string, - validates token content & number of digits. - - This is a hybrid method -- it can be called at the class level, - as ``TOTP.normalize_token()``, or the instance level as ``TOTP().normalize_token()``. - It will normalize to the instance-specific number of :attr:`~TOTP.digits`, - or use the class default. - - :arg token: - token as ascii bytes, unicode, or an integer. - - :raises ValueError: - if token has wrong number of digits, or contains non-numeric characters. - - :returns: - token as :class:`!unicode` string, containing only digits 0-9. - """ - digits = self_or_cls.digits - if isinstance(token, int_types): - token = u("%0*d") % (digits, token) - else: - token = to_unicode(token, param="token") - token = _clean_re.sub(u(""), token) - if not token.isdigit(): - raise MalformedTokenError("Token must contain only the digits 0-9") - if len(token) != digits: - raise MalformedTokenError("Token must have exactly %d digits" % digits) - return token - - #============================================================================= - # token generation - #============================================================================= - -# # debug helper -# def generate_range(self, size, time=None): -# counter = self._time_to_counter(time) - (size + 1) // 2 -# end = counter + size -# while counter <= end: -# token = self._generate(counter) -# yield TotpToken(self, token, counter) -# counter += 1 - - def generate(self, time=None): - """ - Generate token for specified time - (uses current time if none specified). - - :arg time: - Can be ``None``, a :class:`!datetime`, - or class:`!float` / :class:`!int` unix epoch timestamp. - If ``None`` (the default), uses current system time. - Naive datetimes are treated as UTC. - - :returns: - - A :class:`TotpToken` instance, which can be treated - as a sequence of ``(token, expire_time)`` -- see that class - for more details. - - Usage example:: - - >>> # generate a new token, wrapped in a TotpToken instance... - >>> otp = TOTP('s3jdvb7qd2r7jpxx') - >>> otp.generate(1419622739) - - - >>> # when you just need the token... - >>> otp.generate(1419622739).token - '897212' - """ - time = self.normalize_time(time) - counter = self._time_to_counter(time) - if counter < 0: - raise ValueError("timestamp must be >= 0") - token = self._generate(counter) - return TotpToken(self, token, counter) - - def _generate(self, counter): - """ - base implementation of HOTP token generation algorithm. - - :arg counter: HOTP counter, as non-negative integer - :returns: token as unicode string - """ - # generate digest - assert isinstance(counter, int_types), "counter must be integer" - assert counter >= 0, "counter must be non-negative" - keyed_hmac = self._keyed_hmac - if keyed_hmac is None: - keyed_hmac = self._keyed_hmac = compile_hmac(self.alg, self.key) - digest = keyed_hmac(_pack_uint64(counter)) - digest_size = keyed_hmac.digest_info.digest_size - assert len(digest) == digest_size, "digest_size: sanity check failed" - - # derive 31-bit token value - assert digest_size >= 20, "digest_size: sanity check 2 failed" # otherwise 0xF+4 will run off end of hash. - offset = byte_elem_value(digest[-1]) & 0xF - value = _unpack_uint32(digest[offset:offset+4])[0] & 0x7fffffff - - # render to decimal string, return last chars - # NOTE: the 10'th digit is not as secure, as it can only take on values 0-2, not 0-9, - # due to 31-bit mask on int ">I". But some servers / clients use it :| - # if 31-bit mask removed (which breaks spec), would only get values 0-4. - digits = self.digits - assert 0 < digits < 11, "digits: sanity check failed" - return (u("%0*d") % (digits, value))[-digits:] - - #============================================================================= - # token verification - #============================================================================= - - @classmethod - def verify(cls, token, source, **kwds): - r""" - Convenience wrapper around :meth:`TOTP.from_source` and :meth:`TOTP.match`. - - This parses a TOTP key & configuration from the specified source, - and tries and match the token. - It's designed to parallel the :meth:`passlib.ifc.PasswordHash.verify` method. - - :param token: - Token string to match. - - :param source: - Serialized TOTP key. - Can be anything accepted by :meth:`TOTP.from_source`. - - :param \*\*kwds: - All additional keywords passed to :meth:`TOTP.match`. - - :return: - A :class:`TotpMatch` instance, or raises a :exc:`TokenError`. - """ - return cls.from_source(source).match(token, **kwds) - - def match(self, token, time=None, window=30, skew=0, last_counter=None): - """ - Match TOTP token against specified timestamp. - Searches within a window before & after the provided time, - in order to account for transmission delay and small amounts of skew in the client's clock. - - :arg token: - Token to validate. - may be integer or string (whitespace and hyphens are ignored). - - :param time: - Unix epoch timestamp, can be any of :class:`!float`, :class:`!int`, or :class:`!datetime`. - if ``None`` (the default), uses current system time. - *this should correspond to the time the token was received from the client*. - - :param int window: - How far backward and forward in time to search for a match. - Measured in seconds. Defaults to ``30``. Typically only useful if set - to multiples of :attr:`period`. - - :param int skew: - Adjust timestamp by specified value, to account for excessive - client clock skew. Measured in seconds. Defaults to ``0``. - - Negative skew (the common case) indicates transmission delay, - and/or that the client clock is running behind the server. - - Positive skew indicates the client clock is running ahead of the server - (and by enough that it cancels out any negative skew added by - the transmission delay). - - You should ensure the server clock uses a reliable time source such as NTP, - so that only the client clock's inaccuracy needs to be accounted for. - - This is an advanced parameter that should usually be left at ``0``; - The **window** parameter is usually enough to account - for any observed transmission delay. - - :param last_counter: - Optional value of last counter value that was successfully used. - If specified, verify will never search earlier counters, - no matter how large the window is. - - Useful when client has previously authenticated, - and thus should never provide a token older than previously - verified value. - - :raises ~passlib.exc.TokenError: - - If the token is malformed, fails to match, or has already been used. - - :returns TotpMatch: - - Returns a :class:`TotpMatch` instance on successful match. - Can be treated as tuple of ``(counter, time)``. - Raises error if token is malformed / can't be verified. - - Usage example:: - - >>> totp = TOTP('s3jdvb7qd2r7jpxx') - - >>> # valid token for this time period - >>> totp.match('897212', 1419622729) - - - >>> # token from counter step 30 sec ago (within allowed window) - >>> totp.match('000492', 1419622729) - - - >>> # invalid token -- token from 60 sec ago (outside of window) - >>> totp.match('760389', 1419622729) - Traceback: - ... - InvalidTokenError: Token did not match - """ - time = self.normalize_time(time) - self._check_serial(window, "window") - - client_time = time + skew - if last_counter is None: - last_counter = -1 - start = max(last_counter, self._time_to_counter(client_time - window)) - end = self._time_to_counter(client_time + window) + 1 - # XXX: could pass 'expected = _time_to_counter(client_time + TRANSMISSION_DELAY)' - # to the _find_match() method, would help if window set to very large value. - - counter = self._find_match(token, start, end) - assert counter >= last_counter, "sanity check failed: counter went backward" - - if counter == last_counter: - raise UsedTokenError(expire_time=(last_counter + 1) * self.period) - - # NOTE: By returning match tied to