diff --git a/src/googleapiclient/__init__.py b/src/googleapiclient/__init__.py index ab076a4f..26442ace 100644 --- a/src/googleapiclient/__init__.py +++ b/src/googleapiclient/__init__.py @@ -12,4 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.5.0" +__version__ = "1.5.1" + +# Set default logging handler to avoid "No handler found" warnings. +import logging + +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/src/googleapiclient/discovery.py b/src/googleapiclient/discovery.py index cee56284..ecb75fa4 100644 --- a/src/googleapiclient/discovery.py +++ b/src/googleapiclient/discovery.py @@ -82,6 +82,9 @@ URITEMPLATE = re.compile('{[^}]*}') VARNAME = re.compile('[a-zA-Z0-9_-]+') DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/' '{api}/{apiVersion}/rest') +V1_DISCOVERY_URI = DISCOVERY_URI +V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?' + 'version={apiVersion}') DEFAULT_METHOD_DOC = 'A description of how to use this function' HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH']) _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} @@ -196,21 +199,23 @@ def build(serviceName, if http is None: http = httplib2.Http() - requested_url = uritemplate.expand(discoveryServiceUrl, params) + for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,): + requested_url = uritemplate.expand(discovery_url, params) - try: - content = _retrieve_discovery_doc(requested_url, http, cache_discovery, - cache) - except HttpError as e: - if e.resp.status == http_client.NOT_FOUND: - raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName, - version)) - else: - raise e + try: + content = _retrieve_discovery_doc(requested_url, http, cache_discovery, + cache) + return build_from_document(content, base=discovery_url, http=http, + developerKey=developerKey, model=model, requestBuilder=requestBuilder, + credentials=credentials) + except HttpError as e: + if e.resp.status == http_client.NOT_FOUND: + continue + else: + raise e - return build_from_document(content, base=discoveryServiceUrl, http=http, - developerKey=developerKey, model=model, requestBuilder=requestBuilder, - credentials=credentials) + raise UnknownApiNameOrVersion( + "name: %s version: %s" % (serviceName, version)) def _retrieve_discovery_doc(url, http, cache_discovery, cache=None): diff --git a/src/googleapiclient/discovery_cache/__init__.py b/src/googleapiclient/discovery_cache/__init__.py index c56fd659..f86a06de 100644 --- a/src/googleapiclient/discovery_cache/__init__.py +++ b/src/googleapiclient/discovery_cache/__init__.py @@ -19,6 +19,9 @@ from __future__ import absolute_import import logging import datetime + +LOGGER = logging.getLogger(__name__) + DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24 # 1 day @@ -38,5 +41,5 @@ def autodetect(): from . import file_cache return file_cache.cache except Exception as e: - logging.warning(e, exc_info=True) + LOGGER.warning(e, exc_info=True) return None diff --git a/src/googleapiclient/discovery_cache/appengine_memcache.py b/src/googleapiclient/discovery_cache/appengine_memcache.py index a521fc39..7e43e66c 100644 --- a/src/googleapiclient/discovery_cache/appengine_memcache.py +++ b/src/googleapiclient/discovery_cache/appengine_memcache.py @@ -23,6 +23,9 @@ from google.appengine.api import memcache from . import base from ..discovery_cache import DISCOVERY_DOC_MAX_AGE + +LOGGER = logging.getLogger(__name__) + NAMESPACE = 'google-api-client' @@ -41,12 +44,12 @@ class Cache(base.Cache): try: return memcache.get(url, namespace=NAMESPACE) except Exception as e: - logging.warning(e, exc_info=True) + LOGGER.warning(e, exc_info=True) def set(self, url, content): try: memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE) except Exception as e: - logging.warning(e, exc_info=True) + LOGGER.warning(e, exc_info=True) cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE) diff --git a/src/googleapiclient/discovery_cache/file_cache.py b/src/googleapiclient/discovery_cache/file_cache.py index 8b0301d2..31434db4 100644 --- a/src/googleapiclient/discovery_cache/file_cache.py +++ b/src/googleapiclient/discovery_cache/file_cache.py @@ -29,12 +29,16 @@ import os import tempfile import threading -from oauth2client.contrib.locked_file import LockedFile +try: + from oauth2client.contrib.locked_file import LockedFile +except ImportError: + # oauth2client < 2.0.0 + from oauth2client.locked_file import LockedFile from . import base from ..discovery_cache import DISCOVERY_DOC_MAX_AGE -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) FILENAME = 'google-api-python-client-discovery-doc.cache' EPOCH = datetime.datetime.utcfromtimestamp(0) @@ -84,7 +88,7 @@ class Cache(base.Cache): # If we can not obtain the lock, other process or thread must # have initialized the file. except Exception as e: - logging.warning(e, exc_info=True) + LOGGER.warning(e, exc_info=True) finally: f.unlock_and_close() @@ -100,10 +104,10 @@ class Cache(base.Cache): return content return None else: - logger.debug('Could not obtain a lock for the cache file.') + LOGGER.debug('Could not obtain a lock for the cache file.') return None except Exception as e: - logger.warning(e, exc_info=True) + LOGGER.warning(e, exc_info=True) finally: f.unlock_and_close() @@ -122,9 +126,9 @@ class Cache(base.Cache): f.file_handle().seek(0) json.dump(cache, f.file_handle()) else: - logger.debug('Could not obtain a lock for the cache file.') + LOGGER.debug('Could not obtain a lock for the cache file.') except Exception as e: - logger.warning(e, exc_info=True) + LOGGER.warning(e, exc_info=True) finally: f.unlock_and_close() diff --git a/src/googleapiclient/http.py b/src/googleapiclient/http.py index 2245e8d7..5181badb 100644 --- a/src/googleapiclient/http.py +++ b/src/googleapiclient/http.py @@ -20,6 +20,7 @@ actuall HTTP request. """ from __future__ import absolute_import import six +from six.moves import http_client from six.moves import range __author__ = 'jcgregorio@google.com (Joe Gregorio)' @@ -36,11 +37,19 @@ import logging import mimetypes import os import random -import ssl +import socket import sys import time import uuid +# TODO(issue 221): Remove this conditional import jibbajabba. +try: + import ssl +except ImportError: + _ssl_SSLError = object() +else: + _ssl_SSLError = ssl.SSLError + from email.generator import Generator from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart @@ -57,10 +66,57 @@ from googleapiclient.model import JsonModel from oauth2client import util +LOGGER = logging.getLogger(__name__) + DEFAULT_CHUNK_SIZE = 512*1024 MAX_URI_LENGTH = 2048 +_TOO_MANY_REQUESTS = 429 + + +def _should_retry_response(resp_status, content): + """Determines whether a response should be retried. + + Args: + resp_status: The response status received. + content: The response content body. + + Returns: + True if the response should be retried, otherwise False. + """ + # Retry on 5xx errors. + if resp_status >= 500: + return True + + # Retry on 429 errors. + if resp_status == _TOO_MANY_REQUESTS: + return True + + # For 403 errors, we have to check for the `reason` in the response to + # determine if we should retry. + if resp_status == six.moves.http_client.FORBIDDEN: + # If there's no details about the 403 type, don't retry. + if not content: + return False + + # Content is in JSON format. + try: + data = json.loads(content.decode('utf-8')) + reason = data['error']['errors'][0]['reason'] + except (UnicodeDecodeError, ValueError, KeyError): + LOGGER.warning('Invalid JSON content from response: %s', content) + return False + + LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) + + # Only retry on rate limit related failures. + if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): + return True + + # Everything else is a success or non-retriable so break. + return False + def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs): @@ -82,21 +138,37 @@ def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, resp, content - Response from the http request (may be HTTP 5xx). """ resp = None + content = None for retry_num in range(num_retries + 1): if retry_num > 0: - sleep(rand() * 2**retry_num) - logging.warning( - 'Retry #%d for %s: %s %s%s' % (retry_num, req_type, method, uri, - ', following status: %d' % resp.status if resp else '')) + # Sleep before retrying. + sleep_time = rand() * 2 ** retry_num + LOGGER.warning( + 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', + sleep_time, retry_num, num_retries, req_type, method, uri, + resp.status if resp else exception) + sleep(sleep_time) try: + exception = None resp, content = http.request(uri, method, *args, **kwargs) - except ssl.SSLError: - if retry_num == num_retries: + # Retry on SSL errors and socket timeout errors. + except _ssl_SSLError as ssl_error: + exception = ssl_error + except socket.error as socket_error: + # errno's contents differ by platform, so we have to match by name. + if socket.errno.errorcode.get(socket_error.errno) not in ( + 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ): raise + exception = socket_error + + if exception: + if retry_num == num_retries: + raise exception else: continue - if resp.status < 500: + + if not _should_retry_response(resp.status, content): break return resp, content @@ -882,7 +954,7 @@ class HttpRequest(object): for retry_num in range(num_retries + 1): if retry_num > 0: self._sleep(self._rand() * 2**retry_num) - logging.warning( + LOGGER.warning( 'Retry #%d for media upload: %s %s, following status: %d' % (retry_num, self.method, self.uri, resp.status)) @@ -1632,7 +1704,7 @@ def tunnel_patch(http): headers = {} if method == 'PATCH': if 'oauth_token' in headers.get('authorization', ''): - logging.warning( + LOGGER.warning( 'OAuth 1.0 request made with Credentials after tunnel_patch.') headers['x-http-method-override'] = "PATCH" method = 'POST' diff --git a/src/googleapiclient/model.py b/src/googleapiclient/model.py index e8afb63d..dded04ea 100644 --- a/src/googleapiclient/model.py +++ b/src/googleapiclient/model.py @@ -33,6 +33,8 @@ from googleapiclient import __version__ from googleapiclient.errors import HttpError +LOGGER = logging.getLogger(__name__) + dump_request_response = False @@ -105,18 +107,18 @@ class BaseModel(Model): def _log_request(self, headers, path_params, query, body): """Logs debugging information about the request if requested.""" if dump_request_response: - logging.info('--request-start--') - logging.info('-headers-start-') + LOGGER.info('--request-start--') + LOGGER.info('-headers-start-') for h, v in six.iteritems(headers): - logging.info('%s: %s', h, v) - logging.info('-headers-end-') - logging.info('-path-parameters-start-') + LOGGER.info('%s: %s', h, v) + LOGGER.info('-headers-end-') + LOGGER.info('-path-parameters-start-') for h, v in six.iteritems(path_params): - logging.info('%s: %s', h, v) - logging.info('-path-parameters-end-') - logging.info('body: %s', body) - logging.info('query: %s', query) - logging.info('--request-end--') + LOGGER.info('%s: %s', h, v) + LOGGER.info('-path-parameters-end-') + LOGGER.info('body: %s', body) + LOGGER.info('query: %s', query) + LOGGER.info('--request-end--') def request(self, headers, path_params, query_params, body_value): """Updates outgoing requests with a serialized body. @@ -176,12 +178,12 @@ class BaseModel(Model): def _log_response(self, resp, content): """Logs debugging information about the response if requested.""" if dump_request_response: - logging.info('--response-start--') + LOGGER.info('--response-start--') for h, v in six.iteritems(resp): - logging.info('%s: %s', h, v) + LOGGER.info('%s: %s', h, v) if content: - logging.info(content) - logging.info('--response-end--') + LOGGER.info(content) + LOGGER.info('--response-end--') def response(self, resp, content): """Convert the response wire format into a Python object. @@ -206,7 +208,7 @@ class BaseModel(Model): return self.no_content_response return self.deserialize(content) else: - logging.debug('Content from bad request was: %s' % content) + LOGGER.debug('Content from bad request was: %s' % content) raise HttpError(resp, content) def serialize(self, body_value):