Upgrade to googleapiclient 1.5.1

src/googleapiclient/__init__.py

@@ -12,4 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.5.0"
+__version__ = "1.5.1"
+
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+
+try:  # Python 2.7+
+    from logging import NullHandler
+except ImportError:
+    class NullHandler(logging.Handler):
+        def emit(self, record):
+            pass
+
+logging.getLogger(__name__).addHandler(NullHandler())

src/googleapiclient/discovery.py

@@ -82,6 +82,9 @@ URITEMPLATE = re.compile('{[^}]*}')
 VARNAME = re.compile('[a-zA-Z0-9_-]+')
 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
                  '{api}/{apiVersion}/rest')
+V1_DISCOVERY_URI = DISCOVERY_URI
+V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?'
+                    'version={apiVersion}')
 DEFAULT_METHOD_DOC = 'A description of how to use this function'
 HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
 _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
@@ -196,21 +199,23 @@ def build(serviceName,
   if http is None:
     http = httplib2.Http()
 
-  requested_url = uritemplate.expand(discoveryServiceUrl, params)
+  for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,):
+    requested_url = uritemplate.expand(discovery_url, params)
 
-  try:
-    content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
-                                      cache)
-  except HttpError as e:
-    if e.resp.status == http_client.NOT_FOUND:
-      raise UnknownApiNameOrVersion("name: %s  version: %s" % (serviceName,
-                                                               version))
-    else:
-      raise e
+    try:
+      content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
+                                        cache)
+      return build_from_document(content, base=discovery_url, http=http,
+          developerKey=developerKey, model=model, requestBuilder=requestBuilder,
+          credentials=credentials)
+    except HttpError as e:
+      if e.resp.status == http_client.NOT_FOUND:
+        continue
+      else:
+        raise e
 
-  return build_from_document(content, base=discoveryServiceUrl, http=http,
-      developerKey=developerKey, model=model, requestBuilder=requestBuilder,
-      credentials=credentials)
+  raise UnknownApiNameOrVersion(
+        "name: %s  version: %s" % (serviceName, version))
 
 
 def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):

src/googleapiclient/discovery_cache/__init__.py

@@ -19,6 +19,9 @@ from __future__ import absolute_import
 import logging
 import datetime
 
+
+LOGGER = logging.getLogger(__name__)
+
 DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24  # 1 day
 
 
@@ -38,5 +41,5 @@ def autodetect():
       from . import file_cache
       return file_cache.cache
     except Exception as e:
-      logging.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
       return None

src/googleapiclient/discovery_cache/appengine_memcache.py

@@ -23,6 +23,9 @@ from google.appengine.api import memcache
 from . import base
 from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
 
+
+LOGGER = logging.getLogger(__name__)
+
 NAMESPACE = 'google-api-client'
 
 
@@ -41,12 +44,12 @@ class Cache(base.Cache):
     try:
       return memcache.get(url, namespace=NAMESPACE)
     except Exception as e:
-      logging.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
 
   def set(self, url, content):
     try:
       memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
     except Exception as e:
-      logging.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
 
 cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE)

src/googleapiclient/discovery_cache/file_cache.py

@@ -29,12 +29,16 @@ import os
 import tempfile
 import threading
 
-from oauth2client.contrib.locked_file import LockedFile
+try:
+  from oauth2client.contrib.locked_file import LockedFile
+except ImportError:
+  # oauth2client < 2.0.0
+  from oauth2client.locked_file import LockedFile
 
 from . import base
 from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
 
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 FILENAME = 'google-api-python-client-discovery-doc.cache'
 EPOCH = datetime.datetime.utcfromtimestamp(0)
@@ -84,7 +88,7 @@ class Cache(base.Cache):
         # If we can not obtain the lock, other process or thread must
         # have initialized the file.
       except Exception as e:
-        logging.warning(e, exc_info=True)
+        LOGGER.warning(e, exc_info=True)
       finally:
         f.unlock_and_close()
 
@@ -100,10 +104,10 @@ class Cache(base.Cache):
             return content
         return None
       else:
-        logger.debug('Could not obtain a lock for the cache file.')
+        LOGGER.debug('Could not obtain a lock for the cache file.')
         return None
     except Exception as e:
-      logger.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
     finally:
       f.unlock_and_close()
 
@@ -122,9 +126,9 @@ class Cache(base.Cache):
         f.file_handle().seek(0)
         json.dump(cache, f.file_handle())
       else:
-        logger.debug('Could not obtain a lock for the cache file.')
+        LOGGER.debug('Could not obtain a lock for the cache file.')
     except Exception as e:
-      logger.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
     finally:
       f.unlock_and_close()
 

src/googleapiclient/http.py

@@ -20,6 +20,7 @@ actuall HTTP request.
 """
 from __future__ import absolute_import
 import six
+from six.moves import http_client
 from six.moves import range
 
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
@@ -36,11 +37,19 @@ import logging
 import mimetypes
 import os
 import random
-import ssl
+import socket
 import sys
 import time
 import uuid
 
+# TODO(issue 221): Remove this conditional import jibbajabba.
+try:
+  import ssl
+except ImportError:
+  _ssl_SSLError = object()
+else:
+  _ssl_SSLError = ssl.SSLError
+
 from email.generator import Generator
 from email.mime.multipart import MIMEMultipart
 from email.mime.nonmultipart import MIMENonMultipart
@@ -57,10 +66,57 @@ from googleapiclient.model import JsonModel
 from oauth2client import util
 
 
+LOGGER = logging.getLogger(__name__)
+
 DEFAULT_CHUNK_SIZE = 512*1024
 
 MAX_URI_LENGTH = 2048
 
+_TOO_MANY_REQUESTS = 429
+
+
+def _should_retry_response(resp_status, content):
+  """Determines whether a response should be retried.
+
+  Args:
+    resp_status: The response status received.
+    content: The response content body. 
+
+  Returns:
+    True if the response should be retried, otherwise False.
+  """
+  # Retry on 5xx errors.
+  if resp_status >= 500:
+    return True
+
+  # Retry on 429 errors.
+  if resp_status == _TOO_MANY_REQUESTS:
+    return True
+
+  # For 403 errors, we have to check for the `reason` in the response to
+  # determine if we should retry.
+  if resp_status == six.moves.http_client.FORBIDDEN:
+    # If there's no details about the 403 type, don't retry.
+    if not content:
+      return False
+
+    # Content is in JSON format.
+    try:
+      data = json.loads(content.decode('utf-8'))
+      reason = data['error']['errors'][0]['reason']
+    except (UnicodeDecodeError, ValueError, KeyError):
+      LOGGER.warning('Invalid JSON content from response: %s', content)
+      return False
+
+    LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
+
+    # Only retry on rate limit related failures.
+    if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
+      return True
+
+  # Everything else is a success or non-retriable so break.
+  return False
+
 
 def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
                    **kwargs):
@@ -82,21 +138,37 @@ def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
     resp, content - Response from the http request (may be HTTP 5xx).
   """
   resp = None
+  content = None
   for retry_num in range(num_retries + 1):
     if retry_num > 0:
-      sleep(rand() * 2**retry_num)
-      logging.warning(
-          'Retry #%d for %s: %s %s%s' % (retry_num, req_type, method, uri,
-          ', following status: %d' % resp.status if resp else ''))
+      # Sleep before retrying.
+      sleep_time = rand() * 2 ** retry_num
+      LOGGER.warning(
+          'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
+          sleep_time, retry_num, num_retries, req_type, method, uri,
+          resp.status if resp else exception)
+      sleep(sleep_time)
 
     try:
+      exception = None
       resp, content = http.request(uri, method, *args, **kwargs)
-    except ssl.SSLError:
-      if retry_num == num_retries:
+    # Retry on SSL errors and socket timeout errors.
+    except _ssl_SSLError as ssl_error:
+      exception = ssl_error
+    except socket.error as socket_error:
+      # errno's contents differ by platform, so we have to match by name.
+      if socket.errno.errorcode.get(socket_error.errno) not in (
+          'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ):
         raise
+      exception = socket_error
+
+    if exception:
+      if retry_num == num_retries:
+        raise exception
       else:
         continue
-    if resp.status < 500:
+
+    if not _should_retry_response(resp.status, content):
       break
 
   return resp, content
@@ -882,7 +954,7 @@ class HttpRequest(object):
     for retry_num in range(num_retries + 1):
       if retry_num > 0:
         self._sleep(self._rand() * 2**retry_num)
-        logging.warning(
+        LOGGER.warning(
             'Retry #%d for media upload: %s %s, following status: %d'
             % (retry_num, self.method, self.uri, resp.status))
 
@@ -1632,7 +1704,7 @@ def tunnel_patch(http):
       headers = {}
     if method == 'PATCH':
       if 'oauth_token' in headers.get('authorization', ''):
-        logging.warning(
+        LOGGER.warning(
             'OAuth 1.0 request made with Credentials after tunnel_patch.')
       headers['x-http-method-override'] = "PATCH"
       method = 'POST'

src/googleapiclient/model.py

@@ -33,6 +33,8 @@ from googleapiclient import __version__
 from googleapiclient.errors import HttpError
 
 
+LOGGER = logging.getLogger(__name__)
+
 dump_request_response = False
 
 
@@ -105,18 +107,18 @@ class BaseModel(Model):
   def _log_request(self, headers, path_params, query, body):
     """Logs debugging information about the request if requested."""
     if dump_request_response:
-      logging.info('--request-start--')
-      logging.info('-headers-start-')
+      LOGGER.info('--request-start--')
+      LOGGER.info('-headers-start-')
       for h, v in six.iteritems(headers):
-        logging.info('%s: %s', h, v)
-      logging.info('-headers-end-')
-      logging.info('-path-parameters-start-')
+        LOGGER.info('%s: %s', h, v)
+      LOGGER.info('-headers-end-')
+      LOGGER.info('-path-parameters-start-')
       for h, v in six.iteritems(path_params):
-        logging.info('%s: %s', h, v)
-      logging.info('-path-parameters-end-')
-      logging.info('body: %s', body)
-      logging.info('query: %s', query)
-      logging.info('--request-end--')
+        LOGGER.info('%s: %s', h, v)
+      LOGGER.info('-path-parameters-end-')
+      LOGGER.info('body: %s', body)
+      LOGGER.info('query: %s', query)
+      LOGGER.info('--request-end--')
 
   def request(self, headers, path_params, query_params, body_value):
     """Updates outgoing requests with a serialized body.
@@ -176,12 +178,12 @@ class BaseModel(Model):
   def _log_response(self, resp, content):
     """Logs debugging information about the response if requested."""
     if dump_request_response:
-      logging.info('--response-start--')
+      LOGGER.info('--response-start--')
       for h, v in six.iteritems(resp):
-        logging.info('%s: %s', h, v)
+        LOGGER.info('%s: %s', h, v)
       if content:
-        logging.info(content)
-      logging.info('--response-end--')
+        LOGGER.info(content)
+      LOGGER.info('--response-end--')
 
   def response(self, resp, content):
     """Convert the response wire format into a Python object.
@@ -206,7 +208,7 @@ class BaseModel(Model):
         return self.no_content_response
       return self.deserialize(content)
     else:
-      logging.debug('Content from bad request was: %s' % content)
+      LOGGER.debug('Content from bad request was: %s' % content)
       raise HttpError(resp, content)
 
   def serialize(self, body_value):