diff --git a/src/googleapiclient/__init__.py b/src/googleapiclient/__init__.py index ceeae8d1..ab076a4f 100644 --- a/src/googleapiclient/__init__.py +++ b/src/googleapiclient/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.4.2" +__version__ = "1.5.0" diff --git a/src/googleapiclient/discovery.py b/src/googleapiclient/discovery.py index be62cf73..cee56284 100644 --- a/src/googleapiclient/discovery.py +++ b/src/googleapiclient/discovery.py @@ -28,14 +28,17 @@ __all__ = [ 'key2param', ] -from six import StringIO +from six import BytesIO from six.moves import http_client from six.moves.urllib.parse import urlencode, urlparse, urljoin, \ urlunparse, parse_qsl # Standard library imports import copy -from email.generator import Generator +try: + from email.generator import BytesGenerator +except ImportError: + from email.generator import Generator as BytesGenerator from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart import json @@ -102,6 +105,10 @@ STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'} # Library-specific reserved words beyond Python keywords. RESERVED_WORDS = frozenset(['body']) +# patch _write_lines to avoid munging '\r' into '\n' +# ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 ) +class _BytesGenerator(BytesGenerator): + _write_lines = BytesGenerator.write def fix_method_name(name): """Fix method names to avoid reserved word conflicts. @@ -797,8 +804,8 @@ def createMethod(methodName, methodDesc, rootDesc, schema): msgRoot.attach(msg) # encode the body: note that we can't use `as_string`, because # it plays games with `From ` lines. - fp = StringIO() - g = Generator(fp, mangle_from_=False) + fp = BytesIO() + g = _BytesGenerator(fp, mangle_from_=False) g.flatten(msgRoot, unixfrom=False) body = fp.getvalue() diff --git a/src/googleapiclient/discovery_cache/file_cache.py b/src/googleapiclient/discovery_cache/file_cache.py index ce540f02..8b0301d2 100644 --- a/src/googleapiclient/discovery_cache/file_cache.py +++ b/src/googleapiclient/discovery_cache/file_cache.py @@ -29,7 +29,7 @@ import os import tempfile import threading -from oauth2client.locked_file import LockedFile +from oauth2client.contrib.locked_file import LockedFile from . import base from ..discovery_cache import DISCOVERY_DOC_MAX_AGE diff --git a/src/googleapiclient/http.py b/src/googleapiclient/http.py index 5fcd7a11..2245e8d7 100644 --- a/src/googleapiclient/http.py +++ b/src/googleapiclient/http.py @@ -36,6 +36,7 @@ import logging import mimetypes import os import random +import ssl import sys import time import uuid @@ -61,6 +62,46 @@ DEFAULT_CHUNK_SIZE = 512*1024 MAX_URI_LENGTH = 2048 +def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, + **kwargs): + """Retries an HTTP request multiple times while handling errors. + + If after all retries the request still fails, last error is either returned as + return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). + + Args: + http: Http object to be used to execute request. + num_retries: Maximum number of retries. + req_type: Type of the request (used for logging retries). + sleep, rand: Functions to sleep for random time between retries. + uri: URI to be requested. + method: HTTP method to be used. + args, kwargs: Additional arguments passed to http.request. + + Returns: + resp, content - Response from the http request (may be HTTP 5xx). + """ + resp = None + for retry_num in range(num_retries + 1): + if retry_num > 0: + sleep(rand() * 2**retry_num) + logging.warning( + 'Retry #%d for %s: %s %s%s' % (retry_num, req_type, method, uri, + ', following status: %d' % resp.status if resp else '')) + + try: + resp, content = http.request(uri, method, *args, **kwargs) + except ssl.SSLError: + if retry_num == num_retries: + raise + else: + continue + if resp.status < 500: + break + + return resp, content + + class MediaUploadProgress(object): """Status of a resumable upload.""" @@ -425,7 +466,11 @@ class MediaFileUpload(MediaIoBaseUpload): self._filename = filename fd = open(self._filename, 'rb') if mimetype is None: - (mimetype, encoding) = mimetypes.guess_type(filename) + # No mimetype provided, make a guess. + mimetype, _ = mimetypes.guess_type(filename) + if mimetype is None: + # Guess failed, use octet-stream. + mimetype = 'application/octet-stream' super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize, resumable=resumable) @@ -542,16 +587,9 @@ class MediaIoBaseDownload(object): } http = self._request.http - for retry_num in range(num_retries + 1): - if retry_num > 0: - self._sleep(self._rand() * 2**retry_num) - logging.warning( - 'Retry #%d for media download: GET %s, following status: %d' - % (retry_num, self._uri, resp.status)) - - resp, content = http.request(self._uri, headers=headers) - if resp.status < 500: - break + resp, content = _retry_request( + http, num_retries, 'media download', self._sleep, self._rand, self._uri, + 'GET', headers=headers) if resp.status in [200, 206]: if 'content-location' in resp and resp['content-location'] != self._uri: @@ -650,7 +688,7 @@ class HttpRequest(object): # Pull the multipart boundary out of the content-type header. major, minor, params = mimeparse.parse_mime_type( - headers.get('content-type', 'application/json')) + self.headers.get('content-type', 'application/json')) # The size of the non-media part of the request. self.body_size = len(self.body or '') @@ -712,16 +750,9 @@ class HttpRequest(object): self.headers['content-length'] = str(len(self.body)) # Handle retries for server-side errors. - for retry_num in range(num_retries + 1): - if retry_num > 0: - self._sleep(self._rand() * 2**retry_num) - logging.warning('Retry #%d for request: %s %s, following status: %d' - % (retry_num, self.method, self.uri, resp.status)) - - resp, content = http.request(str(self.uri), method=str(self.method), - body=self.body, headers=self.headers) - if resp.status < 500: - break + resp, content = _retry_request( + http, num_retries, 'request', self._sleep, self._rand, str(self.uri), + method=str(self.method), body=self.body, headers=self.headers) for callback in self.response_callbacks: callback(resp) @@ -795,18 +826,9 @@ class HttpRequest(object): start_headers['X-Upload-Content-Length'] = size start_headers['content-length'] = str(self.body_size) - for retry_num in range(num_retries + 1): - if retry_num > 0: - self._sleep(self._rand() * 2**retry_num) - logging.warning( - 'Retry #%d for resumable URI request: %s %s, following status: %d' - % (retry_num, self.method, self.uri, resp.status)) - - resp, content = http.request(self.uri, method=self.method, - body=self.body, - headers=start_headers) - if resp.status < 500: - break + resp, content = _retry_request( + http, num_retries, 'resumable URI request', self._sleep, self._rand, + self.uri, method=self.method, body=self.body, headers=start_headers) if resp.status == 200 and 'location' in resp: self.resumable_uri = resp['location'] @@ -827,10 +849,7 @@ class HttpRequest(object): # The upload was complete. return (status, body) - # The httplib.request method can take streams for the body parameter, but - # only in Python 2.6 or later. If a stream is available under those - # conditions then use it as the body argument. - if self.resumable.has_stream() and sys.version_info[1] >= 6: + if self.resumable.has_stream(): data = self.resumable.stream() if self.resumable.chunksize() == -1: data.seek(self.resumable_progress)