From 842ddc2a2673d60c5ffd52ba233c58bd0496136a Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Sat, 18 Mar 2017 12:17:19 -0400 Subject: [PATCH] httplib 0.10.3, api-client 1.6.2 --- src/googleapiclient/__init__.py | 2 +- src/googleapiclient/_auth.py | 9 +- src/googleapiclient/discovery.py | 134 ++++++++++++++++++++-------- src/googleapiclient/http.py | 21 +++++ src/googleapiclient/sample_tools.py | 4 +- src/googleapiclient/schema.py | 5 +- src/httplib2/__init__.py | 107 +++++++++++----------- 7 files changed, 186 insertions(+), 96 deletions(-) diff --git a/src/googleapiclient/__init__.py b/src/googleapiclient/__init__.py index 1d2baf16..4af0c3c2 100644 --- a/src/googleapiclient/__init__.py +++ b/src/googleapiclient/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.6.1" +__version__ = "1.6.2" # Set default logging handler to avoid "No handler found" warnings. import logging diff --git a/src/googleapiclient/_auth.py b/src/googleapiclient/_auth.py index 044ed12d..87d37095 100644 --- a/src/googleapiclient/_auth.py +++ b/src/googleapiclient/_auth.py @@ -14,8 +14,6 @@ """Helpers for authentication using oauth2client or google-auth.""" -import httplib2 - try: import google.auth import google.auth.credentials @@ -31,6 +29,8 @@ try: except ImportError: # pragma: NO COVER HAS_OAUTH2CLIENT = False +from googleapiclient.http import build_http + def default_credentials(): """Returns Application Default Credentials.""" @@ -86,6 +86,7 @@ def authorized_http(credentials): """ if HAS_GOOGLE_AUTH and isinstance( credentials, google.auth.credentials.Credentials): - return google_auth_httplib2.AuthorizedHttp(credentials) + return google_auth_httplib2.AuthorizedHttp(credentials, + http=build_http()) else: - return credentials.authorize(httplib2.Http()) + return credentials.authorize(build_http()) diff --git a/src/googleapiclient/discovery.py b/src/googleapiclient/discovery.py index 74f0a09e..c8956f43 100644 --- a/src/googleapiclient/discovery.py +++ b/src/googleapiclient/discovery.py @@ -61,6 +61,7 @@ from googleapiclient.errors import MediaUploadSizeError from googleapiclient.errors import UnacceptableMimeTypeError from googleapiclient.errors import UnknownApiNameOrVersion from googleapiclient.errors import UnknownFileType +from googleapiclient.http import build_http from googleapiclient.http import BatchHttpRequest from googleapiclient.http import HttpMock from googleapiclient.http import HttpMockSequence @@ -97,6 +98,7 @@ V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?' 'version={apiVersion}') DEFAULT_METHOD_DOC = 'A description of how to use this function' HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH']) + _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} BODY_PARAMETER_DEFAULT_VALUE = { 'description': 'The request body.', @@ -115,6 +117,7 @@ MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = { 'type': 'string', 'required': False, } +_PAGE_TOKEN_NAMES = ('pageToken', 'nextPageToken') # Parameters accepted by the stack, but not visible via discovery. # TODO(dhermes): Remove 'userip' in 'v2'. @@ -213,7 +216,10 @@ def build(serviceName, 'apiVersion': version } - discovery_http = http if http is not None else httplib2.Http() + if http is None: + discovery_http = build_http() + else: + discovery_http = http for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,): requested_url = uritemplate.expand(discovery_url, params) @@ -328,6 +334,10 @@ def build_from_document( if http is not None and credentials is not None: raise ValueError('Arguments http and credentials are mutually exclusive.') + if developerKey is not None and credentials is not None: + raise ValueError( + 'Arguments developerKey and credentials are mutually exclusive.') + if isinstance(service, six.string_types): service = json.loads(service) @@ -350,8 +360,9 @@ def build_from_document( scopes = list( service.get('auth', {}).get('oauth2', {}).get('scopes', {}).keys()) - # If so, then the we need to setup authentication. - if scopes: + # If so, then the we need to setup authentication if no developerKey is + # specified. + if scopes and not developerKey: # If the user didn't pass in credentials, attempt to acquire application # default credentials. if credentials is None: @@ -366,7 +377,7 @@ def build_from_document( # If the service doesn't require scopes then there is no need for # authentication. else: - http = httplib2.Http() + http = build_http() if model is None: features = service.get('features', []) @@ -718,7 +729,11 @@ def createMethod(methodName, methodDesc, rootDesc, schema): for name in parameters.required_params: if name not in kwargs: - raise TypeError('Missing required parameter "%s"' % name) + # temporary workaround for non-paging methods incorrectly requiring + # page token parameter (cf. drive.changes.watch vs. drive.changes.list) + if name not in _PAGE_TOKEN_NAMES or _findPageTokenName( + _methodProperties(methodDesc, schema, 'response')): + raise TypeError('Missing required parameter "%s"' % name) for name, regex in six.iteritems(parameters.pattern_params): if name in kwargs: @@ -921,13 +936,20 @@ def createMethod(methodName, methodDesc, rootDesc, schema): return (methodName, method) -def createNextMethod(methodName): +def createNextMethod(methodName, + pageTokenName='pageToken', + nextPageTokenName='nextPageToken', + isPageTokenParameter=True): """Creates any _next methods for attaching to a Resource. The _next methods allow for easy iteration through list() responses. Args: methodName: string, name of the method to use. + pageTokenName: string, name of request page token field. + nextPageTokenName: string, name of response page token field. + isPageTokenParameter: Boolean, True if request page token is a query + parameter, False if request page token is a field of the request body. """ methodName = fix_method_name(methodName) @@ -945,24 +967,24 @@ Returns: # Retrieve nextPageToken from previous_response # Use as pageToken in previous_request to create new request. - if 'nextPageToken' not in previous_response or not previous_response['nextPageToken']: + nextPageToken = previous_response.get(nextPageTokenName, None) + if not nextPageToken: return None request = copy.copy(previous_request) - pageToken = previous_response['nextPageToken'] - parsed = list(urlparse(request.uri)) - q = parse_qsl(parsed[4]) - - # Find and remove old 'pageToken' value from URI - newq = [(key, value) for (key, value) in q if key != 'pageToken'] - newq.append(('pageToken', pageToken)) - parsed[4] = urlencode(newq) - uri = urlunparse(parsed) - - request.uri = uri - - logger.info('URL being requested: %s %s' % (methodName,uri)) + if isPageTokenParameter: + # Replace pageToken value in URI + request.uri = _add_query_parameter( + request.uri, pageTokenName, nextPageToken) + logger.info('Next page request URL: %s %s' % (methodName, request.uri)) + else: + # Replace pageToken value in request body + model = self._model + body = model.deserialize(request.body) + body[pageTokenName] = nextPageToken + request.body = model.serialize(body) + logger.info('Next page request body: %s %s' % (methodName, body)) return request @@ -1110,19 +1132,59 @@ class Resource(object): method.__get__(self, self.__class__)) def _add_next_methods(self, resourceDesc, schema): - # Add _next() methods - # Look for response bodies in schema that contain nextPageToken, and methods - # that take a pageToken parameter. - if 'methods' in resourceDesc: - for methodName, methodDesc in six.iteritems(resourceDesc['methods']): - if 'response' in methodDesc: - responseSchema = methodDesc['response'] - if '$ref' in responseSchema: - responseSchema = schema.get(responseSchema['$ref']) - hasNextPageToken = 'nextPageToken' in responseSchema.get('properties', - {}) - hasPageToken = 'pageToken' in methodDesc.get('parameters', {}) - if hasNextPageToken and hasPageToken: - fixedMethodName, method = createNextMethod(methodName + '_next') - self._set_dynamic_attr(fixedMethodName, - method.__get__(self, self.__class__)) + # Add _next() methods if and only if one of the names 'pageToken' or + # 'nextPageToken' occurs among the fields of both the method's response + # type either the method's request (query parameters) or request body. + if 'methods' not in resourceDesc: + return + for methodName, methodDesc in six.iteritems(resourceDesc['methods']): + nextPageTokenName = _findPageTokenName( + _methodProperties(methodDesc, schema, 'response')) + if not nextPageTokenName: + continue + isPageTokenParameter = True + pageTokenName = _findPageTokenName(methodDesc.get('parameters', {})) + if not pageTokenName: + isPageTokenParameter = False + pageTokenName = _findPageTokenName( + _methodProperties(methodDesc, schema, 'request')) + if not pageTokenName: + continue + fixedMethodName, method = createNextMethod( + methodName + '_next', pageTokenName, nextPageTokenName, + isPageTokenParameter) + self._set_dynamic_attr(fixedMethodName, + method.__get__(self, self.__class__)) + + +def _findPageTokenName(fields): + """Search field names for one like a page token. + + Args: + fields: container of string, names of fields. + + Returns: + First name that is either 'pageToken' or 'nextPageToken' if one exists, + otherwise None. + """ + return next((tokenName for tokenName in _PAGE_TOKEN_NAMES + if tokenName in fields), None) + +def _methodProperties(methodDesc, schema, name): + """Get properties of a field in a method description. + + Args: + methodDesc: object, fragment of deserialized discovery document that + describes the method. + schema: object, mapping of schema names to schema descriptions. + name: string, name of top-level field in method description. + + Returns: + Object representing fragment of deserialized discovery document + corresponding to 'properties' field of object corresponding to named field + in method description, if it exists, otherwise empty dict. + """ + desc = methodDesc.get(name, {}) + if '$ref' in desc: + desc = schema.get(desc['$ref'], {}) + return desc.get('properties', {}) diff --git a/src/googleapiclient/http.py b/src/googleapiclient/http.py index 0ef10b95..4330f26e 100644 --- a/src/googleapiclient/http.py +++ b/src/googleapiclient/http.py @@ -80,6 +80,8 @@ MAX_URI_LENGTH = 2048 _TOO_MANY_REQUESTS = 429 +DEFAULT_HTTP_TIMEOUT_SEC = 60 + def _should_retry_response(resp_status, content): """Determines whether a response should be retried. @@ -815,6 +817,7 @@ class HttpRequest(object): if 'content-length' not in self.headers: self.headers['content-length'] = str(self.body_size) # If the request URI is too long then turn it into a POST request. + # Assume that a GET request never contains a request body. if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': self.method = 'POST' self.headers['x-http-method-override'] = 'GET' @@ -1732,3 +1735,21 @@ def tunnel_patch(http): http.request = new_request return http + + +def build_http(): + """Builds httplib2.Http object + + Returns: + A httplib2.Http object, which is used to make http requests, and which has timeout set by default. + To override default timeout call + + socket.setdefaulttimeout(timeout_in_sec) + + before interacting with this method. + """ + if socket.getdefaulttimeout() is not None: + http_timeout = socket.getdefaulttimeout() + else: + http_timeout = DEFAULT_HTTP_TIMEOUT_SEC + return httplib2.Http(timeout=http_timeout) diff --git a/src/googleapiclient/sample_tools.py b/src/googleapiclient/sample_tools.py index 2b4e7b45..5ed632da 100644 --- a/src/googleapiclient/sample_tools.py +++ b/src/googleapiclient/sample_tools.py @@ -23,10 +23,10 @@ __all__ = ['init'] import argparse -import httplib2 import os from googleapiclient import discovery +from googleapiclient.http import build_http from oauth2client import client from oauth2client import file from oauth2client import tools @@ -88,7 +88,7 @@ def init(argv, name, version, doc, filename, scope=None, parents=[], discovery_f credentials = storage.get() if credentials is None or credentials.invalid: credentials = tools.run_flow(flow, storage, flags) - http = credentials.authorize(http = httplib2.Http()) + http = credentials.authorize(http=build_http()) if discovery_filename is None: # Construct a service object via the discovery service. diff --git a/src/googleapiclient/schema.py b/src/googleapiclient/schema.py index 9feaf28a..160d388d 100644 --- a/src/googleapiclient/schema.py +++ b/src/googleapiclient/schema.py @@ -161,13 +161,14 @@ class Schemas(object): # Return with trailing comma and newline removed. return self._prettyPrintSchema(schema, dent=1)[:-2] - def get(self, name): + def get(self, name, default=None): """Get deserialized JSON schema from the schema name. Args: name: string, Schema name. + default: object, return value if name not found. """ - return self.schemas[name] + return self.schemas.get(name, default) class _SchemaToStruct(object): diff --git a/src/httplib2/__init__.py b/src/httplib2/__init__.py index 6c5d61bd..32ec959b 100644 --- a/src/httplib2/__init__.py +++ b/src/httplib2/__init__.py @@ -23,7 +23,7 @@ __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", "Louis Nyffenegger", "Alex Yu"] __license__ = "MIT" -__version__ = "0.9.2" +__version__ = "0.10.3" import re import sys @@ -65,42 +65,54 @@ except ImportError: socks = None # Build the appropriate socket wrapper for ssl +ssl = None +ssl_SSLError = None +ssl_CertificateError = None try: - import ssl # python 2.6 - ssl_SSLError = ssl.SSLError - def _ssl_wrap_socket(sock, key_file, cert_file, disable_validation, - ca_certs, ssl_version, hostname): - if disable_validation: - cert_reqs = ssl.CERT_NONE - else: - cert_reqs = ssl.CERT_REQUIRED - if ssl_version is None: - ssl_version = ssl.PROTOCOL_SSLv23 + import ssl # python 2.6 +except ImportError: + pass +if ssl is not None: + ssl_SSLError = getattr(ssl, 'SSLError', None) + ssl_CertificateError = getattr(ssl, 'CertificateError', None) - if hasattr(ssl, 'SSLContext'): # Python 2.7.9 - context = ssl.SSLContext(ssl_version) - context.verify_mode = cert_reqs - context.check_hostname = (cert_reqs != ssl.CERT_NONE) - if cert_file: - context.load_cert_chain(cert_file, key_file) - if ca_certs: - context.load_verify_locations(ca_certs) - return context.wrap_socket(sock, server_hostname=hostname) - else: - return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, - cert_reqs=cert_reqs, ca_certs=ca_certs, - ssl_version=ssl_version) -except (AttributeError, ImportError): - ssl_SSLError = None - def _ssl_wrap_socket(sock, key_file, cert_file, disable_validation, - ca_certs, ssl_version, hostname): - if not disable_validation: - raise CertificateValidationUnsupported( - "SSL certificate validation is not supported without " - "the ssl module installed. To avoid this error, install " - "the ssl module, or explicity disable validation.") - ssl_sock = socket.ssl(sock, key_file, cert_file) - return httplib.FakeSocket(sock, ssl_sock) + +def _ssl_wrap_socket(sock, key_file, cert_file, disable_validation, + ca_certs, ssl_version, hostname): + if disable_validation: + cert_reqs = ssl.CERT_NONE + else: + cert_reqs = ssl.CERT_REQUIRED + if ssl_version is None: + ssl_version = ssl.PROTOCOL_SSLv23 + + if hasattr(ssl, 'SSLContext'): # Python 2.7.9 + context = ssl.SSLContext(ssl_version) + context.verify_mode = cert_reqs + context.check_hostname = (cert_reqs != ssl.CERT_NONE) + if cert_file: + context.load_cert_chain(cert_file, key_file) + if ca_certs: + context.load_verify_locations(ca_certs) + return context.wrap_socket(sock, server_hostname=hostname) + else: + return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, + cert_reqs=cert_reqs, ca_certs=ca_certs, + ssl_version=ssl_version) + + +def _ssl_wrap_socket_unsupported(sock, key_file, cert_file, disable_validation, + ca_certs, ssl_version, hostname): + if not disable_validation: + raise CertificateValidationUnsupported( + "SSL certificate validation is not supported without " + "the ssl module installed. To avoid this error, install " + "the ssl module, or explicity disable validation.") + ssl_sock = socket.ssl(sock, key_file, cert_file) + return httplib.FakeSocket(sock, ssl_sock) + +if ssl is None: + _ssl_wrap_socket = _ssl_wrap_socket_unsupported if sys.version_info >= (2,3): @@ -269,8 +281,8 @@ def safename(filename): filename = re_slash.sub(",", filename) # limit length of filename - if len(filename)>64: - filename=filename[:64] + if len(filename)>200: + filename=filename[:200] return ",".join((filename, filemd5)) NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') @@ -1066,7 +1078,7 @@ class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): raise CertificateHostnameMismatch( 'Server presented certificate that does not match ' 'host %s: %s' % (hostname, cert), hostname, cert) - except ssl_SSLError, e: + except (ssl_SSLError, ssl_CertificateError, CertificateHostnameMismatch), e: if sock: sock.close() if self.sock: @@ -1076,7 +1088,7 @@ class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): # to get at more detailed error information, in particular # whether the error is due to certificate validation or # something else (such as SSL protocol mismatch). - if e.errno == ssl.SSL_ERROR_SSL: + if getattr(e, 'errno', None) == ssl.SSL_ERROR_SSL: raise SSLHandshakeError(e) else: raise @@ -1155,18 +1167,11 @@ try: server_software.startswith('Development/')): raise NotRunningAppEngineEnvironment() - try: - from google.appengine.api import apiproxy_stub_map - if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: - raise ImportError # Bail out; we're not actually running on App Engine. - from google.appengine.api.urlfetch import fetch - from google.appengine.api.urlfetch import InvalidURLError - except (ImportError, AttributeError): - from google3.apphosting.api import apiproxy_stub_map - if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: - raise ImportError # Bail out; we're not actually running on App Engine. - from google3.apphosting.api.urlfetch import fetch - from google3.apphosting.api.urlfetch import InvalidURLError + from google.appengine.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google.appengine.api.urlfetch import fetch + from google.appengine.api.urlfetch import InvalidURLError # Update the connection classes to use the Googel App Engine specific ones. SCHEME_TO_CONNECTION = {