From 26ebf30da7c50aa35641078dfdf4a2ff84360702 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 4 Jul 2018 16:19:51 -0400 Subject: [PATCH] Google API Client 1.7.3 --- src/googleapiclient/__init__.py | 2 +- src/googleapiclient/_helpers.py | 204 ++++++++++++++++++ src/googleapiclient/channel.py | 8 +- src/googleapiclient/discovery.py | 26 ++- .../discovery_cache/file_cache.py | 4 +- src/googleapiclient/errors.py | 7 +- src/googleapiclient/http.py | 33 ++- src/googleapiclient/sample_tools.py | 10 +- src/googleapiclient/schema.py | 7 +- 9 files changed, 251 insertions(+), 50 deletions(-) create mode 100644 src/googleapiclient/_helpers.py diff --git a/src/googleapiclient/__init__.py b/src/googleapiclient/__init__.py index 0cbe2c3f..5b4ca889 100644 --- a/src/googleapiclient/__init__.py +++ b/src/googleapiclient/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.6.5" +__version__ = "1.7.3" # Set default logging handler to avoid "No handler found" warnings. import logging diff --git a/src/googleapiclient/_helpers.py b/src/googleapiclient/_helpers.py new file mode 100644 index 00000000..5e8184ba --- /dev/null +++ b/src/googleapiclient/_helpers.py @@ -0,0 +1,204 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper functions for commonly used utilities.""" + +import functools +import inspect +import logging +import warnings + +import six +from six.moves import urllib + + +logger = logging.getLogger(__name__) + +POSITIONAL_WARNING = 'WARNING' +POSITIONAL_EXCEPTION = 'EXCEPTION' +POSITIONAL_IGNORE = 'IGNORE' +POSITIONAL_SET = frozenset([POSITIONAL_WARNING, POSITIONAL_EXCEPTION, + POSITIONAL_IGNORE]) + +positional_parameters_enforcement = POSITIONAL_WARNING + +_SYM_LINK_MESSAGE = 'File: {0}: Is a symbolic link.' +_IS_DIR_MESSAGE = '{0}: Is a directory' +_MISSING_FILE_MESSAGE = 'Cannot access {0}: No such file or directory' + + +def positional(max_positional_args): + """A decorator to declare that only the first N arguments my be positional. + + This decorator makes it easy to support Python 3 style keyword-only + parameters. For example, in Python 3 it is possible to write:: + + def fn(pos1, *, kwonly1=None, kwonly1=None): + ... + + All named parameters after ``*`` must be a keyword:: + + fn(10, 'kw1', 'kw2') # Raises exception. + fn(10, kwonly1='kw1') # Ok. + + Example + ^^^^^^^ + + To define a function like above, do:: + + @positional(1) + def fn(pos1, kwonly1=None, kwonly2=None): + ... + + If no default value is provided to a keyword argument, it becomes a + required keyword argument:: + + @positional(0) + def fn(required_kw): + ... + + This must be called with the keyword parameter:: + + fn() # Raises exception. + fn(10) # Raises exception. + fn(required_kw=10) # Ok. + + When defining instance or class methods always remember to account for + ``self`` and ``cls``:: + + class MyClass(object): + + @positional(2) + def my_method(self, pos1, kwonly1=None): + ... + + @classmethod + @positional(2) + def my_method(cls, pos1, kwonly1=None): + ... + + The positional decorator behavior is controlled by + ``_helpers.positional_parameters_enforcement``, which may be set to + ``POSITIONAL_EXCEPTION``, ``POSITIONAL_WARNING`` or + ``POSITIONAL_IGNORE`` to raise an exception, log a warning, or do + nothing, respectively, if a declaration is violated. + + Args: + max_positional_arguments: Maximum number of positional arguments. All + parameters after the this index must be + keyword only. + + Returns: + A decorator that prevents using arguments after max_positional_args + from being used as positional parameters. + + Raises: + TypeError: if a key-word only argument is provided as a positional + parameter, but only if + _helpers.positional_parameters_enforcement is set to + POSITIONAL_EXCEPTION. + """ + + def positional_decorator(wrapped): + @functools.wraps(wrapped) + def positional_wrapper(*args, **kwargs): + if len(args) > max_positional_args: + plural_s = '' + if max_positional_args != 1: + plural_s = 's' + message = ('{function}() takes at most {args_max} positional ' + 'argument{plural} ({args_given} given)'.format( + function=wrapped.__name__, + args_max=max_positional_args, + args_given=len(args), + plural=plural_s)) + if positional_parameters_enforcement == POSITIONAL_EXCEPTION: + raise TypeError(message) + elif positional_parameters_enforcement == POSITIONAL_WARNING: + logger.warning(message) + return wrapped(*args, **kwargs) + return positional_wrapper + + if isinstance(max_positional_args, six.integer_types): + return positional_decorator + else: + args, _, _, defaults = inspect.getargspec(max_positional_args) + return positional(len(args) - len(defaults))(max_positional_args) + + +def parse_unique_urlencoded(content): + """Parses unique key-value parameters from urlencoded content. + + Args: + content: string, URL-encoded key-value pairs. + + Returns: + dict, The key-value pairs from ``content``. + + Raises: + ValueError: if one of the keys is repeated. + """ + urlencoded_params = urllib.parse.parse_qs(content) + params = {} + for key, value in six.iteritems(urlencoded_params): + if len(value) != 1: + msg = ('URL-encoded content contains a repeated value:' + '%s -> %s' % (key, ', '.join(value))) + raise ValueError(msg) + params[key] = value[0] + return params + + +def update_query_params(uri, params): + """Updates a URI with new query parameters. + + If a given key from ``params`` is repeated in the ``uri``, then + the URI will be considered invalid and an error will occur. + + If the URI is valid, then each value from ``params`` will + replace the corresponding value in the query parameters (if + it exists). + + Args: + uri: string, A valid URI, with potential existing query parameters. + params: dict, A dictionary of query parameters. + + Returns: + The same URI but with the new query parameters added. + """ + parts = urllib.parse.urlparse(uri) + query_params = parse_unique_urlencoded(parts.query) + query_params.update(params) + new_query = urllib.parse.urlencode(query_params) + new_parts = parts._replace(query=new_query) + return urllib.parse.urlunparse(new_parts) + + +def _add_query_parameter(url, name, value): + """Adds a query parameter to a url. + + Replaces the current value if it already exists in the URL. + + Args: + url: string, url to add the query parameter to. + name: string, query parameter name. + value: string, query parameter value. + + Returns: + Updated query parameter. Does not update the url if value is None. + """ + if value is None: + return url + else: + return update_query_params(url, {name: value}) diff --git a/src/googleapiclient/channel.py b/src/googleapiclient/channel.py index a38b4ffb..0fdb080f 100644 --- a/src/googleapiclient/channel.py +++ b/src/googleapiclient/channel.py @@ -61,15 +61,9 @@ import datetime import uuid from googleapiclient import errors +from googleapiclient import _helpers as util import six -# Oauth2client < 3 has the positional helper in 'util', >= 3 has it -# in '_helpers'. -try: - from oauth2client import util -except ImportError: - from oauth2client import _helpers as util - # The unix time epoch starts at midnight 1970. EPOCH = datetime.datetime.utcfromtimestamp(0) diff --git a/src/googleapiclient/discovery.py b/src/googleapiclient/discovery.py index fe19022c..7762d84f 100644 --- a/src/googleapiclient/discovery.py +++ b/src/googleapiclient/discovery.py @@ -72,16 +72,9 @@ from googleapiclient.model import JsonModel from googleapiclient.model import MediaModel from googleapiclient.model import RawModel from googleapiclient.schema import Schemas -from oauth2client.client import GoogleCredentials -# Oauth2client < 3 has the positional helper in 'util', >= 3 has it -# in '_helpers'. -try: - from oauth2client.util import _add_query_parameter - from oauth2client.util import positional -except ImportError: - from oauth2client._helpers import _add_query_parameter - from oauth2client._helpers import positional +from googleapiclient._helpers import _add_query_parameter +from googleapiclient._helpers import positional # The client library requires a version of httplib2 that supports RETRIES. @@ -139,7 +132,7 @@ def fix_method_name(name): name: string, method name. Returns: - The name with a '_' prefixed if the name is a reserved word. + The name with an '_' appended if the name is a reserved word. """ if keyword.iskeyword(name) or name in RESERVED_WORDS: return name + '_' @@ -455,7 +448,7 @@ def _media_path_url_from_info(root_desc, path_url): } -def _fix_up_parameters(method_desc, root_desc, http_method): +def _fix_up_parameters(method_desc, root_desc, http_method, schema): """Updates parameters of an API method with values specific to this library. Specifically, adds whatever global parameters are specified by the API to the @@ -473,6 +466,7 @@ def _fix_up_parameters(method_desc, root_desc, http_method): root_desc: Dictionary; the entire original deserialized discovery document. http_method: String; the HTTP method used to call the API method described in method_desc. + schema: Object, mapping of schema names to schema descriptions. Returns: The updated Dictionary stored in the 'parameters' key of the method @@ -493,6 +487,9 @@ def _fix_up_parameters(method_desc, root_desc, http_method): if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc: body = BODY_PARAMETER_DEFAULT_VALUE.copy() body.update(method_desc['request']) + # Make body optional for requests with no parameters. + if not _methodProperties(method_desc, schema, 'request'): + body['required'] = False parameters['body'] = body return parameters @@ -543,7 +540,7 @@ def _fix_up_media_upload(method_desc, root_desc, path_url, parameters): return accept, max_size, media_path_url -def _fix_up_method_description(method_desc, root_desc): +def _fix_up_method_description(method_desc, root_desc, schema): """Updates a method description in a discovery document. SIDE EFFECTS: Changes the parameters dictionary in the method description with @@ -554,6 +551,7 @@ def _fix_up_method_description(method_desc, root_desc): from the dictionary of methods stored in the 'methods' key in the deserialized discovery document. root_desc: Dictionary; the entire original deserialized discovery document. + schema: Object, mapping of schema names to schema descriptions. Returns: Tuple (path_url, http_method, method_id, accept, max_size, media_path_url) @@ -578,7 +576,7 @@ def _fix_up_method_description(method_desc, root_desc): http_method = method_desc['httpMethod'] method_id = method_desc['id'] - parameters = _fix_up_parameters(method_desc, root_desc, http_method) + parameters = _fix_up_parameters(method_desc, root_desc, http_method, schema) # Order is important. `_fix_up_media_upload` needs `method_desc` to have a # 'parameters' key and needs to know if there is a 'body' parameter because it # also sets a 'media_body' parameter. @@ -706,7 +704,7 @@ def createMethod(methodName, methodDesc, rootDesc, schema): """ methodName = fix_method_name(methodName) (pathUrl, httpMethod, methodId, accept, - maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc) + maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc, schema) parameters = ResourceMethodParameters(methodDesc) diff --git a/src/googleapiclient/discovery_cache/file_cache.py b/src/googleapiclient/discovery_cache/file_cache.py index e88e7dc5..48bddea1 100644 --- a/src/googleapiclient/discovery_cache/file_cache.py +++ b/src/googleapiclient/discovery_cache/file_cache.py @@ -36,9 +36,9 @@ except ImportError: try: from oauth2client.locked_file import LockedFile except ImportError: - # oauth2client > 4.0.0 + # oauth2client > 4.0.0 or google-auth raise ImportError( - 'file_cache is unavailable when using oauth2client >= 4.0.0') + 'file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth') from . import base from ..discovery_cache import DISCOVERY_DOC_MAX_AGE diff --git a/src/googleapiclient/errors.py b/src/googleapiclient/errors.py index bab14189..8c4795c4 100644 --- a/src/googleapiclient/errors.py +++ b/src/googleapiclient/errors.py @@ -23,12 +23,7 @@ __author__ = 'jcgregorio@google.com (Joe Gregorio)' import json -# Oauth2client < 3 has the positional helper in 'util', >= 3 has it -# in '_helpers'. -try: - from oauth2client import util -except ImportError: - from oauth2client import _helpers as util +from googleapiclient import _helpers as util class Error(Exception): diff --git a/src/googleapiclient/http.py b/src/googleapiclient/http.py index 66af5d89..a7f14b73 100644 --- a/src/googleapiclient/http.py +++ b/src/googleapiclient/http.py @@ -16,7 +16,7 @@ The classes implement a command pattern, with every object supporting an execute() method that does the -actuall HTTP request. +actual HTTP request. """ from __future__ import absolute_import import six @@ -55,12 +55,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.parser import FeedParser -# Oauth2client < 3 has the positional helper in 'util', >= 3 has it -# in '_helpers'. -try: - from oauth2client import util -except ImportError: - from oauth2client import _helpers as util +from googleapiclient import _helpers as util from googleapiclient import _auth from googleapiclient.errors import BatchError @@ -82,6 +77,8 @@ _TOO_MANY_REQUESTS = 429 DEFAULT_HTTP_TIMEOUT_SEC = 60 +_LEGACY_BATCH_URI = 'https://www.googleapis.com/batch' + def _should_retry_response(resp_status, content): """Determines whether a response should be retried. @@ -166,10 +163,14 @@ def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, # Retry on SSL errors and socket timeout errors. except _ssl_SSLError as ssl_error: exception = ssl_error + except socket.timeout as socket_timeout: + # It's important that this be before socket.error as it's a subclass + # socket.timeout has no errorcode + exception = socket_timeout except socket.error as socket_error: # errno's contents differ by platform, so we have to match by name. - if socket.errno.errorcode.get(socket_error.errno) not in ( - 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ): + if socket.errno.errorcode.get(socket_error.errno) not in { + 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED'}: raise exception = socket_error @@ -1086,7 +1087,17 @@ class BatchHttpRequest(object): batch_uri: string, URI to send batch requests to. """ if batch_uri is None: - batch_uri = 'https://www.googleapis.com/batch' + batch_uri = _LEGACY_BATCH_URI + + if batch_uri == _LEGACY_BATCH_URI: + LOGGER.warn( + "You have constructed a BatchHttpRequest using the legacy batch " + "endpoint %s. This endpoint will be turned down on March 25, 2019. " + "Please provide the API-specific endpoint or use " + "service.new_batch_http_request(). For more details see " + "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" + "and https://developers.google.com/api-client-library/python/guide/batch.", + _LEGACY_BATCH_URI) self._batch_uri = batch_uri # Global callback to be called for each individual response in the batch. @@ -1279,7 +1290,7 @@ class BatchHttpRequest(object): from the server. The default behavior is to have the library generate it's own unique id. If the caller passes in a request_id then they must ensure uniqueness for each request_id, and if they are not an exception is - raised. Callers should either supply all request_ids or nevery supply a + raised. Callers should either supply all request_ids or never supply a request id, to avoid such an error. Args: diff --git a/src/googleapiclient/sample_tools.py b/src/googleapiclient/sample_tools.py index 5ed632da..21fede3e 100644 --- a/src/googleapiclient/sample_tools.py +++ b/src/googleapiclient/sample_tools.py @@ -27,9 +27,13 @@ import os from googleapiclient import discovery from googleapiclient.http import build_http -from oauth2client import client -from oauth2client import file -from oauth2client import tools + +try: + from oauth2client import client + from oauth2client import file + from oauth2client import tools +except ImportError: + raise ImportError('googleapiclient.sample_tools requires oauth2client. Please install oauth2client and try again.') def init(argv, name, version, doc, filename, scope=None, parents=[], discovery_filename=None): diff --git a/src/googleapiclient/schema.py b/src/googleapiclient/schema.py index 160d388d..10d4a1b5 100644 --- a/src/googleapiclient/schema.py +++ b/src/googleapiclient/schema.py @@ -65,12 +65,7 @@ __author__ = 'jcgregorio@google.com (Joe Gregorio)' import copy -# Oauth2client < 3 has the positional helper in 'util', >= 3 has it -# in '_helpers'. -try: - from oauth2client import util -except ImportError: - from oauth2client import _helpers as util +from googleapiclient import _helpers as util class Schemas(object):