From b9237f9f63cd38988e677186c9d0a4f7d895cd10 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Sat, 6 Sep 2025 05:42:53 -0400 Subject: [PATCH] [no ci] Delete src/gam/googleapiclient.shouldnotbehere directory --- .../__init__.py | 27 - .../googleapiclient.shouldnotbehere/_auth.py | 167 -- .../_helpers.py | 207 -- .../channel.py | 315 --- .../discovery.py | 1662 -------------- .../discovery_cache/__init__.py | 78 - .../discovery_cache/appengine_memcache.py | 55 - .../discovery_cache/base.py | 46 - .../discovery_cache/file_cache.py | 145 -- .../googleapiclient.shouldnotbehere/errors.py | 197 -- .../googleapiclient.shouldnotbehere/http.py | 1962 ----------------- .../mimeparse.py | 183 -- .../googleapiclient.shouldnotbehere/model.py | 429 ---- .../googleapiclient.shouldnotbehere/schema.py | 317 --- .../version.py | 15 - 15 files changed, 5805 deletions(-) delete mode 100644 src/gam/googleapiclient.shouldnotbehere/__init__.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/_auth.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/_helpers.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/channel.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/discovery.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/discovery_cache/__init__.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/discovery_cache/appengine_memcache.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/discovery_cache/base.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/discovery_cache/file_cache.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/errors.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/http.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/mimeparse.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/model.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/schema.py delete mode 100644 src/gam/googleapiclient.shouldnotbehere/version.py diff --git a/src/gam/googleapiclient.shouldnotbehere/__init__.py b/src/gam/googleapiclient.shouldnotbehere/__init__.py deleted file mode 100644 index c9218dd8..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Set default logging handler to avoid "No handler found" warnings. -import logging - -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - - class NullHandler(logging.Handler): - def emit(self, record): - pass - - -logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/src/gam/googleapiclient.shouldnotbehere/_auth.py b/src/gam/googleapiclient.shouldnotbehere/_auth.py deleted file mode 100644 index 065b2ecd..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/_auth.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers for authentication using oauth2client or google-auth.""" - -import httplib2 - -try: - import google.auth - import google.auth.credentials - - HAS_GOOGLE_AUTH = True -except ImportError: # pragma: NO COVER - HAS_GOOGLE_AUTH = False - -try: - import google_auth_httplib2 -except ImportError: # pragma: NO COVER - google_auth_httplib2 = None - -try: - import oauth2client - import oauth2client.client - - HAS_OAUTH2CLIENT = True -except ImportError: # pragma: NO COVER - HAS_OAUTH2CLIENT = False - - -def credentials_from_file(filename, scopes=None, quota_project_id=None): - """Returns credentials loaded from a file.""" - if HAS_GOOGLE_AUTH: - credentials, _ = google.auth.load_credentials_from_file( - filename, scopes=scopes, quota_project_id=quota_project_id - ) - return credentials - else: - raise EnvironmentError( - "client_options.credentials_file is only supported in google-auth." - ) - - -def default_credentials(scopes=None, quota_project_id=None): - """Returns Application Default Credentials.""" - if HAS_GOOGLE_AUTH: - credentials, _ = google.auth.default( - scopes=scopes, quota_project_id=quota_project_id - ) - return credentials - elif HAS_OAUTH2CLIENT: - if scopes is not None or quota_project_id is not None: - raise EnvironmentError( - "client_options.scopes and client_options.quota_project_id are not supported in oauth2client." - "Please install google-auth." - ) - return oauth2client.client.GoogleCredentials.get_application_default() - else: - raise EnvironmentError( - "No authentication library is available. Please install either " - "google-auth or oauth2client." - ) - - -def with_scopes(credentials, scopes): - """Scopes the credentials if necessary. - - Args: - credentials (Union[ - google.auth.credentials.Credentials, - oauth2client.client.Credentials]): The credentials to scope. - scopes (Sequence[str]): The list of scopes. - - Returns: - Union[google.auth.credentials.Credentials, - oauth2client.client.Credentials]: The scoped credentials. - """ - if HAS_GOOGLE_AUTH and isinstance(credentials, google.auth.credentials.Credentials): - return google.auth.credentials.with_scopes_if_required(credentials, scopes) - else: - try: - if credentials.create_scoped_required(): - return credentials.create_scoped(scopes) - else: - return credentials - except AttributeError: - return credentials - - -def authorized_http(credentials): - """Returns an http client that is authorized with the given credentials. - - Args: - credentials (Union[ - google.auth.credentials.Credentials, - oauth2client.client.Credentials]): The credentials to use. - - Returns: - Union[httplib2.Http, google_auth_httplib2.AuthorizedHttp]: An - authorized http client. - """ - from googleapiclient.http import build_http - - if HAS_GOOGLE_AUTH and isinstance(credentials, google.auth.credentials.Credentials): - if google_auth_httplib2 is None: - raise ValueError( - "Credentials from google.auth specified, but " - "google-api-python-client is unable to use these credentials " - "unless google-auth-httplib2 is installed. Please install " - "google-auth-httplib2." - ) - return google_auth_httplib2.AuthorizedHttp(credentials, http=build_http()) - else: - return credentials.authorize(build_http()) - - -def refresh_credentials(credentials): - # Refresh must use a new http instance, as the one associated with the - # credentials could be a AuthorizedHttp or an oauth2client-decorated - # Http instance which would cause a weird recursive loop of refreshing - # and likely tear a hole in spacetime. - refresh_http = httplib2.Http() - if HAS_GOOGLE_AUTH and isinstance(credentials, google.auth.credentials.Credentials): - request = google_auth_httplib2.Request(refresh_http) - return credentials.refresh(request) - else: - return credentials.refresh(refresh_http) - - -def apply_credentials(credentials, headers): - # oauth2client and google-auth have the same interface for this. - if not is_valid(credentials): - refresh_credentials(credentials) - return credentials.apply(headers) - - -def is_valid(credentials): - if HAS_GOOGLE_AUTH and isinstance(credentials, google.auth.credentials.Credentials): - return credentials.valid - else: - return ( - credentials.access_token is not None - and not credentials.access_token_expired - ) - - -def get_credentials_from_http(http): - if http is None: - return None - elif hasattr(http.request, "credentials"): - return http.request.credentials - elif hasattr(http, "credentials") and not isinstance( - http.credentials, httplib2.Credentials - ): - return http.credentials - else: - return None diff --git a/src/gam/googleapiclient.shouldnotbehere/_helpers.py b/src/gam/googleapiclient.shouldnotbehere/_helpers.py deleted file mode 100644 index 17b8a171..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/_helpers.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper functions for commonly used utilities.""" - -import functools -import inspect -import logging -import urllib - -logger = logging.getLogger(__name__) - -POSITIONAL_WARNING = "WARNING" -POSITIONAL_EXCEPTION = "EXCEPTION" -POSITIONAL_IGNORE = "IGNORE" -POSITIONAL_SET = frozenset( - [POSITIONAL_WARNING, POSITIONAL_EXCEPTION, POSITIONAL_IGNORE] -) - -positional_parameters_enforcement = POSITIONAL_WARNING - -_SYM_LINK_MESSAGE = "File: {0}: Is a symbolic link." -_IS_DIR_MESSAGE = "{0}: Is a directory" -_MISSING_FILE_MESSAGE = "Cannot access {0}: No such file or directory" - - -def positional(max_positional_args): - """A decorator to declare that only the first N arguments may be positional. - - This decorator makes it easy to support Python 3 style keyword-only - parameters. For example, in Python 3 it is possible to write:: - - def fn(pos1, *, kwonly1=None, kwonly2=None): - ... - - All named parameters after ``*`` must be a keyword:: - - fn(10, 'kw1', 'kw2') # Raises exception. - fn(10, kwonly1='kw1') # Ok. - - Example - ^^^^^^^ - - To define a function like above, do:: - - @positional(1) - def fn(pos1, kwonly1=None, kwonly2=None): - ... - - If no default value is provided to a keyword argument, it becomes a - required keyword argument:: - - @positional(0) - def fn(required_kw): - ... - - This must be called with the keyword parameter:: - - fn() # Raises exception. - fn(10) # Raises exception. - fn(required_kw=10) # Ok. - - When defining instance or class methods always remember to account for - ``self`` and ``cls``:: - - class MyClass(object): - - @positional(2) - def my_method(self, pos1, kwonly1=None): - ... - - @classmethod - @positional(2) - def my_method(cls, pos1, kwonly1=None): - ... - - The positional decorator behavior is controlled by - ``_helpers.positional_parameters_enforcement``, which may be set to - ``POSITIONAL_EXCEPTION``, ``POSITIONAL_WARNING`` or - ``POSITIONAL_IGNORE`` to raise an exception, log a warning, or do - nothing, respectively, if a declaration is violated. - - Args: - max_positional_arguments: Maximum number of positional arguments. All - parameters after this index must be - keyword only. - - Returns: - A decorator that prevents using arguments after max_positional_args - from being used as positional parameters. - - Raises: - TypeError: if a keyword-only argument is provided as a positional - parameter, but only if - _helpers.positional_parameters_enforcement is set to - POSITIONAL_EXCEPTION. - """ - - def positional_decorator(wrapped): - @functools.wraps(wrapped) - def positional_wrapper(*args, **kwargs): - if len(args) > max_positional_args: - plural_s = "" - if max_positional_args != 1: - plural_s = "s" - message = ( - "{function}() takes at most {args_max} positional " - "argument{plural} ({args_given} given)".format( - function=wrapped.__name__, - args_max=max_positional_args, - args_given=len(args), - plural=plural_s, - ) - ) - if positional_parameters_enforcement == POSITIONAL_EXCEPTION: - raise TypeError(message) - elif positional_parameters_enforcement == POSITIONAL_WARNING: - logger.warning(message) - return wrapped(*args, **kwargs) - - return positional_wrapper - - if isinstance(max_positional_args, int): - return positional_decorator - else: - args, _, _, defaults, _, _, _ = inspect.getfullargspec(max_positional_args) - return positional(len(args) - len(defaults))(max_positional_args) - - -def parse_unique_urlencoded(content): - """Parses unique key-value parameters from urlencoded content. - - Args: - content: string, URL-encoded key-value pairs. - - Returns: - dict, The key-value pairs from ``content``. - - Raises: - ValueError: if one of the keys is repeated. - """ - urlencoded_params = urllib.parse.parse_qs(content) - params = {} - for key, value in urlencoded_params.items(): - if len(value) != 1: - msg = "URL-encoded content contains a repeated value:" "%s -> %s" % ( - key, - ", ".join(value), - ) - raise ValueError(msg) - params[key] = value[0] - return params - - -def update_query_params(uri, params): - """Updates a URI with new query parameters. - - If a given key from ``params`` is repeated in the ``uri``, then - the URI will be considered invalid and an error will occur. - - If the URI is valid, then each value from ``params`` will - replace the corresponding value in the query parameters (if - it exists). - - Args: - uri: string, A valid URI, with potential existing query parameters. - params: dict, A dictionary of query parameters. - - Returns: - The same URI but with the new query parameters added. - """ - parts = urllib.parse.urlparse(uri) - query_params = parse_unique_urlencoded(parts.query) - query_params.update(params) - new_query = urllib.parse.urlencode(query_params) - new_parts = parts._replace(query=new_query) - return urllib.parse.urlunparse(new_parts) - - -def _add_query_parameter(url, name, value): - """Adds a query parameter to a url. - - Replaces the current value if it already exists in the URL. - - Args: - url: string, url to add the query parameter to. - name: string, query parameter name. - value: string, query parameter value. - - Returns: - Updated query parameter. Does not update the url if value is None. - """ - if value is None: - return url - else: - return update_query_params(url, {name: value}) diff --git a/src/gam/googleapiclient.shouldnotbehere/channel.py b/src/gam/googleapiclient.shouldnotbehere/channel.py deleted file mode 100644 index 37bda1ea..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/channel.py +++ /dev/null @@ -1,315 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Channel notifications support. - -Classes and functions to support channel subscriptions and notifications -on those channels. - -Notes: - - This code is based on experimental APIs and is subject to change. - - Notification does not do deduplication of notification ids, that's up to - the receiver. - - Storing the Channel between calls is up to the caller. - - -Example setting up a channel: - - # Create a new channel that gets notifications via webhook. - channel = new_webhook_channel("https://example.com/my_web_hook") - - # Store the channel, keyed by 'channel.id'. Store it before calling the - # watch method because notifications may start arriving before the watch - # method returns. - ... - - resp = service.objects().watchAll( - bucket="some_bucket_id", body=channel.body()).execute() - channel.update(resp) - - # Store the channel, keyed by 'channel.id'. Store it after being updated - # since the resource_id value will now be correct, and that's needed to - # stop a subscription. - ... - - -An example Webhook implementation using webapp2. Note that webapp2 puts -headers in a case insensitive dictionary, as headers aren't guaranteed to -always be upper case. - - id = self.request.headers[X_GOOG_CHANNEL_ID] - - # Retrieve the channel by id. - channel = ... - - # Parse notification from the headers, including validating the id. - n = notification_from_headers(channel, self.request.headers) - - # Do app specific stuff with the notification here. - if n.resource_state == 'sync': - # Code to handle sync state. - elif n.resource_state == 'exists': - # Code to handle the exists state. - elif n.resource_state == 'not_exists': - # Code to handle the not exists state. - - -Example of unsubscribing. - - service.channels().stop(channel.body()).execute() -""" -from __future__ import absolute_import - -import datetime -import uuid - -from googleapiclient import _helpers as util -from googleapiclient import errors - -# The unix time epoch starts at midnight 1970. -EPOCH = datetime.datetime(1970, 1, 1) - -# Map the names of the parameters in the JSON channel description to -# the parameter names we use in the Channel class. -CHANNEL_PARAMS = { - "address": "address", - "id": "id", - "expiration": "expiration", - "params": "params", - "resourceId": "resource_id", - "resourceUri": "resource_uri", - "type": "type", - "token": "token", -} - -X_GOOG_CHANNEL_ID = "X-GOOG-CHANNEL-ID" -X_GOOG_MESSAGE_NUMBER = "X-GOOG-MESSAGE-NUMBER" -X_GOOG_RESOURCE_STATE = "X-GOOG-RESOURCE-STATE" -X_GOOG_RESOURCE_URI = "X-GOOG-RESOURCE-URI" -X_GOOG_RESOURCE_ID = "X-GOOG-RESOURCE-ID" - - -def _upper_header_keys(headers): - new_headers = {} - for k, v in headers.items(): - new_headers[k.upper()] = v - return new_headers - - -class Notification(object): - """A Notification from a Channel. - - Notifications are not usually constructed directly, but are returned - from functions like notification_from_headers(). - - Attributes: - message_number: int, The unique id number of this notification. - state: str, The state of the resource being monitored. - uri: str, The address of the resource being monitored. - resource_id: str, The unique identifier of the version of the resource at - this event. - """ - - @util.positional(5) - def __init__(self, message_number, state, resource_uri, resource_id): - """Notification constructor. - - Args: - message_number: int, The unique id number of this notification. - state: str, The state of the resource being monitored. Can be one - of "exists", "not_exists", or "sync". - resource_uri: str, The address of the resource being monitored. - resource_id: str, The identifier of the watched resource. - """ - self.message_number = message_number - self.state = state - self.resource_uri = resource_uri - self.resource_id = resource_id - - -class Channel(object): - """A Channel for notifications. - - Usually not constructed directly, instead it is returned from helper - functions like new_webhook_channel(). - - Attributes: - type: str, The type of delivery mechanism used by this channel. For - example, 'web_hook'. - id: str, A UUID for the channel. - token: str, An arbitrary string associated with the channel that - is delivered to the target address with each event delivered - over this channel. - address: str, The address of the receiving entity where events are - delivered. Specific to the channel type. - expiration: int, The time, in milliseconds from the epoch, when this - channel will expire. - params: dict, A dictionary of string to string, with additional parameters - controlling delivery channel behavior. - resource_id: str, An opaque id that identifies the resource that is - being watched. Stable across different API versions. - resource_uri: str, The canonicalized ID of the watched resource. - """ - - @util.positional(5) - def __init__( - self, - type, - id, - token, - address, - expiration=None, - params=None, - resource_id="", - resource_uri="", - ): - """Create a new Channel. - - In user code, this Channel constructor will not typically be called - manually since there are functions for creating channels for each specific - type with a more customized set of arguments to pass. - - Args: - type: str, The type of delivery mechanism used by this channel. For - example, 'web_hook'. - id: str, A UUID for the channel. - token: str, An arbitrary string associated with the channel that - is delivered to the target address with each event delivered - over this channel. - address: str, The address of the receiving entity where events are - delivered. Specific to the channel type. - expiration: int, The time, in milliseconds from the epoch, when this - channel will expire. - params: dict, A dictionary of string to string, with additional parameters - controlling delivery channel behavior. - resource_id: str, An opaque id that identifies the resource that is - being watched. Stable across different API versions. - resource_uri: str, The canonicalized ID of the watched resource. - """ - self.type = type - self.id = id - self.token = token - self.address = address - self.expiration = expiration - self.params = params - self.resource_id = resource_id - self.resource_uri = resource_uri - - def body(self): - """Build a body from the Channel. - - Constructs a dictionary that's appropriate for passing into watch() - methods as the value of body argument. - - Returns: - A dictionary representation of the channel. - """ - result = { - "id": self.id, - "token": self.token, - "type": self.type, - "address": self.address, - } - if self.params: - result["params"] = self.params - if self.resource_id: - result["resourceId"] = self.resource_id - if self.resource_uri: - result["resourceUri"] = self.resource_uri - if self.expiration: - result["expiration"] = self.expiration - - return result - - def update(self, resp): - """Update a channel with information from the response of watch(). - - When a request is sent to watch() a resource, the response returned - from the watch() request is a dictionary with updated channel information, - such as the resource_id, which is needed when stopping a subscription. - - Args: - resp: dict, The response from a watch() method. - """ - for json_name, param_name in CHANNEL_PARAMS.items(): - value = resp.get(json_name) - if value is not None: - setattr(self, param_name, value) - - -def notification_from_headers(channel, headers): - """Parse a notification from the webhook request headers, validate - the notification, and return a Notification object. - - Args: - channel: Channel, The channel that the notification is associated with. - headers: dict, A dictionary like object that contains the request headers - from the webhook HTTP request. - - Returns: - A Notification object. - - Raises: - errors.InvalidNotificationError if the notification is invalid. - ValueError if the X-GOOG-MESSAGE-NUMBER can't be converted to an int. - """ - headers = _upper_header_keys(headers) - channel_id = headers[X_GOOG_CHANNEL_ID] - if channel.id != channel_id: - raise errors.InvalidNotificationError( - "Channel id mismatch: %s != %s" % (channel.id, channel_id) - ) - else: - message_number = int(headers[X_GOOG_MESSAGE_NUMBER]) - state = headers[X_GOOG_RESOURCE_STATE] - resource_uri = headers[X_GOOG_RESOURCE_URI] - resource_id = headers[X_GOOG_RESOURCE_ID] - return Notification(message_number, state, resource_uri, resource_id) - - -@util.positional(2) -def new_webhook_channel(url, token=None, expiration=None, params=None): - """Create a new webhook Channel. - - Args: - url: str, URL to post notifications to. - token: str, An arbitrary string associated with the channel that - is delivered to the target address with each notification delivered - over this channel. - expiration: datetime.datetime, A time in the future when the channel - should expire. Can also be None if the subscription should use the - default expiration. Note that different services may have different - limits on how long a subscription lasts. Check the response from the - watch() method to see the value the service has set for an expiration - time. - params: dict, Extra parameters to pass on channel creation. Currently - not used for webhook channels. - """ - expiration_ms = 0 - if expiration: - delta = expiration - EPOCH - expiration_ms = ( - delta.microseconds / 1000 + (delta.seconds + delta.days * 24 * 3600) * 1000 - ) - if expiration_ms < 0: - expiration_ms = 0 - - return Channel( - "web_hook", - str(uuid.uuid4()), - token, - url, - expiration=expiration_ms, - params=params, - ) diff --git a/src/gam/googleapiclient.shouldnotbehere/discovery.py b/src/gam/googleapiclient.shouldnotbehere/discovery.py deleted file mode 100644 index f7bbd777..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/discovery.py +++ /dev/null @@ -1,1662 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Client for discovery based APIs. - -A client library for Google's discovery based APIs. -""" -from __future__ import absolute_import - -__author__ = "jcgregorio@google.com (Joe Gregorio)" -__all__ = ["build", "build_from_document", "fix_method_name", "key2param"] - -from collections import OrderedDict -import collections.abc - -# Standard library imports -import copy -from email.generator import BytesGenerator -from email.mime.multipart import MIMEMultipart -from email.mime.nonmultipart import MIMENonMultipart -import http.client as http_client -import io -import json -import keyword -import logging -import mimetypes -import os -import re -import urllib - -import google.api_core.client_options -from google.auth.exceptions import MutualTLSChannelError -from google.auth.transport import mtls -from google.oauth2 import service_account - -# Third-party imports -import httplib2 -import uritemplate - -try: - import google_auth_httplib2 -except ImportError: # pragma: NO COVER - google_auth_httplib2 = None - -try: - from google.api_core import universe - - HAS_UNIVERSE = True -except ImportError: - HAS_UNIVERSE = False - -# Local imports -from googleapiclient import _auth, mimeparse -from googleapiclient._helpers import _add_query_parameter, positional -from googleapiclient.errors import ( - HttpError, - InvalidJsonError, - MediaUploadSizeError, - UnacceptableMimeTypeError, - UnknownApiNameOrVersion, - UnknownFileType, -) -from googleapiclient.http import ( - BatchHttpRequest, - HttpMock, - HttpMockSequence, - HttpRequest, - MediaFileUpload, - MediaUpload, - build_http, -) -from googleapiclient.model import JsonModel, MediaModel, RawModel -from googleapiclient.schema import Schemas - -# The client library requires a version of httplib2 that supports RETRIES. -httplib2.RETRIES = 1 - -logger = logging.getLogger(__name__) - -URITEMPLATE = re.compile("{[^}]*}") -VARNAME = re.compile("[a-zA-Z0-9_-]+") -DISCOVERY_URI = ( - "https://www.googleapis.com/discovery/v1/apis/" "{api}/{apiVersion}/rest" -) -V1_DISCOVERY_URI = DISCOVERY_URI -V2_DISCOVERY_URI = ( - "https://{api}.googleapis.com/$discovery/rest?" "version={apiVersion}" -) -DEFAULT_METHOD_DOC = "A description of how to use this function" -HTTP_PAYLOAD_METHODS = frozenset(["PUT", "POST", "PATCH"]) - -_MEDIA_SIZE_BIT_SHIFTS = {"KB": 10, "MB": 20, "GB": 30, "TB": 40} -BODY_PARAMETER_DEFAULT_VALUE = {"description": "The request body.", "type": "object"} -MEDIA_BODY_PARAMETER_DEFAULT_VALUE = { - "description": ( - "The filename of the media request body, or an instance " - "of a MediaUpload object." - ), - "type": "string", - "required": False, -} -MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE = { - "description": ( - "The MIME type of the media request body, or an instance " - "of a MediaUpload object." - ), - "type": "string", - "required": False, -} -_PAGE_TOKEN_NAMES = ("pageToken", "nextPageToken") - -# Parameters controlling mTLS behavior. See https://google.aip.dev/auth/4114. -GOOGLE_API_USE_CLIENT_CERTIFICATE = "GOOGLE_API_USE_CLIENT_CERTIFICATE" -GOOGLE_API_USE_MTLS_ENDPOINT = "GOOGLE_API_USE_MTLS_ENDPOINT" -GOOGLE_CLOUD_UNIVERSE_DOMAIN = "GOOGLE_CLOUD_UNIVERSE_DOMAIN" -DEFAULT_UNIVERSE = "googleapis.com" -# Parameters accepted by the stack, but not visible via discovery. -# TODO(dhermes): Remove 'userip' in 'v2'. -STACK_QUERY_PARAMETERS = frozenset(["trace", "pp", "userip", "strict"]) -STACK_QUERY_PARAMETER_DEFAULT_VALUE = {"type": "string", "location": "query"} - - -class APICoreVersionError(ValueError): - def __init__(self): - message = ( - "google-api-core >= 2.18.0 is required to use the universe domain feature." - ) - super().__init__(message) - - -# Library-specific reserved words beyond Python keywords. -RESERVED_WORDS = frozenset(["body"]) - -# patch _write_lines to avoid munging '\r' into '\n' -# ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 ) -class _BytesGenerator(BytesGenerator): - _write_lines = BytesGenerator.write - - -def fix_method_name(name): - """Fix method names to avoid '$' characters and reserved word conflicts. - - Args: - name: string, method name. - - Returns: - The name with '_' appended if the name is a reserved word and '$' and '-' - replaced with '_'. - """ - name = name.replace("$", "_").replace("-", "_") - if keyword.iskeyword(name) or name in RESERVED_WORDS: - return name + "_" - else: - return name - - -def key2param(key): - """Converts key names into parameter names. - - For example, converting "max-results" -> "max_results" - - Args: - key: string, the method key name. - - Returns: - A safe method name based on the key name. - """ - result = [] - key = list(key) - if not key[0].isalpha(): - result.append("x") - for c in key: - if c.isalnum(): - result.append(c) - else: - result.append("_") - - return "".join(result) - - -@positional(2) -def build( - serviceName, - version, - http=None, - discoveryServiceUrl=None, - developerKey=None, - model=None, - requestBuilder=HttpRequest, - credentials=None, - cache_discovery=True, - cache=None, - client_options=None, - adc_cert_path=None, - adc_key_path=None, - num_retries=1, - static_discovery=None, - always_use_jwt_access=False, -): - """Construct a Resource for interacting with an API. - - Construct a Resource object for interacting with an API. The serviceName and - version are the names from the Discovery service. - - Args: - serviceName: string, name of the service. - version: string, the version of the service. - http: httplib2.Http, An instance of httplib2.Http or something that acts - like it that HTTP requests will be made through. - discoveryServiceUrl: string, a URI Template that points to the location of - the discovery service. It should have two parameters {api} and - {apiVersion} that when filled in produce an absolute URI to the discovery - document for that service. - developerKey: string, key obtained from - https://code.google.com/apis/console. - model: googleapiclient.Model, converts to and from the wire format. - requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP - request. - credentials: oauth2client.Credentials or - google.auth.credentials.Credentials, credentials to be used for - authentication. - cache_discovery: Boolean, whether or not to cache the discovery doc. - cache: googleapiclient.discovery_cache.base.CacheBase, an optional - cache object for the discovery documents. - client_options: Mapping object or google.api_core.client_options, client - options to set user options on the client. - (1) The API endpoint should be set through client_options. If API endpoint - is not set, `GOOGLE_API_USE_MTLS_ENDPOINT` environment variable can be used - to control which endpoint to use. - (2) client_cert_source is not supported, client cert should be provided using - client_encrypted_cert_source instead. In order to use the provided client - cert, `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable must be - set to `true`. - More details on the environment variables are here: - https://google.aip.dev/auth/4114 - adc_cert_path: str, client certificate file path to save the application - default client certificate for mTLS. This field is required if you want to - use the default client certificate. `GOOGLE_API_USE_CLIENT_CERTIFICATE` - environment variable must be set to `true` in order to use this field, - otherwise this field doesn't nothing. - More details on the environment variables are here: - https://google.aip.dev/auth/4114 - adc_key_path: str, client encrypted private key file path to save the - application default client encrypted private key for mTLS. This field is - required if you want to use the default client certificate. - `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable must be set to - `true` in order to use this field, otherwise this field doesn't nothing. - More details on the environment variables are here: - https://google.aip.dev/auth/4114 - num_retries: Integer, number of times to retry discovery with - randomized exponential backoff in case of intermittent/connection issues. - static_discovery: Boolean, whether or not to use the static discovery docs - included in the library. The default value for `static_discovery` depends - on the value of `discoveryServiceUrl`. `static_discovery` will default to - `True` when `discoveryServiceUrl` is also not provided, otherwise it will - default to `False`. - always_use_jwt_access: Boolean, whether always use self signed JWT for service - account credentials. This only applies to - google.oauth2.service_account.Credentials. - - Returns: - A Resource object with methods for interacting with the service. - - Raises: - google.auth.exceptions.MutualTLSChannelError: if there are any problems - setting up mutual TLS channel. - """ - params = {"api": serviceName, "apiVersion": version} - - # The default value for `static_discovery` depends on the value of - # `discoveryServiceUrl`. `static_discovery` will default to `True` when - # `discoveryServiceUrl` is also not provided, otherwise it will default to - # `False`. This is added for backwards compatability with - # google-api-python-client 1.x which does not support the `static_discovery` - # parameter. - if static_discovery is None: - if discoveryServiceUrl is None: - static_discovery = True - else: - static_discovery = False - - if http is None: - discovery_http = build_http() - else: - discovery_http = http - - service = None - - for discovery_url in _discovery_service_uri_options(discoveryServiceUrl, version): - requested_url = uritemplate.expand(discovery_url, params) - - try: - content = _retrieve_discovery_doc( - requested_url, - discovery_http, - cache_discovery, - serviceName, - version, - cache, - developerKey, - num_retries=num_retries, - static_discovery=static_discovery, - ) - service = build_from_document( - content, - base=discovery_url, - http=http, - developerKey=developerKey, - model=model, - requestBuilder=requestBuilder, - credentials=credentials, - client_options=client_options, - adc_cert_path=adc_cert_path, - adc_key_path=adc_key_path, - always_use_jwt_access=always_use_jwt_access, - ) - break # exit if a service was created - except HttpError as e: - if e.resp.status == http_client.NOT_FOUND: - continue - else: - raise e - - # If discovery_http was created by this function, we are done with it - # and can safely close it - if http is None: - discovery_http.close() - - if service is None: - raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName, version)) - else: - return service - - -def _discovery_service_uri_options(discoveryServiceUrl, version): - """ - Returns Discovery URIs to be used for attempting to build the API Resource. - - Args: - discoveryServiceUrl: - string, the Original Discovery Service URL preferred by the customer. - version: - string, API Version requested - - Returns: - A list of URIs to be tried for the Service Discovery, in order. - """ - - if discoveryServiceUrl is not None: - return [discoveryServiceUrl] - if version is None: - # V1 Discovery won't work if the requested version is None - logger.warning( - "Discovery V1 does not support empty versions. Defaulting to V2..." - ) - return [V2_DISCOVERY_URI] - else: - return [DISCOVERY_URI, V2_DISCOVERY_URI] - - -def _retrieve_discovery_doc( - url, - http, - cache_discovery, - serviceName, - version, - cache=None, - developerKey=None, - num_retries=1, - static_discovery=True, -): - """Retrieves the discovery_doc from cache or the internet. - - Args: - url: string, the URL of the discovery document. - http: httplib2.Http, An instance of httplib2.Http or something that acts - like it through which HTTP requests will be made. - cache_discovery: Boolean, whether or not to cache the discovery doc. - serviceName: string, name of the service. - version: string, the version of the service. - cache: googleapiclient.discovery_cache.base.Cache, an optional cache - object for the discovery documents. - developerKey: string, Key for controlling API usage, generated - from the API Console. - num_retries: Integer, number of times to retry discovery with - randomized exponential backoff in case of intermittent/connection issues. - static_discovery: Boolean, whether or not to use the static discovery docs - included in the library. - - Returns: - A unicode string representation of the discovery document. - """ - from . import discovery_cache - - if cache_discovery: - if cache is None: - cache = discovery_cache.autodetect() - if cache: - content = cache.get(url) - if content: - return content - - # When `static_discovery=True`, use static discovery artifacts included - # with the library - if static_discovery: - content = discovery_cache.get_static_doc(serviceName, version) - if content: - return content - else: - raise UnknownApiNameOrVersion( - "name: %s version: %s" % (serviceName, version) - ) - - actual_url = url - # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment - # variable that contains the network address of the client sending the - # request. If it exists then add that to the request for the discovery - # document to avoid exceeding the quota on discovery requests. - if "REMOTE_ADDR" in os.environ: - actual_url = _add_query_parameter(url, "userIp", os.environ["REMOTE_ADDR"]) - if developerKey: - actual_url = _add_query_parameter(url, "key", developerKey) - logger.debug("URL being requested: GET %s", actual_url) - - # Execute this request with retries build into HttpRequest - # Note that it will already raise an error if we don't get a 2xx response - req = HttpRequest(http, HttpRequest.null_postproc, actual_url) - resp, content = req.execute(num_retries=num_retries) - - try: - content = content.decode("utf-8") - except AttributeError: - pass - - try: - service = json.loads(content) - except ValueError as e: - logger.error("Failed to parse as JSON: " + content) - raise InvalidJsonError() - if cache_discovery and cache: - cache.set(url, content) - return content - - -def _check_api_core_compatible_with_credentials_universe(credentials): - if not HAS_UNIVERSE: - credentials_universe = getattr(credentials, "universe_domain", None) - if credentials_universe and credentials_universe != DEFAULT_UNIVERSE: - raise APICoreVersionError - - -@positional(1) -def build_from_document( - service, - base=None, - future=None, - http=None, - developerKey=None, - model=None, - requestBuilder=HttpRequest, - credentials=None, - client_options=None, - adc_cert_path=None, - adc_key_path=None, - always_use_jwt_access=False, -): - """Create a Resource for interacting with an API. - - Same as `build()`, but constructs the Resource object from a discovery - document that is it given, as opposed to retrieving one over HTTP. - - Args: - service: string or object, the JSON discovery document describing the API. - The value passed in may either be the JSON string or the deserialized - JSON. - base: string, base URI for all HTTP requests, usually the discovery URI. - This parameter is no longer used as rootUrl and servicePath are included - within the discovery document. (deprecated) - future: string, discovery document with future capabilities (deprecated). - http: httplib2.Http, An instance of httplib2.Http or something that acts - like it that HTTP requests will be made through. - developerKey: string, Key for controlling API usage, generated - from the API Console. - model: Model class instance that serializes and de-serializes requests and - responses. - requestBuilder: Takes an http request and packages it up to be executed. - credentials: oauth2client.Credentials or - google.auth.credentials.Credentials, credentials to be used for - authentication. - client_options: Mapping object or google.api_core.client_options, client - options to set user options on the client. - (1) The API endpoint should be set through client_options. If API endpoint - is not set, `GOOGLE_API_USE_MTLS_ENDPOINT` environment variable can be used - to control which endpoint to use. - (2) client_cert_source is not supported, client cert should be provided using - client_encrypted_cert_source instead. In order to use the provided client - cert, `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable must be - set to `true`. - More details on the environment variables are here: - https://google.aip.dev/auth/4114 - adc_cert_path: str, client certificate file path to save the application - default client certificate for mTLS. This field is required if you want to - use the default client certificate. `GOOGLE_API_USE_CLIENT_CERTIFICATE` - environment variable must be set to `true` in order to use this field, - otherwise this field doesn't nothing. - More details on the environment variables are here: - https://google.aip.dev/auth/4114 - adc_key_path: str, client encrypted private key file path to save the - application default client encrypted private key for mTLS. This field is - required if you want to use the default client certificate. - `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable must be set to - `true` in order to use this field, otherwise this field doesn't nothing. - More details on the environment variables are here: - https://google.aip.dev/auth/4114 - always_use_jwt_access: Boolean, whether always use self signed JWT for service - account credentials. This only applies to - google.oauth2.service_account.Credentials. - - Returns: - A Resource object with methods for interacting with the service. - - Raises: - google.auth.exceptions.MutualTLSChannelError: if there are any problems - setting up mutual TLS channel. - """ - - if client_options is None: - client_options = google.api_core.client_options.ClientOptions() - if isinstance(client_options, collections.abc.Mapping): - client_options = google.api_core.client_options.from_dict(client_options) - - if http is not None: - # if http is passed, the user cannot provide credentials - banned_options = [ - (credentials, "credentials"), - (client_options.credentials_file, "client_options.credentials_file"), - ] - for option, name in banned_options: - if option is not None: - raise ValueError( - "Arguments http and {} are mutually exclusive".format(name) - ) - - if isinstance(service, str): - service = json.loads(service) - elif isinstance(service, bytes): - service = json.loads(service.decode("utf-8")) - - if "rootUrl" not in service and isinstance(http, (HttpMock, HttpMockSequence)): - logger.error( - "You are using HttpMock or HttpMockSequence without" - + "having the service discovery doc in cache. Try calling " - + "build() without mocking once first to populate the " - + "cache." - ) - raise InvalidJsonError() - - # If an API Endpoint is provided on client options, use that as the base URL - base = urllib.parse.urljoin(service["rootUrl"], service["servicePath"]) - universe_domain = None - if HAS_UNIVERSE: - universe_domain_env = os.getenv(GOOGLE_CLOUD_UNIVERSE_DOMAIN, None) - universe_domain = universe.determine_domain( - client_options.universe_domain, universe_domain_env - ) - base = base.replace(universe.DEFAULT_UNIVERSE, universe_domain) - else: - client_universe = getattr(client_options, "universe_domain", None) - if client_universe: - raise APICoreVersionError - - audience_for_self_signed_jwt = base - if client_options.api_endpoint: - base = client_options.api_endpoint - - schema = Schemas(service) - - # If the http client is not specified, then we must construct an http client - # to make requests. If the service has scopes, then we also need to setup - # authentication. - if http is None: - # Does the service require scopes? - scopes = list( - service.get("auth", {}).get("oauth2", {}).get("scopes", {}).keys() - ) - - # If so, then the we need to setup authentication if no developerKey is - # specified. - if scopes and not developerKey: - # Make sure the user didn't pass multiple credentials - if client_options.credentials_file and credentials: - raise google.api_core.exceptions.DuplicateCredentialArgs( - "client_options.credentials_file and credentials are mutually exclusive." - ) - # Check for credentials file via client options - if client_options.credentials_file: - credentials = _auth.credentials_from_file( - client_options.credentials_file, - scopes=client_options.scopes, - quota_project_id=client_options.quota_project_id, - ) - # If the user didn't pass in credentials, attempt to acquire application - # default credentials. - if credentials is None: - credentials = _auth.default_credentials( - scopes=client_options.scopes, - quota_project_id=client_options.quota_project_id, - ) - - # Check google-api-core >= 2.18.0 if credentials' universe != "googleapis.com". - _check_api_core_compatible_with_credentials_universe(credentials) - - # The credentials need to be scoped. - # If the user provided scopes via client_options don't override them - if not client_options.scopes: - credentials = _auth.with_scopes(credentials, scopes) - - # For google-auth service account credentials, enable self signed JWT if - # always_use_jwt_access is true. - if ( - credentials - and isinstance(credentials, service_account.Credentials) - and always_use_jwt_access - and hasattr(service_account.Credentials, "with_always_use_jwt_access") - ): - credentials = credentials.with_always_use_jwt_access(always_use_jwt_access) - credentials._create_self_signed_jwt(audience_for_self_signed_jwt) - - # If credentials are provided, create an authorized http instance; - # otherwise, skip authentication. - if credentials: - http = _auth.authorized_http(credentials) - - # If the service doesn't require scopes then there is no need for - # authentication. - else: - http = build_http() - - # Obtain client cert and create mTLS http channel if cert exists. - client_cert_to_use = None - use_client_cert = os.getenv(GOOGLE_API_USE_CLIENT_CERTIFICATE, "false") - if not use_client_cert in ("true", "false"): - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_CLIENT_CERTIFICATE value. Accepted values: true, false" - ) - if client_options and client_options.client_cert_source: - raise MutualTLSChannelError( - "ClientOptions.client_cert_source is not supported, please use ClientOptions.client_encrypted_cert_source." - ) - if use_client_cert == "true": - if ( - client_options - and hasattr(client_options, "client_encrypted_cert_source") - and client_options.client_encrypted_cert_source - ): - client_cert_to_use = client_options.client_encrypted_cert_source - elif ( - adc_cert_path and adc_key_path and mtls.has_default_client_cert_source() - ): - client_cert_to_use = mtls.default_client_encrypted_cert_source( - adc_cert_path, adc_key_path - ) - if client_cert_to_use: - cert_path, key_path, passphrase = client_cert_to_use() - - # The http object we built could be google_auth_httplib2.AuthorizedHttp - # or httplib2.Http. In the first case we need to extract the wrapped - # httplib2.Http object from google_auth_httplib2.AuthorizedHttp. - http_channel = ( - http.http - if google_auth_httplib2 - and isinstance(http, google_auth_httplib2.AuthorizedHttp) - else http - ) - http_channel.add_certificate(key_path, cert_path, "", passphrase) - - # If user doesn't provide api endpoint via client options, decide which - # api endpoint to use. - if "mtlsRootUrl" in service and ( - not client_options or not client_options.api_endpoint - ): - mtls_endpoint = urllib.parse.urljoin( - service["mtlsRootUrl"], service["servicePath"] - ) - use_mtls_endpoint = os.getenv(GOOGLE_API_USE_MTLS_ENDPOINT, "auto") - - if not use_mtls_endpoint in ("never", "auto", "always"): - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always" - ) - - # Switch to mTLS endpoint, if environment variable is "always", or - # environment varibable is "auto" and client cert exists. - if use_mtls_endpoint == "always" or ( - use_mtls_endpoint == "auto" and client_cert_to_use - ): - if HAS_UNIVERSE and universe_domain != universe.DEFAULT_UNIVERSE: - raise MutualTLSChannelError( - f"mTLS is not supported in any universe other than {universe.DEFAULT_UNIVERSE}." - ) - base = mtls_endpoint - else: - # Check google-api-core >= 2.18.0 if credentials' universe != "googleapis.com". - http_credentials = getattr(http, "credentials", None) - _check_api_core_compatible_with_credentials_universe(http_credentials) - - if model is None: - features = service.get("features", []) - model = JsonModel("dataWrapper" in features) - - return Resource( - http=http, - baseUrl=base, - model=model, - developerKey=developerKey, - requestBuilder=requestBuilder, - resourceDesc=service, - rootDesc=service, - schema=schema, - universe_domain=universe_domain, - ) - - -def _cast(value, schema_type): - """Convert value to a string based on JSON Schema type. - - See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on - JSON Schema. - - Args: - value: any, the value to convert - schema_type: string, the type that value should be interpreted as - - Returns: - A string representation of 'value' based on the schema_type. - """ - if schema_type == "string": - if type(value) == type("") or type(value) == type(""): - return value - else: - return str(value) - elif schema_type == "integer": - return str(int(value)) - elif schema_type == "number": - return str(float(value)) - elif schema_type == "boolean": - return str(bool(value)).lower() - else: - if type(value) == type("") or type(value) == type(""): - return value - else: - return str(value) - - -def _media_size_to_long(maxSize): - """Convert a string media size, such as 10GB or 3TB into an integer. - - Args: - maxSize: string, size as a string, such as 2MB or 7GB. - - Returns: - The size as an integer value. - """ - if len(maxSize) < 2: - return 0 - units = maxSize[-2:].upper() - bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units) - if bit_shift is not None: - return int(maxSize[:-2]) << bit_shift - else: - return int(maxSize) - - -def _media_path_url_from_info(root_desc, path_url): - """Creates an absolute media path URL. - - Constructed using the API root URI and service path from the discovery - document and the relative path for the API method. - - Args: - root_desc: Dictionary; the entire original deserialized discovery document. - path_url: String; the relative URL for the API method. Relative to the API - root, which is specified in the discovery document. - - Returns: - String; the absolute URI for media upload for the API method. - """ - return "%(root)supload/%(service_path)s%(path)s" % { - "root": root_desc["rootUrl"], - "service_path": root_desc["servicePath"], - "path": path_url, - } - - -def _fix_up_parameters(method_desc, root_desc, http_method, schema): - """Updates parameters of an API method with values specific to this library. - - Specifically, adds whatever global parameters are specified by the API to the - parameters for the individual method. Also adds parameters which don't - appear in the discovery document, but are available to all discovery based - APIs (these are listed in STACK_QUERY_PARAMETERS). - - SIDE EFFECTS: This updates the parameters dictionary object in the method - description. - - Args: - method_desc: Dictionary with metadata describing an API method. Value comes - from the dictionary of methods stored in the 'methods' key in the - deserialized discovery document. - root_desc: Dictionary; the entire original deserialized discovery document. - http_method: String; the HTTP method used to call the API method described - in method_desc. - schema: Object, mapping of schema names to schema descriptions. - - Returns: - The updated Dictionary stored in the 'parameters' key of the method - description dictionary. - """ - parameters = method_desc.setdefault("parameters", {}) - - # Add in the parameters common to all methods. - for name, description in root_desc.get("parameters", {}).items(): - parameters[name] = description - - # Add in undocumented query parameters. - for name in STACK_QUERY_PARAMETERS: - parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy() - - # Add 'body' (our own reserved word) to parameters if the method supports - # a request payload. - if http_method in HTTP_PAYLOAD_METHODS and "request" in method_desc: - body = BODY_PARAMETER_DEFAULT_VALUE.copy() - body.update(method_desc["request"]) - parameters["body"] = body - - return parameters - - -def _fix_up_media_upload(method_desc, root_desc, path_url, parameters): - """Adds 'media_body' and 'media_mime_type' parameters if supported by method. - - SIDE EFFECTS: If there is a 'mediaUpload' in the method description, adds - 'media_upload' key to parameters. - - Args: - method_desc: Dictionary with metadata describing an API method. Value comes - from the dictionary of methods stored in the 'methods' key in the - deserialized discovery document. - root_desc: Dictionary; the entire original deserialized discovery document. - path_url: String; the relative URL for the API method. Relative to the API - root, which is specified in the discovery document. - parameters: A dictionary describing method parameters for method described - in method_desc. - - Returns: - Triple (accept, max_size, media_path_url) where: - - accept is a list of strings representing what content types are - accepted for media upload. Defaults to empty list if not in the - discovery document. - - max_size is a long representing the max size in bytes allowed for a - media upload. Defaults to 0L if not in the discovery document. - - media_path_url is a String; the absolute URI for media upload for the - API method. Constructed using the API root URI and service path from - the discovery document and the relative path for the API method. If - media upload is not supported, this is None. - """ - media_upload = method_desc.get("mediaUpload", {}) - accept = media_upload.get("accept", []) - max_size = _media_size_to_long(media_upload.get("maxSize", "")) - media_path_url = None - - if media_upload: - media_path_url = _media_path_url_from_info(root_desc, path_url) - parameters["media_body"] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy() - parameters["media_mime_type"] = MEDIA_MIME_TYPE_PARAMETER_DEFAULT_VALUE.copy() - - return accept, max_size, media_path_url - - -def _fix_up_method_description(method_desc, root_desc, schema): - """Updates a method description in a discovery document. - - SIDE EFFECTS: Changes the parameters dictionary in the method description with - extra parameters which are used locally. - - Args: - method_desc: Dictionary with metadata describing an API method. Value comes - from the dictionary of methods stored in the 'methods' key in the - deserialized discovery document. - root_desc: Dictionary; the entire original deserialized discovery document. - schema: Object, mapping of schema names to schema descriptions. - - Returns: - Tuple (path_url, http_method, method_id, accept, max_size, media_path_url) - where: - - path_url is a String; the relative URL for the API method. Relative to - the API root, which is specified in the discovery document. - - http_method is a String; the HTTP method used to call the API method - described in the method description. - - method_id is a String; the name of the RPC method associated with the - API method, and is in the method description in the 'id' key. - - accept is a list of strings representing what content types are - accepted for media upload. Defaults to empty list if not in the - discovery document. - - max_size is a long representing the max size in bytes allowed for a - media upload. Defaults to 0L if not in the discovery document. - - media_path_url is a String; the absolute URI for media upload for the - API method. Constructed using the API root URI and service path from - the discovery document and the relative path for the API method. If - media upload is not supported, this is None. - """ - path_url = method_desc["path"] - http_method = method_desc["httpMethod"] - method_id = method_desc["id"] - - parameters = _fix_up_parameters(method_desc, root_desc, http_method, schema) - # Order is important. `_fix_up_media_upload` needs `method_desc` to have a - # 'parameters' key and needs to know if there is a 'body' parameter because it - # also sets a 'media_body' parameter. - accept, max_size, media_path_url = _fix_up_media_upload( - method_desc, root_desc, path_url, parameters - ) - - return path_url, http_method, method_id, accept, max_size, media_path_url - - -def _fix_up_media_path_base_url(media_path_url, base_url): - """ - Update the media upload base url if its netloc doesn't match base url netloc. - - This can happen in case the base url was overridden by - client_options.api_endpoint. - - Args: - media_path_url: String; the absolute URI for media upload. - base_url: string, base URL for the API. All requests are relative to this URI. - - Returns: - String; the absolute URI for media upload. - """ - parsed_media_url = urllib.parse.urlparse(media_path_url) - parsed_base_url = urllib.parse.urlparse(base_url) - if parsed_media_url.netloc == parsed_base_url.netloc: - return media_path_url - return urllib.parse.urlunparse( - parsed_media_url._replace(netloc=parsed_base_url.netloc) - ) - - -def _urljoin(base, url): - """Custom urljoin replacement supporting : before / in url.""" - # In general, it's unsafe to simply join base and url. However, for - # the case of discovery documents, we know: - # * base will never contain params, query, or fragment - # * url will never contain a scheme or net_loc. - # In general, this means we can safely join on /; we just need to - # ensure we end up with precisely one / joining base and url. The - # exception here is the case of media uploads, where url will be an - # absolute url. - if url.startswith("http://") or url.startswith("https://"): - return urllib.parse.urljoin(base, url) - new_base = base if base.endswith("/") else base + "/" - new_url = url[1:] if url.startswith("/") else url - return new_base + new_url - - -# TODO(dhermes): Convert this class to ResourceMethod and make it callable -class ResourceMethodParameters(object): - """Represents the parameters associated with a method. - - Attributes: - argmap: Map from method parameter name (string) to query parameter name - (string). - required_params: List of required parameters (represented by parameter - name as string). - repeated_params: List of repeated parameters (represented by parameter - name as string). - pattern_params: Map from method parameter name (string) to regular - expression (as a string). If the pattern is set for a parameter, the - value for that parameter must match the regular expression. - query_params: List of parameters (represented by parameter name as string) - that will be used in the query string. - path_params: Set of parameters (represented by parameter name as string) - that will be used in the base URL path. - param_types: Map from method parameter name (string) to parameter type. Type - can be any valid JSON schema type; valid values are 'any', 'array', - 'boolean', 'integer', 'number', 'object', or 'string'. Reference: - http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1 - enum_params: Map from method parameter name (string) to list of strings, - where each list of strings is the list of acceptable enum values. - """ - - def __init__(self, method_desc): - """Constructor for ResourceMethodParameters. - - Sets default values and defers to set_parameters to populate. - - Args: - method_desc: Dictionary with metadata describing an API method. Value - comes from the dictionary of methods stored in the 'methods' key in - the deserialized discovery document. - """ - self.argmap = {} - self.required_params = [] - self.repeated_params = [] - self.pattern_params = {} - self.query_params = [] - # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE - # parsing is gotten rid of. - self.path_params = set() - self.param_types = {} - self.enum_params = {} - - self.set_parameters(method_desc) - - def set_parameters(self, method_desc): - """Populates maps and lists based on method description. - - Iterates through each parameter for the method and parses the values from - the parameter dictionary. - - Args: - method_desc: Dictionary with metadata describing an API method. Value - comes from the dictionary of methods stored in the 'methods' key in - the deserialized discovery document. - """ - parameters = method_desc.get("parameters", {}) - sorted_parameters = OrderedDict(sorted(parameters.items())) - for arg, desc in sorted_parameters.items(): - param = key2param(arg) - self.argmap[param] = arg - - if desc.get("pattern"): - self.pattern_params[param] = desc["pattern"] - if desc.get("enum"): - self.enum_params[param] = desc["enum"] - if desc.get("required"): - self.required_params.append(param) - if desc.get("repeated"): - self.repeated_params.append(param) - if desc.get("location") == "query": - self.query_params.append(param) - if desc.get("location") == "path": - self.path_params.add(param) - self.param_types[param] = desc.get("type", "string") - - # TODO(dhermes): Determine if this is still necessary. Discovery based APIs - # should have all path parameters already marked with - # 'location: path'. - for match in URITEMPLATE.finditer(method_desc["path"]): - for namematch in VARNAME.finditer(match.group(0)): - name = key2param(namematch.group(0)) - self.path_params.add(name) - if name in self.query_params: - self.query_params.remove(name) - - -def createMethod(methodName, methodDesc, rootDesc, schema): - """Creates a method for attaching to a Resource. - - Args: - methodName: string, name of the method to use. - methodDesc: object, fragment of deserialized discovery document that - describes the method. - rootDesc: object, the entire deserialized discovery document. - schema: object, mapping of schema names to schema descriptions. - """ - methodName = fix_method_name(methodName) - ( - pathUrl, - httpMethod, - methodId, - accept, - maxSize, - mediaPathUrl, - ) = _fix_up_method_description(methodDesc, rootDesc, schema) - - parameters = ResourceMethodParameters(methodDesc) - - def method(self, **kwargs): - # Don't bother with doc string, it will be over-written by createMethod. - - # Validate credentials for the configured universe. - self._validate_credentials() - - for name in kwargs: - if name not in parameters.argmap: - raise TypeError("Got an unexpected keyword argument {}".format(name)) - - # Remove args that have a value of None. - keys = list(kwargs.keys()) - for name in keys: - if kwargs[name] is None: - del kwargs[name] - - for name in parameters.required_params: - if name not in kwargs: - # temporary workaround for non-paging methods incorrectly requiring - # page token parameter (cf. drive.changes.watch vs. drive.changes.list) - if name not in _PAGE_TOKEN_NAMES or _findPageTokenName( - _methodProperties(methodDesc, schema, "response") - ): - raise TypeError('Missing required parameter "%s"' % name) - - for name, regex in parameters.pattern_params.items(): - if name in kwargs: - if isinstance(kwargs[name], str): - pvalues = [kwargs[name]] - else: - pvalues = kwargs[name] - for pvalue in pvalues: - if re.match(regex, pvalue) is None: - raise TypeError( - 'Parameter "%s" value "%s" does not match the pattern "%s"' - % (name, pvalue, regex) - ) - - for name, enums in parameters.enum_params.items(): - if name in kwargs: - # We need to handle the case of a repeated enum - # name differently, since we want to handle both - # arg='value' and arg=['value1', 'value2'] - if name in parameters.repeated_params and not isinstance( - kwargs[name], str - ): - values = kwargs[name] - else: - values = [kwargs[name]] - for value in values: - if value not in enums: - raise TypeError( - 'Parameter "%s" value "%s" is not an allowed value in "%s"' - % (name, value, str(enums)) - ) - - actual_query_params = {} - actual_path_params = {} - for key, value in kwargs.items(): - to_type = parameters.param_types.get(key, "string") - # For repeated parameters we cast each member of the list. - if key in parameters.repeated_params and type(value) == type([]): - cast_value = [_cast(x, to_type) for x in value] - else: - cast_value = _cast(value, to_type) - if key in parameters.query_params: - actual_query_params[parameters.argmap[key]] = cast_value - if key in parameters.path_params: - actual_path_params[parameters.argmap[key]] = cast_value - body_value = kwargs.get("body", None) - media_filename = kwargs.get("media_body", None) - media_mime_type = kwargs.get("media_mime_type", None) - - if self._developerKey: - actual_query_params["key"] = self._developerKey - - model = self._model - if methodName.endswith("_media"): - model = MediaModel() - elif "response" not in methodDesc: - model = RawModel() - - api_version = methodDesc.get("apiVersion", None) - - headers = {} - headers, params, query, body = model.request( - headers, actual_path_params, actual_query_params, body_value, api_version - ) - - expanded_url = uritemplate.expand(pathUrl, params) - url = _urljoin(self._baseUrl, expanded_url + query) - - resumable = None - multipart_boundary = "" - - if media_filename: - # Ensure we end up with a valid MediaUpload object. - if isinstance(media_filename, str): - if media_mime_type is None: - logger.warning( - "media_mime_type argument not specified: trying to auto-detect for %s", - media_filename, - ) - media_mime_type, _ = mimetypes.guess_type(media_filename) - if media_mime_type is None: - raise UnknownFileType(media_filename) - if not mimeparse.best_match([media_mime_type], ",".join(accept)): - raise UnacceptableMimeTypeError(media_mime_type) - media_upload = MediaFileUpload(media_filename, mimetype=media_mime_type) - elif isinstance(media_filename, MediaUpload): - media_upload = media_filename - else: - raise TypeError("media_filename must be str or MediaUpload.") - - # Check the maxSize - if media_upload.size() is not None and media_upload.size() > maxSize > 0: - raise MediaUploadSizeError("Media larger than: %s" % maxSize) - - # Use the media path uri for media uploads - expanded_url = uritemplate.expand(mediaPathUrl, params) - url = _urljoin(self._baseUrl, expanded_url + query) - url = _fix_up_media_path_base_url(url, self._baseUrl) - if media_upload.resumable(): - url = _add_query_parameter(url, "uploadType", "resumable") - - if media_upload.resumable(): - # This is all we need to do for resumable, if the body exists it gets - # sent in the first request, otherwise an empty body is sent. - resumable = media_upload - else: - # A non-resumable upload - if body is None: - # This is a simple media upload - headers["content-type"] = media_upload.mimetype() - body = media_upload.getbytes(0, media_upload.size()) - url = _add_query_parameter(url, "uploadType", "media") - else: - # This is a multipart/related upload. - msgRoot = MIMEMultipart("related") - # msgRoot should not write out it's own headers - setattr(msgRoot, "_write_headers", lambda self: None) - - # attach the body as one part - msg = MIMENonMultipart(*headers["content-type"].split("/")) - msg.set_payload(body) - msgRoot.attach(msg) - - # attach the media as the second part - msg = MIMENonMultipart(*media_upload.mimetype().split("/")) - msg["Content-Transfer-Encoding"] = "binary" - - payload = media_upload.getbytes(0, media_upload.size()) - msg.set_payload(payload) - msgRoot.attach(msg) - # encode the body: note that we can't use `as_string`, because - # it plays games with `From ` lines. - fp = io.BytesIO() - g = _BytesGenerator(fp, mangle_from_=False) - g.flatten(msgRoot, unixfrom=False) - body = fp.getvalue() - - multipart_boundary = msgRoot.get_boundary() - headers["content-type"] = ( - "multipart/related; " 'boundary="%s"' - ) % multipart_boundary - url = _add_query_parameter(url, "uploadType", "multipart") - - logger.debug("URL being requested: %s %s" % (httpMethod, url)) - return self._requestBuilder( - self._http, - model.response, - url, - method=httpMethod, - body=body, - headers=headers, - methodId=methodId, - resumable=resumable, - ) - - docs = [methodDesc.get("description", DEFAULT_METHOD_DOC), "\n\n"] - if len(parameters.argmap) > 0: - docs.append("Args:\n") - - # Skip undocumented params and params common to all methods. - skip_parameters = list(rootDesc.get("parameters", {}).keys()) - skip_parameters.extend(STACK_QUERY_PARAMETERS) - - all_args = list(parameters.argmap.keys()) - args_ordered = [key2param(s) for s in methodDesc.get("parameterOrder", [])] - - # Move body to the front of the line. - if "body" in all_args: - args_ordered.append("body") - - for name in sorted(all_args): - if name not in args_ordered: - args_ordered.append(name) - - for arg in args_ordered: - if arg in skip_parameters: - continue - - repeated = "" - if arg in parameters.repeated_params: - repeated = " (repeated)" - required = "" - if arg in parameters.required_params: - required = " (required)" - paramdesc = methodDesc["parameters"][parameters.argmap[arg]] - paramdoc = paramdesc.get("description", "A parameter") - if "$ref" in paramdesc: - docs.append( - (" %s: object, %s%s%s\n The object takes the form of:\n\n%s\n\n") - % ( - arg, - paramdoc, - required, - repeated, - schema.prettyPrintByName(paramdesc["$ref"]), - ) - ) - else: - paramtype = paramdesc.get("type", "string") - docs.append( - " %s: %s, %s%s%s\n" % (arg, paramtype, paramdoc, required, repeated) - ) - enum = paramdesc.get("enum", []) - enumDesc = paramdesc.get("enumDescriptions", []) - if enum and enumDesc: - docs.append(" Allowed values\n") - for (name, desc) in zip(enum, enumDesc): - docs.append(" %s - %s\n" % (name, desc)) - if "response" in methodDesc: - if methodName.endswith("_media"): - docs.append("\nReturns:\n The media object as a string.\n\n ") - else: - docs.append("\nReturns:\n An object of the form:\n\n ") - docs.append(schema.prettyPrintSchema(methodDesc["response"])) - - setattr(method, "__doc__", "".join(docs)) - return (methodName, method) - - -def createNextMethod( - methodName, - pageTokenName="pageToken", - nextPageTokenName="nextPageToken", - isPageTokenParameter=True, -): - """Creates any _next methods for attaching to a Resource. - - The _next methods allow for easy iteration through list() responses. - - Args: - methodName: string, name of the method to use. - pageTokenName: string, name of request page token field. - nextPageTokenName: string, name of response page token field. - isPageTokenParameter: Boolean, True if request page token is a query - parameter, False if request page token is a field of the request body. - """ - methodName = fix_method_name(methodName) - - def methodNext(self, previous_request, previous_response): - """Retrieves the next page of results. - - Args: - previous_request: The request for the previous page. (required) - previous_response: The response from the request for the previous page. (required) - - Returns: - A request object that you can call 'execute()' on to request the next - page. Returns None if there are no more items in the collection. - """ - # Retrieve nextPageToken from previous_response - # Use as pageToken in previous_request to create new request. - - nextPageToken = previous_response.get(nextPageTokenName, None) - if not nextPageToken: - return None - - request = copy.copy(previous_request) - - if isPageTokenParameter: - # Replace pageToken value in URI - request.uri = _add_query_parameter( - request.uri, pageTokenName, nextPageToken - ) - logger.debug("Next page request URL: %s %s" % (methodName, request.uri)) - else: - # Replace pageToken value in request body - model = self._model - body = model.deserialize(request.body) - body[pageTokenName] = nextPageToken - request.body = model.serialize(body) - request.body_size = len(request.body) - if "content-length" in request.headers: - del request.headers["content-length"] - logger.debug("Next page request body: %s %s" % (methodName, body)) - - return request - - return (methodName, methodNext) - - -class Resource(object): - """A class for interacting with a resource.""" - - def __init__( - self, - http, - baseUrl, - model, - requestBuilder, - developerKey, - resourceDesc, - rootDesc, - schema, - universe_domain=universe.DEFAULT_UNIVERSE if HAS_UNIVERSE else "", - ): - """Build a Resource from the API description. - - Args: - http: httplib2.Http, Object to make http requests with. - baseUrl: string, base URL for the API. All requests are relative to this - URI. - model: googleapiclient.Model, converts to and from the wire format. - requestBuilder: class or callable that instantiates an - googleapiclient.HttpRequest object. - developerKey: string, key obtained from - https://code.google.com/apis/console - resourceDesc: object, section of deserialized discovery document that - describes a resource. Note that the top level discovery document - is considered a resource. - rootDesc: object, the entire deserialized discovery document. - schema: object, mapping of schema names to schema descriptions. - universe_domain: string, the universe for the API. The default universe - is "googleapis.com". - """ - self._dynamic_attrs = [] - - self._http = http - self._baseUrl = baseUrl - self._model = model - self._developerKey = developerKey - self._requestBuilder = requestBuilder - self._resourceDesc = resourceDesc - self._rootDesc = rootDesc - self._schema = schema - self._universe_domain = universe_domain - self._credentials_validated = False - - self._set_service_methods() - - def _set_dynamic_attr(self, attr_name, value): - """Sets an instance attribute and tracks it in a list of dynamic attributes. - - Args: - attr_name: string; The name of the attribute to be set - value: The value being set on the object and tracked in the dynamic cache. - """ - self._dynamic_attrs.append(attr_name) - self.__dict__[attr_name] = value - - def __getstate__(self): - """Trim the state down to something that can be pickled. - - Uses the fact that the instance variable _dynamic_attrs holds attrs that - will be wiped and restored on pickle serialization. - """ - state_dict = copy.copy(self.__dict__) - for dynamic_attr in self._dynamic_attrs: - del state_dict[dynamic_attr] - del state_dict["_dynamic_attrs"] - return state_dict - - def __setstate__(self, state): - """Reconstitute the state of the object from being pickled. - - Uses the fact that the instance variable _dynamic_attrs holds attrs that - will be wiped and restored on pickle serialization. - """ - self.__dict__.update(state) - self._dynamic_attrs = [] - self._set_service_methods() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, exc_tb): - self.close() - - def close(self): - """Close httplib2 connections.""" - # httplib2 leaves sockets open by default. - # Cleanup using the `close` method. - # https://github.com/httplib2/httplib2/issues/148 - self._http.close() - - def _set_service_methods(self): - self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema) - self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema) - self._add_next_methods(self._resourceDesc, self._schema) - - def _add_basic_methods(self, resourceDesc, rootDesc, schema): - # If this is the root Resource, add a new_batch_http_request() method. - if resourceDesc == rootDesc: - batch_uri = "%s%s" % ( - rootDesc["rootUrl"], - rootDesc.get("batchPath", "batch"), - ) - - def new_batch_http_request(callback=None): - """Create a BatchHttpRequest object based on the discovery document. - - Args: - callback: callable, A callback to be called for each response, of the - form callback(id, response, exception). The first parameter is the - request id, and the second is the deserialized response object. The - third is an apiclient.errors.HttpError exception object if an HTTP - error occurred while processing the request, or None if no error - occurred. - - Returns: - A BatchHttpRequest object based on the discovery document. - """ - return BatchHttpRequest(callback=callback, batch_uri=batch_uri) - - self._set_dynamic_attr("new_batch_http_request", new_batch_http_request) - - # Add basic methods to Resource - if "methods" in resourceDesc: - for methodName, methodDesc in resourceDesc["methods"].items(): - fixedMethodName, method = createMethod( - methodName, methodDesc, rootDesc, schema - ) - self._set_dynamic_attr( - fixedMethodName, method.__get__(self, self.__class__) - ) - # Add in _media methods. The functionality of the attached method will - # change when it sees that the method name ends in _media. - if methodDesc.get("supportsMediaDownload", False): - fixedMethodName, method = createMethod( - methodName + "_media", methodDesc, rootDesc, schema - ) - self._set_dynamic_attr( - fixedMethodName, method.__get__(self, self.__class__) - ) - - def _add_nested_resources(self, resourceDesc, rootDesc, schema): - # Add in nested resources - if "resources" in resourceDesc: - - def createResourceMethod(methodName, methodDesc): - """Create a method on the Resource to access a nested Resource. - - Args: - methodName: string, name of the method to use. - methodDesc: object, fragment of deserialized discovery document that - describes the method. - """ - methodName = fix_method_name(methodName) - - def methodResource(self): - return Resource( - http=self._http, - baseUrl=self._baseUrl, - model=self._model, - developerKey=self._developerKey, - requestBuilder=self._requestBuilder, - resourceDesc=methodDesc, - rootDesc=rootDesc, - schema=schema, - universe_domain=self._universe_domain, - ) - - setattr(methodResource, "__doc__", "A collection resource.") - setattr(methodResource, "__is_resource__", True) - - return (methodName, methodResource) - - for methodName, methodDesc in resourceDesc["resources"].items(): - fixedMethodName, method = createResourceMethod(methodName, methodDesc) - self._set_dynamic_attr( - fixedMethodName, method.__get__(self, self.__class__) - ) - - def _add_next_methods(self, resourceDesc, schema): - # Add _next() methods if and only if one of the names 'pageToken' or - # 'nextPageToken' occurs among the fields of both the method's response - # type either the method's request (query parameters) or request body. - if "methods" not in resourceDesc: - return - for methodName, methodDesc in resourceDesc["methods"].items(): - nextPageTokenName = _findPageTokenName( - _methodProperties(methodDesc, schema, "response") - ) - if not nextPageTokenName: - continue - isPageTokenParameter = True - pageTokenName = _findPageTokenName(methodDesc.get("parameters", {})) - if not pageTokenName: - isPageTokenParameter = False - pageTokenName = _findPageTokenName( - _methodProperties(methodDesc, schema, "request") - ) - if not pageTokenName: - continue - fixedMethodName, method = createNextMethod( - methodName + "_next", - pageTokenName, - nextPageTokenName, - isPageTokenParameter, - ) - self._set_dynamic_attr( - fixedMethodName, method.__get__(self, self.__class__) - ) - - def _validate_credentials(self): - """Validates client's and credentials' universe domains are consistent. - - Returns: - bool: True iff the configured universe domain is valid. - - Raises: - UniverseMismatchError: If the configured universe domain is not valid. - """ - credentials = getattr(self._http, "credentials", None) - - self._credentials_validated = ( - ( - self._credentials_validated - or universe.compare_domains(self._universe_domain, credentials) - ) - if HAS_UNIVERSE - else True - ) - return self._credentials_validated - - -def _findPageTokenName(fields): - """Search field names for one like a page token. - - Args: - fields: container of string, names of fields. - - Returns: - First name that is either 'pageToken' or 'nextPageToken' if one exists, - otherwise None. - """ - return next( - (tokenName for tokenName in _PAGE_TOKEN_NAMES if tokenName in fields), None - ) - - -def _methodProperties(methodDesc, schema, name): - """Get properties of a field in a method description. - - Args: - methodDesc: object, fragment of deserialized discovery document that - describes the method. - schema: object, mapping of schema names to schema descriptions. - name: string, name of top-level field in method description. - - Returns: - Object representing fragment of deserialized discovery document - corresponding to 'properties' field of object corresponding to named field - in method description, if it exists, otherwise empty dict. - """ - desc = methodDesc.get(name, {}) - if "$ref" in desc: - desc = schema.get(desc["$ref"], {}) - return desc.get("properties", {}) diff --git a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/__init__.py b/src/gam/googleapiclient.shouldnotbehere/discovery_cache/__init__.py deleted file mode 100644 index 6051191e..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Caching utility for the discovery document.""" - -from __future__ import absolute_import - -import logging -import os - -LOGGER = logging.getLogger(__name__) - -DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24 # 1 day -DISCOVERY_DOC_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "documents" -) - - -def autodetect(): - """Detects an appropriate cache module and returns it. - - Returns: - googleapiclient.discovery_cache.base.Cache, a cache object which - is auto detected, or None if no cache object is available. - """ - if "GAE_ENV" in os.environ: - try: - from . import appengine_memcache - - return appengine_memcache.cache - except Exception: - pass - try: - from . import file_cache - - return file_cache.cache - except Exception: - LOGGER.info( - "file_cache is only supported with oauth2client<4.0.0", exc_info=False - ) - return None - - -def get_static_doc(serviceName, version): - """Retrieves the discovery document from the directory defined in - DISCOVERY_DOC_DIR corresponding to the serviceName and version provided. - - Args: - serviceName: string, name of the service. - version: string, the version of the service. - - Returns: - A string containing the contents of the JSON discovery document, - otherwise None if the JSON discovery document was not found. - """ - - content = None - doc_name = "{}.{}.json".format(serviceName, version) - - try: - with open(os.path.join(DISCOVERY_DOC_DIR, doc_name), "r") as f: - content = f.read() - except FileNotFoundError: - # File does not exist. Nothing to do here. - pass - - return content diff --git a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/appengine_memcache.py b/src/gam/googleapiclient.shouldnotbehere/discovery_cache/appengine_memcache.py deleted file mode 100644 index 73a1decb..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/appengine_memcache.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""App Engine memcache based cache for the discovery document.""" - -import logging - -# This is only an optional dependency because we only import this -# module when google.appengine.api.memcache is available. -from google.appengine.api import memcache - -from . import base -from ..discovery_cache import DISCOVERY_DOC_MAX_AGE - -LOGGER = logging.getLogger(__name__) - -NAMESPACE = "google-api-client" - - -class Cache(base.Cache): - """A cache with app engine memcache API.""" - - def __init__(self, max_age): - """Constructor. - - Args: - max_age: Cache expiration in seconds. - """ - self._max_age = max_age - - def get(self, url): - try: - return memcache.get(url, namespace=NAMESPACE) - except Exception as e: - LOGGER.warning(e, exc_info=True) - - def set(self, url, content): - try: - memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE) - except Exception as e: - LOGGER.warning(e, exc_info=True) - - -cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE) diff --git a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/base.py b/src/gam/googleapiclient.shouldnotbehere/discovery_cache/base.py deleted file mode 100644 index 41f3f3f0..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/base.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""An abstract class for caching the discovery document.""" - -import abc - - -class Cache(object): - """A base abstract cache class.""" - - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def get(self, url): - """Gets the content from the memcache with a given key. - - Args: - url: string, the key for the cache. - - Returns: - object, the value in the cache for the given key, or None if the key is - not in the cache. - """ - raise NotImplementedError() - - @abc.abstractmethod - def set(self, url, content): - """Sets the given key and content in the cache. - - Args: - url: string, the key for the cache. - content: string, the discovery document. - """ - raise NotImplementedError() diff --git a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/file_cache.py b/src/gam/googleapiclient.shouldnotbehere/discovery_cache/file_cache.py deleted file mode 100644 index 28a72198..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/discovery_cache/file_cache.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""File based cache for the discovery document. - -The cache is stored in a single file so that multiple processes can -share the same cache. It locks the file whenever accessing to the -file. When the cache content is corrupted, it will be initialized with -an empty cache. -""" - -from __future__ import division - -import datetime -import json -import logging -import os -import tempfile - -try: - from oauth2client.contrib.locked_file import LockedFile -except ImportError: - # oauth2client < 2.0.0 - try: - from oauth2client.locked_file import LockedFile - except ImportError: - # oauth2client > 4.0.0 or google-auth - raise ImportError( - "file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth" - ) - -from . import base -from ..discovery_cache import DISCOVERY_DOC_MAX_AGE - -LOGGER = logging.getLogger(__name__) - -FILENAME = "google-api-python-client-discovery-doc.cache" -EPOCH = datetime.datetime(1970, 1, 1) - - -def _to_timestamp(date): - try: - return (date - EPOCH).total_seconds() - except AttributeError: - # The following is the equivalent of total_seconds() in Python2.6. - # See also: https://docs.python.org/2/library/datetime.html - delta = date - EPOCH - return ( - delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10**6 - ) / 10**6 - - -def _read_or_initialize_cache(f): - f.file_handle().seek(0) - try: - cache = json.load(f.file_handle()) - except Exception: - # This means it opens the file for the first time, or the cache is - # corrupted, so initializing the file with an empty dict. - cache = {} - f.file_handle().truncate(0) - f.file_handle().seek(0) - json.dump(cache, f.file_handle()) - return cache - - -class Cache(base.Cache): - """A file based cache for the discovery documents.""" - - def __init__(self, max_age): - """Constructor. - - Args: - max_age: Cache expiration in seconds. - """ - self._max_age = max_age - self._file = os.path.join(tempfile.gettempdir(), FILENAME) - f = LockedFile(self._file, "a+", "r") - try: - f.open_and_lock() - if f.is_locked(): - _read_or_initialize_cache(f) - # If we can not obtain the lock, other process or thread must - # have initialized the file. - except Exception as e: - LOGGER.warning(e, exc_info=True) - finally: - f.unlock_and_close() - - def get(self, url): - f = LockedFile(self._file, "r+", "r") - try: - f.open_and_lock() - if f.is_locked(): - cache = _read_or_initialize_cache(f) - if url in cache: - content, t = cache.get(url, (None, 0)) - if _to_timestamp(datetime.datetime.now()) < t + self._max_age: - return content - return None - else: - LOGGER.debug("Could not obtain a lock for the cache file.") - return None - except Exception as e: - LOGGER.warning(e, exc_info=True) - finally: - f.unlock_and_close() - - def set(self, url, content): - f = LockedFile(self._file, "r+", "r") - try: - f.open_and_lock() - if f.is_locked(): - cache = _read_or_initialize_cache(f) - cache[url] = (content, _to_timestamp(datetime.datetime.now())) - # Remove stale cache. - for k, (_, timestamp) in list(cache.items()): - if ( - _to_timestamp(datetime.datetime.now()) - >= timestamp + self._max_age - ): - del cache[k] - f.file_handle().truncate(0) - f.file_handle().seek(0) - json.dump(cache, f.file_handle()) - else: - LOGGER.debug("Could not obtain a lock for the cache file.") - except Exception as e: - LOGGER.warning(e, exc_info=True) - finally: - f.unlock_and_close() - - -cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE) diff --git a/src/gam/googleapiclient.shouldnotbehere/errors.py b/src/gam/googleapiclient.shouldnotbehere/errors.py deleted file mode 100644 index 288594a6..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/errors.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Errors for the library. - -All exceptions defined by the library -should be defined in this file. -""" -from __future__ import absolute_import - -__author__ = "jcgregorio@google.com (Joe Gregorio)" - -import json - -from googleapiclient import _helpers as util - - -class Error(Exception): - """Base error for this module.""" - - pass - - -class HttpError(Error): - """HTTP data was invalid or unexpected.""" - - @util.positional(3) - def __init__(self, resp, content, uri=None): - self.resp = resp - if not isinstance(content, bytes): - raise TypeError("HTTP content should be bytes") - self.content = content - self.uri = uri - self.error_details = "" - self.reason = self._get_reason() - - @property - def status_code(self): - """Return the HTTP status code from the response content.""" - return self.resp.status - - def _get_reason(self): - """Calculate the reason for the error from the response content.""" - reason = self.resp.reason - try: - try: - data = json.loads(self.content.decode("utf-8")) - except json.JSONDecodeError: - # In case it is not json - data = self.content.decode("utf-8") - if isinstance(data, dict): - reason = data["error"]["message"] - error_detail_keyword = next( - ( - kw - for kw in ["detail", "details", "errors", "message"] - if kw in data["error"] - ), - "", - ) - if error_detail_keyword: - self.error_details = data["error"][error_detail_keyword] - elif isinstance(data, list) and len(data) > 0: - first_error = data[0] - reason = first_error["error"]["message"] - if "details" in first_error["error"]: - self.error_details = first_error["error"]["details"] - else: - self.error_details = data - except (ValueError, KeyError, TypeError): - pass - if reason is None: - reason = "" - return reason.strip() - - def __repr__(self): - if self.error_details: - return '' % ( - self.resp.status, - self.uri, - self.reason, - self.error_details, - ) - elif self.uri: - return '' % ( - self.resp.status, - self.uri, - self.reason, - ) - else: - return '' % (self.resp.status, self.reason) - - __str__ = __repr__ - - -class InvalidJsonError(Error): - """The JSON returned could not be parsed.""" - - pass - - -class UnknownFileType(Error): - """File type unknown or unexpected.""" - - pass - - -class UnknownLinkType(Error): - """Link type unknown or unexpected.""" - - pass - - -class UnknownApiNameOrVersion(Error): - """No API with that name and version exists.""" - - pass - - -class UnacceptableMimeTypeError(Error): - """That is an unacceptable mimetype for this operation.""" - - pass - - -class MediaUploadSizeError(Error): - """Media is larger than the method can accept.""" - - pass - - -class ResumableUploadError(HttpError): - """Error occurred during resumable upload.""" - - pass - - -class InvalidChunkSizeError(Error): - """The given chunksize is not valid.""" - - pass - - -class InvalidNotificationError(Error): - """The channel Notification is invalid.""" - - pass - - -class BatchError(HttpError): - """Error occurred during batch operations.""" - - @util.positional(2) - def __init__(self, reason, resp=None, content=None): - self.resp = resp - self.content = content - self.reason = reason - - def __repr__(self): - if getattr(self.resp, "status", None) is None: - return '' % (self.reason) - else: - return '' % (self.resp.status, self.reason) - - __str__ = __repr__ - - -class UnexpectedMethodError(Error): - """Exception raised by RequestMockBuilder on unexpected calls.""" - - @util.positional(1) - def __init__(self, methodId=None): - """Constructor for an UnexpectedMethodError.""" - super(UnexpectedMethodError, self).__init__( - "Received unexpected call %s" % methodId - ) - - -class UnexpectedBodyError(Error): - """Exception raised by RequestMockBuilder on unexpected bodies.""" - - def __init__(self, expected, provided): - """Constructor for an UnexpectedMethodError.""" - super(UnexpectedBodyError, self).__init__( - "Expected: [%s] - Provided: [%s]" % (expected, provided) - ) diff --git a/src/gam/googleapiclient.shouldnotbehere/http.py b/src/gam/googleapiclient.shouldnotbehere/http.py deleted file mode 100644 index 187f6f5d..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/http.py +++ /dev/null @@ -1,1962 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Classes to encapsulate a single HTTP request. - -The classes implement a command pattern, with every -object supporting an execute() method that does the -actual HTTP request. -""" -from __future__ import absolute_import - -__author__ = "jcgregorio@google.com (Joe Gregorio)" - -import copy -import http.client as http_client -import io -import json -import logging -import mimetypes -import os -import random -import socket -import time -import urllib -import uuid - -import httplib2 - -# TODO(issue 221): Remove this conditional import jibbajabba. -try: - import ssl -except ImportError: - _ssl_SSLError = object() -else: - _ssl_SSLError = ssl.SSLError - -from email.generator import Generator -from email.mime.multipart import MIMEMultipart -from email.mime.nonmultipart import MIMENonMultipart -from email.parser import FeedParser - -from googleapiclient import _auth -from googleapiclient import _helpers as util -from googleapiclient.errors import ( - BatchError, - HttpError, - InvalidChunkSizeError, - ResumableUploadError, - UnexpectedBodyError, - UnexpectedMethodError, -) -from googleapiclient.model import JsonModel - -LOGGER = logging.getLogger(__name__) - -DEFAULT_CHUNK_SIZE = 100 * 1024 * 1024 - -MAX_URI_LENGTH = 2048 - -MAX_BATCH_LIMIT = 1000 - -_TOO_MANY_REQUESTS = 429 - -DEFAULT_HTTP_TIMEOUT_SEC = 60 - -_LEGACY_BATCH_URI = "https://www.googleapis.com/batch" - - -def _should_retry_response(resp_status, content): - """Determines whether a response should be retried. - - Args: - resp_status: The response status received. - content: The response content body. - - Returns: - True if the response should be retried, otherwise False. - """ - reason = None - - # Retry on 5xx errors. - if resp_status >= 500: - return True - - # Retry on 429 errors. - if resp_status == _TOO_MANY_REQUESTS: - return True - - # For 403 errors, we have to check for the `reason` in the response to - # determine if we should retry. - if resp_status == http_client.FORBIDDEN: - # If there's no details about the 403 type, don't retry. - if not content: - return False - - # Content is in JSON format. - try: - data = json.loads(content.decode("utf-8")) - if isinstance(data, dict): - # There are many variations of the error json so we need - # to determine the keyword which has the error detail. Make sure - # that the order of the keywords below isn't changed as it can - # break user code. If the "errors" key exists, we must use that - # first. - # See Issue #1243 - # https://github.com/googleapis/google-api-python-client/issues/1243 - error_detail_keyword = next( - ( - kw - for kw in ["errors", "status", "message"] - if kw in data["error"] - ), - "", - ) - - if error_detail_keyword: - reason = data["error"][error_detail_keyword] - - if isinstance(reason, list) and len(reason) > 0: - reason = reason[0] - if "reason" in reason: - reason = reason["reason"] - else: - reason = data[0]["error"]["errors"]["reason"] - except (UnicodeDecodeError, ValueError, KeyError): - LOGGER.warning("Invalid JSON content from response: %s", content) - return False - - LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) - - # Only retry on rate limit related failures. - if reason in ("userRateLimitExceeded", "rateLimitExceeded"): - return True - - # Everything else is a success or non-retriable so break. - return False - - -def _retry_request( - http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs -): - """Retries an HTTP request multiple times while handling errors. - - If after all retries the request still fails, last error is either returned as - return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). - - Args: - http: Http object to be used to execute request. - num_retries: Maximum number of retries. - req_type: Type of the request (used for logging retries). - sleep, rand: Functions to sleep for random time between retries. - uri: URI to be requested. - method: HTTP method to be used. - args, kwargs: Additional arguments passed to http.request. - - Returns: - resp, content - Response from the http request (may be HTTP 5xx). - """ - resp = None - content = None - exception = None - for retry_num in range(num_retries + 1): - if retry_num > 0: - # Sleep before retrying. - sleep_time = rand() * 2**retry_num - LOGGER.warning( - "Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s", - sleep_time, - retry_num, - num_retries, - req_type, - method, - uri, - resp.status if resp else exception, - ) - sleep(sleep_time) - - try: - exception = None - resp, content = http.request(uri, method, *args, **kwargs) - # Retry on SSL errors and socket timeout errors. - except _ssl_SSLError as ssl_error: - exception = ssl_error - except socket.timeout as socket_timeout: - # Needs to be before socket.error as it's a subclass of OSError - # socket.timeout has no errorcode - exception = socket_timeout - except ConnectionError as connection_error: - # Needs to be before socket.error as it's a subclass of OSError - exception = connection_error - except OSError as socket_error: - # errno's contents differ by platform, so we have to match by name. - # Some of these same errors may have been caught above, e.g. ECONNRESET *should* be - # raised as a ConnectionError, but some libraries will raise it as a socket.error - # with an errno corresponding to ECONNRESET - if socket.errno.errorcode.get(socket_error.errno) not in { - "WSAETIMEDOUT", - "ETIMEDOUT", - "EPIPE", - "ECONNABORTED", - "ECONNREFUSED", - "ECONNRESET", - }: - raise - exception = socket_error - except httplib2.ServerNotFoundError as server_not_found_error: - exception = server_not_found_error - - if exception: - if retry_num == num_retries: - raise exception - else: - continue - - if not _should_retry_response(resp.status, content): - break - - return resp, content - - -class MediaUploadProgress(object): - """Status of a resumable upload.""" - - def __init__(self, resumable_progress, total_size): - """Constructor. - - Args: - resumable_progress: int, bytes sent so far. - total_size: int, total bytes in complete upload, or None if the total - upload size isn't known ahead of time. - """ - self.resumable_progress = resumable_progress - self.total_size = total_size - - def progress(self): - """Percent of upload completed, as a float. - - Returns: - the percentage complete as a float, returning 0.0 if the total size of - the upload is unknown. - """ - if self.total_size is not None and self.total_size != 0: - return float(self.resumable_progress) / float(self.total_size) - else: - return 0.0 - - -class MediaDownloadProgress(object): - """Status of a resumable download.""" - - def __init__(self, resumable_progress, total_size): - """Constructor. - - Args: - resumable_progress: int, bytes received so far. - total_size: int, total bytes in complete download. - """ - self.resumable_progress = resumable_progress - self.total_size = total_size - - def progress(self): - """Percent of download completed, as a float. - - Returns: - the percentage complete as a float, returning 0.0 if the total size of - the download is unknown. - """ - if self.total_size is not None and self.total_size != 0: - return float(self.resumable_progress) / float(self.total_size) - else: - return 0.0 - - -class MediaUpload(object): - """Describes a media object to upload. - - Base class that defines the interface of MediaUpload subclasses. - - Note that subclasses of MediaUpload may allow you to control the chunksize - when uploading a media object. It is important to keep the size of the chunk - as large as possible to keep the upload efficient. Other factors may influence - the size of the chunk you use, particularly if you are working in an - environment where individual HTTP requests may have a hardcoded time limit, - such as under certain classes of requests under Google App Engine. - - Streams are io.Base compatible objects that support seek(). Some MediaUpload - subclasses support using streams directly to upload data. Support for - streaming may be indicated by a MediaUpload sub-class and if appropriate for a - platform that stream will be used for uploading the media object. The support - for streaming is indicated by has_stream() returning True. The stream() method - should return an io.Base object that supports seek(). On platforms where the - underlying httplib module supports streaming, for example Python 2.6 and - later, the stream will be passed into the http library which will result in - less memory being used and possibly faster uploads. - - If you need to upload media that can't be uploaded using any of the existing - MediaUpload sub-class then you can sub-class MediaUpload for your particular - needs. - """ - - def chunksize(self): - """Chunk size for resumable uploads. - - Returns: - Chunk size in bytes. - """ - raise NotImplementedError() - - def mimetype(self): - """Mime type of the body. - - Returns: - Mime type. - """ - return "application/octet-stream" - - def size(self): - """Size of upload. - - Returns: - Size of the body, or None of the size is unknown. - """ - return None - - def resumable(self): - """Whether this upload is resumable. - - Returns: - True if resumable upload or False. - """ - return False - - def getbytes(self, begin, end): - """Get bytes from the media. - - Args: - begin: int, offset from beginning of file. - length: int, number of bytes to read, starting at begin. - - Returns: - A string of bytes read. May be shorter than length if EOF was reached - first. - """ - raise NotImplementedError() - - def has_stream(self): - """Does the underlying upload support a streaming interface. - - Streaming means it is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - - Returns: - True if the call to stream() will return an instance of a seekable io.Base - subclass. - """ - return False - - def stream(self): - """A stream interface to the data being uploaded. - - Returns: - The returned value is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - """ - raise NotImplementedError() - - @util.positional(1) - def _to_json(self, strip=None): - """Utility function for creating a JSON representation of a MediaUpload. - - Args: - strip: array, An array of names of members to not include in the JSON. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - t = type(self) - d = copy.copy(self.__dict__) - if strip is not None: - for member in strip: - del d[member] - d["_class"] = t.__name__ - d["_module"] = t.__module__ - return json.dumps(d) - - def to_json(self): - """Create a JSON representation of an instance of MediaUpload. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - return self._to_json() - - @classmethod - def new_from_json(cls, s): - """Utility class method to instantiate a MediaUpload subclass from a JSON - representation produced by to_json(). - - Args: - s: string, JSON from to_json(). - - Returns: - An instance of the subclass of MediaUpload that was serialized with - to_json(). - """ - data = json.loads(s) - # Find and call the right classmethod from_json() to restore the object. - module = data["_module"] - m = __import__(module, fromlist=module.split(".")[:-1]) - kls = getattr(m, data["_class"]) - from_json = getattr(kls, "from_json") - return from_json(s) - - -class MediaIoBaseUpload(MediaUpload): - """A MediaUpload for a io.Base objects. - - Note that the Python file object is compatible with io.Base and can be used - with this class also. - - fh = BytesIO('...Some data to upload...') - media = MediaIoBaseUpload(fh, mimetype='image/png', - chunksize=1024*1024, resumable=True) - farm.animals().insert( - id='cow', - name='cow.png', - media_body=media).execute() - - Depending on the platform you are working on, you may pass -1 as the - chunksize, which indicates that the entire file should be uploaded in a single - request. If the underlying platform supports streams, such as Python 2.6 or - later, then this can be very efficient as it avoids multiple connections, and - also avoids loading the entire file into memory before sending it. Note that - Google App Engine has a 5MB limit on request size, so you should never set - your chunksize larger than 5MB, or to -1. - """ - - @util.positional(3) - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, resumable=False): - """Constructor. - - Args: - fd: io.Base or file object, The source of the bytes to upload. MUST be - opened in blocking mode, do not use streams opened in non-blocking mode. - The given stream must be seekable, that is, it must be able to call - seek() on fd. - mimetype: string, Mime-type of the file. - chunksize: int, File will be uploaded in chunks of this many bytes. Only - used if resumable=True. Pass in a value of -1 if the file is to be - uploaded as a single chunk. Note that Google App Engine has a 5MB limit - on request size, so you should never set your chunksize larger than 5MB, - or to -1. - resumable: bool, True if this is a resumable upload. False means upload - in a single request. - """ - super(MediaIoBaseUpload, self).__init__() - self._fd = fd - self._mimetype = mimetype - if not (chunksize == -1 or chunksize > 0): - raise InvalidChunkSizeError() - self._chunksize = chunksize - self._resumable = resumable - - self._fd.seek(0, os.SEEK_END) - self._size = self._fd.tell() - - def chunksize(self): - """Chunk size for resumable uploads. - - Returns: - Chunk size in bytes. - """ - return self._chunksize - - def mimetype(self): - """Mime type of the body. - - Returns: - Mime type. - """ - return self._mimetype - - def size(self): - """Size of upload. - - Returns: - Size of the body, or None of the size is unknown. - """ - return self._size - - def resumable(self): - """Whether this upload is resumable. - - Returns: - True if resumable upload or False. - """ - return self._resumable - - def getbytes(self, begin, length): - """Get bytes from the media. - - Args: - begin: int, offset from beginning of file. - length: int, number of bytes to read, starting at begin. - - Returns: - A string of bytes read. May be shorted than length if EOF was reached - first. - """ - self._fd.seek(begin) - return self._fd.read(length) - - def has_stream(self): - """Does the underlying upload support a streaming interface. - - Streaming means it is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - - Returns: - True if the call to stream() will return an instance of a seekable io.Base - subclass. - """ - return True - - def stream(self): - """A stream interface to the data being uploaded. - - Returns: - The returned value is an io.IOBase subclass that supports seek, i.e. - seekable() returns True. - """ - return self._fd - - def to_json(self): - """This upload type is not serializable.""" - raise NotImplementedError("MediaIoBaseUpload is not serializable.") - - -class MediaFileUpload(MediaIoBaseUpload): - """A MediaUpload for a file. - - Construct a MediaFileUpload and pass as the media_body parameter of the - method. For example, if we had a service that allowed uploading images: - - media = MediaFileUpload('cow.png', mimetype='image/png', - chunksize=1024*1024, resumable=True) - farm.animals().insert( - id='cow', - name='cow.png', - media_body=media).execute() - - Depending on the platform you are working on, you may pass -1 as the - chunksize, which indicates that the entire file should be uploaded in a single - request. If the underlying platform supports streams, such as Python 2.6 or - later, then this can be very efficient as it avoids multiple connections, and - also avoids loading the entire file into memory before sending it. Note that - Google App Engine has a 5MB limit on request size, so you should never set - your chunksize larger than 5MB, or to -1. - """ - - @util.positional(2) - def __init__( - self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False - ): - """Constructor. - - Args: - filename: string, Name of the file. - mimetype: string, Mime-type of the file. If None then a mime-type will be - guessed from the file extension. - chunksize: int, File will be uploaded in chunks of this many bytes. Only - used if resumable=True. Pass in a value of -1 if the file is to be - uploaded in a single chunk. Note that Google App Engine has a 5MB limit - on request size, so you should never set your chunksize larger than 5MB, - or to -1. - resumable: bool, True if this is a resumable upload. False means upload - in a single request. - """ - self._fd = None - self._filename = filename - self._fd = open(self._filename, "rb") - if mimetype is None: - # No mimetype provided, make a guess. - mimetype, _ = mimetypes.guess_type(filename) - if mimetype is None: - # Guess failed, use octet-stream. - mimetype = "application/octet-stream" - super(MediaFileUpload, self).__init__( - self._fd, mimetype, chunksize=chunksize, resumable=resumable - ) - - def __del__(self): - if self._fd: - self._fd.close() - - def to_json(self): - """Creating a JSON representation of an instance of MediaFileUpload. - - Returns: - string, a JSON representation of this instance, suitable to pass to - from_json(). - """ - return self._to_json(strip=["_fd"]) - - @staticmethod - def from_json(s): - d = json.loads(s) - return MediaFileUpload( - d["_filename"], - mimetype=d["_mimetype"], - chunksize=d["_chunksize"], - resumable=d["_resumable"], - ) - - -class MediaInMemoryUpload(MediaIoBaseUpload): - """MediaUpload for a chunk of bytes. - - DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or io.StringIO for - the stream. - """ - - @util.positional(2) - def __init__( - self, - body, - mimetype="application/octet-stream", - chunksize=DEFAULT_CHUNK_SIZE, - resumable=False, - ): - """Create a new MediaInMemoryUpload. - - DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or io.StringIO for - the stream. - - Args: - body: string, Bytes of body content. - mimetype: string, Mime-type of the file or default of - 'application/octet-stream'. - chunksize: int, File will be uploaded in chunks of this many bytes. Only - used if resumable=True. - resumable: bool, True if this is a resumable upload. False means upload - in a single request. - """ - fd = io.BytesIO(body) - super(MediaInMemoryUpload, self).__init__( - fd, mimetype, chunksize=chunksize, resumable=resumable - ) - - -class MediaIoBaseDownload(object): - """ "Download media resources. - - Note that the Python file object is compatible with io.Base and can be used - with this class also. - - - Example: - request = farms.animals().get_media(id='cow') - fh = io.FileIO('cow.png', mode='wb') - downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) - - done = False - while done is False: - status, done = downloader.next_chunk() - if status: - print "Download %d%%." % int(status.progress() * 100) - print "Download Complete!" - """ - - @util.positional(3) - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE): - """Constructor. - - Args: - fd: io.Base or file object, The stream in which to write the downloaded - bytes. - request: googleapiclient.http.HttpRequest, the media request to perform in - chunks. - chunksize: int, File will be downloaded in chunks of this many bytes. - """ - self._fd = fd - self._request = request - self._uri = request.uri - self._chunksize = chunksize - self._progress = 0 - self._total_size = None - self._done = False - - # Stubs for testing. - self._sleep = time.sleep - self._rand = random.random - - self._headers = {} - for k, v in request.headers.items(): - # allow users to supply custom headers by setting them on the request - # but strip out the ones that are set by default on requests generated by - # API methods like Drive's files().get(fileId=...) - if not k.lower() in ("accept", "accept-encoding", "user-agent"): - self._headers[k] = v - - @util.positional(1) - def next_chunk(self, num_retries=0): - """Get the next chunk of the download. - - Args: - num_retries: Integer, number of times to retry with randomized - exponential backoff. If all retries fail, the raised HttpError - represents the last request. If zero (default), we attempt the - request only once. - - Returns: - (status, done): (MediaDownloadProgress, boolean) - The value of 'done' will be True when the media has been fully - downloaded or the total size of the media is unknown. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx. - httplib2.HttpLib2Error if a transport error has occurred. - """ - headers = self._headers.copy() - headers["range"] = "bytes=%d-%d" % ( - self._progress, - self._progress + self._chunksize - 1, - ) - http = self._request.http - - resp, content = _retry_request( - http, - num_retries, - "media download", - self._sleep, - self._rand, - self._uri, - "GET", - headers=headers, - ) - - if resp.status in [200, 206]: - if "content-location" in resp and resp["content-location"] != self._uri: - self._uri = resp["content-location"] - self._progress += len(content) - self._fd.write(content) - - if "content-range" in resp: - content_range = resp["content-range"] - length = content_range.rsplit("/", 1)[1] - self._total_size = int(length) - elif "content-length" in resp: - self._total_size = int(resp["content-length"]) - - if self._total_size is None or self._progress == self._total_size: - self._done = True - return MediaDownloadProgress(self._progress, self._total_size), self._done - elif resp.status == 416: - # 416 is Range Not Satisfiable - # This typically occurs with a zero byte file - content_range = resp["content-range"] - length = content_range.rsplit("/", 1)[1] - self._total_size = int(length) - if self._total_size == 0: - self._done = True - return ( - MediaDownloadProgress(self._progress, self._total_size), - self._done, - ) - raise HttpError(resp, content, uri=self._uri) - - -class _StreamSlice(object): - """Truncated stream. - - Takes a stream and presents a stream that is a slice of the original stream. - This is used when uploading media in chunks. In later versions of Python a - stream can be passed to httplib in place of the string of data to send. The - problem is that httplib just blindly reads to the end of the stream. This - wrapper presents a virtual stream that only reads to the end of the chunk. - """ - - def __init__(self, stream, begin, chunksize): - """Constructor. - - Args: - stream: (io.Base, file object), the stream to wrap. - begin: int, the seek position the chunk begins at. - chunksize: int, the size of the chunk. - """ - self._stream = stream - self._begin = begin - self._chunksize = chunksize - self._stream.seek(begin) - - def read(self, n=-1): - """Read n bytes. - - Args: - n, int, the number of bytes to read. - - Returns: - A string of length 'n', or less if EOF is reached. - """ - # The data left available to read sits in [cur, end) - cur = self._stream.tell() - end = self._begin + self._chunksize - if n == -1 or cur + n > end: - n = end - cur - return self._stream.read(n) - - -class HttpRequest(object): - """Encapsulates a single HTTP request.""" - - @util.positional(4) - def __init__( - self, - http, - postproc, - uri, - method="GET", - body=None, - headers=None, - methodId=None, - resumable=None, - ): - """Constructor for an HttpRequest. - - Args: - http: httplib2.Http, the transport object to use to make a request - postproc: callable, called on the HTTP response and content to transform - it into a data object before returning, or raising an exception - on an error. - uri: string, the absolute URI to send the request to - method: string, the HTTP method to use - body: string, the request body of the HTTP request, - headers: dict, the HTTP request headers - methodId: string, a unique identifier for the API method being called. - resumable: MediaUpload, None if this is not a resumbale request. - """ - self.uri = uri - self.method = method - self.body = body - self.headers = headers or {} - self.methodId = methodId - self.http = http - self.postproc = postproc - self.resumable = resumable - self.response_callbacks = [] - self._in_error_state = False - - # The size of the non-media part of the request. - self.body_size = len(self.body or "") - - # The resumable URI to send chunks to. - self.resumable_uri = None - - # The bytes that have been uploaded. - self.resumable_progress = 0 - - # Stubs for testing. - self._rand = random.random - self._sleep = time.sleep - - @util.positional(1) - def execute(self, http=None, num_retries=0): - """Execute the request. - - Args: - http: httplib2.Http, an http object to be used in place of the - one the HttpRequest request object was constructed with. - num_retries: Integer, number of times to retry with randomized - exponential backoff. If all retries fail, the raised HttpError - represents the last request. If zero (default), we attempt the - request only once. - - Returns: - A deserialized object model of the response body as determined - by the postproc. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx. - httplib2.HttpLib2Error if a transport error has occurred. - """ - if http is None: - http = self.http - - if self.resumable: - body = None - while body is None: - _, body = self.next_chunk(http=http, num_retries=num_retries) - return body - - # Non-resumable case. - - if "content-length" not in self.headers: - self.headers["content-length"] = str(self.body_size) - # If the request URI is too long then turn it into a POST request. - # Assume that a GET request never contains a request body. - if len(self.uri) > MAX_URI_LENGTH and self.method == "GET": - self.method = "POST" - self.headers["x-http-method-override"] = "GET" - self.headers["content-type"] = "application/x-www-form-urlencoded" - parsed = urllib.parse.urlparse(self.uri) - self.uri = urllib.parse.urlunparse( - (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, None) - ) - self.body = parsed.query - self.headers["content-length"] = str(len(self.body)) - - # Handle retries for server-side errors. - resp, content = _retry_request( - http, - num_retries, - "request", - self._sleep, - self._rand, - str(self.uri), - method=str(self.method), - body=self.body, - headers=self.headers, - ) - - for callback in self.response_callbacks: - callback(resp) - if resp.status >= 300: - raise HttpError(resp, content, uri=self.uri) - return self.postproc(resp, content) - - @util.positional(2) - def add_response_callback(self, cb): - """add_response_headers_callback - - Args: - cb: Callback to be called on receiving the response headers, of signature: - - def cb(resp): - # Where resp is an instance of httplib2.Response - """ - self.response_callbacks.append(cb) - - @util.positional(1) - def next_chunk(self, http=None, num_retries=0): - """Execute the next step of a resumable upload. - - Can only be used if the method being executed supports media uploads and - the MediaUpload object passed in was flagged as using resumable upload. - - Example: - - media = MediaFileUpload('cow.png', mimetype='image/png', - chunksize=1000, resumable=True) - request = farm.animals().insert( - id='cow', - name='cow.png', - media_body=media) - - response = None - while response is None: - status, response = request.next_chunk() - if status: - print "Upload %d%% complete." % int(status.progress() * 100) - - - Args: - http: httplib2.Http, an http object to be used in place of the - one the HttpRequest request object was constructed with. - num_retries: Integer, number of times to retry with randomized - exponential backoff. If all retries fail, the raised HttpError - represents the last request. If zero (default), we attempt the - request only once. - - Returns: - (status, body): (ResumableMediaStatus, object) - The body will be None until the resumable media is fully uploaded. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx. - httplib2.HttpLib2Error if a transport error has occurred. - """ - if http is None: - http = self.http - - if self.resumable.size() is None: - size = "*" - else: - size = str(self.resumable.size()) - - if self.resumable_uri is None: - start_headers = copy.copy(self.headers) - start_headers["X-Upload-Content-Type"] = self.resumable.mimetype() - if size != "*": - start_headers["X-Upload-Content-Length"] = size - start_headers["content-length"] = str(self.body_size) - - resp, content = _retry_request( - http, - num_retries, - "resumable URI request", - self._sleep, - self._rand, - self.uri, - method=self.method, - body=self.body, - headers=start_headers, - ) - - if resp.status == 200 and "location" in resp: - self.resumable_uri = resp["location"] - else: - raise ResumableUploadError(resp, content) - elif self._in_error_state: - # If we are in an error state then query the server for current state of - # the upload by sending an empty PUT and reading the 'range' header in - # the response. - headers = {"Content-Range": "bytes */%s" % size, "content-length": "0"} - resp, content = http.request(self.resumable_uri, "PUT", headers=headers) - status, body = self._process_response(resp, content) - if body: - # The upload was complete. - return (status, body) - - if self.resumable.has_stream(): - data = self.resumable.stream() - if self.resumable.chunksize() == -1: - data.seek(self.resumable_progress) - chunk_end = self.resumable.size() - self.resumable_progress - 1 - else: - # Doing chunking with a stream, so wrap a slice of the stream. - data = _StreamSlice( - data, self.resumable_progress, self.resumable.chunksize() - ) - chunk_end = min( - self.resumable_progress + self.resumable.chunksize() - 1, - self.resumable.size() - 1, - ) - else: - data = self.resumable.getbytes( - self.resumable_progress, self.resumable.chunksize() - ) - - # A short read implies that we are at EOF, so finish the upload. - if len(data) < self.resumable.chunksize(): - size = str(self.resumable_progress + len(data)) - - chunk_end = self.resumable_progress + len(data) - 1 - - headers = { - # Must set the content-length header here because httplib can't - # calculate the size when working with _StreamSlice. - "Content-Length": str(chunk_end - self.resumable_progress + 1), - } - - # An empty file results in chunk_end = -1 and size = 0 - # sending "bytes 0--1/0" results in an invalid request - # Only add header "Content-Range" if chunk_end != -1 - if chunk_end != -1: - headers["Content-Range"] = "bytes %d-%d/%s" % ( - self.resumable_progress, - chunk_end, - size, - ) - - for retry_num in range(num_retries + 1): - if retry_num > 0: - self._sleep(self._rand() * 2**retry_num) - LOGGER.warning( - "Retry #%d for media upload: %s %s, following status: %d" - % (retry_num, self.method, self.uri, resp.status) - ) - - try: - resp, content = http.request( - self.resumable_uri, method="PUT", body=data, headers=headers - ) - except: - self._in_error_state = True - raise - if not _should_retry_response(resp.status, content): - break - - return self._process_response(resp, content) - - def _process_response(self, resp, content): - """Process the response from a single chunk upload. - - Args: - resp: httplib2.Response, the response object. - content: string, the content of the response. - - Returns: - (status, body): (ResumableMediaStatus, object) - The body will be None until the resumable media is fully uploaded. - - Raises: - googleapiclient.errors.HttpError if the response was not a 2xx or a 308. - """ - if resp.status in [200, 201]: - self._in_error_state = False - return None, self.postproc(resp, content) - elif resp.status == 308: - self._in_error_state = False - # A "308 Resume Incomplete" indicates we are not done. - try: - self.resumable_progress = int(resp["range"].split("-")[1]) + 1 - except KeyError: - # If resp doesn't contain range header, resumable progress is 0 - self.resumable_progress = 0 - if "location" in resp: - self.resumable_uri = resp["location"] - else: - self._in_error_state = True - raise HttpError(resp, content, uri=self.uri) - - return ( - MediaUploadProgress(self.resumable_progress, self.resumable.size()), - None, - ) - - def to_json(self): - """Returns a JSON representation of the HttpRequest.""" - d = copy.copy(self.__dict__) - if d["resumable"] is not None: - d["resumable"] = self.resumable.to_json() - del d["http"] - del d["postproc"] - del d["_sleep"] - del d["_rand"] - - return json.dumps(d) - - @staticmethod - def from_json(s, http, postproc): - """Returns an HttpRequest populated with info from a JSON object.""" - d = json.loads(s) - if d["resumable"] is not None: - d["resumable"] = MediaUpload.new_from_json(d["resumable"]) - return HttpRequest( - http, - postproc, - uri=d["uri"], - method=d["method"], - body=d["body"], - headers=d["headers"], - methodId=d["methodId"], - resumable=d["resumable"], - ) - - @staticmethod - def null_postproc(resp, contents): - return resp, contents - - -class BatchHttpRequest(object): - """Batches multiple HttpRequest objects into a single HTTP request. - - Example: - from googleapiclient.http import BatchHttpRequest - - def list_animals(request_id, response, exception): - \"\"\"Do something with the animals list response.\"\"\" - if exception is not None: - # Do something with the exception. - pass - else: - # Do something with the response. - pass - - def list_farmers(request_id, response, exception): - \"\"\"Do something with the farmers list response.\"\"\" - if exception is not None: - # Do something with the exception. - pass - else: - # Do something with the response. - pass - - service = build('farm', 'v2') - - batch = BatchHttpRequest() - - batch.add(service.animals().list(), list_animals) - batch.add(service.farmers().list(), list_farmers) - batch.execute(http=http) - """ - - @util.positional(1) - def __init__(self, callback=None, batch_uri=None): - """Constructor for a BatchHttpRequest. - - Args: - callback: callable, A callback to be called for each response, of the - form callback(id, response, exception). The first parameter is the - request id, and the second is the deserialized response object. The - third is an googleapiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no error occurred. - batch_uri: string, URI to send batch requests to. - """ - if batch_uri is None: - batch_uri = _LEGACY_BATCH_URI - - if batch_uri == _LEGACY_BATCH_URI: - LOGGER.warning( - "You have constructed a BatchHttpRequest using the legacy batch " - "endpoint %s. This endpoint will be turned down on August 12, 2020. " - "Please provide the API-specific endpoint or use " - "service.new_batch_http_request(). For more details see " - "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" - "and https://developers.google.com/api-client-library/python/guide/batch.", - _LEGACY_BATCH_URI, - ) - self._batch_uri = batch_uri - - # Global callback to be called for each individual response in the batch. - self._callback = callback - - # A map from id to request. - self._requests = {} - - # A map from id to callback. - self._callbacks = {} - - # List of request ids, in the order in which they were added. - self._order = [] - - # The last auto generated id. - self._last_auto_id = 0 - - # Unique ID on which to base the Content-ID headers. - self._base_id = None - - # A map from request id to (httplib2.Response, content) response pairs - self._responses = {} - - # A map of id(Credentials) that have been refreshed. - self._refreshed_credentials = {} - - def _refresh_and_apply_credentials(self, request, http): - """Refresh the credentials and apply to the request. - - Args: - request: HttpRequest, the request. - http: httplib2.Http, the global http object for the batch. - """ - # For the credentials to refresh, but only once per refresh_token - # If there is no http per the request then refresh the http passed in - # via execute() - creds = None - request_credentials = False - - if request.http is not None: - creds = _auth.get_credentials_from_http(request.http) - request_credentials = True - - if creds is None and http is not None: - creds = _auth.get_credentials_from_http(http) - - if creds is not None: - if id(creds) not in self._refreshed_credentials: - _auth.refresh_credentials(creds) - self._refreshed_credentials[id(creds)] = 1 - - # Only apply the credentials if we are using the http object passed in, - # otherwise apply() will get called during _serialize_request(). - if request.http is None or not request_credentials: - _auth.apply_credentials(creds, request.headers) - - def _id_to_header(self, id_): - """Convert an id to a Content-ID header value. - - Args: - id_: string, identifier of individual request. - - Returns: - A Content-ID header with the id_ encoded into it. A UUID is prepended to - the value because Content-ID headers are supposed to be universally - unique. - """ - if self._base_id is None: - self._base_id = uuid.uuid4() - - # NB: we intentionally leave whitespace between base/id and '+', so RFC2822 - # line folding works properly on Python 3; see - # https://github.com/googleapis/google-api-python-client/issues/164 - return "<%s + %s>" % (self._base_id, urllib.parse.quote(id_)) - - def _header_to_id(self, header): - """Convert a Content-ID header value to an id. - - Presumes the Content-ID header conforms to the format that _id_to_header() - returns. - - Args: - header: string, Content-ID header value. - - Returns: - The extracted id value. - - Raises: - BatchError if the header is not in the expected format. - """ - if header[0] != "<" or header[-1] != ">": - raise BatchError("Invalid value for Content-ID: %s" % header) - if "+" not in header: - raise BatchError("Invalid value for Content-ID: %s" % header) - base, id_ = header[1:-1].split(" + ", 1) - - return urllib.parse.unquote(id_) - - def _serialize_request(self, request): - """Convert an HttpRequest object into a string. - - Args: - request: HttpRequest, the request to serialize. - - Returns: - The request as a string in application/http format. - """ - # Construct status line - parsed = urllib.parse.urlparse(request.uri) - request_line = urllib.parse.urlunparse( - ("", "", parsed.path, parsed.params, parsed.query, "") - ) - status_line = request.method + " " + request_line + " HTTP/1.1\n" - major, minor = request.headers.get("content-type", "application/json").split( - "/" - ) - msg = MIMENonMultipart(major, minor) - headers = request.headers.copy() - - if request.http is not None: - credentials = _auth.get_credentials_from_http(request.http) - if credentials is not None: - _auth.apply_credentials(credentials, headers) - - # MIMENonMultipart adds its own Content-Type header. - if "content-type" in headers: - del headers["content-type"] - - for key, value in headers.items(): - msg[key] = value - msg["Host"] = parsed.netloc - msg.set_unixfrom(None) - - if request.body is not None: - msg.set_payload(request.body) - msg["content-length"] = str(len(request.body)) - - # Serialize the mime message. - fp = io.StringIO() - # maxheaderlen=0 means don't line wrap headers. - g = Generator(fp, maxheaderlen=0) - g.flatten(msg, unixfrom=False) - body = fp.getvalue() - - return status_line + body - - def _deserialize_response(self, payload): - """Convert string into httplib2 response and content. - - Args: - payload: string, headers and body as a string. - - Returns: - A pair (resp, content), such as would be returned from httplib2.request. - """ - # Strip off the status line - status_line, payload = payload.split("\n", 1) - protocol, status, reason = status_line.split(" ", 2) - - # Parse the rest of the response - parser = FeedParser() - parser.feed(payload) - msg = parser.close() - msg["status"] = status - - # Create httplib2.Response from the parsed headers. - resp = httplib2.Response(msg) - resp.reason = reason - resp.version = int(protocol.split("/", 1)[1].replace(".", "")) - - content = payload.split("\r\n\r\n", 1)[1] - - return resp, content - - def _new_id(self): - """Create a new id. - - Auto incrementing number that avoids conflicts with ids already used. - - Returns: - string, a new unique id. - """ - self._last_auto_id += 1 - while str(self._last_auto_id) in self._requests: - self._last_auto_id += 1 - return str(self._last_auto_id) - - @util.positional(2) - def add(self, request, callback=None, request_id=None): - """Add a new request. - - Every callback added will be paired with a unique id, the request_id. That - unique id will be passed back to the callback when the response comes back - from the server. The default behavior is to have the library generate it's - own unique id. If the caller passes in a request_id then they must ensure - uniqueness for each request_id, and if they are not an exception is - raised. Callers should either supply all request_ids or never supply a - request id, to avoid such an error. - - Args: - request: HttpRequest, Request to add to the batch. - callback: callable, A callback to be called for this response, of the - form callback(id, response, exception). The first parameter is the - request id, and the second is the deserialized response object. The - third is an googleapiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no errors occurred. - request_id: string, A unique id for the request. The id will be passed - to the callback with the response. - - Returns: - None - - Raises: - BatchError if a media request is added to a batch. - KeyError is the request_id is not unique. - """ - - if len(self._order) >= MAX_BATCH_LIMIT: - raise BatchError( - "Exceeded the maximum calls(%d) in a single batch request." - % MAX_BATCH_LIMIT - ) - if request_id is None: - request_id = self._new_id() - if request.resumable is not None: - raise BatchError("Media requests cannot be used in a batch request.") - if request_id in self._requests: - raise KeyError("A request with this ID already exists: %s" % request_id) - self._requests[request_id] = request - self._callbacks[request_id] = callback - self._order.append(request_id) - - def _execute(self, http, order, requests): - """Serialize batch request, send to server, process response. - - Args: - http: httplib2.Http, an http object to be used to make the request with. - order: list, list of request ids in the order they were added to the - batch. - requests: list, list of request objects to send. - - Raises: - httplib2.HttpLib2Error if a transport error has occurred. - googleapiclient.errors.BatchError if the response is the wrong format. - """ - message = MIMEMultipart("mixed") - # Message should not write out it's own headers. - setattr(message, "_write_headers", lambda self: None) - - # Add all the individual requests. - for request_id in order: - request = requests[request_id] - - msg = MIMENonMultipart("application", "http") - msg["Content-Transfer-Encoding"] = "binary" - msg["Content-ID"] = self._id_to_header(request_id) - - body = self._serialize_request(request) - msg.set_payload(body) - message.attach(msg) - - # encode the body: note that we can't use `as_string`, because - # it plays games with `From ` lines. - fp = io.StringIO() - g = Generator(fp, mangle_from_=False) - g.flatten(message, unixfrom=False) - body = fp.getvalue() - - headers = {} - headers["content-type"] = ( - "multipart/mixed; " 'boundary="%s"' - ) % message.get_boundary() - - resp, content = http.request( - self._batch_uri, method="POST", body=body, headers=headers - ) - - if resp.status >= 300: - raise HttpError(resp, content, uri=self._batch_uri) - - # Prepend with a content-type header so FeedParser can handle it. - header = "content-type: %s\r\n\r\n" % resp["content-type"] - # PY3's FeedParser only accepts unicode. So we should decode content - # here, and encode each payload again. - content = content.decode("utf-8") - for_parser = header + content - - parser = FeedParser() - parser.feed(for_parser) - mime_response = parser.close() - - if not mime_response.is_multipart(): - raise BatchError( - "Response not in multipart/mixed format.", resp=resp, content=content - ) - - for part in mime_response.get_payload(): - request_id = self._header_to_id(part["Content-ID"]) - response, content = self._deserialize_response(part.get_payload()) - # We encode content here to emulate normal http response. - if isinstance(content, str): - content = content.encode("utf-8") - self._responses[request_id] = (response, content) - - @util.positional(1) - def execute(self, http=None): - """Execute all the requests as a single batched HTTP request. - - Args: - http: httplib2.Http, an http object to be used in place of the one the - HttpRequest request object was constructed with. If one isn't supplied - then use a http object from the requests in this batch. - - Returns: - None - - Raises: - httplib2.HttpLib2Error if a transport error has occurred. - googleapiclient.errors.BatchError if the response is the wrong format. - """ - # If we have no requests return - if len(self._order) == 0: - return None - - # If http is not supplied use the first valid one given in the requests. - if http is None: - for request_id in self._order: - request = self._requests[request_id] - if request is not None: - http = request.http - break - - if http is None: - raise ValueError("Missing a valid http object.") - - # Special case for OAuth2Credentials-style objects which have not yet been - # refreshed with an initial access_token. - creds = _auth.get_credentials_from_http(http) - if creds is not None: - if not _auth.is_valid(creds): - LOGGER.info("Attempting refresh to obtain initial access_token") - _auth.refresh_credentials(creds) - - self._execute(http, self._order, self._requests) - - # Loop over all the requests and check for 401s. For each 401 request the - # credentials should be refreshed and then sent again in a separate batch. - redo_requests = {} - redo_order = [] - - for request_id in self._order: - resp, content = self._responses[request_id] - if resp["status"] == "401": - redo_order.append(request_id) - request = self._requests[request_id] - self._refresh_and_apply_credentials(request, http) - redo_requests[request_id] = request - - if redo_requests: - self._execute(http, redo_order, redo_requests) - - # Now process all callbacks that are erroring, and raise an exception for - # ones that return a non-2xx response? Or add extra parameter to callback - # that contains an HttpError? - - for request_id in self._order: - resp, content = self._responses[request_id] - - request = self._requests[request_id] - callback = self._callbacks[request_id] - - response = None - exception = None - try: - if resp.status >= 300: - raise HttpError(resp, content, uri=request.uri) - response = request.postproc(resp, content) - except HttpError as e: - exception = e - - if callback is not None: - callback(request_id, response, exception) - if self._callback is not None: - self._callback(request_id, response, exception) - - -class HttpRequestMock(object): - """Mock of HttpRequest. - - Do not construct directly, instead use RequestMockBuilder. - """ - - def __init__(self, resp, content, postproc): - """Constructor for HttpRequestMock - - Args: - resp: httplib2.Response, the response to emulate coming from the request - content: string, the response body - postproc: callable, the post processing function usually supplied by - the model class. See model.JsonModel.response() as an example. - """ - self.resp = resp - self.content = content - self.postproc = postproc - if resp is None: - self.resp = httplib2.Response({"status": 200, "reason": "OK"}) - if "reason" in self.resp: - self.resp.reason = self.resp["reason"] - - def execute(self, http=None): - """Execute the request. - - Same behavior as HttpRequest.execute(), but the response is - mocked and not really from an HTTP request/response. - """ - return self.postproc(self.resp, self.content) - - -class RequestMockBuilder(object): - """A simple mock of HttpRequest - - Pass in a dictionary to the constructor that maps request methodIds to - tuples of (httplib2.Response, content, opt_expected_body) that should be - returned when that method is called. None may also be passed in for the - httplib2.Response, in which case a 200 OK response will be generated. - If an opt_expected_body (str or dict) is provided, it will be compared to - the body and UnexpectedBodyError will be raised on inequality. - - Example: - response = '{"data": {"id": "tag:google.c...' - requestBuilder = RequestMockBuilder( - { - 'plus.activities.get': (None, response), - } - ) - googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) - - Methods that you do not supply a response for will return a - 200 OK with an empty string as the response content or raise an excpetion - if check_unexpected is set to True. The methodId is taken from the rpcName - in the discovery document. - - For more details see the project wiki. - """ - - def __init__(self, responses, check_unexpected=False): - """Constructor for RequestMockBuilder - - The constructed object should be a callable object - that can replace the class HttpResponse. - - responses - A dictionary that maps methodIds into tuples - of (httplib2.Response, content). The methodId - comes from the 'rpcName' field in the discovery - document. - check_unexpected - A boolean setting whether or not UnexpectedMethodError - should be raised on unsupplied method. - """ - self.responses = responses - self.check_unexpected = check_unexpected - - def __call__( - self, - http, - postproc, - uri, - method="GET", - body=None, - headers=None, - methodId=None, - resumable=None, - ): - """Implements the callable interface that discovery.build() expects - of requestBuilder, which is to build an object compatible with - HttpRequest.execute(). See that method for the description of the - parameters and the expected response. - """ - if methodId in self.responses: - response = self.responses[methodId] - resp, content = response[:2] - if len(response) > 2: - # Test the body against the supplied expected_body. - expected_body = response[2] - if bool(expected_body) != bool(body): - # Not expecting a body and provided one - # or expecting a body and not provided one. - raise UnexpectedBodyError(expected_body, body) - if isinstance(expected_body, str): - expected_body = json.loads(expected_body) - body = json.loads(body) - if body != expected_body: - raise UnexpectedBodyError(expected_body, body) - return HttpRequestMock(resp, content, postproc) - elif self.check_unexpected: - raise UnexpectedMethodError(methodId=methodId) - else: - model = JsonModel(False) - return HttpRequestMock(None, "{}", model.response) - - -class HttpMock(object): - """Mock of httplib2.Http""" - - def __init__(self, filename=None, headers=None): - """ - Args: - filename: string, absolute filename to read response from - headers: dict, header to return with response - """ - if headers is None: - headers = {"status": "200"} - if filename: - with open(filename, "rb") as f: - self.data = f.read() - else: - self.data = None - self.response_headers = headers - self.headers = None - self.uri = None - self.method = None - self.body = None - self.headers = None - - def request( - self, - uri, - method="GET", - body=None, - headers=None, - redirections=1, - connection_type=None, - ): - self.uri = uri - self.method = method - self.body = body - self.headers = headers - return httplib2.Response(self.response_headers), self.data - - def close(self): - return None - - -class HttpMockSequence(object): - """Mock of httplib2.Http - - Mocks a sequence of calls to request returning different responses for each - call. Create an instance initialized with the desired response headers - and content and then use as if an httplib2.Http instance. - - http = HttpMockSequence([ - ({'status': '401'}, ''), - ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), - ({'status': '200'}, 'echo_request_headers'), - ]) - resp, content = http.request("http://examples.com") - - There are special values you can pass in for content to trigger - behavours that are helpful in testing. - - 'echo_request_headers' means return the request headers in the response body - 'echo_request_headers_as_json' means return the request headers in - the response body - 'echo_request_body' means return the request body in the response body - 'echo_request_uri' means return the request uri in the response body - """ - - def __init__(self, iterable): - """ - Args: - iterable: iterable, a sequence of pairs of (headers, body) - """ - self._iterable = iterable - self.follow_redirects = True - self.request_sequence = list() - - def request( - self, - uri, - method="GET", - body=None, - headers=None, - redirections=1, - connection_type=None, - ): - # Remember the request so after the fact this mock can be examined - self.request_sequence.append((uri, method, body, headers)) - resp, content = self._iterable.pop(0) - if isinstance(content, str): - content = content.encode("utf-8") - - if content == b"echo_request_headers": - content = headers - elif content == b"echo_request_headers_as_json": - content = json.dumps(headers) - elif content == b"echo_request_body": - if hasattr(body, "read"): - content = body.read() - else: - content = body - elif content == b"echo_request_uri": - content = uri - if isinstance(content, str): - content = content.encode("utf-8") - return httplib2.Response(resp), content - - -def set_user_agent(http, user_agent): - """Set the user-agent on every request. - - Args: - http - An instance of httplib2.Http - or something that acts like it. - user_agent: string, the value for the user-agent header. - - Returns: - A modified instance of http that was passed in. - - Example: - - h = httplib2.Http() - h = set_user_agent(h, "my-app-name/6.0") - - Most of the time the user-agent will be set doing auth, this is for the rare - cases where you are accessing an unauthenticated endpoint. - """ - request_orig = http.request - - # The closure that will replace 'httplib2.Http.request'. - def new_request( - uri, - method="GET", - body=None, - headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None, - ): - """Modify the request headers to add the user-agent.""" - if headers is None: - headers = {} - if "user-agent" in headers: - headers["user-agent"] = user_agent + " " + headers["user-agent"] - else: - headers["user-agent"] = user_agent - resp, content = request_orig( - uri, - method=method, - body=body, - headers=headers, - redirections=redirections, - connection_type=connection_type, - ) - return resp, content - - http.request = new_request - return http - - -def tunnel_patch(http): - """Tunnel PATCH requests over POST. - Args: - http - An instance of httplib2.Http - or something that acts like it. - - Returns: - A modified instance of http that was passed in. - - Example: - - h = httplib2.Http() - h = tunnel_patch(h, "my-app-name/6.0") - - Useful if you are running on a platform that doesn't support PATCH. - Apply this last if you are using OAuth 1.0, as changing the method - will result in a different signature. - """ - request_orig = http.request - - # The closure that will replace 'httplib2.Http.request'. - def new_request( - uri, - method="GET", - body=None, - headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, - connection_type=None, - ): - """Modify the request headers to add the user-agent.""" - if headers is None: - headers = {} - if method == "PATCH": - if "oauth_token" in headers.get("authorization", ""): - LOGGER.warning( - "OAuth 1.0 request made with Credentials after tunnel_patch." - ) - headers["x-http-method-override"] = "PATCH" - method = "POST" - resp, content = request_orig( - uri, - method=method, - body=body, - headers=headers, - redirections=redirections, - connection_type=connection_type, - ) - return resp, content - - http.request = new_request - return http - - -def build_http(): - """Builds httplib2.Http object - - Returns: - A httplib2.Http object, which is used to make http requests, and which has timeout set by default. - To override default timeout call - - socket.setdefaulttimeout(timeout_in_sec) - - before interacting with this method. - """ - if socket.getdefaulttimeout() is not None: - http_timeout = socket.getdefaulttimeout() - else: - http_timeout = DEFAULT_HTTP_TIMEOUT_SEC - http = httplib2.Http(timeout=http_timeout) - # 308's are used by several Google APIs (Drive, YouTube) - # for Resumable Uploads rather than Permanent Redirects. - # This asks httplib2 to exclude 308s from the status codes - # it treats as redirects - try: - http.redirect_codes = http.redirect_codes - {308} - except AttributeError: - # Apache Beam tests depend on this library and cannot - # currently upgrade their httplib2 version - # http.redirect_codes does not exist in previous versions - # of httplib2, so pass - pass - - return http diff --git a/src/gam/googleapiclient.shouldnotbehere/mimeparse.py b/src/gam/googleapiclient.shouldnotbehere/mimeparse.py deleted file mode 100644 index d3dedee9..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/mimeparse.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2014 Joe Gregorio -# -# Licensed under the MIT License - -"""MIME-Type Parser - -This module provides basic functions for handling mime-types. It can handle -matching mime-types against a list of media-ranges. See section 14.1 of the -HTTP specification [RFC 2616] for a complete explanation. - - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 - -Contents: - - parse_mime_type(): Parses a mime-type into its component parts. - - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q' - quality parameter. - - quality(): Determines the quality ('q') of a mime-type when - compared against a list of media-ranges. - - quality_parsed(): Just like quality() except the second parameter must be - pre-parsed. - - best_match(): Choose the mime-type with the highest quality ('q') - from a list of candidates. -""" -from __future__ import absolute_import - -from functools import reduce - -__version__ = "0.1.3" -__author__ = "Joe Gregorio" -__email__ = "joe@bitworking.org" -__license__ = "MIT License" -__credits__ = "" - - -def parse_mime_type(mime_type): - """Parses a mime-type into its component parts. - - Carves up a mime-type and returns a tuple of the (type, subtype, params) - where 'params' is a dictionary of all the parameters for the media range. - For example, the media range 'application/xhtml;q=0.5' would get parsed - into: - - ('application', 'xhtml', {'q', '0.5'}) - """ - parts = mime_type.split(";") - params = dict( - [tuple([s.strip() for s in param.split("=", 1)]) for param in parts[1:]] - ) - full_type = parts[0].strip() - # Java URLConnection class sends an Accept header that includes a - # single '*'. Turn it into a legal wildcard. - if full_type == "*": - full_type = "*/*" - (type, subtype) = full_type.split("/") - - return (type.strip(), subtype.strip(), params) - - -def parse_media_range(range): - """Parse a media-range into its component parts. - - Carves up a media range and returns a tuple of the (type, subtype, - params) where 'params' is a dictionary of all the parameters for the media - range. For example, the media range 'application/*;q=0.5' would get parsed - into: - - ('application', '*', {'q', '0.5'}) - - In addition this function also guarantees that there is a value for 'q' - in the params dictionary, filling it in with a proper default if - necessary. - """ - (type, subtype, params) = parse_mime_type(range) - if ( - "q" not in params - or not params["q"] - or not float(params["q"]) - or float(params["q"]) > 1 - or float(params["q"]) < 0 - ): - params["q"] = "1" - - return (type, subtype, params) - - -def fitness_and_quality_parsed(mime_type, parsed_ranges): - """Find the best match for a mime-type amongst parsed media-ranges. - - Find the best match for a given mime-type against a list of media_ranges - that have already been parsed by parse_media_range(). Returns a tuple of - the fitness value and the value of the 'q' quality parameter of the best - match, or (-1, 0) if no match was found. Just as for quality_parsed(), - 'parsed_ranges' must be a list of parsed media ranges. - """ - best_fitness = -1 - best_fit_q = 0 - (target_type, target_subtype, target_params) = parse_media_range(mime_type) - for (type, subtype, params) in parsed_ranges: - type_match = type == target_type or type == "*" or target_type == "*" - subtype_match = ( - subtype == target_subtype or subtype == "*" or target_subtype == "*" - ) - if type_match and subtype_match: - param_matches = reduce( - lambda x, y: x + y, - [ - 1 - for (key, value) in target_params.items() - if key != "q" and key in params and value == params[key] - ], - 0, - ) - fitness = (type == target_type) and 100 or 0 - fitness += (subtype == target_subtype) and 10 or 0 - fitness += param_matches - if fitness > best_fitness: - best_fitness = fitness - best_fit_q = params["q"] - - return best_fitness, float(best_fit_q) - - -def quality_parsed(mime_type, parsed_ranges): - """Find the best match for a mime-type amongst parsed media-ranges. - - Find the best match for a given mime-type against a list of media_ranges - that have already been parsed by parse_media_range(). Returns the 'q' - quality parameter of the best match, 0 if no match was found. This function - bahaves the same as quality() except that 'parsed_ranges' must be a list of - parsed media ranges. - """ - - return fitness_and_quality_parsed(mime_type, parsed_ranges)[1] - - -def quality(mime_type, ranges): - """Return the quality ('q') of a mime-type against a list of media-ranges. - - Returns the quality 'q' of a mime-type when compared against the - media-ranges in ranges. For example: - - >>> quality('text/html','text/*;q=0.3, text/html;q=0.7, - text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5') - 0.7 - - """ - parsed_ranges = [parse_media_range(r) for r in ranges.split(",")] - - return quality_parsed(mime_type, parsed_ranges) - - -def best_match(supported, header): - """Return mime-type with the highest quality ('q') from list of candidates. - - Takes a list of supported mime-types and finds the best match for all the - media-ranges listed in header. The value of header must be a string that - conforms to the format of the HTTP Accept: header. The value of 'supported' - is a list of mime-types. The list of supported mime-types should be sorted - in order of increasing desirability, in case of a situation where there is - a tie. - - >>> best_match(['application/xbel+xml', 'text/xml'], - 'text/*;q=0.5,*/*; q=0.1') - 'text/xml' - """ - split_header = _filter_blank(header.split(",")) - parsed_header = [parse_media_range(r) for r in split_header] - weighted_matches = [] - pos = 0 - for mime_type in supported: - weighted_matches.append( - (fitness_and_quality_parsed(mime_type, parsed_header), pos, mime_type) - ) - pos += 1 - weighted_matches.sort() - - return weighted_matches[-1][0][1] and weighted_matches[-1][2] or "" - - -def _filter_blank(i): - for s in i: - if s.strip(): - yield s diff --git a/src/gam/googleapiclient.shouldnotbehere/model.py b/src/gam/googleapiclient.shouldnotbehere/model.py deleted file mode 100644 index dee1cb7f..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/model.py +++ /dev/null @@ -1,429 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Model objects for requests and responses. - -Each API may support one or more serializations, such -as JSON, Atom, etc. The model classes are responsible -for converting between the wire format and the Python -object representation. -""" -from __future__ import absolute_import - -__author__ = "jcgregorio@google.com (Joe Gregorio)" - -import json -import logging -import platform -import urllib -import warnings - -from googleapiclient import version as googleapiclient_version -from googleapiclient.errors import HttpError - -try: - from google.api_core.version_header import API_VERSION_METADATA_KEY - - HAS_API_VERSION = True -except ImportError: - HAS_API_VERSION = False - -_LIBRARY_VERSION = googleapiclient_version.__version__ -_PY_VERSION = platform.python_version() - -LOGGER = logging.getLogger(__name__) - -dump_request_response = False - - -def _abstract(): - raise NotImplementedError("You need to override this function") - - -class Model(object): - """Model base class. - - All Model classes should implement this interface. - The Model serializes and de-serializes between a wire - format such as JSON and a Python object representation. - """ - - def request(self, headers, path_params, query_params, body_value): - """Updates outgoing requests with a serialized body. - - Args: - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query_params: dict, parameters that appear in the query - body_value: object, the request body as a Python object, which must be - serializable. - Returns: - A tuple of (headers, path_params, query, body) - - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query: string, query part of the request URI - body: string, the body serialized in the desired wire format. - """ - _abstract() - - def response(self, resp, content): - """Convert the response wire format into a Python object. - - Args: - resp: httplib2.Response, the HTTP response headers and status - content: string, the body of the HTTP response - - Returns: - The body de-serialized as a Python object. - - Raises: - googleapiclient.errors.HttpError if a non 2xx response is received. - """ - _abstract() - - -class BaseModel(Model): - """Base model class. - - Subclasses should provide implementations for the "serialize" and - "deserialize" methods, as well as values for the following class attributes. - - Attributes: - accept: The value to use for the HTTP Accept header. - content_type: The value to use for the HTTP Content-type header. - no_content_response: The value to return when deserializing a 204 "No - Content" response. - alt_param: The value to supply as the "alt" query parameter for requests. - """ - - accept = None - content_type = None - no_content_response = None - alt_param = None - - def _log_request(self, headers, path_params, query, body): - """Logs debugging information about the request if requested.""" - if dump_request_response: - LOGGER.info("--request-start--") - LOGGER.info("-headers-start-") - for h, v in headers.items(): - LOGGER.info("%s: %s", h, v) - LOGGER.info("-headers-end-") - LOGGER.info("-path-parameters-start-") - for h, v in path_params.items(): - LOGGER.info("%s: %s", h, v) - LOGGER.info("-path-parameters-end-") - LOGGER.info("body: %s", body) - LOGGER.info("query: %s", query) - LOGGER.info("--request-end--") - - def request(self, headers, path_params, query_params, body_value, api_version=None): - """Updates outgoing requests with a serialized body. - - Args: - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query_params: dict, parameters that appear in the query - body_value: object, the request body as a Python object, which must be - serializable by json. - api_version: str, The precise API version represented by this request, - which will result in an API Version header being sent along with the - HTTP request. - Returns: - A tuple of (headers, path_params, query, body) - - headers: dict, request headers - path_params: dict, parameters that appear in the request path - query: string, query part of the request URI - body: string, the body serialized as JSON - """ - query = self._build_query(query_params) - headers["accept"] = self.accept - headers["accept-encoding"] = "gzip, deflate" - if "user-agent" in headers: - headers["user-agent"] += " " - else: - headers["user-agent"] = "" - headers["user-agent"] += "(gzip)" - if "x-goog-api-client" in headers: - headers["x-goog-api-client"] += " " - else: - headers["x-goog-api-client"] = "" - headers["x-goog-api-client"] += "gdcl/%s gl-python/%s" % ( - _LIBRARY_VERSION, - _PY_VERSION, - ) - - if api_version and HAS_API_VERSION: - headers[API_VERSION_METADATA_KEY] = api_version - elif api_version: - warnings.warn( - "The `api_version` argument is ignored as a newer version of " - "`google-api-core` is required to use this feature." - "Please upgrade `google-api-core` to 2.19.0 or newer." - ) - - if body_value is not None: - headers["content-type"] = self.content_type - body_value = self.serialize(body_value) - self._log_request(headers, path_params, query, body_value) - return (headers, path_params, query, body_value) - - def _build_query(self, params): - """Builds a query string. - - Args: - params: dict, the query parameters - - Returns: - The query parameters properly encoded into an HTTP URI query string. - """ - if self.alt_param is not None: - params.update({"alt": self.alt_param}) - astuples = [] - for key, value in params.items(): - if type(value) == type([]): - for x in value: - x = x.encode("utf-8") - astuples.append((key, x)) - else: - if isinstance(value, str) and callable(value.encode): - value = value.encode("utf-8") - astuples.append((key, value)) - return "?" + urllib.parse.urlencode(astuples) - - def _log_response(self, resp, content): - """Logs debugging information about the response if requested.""" - if dump_request_response: - LOGGER.info("--response-start--") - for h, v in resp.items(): - LOGGER.info("%s: %s", h, v) - if content: - LOGGER.info(content) - LOGGER.info("--response-end--") - - def response(self, resp, content): - """Convert the response wire format into a Python object. - - Args: - resp: httplib2.Response, the HTTP response headers and status - content: string, the body of the HTTP response - - Returns: - The body de-serialized as a Python object. - - Raises: - googleapiclient.errors.HttpError if a non 2xx response is received. - """ - self._log_response(resp, content) - # Error handling is TBD, for example, do we retry - # for some operation/error combinations? - if resp.status < 300: - if resp.status == 204: - # A 204: No Content response should be treated differently - # to all the other success states - return self.no_content_response - return self.deserialize(content) - else: - LOGGER.debug("Content from bad request was: %r" % content) - raise HttpError(resp, content) - - def serialize(self, body_value): - """Perform the actual Python object serialization. - - Args: - body_value: object, the request body as a Python object. - - Returns: - string, the body in serialized form. - """ - _abstract() - - def deserialize(self, content): - """Perform the actual deserialization from response string to Python - object. - - Args: - content: string, the body of the HTTP response - - Returns: - The body de-serialized as a Python object. - """ - _abstract() - - -class JsonModel(BaseModel): - """Model class for JSON. - - Serializes and de-serializes between JSON and the Python - object representation of HTTP request and response bodies. - """ - - accept = "application/json" - content_type = "application/json" - alt_param = "json" - - def __init__(self, data_wrapper=False): - """Construct a JsonModel. - - Args: - data_wrapper: boolean, wrap requests and responses in a data wrapper - """ - self._data_wrapper = data_wrapper - - def serialize(self, body_value): - if ( - isinstance(body_value, dict) - and "data" not in body_value - and self._data_wrapper - ): - body_value = {"data": body_value} - return json.dumps(body_value) - - def deserialize(self, content): - try: - content = content.decode("utf-8") - except AttributeError: - pass - try: - body = json.loads(content) - except json.decoder.JSONDecodeError: - body = content - else: - if self._data_wrapper and "data" in body: - body = body["data"] - return body - - @property - def no_content_response(self): - return {} - - -class RawModel(JsonModel): - """Model class for requests that don't return JSON. - - Serializes and de-serializes between JSON and the Python - object representation of HTTP request, and returns the raw bytes - of the response body. - """ - - accept = "*/*" - content_type = "application/json" - alt_param = None - - def deserialize(self, content): - return content - - @property - def no_content_response(self): - return "" - - -class MediaModel(JsonModel): - """Model class for requests that return Media. - - Serializes and de-serializes between JSON and the Python - object representation of HTTP request, and returns the raw bytes - of the response body. - """ - - accept = "*/*" - content_type = "application/json" - alt_param = "media" - - def deserialize(self, content): - return content - - @property - def no_content_response(self): - return "" - - -class ProtocolBufferModel(BaseModel): - """Model class for protocol buffers. - - Serializes and de-serializes the binary protocol buffer sent in the HTTP - request and response bodies. - """ - - accept = "application/x-protobuf" - content_type = "application/x-protobuf" - alt_param = "proto" - - def __init__(self, protocol_buffer): - """Constructs a ProtocolBufferModel. - - The serialized protocol buffer returned in an HTTP response will be - de-serialized using the given protocol buffer class. - - Args: - protocol_buffer: The protocol buffer class used to de-serialize a - response from the API. - """ - self._protocol_buffer = protocol_buffer - - def serialize(self, body_value): - return body_value.SerializeToString() - - def deserialize(self, content): - return self._protocol_buffer.FromString(content) - - @property - def no_content_response(self): - return self._protocol_buffer() - - -def makepatch(original, modified): - """Create a patch object. - - Some methods support PATCH, an efficient way to send updates to a resource. - This method allows the easy construction of patch bodies by looking at the - differences between a resource before and after it was modified. - - Args: - original: object, the original deserialized resource - modified: object, the modified deserialized resource - Returns: - An object that contains only the changes from original to modified, in a - form suitable to pass to a PATCH method. - - Example usage: - item = service.activities().get(postid=postid, userid=userid).execute() - original = copy.deepcopy(item) - item['object']['content'] = 'This is updated.' - service.activities.patch(postid=postid, userid=userid, - body=makepatch(original, item)).execute() - """ - patch = {} - for key, original_value in original.items(): - modified_value = modified.get(key, None) - if modified_value is None: - # Use None to signal that the element is deleted - patch[key] = None - elif original_value != modified_value: - if type(original_value) == type({}): - # Recursively descend objects - patch[key] = makepatch(original_value, modified_value) - else: - # In the case of simple types or arrays we just replace - patch[key] = modified_value - else: - # Don't add anything to patch if there's no change - pass - for key in modified: - if key not in original: - patch[key] = modified[key] - - return patch diff --git a/src/gam/googleapiclient.shouldnotbehere/schema.py b/src/gam/googleapiclient.shouldnotbehere/schema.py deleted file mode 100644 index 93b07df4..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/schema.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Schema processing for discovery based APIs - -Schemas holds an APIs discovery schemas. It can return those schema as -deserialized JSON objects, or pretty print them as prototype objects that -conform to the schema. - -For example, given the schema: - - schema = \"\"\"{ - "Foo": { - "type": "object", - "properties": { - "etag": { - "type": "string", - "description": "ETag of the collection." - }, - "kind": { - "type": "string", - "description": "Type of the collection ('calendar#acl').", - "default": "calendar#acl" - }, - "nextPageToken": { - "type": "string", - "description": "Token used to access the next - page of this result. Omitted if no further results are available." - } - } - } - }\"\"\" - - s = Schemas(schema) - print s.prettyPrintByName('Foo') - - Produces the following output: - - { - "nextPageToken": "A String", # Token used to access the - # next page of this result. Omitted if no further results are available. - "kind": "A String", # Type of the collection ('calendar#acl'). - "etag": "A String", # ETag of the collection. - }, - -The constructor takes a discovery document in which to look up named schema. -""" -from __future__ import absolute_import - -# TODO(jcgregorio) support format, enum, minimum, maximum - -__author__ = "jcgregorio@google.com (Joe Gregorio)" - - -from collections import OrderedDict - -from googleapiclient import _helpers as util - - -class Schemas(object): - """Schemas for an API.""" - - def __init__(self, discovery): - """Constructor. - - Args: - discovery: object, Deserialized discovery document from which we pull - out the named schema. - """ - self.schemas = discovery.get("schemas", {}) - - # Cache of pretty printed schemas. - self.pretty = {} - - @util.positional(2) - def _prettyPrintByName(self, name, seen=None, dent=0): - """Get pretty printed object prototype from the schema name. - - Args: - name: string, Name of schema in the discovery document. - seen: list of string, Names of schema already seen. Used to handle - recursive definitions. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - if seen is None: - seen = [] - - if name in seen: - # Do not fall into an infinite loop over recursive definitions. - return "# Object with schema name: %s" % name - seen.append(name) - - if name not in self.pretty: - self.pretty[name] = _SchemaToStruct( - self.schemas[name], seen, dent=dent - ).to_str(self._prettyPrintByName) - - seen.pop() - - return self.pretty[name] - - def prettyPrintByName(self, name): - """Get pretty printed object prototype from the schema name. - - Args: - name: string, Name of schema in the discovery document. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - # Return with trailing comma and newline removed. - return self._prettyPrintByName(name, seen=[], dent=0)[:-2] - - @util.positional(2) - def _prettyPrintSchema(self, schema, seen=None, dent=0): - """Get pretty printed object prototype of schema. - - Args: - schema: object, Parsed JSON schema. - seen: list of string, Names of schema already seen. Used to handle - recursive definitions. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - if seen is None: - seen = [] - - return _SchemaToStruct(schema, seen, dent=dent).to_str(self._prettyPrintByName) - - def prettyPrintSchema(self, schema): - """Get pretty printed object prototype of schema. - - Args: - schema: object, Parsed JSON schema. - - Returns: - string, A string that contains a prototype object with - comments that conforms to the given schema. - """ - # Return with trailing comma and newline removed. - return self._prettyPrintSchema(schema, dent=0)[:-2] - - def get(self, name, default=None): - """Get deserialized JSON schema from the schema name. - - Args: - name: string, Schema name. - default: object, return value if name not found. - """ - return self.schemas.get(name, default) - - -class _SchemaToStruct(object): - """Convert schema to a prototype object.""" - - @util.positional(3) - def __init__(self, schema, seen, dent=0): - """Constructor. - - Args: - schema: object, Parsed JSON schema. - seen: list, List of names of schema already seen while parsing. Used to - handle recursive definitions. - dent: int, Initial indentation depth. - """ - # The result of this parsing kept as list of strings. - self.value = [] - - # The final value of the parsing. - self.string = None - - # The parsed JSON schema. - self.schema = schema - - # Indentation level. - self.dent = dent - - # Method that when called returns a prototype object for the schema with - # the given name. - self.from_cache = None - - # List of names of schema already seen while parsing. - self.seen = seen - - def emit(self, text): - """Add text as a line to the output. - - Args: - text: string, Text to output. - """ - self.value.extend([" " * self.dent, text, "\n"]) - - def emitBegin(self, text): - """Add text to the output, but with no line terminator. - - Args: - text: string, Text to output. - """ - self.value.extend([" " * self.dent, text]) - - def emitEnd(self, text, comment): - """Add text and comment to the output with line terminator. - - Args: - text: string, Text to output. - comment: string, Python comment. - """ - if comment: - divider = "\n" + " " * (self.dent + 2) + "# " - lines = comment.splitlines() - lines = [x.rstrip() for x in lines] - comment = divider.join(lines) - self.value.extend([text, " # ", comment, "\n"]) - else: - self.value.extend([text, "\n"]) - - def indent(self): - """Increase indentation level.""" - self.dent += 1 - - def undent(self): - """Decrease indentation level.""" - self.dent -= 1 - - def _to_str_impl(self, schema): - """Prototype object based on the schema, in Python code with comments. - - Args: - schema: object, Parsed JSON schema file. - - Returns: - Prototype object based on the schema, in Python code with comments. - """ - stype = schema.get("type") - if stype == "object": - self.emitEnd("{", schema.get("description", "")) - self.indent() - if "properties" in schema: - properties = schema.get("properties", {}) - sorted_properties = OrderedDict(sorted(properties.items())) - for pname, pschema in sorted_properties.items(): - self.emitBegin('"%s": ' % pname) - self._to_str_impl(pschema) - elif "additionalProperties" in schema: - self.emitBegin('"a_key": ') - self._to_str_impl(schema["additionalProperties"]) - self.undent() - self.emit("},") - elif "$ref" in schema: - schemaName = schema["$ref"] - description = schema.get("description", "") - s = self.from_cache(schemaName, seen=self.seen) - parts = s.splitlines() - self.emitEnd(parts[0], description) - for line in parts[1:]: - self.emit(line.rstrip()) - elif stype == "boolean": - value = schema.get("default", "True or False") - self.emitEnd("%s," % str(value), schema.get("description", "")) - elif stype == "string": - value = schema.get("default", "A String") - self.emitEnd('"%s",' % str(value), schema.get("description", "")) - elif stype == "integer": - value = schema.get("default", "42") - self.emitEnd("%s," % str(value), schema.get("description", "")) - elif stype == "number": - value = schema.get("default", "3.14") - self.emitEnd("%s," % str(value), schema.get("description", "")) - elif stype == "null": - self.emitEnd("None,", schema.get("description", "")) - elif stype == "any": - self.emitEnd('"",', schema.get("description", "")) - elif stype == "array": - self.emitEnd("[", schema.get("description")) - self.indent() - self.emitBegin("") - self._to_str_impl(schema["items"]) - self.undent() - self.emit("],") - else: - self.emit("Unknown type! %s" % stype) - self.emitEnd("", "") - - self.string = "".join(self.value) - return self.string - - def to_str(self, from_cache): - """Prototype object based on the schema, in Python code with comments. - - Args: - from_cache: callable(name, seen), Callable that retrieves an object - prototype for a schema with the given name. Seen is a list of schema - names already seen as we recursively descend the schema definition. - - Returns: - Prototype object based on the schema, in Python code with comments. - The lines of the code will all be properly indented. - """ - self.from_cache = from_cache - return self._to_str_impl(self.schema) diff --git a/src/gam/googleapiclient.shouldnotbehere/version.py b/src/gam/googleapiclient.shouldnotbehere/version.py deleted file mode 100644 index 9f3fec3f..00000000 --- a/src/gam/googleapiclient.shouldnotbehere/version.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version__ = "2.179.0"