From 497251186de91c098e3dbf3c6ee2c0285892dd4c Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Wed, 1 Jan 2020 12:04:45 -0800 Subject: [PATCH] Cleanup (#1066) * Cleanup gam,py: In show sakays, only mark USER-MANAGED keys as current: true/false gapi/__init__.py: pylint cleanup: import order, indentation gapi/errors.py: pylint cleanup: import order, unused import, indentation Recognize 502 Bad Gateway/Gateway Timeout: treat as retryable errors This last change does not directly handle the refresh problem in Issue #1063 but in my version this seems to be the right solution to the 502 gateways errors * In show sakeys, indicate whcij key was used to authenticate * Show all sakeys by default * Include unused import * Remove unused import, fix unit tests --- src/gam.py | 5 +- src/gapi/__init__.py | 88 ++++++++++----------- src/gapi/errors.py | 180 ++++++++++++++++++++++++------------------- 3 files changed, 146 insertions(+), 127 deletions(-) diff --git a/src/gam.py b/src/gam.py index fb512cac..bb1158fa 100755 --- a/src/gam.py +++ b/src/gam.py @@ -7840,7 +7840,7 @@ def _formatOAuth2ServiceData(private_key, private_key_id): def doShowServiceAccountKeys(): iam = buildGAPIServiceObject('iam', None) - keyTypes = 'USER_MANAGED' + keyTypes = None i = 3 while i < len(sys.argv): myarg = sys.argv[i].lower().replace('_', '') @@ -7867,7 +7867,8 @@ def doShowServiceAccountKeys(): print('{0}: {1}'.format(parts[i][:-1], parts[i+1])) for key in keys: key['name'] = key['name'].rsplit('/', 1)[-1] - key['current'] = key['name'] == currentPrivateKeyId + if key['name'] == currentPrivateKeyId: + key['usedToAuthenticateThisRequest'] = True print_json(None, keys) def doRotateServiceAccountKeys(): diff --git a/src/gapi/__init__.py b/src/gapi/__init__.py index a2d147cb..ca0ee685 100644 --- a/src/gapi/__init__.py +++ b/src/gapi/__init__.py @@ -2,17 +2,18 @@ import sys +import googleapiclient.errors +import google.auth.exceptions +import httplib2 + import controlflow import display from gapi import errors -import googleapiclient.errors -import httplib2 from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION, GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER, GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID, MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG, MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE) -import google.auth.exceptions def create_http(cache=None, @@ -33,15 +34,15 @@ def create_http(cache=None, httplib2.Http with the specified options. """ tls_minimum_version = override_min_tls if override_min_tls else GC_Values[ - GC_TLS_MIN_VERSION] + GC_TLS_MIN_VERSION] tls_maximum_version = override_max_tls if override_max_tls else GC_Values[ - GC_TLS_MAX_VERSION] + GC_TLS_MAX_VERSION] return httplib2.Http( - ca_certs=GC_Values[GC_CA_FILE], - tls_maximum_version=tls_maximum_version, - tls_minimum_version=tls_minimum_version, - cache=cache, - timeout=timeout) + ca_certs=GC_Values[GC_CA_FILE], + tls_maximum_version=tls_maximum_version, + tls_minimum_version=tls_minimum_version, + cache=cache, + timeout=timeout) def call(service, @@ -78,16 +79,16 @@ def call(service, method = getattr(service, function) retries = 10 parameters = dict( - list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items())) + list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items())) for n in range(1, retries + 1): try: return method(**parameters).execute() except googleapiclient.errors.HttpError as e: http_status, reason, message = errors.get_gapi_error_detail( - e, - soft_errors=soft_errors, - silent_errors=silent_errors, - retry_on_http_error=n < 3) + e, + soft_errors=soft_errors, + silent_errors=silent_errors, + retry_on_http_error=n < 3) if http_status == -1: # The error detail indicated that we should retry this request # We'll refresh credentials and make another pass @@ -100,9 +101,8 @@ def call(service, if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons: if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION: raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)]( - message) - else: - raise e + message) + raise e if (n != retries) and (is_known_error_reason and errors.ErrorReason( reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons): controlflow.wait_on_failure(n, retries, reason) @@ -114,16 +114,16 @@ def call(service, ': Giving up.'][n > 1])) return None controlflow.system_error_exit( - int(http_status), '{0}: {1} - {2}'.format(http_status, message, - reason)) + int(http_status), '{0}: {1} - {2}'.format(http_status, message, + reason)) except google.auth.exceptions.RefreshError as e: handle_oauth_token_error( - e, soft_errors or - errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons) + e, soft_errors or + errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons) if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons: raise errors.GapiServiceNotAvailableError(str(e)) display.print_error('User {0}: {1}'.format( - GM_Globals[GM_CURRENT_API_USER], str(e))) + GM_Globals[GM_CURRENT_API_USER], str(e))) return None except ValueError as e: if service._http.cache is not None: @@ -165,11 +165,11 @@ def get_items(service, The list of items in the first page of a response. """ results = call( - service, - function, - throw_reasons=throw_reasons, - retry_reasons=retry_reasons, - **kwargs) + service, + function, + throw_reasons=throw_reasons, + retry_reasons=retry_reasons, + **kwargs) if results: return results.get(items, []) return [] @@ -200,7 +200,7 @@ def _get_max_page_size_for_api_call(service, function, **kwargs): return None known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id) max_results = a_method['parameters']['maxResults'].get( - 'maximum', known_api_max) + 'maximum', known_api_max) return {'maxResults': max_results} return None @@ -258,13 +258,13 @@ def get_all_pages(service, total_items = 0 while True: page = call( - service, - function, - soft_errors=soft_errors, - throw_reasons=throw_reasons, - retry_reasons=retry_reasons, - pageToken=page_token, - **kwargs) + service, + function, + soft_errors=soft_errors, + throw_reasons=throw_reasons, + retry_reasons=retry_reasons, + pageToken=page_token, + **kwargs) if page: page_token = page.get('nextPageToken') page_items = page.get(items, []) @@ -282,9 +282,9 @@ def get_all_pages(service, first_item = page_items[0] if num_page_items > 0 else {} last_item = page_items[-1] if num_page_items > 1 else first_item show_message = show_message.replace( - '%%first_item%%', str(first_item.get(message_attribute, ''))) + '%%first_item%%', str(first_item.get(message_attribute, ''))) show_message = show_message.replace( - '%%last_item%%', str(last_item.get(message_attribute, ''))) + '%%last_item%%', str(last_item.get(message_attribute, ''))) sys.stderr.write('\r') sys.stderr.flush() sys.stderr.write(show_message) @@ -314,14 +314,14 @@ def handle_oauth_token_error(e, soft_errors): return if not GM_Globals[GM_CURRENT_API_USER]: display.print_error( - MESSAGE_API_ACCESS_DENIED.format( - GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], - ','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) + MESSAGE_API_ACCESS_DENIED.format( + GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], + ','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG) else: controlflow.system_error_exit( - 19, - MESSAGE_SERVICE_NOT_APPLICABLE.format( - GM_Globals[GM_CURRENT_API_USER])) + 19, + MESSAGE_SERVICE_NOT_APPLICABLE.format( + GM_Globals[GM_CURRENT_API_USER])) controlflow.system_error_exit(18, 'Authentication Token Error - {0}'.format(e)) diff --git a/src/gapi/errors.py b/src/gapi/errors.py index 7829e58e..3dd4dc4f 100644 --- a/src/gapi/errors.py +++ b/src/gapi/errors.py @@ -1,12 +1,11 @@ """GAPI and OAuth Token related errors methods.""" +from enum import Enum import json import controlflow -from enum import Enum -import googleapiclient.errors -from var import UTF8 import display # TODO: Change to relative import when gam is setup as a package +from var import UTF8 class GapiAbortedError(Exception): @@ -17,6 +16,10 @@ class GapiAuthErrorError(Exception): pass +class GapiBadGatewayError(Exception): + pass + + class GapiBadRequestError(Exception): pass @@ -49,6 +52,10 @@ class GapiForbiddenError(Exception): pass +class GapiGatewayTimeoutError(Exception): + pass + + class GapiGroupNotFoundError(Exception): pass @@ -99,6 +106,7 @@ class ErrorReason(Enum): ABORTED = 'aborted' AUTH_ERROR = 'authError' BACKEND_ERROR = 'backendError' + BAD_GATEWAY = 'badGateway' BAD_REQUEST = 'badRequest' CONDITION_NOT_MET = 'conditionNotMet' CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed' @@ -107,6 +115,7 @@ class ErrorReason(Enum): DUPLICATE = 'duplicate' FAILED_PRECONDITION = 'failedPrecondition' FORBIDDEN = 'forbidden' + GATEWAY_TIMEOUT = 'gatewayTimeout' GROUP_NOT_FOUND = 'groupNotFound' INTERNAL_ERROR = 'internalError' INVALID = 'invalid' @@ -130,89 +139,94 @@ class ErrorReason(Enum): # Common sets of GAPI error reasons DEFAULT_RETRY_REASONS = [ - ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED, - ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR, - ErrorReason.INTERNAL_ERROR -] + ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED, + ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR, + ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT, + ErrorReason.INTERNAL_ERROR + ] GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE] GROUP_GET_THROW_REASONS = [ - ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND, - ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN, - ErrorReason.BAD_REQUEST -] + ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND, + ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN, + ErrorReason.BAD_REQUEST + ] GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR] MEMBERS_THROW_REASONS = [ - ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND, - ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID, - ErrorReason.FORBIDDEN -] + ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND, + ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID, + ErrorReason.FORBIDDEN + ] MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR] # A map of GAPI error reasons to the corresponding GAM Python Exception ERROR_REASON_TO_EXCEPTION = { - ErrorReason.ABORTED: - GapiAbortedError, - ErrorReason.AUTH_ERROR: - GapiAuthErrorError, - ErrorReason.BAD_REQUEST: - GapiBadRequestError, - ErrorReason.CONDITION_NOT_MET: - GapiConditionNotMetError, - ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED: - GapiCyclicMembershipsNotAllowedError, - ErrorReason.DOMAIN_CANNOT_USE_APIS: - GapiDomainCannotUseApisError, - ErrorReason.DOMAIN_NOT_FOUND: - GapiDomainNotFoundError, - ErrorReason.DUPLICATE: - GapiDuplicateError, - ErrorReason.FAILED_PRECONDITION: - GapiFailedPreconditionError, - ErrorReason.FORBIDDEN: - GapiForbiddenError, - ErrorReason.GROUP_NOT_FOUND: - GapiGroupNotFoundError, - ErrorReason.INVALID: - GapiInvalidError, - ErrorReason.INVALID_ARGUMENT: - GapiInvalidArgumentError, - ErrorReason.INVALID_MEMBER: - GapiInvalidMemberError, - ErrorReason.MEMBER_NOT_FOUND: - GapiMemberNotFoundError, - ErrorReason.NOT_FOUND: - GapiNotFoundError, - ErrorReason.NOT_IMPLEMENTED: - GapiNotImplementedError, - ErrorReason.PERMISSION_DENIED: - GapiPermissionDeniedError, - ErrorReason.RESOURCE_NOT_FOUND: - GapiResourceNotFoundError, - ErrorReason.SERVICE_NOT_AVAILABLE: - GapiServiceNotAvailableError, - ErrorReason.USER_NOT_FOUND: - GapiUserNotFoundError, -} + ErrorReason.ABORTED: + GapiAbortedError, + ErrorReason.AUTH_ERROR: + GapiAuthErrorError, + ErrorReason.BAD_GATEWAY: + GapiBadGatewayError, + ErrorReason.BAD_REQUEST: + GapiBadRequestError, + ErrorReason.CONDITION_NOT_MET: + GapiConditionNotMetError, + ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED: + GapiCyclicMembershipsNotAllowedError, + ErrorReason.DOMAIN_CANNOT_USE_APIS: + GapiDomainCannotUseApisError, + ErrorReason.DOMAIN_NOT_FOUND: + GapiDomainNotFoundError, + ErrorReason.DUPLICATE: + GapiDuplicateError, + ErrorReason.FAILED_PRECONDITION: + GapiFailedPreconditionError, + ErrorReason.FORBIDDEN: + GapiForbiddenError, + ErrorReason.GATEWAY_TIMEOUT: + GapiGatewayTimeoutError, + ErrorReason.GROUP_NOT_FOUND: + GapiGroupNotFoundError, + ErrorReason.INVALID: + GapiInvalidError, + ErrorReason.INVALID_ARGUMENT: + GapiInvalidArgumentError, + ErrorReason.INVALID_MEMBER: + GapiInvalidMemberError, + ErrorReason.MEMBER_NOT_FOUND: + GapiMemberNotFoundError, + ErrorReason.NOT_FOUND: + GapiNotFoundError, + ErrorReason.NOT_IMPLEMENTED: + GapiNotImplementedError, + ErrorReason.PERMISSION_DENIED: + GapiPermissionDeniedError, + ErrorReason.RESOURCE_NOT_FOUND: + GapiResourceNotFoundError, + ErrorReason.SERVICE_NOT_AVAILABLE: + GapiServiceNotAvailableError, + ErrorReason.USER_NOT_FOUND: + GapiUserNotFoundError, + } # OAuth Token Errors OAUTH2_TOKEN_ERRORS = [ - 'access_denied', - 'access_denied: Requested client not authorized', - 'internal_failure: Backend Error', - 'internal_failure: None', - 'invalid_grant', - 'invalid_grant: Bad Request', - 'invalid_grant: Invalid email or User ID', - 'invalid_grant: Not a valid email', - 'invalid_grant: Invalid JWT: No valid verifier found for issuer', - 'invalid_request: Invalid impersonation prn email address', - 'unauthorized_client: Client is unauthorized to retrieve access tokens ' - 'using this method', - 'unauthorized_client: Client is unauthorized to retrieve access tokens ' - 'using this method, or client not authorized for any of the scopes ' - 'requested', - 'unauthorized_client: Unauthorized client or scope in request', -] + 'access_denied', + 'access_denied: Requested client not authorized', + 'internal_failure: Backend Error', + 'internal_failure: None', + 'invalid_grant', + 'invalid_grant: Bad Request', + 'invalid_grant: Invalid email or User ID', + 'invalid_grant: Not a valid email', + 'invalid_grant: Invalid JWT: No valid verifier found for issuer', + 'invalid_request: Invalid impersonation prn email address', + 'unauthorized_client: Client is unauthorized to retrieve access tokens ' + 'using this method', + 'unauthorized_client: Client is unauthorized to retrieve access tokens ' + 'using this method, or client not authorized for any of the scopes ' + 'requested', + 'unauthorized_client: Unauthorized client or scope in request', + ] def _create_http_error_dict(status_code, reason, message): @@ -227,12 +241,12 @@ def _create_http_error_dict(status_code, reason, message): dict """ return { - 'error': { - 'code': status_code, - 'errors': [{ - 'reason': str(reason), - 'message': message, - }] + 'error': { + 'code': status_code, + 'errors': [{ + 'reason': str(reason), + 'message': message, + }] } } @@ -268,6 +282,10 @@ def get_gapi_error_detail(e, if (e.resp['status'] == '403') and ( error_content.startswith('Request rate higher than configured')): return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content) + if (e.resp['status'] == '502') and ('Bad Gateway' in error_content): + return (e.resp['status'], ErrorReason.BAD_GATEWAY.value, error_content) + if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content): + return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value, error_content) if (e.resp['status'] == '403') and ('Invalid domain.' in error_content): error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value, 'Domain not found')