deepblue/GoogleDriveManagement
1
0
Fork 0
mirror of https://github.com/GAM-team/GAM.git synced 2026-07-04 21:01:36 +00:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Cleanup (#1066)

Browse Source 
* Cleanup

gam,py:
In show sakays, only mark USER-MANAGED keys as current: true/false

gapi/__init__.py:
pylint cleanup: import order, indentation

gapi/errors.py:
pylint cleanup: import order, unused import, indentation

Recognize 502 Bad Gateway/Gateway Timeout: treat as retryable errors

This last change does not directly handle the refresh problem in Issue #1063 but in my version this seems to be the right solution to the 502 gateways errors

* In show sakeys, indicate whcij key was used to authenticate

* Show all sakeys by default

* Include unused import

* Remove unused import, fix unit tests
This commit is contained in:
Ross Scroggs
2020-01-01 12:04:45 -08:00
committed by Jay Lee
parent 2c0a005d3e
commit 497251186d
3 changed files with 146 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/gam.py
View File

@@ -7840,7 +7840,7 @@ def _formatOAuth2ServiceData(private_key, private_key_id):
def doShowServiceAccountKeys(): def doShowServiceAccountKeys():
iam = buildGAPIServiceObject('iam', None) iam = buildGAPIServiceObject('iam', None)
keyTypes = 'USER_MANAGED' keyTypes = None
i = 3 i = 3
while i < len(sys.argv): while i < len(sys.argv):
myarg = sys.argv[i].lower().replace('_', '') myarg = sys.argv[i].lower().replace('_', '')
@@ -7867,7 +7867,8 @@ def doShowServiceAccountKeys():
print('{0}: {1}'.format(parts[i][:-1], parts[i+1])) print('{0}: {1}'.format(parts[i][:-1], parts[i+1]))
for key in keys: for key in keys:
key['name'] = key['name'].rsplit('/', 1)[-1] key['name'] = key['name'].rsplit('/', 1)[-1]
key['current'] = key['name'] == currentPrivateKeyId if key['name'] == currentPrivateKeyId:
key['usedToAuthenticateThisRequest'] = True
print_json(None, keys) print_json(None, keys)
def doRotateServiceAccountKeys(): def doRotateServiceAccountKeys():

88
src/gapi/__init__.py
View File

@@ -2,17 +2,18 @@
import sys import sys
import googleapiclient.errors
import google.auth.exceptions
import httplib2
import controlflow import controlflow
import display import display
from gapi import errors from gapi import errors
import googleapiclient.errors
import httplib2
from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION, from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION,
GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER, GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID, GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG, MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE) MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
import google.auth.exceptions
def create_http(cache=None, def create_http(cache=None,
@@ -33,15 +34,15 @@ def create_http(cache=None,
httplib2.Http with the specified options. httplib2.Http with the specified options.
""" """
tls_minimum_version = override_min_tls if override_min_tls else GC_Values[ tls_minimum_version = override_min_tls if override_min_tls else GC_Values[
GC_TLS_MIN_VERSION] GC_TLS_MIN_VERSION]
tls_maximum_version = override_max_tls if override_max_tls else GC_Values[ tls_maximum_version = override_max_tls if override_max_tls else GC_Values[
GC_TLS_MAX_VERSION] GC_TLS_MAX_VERSION]
return httplib2.Http( return httplib2.Http(
ca_certs=GC_Values[GC_CA_FILE], ca_certs=GC_Values[GC_CA_FILE],
tls_maximum_version=tls_maximum_version, tls_maximum_version=tls_maximum_version,
tls_minimum_version=tls_minimum_version, tls_minimum_version=tls_minimum_version,
cache=cache, cache=cache,
timeout=timeout) timeout=timeout)
def call(service, def call(service,
@@ -78,16 +79,16 @@ def call(service,
method = getattr(service, function) method = getattr(service, function)
retries = 10 retries = 10
parameters = dict( parameters = dict(
list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items())) list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
for n in range(1, retries + 1): for n in range(1, retries + 1):
try: try:
return method(**parameters).execute() return method(**parameters).execute()
except googleapiclient.errors.HttpError as e: except googleapiclient.errors.HttpError as e:
http_status, reason, message = errors.get_gapi_error_detail( http_status, reason, message = errors.get_gapi_error_detail(
e, e,
soft_errors=soft_errors, soft_errors=soft_errors,
silent_errors=silent_errors, silent_errors=silent_errors,
retry_on_http_error=n < 3) retry_on_http_error=n < 3)
if http_status == -1: if http_status == -1:
# The error detail indicated that we should retry this request # The error detail indicated that we should retry this request
# We'll refresh credentials and make another pass # We'll refresh credentials and make another pass
@@ -100,9 +101,8 @@ def call(service,
if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons: if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION: if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)]( raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
message) message)
else: raise e
raise e
if (n != retries) and (is_known_error_reason and errors.ErrorReason( if (n != retries) and (is_known_error_reason and errors.ErrorReason(
reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons): reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
controlflow.wait_on_failure(n, retries, reason) controlflow.wait_on_failure(n, retries, reason)
@@ -114,16 +114,16 @@ def call(service,
': Giving up.'][n > 1])) ': Giving up.'][n > 1]))
return None return None
controlflow.system_error_exit( controlflow.system_error_exit(
int(http_status), '{0}: {1} - {2}'.format(http_status, message, int(http_status), '{0}: {1} - {2}'.format(http_status, message,
reason)) reason))
except google.auth.exceptions.RefreshError as e: except google.auth.exceptions.RefreshError as e:
handle_oauth_token_error( handle_oauth_token_error(
e, soft_errors or e, soft_errors or
errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons) errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons: if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
raise errors.GapiServiceNotAvailableError(str(e)) raise errors.GapiServiceNotAvailableError(str(e))
display.print_error('User {0}: {1}'.format( display.print_error('User {0}: {1}'.format(
GM_Globals[GM_CURRENT_API_USER], str(e))) GM_Globals[GM_CURRENT_API_USER], str(e)))
return None return None
except ValueError as e: except ValueError as e:
if service._http.cache is not None: if service._http.cache is not None:
@@ -165,11 +165,11 @@ def get_items(service,
The list of items in the first page of a response. The list of items in the first page of a response.
""" """
results = call( results = call(
service, service,
function, function,
throw_reasons=throw_reasons, throw_reasons=throw_reasons,
retry_reasons=retry_reasons, retry_reasons=retry_reasons,
**kwargs) **kwargs)
if results: if results:
return results.get(items, []) return results.get(items, [])
return [] return []
@@ -200,7 +200,7 @@ def _get_max_page_size_for_api_call(service, function, **kwargs):
return None return None
known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id) known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
max_results = a_method['parameters']['maxResults'].get( max_results = a_method['parameters']['maxResults'].get(
'maximum', known_api_max) 'maximum', known_api_max)
return {'maxResults': max_results} return {'maxResults': max_results}
return None return None
@@ -258,13 +258,13 @@ def get_all_pages(service,
total_items = 0 total_items = 0
while True: while True:
page = call( page = call(
service, service,
function, function,
soft_errors=soft_errors, soft_errors=soft_errors,
throw_reasons=throw_reasons, throw_reasons=throw_reasons,
retry_reasons=retry_reasons, retry_reasons=retry_reasons,
pageToken=page_token, pageToken=page_token,
**kwargs) **kwargs)
if page: if page:
page_token = page.get('nextPageToken') page_token = page.get('nextPageToken')
page_items = page.get(items, []) page_items = page.get(items, [])
@@ -282,9 +282,9 @@ def get_all_pages(service,
first_item = page_items[0] if num_page_items > 0 else {} first_item = page_items[0] if num_page_items > 0 else {}
last_item = page_items[-1] if num_page_items > 1 else first_item last_item = page_items[-1] if num_page_items > 1 else first_item
show_message = show_message.replace( show_message = show_message.replace(
'%%first_item%%', str(first_item.get(message_attribute, ''))) '%%first_item%%', str(first_item.get(message_attribute, '')))
show_message = show_message.replace( show_message = show_message.replace(
'%%last_item%%', str(last_item.get(message_attribute, ''))) '%%last_item%%', str(last_item.get(message_attribute, '')))
sys.stderr.write('\r') sys.stderr.write('\r')
sys.stderr.flush() sys.stderr.flush()
sys.stderr.write(show_message) sys.stderr.write(show_message)
@@ -314,14 +314,14 @@ def handle_oauth_token_error(e, soft_errors):
return return
if not GM_Globals[GM_CURRENT_API_USER]: if not GM_Globals[GM_CURRENT_API_USER]:
display.print_error( display.print_error(
MESSAGE_API_ACCESS_DENIED.format( MESSAGE_API_ACCESS_DENIED.format(
GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID], GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
','.join(GM_Globals[GM_CURRENT_API_SCOPES]))) ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG) controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
else: else:
controlflow.system_error_exit( controlflow.system_error_exit(
19, 19,
MESSAGE_SERVICE_NOT_APPLICABLE.format( MESSAGE_SERVICE_NOT_APPLICABLE.format(
GM_Globals[GM_CURRENT_API_USER])) GM_Globals[GM_CURRENT_API_USER]))
controlflow.system_error_exit(18, controlflow.system_error_exit(18,
'Authentication Token Error - {0}'.format(e)) 'Authentication Token Error - {0}'.format(e))

180
src/gapi/errors.py
View File

@@ -1,12 +1,11 @@
"""GAPI and OAuth Token related errors methods.""" """GAPI and OAuth Token related errors methods."""
from enum import Enum
import json import json
import controlflow import controlflow
from enum import Enum
import googleapiclient.errors
from var import UTF8
import display # TODO: Change to relative import when gam is setup as a package import display # TODO: Change to relative import when gam is setup as a package
from var import UTF8
class GapiAbortedError(Exception): class GapiAbortedError(Exception):
@@ -17,6 +16,10 @@ class GapiAuthErrorError(Exception):
pass pass
class GapiBadGatewayError(Exception):
pass
class GapiBadRequestError(Exception): class GapiBadRequestError(Exception):
pass pass
@@ -49,6 +52,10 @@ class GapiForbiddenError(Exception):
pass pass
class GapiGatewayTimeoutError(Exception):
pass
class GapiGroupNotFoundError(Exception): class GapiGroupNotFoundError(Exception):
pass pass
@@ -99,6 +106,7 @@ class ErrorReason(Enum):
ABORTED = 'aborted' ABORTED = 'aborted'
AUTH_ERROR = 'authError' AUTH_ERROR = 'authError'
BACKEND_ERROR = 'backendError' BACKEND_ERROR = 'backendError'
BAD_GATEWAY = 'badGateway'
BAD_REQUEST = 'badRequest' BAD_REQUEST = 'badRequest'
CONDITION_NOT_MET = 'conditionNotMet' CONDITION_NOT_MET = 'conditionNotMet'
CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed' CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
@@ -107,6 +115,7 @@ class ErrorReason(Enum):
DUPLICATE = 'duplicate' DUPLICATE = 'duplicate'
FAILED_PRECONDITION = 'failedPrecondition' FAILED_PRECONDITION = 'failedPrecondition'
FORBIDDEN = 'forbidden' FORBIDDEN = 'forbidden'
GATEWAY_TIMEOUT = 'gatewayTimeout'
GROUP_NOT_FOUND = 'groupNotFound' GROUP_NOT_FOUND = 'groupNotFound'
INTERNAL_ERROR = 'internalError' INTERNAL_ERROR = 'internalError'
INVALID = 'invalid' INVALID = 'invalid'
@@ -130,89 +139,94 @@ class ErrorReason(Enum):
# Common sets of GAPI error reasons # Common sets of GAPI error reasons
DEFAULT_RETRY_REASONS = [ DEFAULT_RETRY_REASONS = [
ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED, ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR, ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
ErrorReason.INTERNAL_ERROR ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT,
] ErrorReason.INTERNAL_ERROR
]
GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE] GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
GROUP_GET_THROW_REASONS = [ GROUP_GET_THROW_REASONS = [
ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND, ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN, ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
ErrorReason.BAD_REQUEST ErrorReason.BAD_REQUEST
] ]
GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR] GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
MEMBERS_THROW_REASONS = [ MEMBERS_THROW_REASONS = [
ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND, ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID, ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
ErrorReason.FORBIDDEN ErrorReason.FORBIDDEN
] ]
MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR] MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
# A map of GAPI error reasons to the corresponding GAM Python Exception # A map of GAPI error reasons to the corresponding GAM Python Exception
ERROR_REASON_TO_EXCEPTION = { ERROR_REASON_TO_EXCEPTION = {
ErrorReason.ABORTED: ErrorReason.ABORTED:
GapiAbortedError, GapiAbortedError,
ErrorReason.AUTH_ERROR: ErrorReason.AUTH_ERROR:
GapiAuthErrorError, GapiAuthErrorError,
ErrorReason.BAD_REQUEST: ErrorReason.BAD_GATEWAY:
GapiBadRequestError, GapiBadGatewayError,
ErrorReason.CONDITION_NOT_MET: ErrorReason.BAD_REQUEST:
GapiConditionNotMetError, GapiBadRequestError,
ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED: ErrorReason.CONDITION_NOT_MET:
GapiCyclicMembershipsNotAllowedError, GapiConditionNotMetError,
ErrorReason.DOMAIN_CANNOT_USE_APIS: ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
GapiDomainCannotUseApisError, GapiCyclicMembershipsNotAllowedError,
ErrorReason.DOMAIN_NOT_FOUND: ErrorReason.DOMAIN_CANNOT_USE_APIS:
GapiDomainNotFoundError, GapiDomainCannotUseApisError,
ErrorReason.DUPLICATE: ErrorReason.DOMAIN_NOT_FOUND:
GapiDuplicateError, GapiDomainNotFoundError,
ErrorReason.FAILED_PRECONDITION: ErrorReason.DUPLICATE:
GapiFailedPreconditionError, GapiDuplicateError,
ErrorReason.FORBIDDEN: ErrorReason.FAILED_PRECONDITION:
GapiForbiddenError, GapiFailedPreconditionError,
ErrorReason.GROUP_NOT_FOUND: ErrorReason.FORBIDDEN:
GapiGroupNotFoundError, GapiForbiddenError,
ErrorReason.INVALID: ErrorReason.GATEWAY_TIMEOUT:
GapiInvalidError, GapiGatewayTimeoutError,
ErrorReason.INVALID_ARGUMENT: ErrorReason.GROUP_NOT_FOUND:
GapiInvalidArgumentError, GapiGroupNotFoundError,
ErrorReason.INVALID_MEMBER: ErrorReason.INVALID:
GapiInvalidMemberError, GapiInvalidError,
ErrorReason.MEMBER_NOT_FOUND: ErrorReason.INVALID_ARGUMENT:
GapiMemberNotFoundError, GapiInvalidArgumentError,
ErrorReason.NOT_FOUND: ErrorReason.INVALID_MEMBER:
GapiNotFoundError, GapiInvalidMemberError,
ErrorReason.NOT_IMPLEMENTED: ErrorReason.MEMBER_NOT_FOUND:
GapiNotImplementedError, GapiMemberNotFoundError,
ErrorReason.PERMISSION_DENIED: ErrorReason.NOT_FOUND:
GapiPermissionDeniedError, GapiNotFoundError,
ErrorReason.RESOURCE_NOT_FOUND: ErrorReason.NOT_IMPLEMENTED:
GapiResourceNotFoundError, GapiNotImplementedError,
ErrorReason.SERVICE_NOT_AVAILABLE: ErrorReason.PERMISSION_DENIED:
GapiServiceNotAvailableError, GapiPermissionDeniedError,
ErrorReason.USER_NOT_FOUND: ErrorReason.RESOURCE_NOT_FOUND:
GapiUserNotFoundError, GapiResourceNotFoundError,
} ErrorReason.SERVICE_NOT_AVAILABLE:
GapiServiceNotAvailableError,
ErrorReason.USER_NOT_FOUND:
GapiUserNotFoundError,
}
# OAuth Token Errors # OAuth Token Errors
OAUTH2_TOKEN_ERRORS = [ OAUTH2_TOKEN_ERRORS = [
'access_denied', 'access_denied',
'access_denied: Requested client not authorized', 'access_denied: Requested client not authorized',
'internal_failure: Backend Error', 'internal_failure: Backend Error',
'internal_failure: None', 'internal_failure: None',
'invalid_grant', 'invalid_grant',
'invalid_grant: Bad Request', 'invalid_grant: Bad Request',
'invalid_grant: Invalid email or User ID', 'invalid_grant: Invalid email or User ID',
'invalid_grant: Not a valid email', 'invalid_grant: Not a valid email',
'invalid_grant: Invalid JWT: No valid verifier found for issuer', 'invalid_grant: Invalid JWT: No valid verifier found for issuer',
'invalid_request: Invalid impersonation prn email address', 'invalid_request: Invalid impersonation prn email address',
'unauthorized_client: Client is unauthorized to retrieve access tokens ' 'unauthorized_client: Client is unauthorized to retrieve access tokens '
'using this method', 'using this method',
'unauthorized_client: Client is unauthorized to retrieve access tokens ' 'unauthorized_client: Client is unauthorized to retrieve access tokens '
'using this method, or client not authorized for any of the scopes ' 'using this method, or client not authorized for any of the scopes '
'requested', 'requested',
'unauthorized_client: Unauthorized client or scope in request', 'unauthorized_client: Unauthorized client or scope in request',
] ]
def _create_http_error_dict(status_code, reason, message): def _create_http_error_dict(status_code, reason, message):
@@ -227,12 +241,12 @@ def _create_http_error_dict(status_code, reason, message):
dict dict
""" """
return { return {
'error': { 'error': {
'code': status_code, 'code': status_code,
'errors': [{ 'errors': [{
'reason': str(reason), 'reason': str(reason),
'message': message, 'message': message,
}] }]
} }
} }
@@ -268,6 +282,10 @@ def get_gapi_error_detail(e,
if (e.resp['status'] == '403') and ( if (e.resp['status'] == '403') and (
error_content.startswith('Request rate higher than configured')): error_content.startswith('Request rate higher than configured')):
return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content) return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
return (e.resp['status'], ErrorReason.BAD_GATEWAY.value, error_content)
if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value, error_content)
if (e.resp['status'] == '403') and ('Invalid domain.' in error_content): if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value, error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
'Domain not found') 'Domain not found')