diff --git a/src/gam.py b/src/gam.py index a35706bf..4fb78bd5 100755 --- a/src/gam.py +++ b/src/gam.py @@ -30,6 +30,7 @@ import base64 import codecs import csv import datetime +import httplib import json import mimetypes import platform @@ -341,8 +342,8 @@ def SetGlobalVariables(): value = number GC_Defaults[itemName] = value - def _getOldSignalFile(itemName, fileName, trueValue=True, falseValue=False): - GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], fileName)) else falseValue + def _getOldSignalFile(itemName, fileName, filePresentValue=True, fileAbsentValue=False): + GC_Defaults[itemName] = filePresentValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], fileName)) else fileAbsentValue def _getCfgDirectory(itemName): return GC_Defaults[itemName] @@ -376,10 +377,12 @@ def SetGlobalVariables(): _getOldEnvVar(GC_DEVICE_MAX_RESULTS, u'GAM_DEVICE_MAX_RESULTS') _getOldEnvVar(GC_DRIVE_MAX_RESULTS, u'GAM_DRIVE_MAX_RESULTS') _getOldEnvVar(GC_USER_MAX_RESULTS, u'GAM_USER_MAX_RESULTS') - _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', trueValue=4, falseValue=0) + _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', filePresentValue=4, fileAbsentValue=0) _getOldSignalFile(GC_NO_VERIFY_SSL, u'noverifyssl.txt') _getOldSignalFile(GC_NO_BROWSER, u'nobrowser.txt') - _getOldSignalFile(GC_NO_CACHE, u'nocache.txt') +# _getOldSignalFile(GC_NO_CACHE, u'nocache.txt') +# _getOldSignalFile(GC_CACHE_DISCOVERY_ONLY, u'allcache.txt', filePresentValue=False, fileAbsentValue=True) + _getOldSignalFile(GC_NO_CACHE, u'allcache.txt', filePresentValue=False, fileAbsentValue=True) _getOldSignalFile(GC_NO_UPDATE_CHECK, u'noupdatecheck.txt') # Assign directories first for itemName in GC_VAR_INFO: @@ -406,7 +409,12 @@ def SetGlobalVariables(): ea_config.read(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT)) GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args'))) if GC_Values[GC_NO_CACHE]: - GC_Values[GC_CACHE_DIR] = None + GM_Globals[GM_CACHE_DIR] = None + GM_Globals[GM_CACHE_DISCOVERY_ONLY] = False + else: + GM_Globals[GM_CACHE_DIR] = GC_Values[GC_CACHE_DIR] +# GM_Globals[GM_CACHE_DISCOVERY_ONLY] = GC_Values[GC_CACHE_DISCOVERY_ONLY] + GM_Globals[GM_CACHE_DISCOVERY_ONLY] = False return True def doGAMCheckForUpdates(forceCheck=False): @@ -485,7 +493,7 @@ def doGAMVersion(checkForArgs=True): doGAMCheckForUpdates(forceCheck=True) def handleOAuthTokenError(e, soft_errors): - if e.message in OAUTH2_TOKEN_ERRORS: + if e in OAUTH2_TOKEN_ERRORS or e.startswith(u'Invalid response'): if soft_errors: return None if not GM_Globals[GM_CURRENT_API_USER]: @@ -517,10 +525,9 @@ def getSvcAcctCredentials(scopes, act_as): def waitOnFailure(n, retries, errMsg): wait_on_fail = min(2 ** n, 60) + float(random.randint(1, 1000)) / 1000 if n > 3: - sys.stderr.write(u'Temp error {0}. Backing off {1} seconds...'.format(errMsg, int(wait_on_fail))) + sys.stderr.write(u'Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'.format(errMsg, int(wait_on_fail), n, retries)) + sys.stderr.flush() time.sleep(wait_on_fail) - if n > 3: - sys.stderr.write(u'attempt {0}/{1}\n'.format(n+1, retries)) def checkGAPIError(e, soft_errors=False, silent_errors=False, retryOnHttpError=False, service=None): try: @@ -597,20 +604,24 @@ def callGAPI(service, function, waitOnFailure(n, retries, reason) continue if soft_errors: - stderrErrorMsg(u'{0}: {1} - {2}{3}'.format(http_status, message, reason, u': Giving up.\n' if n > 1 else u'')) + stderrErrorMsg(u'{0}: {1} - {2}{3}'.format(http_status, message, reason, [u'', u': Giving up.'][n > 1])) return None systemErrorExit(int(http_status), u'{0}: {1} - {2}'.format(http_status, message, reason)) except oauth2client.client.AccessTokenRefreshError as e: - handleOAuthTokenError(e, soft_errors or GAPI_SERVICE_NOT_AVAILABLE in throw_reasons) + handleOAuthTokenError(str(e), soft_errors or GAPI_SERVICE_NOT_AVAILABLE in throw_reasons) if GAPI_SERVICE_NOT_AVAILABLE in throw_reasons: - raise GAPI_serviceNotAvailable(e.message) - print u'ERROR: user %s: %s' % (GM_Globals[GM_CURRENT_API_USER], e) - #entityUnknownWarning(u'User', GM_Globals[GM_CURRENT_API_USER], 0, 0) + raise GAPI_serviceNotAvailable(str(e)) + stderrErrorMsg(u'User {0}: {1)'.format(GM_Globals[GM_CURRENT_API_USER], str(e))) return None except httplib2.CertificateValidationUnsupported: noPythonSSLExit() + except ValueError as e: + if service._http.cache is not None: + service._http.cache = None + continue + systemErrorExit(4, str(e)) except TypeError as e: - systemErrorExit(4, e) + systemErrorExit(4, str(e)) def callGAPIpages(service, function, items=u'items', page_message=None, message_attribute=None, @@ -703,32 +714,51 @@ def getOauth2TxtStorageCredentials(): except (KeyError, ValueError): return (storage, None) -def getClientAPIversionHttpService(api): +def getService(api, http): + api, version, api_version = getAPIVersion(api) + retries = 3 + for n in range(1, retries+1): + try: + service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False) + if GM_Globals[GM_CACHE_DISCOVERY_ONLY]: + http.cache = None + return service + except httplib2.ServerNotFoundError as e: + systemErrorExit(4, str(e)) + except httplib2.CertificateValidationUnsupported: + noPythonSSLExit() + except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e: + http.cache = None + if n != retries: + waitOnFailure(n, retries, str(e)) + continue + systemErrorExit(17, str(e)) + except (httplib.ResponseNotReady, httplib2.SSLHandshakeError, socket.error) as e: + if n != retries: + waitOnFailure(n, retries, str(e)) + continue + systemErrorExit(3, str(e)) + except googleapiclient.errors.UnknownApiNameOrVersion: + break + disc_file, discovery = readDiscoveryFile(api_version) + try: + service = googleapiclient.discovery.build_from_document(discovery, http=http) + if GM_Globals[GM_CACHE_DISCOVERY_ONLY]: + http.cache = None + return service + except (KeyError, ValueError): + invalidJSONExit(disc_file) + +def buildGAPIObject(api): + GM_Globals[GM_CURRENT_API_USER] = None storage, credentials = getOauth2TxtStorageCredentials() if not credentials or credentials.invalid: doRequestOAuth() credentials = storage.get() credentials.user_agent = GAM_INFO - api, version, api_version = getAPIVersion(api) http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR])) - try: - return (credentials, googleapiclient.discovery.build(api, version, http=http, cache_discovery=False)) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) - except httplib2.CertificateValidationUnsupported: - noPythonSSLExit() - except googleapiclient.errors.UnknownApiNameOrVersion: - pass - disc_file, discovery = readDiscoveryFile(api_version) - try: - return (credentials, googleapiclient.discovery.build_from_document(discovery, http=http)) - except (ValueError, KeyError): - invalidJSONExit(disc_file) - -def buildGAPIObject(api): - GM_Globals[GM_CURRENT_API_USER] = None - credentials, service = getClientAPIversionHttpService(api) + cache=GM_Globals[GM_CACHE_DIR])) + service = getService(api, http) if GC_Values[GC_DOMAIN]: if not GC_Values[GC_CUSTOMER_ID]: resp, result = service._http.request(u'https://www.googleapis.com/admin/directory/v1/users?domain={0}&maxResults=1&fields=users(customerId)'.format(GC_Values[GC_DOMAIN])) @@ -768,24 +798,10 @@ def convertUserUIDtoEmailAddress(emailAddressOrUID): pass return normalizedEmailAddressOrUID -def getSvcAcctAPIversionHttpService(api): - api, version, api_version = getAPIVersion(api) - http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR]) - try: - return (api_version, http, googleapiclient.discovery.build(api, version, http=http, cache_discovery=False)) - except httplib2.ServerNotFoundError as e: - systemErrorExit(4, e) - except googleapiclient.errors.UnknownApiNameOrVersion: - pass - disc_file, discovery = readDiscoveryFile(api_version) - try: - return (api_version, http, googleapiclient.discovery.build_from_document(discovery, http=http)) - except (ValueError, KeyError): - invalidJSONExit(disc_file) - def buildGAPIServiceObject(api, act_as, use_scopes=None): - _, http, service = getSvcAcctAPIversionHttpService(api) + http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], + cache=GM_Globals[GM_CACHE_DIR]) + service = getService(api, http) GM_Globals[GM_CURRENT_API_USER] = act_as GM_Globals[GM_CURRENT_API_SCOPES] = use_scopes or API_SCOPE_MAPPING[api] credentials = getSvcAcctCredentials(GM_Globals[GM_CURRENT_API_SCOPES], act_as) @@ -794,8 +810,8 @@ def buildGAPIServiceObject(api, act_as, use_scopes=None): except httplib2.ServerNotFoundError as e: systemErrorExit(4, e) except oauth2client.client.AccessTokenRefreshError as e: - print u'ERROR user %s: %s' % (act_as, e) - return handleOAuthTokenError(e, True) + stderrErrorMsg(u'User {0}: {1)'.format(GM_Globals[GM_CURRENT_API_USER], str(e))) + return handleOAuthTokenError(str(e), True) return service def buildActivityGAPIObject(user): @@ -3454,7 +3470,7 @@ def printPermission(permission): for key in permission: if key in [u'name', u'kind', u'etag', u'selfLink',]: continue - print u' %s: %s' % (key, permission[key]) + print utils.convertUTF8(u' %s: %s' % (key, permission[key])) def showDriveFileACL(users): fileId = sys.argv[5] @@ -6504,7 +6520,7 @@ def getCRMService(login_hint): noPythonSSLExit() credentials.user_agent = GAM_INFO http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL], - cache=GC_Values[GC_CACHE_DIR])) + cache=None)) return (googleapiclient.discovery.build(u'cloudresourcemanager', u'v1', http=http, cache_discovery=False), http) def doDelProjects(login_hint=None): @@ -9827,7 +9843,7 @@ def doDeleteOAuth(): try: credentials.revoke(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL])) except oauth2client.client.TokenRevokeError as e: - stderrErrorMsg(e.message) + stderrErrorMsg(str(e)) os.remove(GC_Values[GC_OAUTH2_TXT]) class cmd_flags(object): @@ -10200,7 +10216,7 @@ def ProcessGAMCommand(args): argv = shlex.split(line) except ValueError as e: sys.stderr.write(utils.convertUTF8(u'Command: >>>{0}<<<\n'.format(line.strip()))) - sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e.message)) + sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, str(e))) errors += 1 continue if len(argv) > 0: diff --git a/src/var.py b/src/var.py index 34b63d32..bf231916 100644 --- a/src/var.py +++ b/src/var.py @@ -503,6 +503,10 @@ GM_MAP_ROLE_ID_TO_NAME = u'ri2n' GM_MAP_ROLE_NAME_TO_ID = u'rn2i' # Dictionary mapping User ID to Name GM_MAP_USER_ID_TO_NAME = u'ui2n' +# GAM cache directory. If no_cache is True, this variable will be set to None +GM_CACHE_DIR = u'gacd' +# Reset GAM cache directory after discovery +GM_CACHE_DISCOVERY_ONLY = u'gcdo' # GM_Globals = { GM_SYSEXITRC: 0, @@ -519,6 +523,8 @@ GM_Globals = { GM_MAP_ROLE_ID_TO_NAME: None, GM_MAP_ROLE_NAME_TO_ID: None, GM_MAP_USER_ID_TO_NAME: None, + GM_CACHE_DIR: None, + GM_CACHE_DISCOVERY_ONLY: True, } # # Global variables defined by environment variables/signal files @@ -532,6 +538,8 @@ GC_AUTO_BATCH_MIN = u'auto_batch_min' GC_BATCH_SIZE = u'batch_size' # GAM cache directory. If no_cache is specified, this variable will be set to None GC_CACHE_DIR = u'cache_dir' +# GAM cache discovery only. If no_cache is False, only API discovery calls will be cached +GC_CACHE_DISCOVERY_ONLY = u'cache_discovery_only' # Character set of batch, csv, data files GC_CHARSET = u'charset' # Path to client_secrets.json @@ -581,6 +589,7 @@ GC_Defaults = { GC_AUTO_BATCH_MIN: 0, GC_BATCH_SIZE: 50, GC_CACHE_DIR: u'', + GC_CACHE_DISCOVERY_ONLY: True, GC_CHARSET: DEFAULT_CHARSET, GC_CLIENT_SECRETS_JSON: FN_CLIENT_SECRETS_JSON, GC_CONFIG_DIR: u'', @@ -590,16 +599,16 @@ GC_Defaults = { GC_DOMAIN: u'', GC_DRIVE_DIR: u'', GC_DRIVE_MAX_RESULTS: 1000, - GC_NO_BROWSER: FALSE, - GC_NO_CACHE: FALSE, - GC_NO_UPDATE_CHECK: FALSE, - GC_NO_VERIFY_SSL: FALSE, + GC_NO_BROWSER: False, + GC_NO_CACHE: False, + GC_NO_UPDATE_CHECK: False, + GC_NO_VERIFY_SSL: False, GC_NUM_THREADS: 25, GC_OAUTH2_TXT: FN_OAUTH2_TXT, GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON, GC_SECTION: u'', GC_SHOW_COUNTS_MIN: 0, - GC_SHOW_GETTINGS: TRUE, + GC_SHOW_GETTINGS: True, GC_SITE_DIR: u'', GC_USER_MAX_RESULTS: 500, } @@ -623,6 +632,7 @@ GC_VAR_INFO = { GC_AUTO_BATCH_MIN: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (0, None)}, GC_BATCH_SIZE: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 1000)}, GC_CACHE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY}, + GC_CACHE_DISCOVERY_ONLY: {GC_VAR_TYPE: GC_TYPE_BOOLEAN}, GC_CHARSET: {GC_VAR_TYPE: GC_TYPE_STRING}, GC_CLIENT_SECRETS_JSON: {GC_VAR_TYPE: GC_TYPE_FILE}, GC_CONFIG_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},