Improve API caching options, cache errors (#446)

* Improve API caching options, cache errors

* Clarify argument names

* All or nothing caching

I changed the argument names and implemented your proposal:
	nocache.txt: ignored
	allcache.txt present: all caching
        default: no caching

src/gam.py

@@ -30,6 +30,7 @@ import base64
 import codecs
 import csv
 import datetime
+import httplib
 import json
 import mimetypes
 import platform
@@ -341,8 +342,8 @@ def SetGlobalVariables():
       value = number
     GC_Defaults[itemName] = value
 
-  def _getOldSignalFile(itemName, fileName, trueValue=True, falseValue=False):
+  def _getOldSignalFile(itemName, fileName, filePresentValue=True, fileAbsentValue=False):
-    GC_Defaults[itemName] = trueValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], fileName)) else falseValue
+    GC_Defaults[itemName] = filePresentValue if os.path.isfile(os.path.join(GC_Defaults[GC_CONFIG_DIR], fileName)) else fileAbsentValue
 
   def _getCfgDirectory(itemName):
     return GC_Defaults[itemName]
@@ -376,10 +377,12 @@ def SetGlobalVariables():
   _getOldEnvVar(GC_DEVICE_MAX_RESULTS, u'GAM_DEVICE_MAX_RESULTS')
   _getOldEnvVar(GC_DRIVE_MAX_RESULTS, u'GAM_DRIVE_MAX_RESULTS')
   _getOldEnvVar(GC_USER_MAX_RESULTS, u'GAM_USER_MAX_RESULTS')
-  _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', trueValue=4, falseValue=0)
+  _getOldSignalFile(GC_DEBUG_LEVEL, u'debug.gam', filePresentValue=4, fileAbsentValue=0)
   _getOldSignalFile(GC_NO_VERIFY_SSL, u'noverifyssl.txt')
   _getOldSignalFile(GC_NO_BROWSER, u'nobrowser.txt')
-  _getOldSignalFile(GC_NO_CACHE, u'nocache.txt')
+#  _getOldSignalFile(GC_NO_CACHE, u'nocache.txt')
+#  _getOldSignalFile(GC_CACHE_DISCOVERY_ONLY, u'allcache.txt', filePresentValue=False, fileAbsentValue=True)
+  _getOldSignalFile(GC_NO_CACHE, u'allcache.txt', filePresentValue=False, fileAbsentValue=True)
   _getOldSignalFile(GC_NO_UPDATE_CHECK, u'noupdatecheck.txt')
 # Assign directories first
   for itemName in GC_VAR_INFO:
@@ -406,7 +409,12 @@ def SetGlobalVariables():
     ea_config.read(os.path.join(GC_Values[GC_CONFIG_DIR], FN_EXTRA_ARGS_TXT))
     GM_Globals[GM_EXTRA_ARGS_DICT].update(dict(ea_config.items(u'extra-args')))
   if GC_Values[GC_NO_CACHE]:
-    GC_Values[GC_CACHE_DIR] = None
+    GM_Globals[GM_CACHE_DIR] = None
+    GM_Globals[GM_CACHE_DISCOVERY_ONLY] = False
+  else:
+    GM_Globals[GM_CACHE_DIR] = GC_Values[GC_CACHE_DIR]
+#    GM_Globals[GM_CACHE_DISCOVERY_ONLY] = GC_Values[GC_CACHE_DISCOVERY_ONLY]
+    GM_Globals[GM_CACHE_DISCOVERY_ONLY] = False
   return True
 
 def doGAMCheckForUpdates(forceCheck=False):
@@ -485,7 +493,7 @@ def doGAMVersion(checkForArgs=True):
     doGAMCheckForUpdates(forceCheck=True)
 
 def handleOAuthTokenError(e, soft_errors):
-  if e.message in OAUTH2_TOKEN_ERRORS:
+  if e in OAUTH2_TOKEN_ERRORS or e.startswith(u'Invalid response'):
     if soft_errors:
       return None
     if not GM_Globals[GM_CURRENT_API_USER]:
@@ -517,10 +525,9 @@ def getSvcAcctCredentials(scopes, act_as):
 def waitOnFailure(n, retries, errMsg):
   wait_on_fail = min(2 ** n, 60) + float(random.randint(1, 1000)) / 1000
   if n > 3:
-    sys.stderr.write(u'Temp error {0}. Backing off {1} seconds...'.format(errMsg, int(wait_on_fail)))
+    sys.stderr.write(u'Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'.format(errMsg, int(wait_on_fail), n, retries))
+    sys.stderr.flush()
   time.sleep(wait_on_fail)
-  if n > 3:
-    sys.stderr.write(u'attempt {0}/{1}\n'.format(n+1, retries))
 
 def checkGAPIError(e, soft_errors=False, silent_errors=False, retryOnHttpError=False, service=None):
   try:
@@ -597,20 +604,24 @@ def callGAPI(service, function,
         waitOnFailure(n, retries, reason)
         continue
       if soft_errors:
-        stderrErrorMsg(u'{0}: {1} - {2}{3}'.format(http_status, message, reason, u': Giving up.\n' if n > 1 else u''))
+        stderrErrorMsg(u'{0}: {1} - {2}{3}'.format(http_status, message, reason, [u'', u': Giving up.'][n > 1]))
         return None
       systemErrorExit(int(http_status), u'{0}: {1} - {2}'.format(http_status, message, reason))
     except oauth2client.client.AccessTokenRefreshError as e:
-      handleOAuthTokenError(e, soft_errors or GAPI_SERVICE_NOT_AVAILABLE in throw_reasons)
+      handleOAuthTokenError(str(e), soft_errors or GAPI_SERVICE_NOT_AVAILABLE in throw_reasons)
       if GAPI_SERVICE_NOT_AVAILABLE in throw_reasons:
-        raise GAPI_serviceNotAvailable(e.message)
+        raise GAPI_serviceNotAvailable(str(e))
-      print u'ERROR: user %s: %s' % (GM_Globals[GM_CURRENT_API_USER], e)
+      stderrErrorMsg(u'User {0}: {1)'.format(GM_Globals[GM_CURRENT_API_USER], str(e)))
-      #entityUnknownWarning(u'User', GM_Globals[GM_CURRENT_API_USER], 0, 0)
       return None
     except httplib2.CertificateValidationUnsupported:
       noPythonSSLExit()
+    except ValueError as e:
+      if service._http.cache is not None:
+        service._http.cache = None
+        continue
+      systemErrorExit(4, str(e))
     except TypeError as e:
-      systemErrorExit(4, e)
+      systemErrorExit(4, str(e))
 
 def callGAPIpages(service, function, items=u'items',
                   page_message=None, message_attribute=None,
@@ -703,32 +714,51 @@ def getOauth2TxtStorageCredentials():
   except (KeyError, ValueError):
     return (storage, None)
 
-def getClientAPIversionHttpService(api):
+def getService(api, http):
+  api, version, api_version = getAPIVersion(api)
+  retries = 3
+  for n in range(1, retries+1):
+    try:
+      service = googleapiclient.discovery.build(api, version, http=http, cache_discovery=False)
+      if GM_Globals[GM_CACHE_DISCOVERY_ONLY]:
+        http.cache = None
+      return service
+    except httplib2.ServerNotFoundError as e:
+      systemErrorExit(4, str(e))
+    except httplib2.CertificateValidationUnsupported:
+      noPythonSSLExit()
+    except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
+      http.cache = None
+      if n != retries:
+        waitOnFailure(n, retries, str(e))
+        continue
+      systemErrorExit(17, str(e))
+    except (httplib.ResponseNotReady, httplib2.SSLHandshakeError, socket.error) as e:
+      if n != retries:
+        waitOnFailure(n, retries, str(e))
+        continue
+      systemErrorExit(3, str(e))
+    except googleapiclient.errors.UnknownApiNameOrVersion:
+      break
+  disc_file, discovery = readDiscoveryFile(api_version)
+  try:
+    service = googleapiclient.discovery.build_from_document(discovery, http=http)
+    if GM_Globals[GM_CACHE_DISCOVERY_ONLY]:
+      http.cache = None
+    return service
+  except (KeyError, ValueError):
+    invalidJSONExit(disc_file)
+
+def buildGAPIObject(api):
+  GM_Globals[GM_CURRENT_API_USER] = None
   storage, credentials = getOauth2TxtStorageCredentials()
   if not credentials or credentials.invalid:
     doRequestOAuth()
     credentials = storage.get()
   credentials.user_agent = GAM_INFO
-  api, version, api_version = getAPIVersion(api)
   http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL],
-                                             cache=GC_Values[GC_CACHE_DIR]))
+                                             cache=GM_Globals[GM_CACHE_DIR]))
-  try:
+  service = getService(api, http)
-    return (credentials, googleapiclient.discovery.build(api, version, http=http, cache_discovery=False))
-  except httplib2.ServerNotFoundError as e:
-    systemErrorExit(4, e)
-  except httplib2.CertificateValidationUnsupported:
-    noPythonSSLExit()
-  except googleapiclient.errors.UnknownApiNameOrVersion:
-    pass
-  disc_file, discovery = readDiscoveryFile(api_version)
-  try:
-    return (credentials, googleapiclient.discovery.build_from_document(discovery, http=http))
-  except (ValueError, KeyError):
-    invalidJSONExit(disc_file)
-
-def buildGAPIObject(api):
-  GM_Globals[GM_CURRENT_API_USER] = None
-  credentials, service = getClientAPIversionHttpService(api)
   if GC_Values[GC_DOMAIN]:
     if not GC_Values[GC_CUSTOMER_ID]:
       resp, result = service._http.request(u'https://www.googleapis.com/admin/directory/v1/users?domain={0}&maxResults=1&fields=users(customerId)'.format(GC_Values[GC_DOMAIN]))
@@ -768,24 +798,10 @@ def convertUserUIDtoEmailAddress(emailAddressOrUID):
     pass
   return normalizedEmailAddressOrUID
 
-def getSvcAcctAPIversionHttpService(api):
-  api, version, api_version = getAPIVersion(api)
-  http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL],
-                       cache=GC_Values[GC_CACHE_DIR])
-  try:
-    return (api_version, http, googleapiclient.discovery.build(api, version, http=http, cache_discovery=False))
-  except httplib2.ServerNotFoundError as e:
-    systemErrorExit(4, e)
-  except googleapiclient.errors.UnknownApiNameOrVersion:
-    pass
-  disc_file, discovery = readDiscoveryFile(api_version)
-  try:
-    return (api_version, http, googleapiclient.discovery.build_from_document(discovery, http=http))
-  except (ValueError, KeyError):
-    invalidJSONExit(disc_file)
-
 def buildGAPIServiceObject(api, act_as, use_scopes=None):
-  _, http, service = getSvcAcctAPIversionHttpService(api)
+  http = httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL],
+                       cache=GM_Globals[GM_CACHE_DIR])
+  service = getService(api, http)
   GM_Globals[GM_CURRENT_API_USER] = act_as
   GM_Globals[GM_CURRENT_API_SCOPES] = use_scopes or API_SCOPE_MAPPING[api]
   credentials = getSvcAcctCredentials(GM_Globals[GM_CURRENT_API_SCOPES], act_as)
@@ -794,8 +810,8 @@ def buildGAPIServiceObject(api, act_as, use_scopes=None):
   except httplib2.ServerNotFoundError as e:
     systemErrorExit(4, e)
   except oauth2client.client.AccessTokenRefreshError as e:
-    print u'ERROR user %s: %s' % (act_as, e)
+    stderrErrorMsg(u'User {0}: {1)'.format(GM_Globals[GM_CURRENT_API_USER], str(e)))
-    return handleOAuthTokenError(e, True)
+    return handleOAuthTokenError(str(e), True)
   return service
 
 def buildActivityGAPIObject(user):
@@ -3454,7 +3470,7 @@ def printPermission(permission):
   for key in permission:
     if key in [u'name', u'kind', u'etag', u'selfLink',]:
       continue
-    print u' %s: %s' % (key, permission[key])
+    print utils.convertUTF8(u' %s: %s' % (key, permission[key]))
 
 def showDriveFileACL(users):
   fileId = sys.argv[5]
@@ -6504,7 +6520,7 @@ def getCRMService(login_hint):
     noPythonSSLExit()
   credentials.user_agent = GAM_INFO
   http = credentials.authorize(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL],
-                                             cache=GC_Values[GC_CACHE_DIR]))
+                                             cache=None))
   return (googleapiclient.discovery.build(u'cloudresourcemanager', u'v1', http=http, cache_discovery=False), http)
 
 def doDelProjects(login_hint=None):
@@ -9827,7 +9843,7 @@ def doDeleteOAuth():
   try:
     credentials.revoke(httplib2.Http(disable_ssl_certificate_validation=GC_Values[GC_NO_VERIFY_SSL]))
   except oauth2client.client.TokenRevokeError as e:
-    stderrErrorMsg(e.message)
+    stderrErrorMsg(str(e))
     os.remove(GC_Values[GC_OAUTH2_TXT])
 
 class cmd_flags(object):
@@ -10200,7 +10216,7 @@ def ProcessGAMCommand(args):
           argv = shlex.split(line)
         except ValueError as e:
           sys.stderr.write(utils.convertUTF8(u'Command: >>>{0}<<<\n'.format(line.strip())))
-          sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, e.message))
+          sys.stderr.write(u'{0}{1}\n'.format(ERROR_PREFIX, str(e)))
           errors += 1
           continue
         if len(argv) > 0:

src/var.py

@@ -503,6 +503,10 @@ GM_MAP_ROLE_ID_TO_NAME = u'ri2n'
 GM_MAP_ROLE_NAME_TO_ID = u'rn2i'
 # Dictionary mapping User ID to Name
 GM_MAP_USER_ID_TO_NAME = u'ui2n'
+# GAM cache directory. If no_cache is True, this variable will be set to None
+GM_CACHE_DIR = u'gacd'
+# Reset GAM cache directory after discovery
+GM_CACHE_DISCOVERY_ONLY = u'gcdo'
 #
 GM_Globals = {
   GM_SYSEXITRC: 0,
@@ -519,6 +523,8 @@ GM_Globals = {
   GM_MAP_ROLE_ID_TO_NAME: None,
   GM_MAP_ROLE_NAME_TO_ID: None,
   GM_MAP_USER_ID_TO_NAME: None,
+  GM_CACHE_DIR: None,
+  GM_CACHE_DISCOVERY_ONLY: True,
   }
 #
 # Global variables defined by environment variables/signal files
@@ -532,6 +538,8 @@ GC_AUTO_BATCH_MIN = u'auto_batch_min'
 GC_BATCH_SIZE = u'batch_size'
 # GAM cache directory. If no_cache is specified, this variable will be set to None
 GC_CACHE_DIR = u'cache_dir'
+# GAM cache discovery only. If no_cache is False, only API discovery calls will be cached
+GC_CACHE_DISCOVERY_ONLY = u'cache_discovery_only'
 # Character set of batch, csv, data files
 GC_CHARSET = u'charset'
 # Path to client_secrets.json
@@ -581,6 +589,7 @@ GC_Defaults = {
   GC_AUTO_BATCH_MIN: 0,
   GC_BATCH_SIZE: 50,
   GC_CACHE_DIR: u'',
+  GC_CACHE_DISCOVERY_ONLY: True,
   GC_CHARSET: DEFAULT_CHARSET,
   GC_CLIENT_SECRETS_JSON: FN_CLIENT_SECRETS_JSON,
   GC_CONFIG_DIR: u'',
@@ -590,16 +599,16 @@ GC_Defaults = {
   GC_DOMAIN: u'',
   GC_DRIVE_DIR: u'',
   GC_DRIVE_MAX_RESULTS: 1000,
-  GC_NO_BROWSER: FALSE,
+  GC_NO_BROWSER: False,
-  GC_NO_CACHE: FALSE,
+  GC_NO_CACHE: False,
-  GC_NO_UPDATE_CHECK: FALSE,
+  GC_NO_UPDATE_CHECK: False,
-  GC_NO_VERIFY_SSL: FALSE,
+  GC_NO_VERIFY_SSL: False,
   GC_NUM_THREADS: 25,
   GC_OAUTH2_TXT: FN_OAUTH2_TXT,
   GC_OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON,
   GC_SECTION: u'',
   GC_SHOW_COUNTS_MIN: 0,
-  GC_SHOW_GETTINGS: TRUE,
+  GC_SHOW_GETTINGS: True,
   GC_SITE_DIR: u'',
   GC_USER_MAX_RESULTS: 500,
   }
@@ -623,6 +632,7 @@ GC_VAR_INFO = {
   GC_AUTO_BATCH_MIN: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (0, None)},
   GC_BATCH_SIZE: {GC_VAR_TYPE: GC_TYPE_INTEGER, GC_VAR_LIMITS: (1, 1000)},
   GC_CACHE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
+  GC_CACHE_DISCOVERY_ONLY: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_CHARSET: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_CLIENT_SECRETS_JSON: {GC_VAR_TYPE: GC_TYPE_FILE},
   GC_CONFIG_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},