version bump. document flush

* Use openFile/closeFile to avoid traps

This code isn't necessary, it will happen as part of close
f.flush()
os.fsync(f.fileno())

* Put flush/sync back

src/GamCommands.txt

@@ -40,6 +40,15 @@ If an item contains spaces, it should be surrounded by ".
 	tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen
 <FileFormat> ::=
 	csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
+<LabelColorHex> ::=
+        #000000|#076239|#0b804b|#149e60|#16a766|#1a764d|#1c4587|#285bac|
+        #2a9c68|#3c78d8|#3dc789|#41236d|#434343|#43d692|#44b984|#4a86e8|
+        #653e9b|#666666|#68dfa9|#6d9eeb|#822111|#83334c|#89d3b2|#8e63ce|
+        #999999|#a0eac9|#a46a21|#a479e2|#a4c2f4|#aa8831|#ac2b16|#b65775|
+        #b694e8|#b9e4d0|#c6f3de|#c9daf8|#cc3a21|#cccccc|#cf8933|#d0bcf1|
+        #d5ae49|#e07798|#e4d7f5|#e66550|#eaa041|#efa093|#efefef|#f2c960|
+        #f3f3f3|#f691b3|#f6c5be|#f7a7c0|#fad165|#fb4c2f|#fbc8d9|#fcda83|
+        #fcdee8|#fce8b3|#fef1d1|#ffad47|#ffbc6b|#ffd6a2|#ffe6c7|#ffffff
 <Language> ::=
 	ach|af|ag|ak|am|ar|az|be|bem|bg|bn|br|bs|ca|chr|ckb|co|crs|cs|cy|da|de|ee|el|en|en-gb|en-us|eo|es|es-419|et|eu|
 	fa|fi|fo|fr|fr-ca|fy|ga|gaa|gd|gl|gn|gu|ha|haw|he|hi|hr|ht|hu|hy|ia|id|ig|in|is|it|iw|ja|jw|
@@ -64,7 +73,7 @@ If an item contains spaces, it should be surrounded by ".
 	Google-Coordinate|
 	Google-Drive-storage|
 	Google-Vault|
-	101031
+	101001|101005|101031
 <SKUID> ::=
 	cloudidentity|identity|1010010001|
 	cloudidentitypremium|identitypremium|1010050001|
@@ -189,7 +198,7 @@ If an item contains spaces, it should be surrounded by ".
 <QueryDriveFile> ::= <String> See: https://developers.google.com/drive/v2/web/search-parameters
 <QueryGmail> ::= <String> See: https://support.google.com/mail/answer/7190
 <QueryGroup> ::= <String> See: https://developers.google.com/admin-sdk/directory/v1/guides/search-groups
-<QueryMobile> ::= <String> See: https://support.google.com/a/answer/1408863?hl=en#search
+<QueryMobile> ::= <String> See: https://support.google.com/a/answer/7549103
 <QueryPrinter> ::= <String> See: https://developers.google.com/cloud-print/docs/appInterfaces#search
 <QueryPrintJob> ::= <String> See: https://developers.google.com/cloud-print/docs/appInterfaces#parameters_3
 <QueryUser> ::= <String> See: https://developers.google.com/admin-sdk/directory/v1/guides/search-users
@@ -838,7 +847,7 @@ gam update customer <CustomerAttributes>*
 
 gam info customer
 
-<DataTransferService> ::= googledrive|gdrive|drive|"drive and docs"|calendar
+<DataTransferService> ::= googledrive|gdrive|drive|"drive and docs"|calendar|gplus|google+|googleplus
 
 gam create datatransfer|transfer <OldOwnerID> <DataTransferService> <NewOwnerID> (<ParameterKey> <ParameterValue>)*
 gam info datatransfer|transfer <TransferID>
@@ -1009,12 +1018,13 @@ gam create verify|verification <DomainName>
 gam update verify|verification <DomainName> cname|txt|text|site|file
 gam info verify|verification
 
-gam create course id|alias <CourseAlias> <CourseAttributes>*
+gam create course [id|alias <CourseAlias>] <CourseAttributes>*
 gam update course <CourseID> <CourseAttributes>+
 gam delete course <CourseID>
 gam info course <CourseID>
-gam print courses [todrive] [teacher <UserItem>] [student <UserItem>] [states <CourseStateList>] [alias|aliases] [delimiter <Character>]
-	[show all|students|teachers] [countsonly] [fields <CourseFieldNameList>] [skipfields <CourseFieldNameList>]
+gam print courses [todrive] [teacher <UserItem>] [student <UserItem>] [states <CourseStateList>]
+	[owneremail] [alias|aliases] [delimiter <Character>] [show all|students|teachers] [countsonly]
+	[fields <CourseFieldNameList>] [skipfields <CourseFieldNameList>]
 
 gam course <CourseID> add alias <CourseAlias>
 gam course <CourseID> delete alias <CourseAlias>
@@ -1145,7 +1155,9 @@ gam <UserTypeEntity> update user <UserAttributes>
 gam <UserTypeEntity> deprovision|deprov
 
 gam <UserTypeEntity> [create|add] label|labels <Name> [messagelistvisibility hide|show] [labellistvisibility hide|show|showifunread]
+	[backgroundcolor <LabelColorHex>] [textcolor <LabelColorHex>]
 gam <UserTypeEntity> update labelsettings <LabelName> [name <Name>] [messagelistvisibility hide|show] [labellistvisibility hide|show|showifunread]
+	[backgroundcolor <LabelColorHex>] [textcolor <LabelColorHex>]
 gam <UserTypeEntity> update label|labels [search <RegularExpression>] [replace <LabelReplacement>] [merge]
 gam <UserTypeEntity> delete|del label|labels <LabelName>|regex:<RegularExpression>|--ALL_LABELS--
 gam <UserTypeEntity> show labels|label [onlyuser] [showcounts]

src/gam-install.sh

@@ -135,9 +135,11 @@ if type(release) is list:
     break
 try:
   for asset in release['assets']:
-    if asset[sys.argv[1]].endswith('$gamfile'):
-      print(asset[sys.argv[1]])
+    if asset[attrib].endswith('$gamfile'):
+      print(asset[attrib])
       break
+  else:
+    print('ERROR: Attribute: {0} for $gamfile version {1} not found'.format(attrib, gamversion))
 except KeyError:
   print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
@@ -155,7 +157,15 @@ if (( $rc != 0 )); then
 fi
 
 browser_download_url=$(echo "$release_json" | $pycmd -c "$pycode" browser_download_url $gamversion)
+if [[ ${browser_download_url:0:5} = "ERROR" ]]; then
+  echo_red "${browser_download_url}"
+  exit
+fi
 name=$(echo "$release_json" | $pycmd -c "$pycode" name $gamversion)
+if [[ ${name:0:5} = "ERROR" ]]; then
+  echo_red "${name}"
+  exit
+fi
 # Temp dir for archive
 #temp_archive_dir=$(mktemp -d)
 temp_archive_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')

src/gam.py

@@ -30,6 +30,7 @@ import base64
 import codecs
 import csv
 import datetime
+import hashlib
 import httplib
 import json
 import mimetypes
@@ -40,6 +41,7 @@ import signal
 import socket
 import StringIO
 import uuid
+import zipfile
 
 import googleapiclient
 import googleapiclient.discovery
@@ -204,6 +206,16 @@ def getColor(color):
     return tg.group(0)
   systemErrorExit(2, u'A color must be a valid name or # and six hex characters (#012345); got {0}'.format(color))
 
+def getLabelColor(color):
+  color = color.lower().strip()
+  tg = COLORHEX_PATTERN.match(color)
+  if tg:
+    color = tg.group(0)
+    if color in LABEL_COLORS:
+      return color
+    systemErrorExit(2, u'A label color must be in the list: {0}; got {1}'.format(u'|'.join(LABEL_COLORS), color))
+  systemErrorExit(2, u'A label color must be # and six hex characters (#012345); got {0}'.format(color))
+
 def integerLimits(minVal, maxVal, item=u'integer'):
   if (minVal is not None) and (maxVal is not None):
     return u'{0} {1}<=x<={2}'.format(item, minVal, maxVal)
@@ -2491,11 +2503,9 @@ def doPrintCourses():
   skipFieldsList = []
   titles = [u'id',]
   csvRows = []
-  teacherId = None
-  studentId = None
+  ownerEmails = studentId = teacherId = None
   courseStates = []
-  showAliases = False
-  countsOnly = False
+  countsOnly = showAliases = False
   delimiter = u' '
   showMembers = u''
   i = 3
@@ -2535,13 +2545,24 @@ def doPrintCourses():
     elif myarg == u'skipfields':
       _processFieldsList(myarg, i, skipFieldsList)
       i += 2
+    elif myarg == u'owneremail':
+      ownerEmails = {}
+      cd = buildGAPIObject(u'directory')
+      i += 1
     else:
       systemErrorExit(2, '%s is not a valid argument for "gam print courses"' % sys.argv[i])
+  if ownerEmails is not None and fieldsList:
+    fieldsList.append(u'ownerId')
   fields = u'nextPageToken,courses({0})'.format(u','.join(set(fieldsList))) if fieldsList else None
   printGettingAllItems(u'Courses', None)
   page_message = u'Got %%num_items%% Courses...\n'
   all_courses = callGAPIpages(croom.courses(), u'list', u'courses', page_message=page_message, teacherId=teacherId, studentId=studentId, courseStates=courseStates, fields=fields)
   for course in all_courses:
+    if ownerEmails is not None:
+      ownerId = course[u'ownerId']
+      if ownerId not in ownerEmails:
+        ownerEmails[ownerId] = convertUIDtoEmailAddress(u'uid:%s' % ownerId, cd=cd)
+      course[u'ownerEmail'] = ownerEmails[ownerId]
     for field in skipFieldsList:
       course.pop(field, None)
     addRowTitlesToCSVfile(flatten_json(course), csvRows, titles)
@@ -5379,30 +5400,53 @@ def addSmime(users):
       callGAPI(gmail.users().settings().sendAs().smimeInfo(), u'setDefault', userId=u'me', sendAsEmail=sendAsEmail, id=result[u'id'])
     print u'Added S/MIME certificate for user %s sendas %s issued by %s' % (user, sendAsEmail, result[u'issuerCn'])
 
+def getLabelAttributes(i, myarg, body):
+  if myarg == u'labellistvisibility':
+    value = sys.argv[i+1].lower().replace(u'_', u'')
+    if value == u'hide':
+      body[u'labelListVisibility'] = u'labelHide'
+    elif value == u'show':
+      body[u'labelListVisibility'] = u'labelShow'
+    elif value == u'showifunread':
+      body[u'labelListVisibility'] = u'labelShowIfUnread'
+    else:
+      systemErrorExit(2, 'label_list_visibility must be one of hide, show, show_if_unread; got %s' % value)
+    i += 2
+  elif myarg == u'messagelistvisibility':
+    value = sys.argv[i+1].lower().replace(u'_', u'')
+    if value not in [u'hide', u'show']:
+      systemErrorExit(2, 'message_list_visibility must be show or hide; got %s' % value)
+    body[u'messageListVisibility'] = value
+    i += 2
+  elif myarg == u'backgroundcolor':
+    body.setdefault(u'color', {})
+    body[u'color']['backgroundColor'] = getLabelColor(sys.argv[i+1])
+    i += 2
+  elif myarg == u'textcolor':
+    body.setdefault(u'color', {})
+    body[u'color']['textColor'] = getLabelColor(sys.argv[i+1])
+    i += 2
+  else:
+    systemErrorExit(2, '%s is not a valid argument for this command.' % myarg)
+  return i
+
+def checkLabelColor(body):
+  if u'color' not in body:
+    return
+  if u'backgroundColor' in body[u'color']:
+    if u'textColor' in body[u'color']:
+      return
+    systemErrorExit(2, 'textcolor <LabelColorHex> is required.')
+  systemErrorExit(2, 'backgroundcolor <LabelColorHex> is required.')
+
 def doLabel(users, i):
   label = sys.argv[i]
   i += 1
   body = {u'name': label}
   while i < len(sys.argv):
     myarg = sys.argv[i].lower().replace(u'_', u'')
-    if myarg == u'labellistvisibility':
-      value = sys.argv[i+1].lower().replace(u'_', u'')
-      if value == u'hide':
-        body[u'labelListVisibility'] = u'labelHide'
-      elif value == u'show':
-        body[u'labelListVisibility'] = u'labelShow'
-      elif value == u'showifunread':
-        body[u'labelListVisibility'] = u'labelShowIfUnread'
-      else:
-        systemErrorExit(2, 'label_list_visibility must be one of hide, show, show_if_unread; got %s' % sys.argv[i+1])
-      i += 2
-    elif myarg == u'messagelistvisibility':
-      body[u'messageListVisibility'] = sys.argv[i+1].lower().replace(u'_', u'')
-      if body[u'messageListVisibility'] not in [u'hide', u'show']:
-        systemErrorExit(2, 'message_list_visibility must be show or hide; got %s' % sys.argv[i+1])
-      i += 2
-    else:
-      systemErrorExit(2, '%s is not a valid argument for this command.' % sys.argv[i])
+    i = getLabelAttributes(i, myarg, body)
+  checkLabelColor(body)
   i = 0
   count = len(users)
   for user in users:
@@ -5701,24 +5745,9 @@ def updateLabels(users):
     if myarg == u'name':
       body[u'name'] = sys.argv[i+1]
       i += 2
-    elif myarg == u'labellistvisibility':
-      value = sys.argv[i+1].lower().replace(u'_', u'')
-      if value == u'hide':
-        body[u'labelListVisibility'] = u'labelHide'
-      elif value == u'show':
-        body[u'labelListVisibility'] = u'labelShow'
-      elif value == u'showifunread':
-        body[u'labelListVisibility'] = u'labelShowIfUnread'
-      else:
-        systemErrorExit(2, 'label_list_visibility must be hide, show, show_if_unread; got %s' % sys.argv[i+1])
-      i += 2
-    elif myarg == u'messagelistvisibility':
-      body[u'messageListVisibility'] = sys.argv[i+1].lower().replace(u'_', u'')
-      if body[u'messageListVisibility'] not in [u'hide', u'show']:
-        systemErrorExit(2, 'message_list_visibility must be show or hide; got %s' % sys.argv[i+1])
-      i += 2
     else:
-      systemErrorExit(2, '%s is not a valid argument for "gam <users> update labels"' % sys.argv[i])
+      i = getLabelAttributes(i, myarg, body)
+  checkLabelColor(body)
   for user in users:
     user, gmail = buildGmailGAPIObject(user)
     if not gmail:
@@ -7448,6 +7477,185 @@ def doCreateVaultMatter():
     print u' adding collaborator %s' % collaborator[u'email']
     callGAPI(v.matters(), u'addPermissions', matterId=matterId, body={u'matterPermission': {u'role': u'COLLABORATOR', u'accountId': collaborator[u'id']}})
 
+def doCreateVaultExport():
+  v = buildGAPIObject(u'vault')
+  allowed_corpuses = v._rootDesc[u'schemas'][u'Query'][u'properties'][u'corpus'][u'enum']
+  try:
+    allowed_corpuses.remove(u'CORPUS_TYPE_UNSPECIFIED')
+  except ValueError:
+    pass
+  allowed_scopes = v._rootDesc[u'schemas'][u'Query'][u'properties'][u'dataScope'][u'enum']
+  try:
+    allowed_scopes.remove(u'DATA_SCOPE_UNSPECIFIED')
+  except ValueError:
+    pass
+  allowed_formats = [u'MBOX', u'PST']
+  export_format = u'MBOX'
+  matterId = None
+  body = {u'query': {u'dataScope': u'ALL_DATA'}, u'exportOptions': {}}
+  i = 3
+  while i < len(sys.argv):
+    myarg = sys.argv[i].lower().replace(u'_', u'')
+    if myarg == u'matter':
+      matterId = getMatterItem(v, sys.argv[i+1])
+      body[u'matterId'] = matterId
+      i += 2
+    elif myarg == u'name':
+      body[u'name'] = sys.argv[i+1]
+      i += 2
+    elif myarg == u'corpus':
+      body[u'query'][u'corpus'] = sys.argv[i+1].upper()
+      if body[u'query'][u'corpus'] not in allowed_corpuses:
+        systemErrorExit(3, 'corpus must be one of %s. Got %s' % (u', '.join(allowed_corpuses), sys.argv[i+1]))
+      i += 2
+    elif myarg == u'scope':
+      body[u'query'][u'dataScope'] = sys.argv[i+1].upper()
+      if body[u'query']['dataScope'] not in allowed_scopes:
+        systemErrorExit(3, 'scope must be one of %s. Got %s' % (u', '.join(allowed_scopes), sys.argv[i+1]))
+      i += 2
+    elif myarg in [u'account', u'accounts']:
+      body[u'query'][u'searchMethod'] = u'ACCOUNT'
+      body[u'query'][u'accountInfo'] = {u'emails': sys.argv[i+1].split(u',')}
+      i += 2
+    elif myarg in [u'ou', u'orgunit']:
+      body[u'query'][u'searchMethod'] = u'ORG_UNIT'
+      orgUnitId = getOrgUnitId(sys.argv[i+1])[1]
+      body[u'query'][u'orgUnitInfo'] = {u'orgUnitId': orgUnitId}
+      i += 2
+    elif myarg in [u'teamdrive', u'teamdrives']:
+      body[u'query'][u'searchMethod'] = u'TEAM_DRIVE'
+      body[u'query'][u'teamDriveInfo'] = {u'teamDriveIds': sys.argv[i+1].split(u',')}
+      i += 2
+    elif myarg in [u'room', u'rooms']:
+      body[u'query'][u'searchMethod'] = u'ROOM'
+      body[u'query'][u'hangoutsChatInfo'] = {u'roomId': sys.argv[i+1].split(u',')}
+      i += 2
+    elif myarg in [u'everyone']:
+      body[u'query'][u'searchMethod'] = u'ENTIRE_ORG'
+      i += 2
+    elif myarg in [u'terms']:
+      body[u'query'][u'terms'] = sys.argv[i+1]
+      i += 2
+    elif myarg in [u'start']:
+      body[u'query'][u'startTime'] = getDateZeroTimeOrFullTime(sys.argv[i+1])
+      i += 2
+    elif myarg in [u'end']:
+      body[u'query'][u'endTime'] = getDateZeroTimeOrFullTime(sys.argv[i+1])
+      i += 2
+    elif myarg in [u'timezone']:
+      body[u'query'][u'timeZone'] = sys.argv[i+1]
+      i += 2
+    elif myarg in [u'excludedrafts']:
+      body[u'query'][u'mailOptions'] = {u'excludeDrafts': getBoolean(sys.argv[i+1], myarg)}
+      i += 2
+    elif myarg in [u'driveversiondate']:
+      body[u'query'].setdefault(u'driveOptions', {})[u'versionDate'] = getDateZeroTimeOrFullTime(sys.argv[i+1])
+      i += 2
+    elif myarg in [u'includeteamdrives']:
+      body[u'query'].setdefault(u'driveOptions', {})[u'includeTeamDrives'] = getBoolean(sys.argv[i+1], myarg)
+      i += 2
+    elif myarg in [u'includerooms']:
+      body[u'query'][u'hangoutsChatOptions'] = {u'includeRooms': getBoolean(sys.argv[i+1], myarg)}
+      i += 2
+    elif myarg in [u'format']:
+      export_format = sys.argv[i+1].upper()
+      if export_format not in allowed_formats:
+        print u'ERROR: export format can be one of %s, got %s' % (u', '.join(allowed_formats), export_format)
+        sys.exit(3)
+      i += 2
+    elif myarg in [u'includeaccessinfo']:
+      body[u'exportOptions'].setdefault(u'driveOptions', {})[u'includeAccessInfo'] = getBoolean(sys.argv[i+1], myarg)
+      i += 2
+    else:
+      print u'ERROR %s is not a valid argument for "gam create export".' % sys.argv[i]
+      sys.exit(3)
+  if not matterId:
+    print u'ERROR: you must specify a matterId.'
+    sys.exit(3)
+  if u'corpus' not in body[u'query']:
+    print u'ERROR: you must specify a corpus type. Choose one of %s' % u', '.join(allowed_corpuses)
+    sys.exit(3)
+  if u'name' not in body:
+    body[u'name'] = u'GAM %s export - %s' % (body[u'query'][u'corpus'], datetime.datetime.now())
+  options_field = None
+  if body[u'query'][u'corpus'] == u'MAIL':
+    options_field = u'mailOptions'
+  elif body[u'query'][u'corpus'] == u'GROUPS':
+    options_field = u'groupsOptions'
+  elif body[u'query'][u'corpus'] == u'HANGOUTS_CHAT':
+    options_field = u'hangoutsChatOptions'
+  if options_field:
+    body[u'exportOptions'][options_field] = {u'exportFormat': export_format}
+  results = callGAPI(v.matters().exports(), u'create', matterId=matterId, body=body)
+  print_json(None, results)
+
+def doGetVaultExport():
+  v = buildGAPIObject(u'vault')
+  matterId = getMatterItem(v, sys.argv[3])
+  exportId = convertExportNameToID(v, sys.argv[4], matterId)
+  export = callGAPI(v.matters().exports(), u'get', matterId=matterId, exportId=exportId)
+  print_json(None, export)
+
+def doDownloadVaultExport():
+  verifyFiles = True
+  extractFiles = True
+  v = buildGAPIObject(u'vault')
+  s = buildGAPIObject(u'storage')
+  matterId = getMatterItem(v, sys.argv[3])
+  exportId = convertExportNameToID(v, sys.argv[4], matterId)
+  targetFolder = GC_Values[GC_DRIVE_DIR]
+  export = callGAPI(v.matters().exports(), u'get', matterId=matterId, exportId=exportId)
+  for s_file in export[u'cloudStorageSink']['files']:
+    bucket = s_file['bucketName']
+    s_object = s_file['objectName']
+    filename = os.path.join(targetFolder, s_object.replace(u'/', u'-'))
+    print u'saving to %s' % filename
+    request = s.objects().get_media(bucket=bucket, object=s_object)
+    f = openFile(filename, 'wb')
+    downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+    done = False
+    while not done:
+      status, done = downloader.next_chunk()
+      sys.stdout.write(u' Downloaded: {0:>7.2%}\r'.format(status.progress()))
+      sys.stdout.flush()
+    sys.stdout.write(u'\n Download complete. Flushing to disk...\n')
+    # Necessary to make sure file is flushed by both Python and OS
+    # https://stackoverflow.com/a/13762137/1503886
+    f.flush()
+    os.fsync(f.fileno())
+    closeFile(f)
+    f = openFile(filename, 'rb')
+    if verifyFiles:
+      expected_hash = s_file['md5Hash']
+      sys.stdout.write(u' Verifying file hash is %s...' % expected_hash)
+      sys.stdout.flush()
+      hash_md5 = hashlib.md5()
+      for chunk in iter(lambda: f.read(4096), b""):
+        hash_md5.update(chunk)
+      actual_hash = hash_md5.hexdigest()
+      if actual_hash == expected_hash:
+        print u'VERIFIED'
+      else:
+        print u'ERROR: actual hash was %s. Exiting on corrupt file.' % actual_hash
+        sys.exit(6)
+    closeFile(f)
+    if extractFiles and re.search(r'\.zip$', filename):
+      extract_nested_zip(filename, targetFolder)
+
+def extract_nested_zip(zippedFile, toFolder, spacing=u' '):
+  """ Extract a zip file including any nested zip files
+      Delete the zip file(s) after extraction
+  """
+  print u'%sextracting %s' % (spacing, zippedFile)
+  with zipfile.ZipFile(zippedFile, 'r') as zfile:
+    inner_files = zfile.infolist()
+    for inner_file in inner_files:
+      print u'%s %s' % (spacing, inner_file.filename)
+      zfile.extract(inner_file)
+      if re.search(r'\.zip$', inner_file.filename):
+        extract_nested_zip(inner_file.filename, toFolder, spacing=spacing+u' ')
+  os.remove(zippedFile)
+
 def doCreateVaultHold():
   v = buildGAPIObject(u'vault')
   allowed_corpuses = v._rootDesc[u'schemas'][u'Hold'][u'properties'][u'corpus'][u'enum']
@@ -7485,9 +7693,7 @@ def doCreateVaultHold():
       end_time = getDateZeroTimeOrFullTime(sys.argv[i+1])
       i += 2
     elif myarg == u'matter':
-      matterId = convertMatterNameToID(v, sys.argv[i+1])
-      if not matterId:
-        systemErrorExit(4, 'could not find matter %s' % sys.argv[i+1])
+      matterId = getMatterItem(v, sys.argv[i+1])
       i += 2
     else:
       systemErrorExit(3, '%s is not a valid argument to "gam create hold"' % sys.argv[i])
@@ -7528,12 +7734,8 @@ def doDeleteVaultHold():
   while i < len(sys.argv):
     myarg = sys.argv[i].lower().replace(u'_', u'')
     if myarg == u'matter':
-      matterId = convertMatterNameToID(v, sys.argv[i+1])
-      if not matterId:
-        systemErrorExit(4, 'could not find matter %s' % sys.argv[i+1])
+      matterId = getMatterItem(v, sys.argv[i+1])
       holdId = convertHoldNameToID(v, hold, matterId)
-      if not holdId:
-        systemErrorExit(4, 'could not find hold %s in matter %s' % (sys.argv[3], matterId))
       i += 2
     else:
       systemErrorExit(3, '%s is not a valid argument to "gam delete hold"' % myarg)
@@ -7550,12 +7752,8 @@ def doGetVaultHoldInfo():
   while i < len(sys.argv):
     myarg = sys.argv[i].lower().replace(u'_', u'')
     if myarg == u'matter':
-      matterId = convertMatterNameToID(v, sys.argv[i+1])
-      if not matterId:
-        systemErrorExit(4, 'could not find matter %s' % sys.argv[i+1])
+      matterId = getMatterItem(v, sys.argv[i+1])
       holdId = convertHoldNameToID(v, hold, matterId)
-      if not holdId:
-        systemErrorExit(4, 'could not find hold %s in matter %s' % (hold, matterId))
       i += 2
     else:
       systemErrorExit(3, '%s is not a valid argument for "gam info hold"' % myarg)
@@ -7573,6 +7771,16 @@ def doGetVaultHoldInfo():
     results[u'orgUnit'][u'orgUnitPath'] = doGetOrgInfo(results[u'orgUnit'][u'orgUnitId'], return_attrib=u'orgUnitPath')
   print_json(None, results)
 
+def convertExportNameToID(v, nameOrID, matterId):
+  nameOrID = nameOrID.lower()
+  if nameOrID[:4] == u'uid:':
+    return nameOrID[4:]
+  exports = callGAPIpages(v.matters().exports(), u'list', u'exports', matterId=matterId, fields=u'exports(id,name),nextPageToken')
+  for export in exports:
+    if export[u'name'].lower() == nameOrID:
+      return export[u'id']
+  systemErrorExit(4, 'could not find export name %s in matter %s' % (nameOrID, matterId))
+
 def convertHoldNameToID(v, nameOrID, matterId):
   nameOrID = nameOrID.lower()
   if nameOrID[:4] == u'uid:':
@@ -7581,7 +7789,7 @@ def convertHoldNameToID(v, nameOrID, matterId):
   for hold in holds:
     if hold[u'name'].lower() == nameOrID:
       return hold[u'holdId']
-  return None
+  systemErrorExit(4, 'could not find hold name %s in matter %s' % (nameOrID, matterId))
 
 def convertMatterNameToID(v, nameOrID):
   nameOrID = nameOrID.lower()
@@ -7593,6 +7801,12 @@ def convertMatterNameToID(v, nameOrID):
       return matter[u'matterId']
   return None
 
+def getMatterItem(v, nameOrID):
+  matterId = convertMatterNameToID(v, nameOrID)
+  if not matterId:
+    systemErrorExit(4, 'could not find matter %s' % nameOrID)
+  return matterId
+
 def doUpdateVaultHold():
   v = buildGAPIObject(u'vault')
   hold = sys.argv[3]
@@ -7607,12 +7821,8 @@ def doUpdateVaultHold():
   while i < len(sys.argv):
     myarg = sys.argv[i].lower().replace(u'_', u'')
     if myarg == u'matter':
-      matterId = convertMatterNameToID(v, sys.argv[i+1])
-      if not matterId:
-        systemErrorExit(4, 'could not find matter %s' % sys.argv[i+1])
+      matterId = getMatterItem(v, sys.argv[i+1])
       holdId = convertHoldNameToID(v, hold, matterId)
-      if not holdId:
-        systemErrorExit(4, 'could not find hold %s in matter %s' % (hold, matterId))
       i += 2
     elif myarg == u'query':
       query = sys.argv[i+1]
@@ -7674,9 +7884,7 @@ def doUpdateVaultHold():
 
 def doUpdateVaultMatter(action=None):
   v = buildGAPIObject(u'vault')
-  matterId = convertMatterNameToID(v, sys.argv[3])
-  if not matterId:
-    systemErrorExit(4, 'failed to lookup matter named %s' % sys.argv[3])
+  matterId = getMatterItem(v, sys.argv[3])
   body = {}
   action_kwargs = {u'body': {}}
   add_collaborators = []
@@ -7730,7 +7938,7 @@ def doUpdateVaultMatter(action=None):
 
 def doGetVaultMatterInfo():
   v = buildGAPIObject(u'vault')
-  matterId = convertMatterNameToID(v, sys.argv[3])
+  matterId = getMatterItem(v, sys.argv[3])
   result = callGAPI(v.matters(), u'get', matterId=matterId, view=u'FULL')
   if u'matterPermissions' in result:
     cd = buildGAPIObject(u'directory')
@@ -11569,7 +11777,7 @@ def getUsersToModify(entity_type=None, entity=None, silent=False, member_type=No
                             query=query, maxResults=GC_Values[GC_USER_MAX_RESULTS])
     ou = ou.lower()
     for member in members:
-      if (ou == member[u'orgUnitPath'].lower()) and not (checkNotSuspended and member[u'suspended']):
+      if (ou == member.get(u'orgUnitPath', u'').lower()) and not (checkNotSuspended and member[u'suspended']):
         users.append(member[u'primaryEmail'])
     if not silent:
       sys.stderr.write(u"%s Users are directly in the OU.\n" % len(users))
@@ -11902,6 +12110,9 @@ OAUTH2_SCOPES = [
   {u'name': u'Vault Matters and Holds API',
    u'subscopes': [u'readonly'],
    u'scopes': u'https://www.googleapis.com/auth/ediscovery'},
+  {u'name': u'Cloud Storage (Vault Export - read only)',
+   u'subscopes': [],
+   u'scopes': u'https://www.googleapis.com/auth/devstorage.read_only'},
   ]
 
 OAUTH2_MENU = u'''
@@ -12235,6 +12446,8 @@ def ProcessGAMCommand(args):
         doCreateVaultMatter()
       elif argument in [u'hold', u'vaulthold']:
         doCreateVaultHold()
+      elif argument in [u'export', u'vaultexport']:
+        doCreateVaultExport()
       elif argument in [u'building']:
         doCreateBuilding()
       elif argument in [u'feature']:
@@ -12337,6 +12550,8 @@ def ProcessGAMCommand(args):
         doGetVaultMatterInfo()
       elif argument in [u'hold', u'vaulthold']:
         doGetVaultHoldInfo()
+      elif argument in [u'export', u'vaultexport']:
+        doGetVaultExport()
       elif argument in [u'building']:
         doGetBuildingInfo()
       else:
@@ -12565,6 +12780,13 @@ def ProcessGAMCommand(args):
       else:
         systemErrorExit(2, '%s is not a valid argument for "gam course"' % argument)
       sys.exit(0)
+    elif command == u'download':
+      argument = sys.argv[2].lower()
+      if argument in [u'export', u'vaultexport']:
+        doDownloadVaultExport()
+      else:
+        systemErrorExit(2, '%s is not a valid argument for "gam download"' % argument)
+      sys.exit(0)
     users = getUsersToModify()
     command = sys.argv[3].lower()
     if command == u'print' and len(sys.argv) == 4:
@@ -12701,7 +12923,7 @@ def ProcessGAMCommand(args):
         deleteDelegate(users)
       elif delWhat == u'calendar':
         deleteCalendar(users)
-      elif delWhat == u'label':
+      elif delWhat  in [u'labels', u'label']:
         doDeleteLabel(users)
       elif delWhat in [u'message', u'messages']:
         runCmdForUsers(doProcessMessagesOrThreads, users, default_to_batch=True, function=u'delete', unit=u'messages')

src/var.py

@@ -4,7 +4,7 @@ import platform
 import re
 
 gam_author = u'Jay Lee <jay0lee@gmail.com>'
-gam_version = u'4.50'
+gam_version = u'4.60'
 gam_license = u'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = u'https://git.io/gam'
@@ -110,6 +110,7 @@ API_VER_MAPPING = {
   u'reports': u'reports_v1',
   u'reseller': u'v1',
   u'siteVerification': u'v1',
+  u'storage': u'v1',
   u'urlshortener': u'v1',
   u'vault': u'v1',
   }
@@ -156,6 +157,9 @@ SERVICE_NAME_CHOICES_MAP = {
   u'googledrive': u'Drive and Docs',
   u'gdrive': u'Drive and Docs',
   u'calendar': u'Calendar',
+  u'gplus': u'Google+',
+  u'google+': u'Google+',
+  u'googleplus': u'Google+',
   }
 
 PRINTJOB_ASCENDINGORDER_MAP = {
@@ -1017,6 +1021,19 @@ WEBCOLOR_MAP = {
     u'yellow': u'#ffff00',
     u'yellowgreen': u'#9acd32',
 }
+
+# Gmail label colors
+LABEL_COLORS = [
+  u'#000000', u'#076239', u'#0b804b', u'#149e60', u'#16a766', u'#1a764d', u'#1c4587', u'#285bac',
+  u'#2a9c68', u'#3c78d8', u'#3dc789', u'#41236d', u'#434343', u'#43d692', u'#44b984', u'#4a86e8',
+  u'#653e9b', u'#666666', u'#68dfa9', u'#6d9eeb', u'#822111', u'#83334c', u'#89d3b2', u'#8e63ce',
+  u'#999999', u'#a0eac9', u'#a46a21', u'#a479e2', u'#a4c2f4', u'#aa8831', u'#ac2b16', u'#b65775',
+  u'#b694e8', u'#b9e4d0', u'#c6f3de', u'#c9daf8', u'#cc3a21', u'#cccccc', u'#cf8933', u'#d0bcf1',
+  u'#d5ae49', u'#e07798', u'#e4d7f5', u'#e66550', u'#eaa041', u'#efa093', u'#efefef', u'#f2c960',
+  u'#f3f3f3', u'#f691b3', u'#f6c5be', u'#f7a7c0', u'#fad165', u'#fb4c2f', u'#fbc8d9', u'#fcda83',
+  u'#fcdee8', u'#fce8b3', u'#fef1d1', u'#ffad47', u'#ffbc6b', u'#ffd6a2', u'#ffe6c7', u'#ffffff',
+  ]
+
 # Valid language codes
 LANGUAGE_CODES_MAP = {
   u'ach': u'ach', u'af': u'af', u'ag': u'ga', u'ak': u'ak', u'am': u'am', u'ar': u'ar', u'az': u'az', #Luo, Afrikaans, Irish, Akan, Amharic, Arabica, Azerbaijani