Merge branch 'master' of https://github.com/jay0lee/GAM

src/gam/__init__.py

@@ -1093,7 +1093,7 @@ def buildAlertCenterGAPIObject(user):
 
 def buildActivityGAPIObject(user):
     userEmail = convertUIDtoEmailAddress(user)
-    return (userEmail, buildGAPIServiceObject('appsactivity', userEmail))
+    return (userEmail, buildGAPIServiceObject('driveactivity', userEmail))
 
 
 def buildDriveGAPIObject(user):
@@ -2720,7 +2720,7 @@ def deletePhoto(users):
     for user in users:
         i += 1
         print(f'Deleting photo for {user}{currentCount(i, count)}')
-        gapi.call(cd.users().photos(), 'delete', userKey=user)
+        gapi.call(cd.users().photos(), 'delete', userKey=user, soft_errors=True)
 
 
 def printDriveSettings(users):
@@ -2788,23 +2788,58 @@ def getTeamDriveThemes(users):
 
 
 def printDriveActivity(users):
-    drive_ancestorId = 'root'
-    drive_fileId = None
+    def _get_user_info(user_id):
+        if user_id.startswith('people/'):
+            user_id = user_id[7:]
+        entry = user_info.get(user_id)
+        if entry is None:
+            result = gapi.call(cd.users(), 'get',
+                              soft_errors=True,
+                              userKey=user_id, fields='primaryEmail,name.fullName')
+            if result:
+                entry = (result['primaryEmail'], result['name']['fullName'])
+            else:
+                entry = (f'uid:{user_id}', 'Unknown')
+            user_info[user_id] = entry
+        return entry
+
+    def _update_known_users(structure):
+        if isinstance(structure, list):
+          for v in structure:
+              if isinstance(v, (dict, list)):
+                  _update_known_users(v)
+        elif isinstance(structure, dict):
+          for k, v in sorted(iter(structure.items())):
+              if k != 'knownUser':
+                  if isinstance(v, (dict, list)):
+                      _update_known_users(v)
+              else:
+                  entry = _get_user_info(v['personName'])
+                  v['emailAddress'] = entry[0]
+                  v['personName'] = entry[1]
+                  break
+
+    cd = buildGAPIObject('directory')
+    drive_key = 'ancestorName'
+    drive_fileId = 'root'
+    user_info = {}
     todrive = False
     titles = [
-        'user.name', 'user.permissionId', 'target.id', 'target.name',
-        'target.mimeType'
+        'user.name', 'user.emailAddress', 'target.id', 'target.name',
+        'target.mimeType', 'eventTime'
     ]
+    sort_titles = titles[:]
     csvRows = []
     i = 5
     while i < len(sys.argv):
         activity_object = sys.argv[i].lower().replace('_', '')
         if activity_object == 'fileid':
             drive_fileId = sys.argv[i + 1]
-            drive_ancestorId = None
+            drive_key = 'itemName'
             i += 2
         elif activity_object == 'folderid':
-            drive_ancestorId = sys.argv[i + 1]
+            drive_fileId = sys.argv[i + 1]
+            drive_key = 'ancestorName'
             i += 2
         elif activity_object == 'todrive':
             todrive = True
@@ -2812,23 +2847,57 @@ def printDriveActivity(users):
         else:
             controlflow.invalid_argument_exit(sys.argv[i],
                                               'gam <users> show driveactivity')
+
     for user in users:
         user, activity = buildActivityGAPIObject(user)
         if not activity:
             continue
+        page_token = None
+        total_items = 0
+        kwargs = {drive_key: f'items/{drive_fileId}',
+                  'pageToken': page_token}
         page_message = gapi.got_total_items_msg(f'Activities for {user}', '')
-        feed = gapi.get_all_pages(activity.activities(),
-                                  'list',
-                                  'activities',
-                                  page_message=page_message,
-                                  source='drive.google.com',
-                                  userId='me',
-                                  drive_ancestorId=drive_ancestorId,
-                                  groupingStrategy='none',
-                                  drive_fileId=drive_fileId)
-        for item in feed:
-            display.add_row_titles_to_csv_file(
-                utils.flatten_json(item['combinedEvent']), csvRows, titles)
+        while True:
+          feed = gapi.call(activity.activity(), 'query', body=kwargs)
+          page_token, total_items = gapi.process_page(feed, 'activities', None, total_items, page_message, None)
+          kwargs['pageToken'] = page_token
+          if feed:
+              for activity_event in feed.get('activities', []):
+                  event_row = {}
+                  actors = activity_event.get('actors', [])
+                  if actors:
+                      userId = actors[0].get('user', {}).get('knownUser', {}).get('personName', '')
+                      if not userId:
+                          userId = actors[0].get('impersonation', {}).get('impersonatedUser', {}).get('knownUser', {}).get('personName', '')
+                      if userId:
+                          entry = _get_user_info(userId)
+                          event_row['user.name'] = entry[1]
+                          event_row['user.emailAddress'] = entry[0]
+                  targets = activity_event.get('targets', [])
+                  if targets:
+                      driveItem = targets[0].get('driveItem')
+                      if driveItem:
+                          event_row['target.id'] = driveItem['name'][6:]
+                          event_row['target.name'] = driveItem['title']
+                          event_row['target.mimeType'] = driveItem['mimeType']
+                      else:
+                          teamDrive = targets[0].get('teamDrive')
+                          if teamDrive:
+                              event_row['target.id'] = teamDrive['name'][11:]
+                              event_row['target.name'] = teamDrive['title']
+                  if 'timestamp' in activity_event:
+                      event_row['eventTime'] = activity_event.pop('timestamp')
+                  elif 'timeRange' in activity_event:
+                      timeRange = activity_event.pop('timeRange')
+                      event_row['eventTime'] = f'{timeRange["startTime"]}-{timeRange["endTime"]}'
+                  _update_known_users(activity_event)
+                  display.add_row_titles_to_csv_file(
+                      utils.flatten_json(activity_event, flattened=event_row), csvRows, titles)
+              del feed
+          if not page_token:
+              gapi.finalize_page_message(page_message)
+              break
+    display.sort_csv_titles(sort_titles, titles)
     display.write_csv_file(csvRows, titles, 'Drive Activity', todrive)
 
 
@@ -3570,6 +3639,10 @@ def getDriveFileAttribute(i, body, parameters, myarg, update=False):
     return i
 
 
+def has_multiple_parents(body):
+    return len(body.get('parents', [])) > 1
+
+
 def doUpdateDriveFile(users):
     fileIdSelection = {'fileIds': [], 'query': None}
     media_body = None
@@ -3618,6 +3691,9 @@ def doUpdateDriveFile(users):
             body.setdefault('parents', [])
             for a_parent in more_parents:
                 body['parents'].append({'id': a_parent})
+        if has_multiple_parents(body):
+            sys.stderr.write(f"Multiple parents ({len(body['parents'])}) specified for {user}, only one is allowed.\n")
+            continue
         if fileIdSelection['query']:
             fileIdSelection['fileIds'] = doDriveSearch(
                 drive, query=fileIdSelection['query'])
@@ -3705,6 +3781,9 @@ def createDriveFile(users):
             body.setdefault('parents', [])
             for a_parent in more_parents:
                 body['parents'].append({'id': a_parent})
+        if has_multiple_parents(body):
+            sys.stderr.write(f"Multiple parents ({len(body['parents'])}) specified for {user}, only one is allowed.\n")
+            continue
         if parameters[DFA_LOCALFILEPATH]:
             media_body = googleapiclient.http.MediaFileUpload(
                 parameters[DFA_LOCALFILEPATH],

src/gam/gapi/__init__.py

@@ -218,6 +218,61 @@ def got_total_items_first_last_msg(items):
     return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}' + '\n'
 
 
+def process_page(page, items, all_items, total_items, page_message, message_attribute):
+    """Process one page of a Google service function response.
+
+  Append a list of items to the aggregate list of items
+
+  Args:
+    page: list of items
+    items: see get_all_pages
+    all_items: aggregate list of items
+    total_items: length of all_items
+    page_message: see get_all_pages
+    message_attribute: get_all_pages
+  Returns:
+    The page token and total number of items
+  """
+    if page:
+        page_token = page.get('nextPageToken')
+        page_items = page.get(items, [])
+        num_page_items = len(page_items)
+        total_items += num_page_items
+        if all_items is not None:
+            all_items.extend(page_items)
+    else:
+        page_token = None
+        num_page_items = 0
+
+    # Show a paging message to the user that indicates paging progress
+    if page_message:
+        show_message = page_message.replace(TOTAL_ITEMS_MARKER,
+                                            str(total_items))
+        if message_attribute:
+            first_item = page_items[0] if num_page_items > 0 else {}
+            last_item = page_items[-1] if num_page_items > 1 else first_item
+            if isinstance(message_attribute, str):
+                first_item = str(first_item.get(message_attribute, ''))
+                last_item = str(last_item.get(message_attribute, ''))
+            else:
+                for attr in message_attribute:
+                    first_item = first_item.get(attr, {})
+                    last_item = last_item.get(attr, {})
+                first_item = str(first_item)
+                last_item = str(last_item)
+            show_message = show_message.replace(FIRST_ITEM_MARKER, first_item)
+            show_message = show_message.replace(LAST_ITEM_MARKER, last_item)
+        sys.stderr.write('\r')
+        sys.stderr.flush()
+        sys.stderr.write(show_message)
+    return (page_token, total_items)
+
+def finalize_page_message(page_message):
+    """ Issue final page_message """
+    if page_message and (page_message[-1] != '\n'):
+        sys.stderr.write('\r\n')
+        sys.stderr.flush()
+
 def get_all_pages(service,
                   function,
                   items='items',
@@ -274,46 +329,12 @@ def get_all_pages(service,
                     soft_errors=soft_errors,
                     throw_reasons=throw_reasons,
                     retry_reasons=retry_reasons,
-                    pageToken=page_token,
                     **kwargs)
-        if page:
-            page_token = page.get('nextPageToken')
-            page_items = page.get(items, [])
-            num_page_items = len(page_items)
-            total_items += num_page_items
-            all_items.extend(page_items)
-        else:
-            page_token = None
-            num_page_items = 0
-
-        # Show a paging message to the user that indicates paging progress
-        if page_message:
-            show_message = page_message.replace(TOTAL_ITEMS_MARKER,
-                                                str(total_items))
-            if message_attribute:
-                first_item = page_items[0] if num_page_items > 0 else {}
-                last_item = page_items[-1] if num_page_items > 1 else first_item
-                if type(message_attribute) is str:
-                    first_item = str(first_item.get(message_attribute, ''))
-                    last_item = str(last_item.get(message_attribute, ''))
-                else:
-                    for attr in message_attribute:
-                        first_item = first_item.get(attr, {})
-                        last_item = last_item.get(attr, {})
-                    first_item = str(first_item)
-                    last_item = str(last_item)
-                show_message = show_message.replace(FIRST_ITEM_MARKER, first_item)
-                show_message = show_message.replace(LAST_ITEM_MARKER, last_item)
-            sys.stderr.write('\r')
-            sys.stderr.flush()
-            sys.stderr.write(show_message)
-
+        page_token, total_items = process_page(page, items, all_items, total_items, page_message, message_attribute)
         if not page_token:
-            # End the paging status message and return all items.
-            if page_message and (page_message[-1] != '\n'):
-                sys.stderr.write('\r\n')
-                sys.stderr.flush()
+            finalize_page_message(page_message)
             return all_items
+        kwargs['pageToken'] = page_token
 
 
 # TODO: Make this private once all execution related items that use this method

src/gam/gapi/cloudidentity/groups.py

@@ -1,3 +1,7 @@
+import sys
+
+import googleapiclient
+
 import gam
 from gam.var import *
 from gam import controlflow
@@ -160,6 +164,7 @@ def print_():
     members = membersCountOnly = managers = managersCountOnly = owners = ownersCountOnly = False
     gapi_directory_customer.setTrueCustomerId()
     parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    usemember = None
     memberDelimiter = '\n'
     todrive = False
     titles = []
@@ -171,6 +176,10 @@ def print_():
         if myarg == 'todrive':
             todrive = True
             i += 1
+        elif myarg == 'enterprisemember':
+            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
+            i += 2
         elif myarg == 'delimiter':
             memberDelimiter = sys.argv[i + 1]
             i += 2
@@ -222,16 +231,36 @@ def print_():
                 display.add_titles_to_csv_file([
                     'Owners',
                 ], titles)
-    gam.printGettingAllItems('Groups', None)
+    gam.printGettingAllItems('Groups', usemember)
     page_message = gapi.got_total_items_first_last_msg('Groups')
-    entityList = gapi.get_all_pages(ci.groups(),
-                                    'list',
-                                    'groups',
-                                    page_message=page_message,
-                                    message_attribute=['groupKey', 'id'],
-                                    parent=parent,
-                                    view='FULL',
-                                    pageSize=500)
+    if usemember:
+        try:
+            result = gapi.get_all_pages(ci.groups().memberships(),
+                                        'searchTransitiveGroups',
+                                        'memberships',
+                                        throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
+                                        page_message=page_message,
+                                        message_attribute=['groupKey', 'id'],
+                                        parent='groups/-', query=usemember,
+                                        fields='nextPageToken,memberships(group,groupKey(id),relationType)',
+                                        pageSize=1000)
+        except googleapiclient.errors.HttpError:
+            controlflow.system_error_exit(
+                2,
+                f'enterprisemember requires Enterprise license')
+        entityList = []
+        for entity in result:
+            if entity['relationType'] == 'DIRECT':
+                entityList.append(gapi.call(ci.groups(), 'get', name=entity['group']))
+    else:
+        entityList = gapi.get_all_pages(ci.groups(),
+                                        'list',
+                                        'groups',
+                                        page_message=page_message,
+                                        message_attribute=['groupKey', 'id'],
+                                        parent=parent,
+                                        view='FULL',
+                                        pageSize=500)
     i = 0
     count = len(entityList)
     for groupEntity in entityList:
@@ -319,6 +348,7 @@ def print_members():
     todrive = False
     gapi_directory_customer.setTrueCustomerId()
     parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
+    usemember = None
     roles = []
     titles = ['group']
     csvRows = []
@@ -339,6 +369,10 @@ def print_members():
                         f'{role} is not a valid role for "gam print group-members {myarg}"'
                     )
             i += 2
+        elif myarg == 'enterprisemember':
+            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
+            i += 2
         elif myarg in ['cigroup', 'cigroups']:
             group_email = gam.normalizeEmailAddressOrUID(sys.argv[i + 1])
             groups_to_get = [group_email]
@@ -347,19 +381,36 @@ def print_members():
             controlflow.invalid_argument_exit(sys.argv[i],
                                               'gam print cigroup-members')
     if not groups_to_get:
-        gam.printGettingAllItems('Groups', None)
+        gam.printGettingAllItems('Groups', usemember)
         page_message = gapi.got_total_items_first_last_msg('Groups')
-        groups_to_get = gapi.get_all_pages(
-            ci.groups(),
-            'list',
-            'groups',
-            message_attribute=['groupKey', 'id'],
-            page_message=page_message,
-            parent=parent,
-            view='BASIC',
-            pageSize=1000,
-            fields='nextPageToken,groups(groupKey(id))')
-        groups_to_get = [group['groupKey']['id'] for group in groups_to_get]
+        if usemember:
+            try:
+                groups_to_get = gapi.get_all_pages(ci.groups().memberships(),
+                                                   'searchTransitiveGroups',
+                                                   'memberships',
+                                                   throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
+                                                   message_attribute=['groupKey', 'id'],
+                                                   page_message=page_message,
+                                                   parent='groups/-', query=usemember,
+                                                   pageSize=1000,
+                                                   fields='nextPageToken,memberships(groupKey(id),relationType)')
+            except googleapiclient.errors.HttpError:
+                controlflow.system_error_exit(
+                        2,
+                        f'enterprisemember requires Enterprise license')
+            groups_to_get = [group['groupKey']['id'] for group in groups_to_get if group['relationType'] == 'DIRECT']
+        else:
+            groups_to_get = gapi.get_all_pages(
+                ci.groups(),
+                'list',
+                'groups',
+                message_attribute=['groupKey', 'id'],
+                page_message=page_message,
+                parent=parent,
+                view='BASIC',
+                pageSize=1000,
+                fields='nextPageToken,groups(groupKey(id))')
+            groups_to_get = [group['groupKey']['id'] for group in groups_to_get]
     i = 0
     count = len(groups_to_get)
     for group_email in groups_to_get:

src/gam/gapi/directory/cros.py

@@ -16,11 +16,22 @@ from gam.gapi.directory import orgunits as gapi_directory_orgunits
 from gam import utils
 
 
+def _display_cros_command_result(cd, device_id, command_id, times_to_check_status):
+    print(f'deviceId: {device_id}, commandId: {command_id}')
+    final_states = {'EXPIRED', 'CANCELLED', 'EXECUTED_BY_CLIENT'}
+    for _ in range(0, times_to_check_status):
+        time.sleep(2)
+        result = gapi.call(cd.customer().devices().chromeos().commands(), 'get',
+            customerId=GC_Values[GC_CUSTOMER_ID], deviceId=device_id,
+            commandId=command_id)
+        display.print_json(result)
+        if result.get('state') in final_states:
+            return
+
 def issue_command():
     cd = gapi_directory.build()
     i, devices = getCrOSDeviceEntity(3, cd)
     body = {}
-    final_states = ['EXPIRED', 'CANCELLED', 'EXECUTED_BY_CLIENT']
     valid_commands = gapi.get_enum_values_minus_unspecified(
             cd._rootDesc['schemas']
             ['DirectoryChromeosdevicesIssueCommandRequest']
@@ -68,36 +79,28 @@ def issue_command():
         except googleapiclient.errors.HttpError:
             controlflow.system_error_exit(4, '400 response from Google. This ' \
               'usually indicates the devices was not in a state where it will' \
-              ' accept the command. For example, reboot and take_a_screenshot' \
+              ' accept the command. For example, reboot, set_volume and take_a_screenshot' \
               ' require the device to be in auto-start kiosk app mode.')
-        display.print_json(result)
         command_id = result.get('commandId')
-        for i in range(0, times_to_check_status):
-            time.sleep(2)
-            result = gapi.call(cd.customer().devices().chromeos().commands(), 'get',
-              customerId=GC_Values[GC_CUSTOMER_ID], deviceId=device_id,
-              commandId=command_id)
-            display.print_json(result)
-            state = result.get('state')
-            if state in final_states:
-                break
+        _display_cros_command_result(cd, device_id, command_id, times_to_check_status)
 
 def get_command():
     cd = gapi_directory.build()
     i, devices = getCrOSDeviceEntity(3, cd)
     command_id = None
+    times_to_check_status = 1
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'commandid':
             command_id = sys.argv[i+1]
             i += 2
+        elif myarg == 'timestocheckstatus':
+            times_to_check_status = int(sys.argv[i+1])
+            i += 2
         else:
             controlflow.invalid_argument_exit(sys.argv[i], 'gam getcommand cros')
     for device_id in devices:
-        result = gapi.call(cd.customer().devices().chromeos().commands(), 'get',
-                  customerId=GC_Values[GC_CUSTOMER_ID], deviceId=device_id,
-                  commandId=command_id)
-        display.print_json(result)
+        _display_cros_command_result(cd, device_id, command_id, times_to_check_status)
 
 def doUpdateCros():
     cd = gapi_directory.build()

src/gam/gapi/vault.py

@@ -11,6 +11,7 @@ from gam import display
 from gam import fileutils
 from gam import gapi
 from gam.gapi import storage as gapi_storage
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
 from gam import utils
 
 
@@ -131,7 +132,7 @@ def createExport():
                 i += 2
             elif searchMethod == 'ORG_UNIT':
                 body['query']['orgUnitInfo'] = {
-                    'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+                    'orgUnitId': gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])[1]
                 }
                 i += 2
             elif searchMethod == 'SHARED_DRIVE':
@@ -301,7 +302,7 @@ def createHold():
             i += 2
         elif myarg in ['orgunit', 'ou']:
             body['orgUnit'] = {
-                'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+                'orgUnitId': gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])[1]
             }
             i += 2
         elif myarg in ['start', 'starttime']:
@@ -407,7 +408,7 @@ def getHoldInfo():
             acct_email = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
             results['accounts'][i]['email'] = acct_email
     if 'orgUnit' in results:
-        results['orgUnit']['orgUnitPath'] = gam.doGetOrgInfo(
+        results['orgUnit']['orgUnitPath'] = gapi_directory_orgunits.info(
             results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
     display.print_json(results)
 
@@ -496,7 +497,7 @@ def updateHold():
             i += 2
         elif myarg in ['orgunit', 'ou']:
             body['orgUnit'] = {
-                'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+                'orgUnitId': gapi_directory_orgunits.getOrgUnitId(sys.argv[i + 1])[1]
             }
             i += 2
         elif myarg in ['start', 'starttime']:

src/gam/var.py

@@ -233,7 +233,6 @@ PRODUCTID_NAME_MAPPINGS = {
     '101034': 'G Suite Archived',
     'Google-Apps': 'Google Workspace',
     'Google-Chrome-Device-Management': 'Google Chrome Device Management',
-    'Google-Coordinate': 'Google Coordinate',
     'Google-Drive-storage': 'Google Drive Storage',
     'Google-Vault': 'Google Vault',
 }
@@ -241,7 +240,6 @@ PRODUCTID_NAME_MAPPINGS = {
 # Legacy APIs that use v1 discovery. Newer APIs should all use v2.
 V1_DISCOVERY_APIS = {
     'admin',
-    'appsactivity',
     'calendar',
     'drive',
     'oauth2',
@@ -260,7 +258,7 @@ API_NAME_MAPPING = {
 
 API_VER_MAPPING = {
     'alertcenter': 'v1beta1',
-    'appsactivity': 'v1',
+    'driveactivity': 'v2',
     'calendar': 'v3',
     'cbcm': 'v1.1beta1',
     'classroom': 'v1',
@@ -293,8 +291,8 @@ USERINFO_EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email'
 
 API_SCOPE_MAPPING = {
     'alertcenter': ['https://www.googleapis.com/auth/apps.alerts',],
-    'appsactivity': [
-        'https://www.googleapis.com/auth/activity',
+    'driveactivity': [
+        'https://www.googleapis.com/auth/drive.activity',
         'https://www.googleapis.com/auth/drive',
     ],
     'calendar': ['https://www.googleapis.com/auth/calendar',],
@@ -907,6 +905,7 @@ CROS_ARGUMENT_TO_PROPERTY_MAP = {
     'ethernetmacaddress0': ['ethernetMacAddress0',],
     'firmwareversion': ['firmwareVersion',],
     'lastenrollmenttime': ['lastEnrollmentTime',],
+    'lastknownnetwork': ['lastKnownNetwork'],
     'lastsync': ['lastSync',],
     'location': ['annotatedLocation',],
     'macaddress': ['macAddress',],