re-add ACM API

.github/workflows/build.yml

@@ -109,7 +109,7 @@ jobs:
           path: |
             bin.tar.xz
             src/cpython
-          key: gam-${{ matrix.jid }}-20230405
+          key: gam-${{ matrix.jid }}-20230417
 
       - name: Untar Cache archive
         if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
@@ -467,8 +467,8 @@ jobs:
           git clone https://github.com/pyinstaller/pyinstaller.git
           cd pyinstaller
           export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
-          # temp pin to 5.9
-          export latest_release="v5.9.0"
+
+
           git checkout "${latest_release}"
           # remove pre-compiled bootloaders so we fail if bootloader compile fails
           rm -rvf PyInstaller/bootloader/*-*/*
@@ -564,8 +564,12 @@ jobs:
           if [[ "${RUNNER_OS}" == "macOS" ]]; then
               GAM_ARCHIVE="gam-${GAMVERSION}-macos-universal2.tar.xz"
           elif [[ "${RUNNER_OS}" == "Linux" ]]; then
-              this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
-              GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(arch)-glibc${this_glibc_ver}.tar.xz"
+              if [[ "${staticx}" == "yes" ]]; then
+                libver="legacy"
+              else
+                libver="glibc$(ldd --version | awk '/ldd/{print $NF}')"
+              fi
+              GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(arch)-$libver}.tar.xz"
           fi
           tar -C dist/ --create --verbose --exclude-from "${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" --file $GAM_ARCHIVE --xz gam
 
@@ -670,7 +674,9 @@ jobs:
              $gam update group $newgroup add member $newuser
              rm "${gampath}/enabledasa.txt"
              $gam create admin $newuser _GROUPS_EDITOR_ROLE CUSTOMER # condition nonsecuritygroup
+             $gam create admin $newgroup _HELP_DESK_ADMIN_ROLE org_unit "${newou}"
              GAM_CSV_ROW_FILTER="assignedToUser:regex:${newuser}" $gam print admins | $gam csv - gam delete admin "~roleAssignmentId"
+             GAM_CSV_ROW_FILTER="assignedToGroup:regex:${newgroup}" $gam print admins | $gam csv - gam delete admin "~roleAssignmentId"
              touch "${gampath}/enabledasa.txt"
              $gam csv sample.csv gam create user ~~email~~ firstname "GHA Bulk" lastname ~~email~~ gha.jid $JID ou "${newou}"
              $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random displayname "GitHub Actions Bulk ${JID}"

src/gam/__init__.py

@@ -887,8 +887,7 @@ def doGAMVersion(checkForArgs=True):
         for lib in GAM_VER_LIBS:
             try:
                 print(f'{lib} {lib_version(lib)}')
-            except Exception as e:
-                print(e)
+            except:
                 pass
         tls_ver, cipher_name, used_ip = _getServerTLSUsed(testLocation)
         print(
@@ -1133,39 +1132,66 @@ def buildGAPIObjectNoAuthentication(api):
     service = getService(api, httpObj)
     return service
 
-# Convert UID to email address
+def get_user_email_from_id(uid, cd):
+    try:
+        result = gapi.call(
+                cd.users(),
+                'get',
+                throw_reasons=[gapi_errors.ErrorReason.USER_NOT_FOUND],
+                userKey=uid,
+                fields='primaryEmail')
+        return result.get('primaryEmail')
+    except gapi_errors.GapiUserNotFoundError:
+        return
+
+def get_group_email_from_id(uid, cd):
+    try:
+        result = gapi.call(
+                cd.groups(),
+                'get',
+                throw_reasons=[gapi_errors.ErrorReason.GROUP_NOT_FOUND],
+                groupKey=uid,
+                fields='email')
+        return result.get('email')
+    except gapi_errors.GapiGroupNotFoundError:
+        return
+
 def convertUIDtoEmailAddress(emailAddressOrUID, cd=None, email_types=['user']):
+    '''convert UID to email address
+       returns email address and object type'''
     if isinstance(email_types, str):
         email_types = email_types.split(',')
     normalizedEmailAddressOrUID = normalizeEmailAddressOrUID(emailAddressOrUID)
     if normalizedEmailAddressOrUID.find('@') > 0:
-        return normalizedEmailAddressOrUID
+        return normalizedEmailAddressOrUID, 'email'
     if not cd:
         cd = buildGAPIObject('directory')
-    if 'user' in email_types:
-        try:
-            result = gapi.call(
-                cd.users(),
-                'get',
-                throw_reasons=[gapi_errors.ErrorReason.USER_NOT_FOUND],
-                userKey=normalizedEmailAddressOrUID,
-                fields='primaryEmail')
-            if 'primaryEmail' in result:
-                return result['primaryEmail'].lower()
-        except gapi_errors.GapiUserNotFoundError:
-            pass
-    if 'group' in email_types:
-        try:
-            result = gapi.call(
-                cd.groups(),
-                'get',
-                throw_reasons=[gapi_errors.ErrorReason.GROUP_NOT_FOUND],
-                groupKey=normalizedEmailAddressOrUID,
-                fields='email')
-            if 'email' in result:
-                return result['email'].lower()
-        except gapi_errors.GapiGroupNotFoundError:
-            pass
+    if 'user' in email_types and 'group' in email_types:
+        # Google User IDs *TEND* to be integers while groups tend to have letters
+        # thus we can optimize which check we try first. We'll still check
+        # both since there is no guarantee this will always be true.
+        if normalizedEmailAddressOrUID.isdigit():
+            uid = get_user_email_from_id(normalizedEmailAddressOrUID, cd)
+            if uid:
+                return uid, 'user'
+            uid = get_group_email_from_id(normalizedEmailAddressOrUID, cd)
+            if uid:
+                return uid, 'group'
+        else:
+            uid = get_group_email_from_id(normalizedEmailAddressOrUID, cd)
+            if uid:
+                return uid, 'group'
+            uid = get_user_email_from_id(normalizedEmailAddressOrUID, cd)
+            if uid:
+                return uid, 'user'
+    elif 'user' in email_types:
+        uid = get_user_email_from_id(normalizedEmailAddressOrUID, cd)
+        if uid:
+            return uid, 'user'
+    elif 'group' in email_types:
+        uid = get_group_email_from_id(normalizedEmailAddressOrUID, cd)
+        if uid:
+            return uid, 'group'
     if 'resource' in email_types:
         try:
             result = gapi.call(
@@ -1176,10 +1202,10 @@ def convertUIDtoEmailAddress(emailAddressOrUID, cd=None, email_types=['user']):
                 customer=GC_Values[GC_CUSTOMER_ID],
                 fields='resourceEmail')
             if 'resourceEmail' in result:
-                return result['resourceEmail'].lower()
+                return result['resourceEmail'].lower(), 'resource'
         except gapi_errors.GapiResourceNotFoundError:
             pass
-    return normalizedEmailAddressOrUID
+    return normalizedEmailAddressOrUID, 'unknown'
 
 
 # Convert email address to UID
@@ -1193,12 +1219,13 @@ def convertEmailAddressToUID(emailAddressOrUID, cd=None, email_type='user'):
                 result = gapi.call(
                     cd.users(),
                     'get',
-                    throw_reasons=[gapi_errors.ErrorReason.USER_NOT_FOUND],
+                    throw_reasons=[gapi_errors.ErrorReason.USER_NOT_FOUND,
+                                   gapi_errors.ErrorReason.BAD_REQUEST],
                     userKey=normalizedEmailAddressOrUID,
                     fields='id')
                 if 'id' in result:
                     return result['id']
-            except gapi_errors.GapiUserNotFoundError:
+            except (gapi_errors.GapiUserNotFoundError, gam.gapi.errors.GapiBadRequestError):
                 pass
         try:
             result = gapi.call(
@@ -1250,27 +1277,27 @@ def buildGAPIServiceObject(api, act_as, showAuthError=True, scopes=None):
 
 
 def buildAlertCenterGAPIObject(user):
-    userEmail = convertUIDtoEmailAddress(user)
+    userEmail, _ = convertUIDtoEmailAddress(user)
     return (userEmail, buildGAPIServiceObject('alertcenter', userEmail))
 
 
 def buildActivityGAPIObject(user):
-    userEmail = convertUIDtoEmailAddress(user)
+    userEmail, _ = convertUIDtoEmailAddress(user)
     return (userEmail, buildGAPIServiceObject('driveactivity', userEmail))
 
 
 def buildDriveGAPIObject(user):
-    userEmail = convertUIDtoEmailAddress(user)
+    userEmail, _ = convertUIDtoEmailAddress(user)
     return (userEmail, buildGAPIServiceObject('drive', userEmail))
 
 
 def buildDrive3GAPIObject(user):
-    userEmail = convertUIDtoEmailAddress(user)
+    userEmail, _ = convertUIDtoEmailAddress(user)
     return (userEmail, buildGAPIServiceObject('drive3', userEmail))
 
 
 def buildGmailGAPIObject(user):
-    userEmail = convertUIDtoEmailAddress(user)
+    userEmail, _ = convertUIDtoEmailAddress(user)
     return (userEmail, buildGAPIServiceObject('gmail', userEmail))
 
 
@@ -2294,7 +2321,7 @@ def doGetCourseInfo():
     croom = buildGAPIObject('classroom')
     courseId = addCourseIdScope(sys.argv[3])
     info = gapi.call(croom.courses(), 'get', id=courseId)
-    info['ownerEmail'] = convertUIDtoEmailAddress(f'uid:{info["ownerId"]}')
+    info['ownerEmail'], _ = convertUIDtoEmailAddress(f'uid:{info["ownerId"]}')
     display.print_json(info)
     teachers = gapi.get_all_pages(croom.courses().teachers(),
                                   'list',
@@ -2479,7 +2506,7 @@ def doPrintCourses():
         if ownerEmails is not None:
             ownerId = course['ownerId']
             if ownerId not in ownerEmails:
-                ownerEmails[ownerId] = convertUIDtoEmailAddress(f'uid:{ownerId}',
+                ownerEmails[ownerId], _ = convertUIDtoEmailAddress(f'uid:{ownerId}',
                                                                 cd=cd)
             course['ownerEmail'] = ownerEmails[ownerId]
         for field in skipFieldsList:

src/gam/gapi/calendar.py

@@ -18,9 +18,9 @@ def normalizeCalendarId(calname, checkPrimary=False):
         return calname
     if not GC_Values[GC_DOMAIN]:
         GC_Values[GC_DOMAIN] = gam._getValueFromOAuth('hd')
-    return gam.convertUIDtoEmailAddress(calname,
+    email, _ = gam.convertUIDtoEmailAddress(calname,
                                         email_types=['user', 'resource'])
-
+    return email
 
 def buildCalendarGAPIObject(calname):
     calendarId = normalizeCalendarId(calname)

src/gam/gapi/cloudidentity/devices.py

@@ -253,13 +253,6 @@ def update_state():
 
 
 def print_():
-    # This function is rather messy thanks to
-    # https://github.com/GAM-team/GAM/issues/1534
-    # I'd prefer to keep it all in this function for now but if:
-    #  - we find other list() operations that also hit this bug OR
-    #  - it looks like this issue is going to exist on Google's side
-    #    for a long time.
-    # I'll enterain some cleanup here to "functionalize" (yuck) all of this.
     ci = gapi_cloudidentity.build_dwd()
     customer = _get_device_customerid()
     parent = 'devices/-'
@@ -267,8 +260,6 @@ def print_():
     get_device_users = True
     view = None
     orderByList = []
-    # default sort order needed by our 1 hour bug workaround
-    orderBy = 'create_time'
     titles = []
     csvRows = []
     todrive = False
@@ -328,97 +319,26 @@ def print_():
       }
     if orderByList:
         orderBy = ','.join(orderByList)
-    custom_device_filter = bool(device_filter)
-    # we store the devices in a dict keyed by name which is a unique ID.
-    # that way when we get duplicate devices we just overwrite the name
-    # with the latest copy we saw.
-    devices = {}
+    else:
+        orderBy = None
+    devices = []
     page_message = gapi.got_total_items_msg(view_name_map[view], '...\n')
-    pageToken = None
-    newest_device_date = ''
-    total_items = 0
-    while True:
-        try:
-            a_page = gapi.call(ci.devices(),
-                               'list',
-                               customer=customer,
-                               pageSize=100,
-                               pageToken=pageToken,
-                               filter=device_filter,
-                               view=view,
-                               orderBy=orderBy,
-                               throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O])
-        except googleapiclient.errors.HttpError:
-            sys.stderr.write('WARNING: GAM hit Google internal bug 237397223. Please file a Google Support ticket stating that you are encountering this bug.\n') 
-            if orderBy != 'create_time' or custom_device_filter:
-                controlflow.system_error_exit(5, 'GAM workaround for this issue only works if filter and orderby arguments are not used.\n')
-            sys.stderr.write(f' attempting to work around the bug by filtering for devices created on or after the newest we\'ve seen ({newest_device_date})...')
-            device_filter = f'register:{newest_device_date}..'
-            pageToken = None
-            continue
-        for dev in a_page.get('devices', []):
-            total_items += 1
-            devices[dev['name']] = dev
-            dev_date = dev.get('createTime', '')
-            # remove the Z
-            dev_date = dev_date[:-1]
-            # remove microseconds
-            dev_date = dev_date.split('.')[0]
-            if dev_date > newest_device_date:
-                newest_device_date = dev_date
-        pageToken = a_page.get('nextPageToken')
-        if not pageToken:
-            break
-        sys.stderr.write(page_message.replace('%%total_items%%', str(total_items)))
+    devices += gapi.get_all_pages(ci.devices(), 'list', 'devices',
+        customer=customer, page_message=page_message,
+        pageSize=100, filter=device_filter, view=view, orderBy=orderBy)
     if get_device_users:
         page_message = gapi.got_total_items_msg('Device Users', '...\n')
-        pageToken = None
-        newest_deviceuser_date = ''
-        total_items = 0
-        device_users = {}
-        if not custom_device_filter:
-            device_filter = None
-        while True:
-            try:
-                a_page = gapi.call(ci.devices().deviceUsers(),
-                                   'list',
-                                   customer=customer,
-                                   parent=parent,
-                                   pageSize=20,
-                                   orderBy=orderBy,
-                                   filter=device_filter,
-                                   pageToken=pageToken,
-                                   throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O])
-            except googleapiclient.errors.HttpError:
-                sys.stderr.write('WARNING: GAM hit Google internal bug 237397223. Please file a Google Support ticket stating that you are encountering this bug.\n')
-                if orderBy != 'create_time' or custom_device_filter:
-                    controlflow.system_error_exit(5, 'GAM workaround for this issue only works if filter and orderby arguments are not used.\n')
-                sys.stderr.write(f' attempting to work around the bug by filtering for device users created on or after the newest we\'ve seen ({newest_deviceuser_date})...')
-                device_filter = f'register:{newest_deviceuser_date}..'
-                pageToken = None
-                continue
-            for device_user in a_page.get('deviceUsers', []):
-                total_items += 1
-                dev_date = device_user.get('createTime', '')
-                # remove the Z
-                dev_date = dev_date[:-1]
-                # remove microseconds
-                dev_date = dev_date.split('.')[0]
-                if dev_date > newest_deviceuser_date:
-                    newest_deviceuser_date = dev_date
-                deviceuser_name = device_user['name']
-                device_users[deviceuser_name] = device_user
-            pageToken = a_page.get('nextPageToken')
-            if not pageToken:
-                break
-            sys.stderr.write(page_message.replace('%%total_items%%', str(total_items)))
-        for deviceuser_name, device_user in device_users.items():
-            device_id = deviceuser_name.split('/')[1]
-            device_name = f'devices/{device_id}'
-            if 'users' not in devices[device_name]:
-                devices[device_name]['users'] = []
-            devices[device_name]['users'].append(device_user)
-    for device in devices.values():
+        device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
+            'deviceUsers', customer=customer, parent=parent,
+            page_message=page_message, pageSize=20, filter=device_filter)
+        for device_user in device_users:
+            for device in devices:
+                if device_user.get('name').startswith(device.get('name')):
+                    if 'users' not in device:
+                        device['users'] = []
+                    device['users'].append(device_user)
+                    break
+    for device in devices:
         device = utils.flatten_json(device)
         for a_key in device:
             if a_key not in titles:

src/gam/gapi/cloudidentity/groups.py

@@ -230,7 +230,7 @@ def print_():
             todrive = True
             i += 1
         elif myarg == 'enterprisemember':
-            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            member, _ = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
             usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
             i += 2
         elif myarg == 'delimiter':
@@ -501,7 +501,7 @@ def print_members():
                     )
             i += 2
         elif myarg == 'enterprisemember':
-            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            member, _ = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
             usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
             i += 2
         elif myarg in ['cigroup', 'cigroups']:
@@ -876,6 +876,13 @@ def update():
                     'cloudidentity.googleapis.com/groups.discussion_forum': ''
                 }
                 i += 1
+            elif myarg == 'locked':
+                body['labels'] = {
+                    'cloudidentity.googleapis.com/groups.locked': '',
+                    'cloudidentity.googleapis.com/groups.security': '',
+                    'cloudidentity.googleapis.com/groups.discussion_forum': ''
+                }
+                i += 1
             elif myarg == 'dynamicsecurity':
                 body['labels'] = {
                     'cloudidentity.googleapis.com/groups.dynamic': '',

src/gam/gapi/directory/roleassignments.py

@@ -16,7 +16,9 @@ NONSECURITY_GROUP_CONDITION = f'!{SECURITY_GROUP_CONDITION}'
 def create():
     cd = gapi_directory.build()
     user = gam.normalizeEmailAddressOrUID(sys.argv[3])
-    body = {'assignedTo': gam.convertEmailAddressToUID(user, cd)}
+    body = {'assignedTo': gam.convertEmailAddressToUID(sys.argv[3],
+                          cd=cd,
+                          email_type='any')}
     role = sys.argv[4]
     body['roleId'] = gapi_directory_roles.getRoleId(role)
     body['scopeType'] = sys.argv[5].upper()
@@ -70,7 +72,7 @@ def print_():
     item_fields = ['roleAssignmentId', 'roleId', 'assignedTo', 'scopeType', 'orgUnitId']
     titles = [
         'roleAssignmentId', 'roleId', 'role', 'assignedTo', 'assignedToUser',
-        'scopeType', 'orgUnitId', 'orgUnit'
+        'assignedToGroup', 'scopeType', 'orgUnitId', 'orgUnit'
     ]
     csvRows = []
     i = 3
@@ -107,7 +109,21 @@ def print_():
         admin_attrib = {}
         for key, value in list(admin.items()):
             if key == 'assignedTo':
-                admin_attrib['assignedToUser'] = gam.user_from_userid(value)
+                email_types = admin_attrib.get('assigneeType')
+                if email_types == 'user':
+                    email_field = 'assignedToUser'
+                elif email_types == 'group':
+                    email_field = 'assignedToGroup'
+                else:
+                    email_field = None
+                assignment_email, assignment_type = gam.convertUIDtoEmailAddress(f'uid:{value}', cd, email_types=['user', 'group'])
+                if not email_field and assignment_type in ['user', 'group']:
+                    if assignment_type == 'user':
+                        email_field = 'assignedToUser'
+                    else:
+                        email_field = 'assignedToGroup'
+                if email_field:
+                    admin_attrib[email_field] = assignment_email
             elif key == 'roleId':
                 admin_attrib['role'] = gapi_directory_roles.role_from_roleid(value)
             elif key == 'orgUnitId':

src/gam/gapi/drive/__init__.py

@@ -4,5 +4,5 @@ import gam
 def build(user=None):
     if not user:
         user = gam._get_admin_email()
-    userEmail = gam.convertUIDtoEmailAddress(user)
+    userEmail, _ = gam.convertUIDtoEmailAddress(user)
     return (userEmail, gam.buildGAPIServiceObject('drive3', userEmail))

src/gam/gapi/licensing.py

@@ -204,6 +204,7 @@ def print_(returnFields=None,
                     ],
                     page_message=page_message,
                     customerId=customer_id,
+                    maxResults=100,
                     productId=product,
                     skuId=sku,
                     fields=fields)
@@ -234,6 +235,7 @@ def print_(returnFields=None,
                     ],
                     page_message=page_message,
                     customerId=customer_id,
+                    maxResults=100,
                     productId=productId,
                     fields=fields)
                 if countsOnly:

src/gam/gapi/vault.py

@@ -511,7 +511,7 @@ def getHoldInfo():
         account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
         for i in range(0, len(results['accounts'])):
             uid = f'uid:{results["accounts"][i]["accountId"]}'
-            acct_email = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
+            acct_email, _ = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
             results['accounts'][i]['email'] = acct_email
     if 'orgUnit' in results:
         results['orgUnit']['orgUnitPath'] = gapi_directory_orgunits.info(
@@ -792,7 +792,7 @@ def getMatterInfo():
         cd = gam.buildGAPIObject('directory')
         for i in range(0, len(result['matterPermissions'])):
             uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
-            user_email = gam.convertUIDtoEmailAddress(uid, cd)
+            user_email, _ = gam.convertUIDtoEmailAddress(uid, cd)
             result['matterPermissions'][i]['email'] = user_email
     display.print_json(result)
 

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.56'
+GAM_VERSION = '6.58'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'