Update build.yml

* Unescape \r and \n in chatmessage text so multiline messages can be created

* Bring gam report activity list up to date

.github/actions/decrypt.sh

@@ -13,4 +13,5 @@ fi
 gpg --quiet --batch --yes --decrypt --passphrase="${PASSCODE}" \
     --output "${credsfile}" "${gpgfile}"
 
-tar xf "${credsfile}" --directory "${gampath}"
+tar xvvf "${credsfile}" --directory "${gampath}"
+ls -l "${gampath}"

.github/actions/linux-install.sh

@@ -1,8 +1,10 @@
-export gampath="dist/gam"
+export distpath="dist/"
+export gampath="${distpath}gam"
 rm -rf $gampath
-mkdir -p $gampath
-export gampath=$(readlink -e $gampath)
-$python -OO -m PyInstaller --clean --noupx --strip -F --distpath $gampath gam.spec
+#mkdir -p $gampath
+#export gampath=$(readlink -e $gampath)
+$pip install wheel
+$python -OO -m PyInstaller --clean --noupx --strip --distpath $gampath gam.spec
 export gam="${gampath}/gam"
 export GAMVERSION=`$gam version simple`
 cp LICENSE $gampath
@@ -11,7 +13,7 @@ this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
 GAM_ARCHIVE="gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-glibc${this_glibc_ver}.tar.xz"
 rm $gampath/lastupdatecheck.txt
 # tar will cd to dist and tar up gam/
-tar -C dist/ --create --file $GAM_ARCHIVE --xz gam
+tar -C ${distpath} --create --file $GAM_ARCHIVE --xz gam
 echo "PyInstaller GAM info:"
 du -h $gam
 time $gam version extended

.github/actions/macos-install.sh

@@ -1,18 +1,19 @@
 echo "MacOS Version Info According to Python:"
-python -c "import platform; print(platform.mac_ver())"
-echo "Xcode versionn:"
+macver=$(python -c "import platform; print(platform.mac_ver()[0])")
+echo $macver
+echo "Xcode version:"
 xcodebuild -version
-export gampath=dist/gam
+export distpath="dist/"
+export gampath="${distpath}gam"
 rm -rf $gampath
 export specfile="gam.spec"
-$python -OO -m PyInstaller --clean --noupx --strip -F --distpath "${gampath}" "${specfile}"
+$python -OO -m PyInstaller --clean --noupx --strip --distpath "${gampath}" --target-architecture $PLATFORM "${specfile}"
 export gam="${gampath}/gam"
 $gam version extended
 export GAMVERSION=`$gam version simple`
 cp LICENSE "${gampath}"
 cp GamCommands.txt "${gampath}"
-MACOSVERSION=$(defaults read loginwindow SystemVersionStampAsString)
-GAM_ARCHIVE="gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-MacOS${MACOSVERSION}.tar.xz"
+GAM_ARCHIVE="gam-${GAMVERSION}-${GAMOS}-${PLATFORM}.tar.xz"
 rm "${gampath}/lastupdatecheck.txt"
 # tar will cd to dist/ and tar up gam/
 tar -C dist/ --create --file $GAM_ARCHIVE --xz gam

.github/actions/windows-install.sh

@@ -4,11 +4,12 @@ elif [[ "$PLATFORM" == "x86" ]]; then
   export WIX_BITS="x86"
 fi
 echo "compiling GAM with pyinstaller..."
-export gampath="dist/gam"
+export distpath="dist/"
+export gampath="${distpath}gam"
 rm -rf $gampath
-mkdir -p $gampath
-export gampath=$(readlink -e $gampath)
-pyinstaller --clean --noupx -F --distpath $gampath gam.spec
+#mkdir -p $gampath
+#export gampath=$(readlink -e $gampath)
+pyinstaller --clean --noupx --distpath $gampath gam.spec
 export gam="${gampath}/gam"
 echo "running compiled GAM..."
 $gam version
@@ -18,8 +19,11 @@ cp LICENSE $gampath
 cp GamCommands.txt $gampath
 cp gam-setup.bat $gampath
 GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
-/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE $gampath -xr!.svn
-
+cwd=$(pwd)
+cd "${distpath}"
+/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
+mv "${GAM_ARCHIVE}" "${cwd}"
+cd "${cwd}"
 echo "Running WIX candle $WIX_BITS..."
 /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch $WIX_BITS gam.wxs
 echo "Done with WIX candle..."

.github/workflows/build.yml

@@ -18,7 +18,7 @@ env:
   MIN_OPENSSL_VERSION: "1.1.1k"
   PATCHELF_VERSION: "0.12"
   # PYINSTALLER_VERSION can be full commit hash or version like v4.20
-  PYINSTALLER_VERSION: "e20e74c03768d432d48665b8ef1e02511b16e4be"
+  PYINSTALLER_VERSION: "e5dbb051bd3d53d6c2c70cbd87270eec1765da2e"
 
 jobs:
   build:
@@ -41,16 +41,6 @@ jobs:
             goal: "build"
             gamos: "linux"
             platform: "x86_64"
-#          - os: [self-hosted, linux, ARM]
-#            jid: 10
-#            goal: "build"
-#            gamos: "linux"
-#            platform: "arm"
-#          - os: [self-hosted, linux, ARM64]
-#            jid: 11
-#            goal: "build"
-#            gamos: "linux"
-#            platform: "arm64"
           - os: macos-10.15
             jid: 4
             goal: "build"
@@ -93,6 +83,12 @@ jobs:
             jid: 9
             gamos: "linux"
             platform: "x86_64"
+          - os: ubuntu-20.04
+            goal: test
+            python: "3.10.0-beta.1"
+            jid: 10
+            gamos: linux
+            platform: x86_64
 
     steps:
 
@@ -108,7 +104,7 @@ jobs:
           path: |
             ~/python
             ~/ssl
-          key: ${{ matrix.os }}-${{ matrix.jid }}-20210504
+          key: ${{ matrix.os }}-${{ matrix.jid }}-20210611
 
       - name: Set env variables
         env:
@@ -145,6 +141,7 @@ jobs:
              echo "RUNNING: apt update..."
              sudo apt-get -qq --yes update > /dev/null
              sudo apt-get -qq --yes install swig libpcsclite-dev
+             $pip install --upgrade pip
 
       - name: Build and install Python, OpenSSL and PyInstaller
         if: matrix.goal != 'test' && steps.cache-primes.outputs.cache-hit != 'true'
@@ -156,17 +153,18 @@ jobs:
              echo "pip=$pip" >> $GITHUB_ENV
              echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
              echo -e "Python: $python\nPip: $pip\nLD_LIB...: $LD_LIBRARY_PATH"
+             $pip install wheel
              export url="https://codeload.github.com/pyinstaller/pyinstaller/tar.gz/${PYINSTALLER_VERSION}"
              echo "Downloading ${url}"
              curl -o pyinstaller.tar.gz --compressed "${url}"
              tar xf pyinstaller.tar.gz
              cd "pyinstaller-${PYINSTALLER_VERSION}/bootloader"
              if [ "${PLATFORM}" == "x86" ]; then
-               BITS="32"
+               TARGETARCH="--target-arch=32bit"
              else
-               BITS="64"
+               TARGETARCH=""
              fi
-             $python ./waf all --target-arch=${BITS}bit
+             #$python ./waf all $TARGETARCH
              cd ..
              $python setup.py install
              #$pip install pyinstaller
@@ -184,6 +182,7 @@ jobs:
         run: |
              set +e
              source ../.github/actions/${GAMOS}-install.sh
+             ls -alRF $gampath
              echo "gampath=$gampath" >> $GITHUB_ENV
              echo "gam=$gam" >> $GITHUB_ENV
              echo -e "GAM: ${gam}\nGAMPATH: ${gampath}\nGAMVERSION: ${GAMVERSION}"
@@ -344,3 +343,13 @@ jobs:
                echo "file uploaded as ${fileid}, setting ACL..."
                $gam user $gam_user add drivefileacl $fileid anyone role reader withlink
              done
+
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v2
+        if: github.event_name == 'push' && matrix.goal != 'test'
+        with:
+          name: gam-binaries
+          path: |
+            src/*.tar.xz
+            src/*.zip
+            src/*.msi

src/GamCommands.txt

@@ -158,6 +158,7 @@ If an item contains spaces, it should be surrounded by ".
 <CalendarColorIndex> ::= <Number in range 1-24>
 <CalendarItem> ::= <EmailAddress>|<String>
 <ChatRoom> ::= <String>
+<ChatSpace> ::= <String>
 <ClientID> ::= <String>
 <ColorValue> ::= <ColorName>|<ColorHex>
 <CollaboratorItem> ::= <EmailAddress>|<UniqueID>|<String>
@@ -223,6 +224,7 @@ If an item contains spaces, it should be surrounded by ".
 <QueryGmail> ::= <String> See: https://support.google.com/mail/answer/7190
 <QueryGroup> ::= <String> See: https://developers.google.com/admin-sdk/directory/v1/guides/search-groups
 <QueryMobile> ::= <String> See: https://support.google.com/a/answer/7549103
+<QueryTeamDrive> ::= <String> See: https://developers.google.com/drive/api/v3/search-shareddrives
 <QueryUser> ::= <String> See: https://developers.google.com/admin-sdk/directory/v1/guides/search-users
 <QueryVaultCorpus> ::= <String> See: https://developers.google.com/vault/reference/rest/v1/matters.holds#CorpusQuery
 <RequestID> ::= <String>
@@ -894,23 +896,27 @@ gam delete resoldsubscription <CustomerID> <SKUID> cancel|downgrade|transfer_to_
 gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 
 <ActivityApplicationName> ::=
-	access|accesstransparency|
+	access_transparency|
 	admin|
-	calendar|calendars|
+	calendar|
 	chat|
-	drive|doc|docs|
-	enterprisegroups|groupsenterprise|
+	chrome|
+	context_aware_access|
+	data_studio|
+	drive|
 	gcp|
-	google+|gplus|
-	group|groups|
-	hangoutsmeet|meet|
+	gplus|
+	groups|
+	groups_enterprise|
 	jamboard|
-	login|logins|
+	keep|
+	login|
+	meet|
 	mobile|
-	oauthtoken|token|tokens|
 	rules|
 	saml|
-	useraccounts
+	token|
+	user_accounts
 
 <ReportsApp> ::=
 	accounts|
@@ -1166,6 +1172,14 @@ gam print browsertokens [todrive]
 	[fields <BrowserTokenFieldNameList>]
 	[sortheaders]
 
+gam print chatspaces [todrive]
+gam print chatmembers space <ChatSpace> [todrive]
+gam create chatmessage space <ChatSpace> [thread <String>]
+	(text <String>)|(textfile <FileName> [charset <CharSet>])
+gam delete chatmessage name <String>
+gam update chatmessage name <String>
+	(text <String>)|(textfile <FileName> [charset <CharSet>])
+
 <CrOSAction> ::=
 	deprovision_same_model_replace|
 	deprovision_different_model_replace|
@@ -1365,7 +1379,7 @@ gam update cigroup <GroupItem> sync [owner|manager|member] [notsuspended|suspend
 gam update cigroup <GroupItem> update [owner|manager|member] [notsuspended|suspended] [expires never|<Time>] <UserTypeEntity>
 gam update cigroup <GroupItem> clear [member] [manager] [owner] [notsuspended|suspended]
 gam delete cigroup <GroupItem>
-gam info cigroup <GroupItem> [nousers] [nojoindate] [showupdatedate]
+gam info cigroup <GroupItem> [nousers] [nojoindate] [showupdatedate] [membertree]
 
 gam print cigroups [todrive]
 	[enterprisemember <UserItem>]
@@ -1438,7 +1452,7 @@ gam create user <EmailAddress> <UserAttribute>* [verifynotinvitable]
 gam update user <UserItem> <UserAttribute>* [clearschema <SchemaName>] [clearschema <SchemaName>.<FieldName>] [verifynotinvitable]
 gam delete user <UserItem>
 gam undelete user <UserItem> [org|ou <OrgUnitPath>]
-gam info user [<UserItem>] [noaliases] [nogroups] [nolicenses|nolicences] [noschemas] [schemas|custom <SchemaNameList>] [userview] [skus|sku <SKUIDList>]
+gam info user [<UserItem>] [noaliases] [nogroups] [nolicenses|nolicences] [noschemas] [schemas|custom <SchemaNameList>] [userview] [skus|sku <SKUIDList>] [grouptree]
 
 Print fields for selected users; use domain, query/queries and deleted_only to select users to print;
 if none of these options are specified, all users are printed.
@@ -1600,6 +1614,7 @@ gam <UserTypeEntity> update labelsettings <LabelName> [name <Name>] [messagelist
 gam <UserTypeEntity> update label|labels [search <RegularExpression>] [replace <LabelReplacement>] [merge]
 gam <UserTypeEntity> delete|del label|labels <LabelName>|regex:<RegularExpression>|--ALL_LABELS--
 gam <UserTypeEntity> show labels|label [onlyuser] [showcounts]
+gam <UserTypeEntity> print labels|label [todrive] [onlyuser] [showcounts]
 
 gam <UserTypeEntity> delete messages query <QueryGmail> [doit] [max_to_delete|max_to_process <Number>]
 gam <UserTypeEntity> modify messages query <QueryGmail> [doit] [max_to_modify|max_to_process <Number>] (addlabel <LabelName>)* (removelabel <LabelName>)*
@@ -1690,8 +1705,8 @@ gam <UserTypeEntity> update teamdrive <TeamDriveID> [asadmin] [name <Name>]
 	(<TeamDriveRestrictionsSubfieldName> <Boolean>)*
 gam <UserTypeEntity> delete teamdrive <TeamDriveID>
 gam <UserTypeEntity> show teamdriveinfo <TeamDriveID> [asadmin]
-gam <UserTypeEntity> show teamdrives [asadmin]
-gam <UserTypeEntity> print teamdrives [todrive] [asadmin]
+gam <UserTypeEntity> show teamdrives [query <QueryTeamDrive>] [asadmin]
+gam <UserTypeEntity> print teamdrives [query <QueryTeamDrive>] [todrive] [asadmin]
 gam <UserTypeEntity> show teamdrivethemes
 
 gam <UserTypeEntity> vacation <FalseValues>

src/gam-install.sh

@@ -29,7 +29,7 @@ gamversion="latest"
 adminuser=""
 regularuser=""
 gam_glibc_vers="2.31 2.27 2.23"
-gam_macos_vers="10.15.6 10.14.6 10.13.6"
+#gam_macos_vers="10.15.6 10.14.6 10.13.6"
 
 while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
@@ -128,18 +128,6 @@ case $gamos in
       this_macos_ver=$osversion
     fi
     echo "You are running MacOS $this_macos_ver"
-    use_macos_ver=""
-    for gam_macos_ver in $gam_macos_vers; do
-      if version_gt $this_macos_ver $gam_macos_ver; then
-        use_macos_ver="MacOS$gam_macos_ver"
-        echo_green "Using GAM compiled on $use_macos_ver"
-        break
-      fi
-    done
-    if [ "$use_macos_ver" == "" ]; then
-      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
-      exit
-    fi
     gamfile="macos-x86_64.tar.xz"
     ;;
   MINGW64_NT*)

src/gam.spec

@@ -34,6 +34,7 @@ for d in a.datas:
 
 
 pyz = PYZ(a.pure)
+
 exe = EXE(pyz,
           a.scripts,
           a.binaries,
@@ -43,4 +44,4 @@ exe = EXE(pyz,
           debug=False,
           strip=None,
           upx=False,
-          console=True )
+          console=True)

src/gam/__init__.py

@@ -6,11 +6,14 @@ import configparser
 import csv
 import datetime
 from email import message_from_string
+try:
+    from importlib.metadata import version as lib_version
+except ImportError:
+    from importlib_metadata import version as lib_version
 import io
 import json
 import mimetypes
 import os
-import pkg_resources
 import platform
 from pathlib import Path
 import random
@@ -38,6 +41,7 @@ import googleapiclient.errors
 import googleapiclient.http
 import google.oauth2.service_account
 import httplib2
+from google.auth.jwt import Credentials as JWTCredentials
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -53,6 +57,7 @@ from gam import fileutils
 from gam.gapi import calendar as gapi_calendar
 from gam.gapi import cloudidentity as gapi_cloudidentity
 from gam.gapi import cbcm as gapi_cbcm
+from gam.gapi import chat as gapi_chat
 from gam.gapi import chromehistory as gapi_chromehistory
 from gam.gapi import chromemanagement as gapi_chromemanagement
 from gam.gapi import chromepolicy as gapi_chromepolicy
@@ -761,8 +766,7 @@ def doGAMVersion(checkForArgs=True):
         return
     pyversion = platform.python_version()
     cpu_bits = struct.calcsize('P') * 8
-    api_client_ver = pkg_resources.get_distribution(
-        'google-api-python-client').version
+    api_client_ver = lib_version('google-api-python-client')
     print(
         (f'GAM {GAM_VERSION} - {GAM_URL} - {GM_Globals[GM_GAM_TYPE]}\n'
          f'{GAM_AUTHOR}\n'
@@ -784,6 +788,21 @@ def doGAMVersion(checkForArgs=True):
         doGAMCheckForUpdates(forceCheck=True)
     if extended:
         print(ssl.OPENSSL_VERSION)
+        libs = ['cryptography',
+                'filelock',
+                'google-auth-httplib2',
+                'google-auth-oauthlib',
+                'google-auth',
+                'httplib2',
+                'passlib',
+                'python-dateutil',
+                'yubikey-manager',
+                ]
+        for lib in libs:
+            try:
+                print(f'{lib} {lib_version(lib)}')
+            except:
+                pass
         tls_ver, cipher_name, used_ip = _getServerTLSUsed(testLocation)
         print(
             f'{testLocation} ({used_ip}) connects using {tls_ver} {cipher_name}'
@@ -822,21 +841,33 @@ def _getSvcAcctData():
             controlflow.system_error_exit(6, None)
         GM_Globals[GM_OAUTH2SERVICE_JSON_DATA] = json.loads(json_string)
 
-
-def getSvcAcctCredentials(scopes, act_as):
+jwt_apis = ['chat'] # APIs which can handle OAuthless JWT tokens
+def getSvcAcctCredentials(scopes, act_as, api=None):
     try:
         _getSvcAcctData()
         sign_method = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
-        if sign_method == 'default':
-            credentials = google.oauth2.service_account.Credentials.from_service_account_info(
-                GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
-        elif sign_method == 'yubikey':
-            yksigner = yubikey.YubiKey(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
-            credentials = google.oauth2.service_account.Credentials._from_signer_and_info(yksigner,
-                GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
-        credentials = credentials.with_scopes(scopes)
-        if act_as:
-            credentials = credentials.with_subject(act_as)
+        if act_as or api not in jwt_apis:
+            if sign_method == 'default':
+                credentials = google.oauth2.service_account.Credentials.from_service_account_info(
+                    GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+            elif sign_method == 'yubikey':
+                yksigner = yubikey.YubiKey(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+                credentials = google.oauth2.service_account.Credentials._from_signer_and_info(yksigner,
+                    GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+            credentials = credentials.with_scopes(scopes)
+            if act_as:
+                credentials = credentials.with_subject(act_as)
+        else:
+            audience = f'https://{api}.googleapis.com/'
+            if sign_method == 'default':
+                credentials = JWTCredentials.from_service_account_info(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA],
+                                                            audience=audience)
+            elif sign_method == 'yubikey':
+                yksigner = yubikey.YubiKey(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+                credentials = JWTCredentials._from_signer_and_info(yksigner,
+                                                                   GM_Globals[GM_OAUTH2SERVICE_JSON_DATA],
+                                                                   audience=audience)
+            credentials.project_id = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA]['project_id']
         GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = GM_Globals[
             GM_OAUTH2SERVICE_JSON_DATA]['client_id']
         return credentials
@@ -1087,14 +1118,17 @@ def convertEmailAddressToUID(emailAddressOrUID, cd=None, email_type='user'):
     return normalizedEmailAddressOrUID
 
 
-def buildGAPIServiceObject(api, act_as, showAuthError=True):
+def buildGAPIServiceObject(api, act_as, showAuthError=True, scopes=None):
     httpObj = transport.create_http(cache=GM_Globals[GM_CACHE_DIR])
     service = getService(api, httpObj)
     GM_Globals[GM_CURRENT_API_USER] = act_as
-    GM_Globals[GM_CURRENT_API_SCOPES] = API_SCOPE_MAPPING.get(
-        api, service._rootDesc['auth']['oauth2']['scopes'])
+    if scopes:
+        GM_Globals[GM_CURRENT_API_SCOPES] = scopes
+    else:
+        GM_Globals[GM_CURRENT_API_SCOPES] = API_SCOPE_MAPPING.get(
+              api, service._rootDesc['auth']['oauth2']['scopes'])
     credentials = getSvcAcctCredentials(GM_Globals[GM_CURRENT_API_SCOPES],
-                                        act_as)
+                                        act_as, api)
     request = transport.create_request(httpObj)
     retries = 3
     for n in range(1, retries + 1):
@@ -5417,9 +5451,9 @@ def printShowLabels(users, show=True):
                 label['email'] = user
         if not show:
             display.write_csv_file(labels,
-                           titles,
-                           list_type='Gmail Labels',
-                           todrive=False)
+                                   titles,
+                                   'Gmail Labels',
+                                   todrive)
 
 
 def showGmailProfile(users):
@@ -7220,19 +7254,19 @@ def enableGAMProjectAPIs(GAMProjectAPIs,
     return status
 
 
-def _grantSARotateRights(iam, sa_email):
-    print(f'Giving service account {sa_email} rights to rotate own private key')
+def _grantRotateRights(iam, service_account, email, account_type='serviceAccount'):
+    print(f'Giving account {email} rights to rotate {service_account} private key')
     body = {
         'policy': {
             'bindings': [{
                 'role': 'roles/iam.serviceAccountKeyAdmin',
-                'members': [f'serviceAccount:{sa_email}']
+                'members': [f'{account_type}:{email}']
             }]
         }
     }
     gapi.call(iam.projects().serviceAccounts(),
               'setIamPolicy',
-              resource=f'projects/-/serviceAccounts/{sa_email}',
+              resource=f'projects/-/serviceAccounts/{service_account}',
               body=body)
 
 
@@ -7324,11 +7358,12 @@ def _createClientSecretsOauth2service(httpObj, projectId, login_hint):
                                     })
         GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = service_account[
             'uniqueId']
+    sa_email = service_account['name'].rsplit('/', 1)[-1]
     doCreateOrRotateServiceAccountKeys(iam,
                                        project_id=service_account['projectId'],
                                        client_email=service_account['email'],
                                        client_id=service_account['uniqueId'])
-    _grantSARotateRights(iam, service_account['name'].rsplit('/', 1)[-1])
+    _grantRotateRights(iam, sa_email, sa_email)
     console_url = f'https://console.cloud.google.com/apis/credentials/oauthclient?project={projectId}'
     while True:
         print(f'''Please go to:
@@ -7709,7 +7744,7 @@ def doUpdateProjects():
         iam = getService('iam', httpObj)
         _getSvcAcctData()  # needed to read in GM_OAUTH2SERVICE_JSON_DATA
         sa_email = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA]['client_email']
-        _grantSARotateRights(iam, sa_email)
+        _grantRotateRights(iam, sa_email, sa_email)
 
 
 def _generatePrivateKeyAndPublicCert(client_id, key_size):
@@ -7907,6 +7942,7 @@ def doCreateOrRotateServiceAccountKeys(iam=None,
                     iam.projects().serviceAccounts().keys(),
                     'upload',
                     throw_reasons=throw_reasons,
+                    retry_reasons=[gapi_errors.ErrorReason.FOUR_O_THREE],
                     name=sa_name,
                     body={'publicKeyData': publicKeyData})
                 break
@@ -7931,6 +7967,7 @@ def doCreateOrRotateServiceAccountKeys(iam=None,
         result = gapi.call(iam.projects().serviceAccounts().keys(),
                            'create',
                            name=sa_name,
+                           retry_reasons=[gapi_errors.ErrorReason.FOUR_O_THREE],
                            body=body)
         new_data_str = base64.b64decode(
             result['privateKeyData']).decode(UTF8)
@@ -7956,6 +7993,7 @@ def doCreateOrRotateServiceAccountKeys(iam=None,
                 print(f'  Revoking existing key {keyName} for service account')
                 gapi.call(iam.projects().serviceAccounts().keys(),
                           'delete',
+                          retry_reasons=[gapi_errors.ErrorReason.FOUR_O_THREE],
                           name=key['name'])
                 if mode != 'retainnone':
                     break
@@ -9018,31 +9056,31 @@ def doGetUserInfo(user_email=None):
             print('No access to show user groups.')
     elif getCIGroups:
         memberships = gapi_cloudidentity_groups.get_membership_graph(user_email)
-        print('\nGroup Mmebership Tree:')
-        group_name_mapping = {}
-        group_displayname_mapping = {}
-        groups = memberships.get('groups', [])
-        for group in groups:
-            group_name = group.get('name')
-            group_key = group.get('groupKey', {})
-            group_email = group_key.get('id', '')
-            group_display_name = group.get('displayName', '')
-            group_name_mapping[group_name] = group_email
-            group_displayname_mapping[group_email] = group_display_name
-        edges = []
-        seen_group_count = {}
-        groups_with_multi_memberships = []
-        for adj in memberships.get('adjacencyList', []):
-            group_name = adj.get('group', '')
-            group_email = group_name_mapping[group_name]
-            for edge in adj.get('edges', []):
-                seen_group_count[group_email] = seen_group_count.get(group_email, 0) + 1
-                member_email = edge.get('preferredMemberKey', {}).get('id')
-                edges.append((member_email, group_email))
-        print_group_map(user_email, group_displayname_mapping, seen_group_count, edges, spaces=3, direct=True)
-        if max(seen_group_count.values()) > 1:
-            print()
-            print('   * user has multiple direct or inherited memberships in group')
+        print('Group Membership Tree:')
+        if memberships:
+            group_name_mapping = {}
+            group_displayname_mapping = {}
+            groups = memberships.get('groups', [])
+            for group in groups:
+                group_name = group.get('name')
+                group_key = group.get('groupKey', {})
+                group_email = group_key.get('id', '')
+                group_display_name = group.get('displayName', '')
+                group_name_mapping[group_name] = group_email
+                group_displayname_mapping[group_email] = group_display_name
+            edges = []
+            seen_group_count = {}
+            for adj in memberships.get('adjacencyList', []):
+                group_name = adj.get('group', '')
+                group_email = group_name_mapping[group_name]
+                for edge in adj.get('edges', []):
+                    seen_group_count[group_email] = seen_group_count.get(group_email, 0) + 1
+                    member_email = edge.get('memberKey', {}).get('id')
+                    edges.append((member_email, group_email))
+            print_group_map(user_email, group_displayname_mapping, seen_group_count, edges, 3, 'direct')
+            if seen_group_count and max(seen_group_count.values()) > 1:
+                print()
+                print('   * user has multiple direct or inherited memberships in group')
         print()
     if getLicenses:
         print('Licenses:')
@@ -9059,19 +9097,15 @@ def doGetUserInfo(user_email=None):
         for user_license in user_licenses:
             print(f'  {gapi_licensing._formatSKUIdDisplayName(user_license)}')
 
-def print_group_map(parent, group_name_mappings, seen_group_count, edges, spaces=3, direct=False):
+def print_group_map(parent, group_name_mappings, seen_group_count, edges, spaces, direction):
     for a_parent, a_child in edges:
         if a_parent == parent:
             group_display_name = group_name_mappings[a_child]
-            if direct:
-                direction = 'direct'
-            else:
-                direction = 'inherited'
             output = f'{" " * spaces}{group_display_name} <{a_child}> ({direction})'
             if seen_group_count[a_child] > 1:
                 output += ' *'
             print(output)
-            print_group_map(a_child, group_name_mappings, seen_group_count, edges, spaces+2)
+            print_group_map(a_child, group_name_mappings, seen_group_count, edges, spaces+2, 'inherited')
 
 def doGetAliasInfo(alias_email=None):
     cd = buildGAPIObject('directory')
@@ -11346,6 +11380,8 @@ def ProcessGAMCommand(args):
                 gapi_cbcm.createtoken()
             elif argument in ['printer']:
                 gapi_directory_printers.create()
+            elif argument in ['chatmessage']:
+                gapi_chat.create_message()
             else:
                 controlflow.invalid_argument_exit(argument, 'gam create')
             sys.exit(0)
@@ -11408,6 +11444,8 @@ def ProcessGAMCommand(args):
                 gapi_chromepolicy.update_policy()
             elif argument in ['printer']:
                 gapi_directory_printers.update()
+            elif argument in ['chatmessage']:
+                gapi_chat.update_message()
             else:
                 controlflow.invalid_argument_exit(argument, 'gam update')
             sys.exit(0)
@@ -11544,6 +11582,8 @@ def ProcessGAMCommand(args):
                 gapi_directory_printers.delete()
             elif argument == 'chromepolicy':
                 gapi_chromepolicy.delete_policy()
+            elif argument == 'chatmessage':
+                gapi_chat.delete_message()
             else:
                 controlflow.invalid_argument_exit(argument, 'gam delete')
             sys.exit(0)
@@ -11659,6 +11699,10 @@ def ProcessGAMCommand(args):
                 gapi_chromemanagement.printVersions()
             elif argument in ['chromehistory']:
                 gapi_chromehistory.printHistory()
+            elif argument in ['chatspaces']:
+                gapi_chat.print_spaces()
+            elif argument in ['chatmembers']:
+                gapi_chat.print_members()
             else:
                 controlflow.invalid_argument_exit(argument, 'gam print')
             sys.exit(0)

src/gam/gapi/chat.py

@@ -0,0 +1,207 @@
+import sys
+
+import googleapiclient.errors
+
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
+from gam.gapi import errors as gapi_errors
+
+# Chat scope isn't in discovery doc so need to manually set
+CHAT_SCOPES = ['https://www.googleapis.com/auth/chat.bot']
+
+
+def build():
+    return gam.buildGAPIServiceObject('chat',
+                                      act_as=None,
+                                      scopes=CHAT_SCOPES)
+
+
+THROW_REASONS = [
+        gapi_errors.ErrorReason.FOUR_O_FOUR,     # Chat API not configured
+        ]
+
+def _chat_error_handler(chat, err):
+    if err.status_code == 404:
+        project_id = chat._http.credentials.project_id
+        url = f'https://console.cloud.google.com/apis/api/chat.googleapis.com/hangouts-chat?project={project_id}'
+        print('ERROR: you need to configure Google Chat for your API project. Please go to:')
+        print()
+        print(f'  {url}')
+        print()
+        print('and complete all fields.')
+    else:
+        raise err
+    sys.exit(1)
+
+
+def print_spaces():
+    chat = build()
+    todrive = False
+    i =3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam print chatspaces')
+    try:
+        spaces = gapi.get_all_pages(chat.spaces(), 'list', 'spaces', throw_reasons=THROW_REASONS)
+    except googleapiclient.errors.HttpError as err:
+        _chat_error_handler(chat, err)
+    if not spaces:
+        print('Bot not added to any Chat rooms or users yet.')
+    else:
+        display.write_csv_file(spaces, spaces[0].keys(), 'Chat Spaces', todrive)
+
+
+def print_members():
+    chat = build()
+    space = None
+    todrive = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'space':
+            space = sys.argv[i+1]
+            if space[:7] != 'spaces/':
+                space = f'spaces/{space}'
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print chatmembers")
+    if not space:
+        controlflow.system_error_exit(2,
+                                      'space <ChatSpace> is required.')
+    try:
+        results = gapi.get_all_pages(chat.spaces().members(), 'list', 'memberships', parent=space)
+    except googleapiclient.errors.HttpError as err:
+        _chat_error_handler(chat, err)
+    members = []
+    titles = []
+    for result in results:
+        member = utils.flatten_json(result)
+        for key in member:
+            if key not in titles:
+                titles.append(key)
+        members.append(utils.flatten_json(result))
+    display.write_csv_file(members, titles, 'Chat Members', todrive)
+
+
+def create_message():
+    chat = build()
+    body = {}
+    i = 3
+    while i < len(sys.argv):
+      myarg = sys.argv[i].lower()
+      if myarg == 'text':
+          body['text'] = sys.argv[i+1].replace('\\r', '\r').replace('\\n', '\n')
+          i += 2
+      elif myarg == 'textfile':
+          filename = sys.argv[i + 1]
+          i, encoding = gam.getCharSet(i + 2)
+          body['text'] = fileutils.read_file(filename, encoding=encoding)
+      elif myarg == 'space':
+          space = sys.argv[i+1]
+          if space[:7] != 'spaces/':
+              space = f'spaces/{space}'
+          i += 2
+      elif myarg == 'thread':
+          body['thread'] = {'name': sys.argv[i+1]}
+          i += 2
+      else:
+          controlflow.invalid_argument_exit(myarg, "gam create chat")
+    if not space:
+        controlflow.system_error_exit(2,
+                                      'space <ChatSpace> is required.')
+    if 'text' not in body:
+        controlflow.system_error_exit(2,
+                                      'text <String> or textfile <FileName> is required.')
+    if len(body['text']) > 4096:
+        body['text'] = body['text'][:4095]
+        print('WARNING: trimmed message longer than 4k to be 4k in length.')
+    try:
+        resp = gapi.call(chat.spaces().messages(),
+                         'create',
+                          parent=space,
+                          body=body,
+                          throw_reasons=THROW_REASONS)
+    except googleapiclient.errors.HttpError as err:
+        _chat_error_handler(chat, err)
+    if 'thread' in body:
+        print(f'responded to thread {resp["thread"]["name"]}')
+    else:
+        print(f'started new thread {resp["thread"]["name"]}')
+    print(f'message {resp["name"]}')
+
+def delete_message():
+    chat = build()
+    name = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'name':
+            name = sys.argv[i+1]
+            i += 2
+        else:
+          controlflow.invalid_argument_exit(myarg, "gam delete chat")
+    if not name:
+        controlflow.system_error_exit(2,
+                                      'name <String> is required.')
+    try:
+        gapi.call(chat.spaces().messages(),
+                  'delete',
+                  name=name)
+    except googleapiclient.errors.HttpError as err:
+        _chat_error_handler(chat, err)
+
+
+def update_message():
+    chat = build()
+    body = {}
+    name = None
+    updateMask = 'text'
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'text':
+            body['text'] = sys.argv[i+1].replace('\\r', '\r').replace('\\n', '\n')
+            i += 2
+        elif myarg == 'textfile':
+            filename = sys.argv[i + 1]
+            i, encoding = gam.getCharSet(i + 2)
+            body['text'] = fileutils.read_file(filename, encoding=encoding)
+        elif myarg == 'name':
+            name = sys.argv[i+1]
+            i += 2
+        else:
+          controlflow.invalid_argument_exit(myarg, "gam update chat")
+    if not name:
+        controlflow.system_error_exit(2,
+                                      'name <String> is required.')
+    if 'text' not in body:
+        controlflow.system_error_exit(2,
+                                      'text <String> or textfile <FileName> is required.')
+    if len(body['text']) > 4096:
+        body['text'] = body['text'][:4095]
+        print('WARNING: trimmed message longer than 4k to be 4k in length.')
+    try:
+        resp = gapi.call(chat.spaces().messages(),
+                         'update',
+                         name=name,
+                         updateMask=updateMask,
+                         body=body)
+    except googleapiclient.errors.HttpError as err:
+        _chat_error_handler(chat, err)
+    if 'thread' in body:
+        print(f'updated response to thread {resp["thread"]["name"]}')
+    else:
+        print(f'updated message on thread {resp["thread"]["name"]}')
+    print(f'message {resp["name"]}')

src/gam/gapi/cloudidentity/groups.py

@@ -13,12 +13,8 @@ from gam.gapi import cloudidentity as gapi_cloudidentity
 from gam.gapi.directory import customer as gapi_directory_customer
 
 
-def build():
-    return gapi_cloudidentity.build('cloudidentity')
-
-
 def create():
-    ci = build()
+    ci = gapi_cloudidentity.build()
     initialGroupConfig = 'EMPTY'
     gapi_directory_customer.setTrueCustomerId()
     parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
@@ -48,7 +44,6 @@ def create():
                 body['additionalGroupKeys'].append({'id': alias})
             i += 2
         elif myarg in ['dynamic']:
-            # As of 2020/06/25 this doesn't work (yet?)
             body['dynamicGroupMetadata'] = {
                 'queries': [{
                     'query': sys.argv[i + 1],
@@ -70,7 +65,7 @@ def create():
 
 
 def delete():
-    ci = build()
+    ci = gapi_cloudidentity.build()
     group = sys.argv[3]
     name = group_email_to_id(ci, group)
     print(f'Deleting group {group}')
@@ -78,7 +73,7 @@ def delete():
 
 
 def info():
-    ci = build()
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
     group = gam.normalizeEmailAddressOrUID(sys.argv[3])
     getUsers = True
     showJoinDate = True
@@ -118,10 +113,11 @@ def info():
                                      fields='*',
                                      pageSize=pageSize,
                                      view=view)
-        print('Members:')
+        print(' Members:')
         for member in members:
             role = get_single_role(member.get('roles', [])).lower()
             email = member.get('memberKey', {}).get('id')
+            member_type = member.get('type', 'USER').lower()
             jc_string = ''
             if showJoinDate:
                 joined = member.get('createTime', 'Unknown')
@@ -129,47 +125,39 @@ def info():
             if showUpdateDate:
                 updated = member.get('updateTime', 'Unknown')
                 jc_string += f'  updated {updated}'
-            print(
-                f'{role}: {email}{jc_string}'
-                # f' {member.get("role", ROLE_MEMBER).lower()}: {member.get("email", member["id"])} ({member["type"].lower()})'
-            )
+            print(f'  {role}: {email} ({member_type}){jc_string}')
         print(f'Total {len(members)} users in group')
     elif showMemberTree:
-        print(' Member tree:')
-        global cached_group_members
+        print(' Membership Tree:')
         cached_group_members = {}
-        print_member_tree(ci, name)
+        print_member_tree(ci, name, cached_group_members, 2, True)
 
 
-def print_member_tree(ci, group_id, spaces=2):
+def print_member_tree(ci, group_id, cached_group_members, spaces, show_role):
     if not group_id in cached_group_members:
         cached_group_members[group_id] = gapi.get_all_pages(ci.groups().memberships(),
                                                             'list',
                                                             'memberships',
                                                             parent=group_id,
+                                                            view='FULL',
                                                             fields='*',
                                                             pageSize=1000)
     for member in cached_group_members[group_id]:
         member_id = member.get('name', '')
         member_id = member_id.split('/')[-1]
-        if member_id.isdigit():
-            member_type = 'user'
+        email = member.get('memberKey', {}).get('id')
+        member_type = member.get('type', 'USER').lower()
+        if show_role:
+            role = get_single_role(member.get('roles', [])).lower()
+            print(f'{" " * spaces}{role}: {email} ({member_type})')
         else:
-            member_type = 'group'
-        member_email = member.get('preferredMemberKey', {}).get('id')
-        relation_type = member.get('relationType', '').lower()
-        if member_type == 'user':
-            print(f'{" " * spaces}{member_email} - user')
-        elif member_type == 'group':
-            print(f'{" " * spaces}{member_email} - group')
-            group_id = group_email_to_id(ci, member_email)
-            print_member_tree(ci, group_id, spaces + 2)
-        else:
-            print(f'unknown member type: {member_type} for {member_email}')
+            print(f'{" " * spaces}{email} ({member_type})')
+        if member_type == 'group':
+            print_member_tree(ci, f'groups/{member_id}', cached_group_members, spaces + 2, False)
 
 
 def info_member():
-    ci = build()
+    ci = gapi_cloudidentity.build()
     member = gam.normalizeEmailAddressOrUID(sys.argv[3])
     group = gam.normalizeEmailAddressOrUID(sys.argv[4])
     group_name = gapi.call(ci.groups(),
@@ -199,7 +187,7 @@ GROUP_ROLES_MAP = {
 
 
 def print_():
-    ci = build()
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
     i = 3
     members = membersCountOnly = managers = managersCountOnly = owners = ownersCountOnly = False
     gapi_directory_customer.setTrueCustomerId()
@@ -287,7 +275,7 @@ def print_():
         except googleapiclient.errors.HttpError:
             controlflow.system_error_exit(
                 2,
-                f'enterprisemember requires Enterprise license')
+                'enterprisemember requires Enterprise license')
         entityList = []
         for entity in result:
             if entity['relationType'] == 'DIRECT':
@@ -342,7 +330,7 @@ def print_():
                 ownersCount = 0
             for member in groupMembers:
                 member_email = member['memberKey']['id']
-                role = get_single_role(member.get('roles'))
+                role = get_single_role(member.get('roles', []))
                 if not validRoles or role in validRoles:
                     if role == ROLE_MEMBER:
                         if members:
@@ -385,7 +373,7 @@ def print_():
 
 def _get_groups_list(ci=None, member=None, parent=None):
     if not ci:
-        ci = build()
+        ci = gapi_cloudidentity.build()
     if not parent:
             gapi_directory_customer.setTrueCustomerId()
             parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
@@ -407,7 +395,7 @@ def _get_groups_list(ci=None, member=None, parent=None):
         except googleapiclient.errors.HttpError:
             controlflow.system_error_exit(
                     2,
-                    f'enterprisemember requires Enterprise license')
+                    'enterprisemember requires Enterprise license')
         return [group['groupKey']['id'] for group in groups_to_get if group['relationType'] == 'DIRECT']
     else:
         groups_to_get = gapi.get_all_pages(
@@ -424,7 +412,7 @@ def _get_groups_list(ci=None, member=None, parent=None):
 
 
 def get_membership_graph(member):
-    ci = build()
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
     query = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
     result = gapi.call(ci.groups().memberships(),
                      'getMembershipGraph',
@@ -434,7 +422,7 @@ def get_membership_graph(member):
 
 
 def print_members():
-    ci = build()
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
     todrive = False
     gapi_directory_customer.setTrueCustomerId()
     parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
@@ -550,7 +538,7 @@ def update():
             ]
         return (role, expireTime, users_email)
 
-    ci = build()
+    ci = gapi_cloudidentity.build('cloudidentity_beta')
     group = sys.argv[3]
     myarg = sys.argv[4].lower()
     items = []
@@ -834,6 +822,14 @@ def update():
                     'cloudidentity.googleapis.com/groups.discussion_forum': ''
                 }
                 i += 1
+            elif myarg in ['dynamic']:
+                body['dynamicGroupMetadata'] = {
+                    'queries': [{
+                        'query': sys.argv[i + 1],
+                        'resourceType': 'USER'
+                    }]
+                }
+                i += 2
             else:
                 controlflow.invalid_argument_exit(sys.argv[i],
                                                   'gam update cigroup')

src/gam/gapi/errors.py

@@ -119,6 +119,7 @@ class ErrorReason(Enum):
     FIVE_O_THREE = '503'
     FOUR_O_NINE = '409'
     FOUR_O_O = '400'
+    FOUR_O_FOUR = '404'
     FOUR_O_THREE = '403'
     FOUR_TWO_NINE = '429'
     GATEWAY_TIMEOUT = 'gatewayTimeout'

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.02'
+GAM_VERSION = '6.05'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
@@ -1520,7 +1520,7 @@ USER_EXTERNALID_TYPES = [
 ]
 USER_GENDER_TYPES = ['female', 'male', 'unknown']
 USER_IM_TYPES = ['home', 'work', 'other']
-USER_KEYWORD_TYPES = ['occupation', 'outlook']
+USER_KEYWORD_TYPES = ['occupation', 'outlook', 'mission']
 USER_LOCATION_TYPES = ['default', 'desk']
 USER_ORGANIZATION_TYPES = ['domain_only', 'school', 'unknown', 'work']
 USER_PHONE_TYPES = [
@@ -1535,7 +1535,7 @@ USER_RELATION_TYPES = [
 ]
 USER_WEBSITE_TYPES = [
     'app_install_page', 'blog', 'ftp', 'home', 'home_page', 'other', 'profile',
-    'reservations', 'work'
+    'reservations', 'resume', 'work'
 ]
 
 WEBCOLOR_MAP = {

src/requirements.txt

@@ -6,6 +6,7 @@ google-auth-httplib2
 google-auth-oauthlib>=0.4.1
 google-auth>=1.11.2
 httplib2>=0.17.0
+importlib.metadata; python_version < '3.8'
 passlib>=1.7.2
 python-dateutil
 yubikey-manager>=4.0.0