6.58, workaround for #1625

* Update documentation: gam delete inboundssoassignment <SSOAssignmentSelector>

* tsv is a valid Google Doc extension

.github/actions/decrypt.sh

@@ -14,4 +14,5 @@ gpg --quiet --batch --yes --decrypt --passphrase="${PASSCODE}" \
     --output "${credsfile}" "${gpgfile}"
 
 tar xvvf "${credsfile}" --directory "${gampath}"
-ls -l "${gampath}"
+rm -rvf "${gpgfile}"
+rm -rvf "${credsfile}"

.github/workflows/build.yml

@@ -6,13 +6,17 @@ on:
   schedule:
     - cron: '37 22 * * *'
 
+permissions:
+  contents: read
+  id-token: write
+
 defaults:
   run:
     shell: bash
     working-directory: src
 
 env:
-  OPENSSL_CONFIG_OPTS: no-fips
+  OPENSSL_CONFIG_OPTS: no-fips --api=3.0.0
   OPENSSL_INSTALL_PATH: ${{ github.workspace }}/bin/ssl
   OPENSSL_SOURCE_PATH: ${{ github.workspace }}/src/openssl
   PYTHON_INSTALL_PATH: ${{ github.workspace }}/bin/python
@@ -22,74 +26,114 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - os: ubuntu-22.04
+          - os: ubuntu-20.04
             jid: 1
             goal: build
             arch: x86_64
             openssl_archs: linux-x86_64
-          - os: [self-hosted, linux, arm64]
+          - os: [self-hosted, linux, arm64, gcp]
             jid: 2
             goal: build
             arch: aarch64
             openssl_archs: linux-aarch64
-          - os: macos-12
+          - os: ubuntu-20.04
+            jid: 3
+            goal: build
+            arch: x86_64
+            openssl_archs: linux-x86_64
+            staticx: yes
+          - os: [self-hosted, linux, arm64, gcp]
             jid: 4
             goal: build
+            arch: aarch64
+            openssl_archs: linux-aarch64
+            staticx: yes
+          - os: macos-12
+            jid: 5
+            goal: build
             arch: universal2
             openssl_archs: darwin64-x86_64 darwin64-arm64
           - os: windows-2022
-            jid: 5
+            jid: 6
             goal: build
             arch: Win64
             openssl_archs: VC-WIN64A
           - os: windows-2022
-            jid: 6
+            jid: 7
             goal: build
             arch: Win32
             openssl_archs: VC-WIN32
-          - os: ubuntu-22.04
-            goal: test
-            python: "3.7"
-            jid: 7
-            arch: x86_64
           - os: ubuntu-22.04
             goal: test
             python: "3.8"
-            jid: 8
-            arch: x86_64
-          - os: ubuntu-22.04
-            goal: test
-            python: "3.9"
             jid: 9
             arch: x86_64
           - os: ubuntu-22.04
             goal: test
-            python: "3.11-dev"
+            python: "3.9"
             jid: 10
             arch: x86_64
+          - os: ubuntu-22.04
+            goal: test
+            python: "3.10"
+            jid: 11
+            arch: x86_64
+          - os: ubuntu-22.04
+            goal: test
+            python: "3.12.0-alpha - 3.12"
+            jid: 12
+            arch: x86_64
 
     steps:
-      - uses: actions/checkout@master
+
+      - uses: actions/checkout@v3
         with:
           persist-credentials: false
           fetch-depth: 0
 
+      - id: auth
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: projects/297925809119/locations/global/workloadIdentityPools/gha-pool/providers/gha-provider
+          service_account: github-actions-testing-for-gam@gam-project-wyo-lub-ivl.iam.gserviceaccount.com
+
       - name: Cache multiple paths
-        uses: actions/cache@v2
+        if: matrix.goal == 'build'
+        uses: actions/cache@v3
         id: cache-python-ssl
         with:
           path: |
-            bin
-          key: gam-${{ matrix.jid }}-20220802
+            bin.tar.xz
+            src/cpython
+          key: gam-${{ matrix.jid }}-20230417
+
+      - name: Untar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar xvvf bin.tar.xz
 
       - name: Use pre-compiled Python for testing
         if: matrix.python != ''
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Set cURL retry flag
+        run: |
+          curl_version=$(curl --version | head -n 1 | awk '{ print $2 }')
+          echo "cURL is ${curl_version}"
+          if [ "$curl_version" == "7.68.0" ]; then
+            export curl_retry="--retry 5 --retry-connrefused"
+          else
+            export curl_retry="--retry 5 --retry-all-errors"
+          fi
+          echo "curl_retry=${curl_retry}" >> $GITHUB_ENV
+
       - name: Set env variables for test
         if: matrix.goal == 'test'
         env:
@@ -117,17 +161,11 @@ jobs:
           sudo apt-get -qq --yes update
           sudo apt-get -qq --yes install swig libpcsclite-dev
 
-      - name: MacOS remove Homebrew
-        if: runner.os == 'macOS'
-        run: |
-          # remove everything except the libraries needed by yubikey-manager
-          brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
-
       - name: MacOS install tools
         if: runner.os == 'macOS'
         run: |
           # Install latest Rust
-          curl -fsS -o rust.sh https://sh.rustup.rs
+          curl $curl_retry -fsS -o rust.sh https://sh.rustup.rs
           bash ./rust.sh -y
           source $HOME/.cargo/env
           # needed for Rust to compile cryptography Python package for universal2
@@ -145,8 +183,10 @@ jobs:
           arch: ${{ matrix.arch }}
           jid: ${{ matrix.jid }}
           openssl_archs: ${{ matrix.openssl_archs }}
+          staticx: ${{ matrix.staticx }}
         run: |
           echo "We are running on ${RUNNER_OS}"
+          LD_LIBRARY_PATH="${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib:/usr/local/lib"
           if [[ "${arch}" == "Win64" ]]; then
             PYEXTERNALS_PATH="amd64"
             PYBUILDRELEASE_ARCH="x64"
@@ -168,7 +208,7 @@ jobs:
             PERL=perl
             echo "MACOSX_DEPLOYMENT_TARGET=10.15" >> $GITHUB_ENV
             echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
-            echo "PIP_ARGS=--no-binary=:all:" >> $GITHUB_ENV
+            #echo "PIP_ARGS=--no-binary=:all:" >> $GITHUB_ENV
           elif [[ "${RUNNER_OS}" == "Linux" ]]; then
             MAKE=make
             MAKEOPT="-j$(nproc)"
@@ -178,21 +218,22 @@ jobs:
             MAKE=nmake
             MAKEOPT=""
             PERL="c:\strawberry\perl\bin\perl.exe"
-            echo "PYTHON=${PYTHON_INSTALL_PATH}\python.exe" >> $GITHUB_ENV
+            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}"
+            echo "PYTHON=${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}/python.exe" >> $GITHUB_ENV
             echo "GAM_ARCHIVE_ARCH=${GAM_ARCHIVE_ARCH}" >> $GITHUB_ENV
             echo "WIX_ARCH=${WIX_ARCH}" >> $GITHUB_ENV
           fi
           echo "We'll run make with: ${MAKEOPT}"
           echo "JID=${jid}" >> $GITHUB_ENV
+          echo "staticx=${staticx}" >> $GITHUB_ENV
           echo "arch=${arch}" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
           echo "MAKE=${MAKE}" >> $GITHUB_ENV
           echo "MAKEOPT=${MAKEOPT}" >> $GITHUB_ENV
           echo "PERL=${PERL}" >> $GITHUB_ENV
           echo "PYEXTERNALS_PATH=${PYEXTERNALS_PATH}" >> $GITHUB_ENV
           echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
           echo "openssl_archs=${openssl_archs}" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib" >> $GITHUB_ENV
-          #echo "PATH=${PATH}:${PYTHON_INSTALL_PATH}/scripts" >> $GITHUB_ENV
 
       - name: Get latest stable OpenSSL source
         if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
@@ -268,7 +309,7 @@ jobs:
             rm -rf ${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64
             echo "LDFLAGS=-L${OPENSSL_INSTALL_PATH}/lib" >> $GITHUB_ENV
             echo "CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS=1" >> $GITHUB_ENV
-            echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include -arch arm64 -arch x86_64" >> $GITHUB_ENV
+            echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include -arch arm64 -arch x86_64 ${CFLAGS}" >> $GITHUB_ENV
             echo "ARCHFLAGS=-arch x86_64 -arch arm64" >> $GITHUB_ENV
           else
             cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_archs}"
@@ -280,6 +321,7 @@ jobs:
         if: matrix.goal == 'build'
         run: |
           "${OPENSSL_INSTALL_PATH}/bin/openssl" version
+          "${OPENSSL_INSTALL_PATH}/bin/openssl" version -f
           file "${OPENSSL_INSTALL_PATH}/bin/openssl"
 
       - name: Get latest stable Python source
@@ -330,7 +372,7 @@ jobs:
           $env:OPENSSL_EXT_TARGET_PATH = "${env:OPENSSL_EXT_PATH}${env:PYEXTERNALS_PATH}"
           echo "Copying our OpenSSL to ${env:OPENSSL_EXT_TARGET_PATH}"
           mkdir "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
-          Copy-Item -Path "${env:GITHUB_WORKSPACE}/src/openssl-${env:openssl_archs}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE"
+          Copy-Item -Path "${env:GITHUB_WORKSPACE}/src/openssl-${env:openssl_archs}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE" -Verbose
           cp -v "$env:OPENSSL_INSTALL_PATH\lib\*" "${env:OPENSSL_EXT_TARGET_PATH}"
           cp -v "$env:OPENSSL_INSTALL_PATH\bin\*" "${env:OPENSSL_EXT_TARGET_PATH}"
           cp -v "$env:OPENSSL_INSTALL_PATH\include\openssl\*" "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
@@ -350,22 +392,10 @@ jobs:
         run: |
           cd "${env:PYTHON_SOURCE_PATH}"
           # We need out custom openssl.props which uses OpenSSL 3 DLL names
-          Copy-Item -Path "${env:GITHUB_WORKSPACE}\src\tools\openssl.props" -Destination PCBuild\
+          Copy-Item -Path "${env:GITHUB_WORKSPACE}\src\tools\openssl.props" -Destination PCBuild\ -Verbose
           echo "Building for ${env:PYBUILDRELEASE_ARCH}..."
           PCBuild\build.bat -m --pgo -c Release -p "${env:PYBUILDRELEASE_ARCH}"
 
-      - name: Windows Install Python
-        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
-        shell: powershell
-        run: |
-          cd "${env:PYTHON_SOURCE_PATH}"
-          mkdir "${env:PYTHON_INSTALL_PATH}\lib"
-          mkdir "${env:PYTHON_INSTALL_PATH}\include"
-          Copy-Item -Path "PCBuild\${env:PYEXTERNALS_PATH}\*" "${env:PYTHON_INSTALL_PATH}\"
-          Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Lib\*" "${env:PYTHON_INSTALL_PATH}\lib\" -recurse
-          Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Include\*" "${env:PYTHON_INSTALL_PATH}\include\" -recurse
-          Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\PC\*.h" "${env:PYTHON_INSTALL_PATH}\include\"
-
       - name: Mac/Linux Build Python
         if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
         run: |
@@ -379,6 +409,9 @@ jobs:
           cd "${PYTHON_SOURCE_PATH}"
           $MAKE altinstall
           $MAKE bininstall
+          export PATH="${PATH}:${PYTHON_INSTALL_PATH}/bin"
+          echo "PATH=${PATH}" >> $GITHUB_ENV
+          echo "PATH: ${PATH}"
 
       - name: Run Python
         run: |
@@ -386,56 +419,91 @@ jobs:
 
       - name: Upgrade pip, wheel, etc
         run: |
-          curl -O https://bootstrap.pypa.io/get-pip.py
+          curl $curl_retry -O https://bootstrap.pypa.io/get-pip.py
           "${PYTHON}" get-pip.py
           "${PYTHON}" -m pip install --upgrade pip
           "${PYTHON}" -m pip install --upgrade wheel
           "${PYTHON}" -m pip install --upgrade setuptools
-    
+
+      - name: Install pip requirements
+        run: |
+             echo "before anything..."
+             "${PYTHON}" -m pip list
+             if [[ "${RUNNER_OS}" == "macOS" ]]; then
+               # cffi is a dep of cryptography and doesn't ship
+               # a universal2 wheel so we must build one ourself :-/
+               export CFLAGS="-arch x86_64 -arch arm64"
+               export ARCHFLAGS="-arch x86_64 -arch arm64"
+               "${PYTHON}" -m pip install --upgrade --force-reinstall --no-binary :all: \
+                                 --no-cache-dir --no-deps --use-pep517 \
+                                 --use-feature=no-binary-enable-wheel-cache \
+                                 cffi 
+               echo "before cryptography..."
+               "${PYTHON}" -m pip list
+               # cryptography has a universal2 wheel but getting it installed
+               # on x86-64 MacOS is a royal pain in the keester.
+               "${PYTHON}" -m pip download --only-binary :all: \
+                                             --dest . \
+                                             --no-cache \
+                                             --no-deps \
+                                             --platform macosx_10_15_universal2 \
+                                             cryptography
+               "${PYTHON}" -m pip install --force-reinstall --no-deps cryptography*.whl
+               echo "after cryptography..."
+               "${PYTHON}" -m pip list
+               "${PYTHON}" -m pip install --upgrade --no-binary :all: -r requirements.txt
+             else
+               "${PYTHON}" -m pip install --upgrade -r requirements.txt
+               echo "after requirements..."
+               "${PYTHON}" -m pip list
+               "${PYTHON}" -m pip install --force-reinstall --no-deps --upgrade cryptography
+             fi
+             echo "after everything..."
+             "${PYTHON}" -m pip list
+
       - name: Install PyInstaller
         if: matrix.goal == 'build'
         run: |
           git clone https://github.com/pyinstaller/pyinstaller.git
           cd pyinstaller
           export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
+
+
           git checkout "${latest_release}"
           # remove pre-compiled bootloaders so we fail if bootloader compile fails
           rm -rvf PyInstaller/bootloader/*-*/*
           cd bootloader
-          if [[ "${arch}" == "Win32" ]]; then
-            export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
-          fi
+          case "${arch}" in
+            "Win32")
+              export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
+              ;;
+            "Win64")
+              export PYINSTALLER_BUILD_ARGS="--target-arch=64bit"
+              ;;
+          esac
           echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
           "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
-          cd ../..
+          cd ..
           echo "---- Installing PyInstaller ----"
-          "${PYTHON}" -m pip install pyinstaller
-
-      - name: Install pip requirements
-        run: |
-             if [[ "${RUNNER_OS}" == "macOS" ]]; then
-               for package in cryptography; do
-                 "${PYTHON}" -m pip install --upgrade cffi ${PIP_ARGS}
-                 "${PYTHON}" -m pip download --only-binary :all: \
-                                             --dest . \
-                                             --no-cache \
-                                             --no-deps \
-                                             --platform macosx_10_15_universal2 \
-                                             $package
-               "${PYTHON}" -m pip install --force-reinstall --no-deps $package*.whl
-               done
-               find $PYTHON_INSTALL_PATH/lib/python3.10/site-packages -type f -name "*.so" -exec du -sh "{}" \;
-             fi
-             "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
-             "${PYTHON}" -m pip list
+          "${PYTHON}" -m pip install .
 
       - name: Build GAM with PyInstaller
         if: matrix.goal != 'test'
         run: |
-             export gampath="./dist/gam"
+             if [[ "${staticx}" == "yes" ]]; then
+               export distpath="./dist/gam"
+               export gampath="${distpath}"
+             else
+               export distpath="./dist"
+               export gampath="${distpath}/gam"
+             fi
              mkdir -p -v "${gampath}"
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                export gampath=$($PYTHON -c "import os; print(os.path.realpath('$gampath'))")
+             elif [[ "${RUNNER_OS}" == "Windows" ]]; then
+               # Work around issue where PyInstaller picks up python3.dll from other Python versions
+               # https://github.com/pyinstaller/pyinstaller/issues/7102
+               export PATH="/usr/bin"
              else
                export gampath=$(realpath "${gampath}")
              fi
@@ -443,37 +511,30 @@ jobs:
              echo "gampath=${gampath}" >> $GITHUB_ENV
              echo "gam=${gam}" >> $GITHUB_ENV
              echo -e "GAM: ${gam}\nGAMPATH: ${gampath}"
-             "${PYTHON}" -m PyInstaller --clean --distpath="${gampath}" gam.spec
+             # TEMP force everything back to one file.
+             export PYINSTALLER_BUILD_ONEFILE="yes"
+             export distpath="./dist/gam"
+             export gampath="${distpath}"
+             "${PYTHON}" -m PyInstaller --clean --noconfirm --distpath="${distpath}" gam.spec
  
-      - name: Basic Tests all jobs
-        run: |
-             $PYTHON -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
-             $gam version extended
-             export GAMVERSION=$($gam version simple)
-             echo "GAM Version ${GAMVERSION}"
-             echo "GAMVERSION=${GAMVERSION}" >> $GITHUB_ENV
-
-      - name: Linux/MacOS package
-        if: runner.os != 'Windows' && matrix.goal == 'build'
+      - name: Copy extra package files
+        if: matrix.goal == 'build'
         run: |
+          cp -v roots.pem $gampath
           cp -v LICENSE $gampath
           cp -v GamCommands.txt $gampath
-          if [[ "${RUNNER_OS}" == "macOS" ]]; then
-              GAM_ARCHIVE="gam-${GAMVERSION}-macos-universal2.tar.xz"
-          elif [[ "${RUNNER_OS}" == "Linux" ]]; then
-              this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
-              GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(arch)-glibc${this_glibc_ver}.tar.xz"
+          if [[ "${RUNNER_OS}" == "Windows" ]]; then
+              cp -v gam-setup.bat $gampath
           fi
-          tar -C dist/ --create --verbose --exclude-from "${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" --file $GAM_ARCHIVE --xz gam
 
-      - name: Linux 64-bit install patchelf/staticx
-        if: runner.os == 'Linux' && contains(runner.arch, '64') && matrix.goal != 'test'
+      - name: Install StaticX
+        if: matrix.staticx == 'yes'
         run: |
           "${PYTHON}" -m pip install --upgrade patchelf-wrapper
           "${PYTHON}" -m pip install --upgrade staticx
 
-      - name: Linux 64-bit Make Static
-        if: runner.os == 'Linux' && contains(runner.arch, '64') && matrix.goal != 'test'
+      - name: Make StaticX
+        if: matrix.staticx == 'yes'
         run: |
           case $RUNNER_ARCH in
             X64)
@@ -485,25 +546,36 @@ jobs:
           esac
           echo "ldlib=${ldlib}"
           $PYTHON -m staticx -l "${ldlib}" "${gam}" "${gam}-staticx"
-          
-      - name: Linux Run StaticX-ed
-        if: runner.os == 'Linux' && contains(runner.arch, '64') && matrix.goal != 'test'
-        run: |
-          "${gam}-staticx" version extended
+          rm -v "${gam}"
           mv -v "${gam}-staticx" "${gam}"
 
-      - name: Linux package staticx
-        if: runner.os == 'Linux' && contains(runner.arch, '64') && matrix.goal != 'test'
+      - name: Basic Tests all jobs
+        id: basictests
         run: |
-          GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(uname -m)-legacy.tar.xz"
+             $PYTHON -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
+             $gam version extended
+             export GAMVERSION=$($gam version simple)
+             echo "GAM Version ${GAMVERSION}"
+             echo "GAMVERSION=${GAMVERSION}" >> $GITHUB_ENV
+
+      - name: Linux/MacOS package
+        if: runner.os != 'Windows' && matrix.goal == 'build'
+        run: |
+          if [[ "${RUNNER_OS}" == "macOS" ]]; then
+              GAM_ARCHIVE="gam-${GAMVERSION}-macos-universal2.tar.xz"
+          elif [[ "${RUNNER_OS}" == "Linux" ]]; then
+              if [[ "${staticx}" == "yes" ]]; then
+                libver="legacy"
+              else
+                libver="glibc$(ldd --version | awk '/ldd/{print $NF}')"
+              fi
+              GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(arch)-$libver}.tar.xz"
+          fi
           tar -C dist/ --create --verbose --exclude-from "${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" --file $GAM_ARCHIVE --xz gam
 
       - name: Windows package
         if: runner.os == 'Windows' && matrix.goal != 'test'
         run: |
-          cp -v LICENSE $gampath
-          cp -v GamCommands.txt $gampath
-          cp -v gam-setup.bat $gampath
           cd dist/
           GAM_ARCHIVE="../gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.zip"
           /c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam "-xr@${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" -bb3
@@ -540,47 +612,78 @@ jobs:
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                brew install gnupg
              fi
-             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.gpg creds.tar
+             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.xz.gpg creds.tar.xz
+             rm $gampath/oauth2service.json
              export OAUTHFILE="oauth2.txt-gam-gha-${JID}"
              echo "OAUTHFILE=${OAUTHFILE}" >> $GITHUB_ENV
              export gam_user="gam-gha-${JID}@pdl.jaylee.us"
              echo "gam_user=${gam_user}" >> $GITHUB_ENV
              $gam checkconn
-             $gam oauth info
+             $gam create signjwtserviceaccount
+             export CUSTOMER_ID="C03uzfv2s"
+             export GA_DOMAIN="pdl.jaylee.us"
+             export GA_ADMIN_EMAIL="$gam_user"
+             touch "${gampath}/enabledasa.txt"
+             #$gam oauth info
              $gam info domain
-             $gam oauth refresh
+             #$gam oauth refresh
              $gam info user
-             #$gam info user $gam_user grouptree
              export tstamp=$($PYTHON -c "import time; print(time.time_ns())")
-             export newbase=gha_test_$JID_$tstamp
-             export newuser=$newbase@pdl.jaylee.us
-             export newgroup=$newbase-group@pdl.jaylee.us
-             export newalias=$newbase-alias@pdl.jaylee.us
-             export newbuilding=$newbase-building
-             export newresource=$newbase-resource
+             export newbase="gha_test_${JID}_${tstamp}"
+             export newuser="${newbase}@pdl.jaylee.us"
+             export newgroup="${newbase}-group@pdl.jaylee.us"
+             export newalias="${newbase}-alias@pdl.jaylee.us"
+             export newbuilding="${newbase}-building"
+             export newresource="${newbase}-resource"
+             export newou="aaaGithub Actions/${newbase}"
+             
+             # cleanup old runs
+             rm "${gampath}/enabledasa.txt"
+             GAM_CSV_ROW_FILTER="name:regex:gha_test_${JID}_" $gam print vaultholds | $gam csv - gam delete vaulthold "id:~~holdId~~" matter "id:~~matterId~~"
+             touch "${gampath}/enabledasa.txt"
+             GAM_CSV_ROW_FILTER="name:regex:gha_test_${JID}_" $gam print features | $gam csv - gam delete feature ~name
+             GAM_CSV_ROW_FILTER="name:regex:^gha_test_${JID}_" $gam user $gam_user print shareddrives asadmin | $gam csv - gam user $gam_user delete shareddrive ~id nukefromorbit
+             $gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail
+             GAM_CSV_ROW_FILTER="name:regex:^gha_test_${JID}_" $gam print ous fromparent "aaaGithub Actions" | $gam csv - gam delete ou ~orgUnitId
+             GAM_CSV_ROW_FILTER="groupKey.id:regex:^gha_test_${JID}_" $gam print cigroups | $gam csv - gam delete cigroup ~groupKey.id
+             GAM_CSV_ROW_FILTER="resourceId:regex:^gha_test_${JID}_" $gam print resources | $gam csv - gam delete resource ~resourceId
+             GAM_CSV_ROW_FILTER="buildingId:regex:^gha_test_${JID}_" $gam print buildings | $gam csv - gam delete building ~buildingId
+             
+             echo "Creating OrgUnit ${newou}"
+             $gam create ou "${newou}"
              export GAM_THREADS=5
              echo email > sample.csv;
              for i in {1..10}; do
                echo "${newbase}-bulkuser-$i" >> sample.csv;
              done
-             $gam create user $newuser firstname GHA lastname $JID password random recoveryphone 12125121110 recoveryemail jay0lee@gmail.com gha.jid $JID languages en+,en-GB-
+             $gam create user $newuser firstname GHA lastname $JID displayname "Github Actions ${JID}" password random ou "${newou}" recoveryphone 12125121110 recoveryemail jay0lee@gmail.com gha.jid $JID languages en+,en-GB-
              $gam user $newuser update photo https://dummyimage.com/400x600/000/fff
              $gam user $newuser get photo
              $gam user $newuser delete photo
+             $gam create alias $newalias user $newuser
              $gam create group $newgroup name "GHA $JID group" description "This is a description" isarchived true
              $gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "GHA test message"
              $gam user $gam_user sendemail recipient exchange@pdl.jaylee.us subject "test ${tstamp}" message "test message"
+             rm "${gampath}/enabledasa.txt"
              $gam user $newuser add license workspaceenterpriseplus
              $gam print privileges
-             $gam update cigroup $newgroup memberrestriction 'member.type == 1 || member.customer_id == groupCustomerId()'
+             touch "${gampath}/enabledasa.txt"
+             $gam update cigroup $newgroup security memberrestriction 'member.type == 1 || member.customer_id == groupCustomerId()'
              $gam info cigroup $newgroup
              $gam update group $newgroup add owner $gam_user
              $gam update group $newgroup add member $newuser
+             rm "${gampath}/enabledasa.txt"
              $gam create admin $newuser _GROUPS_EDITOR_ROLE CUSTOMER # condition nonsecuritygroup
-             $gam csv sample.csv gam create user ~~email~~ firstname "GHA Bulk" lastname ~~email~~ gha.jid $JID
-             $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random
+             $gam create admin $newgroup _HELP_DESK_ADMIN_ROLE org_unit "${newou}"
+             GAM_CSV_ROW_FILTER="assignedToUser:regex:${newuser}" $gam print admins | $gam csv - gam delete admin "~roleAssignmentId"
+             GAM_CSV_ROW_FILTER="assignedToGroup:regex:${newgroup}" $gam print admins | $gam csv - gam delete admin "~roleAssignmentId"
+             touch "${gampath}/enabledasa.txt"
+             $gam csv sample.csv gam create user ~~email~~ firstname "GHA Bulk" lastname ~~email~~ gha.jid $JID ou "${newou}"
+             $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random displayname "GitHub Actions Bulk ${JID}"
              $gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""
+             rm "${gampath}/enabledasa.txt"
              $gam csv sample.csv gam user ~email add license workspaceenterpriseplus
+             touch "${gampath}/enabledasa.txt"
              $gam csv sample.csv gam user $gam_user sendemail recipient ~~email~~@pdl.jaylee.us subject "test message $newbase" message "GHA test message"
              $gam csv sample.csv gam update group $newgroup add member ~email
              $gam info group $newgroup
@@ -608,8 +711,8 @@ jobs:
              $gam users "$newbase-bulkuser-7 $newbase-bulkuser-8 $newbase-bulkuser-9" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit
              $gam user $newuser delete label --ALL_LABELS--
              GAM_CSV_ROW_FILTER="name:regex:gha-test-${JID}" $gam print features | $gam csv - gam delete feature ~name
-             $gam create feature name Whiteboard-$newbase
              $gam create feature name VC-$newbase
+             $gam create feature name Whiteboard-$newbase
              $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."
              $gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room
              $gam info resource $newresource
@@ -624,36 +727,44 @@ jobs:
              endtime=$($PYTHON -c "import datetime; print((datetime.datetime.now() + datetime.timedelta(hours=2)).strftime('%Y-%m-%dT%H:%M:%S.%f+00:00'))")
              $gam calendar $gam_user addevent summary "GHA test event" start "${starttime}" end "${endtime}" attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all
              $gam calendar $gam_user printevents after -0d
+             rm "${gampath}/enabledasa.txt"
              matterid=uid:$($gam create vaultmatter name "GHA matter $newbase" description "test matter" collaborators $newuser | head -1 | cut -d ' ' -f 3)
              $gam create vaulthold matter $matterid name "GHA hold $newbase" corpus mail accounts $newuser
              $gam print vaultmatters matterstate open
              $gam print vaultholds matter $matterid
              $gam print vaultcount matter $matterid corpus mail everyone todrive
-             $gam create vaultexport matter $matterid name "GHA export $newbase" corpus mail accounts $newuser use_new_export false
+             $gam create vaultexport matter $matterid name "GHA export $newbase" corpus mail accounts $newuser
              $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~
+             touch "${gampath}/enabledasa.txt"
              $gam csv sample.csv gam user ~email add calendar id:$newresource
              $gam delete resource $newresource
              $gam delete feature Whiteboard-$newbase
              $gam delete feature VC-$newbase
              $gam delete building $newbuilding
              $gam delete group $newgroup
-             $gam create alias $newalias user $newuser
+             rm "${gampath}/enabledasa.txt"
+             echo start
              $gam user $newuser delete license workspaceenterpriseplus
+             echo finish
+             touch "${gampath}/enabledasa.txt"
              $gam whatis $newuser
              $gam user $gam_user show tokens
+             rm "${gampath}/enabledasa.txt"
              $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~
              $gam delete hold "GHA hold $newbase" matter $matterid
              $gam update matter $matterid action close
              $gam update matter $matterid action delete
-             #$gam delete user $newuser
-             #$gam undelete user $newuser
+             # shakes off vault hold on user so we can delete
+             $gam print users query "email:${newuser}" orgunitpath | $gam csv - gam update user ~primaryEmail ou ~orgUnitPath
+             $gam user $newuser show holds
+             export sn="$JID$JID$JID$JID-$(openssl rand -base64 32 | sed 's/[^a-zA-Z0-9]//g')"
+             $gam create device serialnumber $sn devicetype android
+             touch "${gampath}/enabledasa.txt"
              $gam delete user $newuser
              $gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail
              $gam print mobile
              $gam print devices
              $gam print browsers
-             export sn="$JID$JID$JID$JID-$(openssl rand -base64 32 | sed 's/[^a-zA-Z0-9]//g')"
-             $gam create device serialnumber $sn devicetype android
              $gam print cros allfields orderby serialnumber
              $gam show crostelemetry storagepercentonly
              $gam report usageparameters customer
@@ -662,44 +773,87 @@ jobs:
              $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive
              $gam report admin start -3d todrive
              $gam print devices nopersonaldevices nodeviceusers filter "serial:$JID$JID$JID$JID-" | $gam csv - gam delete device id ~name
-             #$gam print userinvitations
-             #$gam print userinvitations | $gam csv - gam send userinvitation ~name
+             rm "${gampath}/enabledasa.txt"
+             $gam print userinvitations
+             $gam print userinvitations | $gam csv - gam send userinvitation ~name
+             touch "${gampath}/enabledasa.txt"
              $gam create caalevel "zzz_${newbase}" basic condition ipsubnetworks 1.1.1.1/32,2.2.2.2/32 endcondition
              $gam print caalevels
              $gam delete caalevel "zzz_${newbase}"
              driveid=$($gam user $gam_user add shareddrive "${newbase}" | awk '{print $NF}')
              echo "Created shared drive ${driveid}"
              $gam user $gam_user add drivefile localfile gam.py parentid "${driveid}"
-             $gam user $gam_user update shareddrive "${driveid}" ou "id:03ph8a2z1t2ph5z"
+             $gam user $gam_user update shareddrive "${driveid}" ou "${newou}"
              $gam user $gam_user show shareddrives asadmin
+             $gam user $gam_user update shareddrive "${driveid}" ou "aaaGithub Actions" # so we can delete our OU...
              $gam user $gam_user delete shareddrive "${driveid}" nukefromorbit
              echo "printer model count:"
+             ssoprofile=$($gam create inboundssoprofile name "El Goog ${newbase}" loginurl https://www.google.com logouturl https://www.google.com changepasswordurl https://www.google.com entityid ElGoog return_name_only)
+             $gam create inboundssocredential profile "id:${ssoprofile}" generate_key
+             $gam create inboundssoassignment profile "id:${ssoprofile}" orgunit "${newou}" mode SAML_SSO
+             $gam delete inboundssoassignment "orgunit:${newou}"
+             $gam delete inboundssoprofile "id:${ssoprofile}"
              $gam print printermodels | wc -l
-             #$gam print printers
-             #$gam create printer displayname "${newbase}" uri ipp://localhost:631 driverless description "made by $(gam_user)" ou /
-             #export CUSTOMER_ID="C01wfv983"
-             #export GA_DOMAIN="pdl.jaylee.us"
-             #touch $gampath/enabledasa.txt
-             #echo "using delegated admin service account"
-             #$gam print users
-
-     # - name: Upload to Google Drive, build only.
-     #   if: github.event_name == 'push' && matrix.goal != 'test'
-     #   run: |
-     #        ls gam-$GAMVERSION-*
-     #        for gamfile in gam-$GAMVERSION-*; do
-     #          echo "Uploading file ${gamfile} to Google Drive..."
-     #          fileid=$($gam user $gam_user add drivefile localfile $gamfile drivefilename $GAMVERSION-${GITHUB_SHA:0:7}-$gamfile parentid 1N2zbO33qzUQFsGM49-m9AQC1ijzd_ru1 returnidonly)
-     #          echo "file uploaded as ${fileid}, setting ACL..."
-     #          $gam user $gam_user add drivefileacl $fileid anyone role reader withlink
-     #        done
+             $gam print printers
+             printerid=$($gam create printer displayname "${newbase}" uri ipp://localhost:631 driverless description "made by $(gam_user)" ou "${newou}" | grep 'id: [a-z,0-9]*$' | cut -d' ' -f3)
+             $gam info printer "$printerid"
+             $gam delete printer "$printerid"
+             $gam delete ou "${newou}"
 
       - name: Archive production artifacts
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'push' && matrix.goal != 'test'
+        uses: actions/upload-artifact@v3
+        if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal != 'test'
         with:
           name: gam-binaries
           path: |
             src/*.tar.xz
             src/*.zip
             src/*.msi
+
+      - name: Tar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar cJvvf bin.tar.xz bin/
+
+  publish:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: write
+      packages: write
+      pull-requests: read
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: VirusTotal Scan
+        uses: crazy-max/ghaction-virustotal@v3
+        with:
+          vt_api_key: ${{ secrets.VT_API_KEY }}
+          files: |
+            gam-binaries/*
+
+      - name: Set datetime version string
+        id: dateversion
+        run: |
+          export dateversion="$(date  +'%Y%m%d.%H%M%S')"
+          echo "Date version: ${dateversion}"
+          echo "dateversion=${dateversion}" >> $GITHUB_OUTPUT
+
+      - uses: "marvinpinto/action-automatic-releases@latest"
+        name: Publish draft release
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "${{ steps.dateversion.outputs.dateversion }}"
+          prerelease: false
+          draft: true
+          files: |
+            gam-binaries/*

.github/workflows/get-roots.yml

@@ -0,0 +1,36 @@
+name: Check for Google Root CA Updates
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '23 23 * * *'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: src
+
+jobs:
+  check-apis:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+      - name: Check for updates
+        run: curl -o ./roots.pem -vvvv https://pki.goog/roots.pem
+
+      - name: Commit file
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git add roots.pem
+          git diff --quiet && git diff --staged --quiet || git commit -am '[ci skip] Updated roots.pem'
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }} 

src/GamCommands.txt

@@ -911,6 +911,12 @@ gam oauth|oauth2 refresh
 
 gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
+gam yubikey resetpiv [yubikeyserialnumber <Number>]
+gam rotate sakey yubikey yubikey_pin yubikey_slot AUTHENTICATION yubikeyserialnumber <Number>
+
+gam create [gcpserviceaccount|signjwtserviceaccount]
+gam enable apis [auto|manual]
+
 gam whatis <EmailItem>
 
 <ResoldCustomerAttribute> ::=
@@ -1392,6 +1398,13 @@ gam print chromeappdevices [todrive]
 	[start <Date>] [end <Date>]
 	[orderby deviceid|machine]
 
+gam print chromeaues [todrive]
+	[ou|org|orgunit <OrgUnitItem>]
+	[minauedate <Date>] [maxauedate <Date>]
+
+gam print chromeneedsattn [todrive]
+	[ou|org|orgunit <OrgUnitItem>]
+
 gam print chromeversions [todrive]
 	[ou|org|orgunit <OrgUnitItem>]
 	[start <Date>] [end <Date>] [recentfirst]
@@ -1548,6 +1561,45 @@ gam print group-members|groups-members [todrive]
 	[roles <GroupRoleList>] [membernames] [fields <MembersFieldNameList>]
 	[includederivedmembership]
 
+<SSOProfileDisplayName> ::= <String>
+<SSOProfileName> ::= id:inboundSamlSsoProfiles/<String>
+<SSOProfileItem> ::= <SSOProfileDisplayName>|<SSOProfileName>
+<SSOProfileItemList> ::= "<SSOProfileItem>(,<SSOProfileItem>)*"
+
+gam create inboundssoprofile [name <SSOProfileDisplayName>]
+	[entityid <String>] [loginurl <URL>] [logouturl <URL>] [changepasswordurl <URL>]
+	[returnnameonly]
+gam update inboundssoprofile <SSOProfileItem>
+	[entityid <String>] [loginurl <URL>] [logouturl <URL>] [changepasswordurl <URL>]
+	[returnnameonly]
+gam delete inboundssoprofile <SSOProfileItem>
+gam info inboundssoprofile <SSOProfileItem>
+gam show inboundssoprofiles
+gam print inboundssoprofiles [todrive]
+
+<SSOCredentialsName> ::= [id:]inboundSamlSsoProfiles/<String>/idpCredentials/<String>
+
+gam create inboundssocredential profile <SSOProfileItem>
+	(pemfile <FileName>)|(generatekey [keysize 1024|2048|4096]) [replaceolddest]
+gam delete inboundssocredential <SSOCredentialsName>
+gam show inboundssocredentials [profile|profiles <SSOProfileItemList>]
+gam print inboundssocredentials [profile|profiles <SSOProfileItemList>] [todrive]
+
+<SSOAssignmentSelector> ::=
+	groups/<String> |
+	group:<EmailAddress> |
+	orgunits/<String> |
+	orgunit:<OrgUnitPath>
+
+gam create inboundssoassignment (group <GroupItem> rank <Number>)|(ou|org|orgunit <OrgUnitItem>)
+	(mode sso_off)|(mode saml_sso profile <SSOProfileItem>)(mode domain_wide_saml_if_enabled) [neverredirect]
+gam update inboundssoassignment [(group <GroupItem> rank <Number>)|(ou|org|orgunit <OrgUnitItem>)]
+	[(mode sso_off)|(mode saml_sso profile <SSOProfileItem>)(mode domain_wide_saml_if_enabled)] [neverredirect]
+gam delete inboundssoassignment <SSOAssignmentSelector>
+gam info inboundssoassignment <SSOAssignmentSelector>
+gam show inboundssoassignments
+gam print inboundssoassignments [todrive]
+
 gam send userinvitation <EmailAddress>
 gam cancel userinvitation <EmailAddress>
 gam check userinvitation|isinvitable <EmailAddress>
@@ -1636,6 +1688,9 @@ gam show guardian|guardians [invitedguardian <EmailAddress>] [student <StudentIt
 gam print guardian|guardians [todrive] [invitedguardian <EmailAddress>] [student <StudentItem>] [invitations [states <GuardianStateList>]] [<UserTypeEntity>]
 gam cancel guardianinvitation|guardianinvitations <GuardianInvitationID> <StudentItem>
 
+gam download storagebucket <URL>
+gam copy storagebucket sourcebucket <URL> targetbucket <URL> [sourceprefix <String>] [targetprefix <String>]
+
 gam create vaultexport|export matter <MatterItem> [name <name>] corpus <drive|mail|groups|hangouts_chat>
 	(accounts <EmailAddressList>) | (orgunit|ou <OrgUnitPath>) | (teamdrives <TeamDriveList>) | (rooms <ChatRoomList>) | everyone
 	[scope <all_data|held_data|unprocessed_data>]
@@ -1648,6 +1703,7 @@ gam delete export <MatterItem> <ExportItem>
 gam info export <MatterItem> <ExportItem>
 gam print exports [todrive] [matters <MatterItemList>]
 gam download export <MatterItem> <ExportItem> [noverify] [noextract] [targetfolder <FilePath>]
+gam copy export <MatterItem> <ExportItem> targetbucket <URL> [targetprefix <String>]
 
 gam create vaulthold|hold corpus drive|groups|mail matter <MatterItem> [name <String>] [query <QueryVaultCorpus>]
 	[(accounts|groups|users <EmailItemList>) | (orgunit|ou <OrgUnit>)]

src/cbcm-v1.1beta1.json

@@ -12,7 +12,7 @@
     }
   },
   "basePath": "",
-  "baseUrl": "https://www.googleapis.com/admin/directory/v1.1beta1/customer/",
+  "baseUrl": "https://admin.googleapis.com/admin/directory/v1.1beta1/customer/",
   "batchPath": "batch",
   "canonicalName": "cbcm",
   "discoveryVersion": "v1",
@@ -412,7 +412,7 @@
     }
   },
   "revision": "20201203",
-  "rootUrl": "https://www.googleapis.com/admin/directory/v1.1beta1/customer/",
+  "rootUrl": "https://admin.googleapis.com/admin/directory/v1.1beta1/customer/",
   "schemas": {
     "ChromeBrowser": {
       "id": "ChromeBrowser",

src/gam-install.sh

@@ -28,7 +28,7 @@ upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.35"
+gam_glibc_vers="2.31"
 
 while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do

src/gam.spec

@@ -1,55 +1,113 @@
-# -*- mode: python -*-
+# -*- mode: python ; coding: utf-8 -*-
+from os import getenv
+from re import search
+from sys import platform
 
-import sys
-
-import importlib
 from PyInstaller.utils.hooks import copy_metadata
 
-# dynamically determine where httplib2/cacerts.txt lives
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-extra_files = [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+from gam.var import GAM_VER_LIBS
 
-extra_files += copy_metadata('google-api-python-client')
+extra_files = []
+for pkg in GAM_VER_LIBS:
+    extra_files += copy_metadata(pkg, recursive=True)
 extra_files += [('cbcm-v1.1beta1.json', '.')]
 extra_files += [('contactdelegation-v1.json', '.')]
 extra_files += [('admin-directory_v1.1beta1.json', '.')]
-
+extra_files += [('roots.pem', '.')]
 hidden_imports = [
      'gam.auth.yubikey',
      ]
-
-a = Analysis(['gam/__main__.py'],
-             hiddenimports=hidden_imports,
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             datas=extra_files,
-             runtime_hooks=None)
-
+a = Analysis(
+    ['gam/__main__.py'],
+    pathex=[],
+    binaries=[],
+    datas=extra_files,
+    hiddenimports=hidden_imports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=None,
+    noarchive=False,
+)
 for d in a.datas:
     if 'pyconfig' in d[0]:
         a.datas.remove(d)
         break
-
-
-pyz = PYZ(a.pure)
-
-
-if sys.platform == "darwin":
-     target_arch="universal2"
+pyz = PYZ(a.pure,
+          a.zipped_data,
+          cipher=None)
+# requires Python 3.10+ but no one should be compiling
+# GAM with older versions anyway
+match platform:
+    case "darwin":
+        target_arch = "universal2"
+        strip = True
+    case "win32":
+        target_arch = None
+        strip = False
+    case _:
+        target_arch = None
+        strip = True
+name = 'gam'
+debug = False
+bootloader_ignore_signals = False
+upx = False
+console = True
+disable_windowed_traceback = False
+argv_emulation = False
+codesign_identity = None
+entitlements_file = None
+if getenv('PYINSTALLER_BUILD_ONEFILE') == 'yes':
+    # Build one file
+    exe = EXE(
+              pyz,
+              a.scripts,
+              a.binaries,
+              a.zipfiles,
+              a.datas,
+              [],
+              name=name,
+              debug=debug,
+              bootloader_ignore_signals=bootloader_ignore_signals,
+              strip=strip,
+              upx=upx,
+              console=console,
+              disable_windowed_traceback=disable_windowed_traceback,
+              argv_emulation=argv_emulation,
+              target_arch=target_arch,
+              codesign_identity=codesign_identity,
+              entitlements_file=entitlements_file,
+              )
 else:
-     target_arch=None
+    # Build one folder
+    exe = EXE(
+              pyz,
+              a.scripts,
+              [],
+              exclude_binaries=True,
+              name=name,
+              debug=debug,
+              bootloader_ignore_signals=bootloader_ignore_signals,
+              strip=strip,
+              upx=upx,
+              console=console,
+              disable_windowed_traceback=disable_windowed_traceback,
+              argv_emulation=argv_emulation,
+              target_arch=target_arch,
+              codesign_identity=codesign_identity,
+              entitlements_file=entitlements_file,
+              )
+    coll = COLLECT(
+        exe,
+        a.binaries,
+        a.zipfiles,
+        a.datas,
+        strip=strip,
+        upx=upx,
+        upx_exclude=[],
+        name=name,
+        )
 
-# use strip on all non-Windows platforms
-strip = not sys.platform == 'win32'
-
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=strip,
-          upx=False,
-          target_arch=target_arch,
-          console=True)

src/gam.wxs

@@ -55,6 +55,9 @@
       <Component Id="gamcommands_txt" Guid="58ff9c45-a7c9-4e22-8845-a9a92610c1f3">
         <File Name="gamcommands.txt" KeyPath="yes" />
       </Component>
+      <Component Id="roots_pem" Guid="18ff9c45-a3c9-4e22-8445-a8a92610c1f3">
+        <File Name="roots.pem" KeyPath="yes" />
+      </Component>
     </ComponentGroup>
   </Fragment>
 

src/gam/__main__.py

@@ -22,25 +22,33 @@ For more information, see https://jaylee.us/gam
 """
 
 import sys
-from multiprocessing import freeze_support
-from multiprocessing import set_start_method
-
-from gam import controlflow
-import gam
-
 
+# Note that this file (and only this file) should remain compatible
+# with older Python versions so we can return a meaningful error
+# instead of a syntax error.
 def main():
+    required_ver = (3, 8, 0)
+    if sys.version_info[:3] < required_ver:
+        err_result = ('ERROR: GAM requires Python %s.%s.%s or newer. You are '
+                'running %s.%s.%s. Please upgrade your Python version '
+                'or use one of the binary GAM downloads.\n' %
+                (required_ver[0],
+                 required_ver[1],
+                 required_ver[2],
+                 sys.version_info[0],
+                 sys.version_info[1],
+                 sys.version_info[2]))
+        sys.stderr.write(err_result)
+        sys.exit(5)
+    from multiprocessing import freeze_support
     freeze_support()
     if sys.platform == 'darwin':
         # https://bugs.python.org/issue33725 in Python 3.8.0 seems
         # to break parallel operations with errors about extra -b
         # command line arguments
+        from multiprocessing import set_start_method
         set_start_method('fork')
-    if sys.version_info[0] < 3 or sys.version_info[1] < 7:
-        controlflow.system_error_exit(
-            5,
-            f'GAM requires Python 3.7 or newer. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.'
-            % sys.version_info[:3])
+    import gam
     sys.exit(gam.ProcessGAMCommand(sys.argv))
 
 

src/gam/auth/__init__.py

@@ -9,6 +9,7 @@ import gam
 from gam import utils
 
 from gam.auth import oauth
+from gam.auth import signjwt
 from gam.var import _FN_OAUTH2_TXT
 from gam.var import _FN_OAUTH2SERVICE_JSON
 from gam.var import GC_OAUTH2_TXT
@@ -28,9 +29,11 @@ def get_admin_credentials_filename():
     # some custom name in it. Otherwise, just use the default name.
     if GC_Values[GC_ENABLE_DASA]:
         return GC_Values[GC_OAUTH2SERVICE_JSON] if GC_Values[GC_OAUTH2SERVICE_JSON] else _FN_OAUTH2SERVICE_JSON
-    else:
-        return GC_Values[GC_OAUTH2_TXT] if GC_Values[GC_OAUTH2_TXT] else _FN_OAUTH2_TXT
+    return GC_Values[GC_OAUTH2_TXT] if GC_Values[GC_OAUTH2_TXT] else _FN_OAUTH2_TXT
 
+APIS_NEEDING_ACCESS_TOKEN = {
+    'cbcm': ['https://www.googleapis.com/auth/admin.directory.device.chromebrowsers']
+    }
 
 def get_admin_credentials(api=None):
     """Gets oauth.Credentials that are authenticated as the domain's admin user."""
@@ -40,17 +43,27 @@ def get_admin_credentials(api=None):
     with open(credential_file) as f:
         creds_data = json.load(f)
     # Validate that enable DASA matches content of authorization file
-    if GC_Values[GC_ENABLE_DASA] and 'private_key_id' in creds_data:
+    if GC_Values[GC_ENABLE_DASA] and creds_data.get('type') == 'service_account':
+        if api in APIS_NEEDING_ACCESS_TOKEN:
+            return gam.getSvcAcctCredentials(scopes=APIS_NEEDING_ACCESS_TOKEN[api],
+                                             act_as=None,
+                                             api=None,
+                                             force_oauth=True)
         audience = f'https://{api}.googleapis.com/'
         key_type = creds_data.get('key_type', 'default')
         if key_type == 'default':
             return JWTCredentials.from_service_account_info(creds_data,
                                                             audience=audience)
-        elif key_type == 'yubikey':
+        if key_type == 'yubikey':
             yksigner = yubikey.YubiKey(creds_data)
             return JWTCredentials._from_signer_and_info(yksigner,
                 creds_data,
                 audience=audience)
+        if key_type == 'signjwt':
+            sjsigner = signjwt.SignJwt(creds_data)
+            return signjwt.JWTCredentials._from_signer_and_info(sjsigner,
+                    creds_data,
+                    audience=audience)
     elif not GC_Values[GC_ENABLE_DASA] and 'token' in creds_data:
         return oauth.Credentials.from_credentials_file(credential_file)
     else:

src/gam/auth/oauth.py

@@ -50,6 +50,7 @@ MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
                                 ' close this browser window and return to GAM.')
 
 MESSAGE_AUTHENTICATION_COMPLETE = ('\nThe authentication flow has completed.\n')
+MESSAGE_AUTHENTICATION_FAILED = ('\nThe authentication flow failed, reissue command')
 
 
 class CredentialsError(Exception):
@@ -272,6 +273,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                             access_type='offline',
                             login_hint=None,
                             filename=None,
+                            open_browser=True,
                             use_console_flow=False):
         """Runs an OAuth Flow from client secrets to generate credentials.
 
@@ -291,8 +293,11 @@ class Credentials(google.oauth2.credentials.Credentials):
       login_hint: String, The email address that will be displayed on the Google
         login page as a hint for the user to login to the correct account.
       filename: String, the path to a file to use to save the credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
+        should be run strictly from a console; False to launch a browser
+        for authentication.
+      open_browser: Boolean: whether or not GAM should try to open the browser
+        automatically.
 
     Returns:
       Credentials
@@ -312,12 +317,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         flow = _ShortURLFlow.from_client_config(client_config,
                                                 scopes,
                                                 autogenerate_code_verifier=True)
-        flow_kwargs = {'access_type': access_type}
+        flow_kwargs = {'access_type': access_type,
+                       'open_browser': open_browser}
         if login_hint:
             flow_kwargs['login_hint'] = login_hint
-
-        flow.run_dual(use_console_flow,
-                **flow_kwargs)
+        flow.run_dual(**flow_kwargs)
         return cls.from_google_oauth2_credentials(flow.credentials,
                                                   filename=filename)
 
@@ -328,6 +332,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                                  access_type='offline',
                                  login_hint=None,
                                  credentials_file=None,
+                                 open_browser=True,
                                  use_console_flow=False):
         """Runs an OAuth Flow from secrets stored on disk to generate credentials.
 
@@ -348,8 +353,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         login page as a hint for the user to login to the correct account.
       credentials_file: String, the path to a file to use to save the
         credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
+        should be run strictly from a console; False to launch a browser for
+        authentication.
+      open_browser: Boolean, whether or not GAM should try to open the browser
+        directly.
 
     Raises:
       InvalidClientSecretsFileError: If the client secrets file cannot be
@@ -378,14 +386,13 @@ class Credentials(google.oauth2.credentials.Credentials):
             raise InvalidClientSecretsFileFormatError(
                 f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
             )
-
         return cls.from_client_secrets(client_id,
                                        client_secret,
                                        scopes,
                                        access_type=access_type,
                                        login_hint=login_hint,
                                        filename=credentials_file,
-                                       use_console_flow=use_console_flow)
+                                       open_browser=open_browser)
 
     def _fetch_id_token_data(self):
         """Fetches verification details from Google for the OAuth2.0 token.
@@ -482,7 +489,11 @@ class Credentials(google.oauth2.credentials.Credentials):
     def _locked_refresh(self, request):
         """Refreshes the credential's access token while the file lock is held."""
         assert self._lock.is_locked
-        super().refresh(request)
+        try:
+            super().refresh(request)
+        except google.auth.exceptions.RefreshError as e:
+            controlflow.system_error_exit(9, str(e))
+
 
     def write(self):
         """Writes credentials to disk."""
@@ -595,7 +606,6 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
 
 
     def run_dual(self,
-                use_console_flow,
                 authorization_prompt_message='',
                 console_prompt_message='',
                 web_success_message='',
@@ -605,7 +615,7 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
         mgr = multiprocessing.Manager()
         d = mgr.dict()
         d['trailing_slash'] = redirect_uri_trailing_slash
-        d['open_browser'] = use_console_flow
+        d['open_browser'] = open_browser
         http_client = multiprocessing.Process(target=_wait_for_http_client,
                                               args=(d,))
         user_input = multiprocessing.Process(target=_wait_for_user_input,
@@ -620,15 +630,22 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
         print(MESSAGE_CONSOLE_AUTHORIZATION_PROMPT.format(url=d['auth_url']))
         user_input.start()
         userInput = False
-        while True:
+        alive = 2
+        while alive > 0:
             sleep(0.1)
             if not http_client.is_alive():
-                user_input.terminate()
-                break
-            elif not user_input.is_alive():
+                if 'code' in d:
+                    user_input.terminate()
+                    break
+                alive -= 1
+            if not user_input.is_alive():
                 userInput = True
-                http_client.terminate()
-                break
+                if 'code' in d:
+                    http_client.terminate()
+                    break
+                alive -= 1
+        if 'code' not in d:
+            controlflow.system_error_exit(8, MESSAGE_AUTHENTICATION_FAILED)
         while True:
             code = d['code']
             if code.startswith('http'):

src/gam/auth/signjwt.py

@@ -0,0 +1,99 @@
+''' Use Google Application Default Credentials '''
+import datetime
+import json
+
+import google.auth
+from google.auth._helpers import datetime_to_secs, scopes_to_string, utcnow
+import google.oauth2.service_account
+
+import gam
+from gam import controlflow
+from gam import gapi
+from gam import transport
+from gam.var import GM_Globals, GM_CACHE_DIR
+
+_DEFAULT_TOKEN_LIFETIME_SECS = 3600  # 1 hour in seconds
+_GOOGLE_OAUTH2_TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
+_IAM_SCOPES = ['https://www.googleapis.com/auth/iam']
+
+# Some Workforce Identity Federation endpoints such as GitHub Actions
+# only allow TLS 1.2 as of April 2023.
+def get_request():
+    httpc = transport.create_http(override_min_tls='TLSv1_2')
+    return transport.create_request(httpc)
+
+
+class JWTCredentials(google.auth.jwt.Credentials):
+    ''' Class used for DASA '''
+    def _make_jwt(self):
+        now = utcnow()
+        lifetime = datetime.timedelta(seconds=self._token_lifetime)
+        expiry = now + lifetime
+        payload = {
+            "iss": self._issuer,
+            "sub": self._subject,
+            "iat": datetime_to_secs(now),
+            "exp": datetime_to_secs(expiry),
+        }
+        if self._audience:
+            payload["aud"] = self._audience
+        payload.update(self._additional_claims)
+        jwt = self._signer.sign(payload)
+        return jwt, expiry
+
+
+class Credentials(google.oauth2.service_account.Credentials):
+    ''' Class used for DwD '''
+
+    def _make_authorization_grant_assertion(self):
+        now = utcnow()
+        lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS)
+        expiry = now + lifetime
+        payload = {
+            "iat": datetime_to_secs(now),
+            "exp": datetime_to_secs(expiry),
+            "iss": self._service_account_email,
+            "aud": _GOOGLE_OAUTH2_TOKEN_ENDPOINT,
+            "scope": scopes_to_string(self._scopes or ()),
+        }
+
+        payload.update(self._additional_claims)
+
+        # The subject can be a user email for domain-wide delegation.
+        if self._subject:
+            payload.setdefault("sub", self._subject)
+        token = self._signer(payload)
+        return token
+
+
+class SignJwt(google.auth.crypt.Signer):
+    ''' Signer class for SignJWT '''
+    def __init__(self, service_account_info):
+        self.service_account_email = service_account_info['client_email']
+        self.name = f'projects/-/serviceAccounts/{self.service_account_email}'
+        self._key_id = None
+
+    @property  # type: ignore
+    def key_id(self):
+        return self._key_id
+
+
+    def sign(self, message):
+        ''' Call IAM Credentials SignJWT API to get our signed JWT '''
+        request = get_request()
+        try:
+            credentials, _ = google.auth.default(scopes=_IAM_SCOPES,
+                    request=request)
+        except (google.auth.exceptions.DefaultCredentialsError,
+                google.auth.exceptions.RefreshError) as e:
+            controlflow.system_error_exit(2, e)
+        httpObj = transport.AuthorizedHttp(
+                credentials,
+                transport.create_http(cache=GM_Globals[GM_CACHE_DIR]))
+        iamc = gam.getService('iamcredentials', httpObj)
+        response = gapi.call(iamc.projects().serviceAccounts(),
+                             'signJwt',
+                             name=self.name,
+                             body={'payload': json.dumps(message)})
+        signed_jwt = response.get('signedJwt')
+        return signed_jwt

src/gam/auth/yubikey.py

@@ -8,7 +8,7 @@ from threading import Timer
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import padding
 from smartcard.Exceptions import CardConnectionException
-from ykman.device import connect_to_device
+from ykman.device import list_all_devices
 from ykman.piv import generate_self_signed_certificate, \
                       generate_chuid
 from yubikit.piv import DEFAULT_MANAGEMENT_KEY, \
@@ -20,11 +20,14 @@ from yubikit.piv import DEFAULT_MANAGEMENT_KEY, \
                         OBJECT_ID, \
                         SLOT, \
                         TOUCH_POLICY
-from yubikit.core.smartcard import ApduError
+from yubikit.core.smartcard import ApduError, \
+                                   SmartCardConnection
 from gam import controlflow
 
+
 class YubiKey():
 
+
     def __init__(self, service_account_info=None):
         self.key_type = None
         self.slot = None
@@ -46,12 +49,16 @@ class YubiKey():
             self.pin = service_account_info.get('yubikey_pin')
             self.key_id = service_account_info.get('private_key_id')
 
+
     def _connect(self):
         try:
-            conn, _, _ = connect_to_device(self.serial_number)
+            devices = list_all_devices()
+            for (device, info) in devices:
+                if info.serial == self.serial_number:
+                    return device.open_connection(SmartCardConnection)
         except CardConnectionException as err:
             controlflow.system_error_exit(9, f'YubiKey - {err}')
-        return conn
+
 
     def get_certificate(self):
         try:
@@ -79,11 +86,25 @@ class YubiKey():
 
     def get_serial_number(self):
         try:
-            _, _, info = connect_to_device(self.serial_number)
-            return info.serial
+            devices = list_all_devices()
+            if not devices:
+                msg = f'Could not find any YubiKey'
+                controlflow.system_error_exit(3, msg)
+            if self.serial_number:
+                for (device, info) in devices:
+                    if info.serial == self.serial_number:
+                        return info.serial
+                msg = f'Could not find YubiKey with serial {self.serial_number}'
+                controlflow.system_error_exit(3, msg)
+            if len(devices) > 1:
+                serials = ', '.join([str(info.serial) for (_, info) in devices])
+                msg = f'Multiple YubiKeys connected. Specify yubikey_serial_number and one of {serials}'
+                controlflow.system_error_exit(4, msg)
+            return devices[0][1].serial
         except ValueError as err:
             controlflow.system_error_exit(9, f'YubiKey - {err}')
 
+
     def reset_piv(self):
         '''Resets YubiKey PIV app and generates new key for GAM to use.'''
         reply = str(input('This will wipe all PIV keys and configuration from your YubiKey. Are you sure? (y/N) ').lower().strip())
@@ -95,7 +116,9 @@ class YubiKey():
                 piv = PivSession(conn)
                 piv.reset()
                 rnd = SystemRandom()
-                pin_puk_chars = string.ascii_letters + string.digits + string.punctuation
+                pin_puk_chars = string.ascii_letters + \
+                                string.digits + \
+                                string.punctuation
                 new_puk = ''.join(rnd.choice(pin_puk_chars) for _ in range(8))
                 new_pin = ''.join(rnd.choice(pin_puk_chars) for _ in range(8))
                 piv.change_puk('12345678', new_puk)
@@ -155,3 +178,4 @@ class YubiKey():
         if 'mplock' in globals():
             mplock.release()
         return signed
+

src/gam/controlflow.py

@@ -34,7 +34,7 @@ def missing_argument_exit(argument, command):
     """Indicate that the argument is missing for the command.
 
   Args:
-    argument: the missingagrument
+    argument: the missing argument
     command: the base GAM command
   """
     system_error_exit(2, f'missing argument {argument} for "{command}"')

src/gam/gapi/__init__.py

@@ -1,6 +1,8 @@
 """Methods related to execution of GAPI requests."""
 
+import os.path
 import sys
+from tempfile import TemporaryDirectory
 
 import googleapiclient.errors
 import google.auth.exceptions
@@ -10,7 +12,8 @@ from gam import controlflow
 from gam import display
 from gam.gapi import errors
 from gam import transport
-from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+from gam.var import (GC_Values, GM_Globals,
+                     GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
                      GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
                      MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
                      MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
@@ -238,8 +241,13 @@ def process_page(page, items, all_items, total_items, page_message, message_attr
         page_items = page.get(items, [])
         num_page_items = len(page_items)
         total_items += num_page_items
-        if all_items is not None:
+        if type(all_items) is list:
             all_items.extend(page_items)
+        elif all_items is not None:
+            i = len(all_items)
+            for item in page_items:
+                all_items[str(i)] = item
+                i += 1
     else:
         page_token = None
         num_page_items = 0
@@ -273,6 +281,7 @@ def finalize_page_message(page_message):
         sys.stderr.write('\r\n')
         sys.stderr.flush()
 
+
 def get_all_pages(service,
                   function,
                   items='items',
@@ -341,13 +350,14 @@ def get_all_pages(service,
         page_token, total_items = process_page(page, items, all_items, total_items, page_message, message_attribute)
         if not page_token:
             finalize_page_message(page_message)
+            if type(all_items) is not list:
+                all_items = all_items.values()
             return all_items
         if page_args_in_body:
             kwargs['body']['pageToken'] = page_token
         else:
             kwargs['pageToken'] = page_token
 
-
 # TODO: Make this private once all execution related items that use this method
 # have been brought into this file
 def handle_oauth_token_error(e, soft_errors):

src/gam/gapi/calendar.py

@@ -18,9 +18,9 @@ def normalizeCalendarId(calname, checkPrimary=False):
         return calname
     if not GC_Values[GC_DOMAIN]:
         GC_Values[GC_DOMAIN] = gam._getValueFromOAuth('hd')
-    return gam.convertUIDtoEmailAddress(calname,
+    email, _ = gam.convertUIDtoEmailAddress(calname,
                                         email_types=['user', 'resource'])
-
+    return email
 
 def buildCalendarGAPIObject(calname):
     calendarId = normalizeCalendarId(calname)

src/gam/gapi/chromemanagement.py

@@ -219,7 +219,7 @@ def printShowCrosTelemetry(mode):
     i = 3
     if mode == 'info':
         if i >= len(sys.argv):
-            controlflow.system_error_exit(3, f'<SerialNumber> required for "gam info crostelemetry"')
+            controlflow.system_error_exit(3, '<SerialNumber> required for "gam info crostelemetry"')
         filter_ = f'serialNumber={sys.argv[i]}'
         i += 1
         mode = 'show'
@@ -286,7 +286,8 @@ def printShowCrosTelemetry(mode):
             device['storageInfo']['percentDiskUsed'] = 100 - device['storageInfo']['percentDiskFree']
         for cpuStatusReport in device.get('cpuStatusReport', []):
             for tempInfo in cpuStatusReport.pop('cpuTemperatureInfo', []):
-                cpuStatusReport[f"cpuTemperatureInfo.{tempInfo['label'].strip()}"] = tempInfo['temperatureCelsius']
+                if 'temperatureCelsius' in tempInfo:
+                    cpuStatusReport[f"cpuTemperatureInfo.{tempInfo['label'].strip()}"] = tempInfo['temperatureCelsius']
         if showOrgUnitPath:
             orgUnitId = device.get('orgUnitId')
             if orgUnitId not in orgUnitIdPathMap:
@@ -306,6 +307,97 @@ def printShowCrosTelemetry(mode):
         display.write_csv_file(csvRows, titles, 'Telemetry Devices', todrive)
 
 
+CHROME_AUES_TITLES = [
+    'model', 'count', 'aueMonth', 'aueYear', 'expired'
+    ]
+def printAUEs():
+    cm = build()
+    customer = _get_customerid()
+    todrive = False
+    titles = CHROME_AUES_TITLES
+    csvRows = []
+    orgunit = None
+    minAueDate = None
+    maxAueDate = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        elif myarg == 'minauedate':
+            minAueDate = _getFilterDate(sys.argv[i + 1]).strftime(YYYYMMDD_FORMAT)
+            i += 2
+        elif myarg == 'maxauedate':
+            maxAueDate = _getFilterDate(sys.argv[i + 1]).strftime(YYYYMMDD_FORMAT)
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromeaues"'
+            controlflow.system_error_exit(3, msg)
+    if orgunit:
+        orgUnitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit, None)
+        query = f'orgUnitPath={orgUnitPath}'
+        titles.append('orgUnitPath')
+    else:
+        orgUnitPath = '/'
+        query = None
+    gam.printGettingAllItems('Chrome Auto Update Expirations', query)
+    aues = gapi.call(cm.customers().reports(),
+                     'countChromeDevicesReachingAutoExpirationDate',
+                     customer=customer, orgUnitId=orgunit,
+                     minAueDate=minAueDate, maxAueDate=maxAueDate).get('deviceAueCountReports', [])
+    for aue in sorted(aues, key=lambda k: k.get('model', 'Unknown')):
+        if orgunit:
+            aue['orgUnitPath'] = orgUnitPath
+        csvRows.append(aue)
+    display.write_csv_file(csvRows, titles, 'Chrome AUEs', todrive)
+
+
+CHROME_NEEDSATTN_TITLES = [
+    'noRecentPolicySyncCount', 'noRecentUserActivityCount', 'pendingUpdate',
+    'osVersionNotCompliantCount', 'unsupportedPolicyCount'
+    ]
+def printNeedsAttn():
+    cm = build()
+    customer = _get_customerid()
+    todrive = False
+    titles = CHROME_NEEDSATTN_TITLES[:]
+    csvRows = []
+    orgunit = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['ou', 'org', 'orgunit']:
+            orgunit = _get_orgunit(sys.argv[i+1])
+            i += 2
+        else:
+            msg = f'{myarg} is not a valid argument to "gam print chromeneedsattn"'
+            controlflow.system_error_exit(3, msg)
+    if orgunit:
+        orgUnitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit, None)
+        query = f'orgUnitPath={orgUnitPath}'
+        titles.append('orgUnitPath')
+    else:
+        orgUnitPath = '/'
+        query = None
+    gam.printGettingAllItems('Chrome Devices Needing Attention', query)
+    result = gapi.call(cm.customers().reports(),
+                     'countChromeDevicesThatNeedAttention',
+                     customer=customer, orgUnitId=orgunit, readMask=','.join(CHROME_NEEDSATTN_TITLES))
+    for field in CHROME_NEEDSATTN_TITLES:
+        result.setdefault(field, 0)
+    if orgunit:
+        result['orgUnitPath'] = orgUnitPath
+    csvRows.append(result)
+    display.write_csv_file(csvRows, titles, 'Chrome Devices Needing Attention', todrive)
+
+
 CHROME_VERSIONS_TITLES = [
     'version', 'count', 'channel', 'deviceOsVersion', 'system'
     ]
@@ -319,6 +411,7 @@ def printVersions():
     startDate = None
     endDate = None
     pfilter = None
+    query = None
     reverse = False
     i = 3
     while i < len(sys.argv):
@@ -349,12 +442,18 @@ def printVersions():
         else:
             pfilter = ''
         pfilter += f'last_active_date>={startDate}'
+    query = pfilter
     if orgunit:
         orgUnitPath = gapi_directory_orgunits.orgunit_from_orgunitid(orgunit, None)
+        if query:
+            query += ' AND '
+        else:
+            query = ''
+        query += f'orgUnitPath={orgUnitPath}'
         titles.append('orgUnitPath')
     else:
         orgUnitPath = '/'
-    gam.printGettingAllItems('Chrome Versions', pfilter)
+    gam.printGettingAllItems('Chrome Versions', query)
     page_message = gapi.got_total_items_msg('Chrome Versions', '...\n')
     versions = gapi.get_all_pages(cm.customers().reports(),
                                   'countChromeVersions',

src/gam/gapi/chromepolicy.py

@@ -167,7 +167,7 @@ def build_schemas(svc=None, sfilter=None):
                     for an_enum in schema['definition']['enumType']:
                         if an_enum['name'] == type_name:
                             setting_dict['enums'] = [enum['name'] for enum in an_enum['value']]
-                            setting_dict['enum_prefix'] = utils.commonprefix(setting_dict['enums'])
+                            setting_dict['enum_prefix'] = utils.commonprefix(setting_dict['enums'], True)
                             prefix_len = len(setting_dict['enum_prefix'])
                             setting_dict['enums'] = [enum[prefix_len:] for enum \
                                                      in setting_dict['enums'] \
@@ -408,7 +408,7 @@ def update_policy():
                               f'{expected_enums}, got {value}'
                         controlflow.system_error_exit(8, msg)
                 elif vtype in ['TYPE_LIST']:
-                    value = value.split(',')
+                    value = value.split(',') if value else []
                 if myarg == 'chrome.users.chromebrowserupdates' and \
                       cased_field == 'targetVersionPrefixSetting':
                     mg = re.compile(r'^([a-z]+)-(\d+)$').match(value)

src/gam/gapi/cloudidentity/devices.py

@@ -80,7 +80,7 @@ def _parse_action(action):
 def info():
     ci = gapi_cloudidentity.build_dwd()
     customer = _get_device_customerid()
-    name = _get_device_name()
+    _, name = _get_deviceuser_name()
     device = gapi.call(ci.devices(), 'get', name=name, customer=customer)
     device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
         'deviceUsers', parent=name, customer=customer)
@@ -253,13 +253,6 @@ def update_state():
 
 
 def print_():
-    # This function is rather messy thanks to
-    # https://github.com/GAM-team/GAM/issues/1534
-    # I'd prefer to keep it all in this function for now but if:
-    #  - we find other list() operations that also hit this bug OR
-    #  - it looks like this issue is going to exist on Google's side
-    #    for a long time.
-    # I'll enterain some cleanup here to "functionalize" (yuck) all of this.
     ci = gapi_cloudidentity.build_dwd()
     customer = _get_device_customerid()
     parent = 'devices/-'
@@ -267,8 +260,6 @@ def print_():
     get_device_users = True
     view = None
     orderByList = []
-    # default sort order needed by our 1 hour bug workaround
-    orderBy = 'create_time'
     titles = []
     csvRows = []
     todrive = False
@@ -328,97 +319,26 @@ def print_():
       }
     if orderByList:
         orderBy = ','.join(orderByList)
-    custom_device_filter = bool(device_filter)
-    # we store the devices in a dict keyed by name which is a unique ID.
-    # that way when we get duplicate devices we just overwrite the name
-    # with the latest copy we saw.
-    devices = {}
+    else:
+        orderBy = None
+    devices = []
     page_message = gapi.got_total_items_msg(view_name_map[view], '...\n')
-    pageToken = None
-    newest_device_date = ''
-    total_items = 0
-    while True:
-        try:
-            a_page = gapi.call(ci.devices(),
-                               'list',
-                               customer=customer,
-                               pageSize=100,
-                               pageToken=pageToken,
-                               filter=device_filter,
-                               view=view,
-                               orderBy=orderBy,
-                               throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O])
-        except googleapiclient.errors.HttpError:
-            sys.stderr.write('WARNING: GAM hit Google internal bug 237397223. Please file a Google Support ticket stating that you are encountering this bug.\n') 
-            if orderBy != 'create_time' or custom_device_filter:
-                controlflow.system_error_exit(5, 'GAM workaround for this issue only works if filter and orderby arguments are not used.\n')
-            sys.stderr.write(f' attempting to work around the bug by filtering for devices created on or after the newest we\'ve seen ({newest_device_date})...')
-            device_filter = f'register:{newest_device_date}..'
-            pageToken = None
-            continue
-        for dev in a_page.get('devices', []):
-            total_items += 1
-            devices[dev['name']] = dev
-            dev_date = dev.get('createTime', '')
-            # remove the Z
-            dev_date = dev_date[:-1]
-            # remove microseconds
-            dev_date = dev_date.split('.')[0]
-            if dev_date > newest_device_date:
-                newest_device_date = dev_date
-        pageToken = a_page.get('nextPageToken')
-        if not pageToken:
-            break
-        sys.stderr.write(page_message.replace('%%total_items%%', str(total_items)))
+    devices += gapi.get_all_pages(ci.devices(), 'list', 'devices',
+        customer=customer, page_message=page_message,
+        pageSize=100, filter=device_filter, view=view, orderBy=orderBy)
     if get_device_users:
         page_message = gapi.got_total_items_msg('Device Users', '...\n')
-        pageToken = None
-        newest_deviceuser_date = ''
-        total_items = 0
-        device_users = {}
-        if not custom_device_filter:
-            device_filter = None
-        while True:
-            try:
-                a_page = gapi.call(ci.devices().deviceUsers(),
-                                   'list',
-                                   customer=customer,
-                                   parent=parent,
-                                   pageSize=20,
-                                   orderBy=orderBy,
-                                   filter=device_filter,
-                                   pageToken=pageToken,
-                                   throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O])
-            except googleapiclient.errors.HttpError:
-                sys.stderr.write('WARNING: GAM hit Google internal bug 237397223. Please file a Google Support ticket stating that you are encountering this bug.\n')
-                if orderBy != 'create_time' or custom_device_filter:
-                    controlflow.system_error_exit(5, 'GAM workaround for this issue only works if filter and orderby arguments are not used.\n')
-                sys.stderr.write(f' attempting to work around the bug by filtering for device users created on or after the newest we\'ve seen ({newest_deviceuser_date})...')
-                device_filter = f'register:{newest_deviceuser_date}..'
-                pageToken = None
-                continue
-            for device_user in a_page.get('deviceUsers', []):
-                total_items += 1
-                dev_date = device_user.get('createTime', '')
-                # remove the Z
-                dev_date = dev_date[:-1]
-                # remove microseconds
-                dev_date = dev_date.split('.')[0]
-                if dev_date > newest_deviceuser_date:
-                    newest_deviceuser_date = dev_date
-                deviceuser_name = device_user['name']
-                device_users[deviceuser_name] = device_user
-            pageToken = a_page.get('nextPageToken')
-            if not pageToken:
-                break
-            sys.stderr.write(page_message.replace('%%total_items%%', str(total_items)))
-        for deviceuser_name, device_user in device_users.items():
-            device_id = deviceuser_name.split('/')[1]
-            device_name = f'devices/{device_id}'
-            if 'users' not in devices[device_name]:
-                devices[device_name]['users'] = []
-            devices[device_name]['users'].append(device_user)
-    for device in devices.values():
+        device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
+            'deviceUsers', customer=customer, parent=parent,
+            page_message=page_message, pageSize=20, filter=device_filter)
+        for device_user in device_users:
+            for device in devices:
+                if device_user.get('name').startswith(device.get('name')):
+                    if 'users' not in device:
+                        device['users'] = []
+                    device['users'].append(device_user)
+                    break
+    for device in devices:
         device = utils.flatten_json(device)
         for a_key in device:
             if a_key not in titles:

src/gam/gapi/cloudidentity/groups.py

@@ -217,6 +217,7 @@ def print_():
     gapi_directory_customer.setTrueCustomerId()
     parent = f'customers/{GC_Values[GC_CUSTOMER_ID]}'
     usemember = None
+    query = None
     memberDelimiter = '\n'
     todrive = False
     titles = []
@@ -229,12 +230,15 @@ def print_():
             todrive = True
             i += 1
         elif myarg == 'enterprisemember':
-            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            member, _ = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
             usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
             i += 2
         elif myarg == 'delimiter':
             memberDelimiter = sys.argv[i + 1]
             i += 2
+        elif myarg == 'query':
+            query = sys.argv[i + 1]
+            i += 2
         elif myarg == 'sortheaders':
             sortHeaders = True
             i += 1
@@ -314,14 +318,20 @@ def print_():
             if entity['relationType'] == 'DIRECT':
                 entityList.append(gapi.call(ci.groups(), 'get', name=entity['group']))
     else:
+        if query:
+            method = 'search'
+            kwargs = {'query': query}
+        else:
+            method = 'list'
+            kwargs = {'parent': parent}
         entityList = gapi.get_all_pages(ci.groups(),
-                                        'list',
+                                        method,
                                         'groups',
                                         page_message=page_message,
                                         message_attribute=['groupKey', 'id'],
-                                        parent=parent,
                                         view='FULL',
-                                        pageSize=500)
+                                        pageSize=500,
+                                        **kwargs)
     i = 0
     count = len(entityList)
     for groupEntity in entityList:
@@ -491,7 +501,7 @@ def print_members():
                     )
             i += 2
         elif myarg == 'enterprisemember':
-            member = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
+            member, _ = gam.convertUIDtoEmailAddress(sys.argv[i + 1], email_types=['user', 'group'])
             usemember = f"member_key_id == '{member}' && 'cloudidentity.googleapis.com/groups.discussion_forum' in labels"
             i += 2
         elif myarg in ['cigroup', 'cigroups']:
@@ -866,6 +876,13 @@ def update():
                     'cloudidentity.googleapis.com/groups.discussion_forum': ''
                 }
                 i += 1
+            elif myarg == 'locked':
+                body['labels'] = {
+                    'cloudidentity.googleapis.com/groups.locked': '',
+                    'cloudidentity.googleapis.com/groups.security': '',
+                    'cloudidentity.googleapis.com/groups.discussion_forum': ''
+                }
+                i += 1
             elif myarg == 'dynamicsecurity':
                 body['labels'] = {
                     'cloudidentity.googleapis.com/groups.dynamic': '',
@@ -935,6 +952,12 @@ def group_email_to_id(ci, group, i=0, count=0):
         return None
 
 
+def group_id_to_email(ci, group_id):
+    return gapi.call(ci.groups(),
+                         'get',
+                         fields='groupKey/id',
+                         name=group_id).get('groupKey', {}).get('id')
+
 def membership_email_to_id(ci, parent, membership, i=0, count=0):
     membership = gam.normalizeEmailAddressOrUID(membership)
     try:

src/gam/gapi/cloudidentity/inboundsso.py

@@ -0,0 +1,548 @@
+"""Methods related to Cloud Identity Inbound (Google as SP) SAML SSO"""
+from datetime import datetime
+import re
+import sys
+
+import dateutil.parser
+import googleapiclient
+
+import gam
+from gam.var import GC_CUSTOMER_ID, GC_Values, MY_CUSTOMER
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
+from gam.gapi import errors as gapi_errors
+from gam.gapi import cloudidentity as gapi_cloudidentity
+from gam.gapi import directory as gapi_directory
+from gam.gapi.cloudidentity import groups as gapi_cloudidentity_groups
+from gam.gapi.directory import orgunits as gapi_directory_orgunits
+
+'''returns customer in the format inboundsso requires'''
+def get_sso_customer():
+    customer = GC_Values[GC_CUSTOMER_ID]
+    return f'customers/{customer}'
+
+
+'''returns org unit in the format inboundsso requires'''
+def get_orgunit_id(orgunit):
+    ou_id = gapi_directory_orgunits.getOrgUnitId(orgunit)[1]
+    if ou_id.startswith('id:'):
+        ou_id = ou_id[3:]
+    return f'orgUnits/{ou_id}'
+
+
+'''build Cloud Identity API'''
+def build():
+    return gapi_cloudidentity.build('cloudidentity')
+
+
+'''parse cmd for profile create/update'''
+def parse_profile(body, i):
+    name_only = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['displayName'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'entityid':
+            body.setdefault('idpConfig', {})['entityId'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'returnnameonly':
+            name_only = True
+            i += 1
+        elif myarg == 'loginurl':
+            body.setdefault('idpConfig', {})['singleSignOnServiceUri'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'logouturl':
+            body.setdefault('idpConfig', {})['logoutRedirectUri'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'changepasswordurl':
+            body.setdefault('idpConfig', {})['changePasswordUri'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam create/update inboundssoprofile')
+    return (name_only, body)
+
+
+'''convert profile nice names to unique ID'''
+def profile_displayname_to_name(displayName, ci=None):
+    if displayName.lower().startswith('id:') or displayName.lower().startswith('uid:'):
+        displayName = displayName.split(':', 1)[1]
+        if not displayName.startswith('inboundSamlSsoProfiles/'):
+            displayName = f'inboundSamlSsoProfiles/{displayName}'
+        return displayName
+    if not ci:
+        ci = build()
+    customer = get_sso_customer()
+    _filter = f'customer=="{customer}"'
+    profiles = gapi.get_all_pages(ci.inboundSamlSsoProfiles(),
+                                  'list',
+                                  'inboundSamlSsoProfiles',
+                                  filter=_filter)
+    matches = []
+    for profile in profiles:
+        if displayName.lower() == profile.get('displayName', '').lower():
+            matches.append(profile)
+    if len(matches) == 1:
+        return matches[0]['name']
+    if len(matches) == 0:
+        controlflow.system_error_exit(3, f'No Inbound SSO profile matches the name {displayName}')
+    else:
+        err_text = f'Multiple profiles match {displayName}:\n\n'
+        for m in matches:
+            err_text += f'  {m["name"]}  {m["displayName"]}\n'
+        controlflow.system_error_exit(3, err_text)
+
+
+'''get an assignment based on target'''
+def assignment_by_target(target, ci=None):
+    if not ci:
+        ci = build()
+    group_pattern = r'^groups/[^/]+$'
+    ou_pattern = r'^orgUnits/[^/]+$'
+    if re.match(group_pattern, target):
+        target_type = 'targetGroup'
+    elif re.match(ou_pattern, target):
+        target_type = 'targetOrgUnit'
+    elif target.lower().startswith('group:'):
+        target_type = 'targetGroup'
+        group_email = target[6:]
+        target = gapi_cloudidentity_groups.group_email_to_id(
+                    ci,
+                    group_email)
+    elif target.lower().startswith('orgunit:'):
+        target_type = 'targetOrgUnit'
+        ou_name = target[8:]
+        target = get_orgunit_id(ou_name)
+    else:
+        controlflow.system_error_exit(3, 'assignments should be prefixed with ' +
+                                         'group:, groups/, orgunit: or orgunits/')
+    customer = get_sso_customer()
+    _filter = f'customer=="{customer}"'
+    assignments = gapi.get_all_pages(ci.inboundSsoAssignments(),
+                                     'list',
+                                     'inboundSsoAssignments',
+                                     filter=_filter)
+    for assignment in assignments:
+        if target_type in assignment and assignment[target_type] == target:
+            return assignment
+    controlflow.system_error_exit(3, f'No SSO profile assigned to {target_type} {target}')
+
+
+'''gam create inboundssoprofile'''
+def create_profile():
+    ci = build()
+    body = {
+          'customer': get_sso_customer(),
+          'displayName': 'SSO Profile'
+          }
+    name_only, body = parse_profile(body, 3)
+    result = gapi.call(ci.inboundSamlSsoProfiles(),
+                       'create',
+                       body=body)
+    if result.get('done'):
+        if name_only:
+            print(result['response']['name'])
+        else:
+            print(f'Created profile {result["response"]["name"]}')
+            display.print_json(result['response'])
+    else:
+        controlflow.system_error_exit(3, 'Create did not finish {result}')
+
+
+'''gam print inboundssoprofiles'''
+def print_show_profiles(action='print'):
+    customer = get_sso_customer()
+    _filter = f'customer=="{customer}"'
+    ci = build()
+    todrive = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, f'gam {action} inboundssoprofiles')
+
+    profiles = gapi.get_all_pages(ci.inboundSamlSsoProfiles(),
+                                  'list',
+                                  'inboundSamlSsoProfiles',
+                                  filter=_filter)
+    if action == 'show':
+        for profile in profiles:
+            display.print_json(profile)
+            print()
+    elif action == 'print':
+        csv_rows = []
+        titles = []
+        for profile in profiles:
+            row = utils.flatten_json(profile)
+            for item in row:
+                if item not in titles:
+                    titles.append(item)
+            csv_rows.append(row)
+        display.write_csv_file(csv_rows,
+                               titles,
+                               'Inbound SSO Profiles',
+                               todrive)
+
+
+'''gam update inboundssoprofile'''
+def update_profile():
+    ci = build()
+    name = profile_displayname_to_name(sys.argv[3], ci)
+    body = {}
+    name_only, body = parse_profile(body, 4)
+    updateMask = ','.join(body.keys())
+    result = gapi.call(ci.inboundSamlSsoProfiles(),
+                       'patch',
+                       name=name,
+                       updateMask=updateMask,
+                       body=body)
+    if name_only:
+        print(result['response']['name'])
+    else:
+        display.print_json(result)
+
+
+'''gam info inboundssoprofile'''
+def info_profile(return_only=False, displayName=None, ci=None):
+    if not ci:
+        ci = build()
+    if not displayName:
+        displayName = sys.argv[3]
+    name = profile_displayname_to_name(displayName, ci)
+    result = gapi.call(ci.inboundSamlSsoProfiles(),
+                       'get',
+                       name=name)
+    if return_only:
+        return result
+    display.print_json(result)
+
+'''gam delete inboundssoprofile'''
+def delete_profile():
+    ci = build()
+    name = profile_displayname_to_name(sys.argv[3], ci)
+    result = gapi.call(ci.inboundSamlSsoProfiles(),
+                       'delete',
+                       name=name)
+    if result.get('done'):
+        print(f'Deleted profile {name}.')
+    else:
+        controlflow.system_error_exit(3, 'Delete did not finish: {result}')
+
+
+'''gam create inboundssocredentials'''
+def create_credentials():
+    allowed_sizes = [1024, 2048, 4096]
+    ci = build()
+    parent = None
+    generate_key = False
+    key_size = 2048
+    pemData = None
+    replace_oldest = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'profile':
+            parent = sys.argv[i+1]
+            parent = profile_displayname_to_name(parent, ci)
+            i += 2
+        elif myarg == 'pemfile':
+            pemfile = sys.argv[i+1]
+            pemData = fileutils.read_file(pemfile)
+            i += 2
+        elif myarg == 'generatekey':
+            generate_key = True
+            i += 1
+        elif myarg == 'replaceoldest':
+            replace_oldest = True
+            i += 1
+        elif myarg == 'keysize':
+            key_size = int(sys.argv[i+1])
+            if key_size not in allowed_sizes:
+                controlflow.expected_argument_exit('key_size',
+                                                   allowed_sizes,
+                                                   key_size)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam create inboundssocredential')
+    if not parent:
+        controlflow.missing_argument_exit('profile', 'gam create inboundssocredential')
+    if replace_oldest:
+        fields='nextPageToken,idpCredentials(name,updateTime)'
+        current_creds = gapi.get_all_pages(
+                ci.inboundSamlSsoProfiles().idpCredentials(),
+                'list',
+                'idpCredentials',
+                parent=parent,
+                fields=fields)
+        if len(current_creds) == 2:
+            oldest_key = min(current_creds,
+                             key=lambda x:x['updateTime'])
+            print(' deleting older key...')
+            delete_credentials(ci=ci,
+                               name=oldest_key['name'])
+        else:
+            print(' profile has {len(current_creds)} credentials. We only replace if there are 2.')
+    if generate_key:
+        privKey, pemData = gam._generatePrivateKeyAndPublicCert('GAM',
+                                                                key_size,
+                                                                b64enc_pub=False)
+        timestamp = datetime.now().strftime('%Y%m%d-%I%M%S')
+        priv_file = f'privatekey-{timestamp}.pem'
+        pub_file = f'publiccert-{timestamp}.pem'
+        fileutils.write_file(priv_file, privKey)
+        print(f' Wrote private key data to {priv_file}')
+        fileutils.write_file(pub_file, pemData)
+        print(f' Wrote public certificate to {pub_file}')
+    if not pemData:
+        controlflow.system_error_exit(3, 'You must either specify "pemfile <filename>" or "generate_key"')
+    body = {
+            'pemData': pemData,
+           }
+    result = gapi.call(ci.inboundSamlSsoProfiles().idpCredentials(),
+                       'add',
+                       parent=parent,
+                       body=body)
+    if result.get('done'):
+       print(f'Created credential {result["response"]["name"]}')
+       display.print_json(result['response'])
+    else:
+        controlflow.system_error_exit(3, 'Create did not finish {result}')
+
+
+'''gam delete inboundssocredential'''
+def delete_credentials(ci=None, name=None):
+    if not ci:
+        ci = build()
+    if not name:
+        name = sys.argv[3]
+    result = gapi.call(ci.inboundSamlSsoProfiles().idpCredentials(),
+                       'delete',
+                       name=name)
+    if result.get('done'):
+        print(f'Deleted credential {name}')
+    else:
+        controlflow.system_error_exit(3, 'Delete did not finish {result}')
+
+
+'''gam print inboundssocredentials'''
+def print_show_credentials(action='print'):
+    ci = build()
+    todrive = False
+    i = 3
+    profiles = []
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['profile', 'profiles']:
+            profiles = [profile_displayname_to_name(profile, ci)
+                            for profile in sys.argv[i+1].split(',')]
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg, f'gam {action} inboundssocredentials')
+    if not profiles:
+        customer = get_sso_customer()
+        _filter = f'customer=="{customer}"'
+        profiles = gapi.get_all_pages(ci.inboundSamlSsoProfiles(),
+                                      'list',
+                                      'inboundSamlSsoProfiles',
+                                      fields='inboundSamlSsoProfiles/name',
+                                      filter=_filter)
+        profiles = [p['name'] for p in profiles]
+    if action == 'print':
+        titles = []
+        csv_rows = []
+    credentials = []
+    for profile in profiles:
+        results = gapi.get_all_pages(ci.inboundSamlSsoProfiles().idpCredentials(),
+                                     'list',
+                                     'idpCredentials',
+                                     parent=profile)
+        credentials.extend(results)
+    if action == 'show':
+        for c in credentials:
+            display.print_json(c)
+            print()
+    elif action == 'print':
+        for c in credentials:
+            csv_row = utils.flatten_json(c)
+            for item in csv_row:
+                if item not in titles:
+                    titles.append(item)
+            csv_rows.append(csv_row)
+        display.write_csv_file(csv_rows,
+                               titles,
+                               'Inbound SSO Credentials',
+                               todrive)
+
+'''parse command for create/update inboundssoassignment'''
+def parse_assignment(body, i, ci):
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'rank':
+            body['rank'] = int(sys.argv[i+1])
+            i += 2
+        elif myarg == 'mode':
+            mode_choices = \
+                gapi.get_enum_values_minus_unspecified(
+                    ci._rootDesc['schemas']['InboundSsoAssignment']['properties']['ssoMode']['enum'])
+            body['ssoMode'] = sys.argv[i+1].upper()
+            if body['ssoMode'] not in mode_choices:
+                controlflow.expected_argument_exit('mode',
+                                                   ', '.join(mode_choices),
+                                                   sys.argv[i+1])
+            i += 2
+        elif myarg == 'profile':
+            profile_name = profile_displayname_to_name(
+                    sys.argv[i+1],
+                    ci)
+            body['samlSsoInfo'] = {
+                    'inboundSamlSsoProfile': profile_name
+                    }
+            i += 2
+        elif myarg == 'neverredirect':
+            body['signInBehavior'] = {
+                    'redirectCondition': 'NEVER'
+                    }
+            i += 1
+        elif myarg == 'group':
+            group = sys.argv[i+1]
+            body['targetGroup'] = gapi_cloudidentity_groups.group_email_to_id(
+                    ci,
+                    group)
+            i += 2
+        elif myarg in ['ou', 'org', 'orgunit']:
+            body['targetOrgUnit'] = get_orgunit_id(sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam create/update inboundssoassignment')
+    return body
+
+
+def update_assignment_target_names(assignment, ci, cd):
+    if 'targetGroup' in assignment:
+        assignment['targetGroupEmail'] = \
+            gapi_cloudidentity_groups.group_id_to_email(ci,
+                                                        assignment['targetGroup'])
+    elif 'targetOrgUnit' in assignment:
+        ou_id = assignment['targetOrgUnit'].split('/')[1]
+        assignment['targetOrgUnitPath'] = \
+            gapi_directory_orgunits.orgunit_from_orgunitid(f'id:{ou_id}', cd)
+
+
+'''gam create inboundssoassignment'''
+def create_assignment():
+    ci = build()
+    cd = gapi_directory.build()
+    body = {
+            'customer': get_sso_customer(),
+           }
+    body = parse_assignment(body, 3, ci)
+    result = gapi.call(ci.inboundSsoAssignments(),
+                       'create',
+                       body=body)
+    if result.get('done'):
+       print(f'Created assignment {result["response"]["name"]}')
+       update_assignment_target_names(result['response'], ci, cd)
+       display.print_json(result['response'])
+    else:
+        controlflow.system_error_exit(3, 'Create did not finish {result}')
+
+
+def get_assignment_name(name):
+  if name.startswith('id:') or name.startswith('uid:'):
+    name = name.split(':', 1)[1]
+  if not name.startswith('inboundSsoAssignments/'):
+    name = f'inboundSsoAssignments/{name}'
+  return name
+
+
+'''gam update inboundssoassignment'''
+def update_assignment():
+    ci = build()
+    cd = gapi_directory.build()
+    name = get_assignment_name(sys.argv[3])
+    body = parse_assignment({}, 4, ci)
+    updateMask = ','.join(list(body.keys()))
+    result = gapi.call(ci.inboundSsoAssignments(),
+                       'patch',
+                       name=name,
+                       updateMask=updateMask,
+                       body=body)
+    if result.get('done'):
+       print(f'Updated assignment {result["response"]["name"]}')
+       update_assignment_target_names(result['response'], ci, cd)
+       display.print_json(result['response'])
+    else:
+        controlflow.system_error_exit(3, 'Update did not finish {result}')
+
+
+'''gam delete inboundssoassignment'''
+def delete_assignment():
+    ci = build()
+    assignment = assignment_by_target(sys.argv[3], ci).get('name')
+    print(f'Deleting Inbound SSO Assignmnet {assignment}...')
+    gapi.call(ci.inboundSsoAssignments(),
+              'delete',
+              name=assignment)
+
+
+'''gam info inboundssoassignment'''
+def info_assignment():
+    ci = build()
+    cd = gapi_directory.build()
+    assignment = assignment_by_target(sys.argv[3], ci)
+    update_assignment_target_names(assignment, ci, cd)
+    profile = assignment.get('samlSsoInfo', {}).get('inboundSamlSsoProfile')
+    if profile:
+        assignment['samlSsoInfo']['inboundSamlSsoProfile'] = \
+            info_profile(return_only=True, displayName=f'id:{profile}', ci=ci)
+    display.print_json(assignment)
+
+
+'''gam print inboundssoassignments'''
+def print_show_assignments(action='print'):
+    ci = build()
+    cd = gapi_directory.build()
+    customer = get_sso_customer()
+    _filter = f'customer=="{customer}"'
+    todrive = False
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(myarg,
+                                              f'gam {action} inboundssoassignments')
+    assignments = gapi.get_all_pages(ci.inboundSsoAssignments(),
+                                     'list',
+                                     'inboundSsoAssignments',
+                                     filter=_filter)
+    if action == 'show':
+        for assignment in assignments:
+            update_assignment_target_names(assignment, ci, cd)
+            display.print_json(assignment)
+            print()
+    elif action == 'print':
+        titles = []
+        csv_rows = []
+        for assignment in assignments:
+            update_assignment_target_names(assignment, ci, cd)
+            csv_row = utils.flatten_json(assignment)
+            for item in csv_row:
+                if item not in titles:
+                    titles.append(item)
+            csv_rows.append(csv_row)
+        display.write_csv_file(csv_rows,
+                               titles,
+                               'Inbound SSO Assignments',
+                               todrive)

src/gam/gapi/cloudresourcemanager.py

@@ -13,12 +13,14 @@ def get_org_id():
     gapi_directory_customer.setTrueCustomerId()
     crm = build()
     query = f'directorycustomerid:{GC_Values[GC_CUSTOMER_ID]}'
-    orgs = gapi.get_all_pages(crm.organizations(),
+    results = gapi.call(crm.organizations(),
                      'search',
-                     'organizations',
+                     pageSize=1,
+                     fields='organizations/name',
                      query=query)
-    if len(orgs) < 1:
+    orgs = results.get('organizations')
+    if not orgs:
         # return nothing and let calling API deal with it
         # since caller knows what GCP role would serve best
         return 
-    return orgs[0]['name']
+    return orgs[0].get('name')

src/gam/gapi/directory/customer.py

@@ -21,36 +21,36 @@ def doGetCustomerInfo():
                               'get',
                               customerKey=customer_id)
     print(f'Customer ID: {customer_info["id"]}')
-    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    fields = 'domains(creationTime,domainName,isPrimary,verified)'
     try:
-        result = gapi.call(
+        domains = gapi.call(
             cd.domains(),
-            'get',
+            'list',
+            fields=fields,
             customer=customer_id,
-            domainName=customer_info['customerDomain'],
-            fields='verified',
-            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND])
+            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND]).get('domains', [])
+        for domain in domains:
+            if domain.get('isPrimary'):
+                primary_domain = domain
+                break
+        else:
+            primary_domain = {}
     except gapi.errors.GapiDomainNotFoundError:
-        result = {'verified': False}
-    print(f'Primary Domain Verified: {result["verified"]}')
-    # If customer has changed primary domain customerCreationTime is date
-    # of current primary being added, not customer create date.
-    # We should also get all domains and use oldest date
-    customer_creation = customer_info['customerCreationTime']
-    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
-    oldest = datetime.datetime.strptime(customer_creation, date_format)
-    domains = gapi.get_items(cd.domains(),
-                             'list',
-                             'domains',
-                             customer=customer_id,
-                             fields='domains(creationTime)')
+        primary_domain = {}
+    print(f'Primary Domain: {primary_domain.get("domainName", "Unknown")}')
+    print(f'Primary Domain Verified: {primary_domain.get("verified", "Unknown")}')
+    # we'll assume creation time is time of oldest domain customer has
+    oldest = 'Unknown' 
     for domain in domains:
         creation_timestamp = int(domain['creationTime']) / 1000
         domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
-        if domain_creation < oldest:
+        if oldest == 'Unknown' or domain_creation < oldest:
             oldest = domain_creation
-    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
-    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    if oldest != 'Unknown':
+        date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+        oldest = oldest.strftime(date_format)
+    print(f'Customer Creation Time: {oldest}')
+    customer_language = customer_info.get('language', 'Unset or Unknown (defaults to en)')
     print(f'Default Language: {customer_language}')
     if 'postalAddress' in customer_info:
         print('Address:')
@@ -59,7 +59,7 @@ def doGetCustomerInfo():
                 print(f' {field}: {customer_info["postalAddress"][field]}')
     if 'phoneNumber' in customer_info:
         print(f'Phone: {customer_info["phoneNumber"]}')
-    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    print(f'Admin Secondary Email: {customer_info.get("alternateEmail", "Unknown")}')
     user_counts_map = {
         'accounts:num_users': 'Total Users',
         'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',

src/gam/gapi/directory/groups.py

@@ -189,7 +189,7 @@ def info(group_name=None):
             pass
     print('')
     print('Group Settings:')
-    for key, value in list(basic_info.items()):
+    for key, value in sorted(list(basic_info.items())):
         if (key in ['kind', 'etag']) or ((key == 'aliases') and
                                          (not getAliases)):
             continue
@@ -199,7 +199,7 @@ def info(group_name=None):
                 print(f'  {val}')
         else:
             print(f' {key}: {value}')
-    for key, value in list(settings.items()):
+    for key, value in sorted(list(settings.items())):
         if key in ['kind', 'etag', 'description', 'email', 'name']:
             continue
         print(f' {key}: {value}')
@@ -1217,6 +1217,8 @@ GROUP_SETTINGS_LIST_PATTERN = re.compile(r'([A-Z][A-Z_]+[A-Z]?)')
 def getGroupAttrValue(myarg, value, gs_object, gs_body, function):
     if myarg == 'collaborative':
         myarg = 'enablecollaborativeinbox'
+    elif myarg == 'gal':
+        myarg = 'includeinglobaladdresslist'
     for (attrib,
          params) in list(gs_object['schemas']['Groups']['properties'].items()):
         if attrib in ['kind', 'etag', 'email']:

src/gam/gapi/directory/roleassignments.py

@@ -16,7 +16,9 @@ NONSECURITY_GROUP_CONDITION = f'!{SECURITY_GROUP_CONDITION}'
 def create():
     cd = gapi_directory.build()
     user = gam.normalizeEmailAddressOrUID(sys.argv[3])
-    body = {'assignedTo': gam.convertEmailAddressToUID(user, cd)}
+    body = {'assignedTo': gam.convertEmailAddressToUID(sys.argv[3],
+                          cd=cd,
+                          email_type='any')}
     role = sys.argv[4]
     body['roleId'] = gapi_directory_roles.getRoleId(role)
     body['scopeType'] = sys.argv[5].upper()
@@ -70,7 +72,7 @@ def print_():
     item_fields = ['roleAssignmentId', 'roleId', 'assignedTo', 'scopeType', 'orgUnitId']
     titles = [
         'roleAssignmentId', 'roleId', 'role', 'assignedTo', 'assignedToUser',
-        'scopeType', 'orgUnitId', 'orgUnit'
+        'assignedToGroup', 'scopeType', 'orgUnitId', 'orgUnit'
     ]
     csvRows = []
     i = 3
@@ -107,7 +109,21 @@ def print_():
         admin_attrib = {}
         for key, value in list(admin.items()):
             if key == 'assignedTo':
-                admin_attrib['assignedToUser'] = gam.user_from_userid(value)
+                email_types = admin_attrib.get('assigneeType')
+                if email_types == 'user':
+                    email_field = 'assignedToUser'
+                elif email_types == 'group':
+                    email_field = 'assignedToGroup'
+                else:
+                    email_field = None
+                assignment_email, assignment_type = gam.convertUIDtoEmailAddress(f'uid:{value}', cd, email_types=['user', 'group'])
+                if not email_field and assignment_type in ['user', 'group']:
+                    if assignment_type == 'user':
+                        email_field = 'assignedToUser'
+                    else:
+                        email_field = 'assignedToGroup'
+                if email_field:
+                    admin_attrib[email_field] = assignment_email
             elif key == 'roleId':
                 admin_attrib['role'] = gapi_directory_roles.role_from_roleid(value)
             elif key == 'orgUnitId':

src/gam/gapi/directory/users.py

@@ -2,6 +2,7 @@ import sys
 from time import sleep
 
 import gam
+from gam import controlflow
 from gam import display
 from gam import gapi
 from gam.gapi import directory as gapi_directory
@@ -18,7 +19,7 @@ def delete():
                   userKey=user_email,
                   throw_reasons=[gapi_errors.ErrorReason.CONDITION_NOT_MET])
     except gam.gapi.errors.GapiConditionNotMetError as err:
-        display.print_error(
+        controlflow.system_error_exit(3,
             f'{err} The user {user_email} may be (or have recently been) on Google Vault Hold and thus not eligible for deletion. You can check holds with "gam user <email> show vaultholds".'
         )
 

src/gam/gapi/drive/__init__.py

@@ -4,5 +4,5 @@ import gam
 def build(user=None):
     if not user:
         user = gam._get_admin_email()
-    userEmail = gam.convertUIDtoEmailAddress(user)
+    userEmail, _ = gam.convertUIDtoEmailAddress(user)
     return (userEmail, gam.buildGAPIServiceObject('drive3', userEmail))

src/gam/gapi/licensing.py

@@ -204,6 +204,7 @@ def print_(returnFields=None,
                     ],
                     page_message=page_message,
                     customerId=customer_id,
+                    maxResults=100,
                     productId=product,
                     skuId=sku,
                     fields=fields)
@@ -234,6 +235,7 @@ def print_(returnFields=None,
                     ],
                     page_message=page_message,
                     customerId=customer_id,
+                    maxResults=100,
                     productId=productId,
                     fields=fields)
                 if countsOnly:

src/gam/gapi/reports.py

@@ -426,6 +426,7 @@ def showReport():
         titles = ['name', 'value', 'client_id']
         csvRows = []
         auth_apps = list()
+        usage = list(usage)
         for item in usage[0]['parameters']:
             if 'name' not in item:
                 continue

src/gam/gapi/storage.py

@@ -2,20 +2,150 @@ import base64
 import os
 import re
 import sys
+import time
 
 import googleapiclient
+from pathvalidate import sanitize_filepath
 
 import gam
+from gam.gapi import errors as gapi_errors
 from gam.var import *
+from gam import controlflow
 from gam import fileutils
 from gam import gapi
 from gam import utils
 
 
-def build_gapi():
+def build():
     return gam.buildGAPIObject('storage')
 
 
+def copy_bucket():
+    s = build()
+    source_bucket = None
+    target_bucket = None
+    prefix = None
+    target_prefix = ''
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'sourcebucket':
+            source_bucket = sys.argv[i+1]
+            i += 2
+        elif myarg == 'targetbucket':
+            target_bucket = sys.argv[i+1]
+            i += 2
+        elif myarg == 'sourceprefix':
+            prefix = sys.argv[i+1]
+            i += 2
+        elif myarg == 'targetprefix':
+            target_prefix = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam copy storagebucket')
+    if not target_bucket:
+        controlflow.missing_argument_exit('target_bucket', 'gam copy storagebucket')
+    if not source_bucket:
+        controlflow.missing_argument_exit('source_bucket', 'gam copy storagebucket')
+    page_message = gapi.got_total_items_msg('Storage Objects', '...\n')
+    objects = gapi.get_all_pages(s.objects(),
+                                 'list',
+                                 items='items',
+                                 page_message=page_message,
+                                 prefix=prefix,
+                                 bucket=source_bucket,
+                                 fields='items(name,bucket,md5Hash),nextPageToken')
+    copy_objects(objects,
+                 target_bucket,
+                 target_prefix)
+
+
+def copy_objects(objects,
+                 target_bucket,
+                 target_prefix):
+    """Copies objects to target_bucket.
+
+    Args:
+      objects: list of object dicts
+        [
+          {
+            bucket: source bucket,
+            name: source object name,
+            (optional) md5Hash: source file hash value
+          },
+          ...
+        ]
+      target_bucket: target bucket id
+      target_prefix: prefix name to prepend to target object
+
+    """
+
+    def process_rewrite(request_id, response, exception):
+        file_ptr = int(request_id)
+        if exception:
+            # Poor man's backoff/retry
+            if exception.status_code == 429 or exception.status_code > 499:
+                print(f'Temporary error {exception.status_code}. Sleeping 10 seconds...')
+                time.sleep(10)
+                next_batch.add(s.objects().rewrite(**files_to_copy[file_ptr]['method']),
+                                                   request_id=request_id)
+                return
+            else:
+                raise exception
+        file_count = file_ptr + 1
+        source_displayname = files_to_copy[file_ptr]['source_displayname']
+        target_displayname = files_to_copy[file_ptr]['target_displayname']
+        if response.get('done'):
+            source_md5 = files_to_copy[file_ptr]['md5Hash']
+            target_md5 = response['resource']['md5Hash']
+            if source_md5 != target_md5:
+                controlflow.system_error_exit(99, f'Target file {target_displayname} checksum {target_md5} does not match source {source_md5}. This should not happen')
+            else:
+                print(f'[ {file_count} / {total_files} ] 100% VERIFIED - finished copying:\n  source: {source_displayname}\n  dest: {target_displayname}')
+        else:
+            total_bytes = float(response.get('objectSize'))
+            done_bytes = float(response.get('totalBytesRewritten'))
+            pct = (done_bytes / total_bytes) * 100
+            print(f'[ {file_count} / {total_files} ] {pct:.2f}%\n  source: {source_displayname}\n  dest:{target_displayname}')
+            files_to_copy[file_ptr]['method']['rewriteToken'] = response.get('rewriteToken')
+            next_batch.add(s.objects().rewrite(**files_to_copy[file_ptr]['method']),
+                                               request_id=request_id)
+
+    s = build()
+    sbatch = s.new_batch_http_request(callback=process_rewrite)
+    files_to_copy = []
+    for object_ in objects:
+        files_to_copy.append(
+                {
+                    'md5Hash': object_['md5Hash'],
+                    'source_displayname': f'{object_["bucket"]}:{object_["name"]}',
+                    'target_displayname': f'{target_bucket}:{target_prefix}{object_["name"]}',
+                    'method': {
+                                'destinationBucket': target_bucket,
+                                'destinationObject': f'{target_prefix}{object_["name"]}',
+                                'sourceBucket': object_['bucket'],
+                                'sourceObject': object_['name'],
+                        #        'maxBytesRewrittenPerCall': 1048576, # uncomment to easily test multiple rewrite API calls per object
+                               },
+                })
+    i = 0
+    total_files = len(files_to_copy)
+    for file in files_to_copy:
+        while len(sbatch._order) == 100:
+            next_batch = s.new_batch_http_request(callback=process_rewrite)
+            sbatch.execute()
+            sbatch = next_batch
+        sbatch.add(s.objects().rewrite(**file['method']),
+                   request_id=str(i))
+        i += 1
+    while len(sbatch._order) > 0:
+        next_batch = s.new_batch_http_request(callback=process_rewrite)
+        sbatch.execute()
+        sbatch = next_batch
+    print('All done!')
+
+
 def get_cloud_storage_object(s,
                              bucket,
                              object_,
@@ -23,11 +153,12 @@ def get_cloud_storage_object(s,
                              expectedMd5=None):
     if not local_file:
         local_file = object_
+    local_file = sanitize_filepath(local_file, platform='auto')
     if os.path.exists(local_file):
-        sys.stdout.write(' File already exists. ')
+        sys.stdout.write(f'File {local_file} already exists.')
         sys.stdout.flush()
         if expectedMd5:
-            sys.stdout.write(f'Verifying {expectedMd5} hash...')
+            sys.stdout.write(f' verifying {expectedMd5} hash...')
             sys.stdout.flush()
             if utils.md5_matches_file(local_file, expectedMd5, False):
                 print('VERIFIED')
@@ -35,7 +166,7 @@ def get_cloud_storage_object(s,
             print('not verified. Downloading again and over-writing...')
         else:
             return  # nothing to verify, just assume we're good.
-    print(f'saving to {local_file}')
+    print(f'Saving to {local_file}')
     request = s.objects().get_media(bucket=bucket, object=object_)
     file_path = os.path.dirname(local_file)
     if not os.path.exists(file_path):
@@ -60,7 +191,7 @@ def get_cloud_storage_object(s,
 
 def download_bucket():
     bucket = sys.argv[3]
-    s = build_gapi()
+    s = build()
     page_message = gapi.got_total_items_msg('Files', '...')
     fields = 'nextPageToken,items(name,id,md5Hash)'
     objects = gapi.get_all_pages(s.objects(),

src/gam/gapi/vault.py

@@ -1,3 +1,4 @@
+from base64 import b64encode
 import datetime
 import json
 import sys
@@ -11,6 +12,7 @@ from gam import controlflow
 from gam import display
 from gam import fileutils
 from gam import gapi
+from gam.gapi import errors as gapi_errors
 from gam.gapi import storage as gapi_storage
 from gam.gapi import directory as gapi_directory
 from gam.gapi.directory import orgunits as gapi_directory_orgunits
@@ -336,9 +338,9 @@ def print_count():
         _validate_query(query, query_discovery)
         body['query'] = query
         operation = gapi.call(v.matters(), 'count', matterId=matterId, body=body)
-    print(f'Watching operation {operation["name"]}...')
+    sys.stderr.write(f'Watching operation {operation["name"]}...\n')
     while not operation.get('done'):
-        print(f' operation {operation["name"]} is not done yet. Checking again in {operation_wait} seconds')
+        sys.stderr.write(f' operation {operation["name"]} is not done yet. Checking again in {operation_wait} seconds\n')
         sleep(operation_wait)
         operation = gapi.call(v.operations(), 'get', name=operation['name'])
     response = operation.get('response', {})
@@ -509,7 +511,7 @@ def getHoldInfo():
         account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
         for i in range(0, len(results['accounts'])):
             uid = f'uid:{results["accounts"][i]["accountId"]}'
-            acct_email = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
+            acct_email, _ = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
             results['accounts'][i]['email'] = acct_email
     if 'orgUnit' in results:
         results['orgUnit']['orgUnitPath'] = gapi_directory_orgunits.info(
@@ -518,7 +520,6 @@ def getHoldInfo():
 
 
 def convertExportNameToID(v, nameOrID, matterId):
-    nameOrID = nameOrID.lower()
     cg = UID_PATTERN.match(nameOrID)
     if cg:
         return cg.group(1)
@@ -529,7 +530,7 @@ def convertExportNameToID(v, nameOrID, matterId):
                                  matterId=matterId,
                                  fields=fields)
     for export in exports:
-        if export['name'].lower() == nameOrID:
+        if export['name'].lower() == nameOrID.lower():
             return export['id']
     controlflow.system_error_exit(
         4, f'could not find export name {nameOrID} '
@@ -682,18 +683,23 @@ def showHoldsForUsers(users):
     v = buildGAPIObject()
     matterIds = _getAllMatterIds(v)
     matterHolds = {}
+    fields = 'holds(holdId,name,accounts(accountId,email),orgUnit),nextPageToken'
     for matterId in matterIds:
-        matterHolds[matterId] = gapi.get_all_pages(v.matters().holds(),
-                                                   'list',
-                                                   'holds',
-                                                   fields='holds(holdId,name,accounts(accountId,email),orgUnit),nextPageToken',
-                                                   matterId=matterId)
+        try:
+            matterHolds[matterId] = gapi.get_all_pages(v.matters().holds(),
+                                                       'list',
+                                                       'holds',
+                                                       fields=fields,
+                                                       throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
+                                                       matterId=matterId)
+        except googleapiclient.errors.HttpError:
+            continue
     totalHolds = 0
     for user in users:
         user = user.lower()
         orgUnits = gapi_directory_orgunits._getAllParentOrgUnitsForUser(user, cd)
-        for matterId in matterIds:
-            for hold in matterHolds[matterId]:
+        for matterId, holds in matterHolds.items():
+            for hold in holds:
                 if 'orgUnit' in hold:
                     orgUnitId = hold['orgUnit'].get('orgUnitId')
                     if orgUnitId in orgUnits:
@@ -786,16 +792,54 @@ def getMatterInfo():
         cd = gam.buildGAPIObject('directory')
         for i in range(0, len(result['matterPermissions'])):
             uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
-            user_email = gam.convertUIDtoEmailAddress(uid, cd)
+            user_email, _ = gam.convertUIDtoEmailAddress(uid, cd)
             result['matterPermissions'][i]['email'] = user_email
     display.print_json(result)
 
 
+def copyExport():
+    v = buildGAPIObject()
+    s = gapi_storage.build()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    target_bucket = None
+    target_prefix = ''
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'targetbucket':
+            target_bucket = sys.argv[i+1]
+            i += 2
+        elif myarg == 'targetprefix':
+            target_prefix = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam copy export')
+    if not target_bucket:
+        controlflow.missing_argument_exit('target_bucket', 'gam copy export')
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
+    objects = []
+    for s_file in export['cloudStorageSink']['files']:
+        # Convert to md5Hash format Storage API uses
+        # because OF COURSE they differ
+        md5Hash = b64encode(bytes.fromhex(s_file['md5Hash'])).decode()
+        objects.append({'bucket': s_file['bucketName'],
+                        'name': s_file['objectName'],
+                        'md5Hash': md5Hash})
+    gapi_storage.copy_objects(objects,
+                              target_bucket,
+                              target_prefix)
+
+
 def downloadExport():
     verifyFiles = True
     extractFiles = True
     v = buildGAPIObject()
-    s = gapi_storage.build_gapi()
+    s = gapi_storage.build()
     matterId = getMatterItem(v, sys.argv[3])
     exportId = convertExportNameToID(v, sys.argv[4], matterId)
     targetFolder = GC_Values[GC_DRIVE_DIR]
@@ -823,24 +867,17 @@ def downloadExport():
     for s_file in export['cloudStorageSink']['files']:
         bucket = s_file['bucketName']
         s_object = s_file['objectName']
-        filename = os.path.join(targetFolder, s_object.replace('/', '-'))
-        print(f'saving to {filename}')
-        request = s.objects().get_media(bucket=bucket, object=s_object)
-        f = fileutils.open_file(filename, 'wb')
-        downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
-        done = False
-        while not done:
-            status, done = downloader.next_chunk()
-            sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
-            sys.stdout.flush()
-        sys.stdout.write('\n Download complete. Flushing to disk...\n')
-        fileutils.close_file(f, True)
         if verifyFiles:
             expected_hash = s_file['md5Hash']
-            sys.stdout.write(f' Verifying file hash is {expected_hash}...')
-            sys.stdout.flush()
-            utils.md5_matches_file(filename, expected_hash, True)
-            print('VERIFIED')
+        else:
+            expected_hash = None
+        local_file = s_object.replace('/', '-').replace(':', '-')
+        filename = os.path.join(targetFolder, local_file)
+        gapi_storage.get_cloud_storage_object(s,
+                             bucket,
+                             s_object,
+                             local_file=filename,
+                             expectedMd5=expected_hash)
         if extractFiles and re.search(r'\.zip$', filename):
             gam.extract_nested_zip(filename, targetFolder)
 
@@ -976,10 +1013,14 @@ def printHolds():
     for matterId in matterIds:
         i += 1
         sys.stderr.write(f'Retrieving holds for matter {matterId} ({i}/{matter_count})\n')
-        holds = gapi.get_all_pages(v.matters().holds(),
+        try:
+            holds = gapi.get_all_pages(v.matters().holds(),
                                    'list',
                                    'holds',
+                                   throw_reasons=[gapi_errors.ErrorReason.FOUR_O_O],
                                    matterId=matterId)
+        except googleapiclient.errors.HttpError:
+            continue
         for hold in holds:
             display.add_row_titles_to_csv_file(
                 utils.flatten_json(hold, flattened={'matterId': matterId}),

src/gam/transport.py

@@ -9,6 +9,8 @@ from gam.var import GC_TLS_MAX_VERSION
 from gam.var import GC_TLS_MIN_VERSION
 from gam.var import GC_Values
 
+# Bump default retries
+httplib2.RETRIES = 5
 
 def create_http(cache=None,
                 timeout=None,

src/gam/utils.py

@@ -96,9 +96,13 @@ class _DeHTMLParser(HTMLParser):
                       re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
 
 
-def commonprefix(m):
+def commonprefix(m, checkEnum=False):
     '''Given a list of strings m, return string which is prefix common to all'''
     s1 = min(m)
+    if checkEnum:
+        loc = s1.find('ENUM_')
+        if loc > 0:
+            return s1[:loc+5]
     s2 = max(m)
     for i, c in enumerate(s1):
         if c != s2[i]:

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.24'
+GAM_VERSION = '6.58'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'
@@ -17,6 +17,18 @@ GAM_INFO = (
     f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
     f'{platform.platform()} {platform.machine()}')
 
+# Packages we want version info available
+# for "gam version extended"
+GAM_VER_LIBS = ['cryptography',
+                'filelock',
+                'google-auth-httplib2',
+                'google-auth-oauthlib',
+                'google-auth',
+                'httplib2',
+                'passlib',
+                'python-dateutil',
+                'yubikey-manager',
+                ]
 GAM_RELEASES = 'https://github.com/GAM-team/GAM/releases'
 GAM_WIKI = 'https://github.com/GAM-team/GAM/wiki'
 GAM_ALL_RELEASES = 'https://api.github.com/repos/GAM-team/GAM/releases'
@@ -62,6 +74,21 @@ SKUS = {
         'aliases': ['cloudsearch'],
         'displayName': 'Google Cloud Search',
     },
+    '1010380001': {
+        'product': '101038',
+        'aliases': ['appsheetcore'],
+        'displayName': 'AppSheet Core',
+    },
+    '1010380002': {
+        'product': '101038',
+        'aliases': ['appsheetstandard', 'appsheetenterprisestandard'],
+        'displayName': 'AppSheet Enterprise Standard',
+    },
+    '1010380003': {
+        'product': '101038',
+        'aliases': ['appsheetplus', 'appsheetenterpriseplus'],
+        'displayName': 'AppSheet Enterprise Plus',
+    },
     '1010310002': {
         'product': '101031',
         'aliases': ['gsefe', 'e4e', 'gsuiteenterpriseeducation'],
@@ -127,6 +154,16 @@ SKUS = {
         'aliases': ['gwetlu', 'workspaceeducationupgrade'],
         'displayName': 'Google Workspace for Education: Teaching and Learning Upgrade'
     },
+    '1010390001': {
+        'product': '101039',
+        'aliases': ['assuredcontrols'],
+        'displayName': 'Assured Controls',
+    },
+    '1010400001': {
+        'product': '101040',
+        'aliases': ['beyondcorp', 'beyondcorpenterprise', 'bce'],
+        'displayName': 'Beyond Corp Enterprise',
+    },
     'Google-Apps': {
         'product': 'Google-Apps',
         'aliases': ['standard', 'free'],
@@ -206,7 +243,7 @@ SKUS = {
     },
     '1010020030': {
         'product': 'Google-Apps',
-        'aliases': ['workspacefrontline', 'workspacefrontlineworker'],
+        'aliases': ['wsflw', 'workspacefrontline', 'workspacefrontlineworker'],
         'displayName': 'Workspace Frontline'
     },
     '1010340002': {
@@ -290,6 +327,9 @@ PRODUCTID_NAME_MAPPINGS = {
     '101035': 'Cloud Search',
     '101036': 'Google Meet Global Dialing',
     '101037': 'G Suite Workspace for Education',
+    '101038': 'AppSheet',
+    '101039': 'Assured Controls',
+    '101040': 'Beyond Corp',
     'Google-Apps': 'Google Workspace',
     'Google-Chrome-Device-Management': 'Google Chrome Device Management',
     'Google-Drive-storage': 'Google Drive Storage',
@@ -629,7 +669,7 @@ GOOGLEDOC_VALID_EXTENSIONS_MAP = {
         '.docx', '.html', '.odt', '.pdf', '.rtf', '.txt', '.zip'
     ],
     MIMETYPE_GA_PRESENTATION: ['.pdf', '.pptx', '.odp', '.txt'],
-    MIMETYPE_GA_SPREADSHEET: ['.csv', '.ods', '.pdf', '.xlsx', '.zip'],
+    MIMETYPE_GA_SPREADSHEET: ['.csv', '.ods', '.pdf', '.tsv', '.xlsx', '.zip'],
 }
 
 MACOS_CODENAMES = {
@@ -648,6 +688,7 @@ MACOS_CODENAMES = {
         },
     11: 'Big Sur',
     12: 'Monterey',
+    13: 'Ventura',
     }
 
 _MICROSOFT_FORMATS_LIST = [{
@@ -971,6 +1012,8 @@ CROS_ARGUMENT_TO_PROPERTY_MAP = {
     'autoupdateexpiration': ['autoUpdateExpiration',],
     'bootmode': ['bootMode',],
     'cpustatusreports': ['cpuStatusReports',],
+    'deprovisionreason': ['deprovisionReason',],
+    'lastDeprovisionTimestamp': ['lastDeprovisionTimestamp',],
     'devicefiles': ['deviceFiles',],
     'deviceid': ['deviceId',],
     'dockmacaddress': ['dockMacAddress',],
@@ -978,6 +1021,7 @@ CROS_ARGUMENT_TO_PROPERTY_MAP = {
     'ethernetmacaddress': ['ethernetMacAddress',],
     'ethernetmacaddress0': ['ethernetMacAddress0',],
     'firmwareversion': ['firmwareVersion',],
+    'firstenrollmenttime': ['firstEnrollmentTime',],
     'lastenrollmenttime': ['lastEnrollmentTime',],
     'lastknownnetwork': ['lastKnownNetwork'],
     'lastsync': ['lastSync',],
@@ -1035,7 +1079,10 @@ CROS_SCALAR_PROPERTY_PRINT_ORDER = [
     'ethernetMacAddress0',
     'macAddress',
     'systemRamTotal',
+    'firstEnrollmentTime',
     'lastEnrollmentTime',
+    'deprovisionReason',
+    'lastDeprovisionTimestamp',
     'orderNumber',
     'manufactureDate',
     'supportEndDate',
@@ -1202,6 +1249,7 @@ _DEFAULT_CHARSET = UTF8
 _FN_CLIENT_SECRETS_JSON = 'client_secrets.json'
 _FN_OAUTH2SERVICE_JSON = 'oauth2service.json'
 _FN_OAUTH2_TXT = 'oauth2.txt'
+_FN_ROOTS_PEM = 'roots.pem'
 #
 GM_Globals = {
     GM_SYSEXITRC: 0,
@@ -1269,6 +1317,9 @@ GC_ENABLE_DASA = 'enabledasa'
 # and doRequestOAuth prints a link and waits for the verification code when
 # oauth2.txt is being created
 GC_NO_BROWSER = 'no_browser'
+# If low memory is True, GAM tries to save RAM by writing pages to disk
+# temporarily
+GC_LOW_MEMORY = 'low_memory'
 # If no_tdemail is True, writeCSVfile won't send an email
 GC_NO_TDEMAIL = 'no_tdemail'
 # oauth_browser forces usage of web server OAuth flow that proved problematic.
@@ -1308,7 +1359,7 @@ GC_TLS_MAX_VERSION = 'tls_max_ver'
 # Path to certificate authority file for validating TLS hosts
 GC_CA_FILE = 'ca_file'
 
-TLS_MIN = 'TLSv1_3' if hasattr(ssl.SSLContext(), 'minimum_version') else None
+TLS_MIN = 'TLSv1_3'
 GC_Defaults = {
     GC_ADMIN_EMAIL: '',
     GC_AUTO_BATCH_MIN: 0,
@@ -1324,6 +1375,7 @@ GC_Defaults = {
     GC_DOMAIN: '',
     GC_DRIVE_DIR: '',
     GC_ENABLE_DASA: False,
+    GC_LOW_MEMORY: False,
     GC_NO_BROWSER: False,
     GC_NO_TDEMAIL: False,
     GC_NO_CACHE: False,
@@ -1343,7 +1395,7 @@ GC_Defaults = {
     GC_CSV_ROW_DROP_FILTER: '',
     GC_TLS_MIN_VERSION: TLS_MIN,
     GC_TLS_MAX_VERSION: None,
-    GC_CA_FILE: None,
+    GC_CA_FILE: _FN_ROOTS_PEM,
 }
 
 GC_Values = {}
@@ -1408,6 +1460,9 @@ GC_VAR_INFO = {
     GC_ENABLE_DASA: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },
+    GC_LOW_MEMORY: {
+        GC_VAR_TYPE: GC_TYPE_BOOLEAN
+    },
     GC_NO_BROWSER: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },

src/requirements.txt

@@ -6,8 +6,7 @@ google-auth-httplib2
 google-auth-oauthlib>=0.4.1
 google-auth>=2.3.2
 httplib2>=0.17.0
-importlib.metadata; python_version < '3.8'
 passlib>=1.7.2
 pathvalidate
 python-dateutil
-yubikey-manager>=4.0.0
+yubikey-manager>=5.0

src/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = GAM for Google Workspace
-version = 6.0.23
+version = attr: gam.var.GAM_VERSION
 description = Command line management for Google Workspaces
 long_description = file: readme.md
 long_description_content_type = text/markdown
@@ -13,15 +13,15 @@ keywords = google, oauth2, gsuite, google-apps, google-admin-sdk, google-drive,
 classifiers =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     License :: OSI Approved :: Apache License
 
 [options]
 packages = find:
-python_requires = >= 3.7
+python_requires = >= 3.8
 install_requires =
     cryptography
     distro; sys_platform == 'linux'
@@ -31,11 +31,13 @@ install_requires =
     google-auth-oauthlib >= 0.4.6
     google-auth >= 2.3.3
     httplib2 >= 0.20.2
-    importlib.metadata; python_version < '3.8'
     passlib >= 1.7.4
-    python-dateutil
-    yubikey-manager >= 4.0.0
     pathvalidate
+    python-dateutil
+    yubikey-manager >= 5.0
+
+[options.package_data]
+* = *.pem
 
 # used during pip install .[test]
 [options.extras_require]