Merge branch 'main' of github.com:GAM-team/GAM

To be released soon after OpenSSL 3.0.7...

.github/actions/decrypt.sh

@@ -14,4 +14,5 @@ gpg --quiet --batch --yes --decrypt --passphrase="${PASSCODE}" \
     --output "${credsfile}" "${gpgfile}"
 
 tar xvvf "${credsfile}" --directory "${gampath}"
-ls -l "${gampath}"
+rm -rvf "${gpgfile}"
+rm -rvf "${credsfile}"

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
             goal: build
             arch: x86_64
             openssl_archs: linux-x86_64
-          - os: [self-hosted, linux, arm64]
+          - os: [self-hosted, linux, arm64, gcp]
             jid: 2
             goal: build
             arch: aarch64
@@ -66,27 +66,36 @@ jobs:
             arch: x86_64
           - os: ubuntu-22.04
             goal: test
-            python: "3.11-dev"
+            python: "3.10"
             jid: 10
             arch: x86_64
 
     steps:
-      - uses: actions/checkout@master
+
+      - uses: actions/checkout@v3
         with:
           persist-credentials: false
           fetch-depth: 0
 
       - name: Cache multiple paths
-        uses: actions/cache@v2
+        if: matrix.goal == 'build'
+        uses: actions/cache@v3
         id: cache-python-ssl
         with:
           path: |
-            bin
-          key: gam-${{ matrix.jid }}-20220802
+            bin.tar.xz
+            src/cpython
+          key: gam-${{ matrix.jid }}-20221101
+
+      - name: Untar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar xvvf bin.tar.xz
 
       - name: Use pre-compiled Python for testing
         if: matrix.python != ''
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 
@@ -117,11 +126,11 @@ jobs:
           sudo apt-get -qq --yes update
           sudo apt-get -qq --yes install swig libpcsclite-dev
 
-      - name: MacOS remove Homebrew
-        if: runner.os == 'macOS'
-        run: |
-          # remove everything except the libraries needed by yubikey-manager
-          brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
+     #- name: MacOS remove Homebrew
+     #  if: runner.os == 'macOS'
+     #  run: |
+     #    # remove everything except the libraries needed by yubikey-manager
+     #    brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
 
       - name: MacOS install tools
         if: runner.os == 'macOS'
@@ -147,6 +156,7 @@ jobs:
           openssl_archs: ${{ matrix.openssl_archs }}
         run: |
           echo "We are running on ${RUNNER_OS}"
+          LD_LIBRARY_PATH="${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib"
           if [[ "${arch}" == "Win64" ]]; then
             PYEXTERNALS_PATH="amd64"
             PYBUILDRELEASE_ARCH="x64"
@@ -178,21 +188,21 @@ jobs:
             MAKE=nmake
             MAKEOPT=""
             PERL="c:\strawberry\perl\bin\perl.exe"
-            echo "PYTHON=${PYTHON_INSTALL_PATH}\python.exe" >> $GITHUB_ENV
+            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}"
+            echo "PYTHON=${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}/python.exe" >> $GITHUB_ENV
             echo "GAM_ARCHIVE_ARCH=${GAM_ARCHIVE_ARCH}" >> $GITHUB_ENV
             echo "WIX_ARCH=${WIX_ARCH}" >> $GITHUB_ENV
           fi
           echo "We'll run make with: ${MAKEOPT}"
           echo "JID=${jid}" >> $GITHUB_ENV
           echo "arch=${arch}" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
           echo "MAKE=${MAKE}" >> $GITHUB_ENV
           echo "MAKEOPT=${MAKEOPT}" >> $GITHUB_ENV
           echo "PERL=${PERL}" >> $GITHUB_ENV
           echo "PYEXTERNALS_PATH=${PYEXTERNALS_PATH}" >> $GITHUB_ENV
           echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
           echo "openssl_archs=${openssl_archs}" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib" >> $GITHUB_ENV
-          #echo "PATH=${PATH}:${PYTHON_INSTALL_PATH}/scripts" >> $GITHUB_ENV
 
       - name: Get latest stable OpenSSL source
         if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
@@ -330,7 +340,7 @@ jobs:
           $env:OPENSSL_EXT_TARGET_PATH = "${env:OPENSSL_EXT_PATH}${env:PYEXTERNALS_PATH}"
           echo "Copying our OpenSSL to ${env:OPENSSL_EXT_TARGET_PATH}"
           mkdir "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
-          Copy-Item -Path "${env:GITHUB_WORKSPACE}/src/openssl-${env:openssl_archs}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE"
+          Copy-Item -Path "${env:GITHUB_WORKSPACE}/src/openssl-${env:openssl_archs}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE" -Verbose
           cp -v "$env:OPENSSL_INSTALL_PATH\lib\*" "${env:OPENSSL_EXT_TARGET_PATH}"
           cp -v "$env:OPENSSL_INSTALL_PATH\bin\*" "${env:OPENSSL_EXT_TARGET_PATH}"
           cp -v "$env:OPENSSL_INSTALL_PATH\include\openssl\*" "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
@@ -350,22 +360,10 @@ jobs:
         run: |
           cd "${env:PYTHON_SOURCE_PATH}"
           # We need out custom openssl.props which uses OpenSSL 3 DLL names
-          Copy-Item -Path "${env:GITHUB_WORKSPACE}\src\tools\openssl.props" -Destination PCBuild\
+          Copy-Item -Path "${env:GITHUB_WORKSPACE}\src\tools\openssl.props" -Destination PCBuild\ -Verbose
           echo "Building for ${env:PYBUILDRELEASE_ARCH}..."
           PCBuild\build.bat -m --pgo -c Release -p "${env:PYBUILDRELEASE_ARCH}"
 
-      - name: Windows Install Python
-        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
-        shell: powershell
-        run: |
-          cd "${env:PYTHON_SOURCE_PATH}"
-          mkdir "${env:PYTHON_INSTALL_PATH}\lib"
-          mkdir "${env:PYTHON_INSTALL_PATH}\include"
-          Copy-Item -Path "PCBuild\${env:PYEXTERNALS_PATH}\*" "${env:PYTHON_INSTALL_PATH}\"
-          Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Lib\*" "${env:PYTHON_INSTALL_PATH}\lib\" -recurse
-          Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Include\*" "${env:PYTHON_INSTALL_PATH}\include\" -recurse
-          Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\PC\*.h" "${env:PYTHON_INSTALL_PATH}\include\"
-
       - name: Mac/Linux Build Python
         if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
         run: |
@@ -379,6 +377,9 @@ jobs:
           cd "${PYTHON_SOURCE_PATH}"
           $MAKE altinstall
           $MAKE bininstall
+          export PATH="${PATH}:${PYTHON_INSTALL_PATH}/bin"
+          echo "PATH=${PATH}" >> $GITHUB_ENV
+          echo "PATH: ${PATH}"
 
       - name: Run Python
         run: |
@@ -391,7 +392,22 @@ jobs:
           "${PYTHON}" -m pip install --upgrade pip
           "${PYTHON}" -m pip install --upgrade wheel
           "${PYTHON}" -m pip install --upgrade setuptools
-    
+
+      - name: Install pip requirements
+        run: |
+             if [[ "${RUNNER_OS}" == "macOS" ]]; then
+               "${PYTHON}" -m pip install --upgrade cffi ${PIP_ARGS}
+               "${PYTHON}" -m pip download --only-binary :all: \
+                                             --dest . \
+                                             --no-cache \
+                                             --no-deps \
+                                             --platform macosx_10_15_universal2 \
+                                             cryptography
+               "${PYTHON}" -m pip install --force-reinstall --no-deps cryptography*.whl
+             fi
+             "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
+             "${PYTHON}" -m pip list
+
       - name: Install PyInstaller
         if: matrix.goal == 'build'
         run: |
@@ -407,27 +423,9 @@ jobs:
           fi
           echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
           "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
-          cd ../..
+          cd ..
           echo "---- Installing PyInstaller ----"
-          "${PYTHON}" -m pip install pyinstaller
-
-      - name: Install pip requirements
-        run: |
-             if [[ "${RUNNER_OS}" == "macOS" ]]; then
-               for package in cryptography; do
-                 "${PYTHON}" -m pip install --upgrade cffi ${PIP_ARGS}
-                 "${PYTHON}" -m pip download --only-binary :all: \
-                                             --dest . \
-                                             --no-cache \
-                                             --no-deps \
-                                             --platform macosx_10_15_universal2 \
-                                             $package
-               "${PYTHON}" -m pip install --force-reinstall --no-deps $package*.whl
-               done
-               find $PYTHON_INSTALL_PATH/lib/python3.10/site-packages -type f -name "*.so" -exec du -sh "{}" \;
-             fi
-             "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
-             "${PYTHON}" -m pip list
+          "${PYTHON}" -m pip install .
 
       - name: Build GAM with PyInstaller
         if: matrix.goal != 'test'
@@ -436,6 +434,10 @@ jobs:
              mkdir -p -v "${gampath}"
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                export gampath=$($PYTHON -c "import os; print(os.path.realpath('$gampath'))")
+             elif [[ "${RUNNER_OS}" == "Windows" ]]; then
+               # Work around issue where PyInstaller picks up python3.dll from other Python versions
+               # https://github.com/pyinstaller/pyinstaller/issues/7102
+               export PATH="/usr/bin"
              else
                export gampath=$(realpath "${gampath}")
              fi
@@ -445,6 +447,16 @@ jobs:
              echo -e "GAM: ${gam}\nGAMPATH: ${gampath}"
              "${PYTHON}" -m PyInstaller --clean --distpath="${gampath}" gam.spec
  
+      - name: Copy extra package files
+        if: matrix.goal == 'build'
+        run: |
+          cp -v roots.pem $gampath
+          cp -v LICENSE $gampath
+          cp -v GamCommands.txt $gampath
+          if [[ "${RUNNER_OS}" == "Windows" ]]; then
+              cp -v gam-setup.bat $gampath
+          fi
+
       - name: Basic Tests all jobs
         run: |
              $PYTHON -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
@@ -456,8 +468,6 @@ jobs:
       - name: Linux/MacOS package
         if: runner.os != 'Windows' && matrix.goal == 'build'
         run: |
-          cp -v LICENSE $gampath
-          cp -v GamCommands.txt $gampath
           if [[ "${RUNNER_OS}" == "macOS" ]]; then
               GAM_ARCHIVE="gam-${GAMVERSION}-macos-universal2.tar.xz"
           elif [[ "${RUNNER_OS}" == "Linux" ]]; then
@@ -501,9 +511,6 @@ jobs:
       - name: Windows package
         if: runner.os == 'Windows' && matrix.goal != 'test'
         run: |
-          cp -v LICENSE $gampath
-          cp -v GamCommands.txt $gampath
-          cp -v gam-setup.bat $gampath
           cd dist/
           GAM_ARCHIVE="../gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.zip"
           /c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam "-xr@${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" -bb3
@@ -540,11 +547,12 @@ jobs:
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                brew install gnupg
              fi
-             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.gpg creds.tar
+             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.xz.gpg creds.tar.xz
              export OAUTHFILE="oauth2.txt-gam-gha-${JID}"
              echo "OAUTHFILE=${OAUTHFILE}" >> $GITHUB_ENV
              export gam_user="gam-gha-${JID}@pdl.jaylee.us"
              echo "gam_user=${gam_user}" >> $GITHUB_ENV
+             touch "${gampath}/lowmemory.txt"
              $gam checkconn
              $gam oauth info
              $gam info domain
@@ -683,23 +691,55 @@ jobs:
              #echo "using delegated admin service account"
              #$gam print users
 
-     # - name: Upload to Google Drive, build only.
-     #   if: github.event_name == 'push' && matrix.goal != 'test'
-     #   run: |
-     #        ls gam-$GAMVERSION-*
-     #        for gamfile in gam-$GAMVERSION-*; do
-     #          echo "Uploading file ${gamfile} to Google Drive..."
-     #          fileid=$($gam user $gam_user add drivefile localfile $gamfile drivefilename $GAMVERSION-${GITHUB_SHA:0:7}-$gamfile parentid 1N2zbO33qzUQFsGM49-m9AQC1ijzd_ru1 returnidonly)
-     #          echo "file uploaded as ${fileid}, setting ACL..."
-     #          $gam user $gam_user add drivefileacl $fileid anyone role reader withlink
-     #        done
-
       - name: Archive production artifacts
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'push' && matrix.goal != 'test'
+        uses: actions/upload-artifact@v3
+        if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal != 'test'
         with:
           name: gam-binaries
           path: |
             src/*.tar.xz
             src/*.zip
             src/*.msi
+
+      - name: Tar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar cJvvf bin.tar.xz bin/
+
+  publish:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Set datetime version string
+        id: dateversion
+        run: |
+          echo "::set-output name=dateversion::$(date +'%Y%m%d.%k%M%S')"
+
+      - name: VirusTotal Scan
+        uses: crazy-max/ghaction-virustotal@v3
+        with:
+          vt_api_key: ${{ secrets.VT_API_KEY }}
+          files: |
+            gam-binaries/*
+
+      - uses: "marvinpinto/action-automatic-releases@latest"
+        name: Publish draft release
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: ${{ steps.dateversion.outputs.dateversion }}
+          prerelease: false
+          draft: true
+          title: "GAM ${{ steps.dateversion.outputs.dateversion }}"
+          files: |
+            gam-binaries/*

.github/workflows/get-roots.yml

@@ -0,0 +1,36 @@
+name: Check for Google Root CA Updates
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '23 23 * * *'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: src
+
+jobs:
+  check-apis:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+      - name: Check for updates
+        run: curl -o ./roots.pem -vvvv https://pki.goog/roots.pem
+
+      - name: Commit file
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git add roots.pem
+          git diff --quiet && git diff --staged --quiet || git commit -am '[ci skip] Updated roots.pem'
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }} 

src/gam.spec

@@ -5,9 +5,7 @@ import sys
 import importlib
 from PyInstaller.utils.hooks import copy_metadata
 
-# dynamically determine where httplib2/cacerts.txt lives
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-extra_files = [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+extra_files = []
 
 extra_files += copy_metadata('google-api-python-client')
 extra_files += [('cbcm-v1.1beta1.json', '.')]

src/gam.wxs

@@ -55,6 +55,9 @@
       <Component Id="gamcommands_txt" Guid="58ff9c45-a7c9-4e22-8845-a9a92610c1f3">
         <File Name="gamcommands.txt" KeyPath="yes" />
       </Component>
+      <Component Id="roots_pem" Guid="18ff9c45-a3c9-4e22-8445-a8a92610c1f3">
+        <File Name="roots.pem" KeyPath="yes" />
+      </Component>
     </ComponentGroup>
   </Fragment>
 

src/gam/__init__.py

@@ -553,9 +553,12 @@ def SetGlobalVariables():
                       'debug.gam',
                       filePresentValue=4,
                       fileAbsentValue=0)
+    _getOldSignalFile(GC_LOW_MEMORY, 'lowmemory.txt')
     _getOldSignalFile(GC_NO_BROWSER, 'nobrowser.txt')
     _getOldSignalFile(GC_NO_TDEMAIL, 'notdemail.txt')
-    _getOldSignalFile(GC_OAUTH_BROWSER, 'oauthbrowser.txt')
+    # oauthbrowser.txt is deprecated as we now always
+    # use the localhost flow.
+    #_getOldSignalFile(GC_OAUTH_BROWSER, 'oauthbrowser.txt')
     #  _getOldSignalFile(GC_NO_CACHE, u'nocache.txt')
     #  _getOldSignalFile(GC_CACHE_DISCOVERY_ONLY, u'allcache.txt', filePresentValue=False, fileAbsentValue=True)
     _getOldSignalFile(GC_NO_CACHE,
@@ -631,27 +634,28 @@ TIME_OFFSET_UNITS = [('day', 86400), ('hour', 3600), ('minute', 60),
 
 
 def getLocalGoogleTimeOffset(testLocation='admin.googleapis.com'):
-    localUTC = datetime.datetime.now(datetime.timezone.utc)
-    try:
-        # we disable SSL verify so we can still get time even if clock
-        # is way off. This could be spoofed / MitM but we'll fail for those
-        # situations everywhere else but here.
-        badhttp = transport.create_http()
-        badhttp.disable_ssl_certificate_validation = True
-        googleUTC = dateutil.parser.parse(
-            badhttp.request('https://' + testLocation, 'HEAD')[0]['date'])
-    except (httplib2.ServerNotFoundError, RuntimeError, ValueError) as e:
-        controlflow.system_error_exit(4, str(e))
-    offset = remainder = int(abs((localUTC - googleUTC).total_seconds()))
-    timeoff = []
-    for tou in TIME_OFFSET_UNITS:
-        uval, remainder = divmod(remainder, tou[1])
-        if uval:
-            timeoff.append(f'{uval} {tou[0]}{"s" if uval != 1 else ""}')
-    if not timeoff:
-        timeoff.append('less than 1 second')
-    nicetime = ', '.join(timeoff)
-    return (offset, nicetime)
+    # Try with http first, if time is close (<MAX_LOCAL_GOOGLE_TIME_OFFSET seconds),
+    # retry with https
+    badhttp = transport.create_http()
+    for prot in ['http', 'https']:
+        localUTC = datetime.datetime.now(datetime.timezone.utc)
+        try:
+            googleUTC = dateutil.parser.parse(
+                badhttp.request(f'{prot}://' + testLocation, 'HEAD')[0]['date'])
+        except (httplib2.ServerNotFoundError, RuntimeError, ValueError) as e:
+            controlflow.system_error_exit(4, str(e))
+        offset = remainder = int(abs((localUTC - googleUTC).total_seconds()))
+        if offset < MAX_LOCAL_GOOGLE_TIME_OFFSET and prot == 'http':
+            continue
+        timeoff = []
+        for tou in TIME_OFFSET_UNITS:
+            uval, remainder = divmod(remainder, tou[1])
+            if uval:
+                timeoff.append(f'{uval} {tou[0]}{"s" if uval != 1 else ""}')
+        if not timeoff:
+            timeoff.append('less than 1 second')
+        nicetime = ', '.join(timeoff)
+        return (offset, nicetime)
 
 
 def doGAMCheckForUpdates(forceCheck=False):
@@ -777,8 +781,9 @@ def checkConnection():
     success_count = 0
     for host in hosts:
         try_count += 1
-        check_line = f'Checking {host} ({try_count}/{host_count})...'
-        sys.stdout.write(f'{check_line:<60}')
+        ip = socket.gethostbyname(host)
+        check_line = f'Checking {host} ({ip}) ({try_count}/{host_count})...'
+        sys.stdout.write(f'{check_line:<80}')
         sys.stdout.flush()
         try:
             httpc.request(f'https://{host}/', 'HEAD', headers=headers)
@@ -7128,7 +7133,7 @@ def getCRMService(login_hint):
         scopes,
         'online',
         login_hint=login_hint,
-        use_console_flow=not GC_Values[GC_OAUTH_BROWSER])
+        open_browser=not GC_Values[GC_NO_BROWSER])
     httpc = transport.AuthorizedHttp(creds, transport.create_http())
     return getService('cloudresourcemanager', httpc), httpc
 
@@ -7284,7 +7289,7 @@ def _createClientSecretsOauth2service(httpObj, projectId, login_hint):
             'code':
                 'ThisIsAnInvalidCodeOnlyBeingUsedToTestIfClientAndSecretAreValid',
             'redirect_uri':
-                'urn:ietf:wg:oauth:2.0:oob',
+                'http://127.0.0.1:8080/',
             'grant_type':
                 'authorization_code'
         }
@@ -10542,7 +10547,7 @@ def doRequestOAuth(login_hint=None, scopes=None):
             access_type='offline',
             login_hint=login_hint,
             credentials_file=GC_Values[GC_OAUTH2_TXT],
-            use_console_flow=not GC_Values[GC_OAUTH_BROWSER])
+            open_browser=not GC_Values[GC_NO_BROWSER])
         creds.write()
     except gam.auth.oauth.InvalidClientSecretsFileError:
         controlflow.system_error_exit(14, missing_client_secrets_message)
@@ -10593,7 +10598,6 @@ OAUTH2_SCOPES = [
         'name': 'Cloud Identity - User Invitations',
         'subscopes': ['readonly'],
         'scopes': 'https://www.googleapis.com/auth/cloud-identity.userinvitations',
-        'offByDefault': True,
     },
     {
         'name': 'Contact Delegation',
@@ -11241,7 +11245,7 @@ def run_batch(items):
                 )
                 pool.close()
                 pool.join()
-                pool = mp_pool(num_worker_threads, init_gam_worker, maxtasksperchild=200, initargs=(1,))
+                pool = mp_pool(num_worker_threads, init_gam_worker, maxtasksperchild=200, initargs=(l,))
                 sys.stderr.write(
                     'commit-batch - running processes finished, proceeding\n')
                 continue

src/gam/auth/oauth.py

@@ -272,6 +272,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                             access_type='offline',
                             login_hint=None,
                             filename=None,
+                            open_browser=True,
                             use_console_flow=False):
         """Runs an OAuth Flow from client secrets to generate credentials.
 
@@ -291,8 +292,11 @@ class Credentials(google.oauth2.credentials.Credentials):
       login_hint: String, The email address that will be displayed on the Google
         login page as a hint for the user to login to the correct account.
       filename: String, the path to a file to use to save the credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
+        should be run strictly from a console; False to launch a browser
+        for authentication.
+      open_browser: Boolean: whether or not GAM should try to open the browser
+        automatically.
 
     Returns:
       Credentials
@@ -312,12 +316,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         flow = _ShortURLFlow.from_client_config(client_config,
                                                 scopes,
                                                 autogenerate_code_verifier=True)
-        flow_kwargs = {'access_type': access_type}
+        flow_kwargs = {'access_type': access_type,
+                       'open_browser': open_browser}
         if login_hint:
             flow_kwargs['login_hint'] = login_hint
-
-        flow.run_dual(use_console_flow,
-                **flow_kwargs)
+        flow.run_dual(**flow_kwargs)
         return cls.from_google_oauth2_credentials(flow.credentials,
                                                   filename=filename)
 
@@ -328,6 +331,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                                  access_type='offline',
                                  login_hint=None,
                                  credentials_file=None,
+                                 open_browser=True,
                                  use_console_flow=False):
         """Runs an OAuth Flow from secrets stored on disk to generate credentials.
 
@@ -348,8 +352,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         login page as a hint for the user to login to the correct account.
       credentials_file: String, the path to a file to use to save the
         credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
+        should be run strictly from a console; False to launch a browser for
+        authentication.
+      open_browser: Boolean, whether or not GAM should try to open the browser
+        directly.
 
     Raises:
       InvalidClientSecretsFileError: If the client secrets file cannot be
@@ -378,14 +385,13 @@ class Credentials(google.oauth2.credentials.Credentials):
             raise InvalidClientSecretsFileFormatError(
                 f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
             )
-
         return cls.from_client_secrets(client_id,
                                        client_secret,
                                        scopes,
                                        access_type=access_type,
                                        login_hint=login_hint,
                                        filename=credentials_file,
-                                       use_console_flow=use_console_flow)
+                                       open_browser=open_browser)
 
     def _fetch_id_token_data(self):
         """Fetches verification details from Google for the OAuth2.0 token.
@@ -482,7 +488,11 @@ class Credentials(google.oauth2.credentials.Credentials):
     def _locked_refresh(self, request):
         """Refreshes the credential's access token while the file lock is held."""
         assert self._lock.is_locked
-        super().refresh(request)
+        try:
+            super().refresh(request)
+        except google.auth.exceptions.RefreshError as e:
+            controlflow.system_error_exit(9, str(e))
+
 
     def write(self):
         """Writes credentials to disk."""
@@ -595,7 +605,6 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
 
 
     def run_dual(self,
-                use_console_flow,
                 authorization_prompt_message='',
                 console_prompt_message='',
                 web_success_message='',
@@ -605,7 +614,7 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
         mgr = multiprocessing.Manager()
         d = mgr.dict()
         d['trailing_slash'] = redirect_uri_trailing_slash
-        d['open_browser'] = use_console_flow
+        d['open_browser'] = open_browser
         http_client = multiprocessing.Process(target=_wait_for_http_client,
                                               args=(d,))
         user_input = multiprocessing.Process(target=_wait_for_user_input,
@@ -637,8 +646,8 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
                 code = parsed_params.get('code', [None])[0]
             try:
                 fetch_args = {'code': code}
-                if GC_Values.get('GC_CA_FILE'):
-                    fetch_args['verify'] = GC_Values.get('GC_CA_FILE')
+                if GC_Values.get(GC_CA_FILE):
+                    fetch_args['verify'] = GC_Values.get(GC_CA_FILE)
                 self.fetch_token(**fetch_args)
                 break
             except Exception as e:

src/gam/gapi/__init__.py

@@ -1,6 +1,9 @@
 """Methods related to execution of GAPI requests."""
 
+import os.path
+import shelve
 import sys
+from tempfile import TemporaryDirectory
 
 import googleapiclient.errors
 import google.auth.exceptions
@@ -10,7 +13,8 @@ from gam import controlflow
 from gam import display
 from gam.gapi import errors
 from gam import transport
-from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+from gam.var import (GC_Values, GC_LOW_MEMORY, GM_Globals,
+                     GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
                      GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
                      MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
                      MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
@@ -238,8 +242,13 @@ def process_page(page, items, all_items, total_items, page_message, message_attr
         page_items = page.get(items, [])
         num_page_items = len(page_items)
         total_items += num_page_items
-        if all_items is not None:
+        if type(all_items) is list:
             all_items.extend(page_items)
+        elif all_items is not None:
+            i = len(all_items)
+            for item in page_items:
+                all_items[str(i)] = item
+                i += 1
     else:
         page_token = None
         num_page_items = 0
@@ -273,6 +282,7 @@ def finalize_page_message(page_message):
         sys.stderr.write('\r\n')
         sys.stderr.flush()
 
+
 def get_all_pages(service,
                   function,
                   items='items',
@@ -328,7 +338,18 @@ def get_all_pages(service,
                 kwargs['body'].update(page_key)
             else:
                 kwargs.update(page_key)
-    all_items = []
+    if GC_Values[GC_LOW_MEMORY]:
+        td_args = {'prefix': 'GAM-'}
+        if sys.version_info.minor >= 10:
+            td_args['ignore_cleanup_errors'] = True
+        tempdir = TemporaryDirectory(**td_args)
+        tempfile = os.path.join(tempdir.name, 'gapi_pages')
+        all_items = shelve.open(tempfile)
+        # attach tempdir to all_items so we
+        # don't cleanup tempdir early
+        all_items._tempdir = tempdir
+    else:
+        all_items = []
     page_token = None
     total_items = 0
     while True:
@@ -341,13 +362,14 @@ def get_all_pages(service,
         page_token, total_items = process_page(page, items, all_items, total_items, page_message, message_attribute)
         if not page_token:
             finalize_page_message(page_message)
+            if type(all_items) is not list:
+                all_items = all_items.values()
             return all_items
         if page_args_in_body:
             kwargs['body']['pageToken'] = page_token
         else:
             kwargs['pageToken'] = page_token
 
-
 # TODO: Make this private once all execution related items that use this method
 # have been brought into this file
 def handle_oauth_token_error(e, soft_errors):

src/gam/gapi/cloudidentity/devices.py

@@ -80,7 +80,7 @@ def _parse_action(action):
 def info():
     ci = gapi_cloudidentity.build_dwd()
     customer = _get_device_customerid()
-    name = _get_device_name()
+    _, name = _get_deviceuser_name()
     device = gapi.call(ci.devices(), 'get', name=name, customer=customer)
     device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
         'deviceUsers', parent=name, customer=customer)

src/gam/gapi/cloudidentity/userinvitations.py

@@ -25,7 +25,7 @@ def _reduce_name(name):
 
 def is_invitable_user(email):
     '''return email isInvitableUser'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     encoded_email = quote_plus(email)
     name = f'{customer}/userinvitations/{encoded_email}'
@@ -35,7 +35,7 @@ def is_invitable_user(email):
 
 def _generic_action(action):
     '''generic function to call actionable APIs'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     email = sys.argv[3].lower()
     encoded_email = quote_plus(email)
@@ -55,7 +55,7 @@ def _generic_action(action):
 
 def _generic_get(get_type):
     '''generic function to call read data APIs'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     email = sys.argv[3].lower()
     encoded_email = quote_plus(email)
@@ -75,7 +75,7 @@ def bulk_is_invitable(emails):
         if response.get('isInvitableUser'):
             rows.append({'invitableUsers': request_id})
 
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     todrive = False
     #batch_size = 1000
@@ -139,7 +139,7 @@ USERINVITATION_STATE_CHOICES_MAP = {
 
 def print_():
     '''gam print userinvitations'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     todrive = False
     titles = ['name', 'state', 'updateTime']

src/gam/gapi/cloudresourcemanager.py

@@ -13,12 +13,14 @@ def get_org_id():
     gapi_directory_customer.setTrueCustomerId()
     crm = build()
     query = f'directorycustomerid:{GC_Values[GC_CUSTOMER_ID]}'
-    orgs = gapi.get_all_pages(crm.organizations(),
+    results = gapi.call(crm.organizations(),
                      'search',
-                     'organizations',
+                     pageSize=1,
+                     fields='organizations/name',
                      query=query)
-    if len(orgs) < 1:
+    orgs = results.get('organizations')
+    if not orgs:
         # return nothing and let calling API deal with it
         # since caller knows what GCP role would serve best
         return 
-    return orgs[0]['name']
+    return orgs[0].get('name')

src/gam/gapi/directory/customer.py

@@ -21,36 +21,36 @@ def doGetCustomerInfo():
                               'get',
                               customerKey=customer_id)
     print(f'Customer ID: {customer_info["id"]}')
-    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    fields = 'domains(creationTime,domainName,isPrimary,verified)'
     try:
-        result = gapi.call(
+        domains = gapi.call(
             cd.domains(),
-            'get',
+            'list',
+            fields=fields,
             customer=customer_id,
-            domainName=customer_info['customerDomain'],
-            fields='verified',
-            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND])
+            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND]).get('domains', [])
+        for domain in domains:
+            if domain.get('isPrimary'):
+                primary_domain = domain
+                break
+        else:
+            primary_domain = {}
     except gapi.errors.GapiDomainNotFoundError:
-        result = {'verified': False}
-    print(f'Primary Domain Verified: {result["verified"]}')
-    # If customer has changed primary domain customerCreationTime is date
-    # of current primary being added, not customer create date.
-    # We should also get all domains and use oldest date
-    customer_creation = customer_info['customerCreationTime']
-    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
-    oldest = datetime.datetime.strptime(customer_creation, date_format)
-    domains = gapi.get_items(cd.domains(),
-                             'list',
-                             'domains',
-                             customer=customer_id,
-                             fields='domains(creationTime)')
+        primary_domain = {}
+    print(f'Primary Domain: {primary_domain.get("domainName", "Unknown")}')
+    print(f'Primary Domain Verified: {primary_domain.get("verified", "Unknown")}')
+    # we'll assume creation time is time of oldest domain customer has
+    oldest = 'Unknown' 
     for domain in domains:
         creation_timestamp = int(domain['creationTime']) / 1000
         domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
-        if domain_creation < oldest:
+        if oldest == 'Unknown' or domain_creation < oldest:
             oldest = domain_creation
-    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
-    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    if oldest != 'Unknown':
+        date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+        oldest = oldest.strftime(date_format)
+    print(f'Customer Creation Time: {oldest}')
+    customer_language = customer_info.get('language', 'Unset or Unknown (defaults to en)')
     print(f'Default Language: {customer_language}')
     if 'postalAddress' in customer_info:
         print('Address:')
@@ -59,7 +59,7 @@ def doGetCustomerInfo():
                 print(f' {field}: {customer_info["postalAddress"][field]}')
     if 'phoneNumber' in customer_info:
         print(f'Phone: {customer_info["phoneNumber"]}')
-    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    print(f'Admin Secondary Email: {customer_info.get("alternateEmail", "Unknown")}')
     user_counts_map = {
         'accounts:num_users': 'Total Users',
         'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',

src/gam/gapi/reports.py

@@ -426,6 +426,7 @@ def showReport():
         titles = ['name', 'value', 'client_id']
         csvRows = []
         auth_apps = list()
+        usage = list(usage)
         for item in usage[0]['parameters']:
             if 'name' not in item:
                 continue

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.23'
+GAM_VERSION = '6.30'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'
@@ -206,7 +206,7 @@ SKUS = {
     },
     '1010020030': {
         'product': 'Google-Apps',
-        'aliases': ['workspacefrontline', 'workspacefrontlineworker'],
+        'aliases': ['wsflw', 'workspacefrontline', 'workspacefrontlineworker'],
         'displayName': 'Workspace Frontline'
     },
     '1010340002': {
@@ -648,6 +648,7 @@ MACOS_CODENAMES = {
         },
     11: 'Big Sur',
     12: 'Monterey',
+    13: 'Ventura',
     }
 
 _MICROSOFT_FORMATS_LIST = [{
@@ -1202,6 +1203,7 @@ _DEFAULT_CHARSET = UTF8
 _FN_CLIENT_SECRETS_JSON = 'client_secrets.json'
 _FN_OAUTH2SERVICE_JSON = 'oauth2service.json'
 _FN_OAUTH2_TXT = 'oauth2.txt'
+_FN_ROOTS_PEM = 'roots.pem'
 #
 GM_Globals = {
     GM_SYSEXITRC: 0,
@@ -1269,6 +1271,9 @@ GC_ENABLE_DASA = 'enabledasa'
 # and doRequestOAuth prints a link and waits for the verification code when
 # oauth2.txt is being created
 GC_NO_BROWSER = 'no_browser'
+# If low memory is True, GAM tries to save RAM by writing pages to disk
+# temporarily
+GC_LOW_MEMORY = 'low_memory'
 # If no_tdemail is True, writeCSVfile won't send an email
 GC_NO_TDEMAIL = 'no_tdemail'
 # oauth_browser forces usage of web server OAuth flow that proved problematic.
@@ -1324,6 +1329,7 @@ GC_Defaults = {
     GC_DOMAIN: '',
     GC_DRIVE_DIR: '',
     GC_ENABLE_DASA: False,
+    GC_LOW_MEMORY: False,
     GC_NO_BROWSER: False,
     GC_NO_TDEMAIL: False,
     GC_NO_CACHE: False,
@@ -1343,7 +1349,7 @@ GC_Defaults = {
     GC_CSV_ROW_DROP_FILTER: '',
     GC_TLS_MIN_VERSION: TLS_MIN,
     GC_TLS_MAX_VERSION: None,
-    GC_CA_FILE: None,
+    GC_CA_FILE: _FN_ROOTS_PEM,
 }
 
 GC_Values = {}
@@ -1408,6 +1414,9 @@ GC_VAR_INFO = {
     GC_ENABLE_DASA: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },
+    GC_LOW_MEMORY: {
+        GC_VAR_TYPE: GC_TYPE_BOOLEAN
+    },
     GC_NO_BROWSER: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },

src/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = GAM for Google Workspace
-version = 6.0.23
+version = attr: gam.var.GAM_VERSION
 description = Command line management for Google Workspaces
 long_description = file: readme.md
 long_description_content_type = text/markdown
@@ -37,6 +37,9 @@ install_requires =
     yubikey-manager >= 4.0.0
     pathvalidate
 
+[options.package_data]
+* = *.pem
+
 # used during pip install .[test]
 [options.extras_require]
 test = pre-commit