Update build.yml

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
             goal: build
             arch: x86_64
             openssl_archs: linux-x86_64
-          - os: [self-hosted, linux, arm64]
+          - os: [self-hosted, linux, arm64, gcp]
             jid: 2
             goal: build
             arch: aarch64
@@ -71,22 +71,30 @@ jobs:
             arch: x86_64
 
     steps:
-      - uses: actions/checkout@master
+
+      - uses: actions/checkout@v3
         with:
           persist-credentials: false
           fetch-depth: 0
 
       - name: Cache multiple paths
-        uses: actions/cache@v2
+        if: matrix.goal == 'build'
+        uses: actions/cache@v3
         id: cache-python-ssl
         with:
           path: |
-            bin
+            bin.tar.xz
-          key: gam-${{ matrix.jid }}-20220802
+          key: gam-${{ matrix.jid }}-20221012
+
+      - name: Untar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar xvvf bin.tar.xz
 
       - name: Use pre-compiled Python for testing
         if: matrix.python != ''
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 
@@ -117,11 +125,11 @@ jobs:
           sudo apt-get -qq --yes update
           sudo apt-get -qq --yes install swig libpcsclite-dev
 
-      - name: MacOS remove Homebrew
+     #- name: MacOS remove Homebrew
-        if: runner.os == 'macOS'
+     #  if: runner.os == 'macOS'
-        run: |
+     #  run: |
-          # remove everything except the libraries needed by yubikey-manager
+     #    # remove everything except the libraries needed by yubikey-manager
-          brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
+     #    brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
 
       - name: MacOS install tools
         if: runner.os == 'macOS'
@@ -192,7 +200,6 @@ jobs:
           echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
           echo "openssl_archs=${openssl_archs}" >> $GITHUB_ENV
           echo "LD_LIBRARY_PATH=${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib" >> $GITHUB_ENV
-          #echo "PATH=${PATH}:${PYTHON_INSTALL_PATH}/scripts" >> $GITHUB_ENV
 
       - name: Get latest stable OpenSSL source
         if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
@@ -365,6 +372,9 @@ jobs:
           Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Lib\*" "${env:PYTHON_INSTALL_PATH}\lib\" -recurse
           Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Include\*" "${env:PYTHON_INSTALL_PATH}\include\" -recurse
           Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\PC\*.h" "${env:PYTHON_INSTALL_PATH}\include\"
+          $env:Path = '${PYTHON_INSTALL_PATH}/bin' + $env:Path
+          echo "Path=$env:Path" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "PATH: ${env:Path}"
 
       - name: Mac/Linux Build Python
         if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
@@ -379,6 +389,9 @@ jobs:
           cd "${PYTHON_SOURCE_PATH}"
           $MAKE altinstall
           $MAKE bininstall
+          export PATH="${PATH}:${PYTHON_INSTALL_PATH}/bin"
+          echo "PATH=${PATH}" >> $GITHUB_ENV
+          echo "PATH: ${PATH}"
 
       - name: Run Python
         run: |
@@ -392,25 +405,6 @@ jobs:
           "${PYTHON}" -m pip install --upgrade wheel
           "${PYTHON}" -m pip install --upgrade setuptools
 
-      - name: Install PyInstaller
-        if: matrix.goal == 'build'
-        run: |
-          git clone https://github.com/pyinstaller/pyinstaller.git
-          cd pyinstaller
-          export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
-          git checkout "${latest_release}"
-          # remove pre-compiled bootloaders so we fail if bootloader compile fails
-          rm -rvf PyInstaller/bootloader/*-*/*
-          cd bootloader
-          if [[ "${arch}" == "Win32" ]]; then
-            export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
-          fi
-          echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
-          "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
-          cd ../..
-          echo "---- Installing PyInstaller ----"
-          "${PYTHON}" -m pip install pyinstaller
-
       - name: Install pip requirements
         run: |
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
@@ -429,6 +423,27 @@ jobs:
              "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
              "${PYTHON}" -m pip list
 
+      - name: Install PyInstaller
+        if: matrix.goal == 'build'
+        run: |
+          git clone https://github.com/pyinstaller/pyinstaller.git
+          cd pyinstaller
+          export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
+          # temp freeze PyInstaller at 5.3
+          export latest_release="v5.3"
+          git checkout "${latest_release}"
+          # remove pre-compiled bootloaders so we fail if bootloader compile fails
+          rm -rvf PyInstaller/bootloader/*-*/*
+          cd bootloader
+          if [[ "${arch}" == "Win32" ]]; then
+            export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
+          fi
+          echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
+          "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
+          cd ..
+          echo "---- Installing PyInstaller ----"
+          "${PYTHON}" -m pip install .
+
       - name: Build GAM with PyInstaller
         if: matrix.goal != 'test'
         run: |
@@ -445,6 +460,16 @@ jobs:
              echo -e "GAM: ${gam}\nGAMPATH: ${gampath}"
              "${PYTHON}" -m PyInstaller --clean --distpath="${gampath}" gam.spec
  
+      - name: Copy extra package files
+        if: matrix.goal == 'build'
+        run: |
+          cp -v roots.pem $gampath
+          cp -v LICENSE $gampath
+          cp -v GamCommands.txt $gampath
+          if [[ "${RUNNER_OS}" == "Windows" ]]; then
+              cp -v gam-setup.bat $gampath
+          fi
+
       - name: Basic Tests all jobs
         run: |
              $PYTHON -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
@@ -456,8 +481,6 @@ jobs:
       - name: Linux/MacOS package
         if: runner.os != 'Windows' && matrix.goal == 'build'
         run: |
-          cp -v LICENSE $gampath
-          cp -v GamCommands.txt $gampath
           if [[ "${RUNNER_OS}" == "macOS" ]]; then
               GAM_ARCHIVE="gam-${GAMVERSION}-macos-universal2.tar.xz"
           elif [[ "${RUNNER_OS}" == "Linux" ]]; then
@@ -501,9 +524,6 @@ jobs:
       - name: Windows package
         if: runner.os == 'Windows' && matrix.goal != 'test'
         run: |
-          cp -v LICENSE $gampath
-          cp -v GamCommands.txt $gampath
-          cp -v gam-setup.bat $gampath
           cd dist/
           GAM_ARCHIVE="../gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.zip"
           /c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam "-xr@${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" -bb3
@@ -540,11 +560,12 @@ jobs:
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                brew install gnupg
              fi
-             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.gpg creds.tar
+             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.xz.gpg creds.tar.xz
              export OAUTHFILE="oauth2.txt-gam-gha-${JID}"
              echo "OAUTHFILE=${OAUTHFILE}" >> $GITHUB_ENV
              export gam_user="gam-gha-${JID}@pdl.jaylee.us"
              echo "gam_user=${gam_user}" >> $GITHUB_ENV
+             touch "${gampath}/lowmemory.txt"
              $gam checkconn
              $gam oauth info
              $gam info domain
@@ -695,11 +716,54 @@ jobs:
      #        done
 
       - name: Archive production artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
-        if: github.event_name == 'push' && matrix.goal != 'test'
+        if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal != 'test'
         with:
           name: gam-binaries
           path: |
             src/*.tar.xz
             src/*.zip
             src/*.msi
+
+      - name: Tar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar cJvvf bin.tar.xz bin/
+
+  publish:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Set datetime version string
+        id: dateversion
+        run: |
+          echo "::set-output name=dateversion::$(date +'%Y%m%d.%k%M%S')"
+
+      - name: VirusTotal Scan
+        uses: crazy-max/ghaction-virustotal@v3
+        with:
+          vt_api_key: ${{ secrets.VT_API_KEY }}
+          files: |
+            gam-binaries/*
+
+      - uses: "marvinpinto/action-automatic-releases@latest"
+        name: Publish draft release
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: ${{ steps.dateversion.outputs.dateversion }}
+          prerelease: false
+          draft: true
+          title: "GAM ${{ steps.dateversion.outputs.dateversion }}"
+          files: |
+            gam-binaries/*

.github/workflows/get-roots.yml

@@ -0,0 +1,36 @@
+name: Check for Google Root CA Updates
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '23 23 * * *'
+
+defaults:
+  run:
+    shell: bash
+    working-directory: src
+
+jobs:
+  check-apis:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+      - name: Check for updates
+        run: curl -o ./roots.pem -vvvv https://pki.goog/roots.pem
+
+      - name: Commit file
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git add roots.pem
+          git diff --quiet && git diff --staged --quiet || git commit -am '[ci skip] Updated roots.pem'
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }} 

src/gam.spec

@@ -5,9 +5,7 @@ import sys
 import importlib
 from PyInstaller.utils.hooks import copy_metadata
 
-# dynamically determine where httplib2/cacerts.txt lives
+extra_files = []
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-extra_files = [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
 
 extra_files += copy_metadata('google-api-python-client')
 extra_files += [('cbcm-v1.1beta1.json', '.')]

src/gam.wxs

@@ -55,6 +55,9 @@
       <Component Id="gamcommands_txt" Guid="58ff9c45-a7c9-4e22-8845-a9a92610c1f3">
         <File Name="gamcommands.txt" KeyPath="yes" />
       </Component>
+      <Component Id="roots_pem" Guid="18ff9c45-a3c9-4e22-8445-a8a92610c1f3">
+        <File Name="roots.pem" KeyPath="yes" />
+      </Component>
     </ComponentGroup>
   </Fragment>
 

src/gam/__init__.py

@@ -553,9 +553,12 @@ def SetGlobalVariables():
                       'debug.gam',
                       filePresentValue=4,
                       fileAbsentValue=0)
+    _getOldSignalFile(GC_LOW_MEMORY, 'lowmemory.txt')
     _getOldSignalFile(GC_NO_BROWSER, 'nobrowser.txt')
     _getOldSignalFile(GC_NO_TDEMAIL, 'notdemail.txt')
-    _getOldSignalFile(GC_OAUTH_BROWSER, 'oauthbrowser.txt')
+    # oauthbrowser.txt is deprecated as we now always
+    # use the localhost flow.
+    #_getOldSignalFile(GC_OAUTH_BROWSER, 'oauthbrowser.txt')
     #  _getOldSignalFile(GC_NO_CACHE, u'nocache.txt')
     #  _getOldSignalFile(GC_CACHE_DISCOVERY_ONLY, u'allcache.txt', filePresentValue=False, fileAbsentValue=True)
     _getOldSignalFile(GC_NO_CACHE,
@@ -777,8 +780,9 @@ def checkConnection():
     success_count = 0
     for host in hosts:
         try_count += 1
-        check_line = f'Checking {host} ({try_count}/{host_count})...'
+        ip = socket.gethostbyname(host)
-        sys.stdout.write(f'{check_line:<60}')
+        check_line = f'Checking {host} ({ip}) ({try_count}/{host_count})...'
+        sys.stdout.write(f'{check_line:<80}')
         sys.stdout.flush()
         try:
             httpc.request(f'https://{host}/', 'HEAD', headers=headers)
@@ -7128,7 +7132,7 @@ def getCRMService(login_hint):
         scopes,
         'online',
         login_hint=login_hint,
-        use_console_flow=not GC_Values[GC_OAUTH_BROWSER])
+        open_browser=not GC_Values[GC_NO_BROWSER])
     httpc = transport.AuthorizedHttp(creds, transport.create_http())
     return getService('cloudresourcemanager', httpc), httpc
 
@@ -7284,7 +7288,7 @@ def _createClientSecretsOauth2service(httpObj, projectId, login_hint):
             'code':
                 'ThisIsAnInvalidCodeOnlyBeingUsedToTestIfClientAndSecretAreValid',
             'redirect_uri':
-                'urn:ietf:wg:oauth:2.0:oob',
+                'http://127.0.0.1:8080/',
             'grant_type':
                 'authorization_code'
         }
@@ -10542,7 +10546,7 @@ def doRequestOAuth(login_hint=None, scopes=None):
             access_type='offline',
             login_hint=login_hint,
             credentials_file=GC_Values[GC_OAUTH2_TXT],
-            use_console_flow=not GC_Values[GC_OAUTH_BROWSER])
+            open_browser=not GC_Values[GC_NO_BROWSER])
         creds.write()
     except gam.auth.oauth.InvalidClientSecretsFileError:
         controlflow.system_error_exit(14, missing_client_secrets_message)
@@ -10593,7 +10597,6 @@ OAUTH2_SCOPES = [
         'name': 'Cloud Identity - User Invitations',
         'subscopes': ['readonly'],
         'scopes': 'https://www.googleapis.com/auth/cloud-identity.userinvitations',
-        'offByDefault': True,
     },
     {
         'name': 'Contact Delegation',

src/gam/auth/oauth.py

@@ -272,6 +272,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                             access_type='offline',
                             login_hint=None,
                             filename=None,
+                            open_browser=True,
                             use_console_flow=False):
         """Runs an OAuth Flow from client secrets to generate credentials.
 
@@ -291,8 +292,11 @@ class Credentials(google.oauth2.credentials.Credentials):
       login_hint: String, The email address that will be displayed on the Google
         login page as a hint for the user to login to the correct account.
       filename: String, the path to a file to use to save the credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
-        strictly from a console; False to launch a browser for authentication.
+        should be run strictly from a console; False to launch a browser
+        for authentication.
+      open_browser: Boolean: whether or not GAM should try to open the browser
+        automatically.
 
     Returns:
       Credentials
@@ -312,12 +316,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         flow = _ShortURLFlow.from_client_config(client_config,
                                                 scopes,
                                                 autogenerate_code_verifier=True)
-        flow_kwargs = {'access_type': access_type}
+        flow_kwargs = {'access_type': access_type,
+                       'open_browser': open_browser}
         if login_hint:
             flow_kwargs['login_hint'] = login_hint
-
+        flow.run_dual(**flow_kwargs)
-        flow.run_dual(use_console_flow,
-                **flow_kwargs)
         return cls.from_google_oauth2_credentials(flow.credentials,
                                                   filename=filename)
 
@@ -328,6 +331,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                                  access_type='offline',
                                  login_hint=None,
                                  credentials_file=None,
+                                 open_browser=True,
                                  use_console_flow=False):
         """Runs an OAuth Flow from secrets stored on disk to generate credentials.
 
@@ -348,8 +352,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         login page as a hint for the user to login to the correct account.
       credentials_file: String, the path to a file to use to save the
         credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
-        strictly from a console; False to launch a browser for authentication.
+        should be run strictly from a console; False to launch a browser for
+        authentication.
+      open_browser: Boolean, whether or not GAM should try to open the browser
+        directly.
 
     Raises:
       InvalidClientSecretsFileError: If the client secrets file cannot be
@@ -378,14 +385,13 @@ class Credentials(google.oauth2.credentials.Credentials):
             raise InvalidClientSecretsFileFormatError(
                 f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
             )
-
         return cls.from_client_secrets(client_id,
                                        client_secret,
                                        scopes,
                                        access_type=access_type,
                                        login_hint=login_hint,
                                        filename=credentials_file,
-                                       use_console_flow=use_console_flow)
+                                       open_browser=open_browser)
 
     def _fetch_id_token_data(self):
         """Fetches verification details from Google for the OAuth2.0 token.
@@ -482,7 +488,11 @@ class Credentials(google.oauth2.credentials.Credentials):
     def _locked_refresh(self, request):
         """Refreshes the credential's access token while the file lock is held."""
         assert self._lock.is_locked
-        super().refresh(request)
+        try:
+            super().refresh(request)
+        except google.auth.exceptions.RefreshError as e:
+            controlflow.system_error_exit(9, str(e))
+
 
     def write(self):
         """Writes credentials to disk."""
@@ -595,7 +605,6 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
 
 
     def run_dual(self,
-                use_console_flow,
                 authorization_prompt_message='',
                 console_prompt_message='',
                 web_success_message='',
@@ -605,7 +614,7 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
         mgr = multiprocessing.Manager()
         d = mgr.dict()
         d['trailing_slash'] = redirect_uri_trailing_slash
-        d['open_browser'] = use_console_flow
+        d['open_browser'] = open_browser
         http_client = multiprocessing.Process(target=_wait_for_http_client,
                                               args=(d,))
         user_input = multiprocessing.Process(target=_wait_for_user_input,
@@ -637,8 +646,8 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
                 code = parsed_params.get('code', [None])[0]
             try:
                 fetch_args = {'code': code}
-                if GC_Values.get('GC_CA_FILE'):
+                if GC_Values.get(GC_CA_FILE):
-                    fetch_args['verify'] = GC_Values.get('GC_CA_FILE')
+                    fetch_args['verify'] = GC_Values.get(GC_CA_FILE)
                 self.fetch_token(**fetch_args)
                 break
             except Exception as e:

src/gam/gapi/__init__.py

@@ -1,6 +1,9 @@
 """Methods related to execution of GAPI requests."""
 
+import os.path
+import shelve
 import sys
+from tempfile import TemporaryDirectory
 
 import googleapiclient.errors
 import google.auth.exceptions
@@ -10,7 +13,8 @@ from gam import controlflow
 from gam import display
 from gam.gapi import errors
 from gam import transport
-from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+from gam.var import (GC_Values, GC_LOW_MEMORY, GM_Globals,
+                     GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
                      GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
                      MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
                      MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
@@ -238,8 +242,13 @@ def process_page(page, items, all_items, total_items, page_message, message_attr
         page_items = page.get(items, [])
         num_page_items = len(page_items)
         total_items += num_page_items
-        if all_items is not None:
+        if type(all_items) is list:
             all_items.extend(page_items)
+        elif all_items is not None:
+            i = len(all_items)
+            for item in page_items:
+                all_items[str(i)] = item
+                i += 1
     else:
         page_token = None
         num_page_items = 0
@@ -273,6 +282,7 @@ def finalize_page_message(page_message):
         sys.stderr.write('\r\n')
         sys.stderr.flush()
 
+
 def get_all_pages(service,
                   function,
                   items='items',
@@ -328,7 +338,18 @@ def get_all_pages(service,
                 kwargs['body'].update(page_key)
             else:
                 kwargs.update(page_key)
-    all_items = []
+    if GC_Values[GC_LOW_MEMORY]:
+        td_args = {'prefix': 'GAM-'}
+        if sys.version_info.minor >= 10:
+            td_args['ignore_cleanup_errors'] = True
+        tempdir = TemporaryDirectory(**td_args)
+        tempfile = os.path.join(tempdir.name, 'gapi_pages')
+        all_items = shelve.open(tempfile)
+        # attach tempdir to all_items so we
+        # don't cleanup tempdir early
+        all_items._tempdir = tempdir
+    else:
+        all_items = []
     page_token = None
     total_items = 0
     while True:
@@ -341,13 +362,14 @@ def get_all_pages(service,
         page_token, total_items = process_page(page, items, all_items, total_items, page_message, message_attribute)
         if not page_token:
             finalize_page_message(page_message)
+            if type(all_items) is not list:
+                all_items = all_items.values()
             return all_items
         if page_args_in_body:
             kwargs['body']['pageToken'] = page_token
         else:
             kwargs['pageToken'] = page_token
 
-
 # TODO: Make this private once all execution related items that use this method
 # have been brought into this file
 def handle_oauth_token_error(e, soft_errors):

src/gam/gapi/cloudidentity/devices.py

@@ -80,7 +80,7 @@ def _parse_action(action):
 def info():
     ci = gapi_cloudidentity.build_dwd()
     customer = _get_device_customerid()
-    name = _get_device_name()
+    _, name = _get_deviceuser_name()
     device = gapi.call(ci.devices(), 'get', name=name, customer=customer)
     device_users = gapi.get_all_pages(ci.devices().deviceUsers(), 'list',
         'deviceUsers', parent=name, customer=customer)

src/gam/gapi/cloudidentity/userinvitations.py

@@ -25,7 +25,7 @@ def _reduce_name(name):
 
 def is_invitable_user(email):
     '''return email isInvitableUser'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     encoded_email = quote_plus(email)
     name = f'{customer}/userinvitations/{encoded_email}'
@@ -35,7 +35,7 @@ def is_invitable_user(email):
 
 def _generic_action(action):
     '''generic function to call actionable APIs'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     email = sys.argv[3].lower()
     encoded_email = quote_plus(email)
@@ -55,7 +55,7 @@ def _generic_action(action):
 
 def _generic_get(get_type):
     '''generic function to call read data APIs'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     email = sys.argv[3].lower()
     encoded_email = quote_plus(email)
@@ -75,7 +75,7 @@ def bulk_is_invitable(emails):
         if response.get('isInvitableUser'):
             rows.append({'invitableUsers': request_id})
 
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     todrive = False
     #batch_size = 1000
@@ -139,7 +139,7 @@ USERINVITATION_STATE_CHOICES_MAP = {
 
 def print_():
     '''gam print userinvitations'''
-    svc = gapi_cloudidentity.build_dwd('cloudidentity')
+    svc = gapi_cloudidentity.build('cloudidentity')
     customer = _get_customerid()
     todrive = False
     titles = ['name', 'state', 'updateTime']

src/gam/gapi/cloudresourcemanager.py

@@ -13,12 +13,14 @@ def get_org_id():
     gapi_directory_customer.setTrueCustomerId()
     crm = build()
     query = f'directorycustomerid:{GC_Values[GC_CUSTOMER_ID]}'
-    orgs = gapi.get_all_pages(crm.organizations(),
+    results = gapi.call(crm.organizations(),
                      'search',
-                     'organizations',
+                     pageSize=1,
+                     fields='organizations/name',
                      query=query)
-    if len(orgs) < 1:
+    orgs = results.get('organizations')
+    if not orgs:
         # return nothing and let calling API deal with it
         # since caller knows what GCP role would serve best
         return 
-    return orgs[0]['name']
+    return orgs[0].get('name')

src/gam/gapi/directory/customer.py

@@ -21,36 +21,36 @@ def doGetCustomerInfo():
                               'get',
                               customerKey=customer_id)
     print(f'Customer ID: {customer_info["id"]}')
-    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    fields = 'domains(creationTime,domainName,isPrimary,verified)'
     try:
-        result = gapi.call(
+        domains = gapi.call(
             cd.domains(),
-            'get',
+            'list',
+            fields=fields,
             customer=customer_id,
-            domainName=customer_info['customerDomain'],
+            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND]).get('domains', [])
-            fields='verified',
+        for domain in domains:
-            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND])
+            if domain.get('isPrimary'):
+                primary_domain = domain
+                break
+        else:
+            primary_domain = {}
     except gapi.errors.GapiDomainNotFoundError:
-        result = {'verified': False}
+        primary_domain = {}
-    print(f'Primary Domain Verified: {result["verified"]}')
+    print(f'Primary Domain: {primary_domain.get("domainName", "Unknown")}')
-    # If customer has changed primary domain customerCreationTime is date
+    print(f'Primary Domain Verified: {primary_domain.get("verified", "Unknown")}')
-    # of current primary being added, not customer create date.
+    # we'll assume creation time is time of oldest domain customer has
-    # We should also get all domains and use oldest date
+    oldest = 'Unknown' 
-    customer_creation = customer_info['customerCreationTime']
-    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
-    oldest = datetime.datetime.strptime(customer_creation, date_format)
-    domains = gapi.get_items(cd.domains(),
-                             'list',
-                             'domains',
-                             customer=customer_id,
-                             fields='domains(creationTime)')
     for domain in domains:
         creation_timestamp = int(domain['creationTime']) / 1000
         domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
-        if domain_creation < oldest:
+        if oldest == 'Unknown' or domain_creation < oldest:
             oldest = domain_creation
-    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
+    if oldest != 'Unknown':
-    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+        date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+        oldest = oldest.strftime(date_format)
+    print(f'Customer Creation Time: {oldest}')
+    customer_language = customer_info.get('language', 'Unset or Unknown (defaults to en)')
     print(f'Default Language: {customer_language}')
     if 'postalAddress' in customer_info:
         print('Address:')
@@ -59,7 +59,7 @@ def doGetCustomerInfo():
                 print(f' {field}: {customer_info["postalAddress"][field]}')
     if 'phoneNumber' in customer_info:
         print(f'Phone: {customer_info["phoneNumber"]}')
-    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    print(f'Admin Secondary Email: {customer_info.get("alternateEmail", "Unknown")}')
     user_counts_map = {
         'accounts:num_users': 'Total Users',
         'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',

src/gam/gapi/reports.py

@@ -426,6 +426,7 @@ def showReport():
         titles = ['name', 'value', 'client_id']
         csvRows = []
         auth_apps = list()
+        usage = list(usage)
         for item in usage[0]['parameters']:
             if 'name' not in item:
                 continue

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.23'
+GAM_VERSION = '6.26'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'
@@ -206,7 +206,7 @@ SKUS = {
     },
     '1010020030': {
         'product': 'Google-Apps',
-        'aliases': ['workspacefrontline', 'workspacefrontlineworker'],
+        'aliases': ['wsflw', 'workspacefrontline', 'workspacefrontlineworker'],
         'displayName': 'Workspace Frontline'
     },
     '1010340002': {
@@ -1202,6 +1202,7 @@ _DEFAULT_CHARSET = UTF8
 _FN_CLIENT_SECRETS_JSON = 'client_secrets.json'
 _FN_OAUTH2SERVICE_JSON = 'oauth2service.json'
 _FN_OAUTH2_TXT = 'oauth2.txt'
+_FN_ROOTS_PEM = 'roots.pem'
 #
 GM_Globals = {
     GM_SYSEXITRC: 0,
@@ -1269,6 +1270,9 @@ GC_ENABLE_DASA = 'enabledasa'
 # and doRequestOAuth prints a link and waits for the verification code when
 # oauth2.txt is being created
 GC_NO_BROWSER = 'no_browser'
+# If low memory is True, GAM tries to save RAM by writing pages to disk
+# temporarily
+GC_LOW_MEMORY = 'low_memory'
 # If no_tdemail is True, writeCSVfile won't send an email
 GC_NO_TDEMAIL = 'no_tdemail'
 # oauth_browser forces usage of web server OAuth flow that proved problematic.
@@ -1324,6 +1328,7 @@ GC_Defaults = {
     GC_DOMAIN: '',
     GC_DRIVE_DIR: '',
     GC_ENABLE_DASA: False,
+    GC_LOW_MEMORY: False,
     GC_NO_BROWSER: False,
     GC_NO_TDEMAIL: False,
     GC_NO_CACHE: False,
@@ -1343,7 +1348,7 @@ GC_Defaults = {
     GC_CSV_ROW_DROP_FILTER: '',
     GC_TLS_MIN_VERSION: TLS_MIN,
     GC_TLS_MAX_VERSION: None,
-    GC_CA_FILE: None,
+    GC_CA_FILE: _FN_ROOTS_PEM,
 }
 
 GC_Values = {}
@@ -1408,6 +1413,9 @@ GC_VAR_INFO = {
     GC_ENABLE_DASA: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },
+    GC_LOW_MEMORY: {
+        GC_VAR_TYPE: GC_TYPE_BOOLEAN
+    },
     GC_NO_BROWSER: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },

src/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = GAM for Google Workspace
-version = 6.0.23
+version = attr: gam.var.GAM_VERSION
 description = Command line management for Google Workspaces
 long_description = file: readme.md
 long_description_content_type = text/markdown
@@ -37,6 +37,9 @@ install_requires =
     yubikey-manager >= 4.0.0
     pathvalidate
 
+[options.package_data]
+* = *.pem
+
 # used during pip install .[test]
 [options.extras_require]
 test = pre-commit