Update var.py

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
             goal: build
             arch: x86_64
             openssl_archs: linux-x86_64
-          - os: [self-hosted, linux, arm64]
+          - os: [self-hosted, linux, arm64, gcp]
             jid: 2
             goal: build
             arch: aarch64
@@ -71,18 +71,26 @@ jobs:
             arch: x86_64
 
     steps:
+
       - uses: actions/checkout@v3
         with:
           persist-credentials: false
           fetch-depth: 0
 
       - name: Cache multiple paths
+        if: matrix.goal == 'build'
         uses: actions/cache@v3
         id: cache-python-ssl
         with:
           path: |
-            bin
-          key: gam-${{ matrix.jid }}-20220901
+            bin.tar.xz
+          key: gam-${{ matrix.jid }}-20221011
+
+      - name: Untar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar xvvf bin.tar.xz
 
       - name: Use pre-compiled Python for testing
         if: matrix.python != ''
@@ -117,11 +125,11 @@ jobs:
           sudo apt-get -qq --yes update
           sudo apt-get -qq --yes install swig libpcsclite-dev
 
-      - name: MacOS remove Homebrew
-        if: runner.os == 'macOS'
-        run: |
-          # remove everything except the libraries needed by yubikey-manager
-          brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
+     #- name: MacOS remove Homebrew
+     #  if: runner.os == 'macOS'
+     #  run: |
+     #    # remove everything except the libraries needed by yubikey-manager
+     #    brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
 
       - name: MacOS install tools
         if: runner.os == 'macOS'
@@ -192,7 +200,6 @@ jobs:
           echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
           echo "openssl_archs=${openssl_archs}" >> $GITHUB_ENV
           echo "LD_LIBRARY_PATH=${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib" >> $GITHUB_ENV
-          #echo "PATH=${PATH}:${PYTHON_INSTALL_PATH}/scripts" >> $GITHUB_ENV
 
       - name: Get latest stable OpenSSL source
         if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
@@ -365,6 +372,9 @@ jobs:
           Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Lib\*" "${env:PYTHON_INSTALL_PATH}\lib\" -recurse
           Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\Include\*" "${env:PYTHON_INSTALL_PATH}\include\" -recurse
           Copy-Item -Path "${env:PYTHON_SOURCE_PATH}\PC\*.h" "${env:PYTHON_INSTALL_PATH}\include\"
+          $env:Path = '${PYTHON_INSTALL_PATH}/bin' + $env:Path
+          echo "Path=$env:Path" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "PATH: ${env:Path}"
 
       - name: Mac/Linux Build Python
         if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
@@ -379,6 +389,9 @@ jobs:
           cd "${PYTHON_SOURCE_PATH}"
           $MAKE altinstall
           $MAKE bininstall
+          export PATH="${PATH}:${PYTHON_INSTALL_PATH}/bin"
+          echo "PATH=${PATH}" >> $GITHUB_ENV
+          echo "PATH: ${PATH}"
 
       - name: Run Python
         run: |
@@ -391,25 +404,6 @@ jobs:
           "${PYTHON}" -m pip install --upgrade pip
           "${PYTHON}" -m pip install --upgrade wheel
           "${PYTHON}" -m pip install --upgrade setuptools
-    
-      - name: Install PyInstaller
-        if: matrix.goal == 'build'
-        run: |
-          git clone https://github.com/pyinstaller/pyinstaller.git
-          cd pyinstaller
-          export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
-          git checkout "${latest_release}"
-          # remove pre-compiled bootloaders so we fail if bootloader compile fails
-          rm -rvf PyInstaller/bootloader/*-*/*
-          cd bootloader
-          if [[ "${arch}" == "Win32" ]]; then
-            export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
-          fi
-          echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
-          "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
-          cd ../..
-          echo "---- Installing PyInstaller ----"
-          "${PYTHON}" -m pip install pyinstaller
 
       - name: Install pip requirements
         run: |
@@ -429,6 +423,27 @@ jobs:
              "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
              "${PYTHON}" -m pip list
 
+      - name: Install PyInstaller
+        if: matrix.goal == 'build'
+        run: |
+          git clone https://github.com/pyinstaller/pyinstaller.git
+          cd pyinstaller
+          export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
+          # temp freeze PyInstaller at 5.3
+          export latest_release="v5.3"
+          git checkout "${latest_release}"
+          # remove pre-compiled bootloaders so we fail if bootloader compile fails
+          rm -rvf PyInstaller/bootloader/*-*/*
+          cd bootloader
+          if [[ "${arch}" == "Win32" ]]; then
+            export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
+          fi
+          echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
+          "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
+          cd ..
+          echo "---- Installing PyInstaller ----"
+          "${PYTHON}" -m pip install .
+
       - name: Build GAM with PyInstaller
         if: matrix.goal != 'test'
         run: |
@@ -545,11 +560,12 @@ jobs:
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                brew install gnupg
              fi
-             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.gpg creds.tar
+             source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.xz.gpg creds.tar.xz
              export OAUTHFILE="oauth2.txt-gam-gha-${JID}"
              echo "OAUTHFILE=${OAUTHFILE}" >> $GITHUB_ENV
              export gam_user="gam-gha-${JID}@pdl.jaylee.us"
              echo "gam_user=${gam_user}" >> $GITHUB_ENV
+             touch "${gampath}/lowmemory.txt"
              $gam checkconn
              $gam oauth info
              $gam info domain
@@ -701,7 +717,7 @@ jobs:
 
       - name: Archive production artifacts
         uses: actions/upload-artifact@v3
-        if: github.event_name == 'push' && matrix.goal != 'test'
+        if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal != 'test'
         with:
           name: gam-binaries
           path: |
@@ -709,7 +725,14 @@ jobs:
             src/*.zip
             src/*.msi
 
+      - name: Tar Cache archive
+        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
+        working-directory: ${{ github.workspace }}
+        run: |
+          tar cJvvf bin.tar.xz bin/
+
   publish:
+    if: github.event_name == 'push'
     runs-on: ubuntu-latest
     needs: build
     steps:
@@ -721,12 +744,19 @@ jobs:
 
       - name: Download artifacts
         uses: actions/download-artifact@v3
-        
+
       - name: Set datetime version string
         id: dateversion
         run: |
           echo "::set-output name=dateversion::$(date +'%Y%m%d.%k%M%S')"
 
+      - name: VirusTotal Scan
+        uses: crazy-max/ghaction-virustotal@v3
+        with:
+          vt_api_key: ${{ secrets.VT_API_KEY }}
+          files: |
+            gam-binaries/*
+
       - uses: "marvinpinto/action-automatic-releases@latest"
         name: Publish draft release
         with:

src/gam/__init__.py

@@ -553,9 +553,12 @@ def SetGlobalVariables():
                       'debug.gam',
                       filePresentValue=4,
                       fileAbsentValue=0)
+    _getOldSignalFile(GC_LOW_MEMORY, 'lowmemory.txt')
     _getOldSignalFile(GC_NO_BROWSER, 'nobrowser.txt')
     _getOldSignalFile(GC_NO_TDEMAIL, 'notdemail.txt')
-    _getOldSignalFile(GC_OAUTH_BROWSER, 'oauthbrowser.txt')
+    # oauthbrowser.txt is deprecated as we now always
+    # use the localhost flow.
+    #_getOldSignalFile(GC_OAUTH_BROWSER, 'oauthbrowser.txt')
     #  _getOldSignalFile(GC_NO_CACHE, u'nocache.txt')
     #  _getOldSignalFile(GC_CACHE_DISCOVERY_ONLY, u'allcache.txt', filePresentValue=False, fileAbsentValue=True)
     _getOldSignalFile(GC_NO_CACHE,
@@ -7129,7 +7132,7 @@ def getCRMService(login_hint):
         scopes,
         'online',
         login_hint=login_hint,
-        use_console_flow=not GC_Values[GC_OAUTH_BROWSER])
+        open_browser=not GC_Values[GC_NO_BROWSER])
     httpc = transport.AuthorizedHttp(creds, transport.create_http())
     return getService('cloudresourcemanager', httpc), httpc
 
@@ -10543,7 +10546,7 @@ def doRequestOAuth(login_hint=None, scopes=None):
             access_type='offline',
             login_hint=login_hint,
             credentials_file=GC_Values[GC_OAUTH2_TXT],
-            use_console_flow=not GC_Values[GC_OAUTH_BROWSER])
+            open_browser=not GC_Values[GC_NO_BROWSER])
         creds.write()
     except gam.auth.oauth.InvalidClientSecretsFileError:
         controlflow.system_error_exit(14, missing_client_secrets_message)

src/gam/auth/oauth.py

@@ -272,6 +272,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                             access_type='offline',
                             login_hint=None,
                             filename=None,
+                            open_browser=True,
                             use_console_flow=False):
         """Runs an OAuth Flow from client secrets to generate credentials.
 
@@ -291,8 +292,11 @@ class Credentials(google.oauth2.credentials.Credentials):
       login_hint: String, The email address that will be displayed on the Google
         login page as a hint for the user to login to the correct account.
       filename: String, the path to a file to use to save the credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
+        should be run strictly from a console; False to launch a browser
+        for authentication.
+      open_browser: Boolean: whether or not GAM should try to open the browser
+        automatically.
 
     Returns:
       Credentials
@@ -312,12 +316,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         flow = _ShortURLFlow.from_client_config(client_config,
                                                 scopes,
                                                 autogenerate_code_verifier=True)
-        flow_kwargs = {'access_type': access_type}
+        flow_kwargs = {'access_type': access_type,
+                       'open_browser': open_browser}
         if login_hint:
             flow_kwargs['login_hint'] = login_hint
-
-        flow.run_dual(use_console_flow,
-                **flow_kwargs)
+        flow.run_dual(**flow_kwargs)
         return cls.from_google_oauth2_credentials(flow.credentials,
                                                   filename=filename)
 
@@ -328,6 +331,7 @@ class Credentials(google.oauth2.credentials.Credentials):
                                  access_type='offline',
                                  login_hint=None,
                                  credentials_file=None,
+                                 open_browser=True,
                                  use_console_flow=False):
         """Runs an OAuth Flow from secrets stored on disk to generate credentials.
 
@@ -348,8 +352,11 @@ class Credentials(google.oauth2.credentials.Credentials):
         login page as a hint for the user to login to the correct account.
       credentials_file: String, the path to a file to use to save the
         credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
+      use_console_flow: OBSOLETE: Boolean, True if the authentication flow
+        should be run strictly from a console; False to launch a browser for
+        authentication.
+      open_browser: Boolean, whether or not GAM should try to open the browser
+        directly.
 
     Raises:
       InvalidClientSecretsFileError: If the client secrets file cannot be
@@ -378,14 +385,13 @@ class Credentials(google.oauth2.credentials.Credentials):
             raise InvalidClientSecretsFileFormatError(
                 f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
             )
-
         return cls.from_client_secrets(client_id,
                                        client_secret,
                                        scopes,
                                        access_type=access_type,
                                        login_hint=login_hint,
                                        filename=credentials_file,
-                                       use_console_flow=use_console_flow)
+                                       open_browser=open_browser)
 
     def _fetch_id_token_data(self):
         """Fetches verification details from Google for the OAuth2.0 token.
@@ -482,7 +488,11 @@ class Credentials(google.oauth2.credentials.Credentials):
     def _locked_refresh(self, request):
         """Refreshes the credential's access token while the file lock is held."""
         assert self._lock.is_locked
-        super().refresh(request)
+        try:
+            super().refresh(request)
+        except google.auth.exceptions.RefreshError as e:
+            controlflow.system_error_exit(9, str(e))
+
 
     def write(self):
         """Writes credentials to disk."""
@@ -595,7 +605,6 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
 
 
     def run_dual(self,
-                use_console_flow,
                 authorization_prompt_message='',
                 console_prompt_message='',
                 web_success_message='',
@@ -605,7 +614,7 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
         mgr = multiprocessing.Manager()
         d = mgr.dict()
         d['trailing_slash'] = redirect_uri_trailing_slash
-        d['open_browser'] = use_console_flow
+        d['open_browser'] = open_browser
         http_client = multiprocessing.Process(target=_wait_for_http_client,
                                               args=(d,))
         user_input = multiprocessing.Process(target=_wait_for_user_input,

src/gam/gapi/__init__.py

@@ -1,6 +1,9 @@
 """Methods related to execution of GAPI requests."""
 
+import os.path
+import shelve
 import sys
+from tempfile import TemporaryDirectory
 
 import googleapiclient.errors
 import google.auth.exceptions
@@ -10,7 +13,8 @@ from gam import controlflow
 from gam import display
 from gam.gapi import errors
 from gam import transport
-from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+from gam.var import (GC_Values, GC_LOW_MEMORY, GM_Globals,
+                     GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
                      GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
                      MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
                      MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
@@ -238,8 +242,13 @@ def process_page(page, items, all_items, total_items, page_message, message_attr
         page_items = page.get(items, [])
         num_page_items = len(page_items)
         total_items += num_page_items
-        if all_items is not None:
+        if type(all_items) is list:
             all_items.extend(page_items)
+        elif all_items is not None:
+            i = len(all_items)
+            for item in page_items:
+                all_items[str(i)] = item
+                i += 1
     else:
         page_token = None
         num_page_items = 0
@@ -273,6 +282,7 @@ def finalize_page_message(page_message):
         sys.stderr.write('\r\n')
         sys.stderr.flush()
 
+
 def get_all_pages(service,
                   function,
                   items='items',
@@ -328,7 +338,18 @@ def get_all_pages(service,
                 kwargs['body'].update(page_key)
             else:
                 kwargs.update(page_key)
-    all_items = []
+    if GC_Values[GC_LOW_MEMORY]:
+        td_args = {'prefix': 'GAM-'}
+        if sys.version_info.minor >= 10:
+            td_args['ignore_cleanup_errors'] = True
+        tempdir = TemporaryDirectory(**td_args)
+        tempfile = os.path.join(tempdir.name, 'gapi_pages')
+        all_items = shelve.open(tempfile)
+        # attach tempdir to all_items so we
+        # don't cleanup tempdir early
+        all_items._tempdir = tempdir
+    else:
+        all_items = []
     page_token = None
     total_items = 0
     while True:
@@ -341,13 +362,14 @@ def get_all_pages(service,
         page_token, total_items = process_page(page, items, all_items, total_items, page_message, message_attribute)
         if not page_token:
             finalize_page_message(page_message)
+            if type(all_items) is not list:
+                all_items = all_items.values()
             return all_items
         if page_args_in_body:
             kwargs['body']['pageToken'] = page_token
         else:
             kwargs['pageToken'] = page_token
 
-
 # TODO: Make this private once all execution related items that use this method
 # have been brought into this file
 def handle_oauth_token_error(e, soft_errors):

src/gam/gapi/cloudresourcemanager.py

@@ -13,12 +13,14 @@ def get_org_id():
     gapi_directory_customer.setTrueCustomerId()
     crm = build()
     query = f'directorycustomerid:{GC_Values[GC_CUSTOMER_ID]}'
-    orgs = gapi.get_all_pages(crm.organizations(),
+    results = gapi.call(crm.organizations(),
                      'search',
-                     'organizations',
+                     pageSize=1,
+                     fields='organizations/name',
                      query=query)
-    if len(orgs) < 1:
+    orgs = results.get('organizations')
+    if not orgs:
         # return nothing and let calling API deal with it
         # since caller knows what GCP role would serve best
         return 
-    return orgs[0]['name']
+    return orgs[0].get('name')

src/gam/gapi/directory/customer.py

@@ -21,36 +21,36 @@ def doGetCustomerInfo():
                               'get',
                               customerKey=customer_id)
     print(f'Customer ID: {customer_info["id"]}')
-    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    fields = 'domains(creationTime,domainName,isPrimary,verified)'
     try:
-        result = gapi.call(
+        domains = gapi.call(
             cd.domains(),
-            'get',
+            'list',
+            fields=fields,
             customer=customer_id,
-            domainName=customer_info['customerDomain'],
-            fields='verified',
-            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND])
+            throw_reasons=[gapi.errors.ErrorReason.DOMAIN_NOT_FOUND]).get('domains', [])
+        for domain in domains:
+            if domain.get('isPrimary'):
+                primary_domain = domain
+                break
+        else:
+            primary_domain = {}
     except gapi.errors.GapiDomainNotFoundError:
-        result = {'verified': False}
-    print(f'Primary Domain Verified: {result["verified"]}')
-    # If customer has changed primary domain customerCreationTime is date
-    # of current primary being added, not customer create date.
-    # We should also get all domains and use oldest date
-    customer_creation = customer_info['customerCreationTime']
-    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
-    oldest = datetime.datetime.strptime(customer_creation, date_format)
-    domains = gapi.get_items(cd.domains(),
-                             'list',
-                             'domains',
-                             customer=customer_id,
-                             fields='domains(creationTime)')
+        primary_domain = {}
+    print(f'Primary Domain: {primary_domain.get("domainName", "Unknown")}')
+    print(f'Primary Domain Verified: {primary_domain.get("verified", "Unknown")}')
+    # we'll assume creation time is time of oldest domain customer has
+    oldest = 'Unknown' 
     for domain in domains:
         creation_timestamp = int(domain['creationTime']) / 1000
         domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
-        if domain_creation < oldest:
+        if oldest == 'Unknown' or domain_creation < oldest:
             oldest = domain_creation
-    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
-    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    if oldest != 'Unknown':
+        date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+        oldest = oldest.strftime(date_format)
+    print(f'Customer Creation Time: {oldest}')
+    customer_language = customer_info.get('language', 'Unset or Unknown (defaults to en)')
     print(f'Default Language: {customer_language}')
     if 'postalAddress' in customer_info:
         print('Address:')
@@ -59,7 +59,7 @@ def doGetCustomerInfo():
                 print(f' {field}: {customer_info["postalAddress"][field]}')
     if 'phoneNumber' in customer_info:
         print(f'Phone: {customer_info["phoneNumber"]}')
-    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    print(f'Admin Secondary Email: {customer_info.get("alternateEmail", "Unknown")}')
     user_counts_map = {
         'accounts:num_users': 'Total Users',
         'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',

src/gam/gapi/reports.py

@@ -426,6 +426,7 @@ def showReport():
         titles = ['name', 'value', 'client_id']
         csvRows = []
         auth_apps = list()
+        usage = list(usage)
         for item in usage[0]['parameters']:
             if 'name' not in item:
                 continue

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.25'
+GAM_VERSION = '6.26'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'
@@ -206,7 +206,7 @@ SKUS = {
     },
     '1010020030': {
         'product': 'Google-Apps',
-        'aliases': ['workspacefrontline', 'workspacefrontlineworker'],
+        'aliases': ['wsflw', 'workspacefrontline', 'workspacefrontlineworker'],
         'displayName': 'Workspace Frontline'
     },
     '1010340002': {
@@ -1270,6 +1270,9 @@ GC_ENABLE_DASA = 'enabledasa'
 # and doRequestOAuth prints a link and waits for the verification code when
 # oauth2.txt is being created
 GC_NO_BROWSER = 'no_browser'
+# If low memory is True, GAM tries to save RAM by writing pages to disk
+# temporarily
+GC_LOW_MEMORY = 'low_memory'
 # If no_tdemail is True, writeCSVfile won't send an email
 GC_NO_TDEMAIL = 'no_tdemail'
 # oauth_browser forces usage of web server OAuth flow that proved problematic.
@@ -1325,6 +1328,7 @@ GC_Defaults = {
     GC_DOMAIN: '',
     GC_DRIVE_DIR: '',
     GC_ENABLE_DASA: False,
+    GC_LOW_MEMORY: False,
     GC_NO_BROWSER: False,
     GC_NO_TDEMAIL: False,
     GC_NO_CACHE: False,
@@ -1409,6 +1413,9 @@ GC_VAR_INFO = {
     GC_ENABLE_DASA: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },
+    GC_LOW_MEMORY: {
+        GC_VAR_TYPE: GC_TYPE_BOOLEAN
+    },
     GC_NO_BROWSER: {
         GC_VAR_TYPE: GC_TYPE_BOOLEAN
     },