Update var.py

* Handle socks error when checking local time offset

* Keep pylint happy

* Avoid trap when authentication flow blows up

* Fix bug that causes trap on create project

.github/workflows/build.yml

@@ -12,7 +12,7 @@ defaults:
     working-directory: src
 
 env:
-  OPENSSL_CONFIG_OPTS: no-fips
+  OPENSSL_CONFIG_OPTS: no-fips --api=3.0.0
   OPENSSL_INSTALL_PATH: ${{ github.workspace }}/bin/ssl
   OPENSSL_SOURCE_PATH: ${{ github.workspace }}/src/openssl
   PYTHON_INSTALL_PATH: ${{ github.workspace }}/bin/python
@@ -103,7 +103,7 @@ jobs:
           path: |
             bin.tar.xz
             src/cpython
-          key: gam-${{ matrix.jid }}-20230208
+          key: gam-${{ matrix.jid }}-20230326
 
       - name: Untar Cache archive
         if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
@@ -144,12 +144,6 @@ jobs:
           sudo apt-get -qq --yes update
           sudo apt-get -qq --yes install swig libpcsclite-dev
 
-     #- name: MacOS remove Homebrew
-     #  if: runner.os == 'macOS'
-     #  run: |
-     #    # remove everything except the libraries needed by yubikey-manager
-     #    brew uninstall $(brew list | grep -v 'pcre\|swig\|pcsc-lite')
-
       - name: MacOS install tools
         if: runner.os == 'macOS'
         run: |
@@ -175,7 +169,7 @@ jobs:
           staticx: ${{ matrix.staticx }}
         run: |
           echo "We are running on ${RUNNER_OS}"
-          LD_LIBRARY_PATH="${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib"
+          LD_LIBRARY_PATH="${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib:/usr/local/lib"
           if [[ "${arch}" == "Win64" ]]; then
             PYEXTERNALS_PATH="amd64"
             PYBUILDRELEASE_ARCH="x64"
@@ -298,7 +292,7 @@ jobs:
             rm -rf ${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64
             echo "LDFLAGS=-L${OPENSSL_INSTALL_PATH}/lib" >> $GITHUB_ENV
             echo "CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS=1" >> $GITHUB_ENV
-            echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include -arch arm64 -arch x86_64" >> $GITHUB_ENV
+            echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include -arch arm64 -arch x86_64 ${CFLAGS}" >> $GITHUB_ENV
             echo "ARCHFLAGS=-arch x86_64 -arch arm64" >> $GITHUB_ENV
           else
             cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_archs}"
@@ -310,6 +304,7 @@ jobs:
         if: matrix.goal == 'build'
         run: |
           "${OPENSSL_INSTALL_PATH}/bin/openssl" version
+          "${OPENSSL_INSTALL_PATH}/bin/openssl" version -f
           file "${OPENSSL_INSTALL_PATH}/bin/openssl"
 
       - name: Get latest stable Python source
@@ -415,6 +410,7 @@ jobs:
 
       - name: Install pip requirements
         run: |
+             "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
              if [[ "${RUNNER_OS}" == "macOS" ]]; then
                "${PYTHON}" -m pip install --upgrade cffi ${PIP_ARGS}
                "${PYTHON}" -m pip download --only-binary :all: \
@@ -424,8 +420,9 @@ jobs:
                                              --platform macosx_10_15_universal2 \
                                              cryptography
                "${PYTHON}" -m pip install --force-reinstall --no-deps cryptography*.whl
+             else
+               "${PYTHON}" -m pip install --force-reinstall --no-deps --upgrade cryptography
              fi
-             "${PYTHON}" -m pip install --upgrade -r requirements.txt ${PIP_ARGS}
              "${PYTHON}" -m pip list
 
       - name: Install PyInstaller

src/GamCommands.txt

@@ -873,7 +873,7 @@ Specify a collection of Users by directly specifying them or by specifying items
 	<UserMultiAttribute>
 
 gam checkconnection
-gam version [check|simple|extended] [timeoffset] [location <HostName>]
+gam version [check|checkrc|simple|extended] [timeoffset] [location <HostName>]
 gam help
 
 gam batch <FileName>|- [charset <Charset>]
@@ -911,8 +911,8 @@ gam oauth|oauth2 refresh
 
 gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
-gam yubikey [resetpiv]
-gam rotate sakey yubikey yubikey_pin yubikey_slot AUTHENTICATION yubikeypin <String> yubikeyserialnumber <String>
+gam yubikey resetpiv [yubikeyserialnumber <Number>]
+gam rotate sakey yubikey yubikey_pin yubikey_slot AUTHENTICATION yubikeyserialnumber <Number>
 
 gam create [gcpserviceaccount|signjwtserviceaccount]
 gam enable apis [auto|manual]

src/gam/__init__.py

@@ -589,7 +589,7 @@ def SetGlobalVariables():
     GM_Globals[GM_ENABLEDASA_TXT] = os.path.join(
         GC_Values[GC_CONFIG_DIR], FN_ENABLEDASA_TXT)
     if not GC_Values[GC_NO_UPDATE_CHECK]:
-        doGAMCheckForUpdates()
+        doGAMCheckForUpdates(forceCheck=0)
 
 # domain must be set and customer_id must be set and != my_customer when enable_dasa = true
     if GC_Values[GC_ENABLE_DASA]:
@@ -636,8 +636,10 @@ TIME_OFFSET_UNITS = [('day', 86400), ('hour', 3600), ('minute', 60),
 
 
 def getLocalGoogleTimeOffset(testLocation='admin.googleapis.com'):
+    # If local time is well off, it breaks https because the server certificate
+    # will be seen as too old or new and thus invalid; http doesn't have that issue.
     # Try with http first, if time is close (<MAX_LOCAL_GOOGLE_TIME_OFFSET seconds),
-    # retry with https
+    # retry with https as it should be OK
     badhttp = transport.create_http()
     for prot in ['http', 'https']:
         localUTC = datetime.datetime.now(datetime.timezone.utc)
@@ -646,6 +648,12 @@ def getLocalGoogleTimeOffset(testLocation='admin.googleapis.com'):
                 badhttp.request(f'{prot}://' + testLocation, 'HEAD')[0]['date'])
         except (httplib2.ServerNotFoundError, RuntimeError, ValueError) as e:
             controlflow.system_error_exit(4, str(e))
+        except httplib2.socks.HTTPError as e:
+            # If user has specified an HTTPS proxy, the http request will probably fail as httplib2
+            # turns a GET into a CONNECT which is not valid for an http address
+            if prot == 'http':
+                continue
+            handleServerError(e)
         offset = remainder = int(abs((localUTC - googleUTC).total_seconds()))
         if offset < MAX_LOCAL_GOOGLE_TIME_OFFSET and prot == 'http':
             continue
@@ -660,7 +668,7 @@ def getLocalGoogleTimeOffset(testLocation='admin.googleapis.com'):
         return (offset, nicetime)
 
 
-def doGAMCheckForUpdates(forceCheck=False):
+def doGAMCheckForUpdates(forceCheck=0):
 
     def _gamLatestVersionNotAvailable():
         if forceCheck:
@@ -703,6 +711,8 @@ def doGAMCheckForUpdates(forceCheck=False):
             print(
                 f'Version Check:\n Current: {current_version}\n Latest: {latest_version}'
             )
+            if forceCheck < 0:
+                sys.exit(1 if latest_version > current_version else 0)
         if latest_version <= current_version:
             fileutils.write_file(GM_Globals[GM_LAST_UPDATE_CHECK_TXT],
                                  str(now_time),
@@ -822,14 +832,18 @@ def checkConnection():
         controlflow.system_error_exit(3, createYellowText('Some hosts failed to connect! Please follow the recommendations for those hosts to correct any issues and try again.'))
 
 def doGAMVersion(checkForArgs=True):
-    force_check = extended = simple = timeOffset = False
+    forceCheck = 0
+    extended = simple = timeOffset = False
     testLocation = 'admin.googleapis.com'
     if checkForArgs:
         i = 2
         while i < len(sys.argv):
             myarg = sys.argv[i].lower().replace('_', '')
             if myarg == 'check':
-                force_check = True
+                forceCheck = 1
+                i += 1
+            elif myarg == 'checkrc':
+                forceCheck = -1
                 i += 1
             elif myarg == 'simple':
                 simple = True
@@ -869,8 +883,8 @@ def doGAMVersion(checkForArgs=True):
               (testLocation, nicetime))
         if offset > MAX_LOCAL_GOOGLE_TIME_OFFSET:
             controlflow.system_error_exit(4, 'Please fix your system time.')
-    if force_check:
-        doGAMCheckForUpdates(forceCheck=True)
+    if forceCheck:
+        doGAMCheckForUpdates(forceCheck)
     if extended:
         print(ssl.OPENSSL_VERSION)
         libs = ['cryptography',
@@ -7412,7 +7426,7 @@ def _createClientSecretsOauth2service(httpObj, projectId, login_hint):
     while True:
         print(f'''Please go to:
 
-{console_url}
+  {console_url}
 
 1. Choose "Desktop App" or "Other" for "Application type".
 2. Enter a desired value for "Name" or leave as is.
@@ -7451,6 +7465,24 @@ def _createClientSecretsOauth2service(httpObj, projectId, login_hint):
     fileutils.write_file(GC_Values[GC_CLIENT_SECRETS_JSON],
                          cs_data,
                          continue_on_error=False)
+    print(f'''
+Now it's important to mark the GAM Client ID as trusted by your Workspace instance.
+
+1. Please go to:
+
+  https://admin.google.com/ac/owl/list?tab=configuredApps
+
+2. Click on: Add app > OAuth App Name Or Client ID.
+3. Enter the following Client ID value:
+
+  {client_id}
+
+4. Search for the ID, select the GAM app, check the box and press Select.
+5. Keep the default scope or select a preferred scope that includes your GAM admin.
+6. Press Continue
+7. Select Trusted radio button, Continue and Finish.
+''')
+    input('Press Enter when complete.')
     print('That\'s it! Your GAM Project is created and ready to use.')
 
 
@@ -7791,7 +7823,7 @@ def doUpdateProjects():
         _grantRotateRights(iam, sa_email, sa_email)
 
 
-def _generatePrivateKeyAndPublicCert(client_id, key_size, b64enc_pub=True):
+def _generatePrivateKeyAndPublicCert(client_id, key_size, b64enc_pub=True, validity_hours=0):
     print(' Generating new private key...')
     private_key = rsa.generate_private_key(public_exponent=65537,
                                            key_size=key_size,
@@ -7808,10 +7840,16 @@ def _generatePrivateKeyAndPublicCert(client_id, key_size, b64enc_pub=True):
     builder = builder.issuer_name(
         x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, client_id)]))
     # Gooogle seems to enforce the not before date strictly. Set the not before
-    # date to be UTC one hour ago should cover any clock skew.
-    builder = builder.not_valid_before(datetime.datetime.utcnow() - datetime.timedelta(hours=1))
-    # Google uses 12/31/9999 date for end time
-    builder = builder.not_valid_after(datetime.datetime(9999, 12, 31, 23, 59))
+    # date to be UTC two minutes ago which should cover any clock skew.
+    now = datetime.datetime.utcnow()
+    builder = builder.not_valid_before(now - datetime.timedelta(minutes=2))
+    # Google defaults to 12/31/9999 date for end time if there's no
+    # policy to restrict key age
+    if validity_hours:
+        expires = now + datetime.timedelta(hours=validity_hours) - datetime.timedelta(minutes=2)
+        builder = builder.not_valid_after(expires)
+    else:
+        builder = builder.not_valid_after(datetime.datetime(9999, 12, 31, 23, 59))
     builder = builder.serial_number(x509.random_serial_number())
     builder = builder.public_key(public_key)
     builder = builder.add_extension(x509.BasicConstraints(ca=False,
@@ -7877,6 +7915,7 @@ def doShowServiceAccountKeys():
                           'list',
                           'keys',
                           name=name,
+                          fields='*',
                           keyTypes=keyTypes)
     if not keys:
         print('No keys')
@@ -7891,6 +7930,28 @@ def doShowServiceAccountKeys():
     display.print_json(keys)
 
 
+def getYubiKeySerialNumber(new_data, serial_number):
+    try:
+        new_data['yubikey_serial_number'] = int(serial_number)
+    except ValueError:
+        controlflow.system_error_exit(
+            3,
+            'yubikey_serial_number must be a number')
+        
+def doResetYubiKeyPIV():
+    new_data = {}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'yubikeyserialnumber':
+            getYubiKeySerialNumber(new_data, sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, 'gam yubikey resetpiv')
+    yk = yubikey.YubiKey(new_data)
+    yk.serial_number = yk.get_serial_number()
+    yk.reset_piv()
+
 def create_signjwt_serviceaccount():
     i = 3
     if i < len(sys.argv):
@@ -7922,6 +7983,7 @@ def doCreateOrRotateServiceAccountKeys(iam=None,
                                        client_email=None,
                                        client_id=None):
     local_key_size = 2048
+    validity_hours = 0
     mode = 'retainexisting'
     body = {}
     if iam:
@@ -7972,12 +8034,10 @@ def doCreateOrRotateServiceAccountKeys(iam=None,
                 new_data['yubikey_pin'] = input('Enter your YubiKey PIN: ')
                 i += 1
             elif myarg == 'yubikeyserialnumber':
-                try:
-                    new_data['yubikey_serial_number'] = int(sys.argv[i+1])
-                except ValueError:
-                    controlflow.system_error_exit(
-                            3,
-                            'yubikey_serial_number must be a number')
+                getYubiKeySerialNumber(new_data, sys.argv[i+1])
+                i += 2
+            elif myarg == 'validityhours':
+                validity_hours = int(sys.argv[i + 1])
                 i += 2
             elif myarg in ['retainnone', 'retainexisting', 'replacecurrent']:
                 mode = myarg
@@ -7999,7 +8059,7 @@ def doCreateOrRotateServiceAccountKeys(iam=None,
     elif local_key_size:
         # Generate private key locally, store in file
         new_data['private_key'], publicKeyData = _generatePrivateKeyAndPublicCert(
-            sa_name, local_key_size)
+            sa_name, local_key_size, validity_hours=validity_hours)
         new_data['key_type'] = 'default'
         for key in list(new_data):
             if key.startswith('yubikey_'):
@@ -11877,6 +11937,8 @@ def ProcessGAMCommand(args):
             argument = sys.argv[2].lower()
             if argument in ['browsertoken', 'browserokens']:
                 gapi_cbcm.revoketoken()
+            else:
+                controlflow.invalid_argument_exit(argument, 'gam revoke')
             sys.exit(0)
         elif command in ['close', 'reopen']:
             # close and reopen will have to be split apart if either takes a new argument
@@ -12041,6 +12103,8 @@ def ProcessGAMCommand(args):
             argument = sys.argv[2].lower()
             if argument in ['browser', 'browsers']:
                 gapi_cbcm.move()
+            else:
+                controlflow.invalid_argument_exit(argument, 'gam move')
             sys.exit(0)
         elif command in ['oauth', 'oauth2']:
             argument = sys.argv[2].lower()
@@ -12138,6 +12202,8 @@ def ProcessGAMCommand(args):
             argument = sys.argv[2].lower()
             if argument in ['isinvitable', 'userinvitation', 'userinvitations']:
                 gapi_cloudidentity_userinvitations.check()
+            else:
+                controlflow.invalid_argument_exit(argument, 'gam check')
             sys.exit(0)
         elif command in ['cancelwipe', 'wipe', 'approve', 'block', 'sync']:
             target = sys.argv[2].lower().replace('_', '')
@@ -12157,6 +12223,8 @@ def ProcessGAMCommand(args):
                     gapi_cloudidentity_devices.approve_user()
                 elif command == 'block':
                     gapi_cloudidentity_devices.block_user()
+            else:
+                controlflow.invalid_argument_exit(target, f'gam {command}')
             sys.exit(0)
         elif command in ['issuecommand', 'getcommand']:
             target = sys.argv[2].lower().replace('_', '')
@@ -12165,18 +12233,22 @@ def ProcessGAMCommand(args):
                     gapi_directory_cros.issue_command()
                 elif command == 'getcommand':
                     gapi_directory_cros.get_command()
+            else:
+                controlflow.invalid_argument_exit(target, f'gam {command}')
             sys.exit(0)
         elif command in ['yubikey']:
             action = sys.argv[2].lower().replace('_', '')
             if action == 'resetpiv':
-                yk = yubikey.YubiKey()
-                yk.serial_number = yk.get_serial_number()
-                yk.reset_piv()
+                doResetYubiKeyPIV()
+            else:
+                controlflow.invalid_argument_exit(action, f'gam yubikey')
             sys.exit(0)
         elif command == 'enable':
             enable_what = sys.argv[2].lower().replace('_', '')
             if enable_what in ['api', 'apis']:
                 enable_apis()
+            else:
+                controlflow.invalid_argument_exit(enable_what, 'gam enable')
             sys.exit(0)
         users = getUsersToModify()
         command = sys.argv[3].lower()

src/gam/auth/__init__.py

@@ -5,7 +5,6 @@ import os
 
 from google.auth.jwt import Credentials as JWTCredentials
 
-import gam
 from gam import utils
 
 from gam.auth import oauth
@@ -29,8 +28,7 @@ def get_admin_credentials_filename():
     # some custom name in it. Otherwise, just use the default name.
     if GC_Values[GC_ENABLE_DASA]:
         return GC_Values[GC_OAUTH2SERVICE_JSON] if GC_Values[GC_OAUTH2SERVICE_JSON] else _FN_OAUTH2SERVICE_JSON
-    else:
-        return GC_Values[GC_OAUTH2_TXT] if GC_Values[GC_OAUTH2_TXT] else _FN_OAUTH2_TXT
+    return GC_Values[GC_OAUTH2_TXT] if GC_Values[GC_OAUTH2_TXT] else _FN_OAUTH2_TXT
 
 
 def get_admin_credentials(api=None):
@@ -47,12 +45,12 @@ def get_admin_credentials(api=None):
         if key_type == 'default':
             return JWTCredentials.from_service_account_info(creds_data,
                                                             audience=audience)
-        elif key_type == 'yubikey':
+        if key_type == 'yubikey':
             yksigner = yubikey.YubiKey(creds_data)
             return JWTCredentials._from_signer_and_info(yksigner,
                 creds_data,
                 audience=audience)
-        elif key_type == 'signjwt':
+        if key_type == 'signjwt':
             sjsigner = signjwt.SignJwt(creds_data)
             return signjwt.JWTCredentials._from_signer_and_info(sjsigner,
                     creds_data,

src/gam/auth/oauth.py

@@ -50,6 +50,7 @@ MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
                                 ' close this browser window and return to GAM.')
 
 MESSAGE_AUTHENTICATION_COMPLETE = ('\nThe authentication flow has completed.\n')
+MESSAGE_AUTHENTICATION_FAILED = ('\nThe authentication flow failed, reissue command')
 
 
 class CredentialsError(Exception):
@@ -629,15 +630,22 @@ class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
         print(MESSAGE_CONSOLE_AUTHORIZATION_PROMPT.format(url=d['auth_url']))
         user_input.start()
         userInput = False
-        while True:
+        alive = 2
+        while alive > 0:
             sleep(0.1)
             if not http_client.is_alive():
-                user_input.terminate()
-                break
-            elif not user_input.is_alive():
+                if 'code' in d:
+                    user_input.terminate()
+                    break
+                alive -= 1
+            if not user_input.is_alive():
                 userInput = True
-                http_client.terminate()
-                break
+                if 'code' in d:
+                    http_client.terminate()
+                    break
+                alive -= 1
+        if 'code' not in d:
+            controlflow.system_error_exit(8, MESSAGE_AUTHENTICATION_FAILED)
         while True:
             code = d['code']
             if code.startswith('http'):

src/gam/auth/yubikey.py

@@ -87,6 +87,9 @@ class YubiKey():
     def get_serial_number(self):
         try:
             devices = list_all_devices()
+            if not devices:
+                msg = f'Could not find any YubiKey'
+                controlflow.system_error_exit(3, msg)
             if self.serial_number:
                 for (device, info) in devices:
                     if info.serial == self.serial_number:

src/gam/gapi/chromepolicy.py

@@ -408,7 +408,7 @@ def update_policy():
                               f'{expected_enums}, got {value}'
                         controlflow.system_error_exit(8, msg)
                 elif vtype in ['TYPE_LIST']:
-                    value = value.split(',')
+                    value = value.split(',') if value else []
                 if myarg == 'chrome.users.chromebrowserupdates' and \
                       cased_field == 'targetVersionPrefixSetting':
                     mg = re.compile(r'^([a-z]+)-(\d+)$').match(value)

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.50'
+GAM_VERSION = '6.53'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'
@@ -62,6 +62,21 @@ SKUS = {
         'aliases': ['cloudsearch'],
         'displayName': 'Google Cloud Search',
     },
+    '1010380001': {
+        'product': '101038',
+        'aliases': ['appsheetcore'],
+        'displayName': 'AppSheet Core',
+    },
+    '1010380002': {
+        'product': '101038',
+        'aliases': ['appsheetstandard', 'appsheetenterprisestandard'],
+        'displayName': 'AppSheet Enterprise Standard',
+    },
+    '1010380003': {
+        'product': '101038',
+        'aliases': ['appsheetplus', 'appsheetenterpriseplus'],
+        'displayName': 'AppSheet Enterprise Plus',
+    },
     '1010310002': {
         'product': '101031',
         'aliases': ['gsefe', 'e4e', 'gsuiteenterpriseeducation'],
@@ -300,6 +315,7 @@ PRODUCTID_NAME_MAPPINGS = {
     '101035': 'Cloud Search',
     '101036': 'Google Meet Global Dialing',
     '101037': 'G Suite Workspace for Education',
+    '101038': 'AppSheet',
     '101039': 'Assured Controls',
     '101040': 'Beyond Corp',
     'Google-Apps': 'Google Workspace',