GCP cleanup (#1602)

.github/workflows/build.yml

@@ -25,16 +25,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04
-            jid: 1
-            goal: build
-            arch: x86_64
-            openssl_archs: linux-x86_64
-          - os: [self-hosted, linux, arm64, gcp]
-            jid: 2
-            goal: build
-            arch: aarch64
-            openssl_archs: linux-aarch64
+          #- os: ubuntu-20.04
+          #  jid: 1
+          #  goal: build
+          #  arch: x86_64
+          #  openssl_archs: linux-x86_64
+          #- os: [self-hosted, linux, arm64, gcp]
+          #  jid: 2
+          #  goal: build
+          #  arch: aarch64
+          #  openssl_archs: linux-aarch64
           - os: ubuntu-20.04
             jid: 3
             goal: build

src/GamCommands.txt

@@ -911,6 +911,12 @@ gam oauth|oauth2 refresh
 
 gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
+gam yubikey [resetpiv]
+gam rotate sakey yubikey yubikey_pin yubikey_slot AUTHENTICATION yubikeypin <String> yubikeyserialnumber <String>
+
+gam create [gcpserviceaccount|signjwtserviceaccount]
+gam enable apis [auto|manual]
+
 gam whatis <EmailItem>
 
 <ResoldCustomerAttribute> ::=

src/gam/__init__.py

@@ -51,6 +51,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.x509.oid import NameOID
 
 import gam.auth.oauth
+from gam.auth import signjwt
 from gam import auth
 from gam import controlflow
 from gam import display
@@ -782,9 +783,9 @@ def checkConnection():
     success_count = 0
     for host in hosts:
         try_count += 1
-        ip = socket.gethostbyname(host)
+        ip = socket.getaddrinfo(host, None)[0][-1][0] # works with ipv6
         check_line = f'Checking {host} ({ip}) ({try_count}/{host_count})...'
-        sys.stdout.write(f'{check_line:<80}')
+        sys.stdout.write(f'{check_line:<100}')
         sys.stdout.flush()
         try:
             httpc.request(f'https://{host}/', 'HEAD', headers=headers)
@@ -925,14 +926,12 @@ def _getSvcAcctData():
             controlflow.system_error_exit(6, None)
         GM_Globals[GM_OAUTH2SERVICE_JSON_DATA] = json.loads(json_string)
 
-jwt_apis = ['chat',
-            'cloudresourcemanager',
-            'accesscontextmanager'] # APIs which can handle OAuthless JWT tokens
-def getSvcAcctCredentials(scopes, act_as, api=None):
+def getSvcAcctCredentials(scopes, act_as, api=None, force_oauth=False):
     try:
         _getSvcAcctData()
         sign_method = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
-        if act_as or api not in jwt_apis:
+        if act_as or force_oauth:
+            # DwD means we need to go about things differently...
             if sign_method == 'default':
                 credentials = google.oauth2.service_account.Credentials.from_service_account_info(
                     GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
@@ -940,6 +939,10 @@ def getSvcAcctCredentials(scopes, act_as, api=None):
                 yksigner = yubikey.YubiKey(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
                 credentials = google.oauth2.service_account.Credentials._from_signer_and_info(yksigner,
                     GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+            elif sign_method == 'signjwt':
+                sjsigner = signjwt.SignJwt(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+                credentials = signjwt.Credentials._from_signer_and_info(sjsigner.sign,
+                        GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
             credentials = credentials.with_scopes(scopes)
             if act_as:
                 credentials = credentials.with_subject(act_as)
@@ -953,6 +956,11 @@ def getSvcAcctCredentials(scopes, act_as, api=None):
                 credentials = JWTCredentials._from_signer_and_info(yksigner,
                                                                    GM_Globals[GM_OAUTH2SERVICE_JSON_DATA],
                                                                    audience=audience)
+            elif sign_method == 'signjwt':
+                sjsigner = signjwt.SignJwt(GM_Globals[GM_OAUTH2SERVICE_JSON_DATA])
+                credentials = signjwt.JWTCredentials._from_signer_and_info(sjsigner,
+                                                                   GM_Globals[GM_OAUTH2SERVICE_JSON_DATA],
+                                                                   audience=audience)
             credentials.project_id = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA]['project_id']
         GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID] = GM_Globals[
             GM_OAUTH2SERVICE_JSON_DATA]['client_id']
@@ -1078,9 +1086,10 @@ def getService(api, httpObj):
         controlflow.invalid_json_exit(disc_file)
 
 
-def buildGAPIObject(api):
+def buildGAPIObject(api, credentials=None):
     GM_Globals[GM_CURRENT_API_USER] = None
-    credentials = getValidOauth2TxtCredentials(api=getAPIVersion(api)[0])
+    if not credentials:
+        credentials = getValidOauth2TxtCredentials(api=getAPIVersion(api)[0])
     credentials.user_agent = GAM_INFO
     httpObj = transport.AuthorizedHttp(
         credentials, transport.create_http(cache=GM_Globals[GM_CACHE_DIR]))
@@ -1295,7 +1304,9 @@ def doCheckServiceAccount(users):
     # We are explicitly not doing DwD here, just confirming service account can auth
     auth_error = ''
     try:
-        credentials = getSvcAcctCredentials([USERINFO_EMAIL_SCOPE], None)
+        credentials = getSvcAcctCredentials([USERINFO_EMAIL_SCOPE],
+                                            None,
+                                            force_oauth=True)
         request = transport.create_request()
         credentials.refresh(request)
         sa_token_info = gapi.call(oa2,
@@ -1315,12 +1326,10 @@ def doCheckServiceAccount(users):
             'Invalid private key in oauth2service.json. Please delete the file and then\nrecreate with "gam create project" or "gam use project"'
         )
     key_type = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
-    if key_type == 'yubikey':
-        printPassFail('Skipping age check. YubiKey rotation not necessary.', test_pass)
-    else:
+    if key_type == 'default':
         print(
             'Checking key age. Google recommends rotating keys on a routine basis...'
-    )
+        )
         try:
             iam = buildGAPIServiceObject('iam', None)
             project = GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID]
@@ -1346,6 +1355,8 @@ def doCheckServiceAccount(users):
             key_days = 'UNKNOWN'
             print('Unable to check key age, please run "gam update project"')
         printPassFail(f'Key is {key_days} days old', key_age_result)
+    else:
+        printPassFail(f'Skipping age check. {key_type} rotation not necessary.', test_pass)
     if not check_scopes:
         for _, scopes in list(API_SCOPE_MAPPING.items()):
             for scope in scopes:
@@ -7158,6 +7169,43 @@ def getGAMProjectFile(filepath):
     return c.decode(UTF8)
 
 
+def enable_apis():
+    a_or_m = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg in ['auto', 'manual']:
+            a_or_m = myarg[0]
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam enable apis')
+    GAMProjectAPIs = getGAMProjectFile('project-apis.txt').splitlines()
+    try:
+        _, projectId = google.auth.default()
+    except google.auth.exceptions.DefaultCredentialsError as e:
+        projectId = input('Please enter your project ID: ')
+    while a_or_m not in ['a', 'm']:
+        a_or_m = input('Do you want to enable projects [a]utomatically or [m]anually? (a/m): ').strip().lower()
+        if a_or_m in ['a', 'm']:
+            break
+        print('Please enter A or M....')
+    if a_or_m == 'a':
+        login_hint = _getValidateLoginHint()
+        _, httpObj = getCRMService(login_hint)
+        enableGAMProjectAPIs(GAMProjectAPIs,
+                httpObj,
+                projectId=projectId,
+                checkEnabled=True)
+    else:
+        chunk_size = 20
+        print('Using an account with project access, please use ALL of these URLs to enable 20 APIs at a time:\n\n')
+        for chunk in range(0, len(GAMProjectAPIs), chunk_size):
+            apiid = ",".join(GAMProjectAPIs[chunk:chunk+chunk_size])
+            url = f'https://console.cloud.google.com/apis/enableflow?apiid={apiid}&project={projectId}'
+            print(f'    {url}\n\n')
+
+
 def enableGAMProjectAPIs(GAMProjectAPIs,
                          httpObj,
                          projectId,
@@ -7824,8 +7872,7 @@ def doShowServiceAccountKeys():
         else:
             controlflow.invalid_argument_exit(myarg, 'gam show sakeys')
     name = f'projects/-/serviceAccounts/{GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID]}'
-    currentPrivateKeyId = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA][
-        'private_key_id']
+    currentPrivateKeyId = GM_Globals[GM_OAUTH2SERVICE_JSON_DATA].get('private_key_id')
     keys = gapi.get_items(iam.projects().serviceAccounts().keys(),
                           'list',
                           'keys',
@@ -7844,6 +7891,32 @@ def doShowServiceAccountKeys():
     display.print_json(keys)
 
 
+def create_signjwt_serviceaccount():
+    i = 3
+    if i < len(sys.argv):
+        controlflow.invalid_argument_exit(sys.argv[i], f'gam create {sys.argv[2]}')
+    _checkForExistingProjectFiles()
+    sa_info = {
+            'type': 'service_account',
+            'key_type': 'signjwt',
+            'token_uri': 'https://oauth2.googleapis.com/token'
+            }
+    try:
+        creds, sa_info['project_id'] = google.auth.default()
+    except google.auth.exceptions.DefaultCredentialsError as e:
+        controlflow.system_error_exit(2, e)
+    request = transport.create_request()
+    creds.refresh(request)
+    sa_info['client_email'] = creds.service_account_email
+    oa2 = buildGAPIObjectNoAuthentication('oauth2')
+    token_info = gapi.call(oa2, 'tokeninfo', access_token=creds.token)
+    sa_info['client_id'] = token_info['issued_to']
+    sa_output = json.dumps(sa_info, indent=4, sort_keys=True)
+    fileutils.write_file(GC_Values[GC_OAUTH2SERVICE_JSON],
+                          sa_output,
+                          continue_on_error=False)
+
+
 def doCreateOrRotateServiceAccountKeys(iam=None,
                                        project_id=None,
                                        client_email=None,
@@ -11564,6 +11637,8 @@ def ProcessGAMCommand(args):
                 gapi_chat.create_message()
             elif argument in ['caalevel']:
                 gapi_caa.create_access_level()
+            elif argument in ['gcpserviceaccount', 'signjwtserviceaccount']:
+                create_signjwt_serviceaccount()
             else:
                 controlflow.invalid_argument_exit(argument, 'gam create')
             sys.exit(0)
@@ -12098,6 +12173,11 @@ def ProcessGAMCommand(args):
                 yk.serial_number = yk.get_serial_number()
                 yk.reset_piv()
             sys.exit(0)
+        elif command == 'enable':
+            enable_what = sys.argv[2].lower().replace('_', '')
+            if enable_what in ['api', 'apis']:
+                enable_apis()
+            sys.exit(0)
         users = getUsersToModify()
         command = sys.argv[3].lower()
         if command == 'print' and len(sys.argv) == 4:

src/gam/auth/__init__.py

@@ -9,6 +9,7 @@ import gam
 from gam import utils
 
 from gam.auth import oauth
+from gam.auth import signjwt
 from gam.var import _FN_OAUTH2_TXT
 from gam.var import _FN_OAUTH2SERVICE_JSON
 from gam.var import GC_OAUTH2_TXT
@@ -40,7 +41,7 @@ def get_admin_credentials(api=None):
     with open(credential_file) as f:
         creds_data = json.load(f)
     # Validate that enable DASA matches content of authorization file
-    if GC_Values[GC_ENABLE_DASA] and 'private_key_id' in creds_data:
+    if GC_Values[GC_ENABLE_DASA] and 'key_type' in creds_data:
         audience = f'https://{api}.googleapis.com/'
         key_type = creds_data.get('key_type', 'default')
         if key_type == 'default':
@@ -51,6 +52,11 @@ def get_admin_credentials(api=None):
             return JWTCredentials._from_signer_and_info(yksigner,
                 creds_data,
                 audience=audience)
+        elif key_type == 'signjwt':
+            sjsigner = signjwt.SignJwt(creds_data)
+            return signjwt.JWTCredentials._from_signer_and_info(sjsigner,
+                    creds_data,
+                    audience=audience)
     elif not GC_Values[GC_ENABLE_DASA] and 'token' in creds_data:
         return oauth.Credentials.from_credentials_file(credential_file)
     else:

src/gam/auth/signjwt.py

@@ -0,0 +1,89 @@
+''' Use Google Application Default Credentials '''
+import datetime
+import json
+
+import google.auth
+from google.auth._helpers import datetime_to_secs, scopes_to_string, utcnow
+import google.oauth2.service_account
+
+import gam
+from gam import controlflow
+from gam import gapi
+from gam import transport
+from gam.var import GM_Globals, GM_CACHE_DIR
+
+_DEFAULT_TOKEN_LIFETIME_SECS = 3600  # 1 hour in seconds
+_GOOGLE_OAUTH2_TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
+
+
+class JWTCredentials(google.auth.jwt.Credentials):
+    ''' Class used for DASA '''
+    def _make_jwt(self):
+        now = utcnow()
+        lifetime = datetime.timedelta(seconds=self._token_lifetime)
+        expiry = now + lifetime
+        payload = {
+            "iss": self._issuer,
+            "sub": self._subject,
+            "iat": datetime_to_secs(now),
+            "exp": datetime_to_secs(expiry),
+        }
+        if self._audience:
+            payload["aud"] = self._audience
+        payload.update(self._additional_claims)
+        jwt = self._signer.sign(payload)
+        return jwt, expiry
+
+
+class Credentials(google.oauth2.service_account.Credentials):
+    ''' Class used for DwD '''
+
+    def _make_authorization_grant_assertion(self):
+        now = utcnow()
+        lifetime = datetime.timedelta(seconds=_DEFAULT_TOKEN_LIFETIME_SECS)
+        expiry = now + lifetime
+        payload = {
+            "iat": datetime_to_secs(now),
+            "exp": datetime_to_secs(expiry),
+            "iss": self._service_account_email,
+            "aud": _GOOGLE_OAUTH2_TOKEN_ENDPOINT,
+            "scope": scopes_to_string(self._scopes or ()),
+        }
+
+        payload.update(self._additional_claims)
+
+        # The subject can be a user email for domain-wide delegation.
+        if self._subject:
+            payload.setdefault("sub", self._subject)
+        token = self._signer(payload)
+        return token
+
+
+class SignJwt(google.auth.crypt.Signer):
+    ''' Signer class for SignJWT '''
+    def __init__(self, service_account_info):
+        self.service_account_email = service_account_info['client_email']
+        self.name = f'projects/-/serviceAccounts/{self.service_account_email}'
+        self._key_id = None
+
+    @property  # type: ignore
+    def key_id(self):
+        return self._key_id
+
+
+    def sign(self, message):
+        ''' Call IAM Credentials SignJWT API to get our signed JWT '''
+        try:
+            credentials, _ = google.auth.default()
+        except google.auth.exceptions.DefaultCredentialsError as e:
+            controlflow.system_error_exit(2, e)
+        httpObj = transport.AuthorizedHttp(
+                credentials,
+                transport.create_http(cache=GM_Globals[GM_CACHE_DIR]))
+        iamc = gam.getService('iamcredentials', httpObj)
+        response = gapi.call(iamc.projects().serviceAccounts(),
+                             'signJwt',
+                             name=self.name,
+                             body={'payload': json.dumps(message)})
+        signed_jwt = response.get('signedJwt')
+        return signed_jwt

src/gam/var.py

@@ -8,7 +8,7 @@ import platform
 import re
 
 GAM_AUTHOR = 'Jay Lee <jay0lee@gmail.com>'
-GAM_VERSION = '6.42'
+GAM_VERSION = '6.50'
 GAM_LICENSE = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://jaylee.us/gam'