deepblue/GoogleDriveManagement
1
0
Fork 0
mirror of https://github.com/GAM-team/GAM.git synced 2026-07-05 05:11:35 +00:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

Phase 3 complete

Browse Source
This commit is contained in:
Jay Lee
2026-07-04 05:37:34 -04:00
parent 5b27b7b875
commit c41149bedb
58 changed files with 1716 additions and 1983 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/gam/util/access.py
View File

@@ -14,9 +14,14 @@ from gamlib import glgapi as GAPI
from gamlib import glglobals as GM
from gamlib import glindent
from gamlib import glmsgs as Msg
from gam.constants import API_ACCESS_DENIED_RC, INVALID_DOMAIN_RC
from util.api import _getAdminEmail, _getSvcAcctData, buildGAPIObject, callGAPI
from util.args import getEmailAddressDomain, getPhraseDNEorSNA
from util.display import ENTITY_DOES_NOT_EXIST_RC, ENTITY_DUPLICATE_RC, entityActionFailedWarning, entityDoesNotExistWarning, entityServiceNotApplicableWarning
from util.errors import OAUTH2SERVICE_JSON_REQUIRED_RC
from util.output import currentCountNL, formatKeyValueList, setSysExitRC, stderrErrorMsg, systemErrorExit, writeStderr
_gam = lambda: sys.modules['gam']
Act = glaction.GamAction()
@@ -27,41 +32,41 @@ Ind = glindent.GamIndent()
# Something's wrong with CustomerID??
def accessErrorMessage(cd, errMsg=None):
if cd is None:
cd = _gam().buildGAPIObject(API.DIRECTORY)
cd = buildGAPIObject(API.DIRECTORY)
try:
_gam().callGAPI(cd.customers(), 'get',
callGAPI(cd.customers(), 'get',
throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND,
GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
customerKey=GC.Values[GC.CUSTOMER_ID], fields='id')
except (GAPI.badRequest, GAPI.invalidInput):
return _gam().formatKeyValueList('',
return formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
Msg.INVALID],
'')
except GAPI.resourceNotFound:
return _gam().formatKeyValueList('',
return formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
Msg.DOES_NOT_EXIST],
'')
except (GAPI.forbidden, GAPI.permissionDenied):
return _gam().formatKeyValueList('',
return formatKeyValueList('',
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
Ent.DOMAIN, GC.Values[GC.DOMAIN],
Ent.USER, GM.Globals[GM.ADMIN]])+[Msg.ACCESS_FORBIDDEN],
'')
if errMsg:
return _gam().formatKeyValueList('',
return formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
errMsg],
'')
return None
def accessErrorExit(cd, errMsg=None):
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, accessErrorMessage(cd or _gam().buildGAPIObject(API.DIRECTORY), errMsg))
systemErrorExit(INVALID_DOMAIN_RC, accessErrorMessage(cd or buildGAPIObject(API.DIRECTORY), errMsg))
def accessErrorExitNonDirectory(api, errMsg):
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC,
_gam().formatKeyValueList('',
systemErrorExit(API_ACCESS_DENIED_RC,
formatKeyValueList('',
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
Ent.DOMAIN, GC.Values[GC.DOMAIN],
Ent.API, api])+[errMsg],
@@ -69,82 +74,82 @@ def accessErrorExitNonDirectory(api, errMsg):
def ClientAPIAccessDeniedExit(errMsg=None):
if errMsg is None:
_gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
stderrErrorMsg(Msg.API_ACCESS_DENIED)
missingScopes = API.getClientScopesSet(GM.Globals[GM.CURRENT_CLIENT_API])-GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
if missingScopes:
_gam().writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
','.join(sorted(missingScopes))))
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
systemErrorExit(API_ACCESS_DENIED_RC, None)
else:
_gam().stderrErrorMsg(errMsg)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
stderrErrorMsg(errMsg)
systemErrorExit(API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
def SvcAcctAPIAccessDenied():
_gam()._getSvcAcctData()
_getSvcAcctData()
if (GM.Globals[GM.CURRENT_SVCACCT_API] == API.GMAIL and
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] and
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES][0] == API.GMAIL_SEND_SCOPE):
_gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
_gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
systemErrorExit(OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
stderrErrorMsg(Msg.API_ACCESS_DENIED)
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
_gam().writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
apiOrScopes,
GM.Globals[GM.CURRENT_SVCACCT_USER] or _gam()._getAdminEmail()))
GM.Globals[GM.CURRENT_SVCACCT_USER] or _getAdminEmail()))
def SvcAcctAPIAccessDeniedExit():
SvcAcctAPIAccessDenied()
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
systemErrorExit(API_ACCESS_DENIED_RC, None)
def SvcAcctAPIDisabledExit():
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
ClientAPIAccessDeniedExit()
if GM.Globals[GM.CURRENT_SVCACCT_API]:
_gam().stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
systemErrorExit(API_ACCESS_DENIED_RC, None)
systemErrorExit(API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
def APIAccessDeniedExit():
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
ClientAPIAccessDeniedExit()
if GM.Globals[GM.CURRENT_SVCACCT_API]:
SvcAcctAPIAccessDeniedExit()
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
systemErrorExit(API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
def checkEntityDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
message = accessErrorMessage(cd)
if message:
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_gam().entityDoesNotExistWarning(entityType, entityName, i, count)
systemErrorExit(INVALID_DOMAIN_RC, message)
entityDoesNotExistWarning(entityType, entityName, i, count)
def checkEntityAFDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
message = accessErrorMessage(cd)
if message:
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_gam().entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
systemErrorExit(INVALID_DOMAIN_RC, message)
entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
def checkEntityItemValueAFDNEorAccessErrorExit(cd, entityType, entityName, itemType, itemValue, i=0, count=0):
message = accessErrorMessage(cd)
if message:
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_gam().entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
systemErrorExit(INVALID_DOMAIN_RC, message)
entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
def entityUnknownWarning(entityType, entityName, i=0, count=0):
domain = _gam().getEmailAddressDomain(entityName)
domain = getEmailAddressDomain(entityName)
if (domain.endswith(GC.Values[GC.DOMAIN])) or (domain.endswith('google.com')):
_gam().entityDoesNotExistWarning(entityType, entityName, i, count)
entityDoesNotExistWarning(entityType, entityName, i, count)
else:
_gam().entityServiceNotApplicableWarning(entityType, entityName, i, count)
entityServiceNotApplicableWarning(entityType, entityName, i, count)
def entityOrEntityUnknownWarning(entity1Type, entity1Name, entity2Type, entity2Name, i=0, count=0):
_gam().setSysExitRC(_gam().ENTITY_DOES_NOT_EXIST_RC)
_gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _gam().getPhraseDNEorSNA(entity1Name), None,
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _gam().getPhraseDNEorSNA(entity2Name)],
_gam().currentCountNL(i, count)))
setSysExitRC(ENTITY_DOES_NOT_EXIST_RC)
writeStderr(formatKeyValueList(Ind.Spaces(),
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, getPhraseDNEorSNA(entity1Name), None,
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, getPhraseDNEorSNA(entity2Name)],
currentCountNL(i, count)))
def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
email = entityValueList[1]
try:
result = _gam().callGAPI(cd.users(), 'get',
result = callGAPI(cd.users(), 'get',
throwReasons=GAPI.USER_GET_THROW_REASONS,
userKey=email, fields='id,primaryEmail')
if (result['primaryEmail'].lower() == email) or (result['id'] == email):
@@ -154,7 +159,7 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
except (GAPI.userNotFound, GAPI.badRequest,
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.backendError, GAPI.systemError):
try:
result = _gam().callGAPI(cd.groups(), 'get',
result = callGAPI(cd.groups(), 'get',
throwReasons=GAPI.GROUP_GET_THROW_REASONS,
groupKey=email, fields='id,email')
if (result['email'].lower() == email) or (result['id'] == email):
@@ -164,10 +169,10 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
except (GAPI.groupNotFound,
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.badRequest):
kvList = [Ent.EMAIL, email]
_gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
writeStderr(formatKeyValueList(Ind.Spaces(),
Ent.FormatEntityValueList(entityValueList)+
[Act.Failed(), Msg.DUPLICATE]+
Ent.FormatEntityValueList(kvList),
_gam().currentCountNL(i, count)))
_gam().setSysExitRC(_gam().ENTITY_DUPLICATE_RC)
currentCountNL(i, count)))
setSysExitRC(ENTITY_DUPLICATE_RC)
return kvList[0]

265
src/gam/util/api.py
View File

@@ -47,6 +47,15 @@ from gamlib import glgdata as GDATA
from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
from gamlib import yubikey
from gam.constants import API_ACCESS_DENIED_RC, GOOGLE_API_ERROR_RC, HTTP_ERROR_RC, NETWORK_ERROR_RC, NO_SCOPES_FOR_API_RC, REFRESH_EXPIRY, SOCKET_ERROR_RC, SYSTEM_ERROR_RC
from util.args import UTF8, YYYYMMDDTHHMMSSZ_FORMAT, formatHTTPError
from util.display import FIRST_ITEM_MARKER, LAST_ITEM_MARKER, SERVICE_NOT_APPLICABLE_RC, TOTAL_ITEMS_MARKER, entityActionFailedWarning, printBlankLine, printKeyValueList, userServiceNotEnabledWarning
from util.errors import INVALID_JSON_RC, OAUTH2SERVICE_JSON_REQUIRED_RC, OAUTH2_TXT_REQUIRED_RC, expiredRevokedOauth2TxtExit, invalidDiscoveryJsonExit, invalidOauth2TxtExit, invalidOauth2serviceJsonExit
from util.fileio import FILE_ERROR_RC, UNKNOWN, checkAPICallsRate, incrAPICallsRetryData, readFile, writeFile
from util.output import ERROR_PREFIX, flushStderr, setSysExitRC, stderrErrorMsg, systemErrorExit, writeStderr, writeStdout
HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>')
from gam.constants import GAM_LATEST_RELEASE, GAM_USER_AGENT, __author__, __version__
# Constants only used in this module
@@ -56,7 +65,6 @@ DEVELOPER_PREVIEW_DISCOVERY_URI = "https://{api}.googleapis.com/$discovery/rest?
_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds
_gam = lambda: sys.modules['gam']
def _getEnt():
return sys.modules['gam'].Ent
@@ -68,10 +76,10 @@ def _getInd():
def handleServerError(e):
errMsg = str(e)
if 'setting tls' not in errMsg:
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, errMsg)
_gam().stderrErrorMsg(errMsg)
_gam().writeStderr(Msg.DISABLE_TLS_MIN_MAX)
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, None)
systemErrorExit(NETWORK_ERROR_RC, errMsg)
stderrErrorMsg(errMsg)
writeStderr(Msg.DISABLE_TLS_MIN_MAX)
systemErrorExit(NETWORK_ERROR_RC, None)
def getHttpObj(cache=None, timeout=None, override_min_tls=None, override_max_tls=None):
tls_minimum_version = override_min_tls if override_min_tls else GC.Values[GC.TLS_MIN_VERSION] if GC.Values[GC.TLS_MIN_VERSION] else None
@@ -111,7 +119,7 @@ def _force_user_agent(user_agent):
def _lazy_force_user_agent(request_method):
"""Wraps a request method to lazily insert GAM_USER_AGENT at call time."""
def wrapped_request_method(*args, **kwargs):
user_agent = _gam().GAM_USER_AGENT
user_agent = GAM_USER_AGENT
if kwargs.get('headers') is not None:
if kwargs['headers'].get('user-agent'):
if user_agent not in kwargs['headers']['user-agent']:
@@ -158,10 +166,10 @@ def doGAMCheckForUpdates(forceCheck):
Ind = _getInd()
def _gamLatestVersionNotAvailable():
if forceCheck:
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
systemErrorExit(NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
try:
_, c = getHttpObj(timeout=10).request(_gam().GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
_, c = getHttpObj(timeout=10).request(GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
try:
release_data = json.loads(c)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
@@ -170,17 +178,17 @@ def doGAMCheckForUpdates(forceCheck):
if not isinstance(release_data, dict) or 'tag_name' not in release_data:
_gamLatestVersionNotAvailable()
return
current_version = _gam().__version__
current_version = __version__
latest_version = release_data['tag_name']
if latest_version[0].lower() == 'v':
latest_version = latest_version[1:]
_gam().printKeyValueList(['Version Check', None])
printKeyValueList(['Version Check', None])
Ind.Increment()
_gam().printKeyValueList(['Current', current_version])
_gam().printKeyValueList([' Latest', latest_version])
printKeyValueList(['Current', current_version])
printKeyValueList([' Latest', latest_version])
Ind.Decrement()
if forceCheck < 0:
_gam().setSysExitRC(1 if latest_version > current_version else 0)
setSysExitRC(1 if latest_version > current_version else 0)
return
except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError,
google.auth.exceptions.TransportError,
@@ -252,7 +260,7 @@ class signjwtSignJwt(google.auth.crypt.Signer):
credentials, _ = google.auth.default(scopes=[API.IAM_SCOPE],
request=request)
except (google.auth.exceptions.DefaultCredentialsError, google.auth.exceptions.RefreshError) as e:
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, str(e))
systemErrorExit(API_ACCESS_DENIED_RC, str(e))
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj())
# refresh here so we can use the proper request from above
httpObj.credentials.refresh(request)
@@ -263,40 +271,41 @@ class signjwtSignJwt(google.auth.crypt.Signer):
return signed_jwt
def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0):
from util.access import APIAccessDeniedExit, ClientAPIAccessDeniedExit, SvcAcctAPIAccessDeniedExit
Ent = _getEnt()
errMsg = str(e).replace('.', '')
if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or
errMsg.startswith('Invalid response') or
errMsg.startswith('invalid_request: Invalid impersonation &quot;sub&quot; field')):
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
_gam().ClientAPIAccessDeniedExit()
ClientAPIAccessDeniedExit()
# 403 Forbidden, API disabled, user not enabled
# 400 Bad Request, user not defined
if softErrors:
_gam().entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
return None
_gam().systemErrorExit(_gam().SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
systemErrorExit(SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
if errMsg in API.OAUTH2_UNAUTHORIZED_ERRORS:
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
_gam().ClientAPIAccessDeniedExit()
ClientAPIAccessDeniedExit()
# 401 Unauthorized, API disabled, user enabled
if softErrors:
if displayError:
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
_gam().userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
return None
_gam().SvcAcctAPIAccessDeniedExit()
SvcAcctAPIAccessDeniedExit()
if errMsg in API.REFRESH_PERM_ERRORS:
if softErrors:
return None
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
_gam().expiredRevokedOauth2TxtExit()
_gam().stderrErrorMsg(f'Authentication Token Error - {errMsg}')
_gam().APIAccessDeniedExit()
expiredRevokedOauth2TxtExit()
stderrErrorMsg(f'Authentication Token Error - {errMsg}')
APIAccessDeniedExit()
def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnly=False, noScopes=False):
if not noDASA and GC.Values[GC.ENABLE_DASA]:
jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
jsonData = readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
if jsonData:
try:
if api in API.APIS_NEEDING_ACCESS_TOKEN:
@@ -314,9 +323,9 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
sjsigner = signjwtSignJwt(jsonDict)
return (True, signjwtJWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience))
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
_gam().invalidOauth2serviceJsonExit(str(e))
_gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
jsonData = _gam().readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
invalidOauth2serviceJsonExit(str(e))
invalidOauth2serviceJsonExit(Msg.NO_DATA)
jsonData = readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
if jsonData:
try:
jsonDict = json.loads(jsonData)
@@ -326,11 +335,11 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
if not refreshOnly:
if set(jsonDict.get('scopes', API.REQUIRED_SCOPES)) == API.REQUIRED_SCOPES_SET:
if exitOnError:
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
systemErrorExit(OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
return (False, None)
else:
GM.Globals[GM.CREDENTIALS_SCOPES] = set(jsonDict.pop('scopes', API.REQUIRED_SCOPES))
token_expiry = jsonDict.get('token_expiry', _gam().REFRESH_EXPIRY)
token_expiry = jsonDict.get('token_expiry', REFRESH_EXPIRY)
if GC.Values[GC.TRUNCATE_CLIENT_ID]:
# chop off .apps.googleusercontent.com suffix as it's not needed and we need to keep things short for the Auth URL.
jsonDict['client_id'] = re.sub(r'\.apps\.googleusercontent\.com$', '', jsonDict['client_id'])
@@ -343,15 +352,15 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
creds.token = jsonDict['access_token']
creds._id_token = jsonDict['id_token_jwt']
GM.Globals[GM.DECODED_ID_TOKEN] = jsonDict['id_token']
creds.expiry = arrow.Arrow.strptime(token_expiry, _gam().YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
creds.expiry = arrow.Arrow.strptime(token_expiry, YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
return (not noScopes, creds)
if jsonDict and exitOnError:
_gam().invalidOauth2TxtExit(Msg.INVALID)
invalidOauth2TxtExit(Msg.INVALID)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
if exitOnError:
_gam().invalidOauth2TxtExit(str(e))
invalidOauth2TxtExit(str(e))
if exitOnError:
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
systemErrorExit(OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
return (False, None)
def _getValueFromOAuth(field, credentials=None):
@@ -366,9 +375,9 @@ def _getValueFromOAuth(field, credentials=None):
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
except ValueError as e:
if 'Token used too early' in str(e):
_gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, _gam().UNKNOWN)
stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
systemErrorExit(SYSTEM_ERROR_RC, str(e))
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, UNKNOWN)
def _getAdminEmail():
if GC.Values[GC.ADMIN_EMAIL]:
@@ -383,25 +392,25 @@ def writeClientCredentials(creds, filename):
'refresh_token': creds.refresh_token,
'scopes': sorted(creds.scopes or GM.Globals[GM.CREDENTIALS_SCOPES]),
'token': creds.token,
'token_expiry': creds.expiry.strftime(_gam().YYYYMMDDTHHMMSSZ_FORMAT),
'token_expiry': creds.expiry.strftime(YYYYMMDDTHHMMSSZ_FORMAT),
'token_uri': creds.token_uri,
}
expected_iss = ['https://accounts.google.com', 'accounts.google.com']
if _getValueFromOAuth('iss', creds) not in expected_iss:
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
systemErrorExit(OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
request = transportCreateRequest()
try:
creds_data['decoded_id_token'] = google.oauth2.id_token.verify_oauth2_token(creds.id_token, request,
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
except ValueError as e:
if 'Token used too early' in str(e):
_gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
systemErrorExit(SYSTEM_ERROR_RC, str(e))
GM.Globals[GM.DECODED_ID_TOKEN] = creds_data['decoded_id_token']
if filename != '-':
_gam().writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
else:
_gam().writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
def shortenURL(long_url):
if GC.Values[GC.NO_SHORT_URLS]:
@@ -412,7 +421,7 @@ def shortenURL(long_url):
resp, content = httpObj.request(URL_SHORTENER_ENDPOINT, 'POST',
payload,
headers={'Content-Type': 'application/json',
'User-Agent': _gam().GAM_USER_AGENT})
'User-Agent': GAM_USER_AGENT})
except:
return long_url
if resp.status != 200:
@@ -432,11 +441,11 @@ def runSqliteQuery(db_file, query):
def refreshCredentialsWithReauth(credentials):
def gcloudError():
_gam().writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
e = Msg.REAUTHENTICATION_IS_NEEDED
handleOAuthTokenError(e, False)
_gam().writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
if 'termios' in sys.modules:
import termios
old_settings = termios.tcgetattr(sys.stdin)
@@ -444,7 +453,7 @@ def refreshCredentialsWithReauth(credentials):
# First makes sure gcloud has a valid access token and thus
# should also have a valid RAPT token
try:
devnull = open(os.devnull, 'w', encoding=_gam().UTF8)
devnull = open(os.devnull, 'w', encoding=UTF8)
subprocess.run(['gcloud',
'auth',
'print-identity-token',
@@ -462,7 +471,7 @@ def refreshCredentialsWithReauth(credentials):
if 'termios' in sys.modules:
import termios
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
_gam().printBlankLine()
printBlankLine()
raise KeyboardInterrupt from e
token_path = gcloud_path_result.stdout.decode().strip()
if not token_path:
@@ -474,7 +483,7 @@ def refreshCredentialsWithReauth(credentials):
except TypeError:
gcloudError()
if not credentials._rapt_token:
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC,
systemErrorExit(SYSTEM_ERROR_RC,
'Failed to retrieve reauth token from gcloud. You may need to wait until gcloud is also prompted for reauth.')
def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, api=None, noDASA=False, refreshOnly=False, noScopes=False):
@@ -485,7 +494,7 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
with lock:
writeCreds, credentials = getOauth2TxtCredentials(api=api, noDASA=noDASA, refreshOnly=refreshOnly, noScopes=noScopes)
if not credentials:
_gam().invalidOauth2TxtExit('')
invalidOauth2TxtExit('')
if credentials.expired or forceRefresh:
triesLimit = 3
for n in range(1, triesLimit+1):
@@ -496,7 +505,7 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
break
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -513,11 +522,11 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
def waitOnFailure(n, triesLimit, error_code, error_message):
delta = min(2 ** n, 60)+float(random.randint(1, 1000))/1000
if n > 3:
_gam().writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
_gam().flushStderr()
writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
flushStderr()
time.sleep(delta)
if GC.Values[GC.SHOW_API_CALLS_RETRY_DATA]:
_gam().incrAPICallsRetryData(error_message, delta)
incrAPICallsRetryData(error_message, delta)
def clearServiceCache(service):
if hasattr(service._http, 'http') and hasattr(service._http.http, 'cache'):
@@ -564,22 +573,22 @@ def getService(api, httpObj):
clearServiceCache(service)
return service
except googleapiclient.errors.UnknownApiNameOrVersion as e:
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), _gam().__author__))
systemErrorExit(GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), __author__))
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
if n != triesLimit:
waitOnFailure(n, triesLimit, _gam().INVALID_JSON_RC, str(e))
waitOnFailure(n, triesLimit, INVALID_JSON_RC, str(e))
continue
_gam().systemErrorExit(_gam().INVALID_JSON_RC, str(e))
systemErrorExit(INVALID_JSON_RC, str(e))
except (http.client.ResponseNotReady, OSError, googleapiclient.errors.HttpError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit:
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg)
continue
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
systemErrorExit(SOCKET_ERROR_RC, errMsg)
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
httpObj.connections = {}
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
disc_file, discovery = readDiscoveryFile(f'{api}-{version}')
@@ -591,9 +600,9 @@ def getService(api, httpObj):
clearServiceCache(service)
return service
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
_gam().invalidDiscoveryJsonExit(disc_file, str(e))
invalidDiscoveryJsonExit(disc_file, str(e))
except IOError as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, str(e))
systemErrorExit(FILE_ERROR_RC, str(e))
def defaultSvcAcctScopes():
scopesList = API.getSvcAcctScopesList(GC.Values[GC.USER_SERVICE_ACCOUNT_ACCESS_ONLY], False)
@@ -610,15 +619,15 @@ def defaultSvcAcctScopes():
def _getSvcAcctData():
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
jsonData = readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
if not jsonData:
_gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
invalidOauth2serviceJsonExit(Msg.NO_DATA)
try:
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = json.loads(jsonData)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
_gam().invalidOauth2serviceJsonExit(str(e))
invalidOauth2serviceJsonExit(str(e))
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
_gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
systemErrorExit(OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
requiredFields = ['client_email', 'client_id', 'project_id', 'token_uri']
key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
if key_type == 'default':
@@ -628,7 +637,7 @@ def _getSvcAcctData():
if field not in GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
missingFields.append(field)
if missingFields:
_gam().invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
# Some old oauth2service.json files have: 'https://accounts.google.com/o/oauth2/auth' which no longer works
if GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] == 'https://accounts.google.com/o/oauth2/auth':
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] = API.GOOGLE_OAUTH2_TOKEN_ENDPOINT
@@ -640,6 +649,7 @@ def _getSvcAcctData():
GM.Globals[GM.SVCACCT_SCOPES] = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].pop(API.OAUTH2SA_SCOPES)
def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=False):
from util.access import SvcAcctAPIAccessDeniedExit
_getSvcAcctData()
if isinstance(scopesOrAPI, str):
GM.Globals[GM.CURRENT_SVCACCT_API] = scopesOrAPI
@@ -650,7 +660,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
if scopesOrAPI != API.CHAT_EVENTS and not GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]:
if softErrors:
return None
_gam().SvcAcctAPIAccessDeniedExit()
SvcAcctAPIAccessDeniedExit()
if scopesOrAPI in {API.PEOPLE, API.PEOPLE_DIRECTORY, API.PEOPLE_OTHERCONTACTS}:
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES].append(API.USERINFO_PROFILE_SCOPE)
if scopesOrAPI in {API.PEOPLE_OTHERCONTACTS}:
@@ -677,7 +687,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
except (ValueError, IndexError, KeyError) as e:
if softErrors:
return None
_gam().invalidOauth2serviceJsonExit(str(e))
invalidOauth2serviceJsonExit(str(e))
credentials = credentials.with_scopes(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])
else:
audience = f'https://{scopesOrAPI}.googleapis.com/'
@@ -699,7 +709,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
except (ValueError, IndexError, KeyError) as e:
if softErrors:
return None
_gam().invalidOauth2serviceJsonExit(str(e))
invalidOauth2serviceJsonExit(str(e))
GM.Globals[GM.CURRENT_SVCACCT_USER] = userEmail
if userEmail:
credentials = credentials.with_subject(userEmail)
@@ -726,13 +736,13 @@ def getGDataOAuthToken(gdataObj, credentials=None):
GM.Globals[GM.ADMIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('email', 'UNKNOWN').lower()
GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id
gdataObj.domain = GC.Values[GC.DOMAIN]
gdataObj.source = _gam().GAM_USER_AGENT
gdataObj.source = GAM_USER_AGENT
return True
def checkGDataError(e, service):
error = e.args
reason = error[0].get('reason', '')
body = error[0].get('body', '').decode(_gam().UTF8)
body = error[0].get('body', '').decode(UTF8)
# First check for errors that need special handling
if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']:
keep_domain = service.domain
@@ -741,7 +751,7 @@ def checkGDataError(e, service):
return (GDATA.TOKEN_EXPIRED, reason)
error_code = getattr(e, 'error_code', 600)
if GC.Values[GC.DEBUG_LEVEL] > 0:
_gam().writeStdout(f'{_gam().ERROR_PREFIX} {error_code}: {reason}, {body}\n')
writeStdout(f'{ERROR_PREFIX} {error_code}: {reason}, {body}\n')
if error_code == 600:
if (body.startswith('Quota exceeded for the current request') or
body.startswith('Quota exceeded for quota metric') or
@@ -842,6 +852,7 @@ def callGData(service, function,
bailOnInternalServerError=False, softErrors=False,
throwErrors=None, retryErrors=None, triesLimit=0,
**kwargs):
from util.access import APIAccessDeniedExit
if throwErrors is None:
throwErrors = []
if retryErrors is None:
@@ -851,7 +862,7 @@ def callGData(service, function,
allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors
method = getattr(service, function)
if GC.Values[GC.API_CALLS_RATE_CHECK]:
_gam().checkAPICallsRate()
checkAPICallsRate()
for n in range(1, triesLimit+1):
try:
return method(**kwargs)
@@ -868,14 +879,14 @@ def callGData(service, function,
raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message)
raise
if softErrors:
_gam().stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
return None
if error_code == GDATA.INSUFFICIENT_PERMISSIONS:
_gam().APIAccessDeniedExit()
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
APIAccessDeniedExit()
systemErrorExit(GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -886,25 +897,25 @@ def callGData(service, function,
except (http.client.ResponseNotReady, OSError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit:
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg)
continue
if softErrors:
_gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
writeStderr(f'\n{ERROR_PREFIX}{errMsg} - Giving up.\n')
return None
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
systemErrorExit(SOCKET_ERROR_RC, errMsg)
def writeGotMessage(msg):
if GC.Values[GC.SHOW_GETTINGS_GOT_NL]:
_gam().writeStderr(msg)
writeStderr(msg)
else:
_gam().writeStderr('\r')
writeStderr('\r')
msgLen = len(msg)
if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]:
_gam().writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
else:
_gam().writeStderr(msg)
writeStderr(msg)
GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen
_gam().flushStderr()
flushStderr()
def callGDataPages(service, function,
pageMessage=None,
@@ -935,12 +946,12 @@ def callGDataPages(service, function,
nextLink = None
pageItems = 0
if pageMessage:
show_message = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
show_message = pageMessage.replace(TOTAL_ITEMS_MARKER, str(totalItems))
writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems)))
if nextLink is None:
if pageMessage and (pageMessage[-1] != '\n'):
_gam().writeStderr('\r\n')
_gam().flushStderr()
writeStderr('\r\n')
flushStderr()
return allResults
uri = nextLink.href
if 'url_params' in kwargs:
@@ -951,14 +962,14 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}}
try:
error = json.loads(e.content.decode(_gam().UTF8))
error = json.loads(e.content.decode(UTF8))
if GC.Values[GC.DEBUG_LEVEL] > 0:
_gam().writeStdout(f'{_gam().ERROR_PREFIX} JSON: {str(error)}\n')
writeStdout(f'{ERROR_PREFIX} JSON: {str(error)}\n')
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
eContent = e.content.decode(UTF8) if isinstance(e.content, bytes) else e.content
lContent = eContent.lower()
if GC.Values[GC.DEBUG_LEVEL] > 0:
_gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
writeStdout(f'{ERROR_PREFIX} HTTP: {str(eContent)}\n')
if eContent[0:15] != '<!DOCTYPE html>':
if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')):
return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent)
@@ -971,7 +982,7 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
if (e.resp['status'] == '504') and ('gateway timeout' in lContent):
return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent)
else:
tg = _gam().HTML_TITLE_PATTERN.match(lContent)
tg = HTML_TITLE_PATTERN.match(lContent)
lContent = tg.group(1) if tg else 'bad request'
if (e.resp['status'] == '403') and ('invalid domain.' in lContent):
error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found')
@@ -998,10 +1009,10 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
elif retryOnHttpError:
return (-1, None, eContent)
elif softErrors:
_gam().stderrErrorMsg(eContent)
stderrErrorMsg(eContent)
return (0, None, None)
else:
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, eContent)
systemErrorExit(HTTP_ERROR_RC, eContent)
requiredScopes = ''
wwwAuthenticate = e.resp.get('www-authenticate', '')
if 'insufficient_scope' in wwwAuthenticate:
@@ -1107,9 +1118,9 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
http_status = 400
error = makeErrorDict(http_status, GAPI.INVALID, message)
else:
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
systemErrorExit(GOOGLE_API_ERROR_RC, str(error))
else:
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
systemErrorExit(GOOGLE_API_ERROR_RC, str(error))
try:
reason = error['error']['errors'][0]['reason']
for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []):
@@ -1135,6 +1146,7 @@ def callGAPI(service, function,
softErrors=False, mapNotFound=True,
throwReasons=None, retryReasons=None, triesLimit=0,
**kwargs):
from util.access import APIAccessDeniedExit
if throwReasons is None:
throwReasons = []
if retryReasons is None:
@@ -1145,7 +1157,7 @@ def callGAPI(service, function,
method = getattr(service, function)
svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST])
if GC.Values[GC.API_CALLS_RATE_CHECK]:
_gam().checkAPICallsRate()
checkAPICallsRate()
for n in range(1, triesLimit+1):
try:
return method(**svcparms).execute()
@@ -1158,7 +1170,7 @@ def callGAPI(service, function,
# service._http.credentials.refresh(getHttpObj())
service._http.credentials.refresh(transportCreateRequest())
except TypeError:
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, message)
systemErrorExit(HTTP_ERROR_RC, message)
continue
if http_status == 0:
return None
@@ -1179,17 +1191,17 @@ def callGAPI(service, function,
raise GAPI.REASON_EXCEPTION_MAP[reason](message)
raise e
if softErrors:
_gam().stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
return None
if reason == GAPI.INSUFFICIENT_PERMISSIONS:
_gam().APIAccessDeniedExit()
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, _gam().formatHTTPError(http_status, reason, message))
APIAccessDeniedExit()
systemErrorExit(HTTP_ERROR_RC, formatHTTPError(http_status, reason, message))
except googleapiclient.errors.MediaUploadSizeError as e:
raise e
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
service._http.connections = {}
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -1200,22 +1212,22 @@ def callGAPI(service, function,
except (http.client.ResponseNotReady, OSError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit:
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
waitOnFailure(n, triesLimit, SOCKET_ERROR_RC, errMsg)
continue
if softErrors:
_gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
writeStderr(f'\n{ERROR_PREFIX}{errMsg} - Giving up.\n')
return None
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
systemErrorExit(SOCKET_ERROR_RC, errMsg)
except ValueError as e:
if clearServiceCache(service):
continue
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
systemErrorExit(GOOGLE_API_ERROR_RC, str(e))
except TypeError as e:
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
systemErrorExit(GOOGLE_API_ERROR_RC, str(e))
def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType):
Ent = _getEnt()
showMessage = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
showMessage = pageMessage.replace(TOTAL_ITEMS_MARKER, str(totalItems))
if pageItems:
if messageAttribute:
firstItem = results[0] if pageItems > 0 else {}
@@ -1229,11 +1241,11 @@ def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAtt
lastItem = lastItem.get(attr, {})
firstItem = str(firstItem)
lastItem = str(lastItem)
showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, firstItem)
showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, lastItem)
showMessage = showMessage.replace(FIRST_ITEM_MARKER, firstItem)
showMessage = showMessage.replace(LAST_ITEM_MARKER, lastItem)
else:
showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, '')
showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, '')
showMessage = showMessage.replace(FIRST_ITEM_MARKER, '')
showMessage = showMessage.replace(LAST_ITEM_MARKER, '')
writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems))))
def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType):
@@ -1257,8 +1269,8 @@ def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage,
def _finalizeGAPIpagesResult(pageMessage):
if pageMessage and (pageMessage[-1] != '\n'):
_gam().writeStderr('\r\n')
_gam().flushStderr()
writeStderr('\r\n')
flushStderr()
def _setMaxArgResults(maxItems, pageArgsInBody, kwargs):
if pageArgsInBody:
@@ -1381,18 +1393,18 @@ def readDiscoveryFile(api_version):
disc_filename = f'{api_version}.json'
disc_file = os.path.join(GM.Globals[GM.GAM_PATH], disc_filename)
if hasattr(sys, '_MEIPASS'):
json_string = _gam().readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
json_string = readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
elif os.path.isfile(disc_file):
json_string = _gam().readFile(disc_file, continueOnError=True, displayError=True)
json_string = readFile(disc_file, continueOnError=True, displayError=True)
else:
json_string = None
if not json_string:
_gam().invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
try:
discovery = json.loads(json_string)
return (disc_file, discovery)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
_gam().invalidDiscoveryJsonExit(disc_file, str(e))
invalidDiscoveryJsonExit(disc_file, str(e))
def buildGAPIObject(api, credentials=None):
if credentials is None:
@@ -1406,7 +1418,7 @@ def buildGAPIObject(api, credentials=None):
GM.Globals[GM.CURRENT_CLIENT_API] = api
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API_Scopes.intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
if api not in API.SCOPELESS_APIS and not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
_gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
systemErrorExit(NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
if not GC.Values[GC.DOMAIN]:
GC.Values[GC.DOMAIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('hd', 'UNKNOWN').lower()
if not GC.Values[GC.CUSTOMER_ID]:
@@ -1416,9 +1428,10 @@ def buildGAPIObject(api, credentials=None):
return service
def getSaUser(user):
from util.entity import convertUIDtoEmailAddress
currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API]
currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
userEmail = _gam().convertUIDtoEmailAddress(user) if user else None
userEmail = convertUIDtoEmailAddress(user) if user else None
GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes
return userEmail
@@ -1446,7 +1459,7 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
httpObj.connections = {}
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -1456,12 +1469,12 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
if isinstance(e, str):
eContent = e
else:
eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
eContent = e.content.decode(UTF8) if isinstance(e.content, bytes) else e.content
if eContent[0:15] == '<!DOCTYPE html>':
if GC.Values[GC.DEBUG_LEVEL] > 0:
_gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
writeStdout(f'{ERROR_PREFIX} HTTP: {str(eContent)}\n')
lContent = eContent.lower()
tg = _gam().HTML_TITLE_PATTERN.match(lContent)
tg = HTML_TITLE_PATTERN.match(lContent)
lContent = tg.group(1) if tg else ''
if lContent.startswith('Error 502 (Server Error)'):
time.sleep(30)
@@ -1479,7 +1492,7 @@ def initGDataObject(gdataObj, api):
credentials = getClientCredentials(noDASA=True, refreshOnly=True)
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API.getClientScopesSet(api).intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
if not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
_gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
systemErrorExit(NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
getGDataOAuthToken(gdataObj, credentials)
if GC.Values[GC.DEBUG_LEVEL] > 0:
gdataObj.debug = True

14
src/gam/util/args.py
View File

@@ -88,15 +88,6 @@ from gamlib import glmsgs as Msg
from gamlib import glskus as SKU
class _InstanceProxy:
"""Lazy proxy that delegates attribute access to a named instance in the gam module."""
def __init__(self, name):
self._name = name
def __getattr__(self, attr):
return getattr(getattr(sys.modules['gam'], self._name), attr)
Cmd = _InstanceProxy('Cmd')
from util.errors import (
blankArgumentExit,
csvFieldErrorExit,
@@ -109,13 +100,13 @@ from util.errors import (
)
from util.fileio import readFile
from gam.var import Cmd
# Lazy accessor for Ent (runtime instance)
def _getEnt():
return sys.modules['gam'].Ent
# Lazy accessor for main module
_gam = lambda: sys.modules['gam']
# --- Constants duplicated from __init__.py ---
# These are simple literals that never change, duplicated to avoid
@@ -147,6 +138,9 @@ SECONDS_PER_WEEK = 604800
def ISOformatTimeStamp(timestamp):
return timestamp.isoformat('T', 'seconds')
def currentISOformatTimeStamp(timespec='milliseconds'):
return arrow.now(GC.Values[GC.TIMEZONE]).isoformat('T', timespec)
def checkArgumentPresent(choices, required=False):
choiceList = choices if isinstance(choices, (list, set)) else [choices]
if Cmd.ArgumentsRemaining():

326
src/gam/util/batch.py
View File

@@ -25,11 +25,20 @@ from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
from util.csv_pf import CSVPrintFile
from gam.constants import HARD_ERROR_RC, KEYBOARD_INTERRUPT_RC
from util.api import buildGAPIObject
from util.args import UTF8, checkArgumentPresent, currentISOformatTimeStamp, checkForExtraneousArguments, checkMatchSkipFields, getBoolean, getCharSet, getDelimiter, getInteger, getMatchSkipFields, getString, normalizeEmailAddressOrUID, todaysTime
from util.csv_pf import CheckInputRowFilterHeaders
from util.display import actionPerformedNumItems
from util.entity import getEntityArgument, getEntityList, getEntityToModify
from util.errors import USAGE_ERROR_RC, csvFieldErrorExit, formatChoiceList, missingArgumentExit, usageErrorExit
from util.fileio import FILE_ERROR_RC, StringIOobject, closeFile, closeGAMCommandLog, fdErrorMessage, fileErrorMessage, openFile, openGAMCommandLog, setFilePath, writeGAMCommandLog
from util.gdoc import getGDocData, getStorageFileData, openCSVFileReader
from util.output import ERROR_PREFIX, flushStderr, flushStdout, readStdin, setSysExitRC, systemErrorExit, writeStderr, writeStdout
from gam.constants import GAM
from gam.var import Cmd, Ind
_gam = lambda: sys.modules['gam']
Cmd = glclargs.GamCLArgs()
class NullHandler(logging.Handler):
@@ -71,18 +80,18 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
def reopenSTDFile(stdtype):
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][stdtype == GM.STDOUT]),
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
elif stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
GM.Globals[stdtype][GM.REDIRECT_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_FD]
else:
GM.Globals[stdtype][GM.REDIRECT_FD] = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
GM.Globals[stdtype][GM.REDIRECT_FD] = openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
if stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
else:
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _gam().StringIOobject()
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else StringIOobject()
GM.Globals[GM.DATETIME_NOW] = datetimeNow
GC.Values[GC.TIMEZONE] = tzinfo
@@ -157,11 +166,11 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
if mpQueueStdout:
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()))
else:
_gam().flushStdout()
flushStdout()
if mpQueueStderr and mpQueueStderr is not mpQueueStdout:
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD].getvalue()))
else:
_gam().flushStderr()
flushStderr()
def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr):
mpQueue = mpManager.Queue()
@@ -177,7 +186,7 @@ def terminateCSVFileQueueHandler(mpQueue, mpQueueHandler):
GM.Globals[GM.PARSER] = None
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None
if multiprocessing.get_start_method() != 'fork':
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _gam().Cmd.AllArguments()))
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, Cmd.AllArguments()))
savedValues = saveNonPickleableValues()
mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals))
restoreNonPickleableValues(savedValues)
@@ -199,27 +208,27 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
if data[1] is not None:
_writeData(data[1])
if GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _gam().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
fd.flush()
except IOError as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
systemErrorExit(FILE_ERROR_RC, fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
if multiprocessing.get_start_method() != 'fork':
signal.signal(signal.SIGINT, signal.SIG_IGN)
GM.Globals = gmGlobals.copy()
GC.Values = gcValues.copy()
pid0DataItem = [_gam().KEYBOARD_INTERRUPT_RC, None]
pid0DataItem = [KEYBOARD_INTERRUPT_RC, None]
pidData = {}
if multiprocessing.get_start_method() != 'fork':
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
fd = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stdout']),
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout' and GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stderr':
fd = os.fdopen(os.dup(sys.stdout.fileno()), GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
else:
fd = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
fd = openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
else:
fd = GM.Globals[stdtype][GM.REDIRECT_FD]
while True:
@@ -229,7 +238,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
break
if dataType == GM.REDIRECT_QUEUE_START:
pidData[pid] = {'queue': GM.Globals[stdtype][GM.REDIRECT_QUEUE],
'start': _gam().currentISOformatTimeStamp(),
'start': currentISOformatTimeStamp(),
'cmd': Cmd.QuotedArgumentList(dataItem)}
if pid == 0 and GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
fd.write(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'Start', pidData[pid]['start'], 0, pidData[pid]['cmd']))
@@ -245,7 +254,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
break
for pid in pidData:
if pid != 0:
_writePidData(pid, [_gam().KEYBOARD_INTERRUPT_RC, None])
_writePidData(pid, [KEYBOARD_INTERRUPT_RC, None])
_writePidData(0, pid0DataItem)
if fd not in [sys.stdout, sys.stderr]:
try:
@@ -266,7 +275,7 @@ def batchWriteStderr(data):
sys.stderr.write(data)
sys.stderr.flush()
except IOError as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage('stderr', e))
systemErrorExit(FILE_ERROR_RC, fileErrorMessage('stderr', e))
def writeStdQueueHandler(mpQueue, item):
while True:
@@ -275,7 +284,7 @@ def writeStdQueueHandler(mpQueue, item):
return
except Exception as e:
time.sleep(1)
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
def terminateStdQueueHandler(mpQueue, mpQueueHandler):
mpQueue.put((0, GM.REDIRECT_QUEUE_EOF, None))
@@ -293,6 +302,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
csvRowLimit,
showGettings, showGettingsGotNL,
args):
from gam import ProcessGAMCommand
global mplock
with mplock:
@@ -338,7 +348,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
if mpQueueCSVFile:
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = mpQueueCSVFile
if mpQueueStdout:
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: StringIOobject()}
if debugLevel:
sys.stdout = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_START, args))
@@ -347,14 +357,14 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
GM.Globals[GM.STDOUT] = {}
if mpQueueStderr:
if mpQueueStderr is not mpQueueStdout:
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: StringIOobject()}
# mpQueueStderr.put((pid, GM.REDIRECT_QUEUE_START, args))
writeStdQueueHandler(mpQueueStderr, (pid, GM.REDIRECT_QUEUE_START, args))
else:
GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
else:
GM.Globals[GM.STDERR] = {}
sysRC = _gam().ProcessGAMCommand(args)
sysRC = ProcessGAMCommand(args)
with mplock:
if mpQueueStdout:
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_END, [sysRC, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()]))
@@ -404,9 +414,9 @@ def MultiprocessGAMCommands(items, showCmds):
def poolCallback(result):
poolProcessResults[0] -= 1
if showCmds:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
if GM.Globals[GM.CMDLOG_LOGGER]:
GM.Globals[GM.CMDLOG_LOGGER].info(f'{_gam().currentISOformatTimeStamp()},{result[1]},{result[2]}')
GM.Globals[GM.CMDLOG_LOGGER].info(f'{currentISOformatTimeStamp()},{result[1]},{result[2]}')
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(result[1]):
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
@@ -417,8 +427,8 @@ def MultiprocessGAMCommands(items, showCmds):
def handleControlC(source):
nonlocal controlC
batchWriteStderr(f'Control-C (Multiprocess-{source})\n')
_gam().setSysExitRC(_gam().KEYBOARD_INTERRUPT_RC)
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
setSysExitRC(KEYBOARD_INTERRUPT_RC)
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
pool.terminate()
@@ -444,21 +454,21 @@ def MultiprocessGAMCommands(items, showCmds):
else:
pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
except IOError as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, e)
systemErrorExit(FILE_ERROR_RC, e)
except AssertionError as e:
_gam().Cmd.SetLocation(0)
_gam().usageErrorExit(str(e))
Cmd.SetLocation(0)
usageErrorExit(str(e))
if multiprocessing.get_start_method() != 'fork':
savedValues = saveNonPickleableValues()
if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]:
mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values)
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, Cmd.AllArguments()))
else:
mpQueueStdout = None
if GM.Globals[GM.STDERR][GM.REDIRECT_MULTIPROCESS]:
if GM.Globals[GM.STDERR][GM.REDIRECT_NAME] != 'stdout':
mpQueueStderr, mpQueueHandlerStderr = initializeStdQueueHandler(mpManager, GM.STDERR, GM.Globals, GC.Values)
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, Cmd.AllArguments()))
else:
mpQueueStderr = mpQueueStdout
else:
@@ -478,7 +488,7 @@ def MultiprocessGAMCommands(items, showCmds):
# signal.signal(signal.SIGINT, origSigintHandler)
controlC = False
signal.signal(signal.SIGINT, signal_handler)
batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
batchWriteStderr(Msg.USING_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, numPoolProcesses,
PROCESS_PLURAL_SINGULAR[numPoolProcesses == 1]))
try:
@@ -490,7 +500,7 @@ def MultiprocessGAMCommands(items, showCmds):
if controlC:
break
if item[0] == Cmd.COMMIT_BATCH_CMD:
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
while poolProcessResults[0] > 0:
@@ -502,24 +512,24 @@ def MultiprocessGAMCommands(items, showCmds):
completedProcesses.append(p)
for p in completedProcesses:
del poolProcessResults[p]
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
if len(item) > 1:
_gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
readStdin(f'{currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
continue
if item[0] == Cmd.PRINT_CMD:
batchWriteStderr(Cmd.QuotedArgumentList(item[1:])+'\n')
continue
if item[0] == Cmd.SLEEP_CMD:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
time.sleep(int(item[1]))
continue
pid += 1
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(currentISOformatTimeStamp(), pid, numItems))
if showCmds or GM.Globals[GM.CMDLOG_LOGGER]:
logCmd = Cmd.QuotedArgumentList(item)
if showCmds:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
else:
logCmd = ''
poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti,
@@ -565,7 +575,7 @@ def MultiprocessGAMCommands(items, showCmds):
else:
waitRemaining = 'unlimited'
while poolProcessResults[0] > 0:
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1],
Msg.BATCH_CSV_WAIT_LIMIT.format(waitRemaining)))
@@ -584,7 +594,7 @@ def MultiprocessGAMCommands(items, showCmds):
if GC.Values[GC.PROCESS_WAIT_LIMIT] > 0:
delta = int(time.time()-processWaitStart)
if delta >= GC.Values[GC.PROCESS_WAIT_LIMIT]:
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
pool.terminate()
@@ -596,7 +606,7 @@ def MultiprocessGAMCommands(items, showCmds):
except KeyboardInterrupt:
handleControlC('KBI')
pool.join()
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems))
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(currentISOformatTimeStamp(), numItems))
if mpQueueCSVFile:
terminateCSVFileQueueHandler(mpQueueCSVFile, mpQueueHandlerCSVFile)
if mpQueueStdout:
@@ -617,11 +627,11 @@ def threadBatchWorker(showCmds=False, numItems=0):
sysRC = subprocess.call(item, stdout=GM.Globals[GM.STDOUT].get(GM.REDIRECT_MULTI_FD, sys.stdout),
stderr=GM.Globals[GM.STDERR].get(GM.REDIRECT_MULTI_FD, sys.stderr))
if showCmds:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(sysRC):
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
except Exception as e:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
GM.Globals[GM.TBATCH_QUEUE].task_done()
BATCH_COMMANDS = [Cmd.GAM_CMD, Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD, Cmd.SLEEP_CMD, Cmd.DATETIME_CMD, Cmd.SET_CMD, Cmd.CLEAR_CMD]
@@ -632,12 +642,12 @@ def ThreadBatchGAMCommands(items, showCmds):
return
pythonCmd = [sys.executable]
if not getattr(sys, 'frozen', False): # we're not frozen
pythonCmd.append(os.path.realpath(_gam().Cmd.Argument(0)))
pythonCmd.append(os.path.realpath(Cmd.Argument(0)))
GM.Globals[GM.NUM_BATCH_ITEMS] = numItems = len(items)
numWorkerThreads = min(numItems, GC.Values[GC.NUM_TBATCH_THREADS])
# GM.Globals[GM.TBATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
GM.Globals[GM.TBATCH_QUEUE] = queue.Queue(maxsize=numWorkerThreads)
batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
batchWriteStderr(Msg.USING_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, numWorkerThreads,
THREAD_PLURAL_SINGULAR[numWorkerThreads == 1]))
for _ in range(numWorkerThreads):
@@ -650,28 +660,28 @@ def ThreadBatchGAMCommands(items, showCmds):
if GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING]:
break
if item[0] == Cmd.COMMIT_BATCH_CMD:
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(currentISOformatTimeStamp(),
numItems, numThreadsInUse,
THREAD_PLURAL_SINGULAR[numThreadsInUse == 1]))
GM.Globals[GM.TBATCH_QUEUE].join()
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.THREADS))
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(currentISOformatTimeStamp(), numItems, Msg.THREADS))
numThreadsInUse = 0
if len(item) > 1:
_gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
readStdin(f'{currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
continue
if item[0] == Cmd.PRINT_CMD:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
continue
if item[0] == Cmd.SLEEP_CMD:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
time.sleep(int(item[1]))
continue
pid += 1
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(currentISOformatTimeStamp(), pid, numItems))
if showCmds:
logCmd = Cmd.QuotedArgumentList(item)
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
else:
logCmd = ''
if item[0] == Cmd.GAM_CMD:
@@ -681,43 +691,43 @@ def ThreadBatchGAMCommands(items, showCmds):
numThreadsInUse += 1
GM.Globals[GM.TBATCH_QUEUE].join()
if showCmds:
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Complete\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},0/{numItems},Complete\n')
def _getShowCommands():
if _gam().checkArgumentPresent('showcmds'):
return _gam().getBoolean()
if checkArgumentPresent('showcmds'):
return getBoolean()
return GC.Values[GC.SHOW_COMMANDS]
def _getSkipRows():
if _gam().checkArgumentPresent('skiprows'):
return _gam().getInteger(minVal=0)
if checkArgumentPresent('skiprows'):
return getInteger(minVal=0)
# return GC.Values[GC.CSV_INPUT_ROW_SKIP]
return 0
def _getMaxRows():
if _gam().checkArgumentPresent('maxrows'):
return _gam().getInteger(minVal=0)
if checkArgumentPresent('maxrows'):
return getInteger(minVal=0)
return GC.Values[GC.CSV_INPUT_ROW_LIMIT]
# gam batch <BatchContent> [showcmds [<Boolean>]]
def doBatch(threadBatch=False):
filename = _gam().getString(Cmd.OB_FILE_NAME)
filename = getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_gam().Cmd.Backup()
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
Cmd.Backup()
usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
filenameLower = filename.lower()
if filenameLower not in {'gdoc', 'gcsdoc'}:
encoding = _gam().getCharSet()
filename = _gam().setFilePath(filename, GC.INPUT_DIR)
f = _gam().openFile(filename, encoding=encoding, stripUTFBOM=True)
encoding = getCharSet()
filename = setFilePath(filename, GC.INPUT_DIR)
f = openFile(filename, encoding=encoding, stripUTFBOM=True)
elif filenameLower == 'gdoc':
f = _gam().getGDocData(filenameLower)
_gam().getCharSet()
f = getGDocData(filenameLower)
getCharSet()
else: #filenameLower == 'gcsdoc':
f = _gam().getStorageFileData(filenameLower)
_gam().getCharSet()
f = getStorageFileData(filenameLower)
getCharSet()
showCmds = _getShowCommands()
_gam().checkForExtraneousArguments()
checkForExtraneousArguments()
validCommands = BATCH_COMMANDS if not threadBatch else TBATCH_COMMANDS
kwValues = {}
items = []
@@ -732,40 +742,40 @@ def doBatch(threadBatch=False):
try:
argv = shlex.split(line)
except ValueError as e:
_gam().writeStderr(f'Command: >>>{line.strip()}<<<\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{str(e)}\n')
writeStderr(f'Command: >>>{line.strip()}<<<\n')
writeStderr(f'{ERROR_PREFIX}{str(e)}\n')
errors += 1
continue
if argv:
cmd = argv[0].strip().lower()
if cmd == Cmd.DATETIME_CMD:
if len(argv) == 2:
kwValues['datetime'] = _gam().todaysTime().strftime(argv[1])
kwValues['datetime'] = todaysTime().strftime(argv[1])
else:
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
writeStderr(f'{ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
errors += 1
continue
if cmd == Cmd.SET_CMD:
if len(argv) == 3:
kwValues[argv[1]] = argv[2]
else:
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
writeStderr(f'{ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
errors += 1
continue
if cmd == Cmd.CLEAR_CMD:
if len(argv) == 2:
kwValues.pop(argv[1], None)
else:
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
writeStderr(f'{ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
errors += 1
continue
if cmd == Cmd.SLEEP_CMD:
if len(argv) != 2 or not argv[1].isdigit():
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
writeStderr(f'{ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
errors += 1
continue
if (not cmd) or ((len(argv) == 1) and (cmd not in [Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD])):
@@ -773,28 +783,29 @@ def doBatch(threadBatch=False):
if cmd in validCommands:
items.append(argv)
else:
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_gam().formatChoiceList(validCommands)}>\n')
writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
writeStderr(f'{ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{formatChoiceList(validCommands)}>\n')
errors += 1
except IOError as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
_gam().closeFile(f)
systemErrorExit(FILE_ERROR_RC, fileErrorMessage(filename, e))
closeFile(f)
if errors == 0:
if not threadBatch:
MultiprocessGAMCommands(items, showCmds)
else:
ThreadBatchGAMCommands(items, showCmds)
else:
_gam().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_gam().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
_gam().setSysExitRC(_gam().USAGE_ERROR_RC)
writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
setSysExitRC(USAGE_ERROR_RC)
# gam tbatch <BatchContent> [showcmds [<Boolean>]]
def doThreadBatch():
_gam().adjustRedirectedSTDFilesIfNotMultiprocessing()
from util.fileio import adjustRedirectedSTDFilesIfNotMultiprocessing
adjustRedirectedSTDFilesIfNotMultiprocessing()
doBatch(True)
def doAutoBatch(entityType, entityList, CL_command):
remaining = _gam().Cmd.Remaining()
remaining = Cmd.Remaining()
items = []
initial_argv = [Cmd.GAM_CMD]
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION]:
@@ -823,8 +834,8 @@ def getSubFields(initial_argv, fieldNames):
subFields = {}
GAM_argv = initial_argv[:]
GAM_argvI = len(GAM_argv)
while _gam().Cmd.ArgumentsRemaining():
myarg = _gam().Cmd.Current()
while Cmd.ArgumentsRemaining():
myarg = Cmd.Current()
if not myarg:
GAM_argv.append(myarg)
elif SUB_PATTERN.search(myarg):
@@ -838,17 +849,17 @@ def getSubFields(initial_argv, fieldNames):
if not rematch:
fieldName = submatch.group(1)
if fieldName not in fieldNames:
_gam().csvFieldErrorExit(fieldName, fieldNames)
csvFieldErrorExit(fieldName, fieldNames)
subFields[GAM_argvI].append((SUB_TYPE, fieldName, submatch.start(), submatch.end()))
else:
fieldName = rematch.group(1)
if fieldName not in fieldNames:
_gam().csvFieldErrorExit(fieldName, fieldNames)
csvFieldErrorExit(fieldName, fieldNames)
try:
re.compile(rematch.group(2))
subFields[GAM_argvI].append((RE_TYPE, fieldName, submatch.start(), submatch.end(), rematch.group(2), rematch.group(3)))
except re.error as e:
_gam().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
pos = submatch.end()
GAM_argv.append(myarg)
elif myarg[0] == '~':
@@ -857,11 +868,11 @@ def getSubFields(initial_argv, fieldNames):
subFields[GAM_argvI] = [(SUB_TYPE, fieldName, 0, len(myarg))]
GAM_argv.append(myarg)
else:
_gam().csvFieldErrorExit(fieldName, fieldNames)
csvFieldErrorExit(fieldName, fieldNames)
else:
GAM_argv.append(myarg)
GAM_argvI += 1
_gam().Cmd.Advance()
Cmd.Advance()
return(GAM_argv, subFields)
def processSubFields(GAM_argv, row, subFields):
@@ -888,28 +899,28 @@ def processSubFields(GAM_argv, row, subFields):
# [skiprows <Integer>] [maxrows <Integer>]
# gam <GAM argument list>
def doCSV(testMode=False):
filename = _gam().getString(Cmd.OB_FILE_NAME)
filename = getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_gam().Cmd.Backup()
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
Cmd.Backup()
usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
f, csvFile, fieldnames = openCSVFileReader(filename)
matchFields, skipFields = getMatchSkipFields(fieldnames)
showCmds = _getShowCommands()
skipRows = _getSkipRows()
maxRows = _getMaxRows()
_gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _gam().Cmd.ArgumentsRemaining():
_gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not Cmd.ArgumentsRemaining():
missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
initial_argv = [Cmd.GAM_CMD]
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _gam().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
initial_argv.extend([Cmd.SELECT_CMD, GM.Globals[GM.SECTION]])
GAM_argv, subFields = getSubFields(initial_argv, fieldnames)
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
_gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
items = []
i = 0
for row in csvFile:
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
if checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1
if skipRows:
if i <= skipRows:
@@ -919,21 +930,21 @@ def doCSV(testMode=False):
items.append(processSubFields(GAM_argv, row, subFields))
if maxRows and i >= maxRows:
break
_gam().closeFile(f)
closeFile(f)
if not testMode:
MultiprocessGAMCommands(items, showCmds)
else:
numItems = min(len(items), 10)
_gam().writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
_gam().Ind.Increment()
writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
Ind.Increment()
for field in fieldnames:
_gam().writeStdout(f'{_gam().Ind.Spaces()}{field}\n')
_gam().Ind.Decrement()
_gam().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _gam().GAM))
_gam().Ind.Increment()
writeStdout(f'{Ind.Spaces()}{field}\n')
Ind.Decrement()
writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, GAM))
Ind.Increment()
for i in range(numItems):
_gam().writeStdout(f'{_gam().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
_gam().Ind.Decrement()
writeStdout(f'{Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
Ind.Decrement()
def doCSVTest():
doCSV(testMode=True)
@@ -944,23 +955,24 @@ def doCSVTest():
# [skiprows <Integer>] [maxrows <Integer>]
# gam <GAM argument list>
def doLoop(loopCmd):
filename = _gam().getString(Cmd.OB_FILE_NAME)
from gam import ProcessGAMCommand
filename = getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_gam().Cmd.Backup()
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
Cmd.Backup()
usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
f, csvFile, fieldnames = openCSVFileReader(filename)
matchFields, skipFields = getMatchSkipFields(fieldnames)
showCmds = _getShowCommands()
skipRows = _getSkipRows()
maxRows = _getMaxRows()
_gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _gam().Cmd.ArgumentsRemaining():
_gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not Cmd.ArgumentsRemaining():
missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
_gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
choice = _gam().Cmd.Current().strip().lower()
CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
choice = Cmd.Current().strip().lower()
if choice == Cmd.LOOP_CMD:
_gam().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
# gam loop ... gam redirect|select|config ... process gam.cfg on each iteration
# gam redirect|select|config ... loop ... gam redirect|select|config ... process gam.cfg on each iteration
# gam loop ... gam !redirect|select|config ... no further processing of gam.cfg
@@ -980,13 +992,13 @@ def doLoop(loopCmd):
else:
LoopGlobals = {GM.CMDLOG_LOGGER: None, GM.CMDLOG_HANDLER: None}
if (GM.Globals[GM.PID] > 0) and GC.Values[GC.CMDLOG]:
_gam().openGAMCommandLog(LoopGlobals, 'looplog')
openGAMCommandLog(LoopGlobals, 'looplog')
if LoopGlobals[GM.CMDLOG_LOGGER]:
_gam().writeGAMCommandLog(LoopGlobals, loopCmd, '*')
writeGAMCommandLog(LoopGlobals, loopCmd, '*')
if not showCmds:
i = 0
for row in csvFile:
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
if checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1
if skipRows:
if i <= skipRows:
@@ -996,20 +1008,20 @@ def doLoop(loopCmd):
item = processSubFields(GAM_argv, row, subFields)
logCmd = Cmd.QuotedArgumentList(item)
if i % 100 == 0:
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_gam().currentISOformatTimeStamp(), i))
sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(currentISOformatTimeStamp(), i))
sysRC = ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= HARD_ERROR_RC):
break
if maxRows and i >= maxRows:
break
_gam().closeFile(f)
closeFile(f)
else:
items = []
i = 0
for row in csvFile:
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
if checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1
if skipRows:
if i <= skipRows:
@@ -1019,26 +1031,26 @@ def doLoop(loopCmd):
items.append(processSubFields(GAM_argv, row, subFields))
if maxRows and i >= maxRows:
break
_gam().closeFile(f)
closeFile(f)
numItems = len(items)
pid = 0
for item in items:
pid += 1
logCmd = Cmd.QuotedArgumentList(item)
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
batchWriteStderr(f'{currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
sysRC = ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(f'{currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= HARD_ERROR_RC):
break
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_gam().closeGAMCommandLog(LoopGlobals)
closeGAMCommandLog(LoopGlobals)
if multi:
terminateCSVFileQueueHandler(mpQueue, mpQueueHandler)
def _doList(entityList, entityType):
_gam().buildGAPIObject(API.DIRECTORY)
buildGAPIObject(API.DIRECTORY)
if GM.Globals[GM.CSV_DATA_DICT]:
keyField = GM.Globals[GM.CSV_KEY_FIELD]
dataField = GM.Globals[GM.CSV_DATA_FIELD]
@@ -1046,26 +1058,26 @@ def _doList(entityList, entityType):
keyField = 'Entity'
dataField = 'Data'
csvPF = CSVPrintFile(keyField)
if _gam().checkArgumentPresent('todrive'):
if checkArgumentPresent('todrive'):
csvPF.GetTodriveParameters()
if entityList is None:
entityList = _gam().getEntityList(Cmd.OB_ENTITY)
showData = _gam().checkArgumentPresent('data')
entityList = getEntityList(Cmd.OB_ENTITY)
showData = checkArgumentPresent('data')
if showData:
if not entityType:
itemType, itemList = _gam().getEntityToModify(crosAllowed=True)
itemType, itemList = getEntityToModify(crosAllowed=True)
else:
itemType = None
itemList = _gam().getEntityList(Cmd.OB_ENTITY)
itemList = getEntityList(Cmd.OB_ENTITY)
entityItemLists = itemList if isinstance(itemList, dict) else None
csvPF.AddTitle(dataField)
else:
entityItemLists = None
dataDelimiter = _gam().getDelimiter()
_gam().checkForExtraneousArguments()
_, _, entityList = _gam().getEntityArgument(entityList)
dataDelimiter = getDelimiter()
checkForExtraneousArguments()
_, _, entityList = getEntityArgument(entityList)
for entity in entityList:
entityEmail = _gam().normalizeEmailAddressOrUID(entity)
entityEmail = normalizeEmailAddressOrUID(entity)
if showData:
if entityItemLists:
if entity not in entityItemLists:
@@ -1074,7 +1086,7 @@ def _doList(entityList, entityType):
itemList = entityItemLists[entity]
if itemType == Cmd.ENTITY_USERS:
for i, item in enumerate(itemList):
itemList[i] = _gam().normalizeEmailAddressOrUID(item)
itemList[i] = normalizeEmailAddressOrUID(item)
if dataDelimiter:
csvPF.WriteRow({keyField: entityEmail, dataField: dataDelimiter.join(itemList)})
else:
@@ -1097,10 +1109,10 @@ def doListUser(entityList):
_doList(entityList, Cmd.ENTITY_USERS)
def _showCount(entityList, entityType):
_gam().buildGAPIObject(API.DIRECTORY)
_gam().checkForExtraneousArguments()
_, count, entityList = _gam().getEntityArgument(entityList)
_gam().actionPerformedNumItems(count, entityType)
buildGAPIObject(API.DIRECTORY)
checkForExtraneousArguments()
_, count, entityList = getEntityArgument(entityList)
actionPerformedNumItems(count, entityType)
# gam <CrOSTypeEntity> show count
def showCountCrOS(entityList):

13
src/gam/util/config.py
View File

@@ -18,16 +18,11 @@ import time
import httplib2
import arrow
from gamlib import glaction
Act = glaction.GamAction()
from gam.var import Act, Cmd, Ent, Ind
from gamlib import glapi as API
from gamlib import glcfg as GC
from gam.util.fileio import setFilePath, readFile, writeFile, openFile
from gamlib import glentity
Ent = glentity.GamEntity()
from gamlib import glglobals as GM
from gamlib import glindent
Ind = glindent.GamIndent()
from gamlib import glmsgs as Msg
from gamlib import glskus as SKU
@@ -41,10 +36,11 @@ from util.args import (
checkArgumentPresent, getArgument, getBoolean, getCharacter, getChoice,
getFloat, getInteger, getLanguageCode, getREPattern, getString,
shlexSplitList, shlexSplitListStatus,
Cmd, FALSE, FALSE_VALUES, TRUE, TRUE_FALSE, TRUE_VALUES, UTF8,
FALSE, FALSE_VALUES, TRUE, TRUE_FALSE, TRUE_VALUES, UTF8,
)
from util.csv_pf import CSVPrintFile
from util.display import printKeyValueList, printLine
from util.output import redactable_debug_print
from util.entity import getEntitiesFromCSVFile
from util.entity import getEntitiesFromFile
from util.errors import formatChoiceList, usageErrorExit, USAGE_ERROR_RC
@@ -66,9 +62,6 @@ from gam.constants import (
def SetGlobalVariables():
# redactable_debug_print is defined in gam.__init__ — access at call time
# to avoid circular import (gam imports from us during init)
redactable_debug_print = sys.modules['gam'].redactable_debug_print
def _stringInQuotes(value):

17
src/gam/util/connection.py
View File

@@ -34,10 +34,11 @@ from util.output import (
flushStdout, stderrWarningMsg, systemErrorExit, writeStdout,
)
from util.api import doGAMCheckForUpdates, getHttpObj, getService, handleServerError, waitOnFailure
from gam.constants import GAM_USER_AGENT, __author__, __version__
from gam.var import Cmd, Ent
# gam.__init__ attributes that can't be imported at module level
# (connection.py is imported BY __init__.py during init)
_gam = lambda: sys.modules['gam']
# --- Constants ---
@@ -114,7 +115,7 @@ def _getServerTLSUsed(location):
triesLimit = 5
for n in range(1, triesLimit+1):
try:
httpObj.request(url, headers={'user-agent': _gam().GAM_USER_AGENT})
httpObj.request(url, headers={'user-agent': GAM_USER_AGENT})
cipher_name, tls_ver, _ = httpObj.connections[conn].sock.cipher()
return tls_ver, cipher_name
except (httplib2.HttpLib2Error, RuntimeError) as e:
@@ -228,7 +229,7 @@ def doCheckConnection():
try_count = 0
httpObj = getHttpObj(timeout=30)
httpObj.follow_redirects = False
headers = {'user-agent': _gam().GAM_USER_AGENT}
headers = {'user-agent': GAM_USER_AGENT}
okay = createGreenText('OK')
not_okay = createRedText('ERROR')
success_count = 0
@@ -278,12 +279,10 @@ def doCheckConnection():
# gam comment
def doComment():
writeStdout(_gam().Cmd.QuotedArgumentList(_gam().Cmd.Remaining())+'\n')
writeStdout(Cmd.QuotedArgumentList(Cmd.Remaining())+'\n')
# gam version [check|checkrc|simple|extended] [timeoffset] [nooffseterror] [location <HostName>]
def doVersion(checkForArgs=True):
Ent = _gam().Ent
Cmd = _gam().Cmd
forceCheck = 0
extended = noOffsetError = timeOffset = simple = False
testLocation = GOOGLE_TIMECHECK_LOCATION
@@ -307,10 +306,10 @@ def doVersion(checkForArgs=True):
else:
unknownArgumentExit()
if simple:
writeStdout(_gam().__version__)
writeStdout(__version__)
return
writeStdout((f'{GAM} {_gam().__version__} - {GAM_URL} - {GM.Globals[GM.GAM_TYPE]}\n'
f'{_gam().__author__}\n'
writeStdout((f'{GAM} {__version__} - {GAM_URL} - {GM.Globals[GM.GAM_TYPE]}\n'
f'{__author__}\n'
f'Python {sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]} {struct.calcsize("P")*8}-bit {sys.version_info[3]}\n'
f'{getOSPlatform()} {platform.machine()}\n'
f'Path: {GM.Globals[GM.GAM_PATH]}\n'

498
src/gam/util/csv_pf.py
View File

File diff suppressed because it is too large Load Diff

12
src/gam/util/display.py
View File

@@ -12,17 +12,7 @@ from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
class _InstanceProxy:
"""Lazy proxy that delegates attribute access to a named instance in the gam module."""
def __init__(self, name):
self._name = name
def __getattr__(self, attr):
return getattr(getattr(sys.modules['gam'], self._name), attr)
Act = _InstanceProxy('Act')
Ind = _InstanceProxy('Ind')
from gam.var import Act, Ind
from util.output import (
currentCountNL,
formatKeyValueList,

60
src/gam/util/email.py
View File

@@ -20,65 +20,67 @@ from email.mime.text import MIMEText
from gamlib import glapi as API
from gamlib import glcfg as GC
from gamlib import glgapi as GAPI
from util.api import _getAdminEmail, buildGAPIObject, buildGAPIServiceObject, callGAPI
from util.args import NAME_EMAIL_ADDRESS_PATTERN, UTF8, normalizeEmailAddressOrUID
from util.display import entityActionFailedWarning, entityActionPerformed, entityActionPerformedMessage
from util.errors import usageErrorExit
from util.fileio import readFile, setFilePath
_gam = lambda: sys.modules['gam']
# Add attachements to an email message
def _addAttachmentsToMessage(message, attachments):
for attachment in attachments:
try:
attachFilename = _gam().setFilePath(attachment[0], GC.INPUT_DIR)
attachFilename = setFilePath(attachment[0], GC.INPUT_DIR)
attachContentType, attachEncoding = mimetypes.guess_type(attachFilename)
if attachContentType is None or attachEncoding is not None:
attachContentType = 'application/octet-stream'
main_type, sub_type = attachContentType.split('/', 1)
if main_type == 'text':
msg = MIMEText(_gam().readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=_gam().UTF8)
msg = MIMEText(readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=UTF8)
elif main_type == 'image':
msg = MIMEImage(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
msg = MIMEImage(readFile(attachFilename, 'rb'), _subtype=sub_type)
elif main_type == 'audio':
msg = MIMEAudio(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
msg = MIMEAudio(readFile(attachFilename, 'rb'), _subtype=sub_type)
elif main_type == 'application':
msg = MIMEApplication(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
msg = MIMEApplication(readFile(attachFilename, 'rb'), _subtype=sub_type)
else:
msg = MIMEBase(main_type, sub_type)
msg.set_payload(_gam().readFile(attachFilename, 'rb'))
msg.set_payload(readFile(attachFilename, 'rb'))
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFilename))
message.attach(msg)
except (IOError, UnicodeDecodeError) as e:
_gam().usageErrorExit(f'{attachFilename}: {str(e)}')
usageErrorExit(f'{attachFilename}: {str(e)}')
# Add embedded images to an email message
def _addEmbeddedImagesToMessage(message, embeddedImages):
for embeddedImage in embeddedImages:
try:
imageFilename = _gam().setFilePath(embeddedImage[0], GC.INPUT_DIR)
imageFilename = setFilePath(embeddedImage[0], GC.INPUT_DIR)
imageContentType, imageEncoding = mimetypes.guess_type(imageFilename)
if imageContentType is None or imageEncoding is not None:
imageContentType = 'application/octet-stream'
main_type, sub_type = imageContentType.split('/', 1)
if main_type == 'image':
msg = MIMEImage(_gam().readFile(imageFilename, 'rb'), _subtype=sub_type)
msg = MIMEImage(readFile(imageFilename, 'rb'), _subtype=sub_type)
else:
msg = MIMEBase(main_type, sub_type)
msg.set_payload(_gam().readFile(imageFilename, 'rb'))
msg.set_payload(readFile(imageFilename, 'rb'))
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(imageFilename))
msg.add_header('Content-ID', f'<{embeddedImage[1]}>')
message.attach(msg)
except (IOError, UnicodeDecodeError) as e:
_gam().usageErrorExit(f'{imageFilename}: {str(e)}')
usageErrorExit(f'{imageFilename}: {str(e)}')
# Send an email
def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msgFrom=None, msgReplyTo=None,
html=False, charset=None, attachments=None, embeddedImages=None,
msgHeaders=None, ccRecipients=None, bccRecipients=None, mailBox=None, threadId=None,
action=None):
Act = _gam().Act
Ent = _gam().Ent
if charset is None:
charset = _gam().UTF8
charset = UTF8
if action is None:
action = Act.SENDEMAIL
@@ -92,26 +94,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
toSent.remove(addr)
toFailed[addr] = f'{err[0]}: {err[1]}'
if toSent:
_gam().entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
for addr, errMsg in toFailed.items():
_gam().entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
def cleanAddr(emailAddr):
match = _gam().NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
match = NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
if match:
emailName = match.group(1)
emailAddr = _gam().normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
emailAddr = normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
return (f'{emailName} <{emailAddr}>', emailAddr)
emailAddr = _gam().normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
emailAddr = normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
return (emailAddr, emailAddr)
if msgFrom is None:
msgFrom = _gam()._getAdminEmail()
msgFrom = _getAdminEmail()
# Force ASCII for RFC compliance
# xmlcharref seems to work to display at least
# some unicode in HTML body and is ignored in
# plain text body.
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(_gam().UTF8)
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(UTF8)
if not attachments and not embeddedImages:
message = MIMEText(msgBody, ['plain', 'html'][html], charset)
else:
@@ -141,26 +143,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
Act.Set(action)
if not GC.Values[GC.SMTP_HOST]:
if not clientAccess:
userId, gmail = _gam().buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
userId, gmail = buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
if not gmail:
Act.Set(parentAction)
return
else:
userId = mailBoxAddr
gmail = _gam().buildGAPIObject(API.GMAIL)
gmail = buildGAPIObject(API.GMAIL)
message['To'] = msgTo if msgTo else userId
body = {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}
if threadId is not None:
body['threadId'] = threadId
try:
result = _gam().callGAPI(gmail.users().messages(), 'send',
result = callGAPI(gmail.users().messages(), 'send',
throwReasons=[GAPI.SERVICE_NOT_AVAILABLE, GAPI.AUTH_ERROR, GAPI.DOMAIN_POLICY,
GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
userId=userId, body=body, fields='id')
_gam().entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy,
GAPI.invalid, GAPI.invalidArgument, GAPI.forbidden, GAPI.permissionDenied) as e:
_gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
else:
message['To'] = msgTo if msgTo else mailBoxAddr
server = None
@@ -171,7 +173,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
server.starttls(context=ssl.create_default_context(cafile=GC.Values[GC.CACERTS_PEM]))
if GC.Values[GC.SMTP_USERNAME] and GC.Values[GC.SMTP_PASSWORD]:
if isinstance(GC.Values[GC.SMTP_PASSWORD], bytes):
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(_gam().UTF8))
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(UTF8))
else:
server.login(GC.Values[GC.SMTP_USERNAME], GC.Values[GC.SMTP_PASSWORD])
result = server.send_message(message)
@@ -179,7 +181,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
checkResult(Ent.RECIPIENT_CC, ccRecipients)
checkResult(Ent.RECIPIENT_BCC, bccRecipients)
except smtplib.SMTPException as e:
_gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
if server:
try:
server.quit()

926
src/gam/util/entity.py
View File

File diff suppressed because it is too large Load Diff

14
src/gam/util/errors.py
View File

@@ -8,19 +8,7 @@ from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
class _InstanceProxy:
"""Lazy proxy that delegates attribute access to a named instance in the gam module."""
def __init__(self, name):
self._name = name
def __getattr__(self, attr):
return getattr(getattr(sys.modules['gam'], self._name), attr)
Act = _InstanceProxy('Act')
Cmd = _InstanceProxy('Cmd')
Ind = _InstanceProxy('Ind')
from gam.var import Act, Cmd, Ind
from util.output import (
currentCountNL,
formatKeyValueList,

50
src/gam/util/fileio.py
View File

@@ -19,17 +19,7 @@ from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
class _InstanceProxy:
"""Lazy proxy that delegates attribute access to a named instance in the gam module."""
def __init__(self, name):
self._name = name
def __getattr__(self, attr):
return getattr(getattr(sys.modules['gam'], self._name), attr)
Act = _InstanceProxy('Act')
Ind = _InstanceProxy('Ind')
from gam.var import Act, Ind
from util.output import (
stderrErrorMsg,
stderrWarningMsg,
@@ -287,3 +277,41 @@ def closeGAMCommandLog(Globals):
except Exception:
pass
Globals[GM.CMDLOG_LOGGER] = None
def adjustRedirectedSTDFilesIfNotMultiprocessing():
def adjustRedirectedSTDFile(stdtype):
rdFd = GM.Globals[stdtype].get(GM.REDIRECT_FD)
rdMultiFd = GM.Globals[stdtype].get(GM.REDIRECT_MULTI_FD)
if rdFd and rdMultiFd and rdFd != rdMultiFd:
try:
rdFd.write(rdMultiFd.getvalue())
rdMultiFd.close()
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = rdFd
if (stdtype == GM.STDOUT) and (GM.Globals.get(GM.SAVED_STDOUT) is not None):
sys.stdout = rdFd
except IOError as e:
systemErrorExit(FILE_ERROR_RC, e)
adjustRedirectedSTDFile(GM.STDOUT)
if GM.Globals[GM.STDERR].get(GM.REDIRECT_NAME) != 'stdout':
adjustRedirectedSTDFile(GM.STDERR)
else:
GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
def closeSTDFilesIfNotMultiprocessing(closeSTD):
def closeSTDFile(stdtype, stdfile):
rdFd = GM.Globals[stdtype].get(GM.REDIRECT_FD)
rdMultiFd = GM.Globals[stdtype].get(GM.REDIRECT_MULTI_FD)
if rdFd and rdMultiFd and (rdFd == rdMultiFd) and (rdFd != stdfile):
try:
rdFd.flush()
if closeSTD:
rdFd.close()
except BrokenPipeError:
pass
closeSTDFile(GM.STDOUT, sys.stdout)
if GM.Globals[GM.STDERR].get(GM.REDIRECT_NAME) != 'stdout':
closeSTDFile(GM.STDERR, sys.stderr)

159
src/gam/util/gdoc.py
View File

@@ -22,9 +22,14 @@ Ent = glentity.GamEntity()
from gamlib import glgapi as GAPI
from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
from gam.constants import DEFAULT_CSV_READ_MODE, NO_ENTITIES_FOUND_RC
from util.args import UTF8, checkArgumentPresent, getBoolean, getCharSet, getCharacter, getEmailAddress, getSheetEntity, getSheetIdFromSheetEntity, getString, shlexSplitList
from util.display import ACTION_NOT_PERFORMED_RC, userDriveServiceNotEnabledWarning
from util.errors import entityActionFailedExit, entityDoesNotExistExit
from util.fileio import FILE_ERROR_RC, UTF8_SIG, fileErrorMessage, getGDocSheetDataFailedExit, getGDocSheetDataRetryWarning, openFile, setFilePath
from util.output import stderrWarningMsg, systemErrorExit
_gam = lambda: sys.modules['gam']
GDOC_FORMAT_MIME_TYPES = {
@@ -35,40 +40,41 @@ GDOC_FORMAT_MIME_TYPES = {
# gdoc <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity>
def getGDocData(gformat):
from util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI
mimeType = GDOC_FORMAT_MIME_TYPES[gformat]
user = _gam().getEmailAddress()
fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
user = getEmailAddress()
fileIdEntity = getDriveFileEntity(queryShortcutsOK=False)
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
_, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
_, drive = buildGAPIServiceObject(API.DRIVE3, user)
else:
drive = _gam().buildGAPIObject(API.DRIVE3)
drive = buildGAPIObject(API.DRIVE3)
if not drive:
sys.exit(GM.Globals[GM.SYSEXITRC])
_, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
_, _, jcount = _validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
if jcount == 0:
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
if jcount > 1:
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
fileId = fileIdEntity['list'][0]
f = None
try:
result = _gam().callGAPI(drive.files(), 'get',
result = callGAPI(drive.files(), 'get',
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
fileId=fileId, fields='name,mimeType,exportLinks',
supportsAllDrives=True)
# Google Doc
if 'exportLinks' in result:
if mimeType not in result['exportLinks']:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
f = TemporaryFile(mode='w+', encoding=UTF8)
_, content = drive._http.request(uri=result['exportLinks'][mimeType], method='GET')
f.write(content.decode(_gam().UTF8_SIG))
f.write(content.decode(UTF8_SIG))
f.seek(0)
return f
# Drive File
if result['mimeType'] != mimeType:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
fb = TemporaryFile(mode='wb+')
request = drive.files().get_media(fileId=fileId)
@@ -76,63 +82,64 @@ def getGDocData(gformat):
done = False
while not done:
_, done = downloader.next_chunk()
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
f = TemporaryFile(mode='w+', encoding=UTF8)
fb.seek(0)
f.write(fb.read().decode(_gam().UTF8_SIG))
f.write(fb.read().decode(UTF8_SIG))
fb.close()
f.seek(0)
return f
except GAPI.fileNotFound:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
except (IOError, httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if f:
f.close()
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
_gam().userDriveServiceNotEnabledWarning(user, str(e))
userDriveServiceNotEnabledWarning(user, str(e))
sys.exit(GM.Globals[GM.SYSEXITRC])
HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>')
# gsheet <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> <SheetEntity>
def getGSheetData():
user = _gam().getEmailAddress()
fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
sheetEntity = _gam().getSheetEntity(False)
from util.api import buildGAPIObject, buildGAPIServiceObject, callGAPI
user = getEmailAddress()
fileIdEntity = getDriveFileEntity(queryShortcutsOK=False)
sheetEntity = getSheetEntity(False)
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
user, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
user, drive = buildGAPIServiceObject(API.DRIVE3, user)
else:
drive = _gam().buildGAPIObject(API.DRIVE3)
drive = buildGAPIObject(API.DRIVE3)
if not drive:
sys.exit(GM.Globals[GM.SYSEXITRC])
_, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
_, _, jcount = _validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
if jcount == 0:
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
if jcount > 1:
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
_, sheet = _gam().buildGAPIServiceObject(API.SHEETS, user)
_, sheet = buildGAPIServiceObject(API.SHEETS, user)
else:
sheet = _gam().buildGAPIObject(API.SHEETS)
sheet = buildGAPIObject(API.SHEETS)
if not sheet:
sys.exit(GM.Globals[GM.SYSEXITRC])
fileId = fileIdEntity['list'][0]
f = None
try:
result = _gam().callGAPI(drive.files(), 'get',
result = callGAPI(drive.files(), 'get',
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
fileId=fileId, fields='name,mimeType', supportsAllDrives=True)
if result['mimeType'] != _gam().MIMETYPE_GA_SPREADSHEET:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], _gam().MIMETYPE_GA_SPREADSHEET))
spreadsheet = _gam().callGAPI(sheet.spreadsheets(), 'get',
if result['mimeType'] != MIMETYPE_GA_SPREADSHEET:
getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], MIMETYPE_GA_SPREADSHEET))
spreadsheet = callGAPI(sheet.spreadsheets(), 'get',
throwReasons=GAPI.SHEETS_ACCESS_THROW_REASONS,
spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))')
sheetId = _gam().getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
sheetId = getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
if sheetId is None:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
spreadsheetUrl = f'{re.sub("/edit.*$", "/export", spreadsheet["spreadsheetUrl"])}?format=csv&id={fileId}&gid={sheetId}'
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
f = TemporaryFile(mode='w+', encoding=UTF8)
if GC.Values[GC.DEBUG_LEVEL] > 0:
sys.stderr.write(f'Debug: spreadsheetUrl: {spreadsheetUrl}\n')
triesLimit = 3
@@ -143,25 +150,25 @@ def getGSheetData():
break
tg = HTML_TITLE_PATTERN.match(content[0:600].decode('utf-8'))
errMsg = tg.group(1) if tg else 'Unknown error'
_gam().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
time.sleep(20)
else:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
f.write(content.decode(_gam().UTF8_SIG))
getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
f.write(content.decode(UTF8_SIG))
f.seek(0)
return f
except GAPI.fileNotFound:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
except (GAPI.notFound, GAPI.forbidden, GAPI.permissionDenied,
GAPI.internalError, GAPI.insufficientFilePermissions, GAPI.badRequest,
GAPI.invalid, GAPI.invalidArgument, GAPI.failedPrecondition) as e:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
except (IOError, httplib2.HttpLib2Error) as e:
if f:
f.close()
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
_gam().userDriveServiceNotEnabledWarning(user, str(e))
userDriveServiceNotEnabledWarning(user, str(e))
sys.exit(GM.Globals[GM.SYSEXITRC])
@@ -172,15 +179,14 @@ BUCKET_OBJECT_PATTERNS = [
]
def getBucketObjectName():
Cmd = _gam().Cmd
uri = _gam().getString(Cmd.OB_STRING)
uri = getString(Cmd.OB_STRING)
for pattern in BUCKET_OBJECT_PATTERNS:
mg = re.search(pattern['pattern'], uri)
if mg:
bucket = mg.group(1)
s_object = mg.group(2) if not pattern['unquote'] else unquote(mg.group(2))
return (bucket, s_object, f'{bucket}/{s_object}')
_gam().systemErrorExit(_gam().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
systemErrorExit(ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
GCS_FORMAT_MIME_TYPES = {
'gcscsv': 'text/csv',
@@ -190,19 +196,20 @@ GCS_FORMAT_MIME_TYPES = {
# gcscsv|gcshtml|gcsdoc <StorageBucketObjectName>
def getStorageFileData(gcsformat, returnData=True):
from util.api import buildGAPIObject, callGAPI
mimeType = GCS_FORMAT_MIME_TYPES[gcsformat]
bucket, s_object, bucketObject = getBucketObjectName()
s = _gam().buildGAPIObject(API.STORAGEREAD)
s = buildGAPIObject(API.STORAGEREAD)
try:
result = _gam().callGAPI(s.objects(), 'get',
result = callGAPI(s.objects(), 'get',
throwReasons=[GAPI.NOT_FOUND, GAPI.FORBIDDEN],
bucket=bucket, object=s_object, projection='noAcl', fields='contentType')
except GAPI.notFound:
_gam().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
except GAPI.forbidden as e:
_gam().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
if result['contentType'] != mimeType:
_gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
Msg.INVALID_MIMETYPE.format(result['contentType'], mimeType))
fb = TemporaryFile(mode='wb+')
try:
@@ -213,58 +220,58 @@ def getStorageFileData(gcsformat, returnData=True):
_, done = downloader.next_chunk()
fb.seek(0)
if returnData:
data = fb.read().decode(_gam().UTF8)
data = fb.read().decode(UTF8)
fb.close()
return data
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
f.write(fb.read().decode(_gam().UTF8_SIG))
f = TemporaryFile(mode='w+', encoding=UTF8)
f.write(fb.read().decode(UTF8_SIG))
fb.close()
f.seek(0)
return f
except googleapiclient.http.HttpError as e:
mg = _gam().HTTP_ERROR_PATTERN.match(str(e))
_gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
mg = HTTP_ERROR_PATTERN.match(str(e))
getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
# <CSVFileInput>
def openCSVFileReader(filename, fieldnames=None):
Cmd = _gam().Cmd
from gam.cmd.drive.transfer.fileops import HTTP_ERROR_PATTERN
filenameLower = filename.lower()
if filenameLower == 'gsheet':
f = getGSheetData()
_gam().getCharSet()
getCharSet()
elif filenameLower in {'gcsv', 'gdoc'}:
f = getGDocData(filenameLower)
_gam().getCharSet()
getCharSet()
elif filenameLower in {'gcscsv', 'gcsdoc'}:
f = getStorageFileData(filenameLower, False)
_gam().getCharSet()
getCharSet()
else:
encoding = _gam().getCharSet()
filename = _gam().setFilePath(filename, GC.INPUT_DIR)
f = _gam().openFile(filename, mode=_gam().DEFAULT_CSV_READ_MODE, encoding=encoding)
if _gam().checkArgumentPresent('warnifnodata'):
encoding = getCharSet()
filename = setFilePath(filename, GC.INPUT_DIR)
f = openFile(filename, mode=DEFAULT_CSV_READ_MODE, encoding=encoding)
if checkArgumentPresent('warnifnodata'):
loc = f.tell()
try:
if not f.readline() or not f.readline():
_gam().stderrWarningMsg(_gam().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
sys.exit(_gam().NO_ENTITIES_FOUND_RC)
stderrWarningMsg(fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
sys.exit(NO_ENTITIES_FOUND_RC)
f.seek(loc)
except (IOError, UnicodeDecodeError, UnicodeError) as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
if _gam().checkArgumentPresent('columndelimiter'):
columnDelimiter = _gam().getCharacter()
systemErrorExit(FILE_ERROR_RC, fileErrorMessage(filename, e))
if checkArgumentPresent('columndelimiter'):
columnDelimiter = getCharacter()
else:
columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER]
if _gam().checkArgumentPresent('noescapechar'):
noEscapeChar = _gam().getBoolean()
if checkArgumentPresent('noescapechar'):
noEscapeChar = getBoolean()
else:
noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR]
if _gam().checkArgumentPresent('quotechar'):
quotechar = _gam().getCharacter()
if checkArgumentPresent('quotechar'):
quotechar = getCharacter()
else:
quotechar = GC.Values[GC.CSV_INPUT_QUOTE_CHAR]
if not _gam().checkArgumentPresent('endcsv') and _gam().checkArgumentPresent('fields'):
fieldnames = _gam().shlexSplitList(_gam().getString(Cmd.OB_FIELD_NAME_LIST))
if not checkArgumentPresent('endcsv') and checkArgumentPresent('fields'):
fieldnames = shlexSplitList(getString(Cmd.OB_FIELD_NAME_LIST))
try:
csvFile = csv.DictReader(f, fieldnames=fieldnames,
delimiter=columnDelimiter,
@@ -272,4 +279,4 @@ def openCSVFileReader(filename, fieldnames=None):
quotechar=quotechar)
return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else [])
except (csv.Error, UnicodeDecodeError, UnicodeError) as e:
_gam().systemErrorExit(_gam().FILE_ERROR_RC, e)
systemErrorExit(FILE_ERROR_RC, e)

62
src/gam/util/orgunits.py
View File

@@ -9,13 +9,16 @@ from gamlib import glapi as API
from gamlib import glcfg as GC
from gamlib import glgapi as GAPI
from gamlib import glglobals as GM
from util.access import accessErrorExit, checkEntityAFDNEorAccessErrorExit
from util.api import buildGAPIObject, callGAPI
from util.args import encodeOrgUnitPath, makeOrgUnitPathAbsolute, makeOrgUnitPathRelative
from util.errors import entityDoesNotExistExit, invalidArgumentExit, missingArgumentExit
from gam.var import Cmd, Ent
_gam = lambda: sys.modules['gam']
def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
Cmd = _gam().Cmd
if Cmd.ArgumentsRemaining():
path = Cmd.Current().strip()
if path == 'root':
@@ -23,40 +26,39 @@ def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
if path:
if pathOnly and (path.startswith('id:') or path.startswith('uid:')) and cd is not None:
try:
result = _gam().callGAPI(cd.orgunits(), 'get',
result = callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=path,
fields='orgUnitPath')
Cmd.Advance()
if absolutePath:
return _gam().makeOrgUnitPathAbsolute(result['orgUnitPath'])
return _gam().makeOrgUnitPathRelative(result['orgUnitPath'])
return makeOrgUnitPathAbsolute(result['orgUnitPath'])
return makeOrgUnitPathRelative(result['orgUnitPath'])
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError,
GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_gam().checkEntityAFDNEorAccessErrorExit(cd, _gam().Ent.ORGANIZATIONAL_UNIT, path)
_gam().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, path)
invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
Cmd.Advance()
if absolutePath:
return _gam().makeOrgUnitPathAbsolute(path)
return _gam().makeOrgUnitPathRelative(path)
_gam().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
return makeOrgUnitPathAbsolute(path)
return makeOrgUnitPathRelative(path)
missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
def getTopLevelOrgId(cd, parentOrgUnitPath):
Ent = _gam().Ent
if parentOrgUnitPath != '/':
try:
result = _gam().callGAPI(cd.orgunits(), 'get',
result = callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentOrgUnitPath)),
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(parentOrgUnitPath)),
fields='orgUnitId')
return result['orgUnitId']
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
return None
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
return None
try:
result = _gam().callGAPI(cd.orgunits(), 'list',
result = callGAPI(cd.orgunits(), 'list',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='allIncludingParent',
fields='organizationUnits(orgUnitId,orgUnitPath)')
@@ -67,18 +69,17 @@ def getTopLevelOrgId(cd, parentOrgUnitPath):
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
return None
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
return None
def getOrgUnitId(cd=None, orgUnit=None):
Ent = _gam().Ent
if cd is None:
cd = _gam().buildGAPIObject(API.DIRECTORY)
cd = buildGAPIObject(API.DIRECTORY)
if orgUnit is None:
orgUnit = getOrgUnitItem()
try:
if orgUnit == '/':
result = _gam().callGAPI(cd.orgunits(), 'list',
result = callGAPI(cd.orgunits(), 'list',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='children',
fields='organizationUnits(parentOrgUnitId,parentOrgUnitPath)')
@@ -88,35 +89,34 @@ def getOrgUnitId(cd=None, orgUnit=None):
if topLevelOrgId:
return (orgUnit, topLevelOrgId)
return (orgUnit, '/') #Bogus but should never happen
result = _gam().callGAPI(cd.orgunits(), 'get',
result = callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(orgUnit)),
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=encodeOrgUnitPath(makeOrgUnitPathRelative(orgUnit)),
fields='orgUnitId,orgUnitPath')
return (result['orgUnitPath'], result['orgUnitId'])
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
_gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_gam().accessErrorExit(cd)
accessErrorExit(cd)
def getAllParentOrgUnitsForUser(cd, user):
Ent = _gam().Ent
try:
result = _gam().callGAPI(cd.users(), 'get',
result = callGAPI(cd.users(), 'get',
throwReasons=GAPI.USER_GET_THROW_REASONS,
userKey=user, fields='orgUnitPath', projection='basic')
except (GAPI.userNotFound, GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden):
_gam().entityDoesNotExistExit(Ent.USER, user)
entityDoesNotExistExit(Ent.USER, user)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_gam().accessErrorExit(cd)
accessErrorExit(cd)
parentPath = result['orgUnitPath']
if parentPath == '/':
orgUnitPath, orgUnitId = getOrgUnitId(cd, '/')
return {orgUnitId: orgUnitPath}
parentPath = _gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentPath))
parentPath = encodeOrgUnitPath(makeOrgUnitPathRelative(parentPath))
orgUnits = {}
while True:
try:
result = _gam().callGAPI(cd.orgunits(), 'get',
result = callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=parentPath,
fields='orgUnitId,orgUnitPath,parentOrgUnitId')
@@ -125,9 +125,9 @@ def getAllParentOrgUnitsForUser(cd, user):
break
parentPath = result['parentOrgUnitId']
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
_gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_gam().accessErrorExit(cd)
accessErrorExit(cd)
return orgUnits
def _getOrgunitsOrgUnitIdPath(cd, orgUnit):

40
src/gam/util/output.py
View File

@@ -5,21 +5,15 @@ No circular dependency risk — they only depend on gamlib modules and
simple string constants.
"""
import re
import sys
import time
from gamlib import glcfg as GC
from gamlib import glglobals as GM
class _InstanceProxy:
"""Lazy proxy that delegates attribute access to a named instance in the gam module."""
def __init__(self, name):
self._name = name
def __getattr__(self, attr):
return getattr(getattr(sys.modules['gam'], self._name), attr)
Ind = _InstanceProxy('Ind')
from gamlib import glmsgs as Msg
from gam.constants import DEBUG_REDACTION_PATTERNS
from gam.var import Ind
# These constants are duplicated from __init__.py to avoid circular imports.
@@ -182,3 +176,29 @@ def formatKeyValueList(prefixStr, kvList, suffixStr):
msg += ' '
msg += suffixStr
return msg
def redactable_debug_print(*args):
processed_args = []
for arg in args:
if arg.startswith('b\''):
sbytes = arg[2:-1]
sbytes = bytes(sbytes, 'utf-8')
arg = sbytes.decode()
arg = arg.replace('\\r\\n', "\n ")
if GC.Values[GC.DEBUG_REDACTION]:
for pattern, replace in DEBUG_REDACTION_PATTERNS:
arg = re.sub(pattern, replace, arg)
processed_args.append(arg)
print(*processed_args)
def showAPICallsRetryData():
if GC.Values.get(GC.SHOW_API_CALLS_RETRY_DATA) and GM.Globals[GM.API_CALLS_RETRY_DATA]:
Ind.Reset()
writeStderr(Msg.API_CALLS_RETRY_DATA)
Ind.Increment()
for k, v in sorted(GM.Globals[GM.API_CALLS_RETRY_DATA].items()):
m, s = divmod(int(v[1]), 60)
h, m = divmod(m, 60)
writeStderr(formatKeyValueList(Ind.Spaces(), [k, f'{v[0]}/{h}:{m:02d}:{s:02d}'], '\n'))
Ind.Decrement()