phase 2 code complete
Some checks failed
Build and test GAM / build (false, build, 1, Build Intel Ubuntu Jammy, ubuntu-22.04) (push) Has been cancelled
Build and test GAM / build (false, build, 10, Build x86_64 macOS 15, macos-15-intel) (push) Has been cancelled
Build and test GAM / build (false, build, 11, Build x86_64 macOS 26, macos-26-intel) (push) Has been cancelled
Build and test GAM / build (false, build, 12, Build Arm MacOS 26, macos-26) (push) Has been cancelled
Build and test GAM / build (false, build, 13, Build Intel Windows, windows-2025-vs2026) (push) Has been cancelled
Build and test GAM / build (false, build, 14, Build Arm Windows, windows-11-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 2, Build Intel Ubuntu Noble, ubuntu-24.04) (push) Has been cancelled
Build and test GAM / build (false, build, 3, Build Arm Ubuntu Noble, ubuntu-24.04-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 4, Build Arm Ubuntu Jammy, ubuntu-22.04-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 5, Build Intel StaticX Legacy, ubuntu-22.04, yes) (push) Has been cancelled
Build and test GAM / build (false, build, 6, Build Arm StaticX Legacy, ubuntu-22.04-arm, yes) (push) Has been cancelled
Build and test GAM / build (false, build, 8, Build Arm MacOS 14, macos-14) (push) Has been cancelled
Build and test GAM / build (false, build, 9, Build Arm MacOS 15, macos-15) (push) Has been cancelled
Build and test GAM / build (false, test, 16, Test Python 3.11, ubuntu-24.04, 3.11) (push) Has been cancelled
Build and test GAM / build (false, test, 17, Test Python 3.12, ubuntu-24.04, 3.12) (push) Has been cancelled
Build and test GAM / build (false, test, 18, Test Python 3.13, ubuntu-24.04, 3.13) (push) Has been cancelled
Build and test GAM / build (false, test, 19, Test Python 3.15-dev, ubuntu-24.04, 3.15-dev) (push) Has been cancelled
Build and test GAM / build (true, test, 20, Test Python 3.14 freethread, ubuntu-24.04, 3.14) (push) Has been cancelled
Build and test GAM / publish (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled

This commit is contained in:
Jay Lee
2026-07-03 19:52:57 -04:00
parent 7c639d3487
commit 5b27b7b875
10 changed files with 1210 additions and 1206 deletions

View File

@@ -3207,3 +3207,62 @@ def ProcessGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=True):
# Process GAM command # Process GAM command
def CallGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=False): def CallGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=False):
return ProcessGAMCommand(args, processGamCfg=processGamCfg, inLoop=inLoop, closeSTD=closeSTD) return ProcessGAMCommand(args, processGamCfg=processGamCfg, inLoop=inLoop, closeSTD=closeSTD)
# Re-export util names for _gam().X access by util/ modules.
# This block runs AFTER all modules are fully loaded (no circular import risk).
# Only names actually referenced via _gam() are included.
from gamlib import glskus as SKU # noqa: E402,F811
from tempfile import TemporaryFile # noqa: E402,F401
from util.access import ( # noqa: E402,F401
APIAccessDeniedExit, ClientAPIAccessDeniedExit, SvcAcctAPIAccessDeniedExit, accessErrorExit,
accessErrorExitNonDirectory, checkEntityAFDNEorAccessErrorExit,
checkEntityDNEorAccessErrorExit, entityUnknownWarning,
)
from util.api import ( # noqa: E402,F401,F811
_getAdminEmail, _getSvcAcctData, buildGAPIObject, buildGAPIServiceObject,
callGAPI, callGAPIitems, callGAPIpages, chooseSaAPI, yieldGAPIpages,
)
from util.args import ( # noqa: E402,F401,F811
ARCHIVED_ARGUMENTS, ISOformatTimeStamp, LOCALE_CODES_MAP, NAME_EMAIL_ADDRESS_PATTERN,
SUSPENDED_ARGUMENTS, YYYYMMDDTHHMMSSZ_FORMAT, YYYYMMDD_FORMAT, YYYYMMDD_PATTERN,
_getIsArchived, _getIsSuspended, checkDataField, checkForExtraneousArguments,
checkMatchSkipFields, checkSubkeyField, encodeOrgUnitPath, escapeCRsNLs,
formatLocalTime, formatLocalTimestamp, getBoolean, getCharSet, getCharacter,
getDelimiter, getEmailAddress, getEmailAddressDomain, getLanguageCode,
getMatchSkipFields, getPhraseDNEorSNA, getREPattern, getString,
getSheetEntity, getSheetIdFromSheetEntity, makeOrgUnitPathAbsolute,
makeOrgUnitPathRelative, orgUnitPathQuery, protectedSheetId, removeCourseIdScope,
shlexSplitList, splitEmailAddress, todaysTime, validateEmailAddressOrUID,
)
from util.csv_pf import CheckInputRowFilterHeaders # noqa: E402,F401
from util.display import ( # noqa: E402,F401
FIRST_ITEM_MARKER, LAST_ITEM_MARKER, TOTAL_ITEMS_MARKER, actionPerformedNumItems,
entityActionFailedWarning, entityActionNotPerformedWarning, entityActionPerformed,
entityActionPerformedMessage, entityDoesNotExistWarning, entityPerformActionNumItems,
entityServiceNotApplicableWarning, getPageMessage, getPageMessageForWhom, printBlankLine,
printGettingAllAccountEntities, printGettingAllEntityItemsForWhom,
printGotEntityItemsForWhom, printJSONKey, printJSONValue, printKeyValueList,
setGettingAllEntityItemsForWhom, userDriveServiceNotEnabledWarning,
userServiceNotEnabledWarning,
)
from util.email import send_email # noqa: E402,F401,F811
from util.entity import checkUserExists, convertUIDtoEmailAddress # noqa: E402,F401,F811
from util.errors import ( # noqa: E402,F401,F811
csvDataAlreadySavedErrorExit, csvFieldErrorExit, entityActionFailedExit,
entityDoesNotExistExit, expiredRevokedOauth2TxtExit, formatChoiceList,
invalidArgumentExit, invalidChoiceExit, invalidDiscoveryJsonExit,
invalidOauth2TxtExit, invalidOauth2serviceJsonExit, missingArgumentExit,
)
from util.fileio import ( # noqa: E402,F401,F811
StringIOobject, checkAPICallsRate, closeFile, fdErrorMessage, fileErrorMessage,
getGDocSheetDataFailedExit, getGDocSheetDataRetryWarning, incrAPICallsRetryData,
openFile, readFile, writeFile,
)
from util.gdoc import ( # noqa: E402,F401
getGDocData, getStorageFileData, openCSVFileReader,
)
from util.output import ( # noqa: E402,F401,F811
currentCountNL, flushStderr, flushStdout,
printWarningMessage, readStdin, stderrWarningMsg, writeStdout,
)

View File

@@ -16,8 +16,7 @@ from gamlib import glindent
from gamlib import glmsgs as Msg from gamlib import glmsgs as Msg
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
Act = glaction.GamAction() Act = glaction.GamAction()
@@ -27,139 +26,125 @@ Ind = glindent.GamIndent()
# Something's wrong with CustomerID?? # Something's wrong with CustomerID??
def accessErrorMessage(cd, errMsg=None): def accessErrorMessage(cd, errMsg=None):
_m = _getMain()
if cd is None: if cd is None:
cd = _m.buildGAPIObject(API.DIRECTORY) cd = _gam().buildGAPIObject(API.DIRECTORY)
try: try:
_m.callGAPI(cd.customers(), 'get', _gam().callGAPI(cd.customers(), 'get',
throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND, throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND,
GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED], GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
customerKey=GC.Values[GC.CUSTOMER_ID], fields='id') customerKey=GC.Values[GC.CUSTOMER_ID], fields='id')
except (GAPI.badRequest, GAPI.invalidInput): except (GAPI.badRequest, GAPI.invalidInput):
return _m.formatKeyValueList('', return _gam().formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID], [Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
Msg.INVALID], Msg.INVALID],
'') '')
except GAPI.resourceNotFound: except GAPI.resourceNotFound:
return _m.formatKeyValueList('', return _gam().formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID], [Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
Msg.DOES_NOT_EXIST], Msg.DOES_NOT_EXIST],
'') '')
except (GAPI.forbidden, GAPI.permissionDenied): except (GAPI.forbidden, GAPI.permissionDenied):
return _m.formatKeyValueList('', return _gam().formatKeyValueList('',
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID], Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
Ent.DOMAIN, GC.Values[GC.DOMAIN], Ent.DOMAIN, GC.Values[GC.DOMAIN],
Ent.USER, GM.Globals[GM.ADMIN]])+[Msg.ACCESS_FORBIDDEN], Ent.USER, GM.Globals[GM.ADMIN]])+[Msg.ACCESS_FORBIDDEN],
'') '')
if errMsg: if errMsg:
return _m.formatKeyValueList('', return _gam().formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID], [Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
errMsg], errMsg],
'') '')
return None return None
def accessErrorExit(cd, errMsg=None): def accessErrorExit(cd, errMsg=None):
_m = _getMain() _gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, accessErrorMessage(cd or _gam().buildGAPIObject(API.DIRECTORY), errMsg))
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, accessErrorMessage(cd or _m.buildGAPIObject(API.DIRECTORY), errMsg))
def accessErrorExitNonDirectory(api, errMsg): def accessErrorExitNonDirectory(api, errMsg):
_m = _getMain() _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC,
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, _gam().formatKeyValueList('',
_m.formatKeyValueList('',
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID], Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
Ent.DOMAIN, GC.Values[GC.DOMAIN], Ent.DOMAIN, GC.Values[GC.DOMAIN],
Ent.API, api])+[errMsg], Ent.API, api])+[errMsg],
'')) ''))
def ClientAPIAccessDeniedExit(errMsg=None): def ClientAPIAccessDeniedExit(errMsg=None):
_m = _getMain()
if errMsg is None: if errMsg is None:
_m.stderrErrorMsg(Msg.API_ACCESS_DENIED) _gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
missingScopes = API.getClientScopesSet(GM.Globals[GM.CURRENT_CLIENT_API])-GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] missingScopes = API.getClientScopesSet(GM.Globals[GM.CURRENT_CLIENT_API])-GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
if missingScopes: if missingScopes:
_m.writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID], _gam().writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
','.join(sorted(missingScopes)))) ','.join(sorted(missingScopes))))
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
else: else:
_m.stderrErrorMsg(errMsg) _gam().stderrErrorMsg(errMsg)
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
def SvcAcctAPIAccessDenied(): def SvcAcctAPIAccessDenied():
_m = _getMain() _gam()._getSvcAcctData()
_m._getSvcAcctData()
if (GM.Globals[GM.CURRENT_SVCACCT_API] == API.GMAIL and if (GM.Globals[GM.CURRENT_SVCACCT_API] == API.GMAIL and
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] and GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] and
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES][0] == API.GMAIL_SEND_SCOPE): GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES][0] == API.GMAIL_SEND_SCOPE):
_m.systemErrorExit(_m.OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED) _gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
_m.stderrErrorMsg(Msg.API_ACCESS_DENIED) _gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])) apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
_m.writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'], _gam().writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
apiOrScopes, apiOrScopes,
GM.Globals[GM.CURRENT_SVCACCT_USER] or _m._getAdminEmail())) GM.Globals[GM.CURRENT_SVCACCT_USER] or _gam()._getAdminEmail()))
def SvcAcctAPIAccessDeniedExit(): def SvcAcctAPIAccessDeniedExit():
_m = _getMain()
SvcAcctAPIAccessDenied() SvcAcctAPIAccessDenied()
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
def SvcAcctAPIDisabledExit(): def SvcAcctAPIDisabledExit():
_m = _getMain()
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]: if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
ClientAPIAccessDeniedExit() ClientAPIAccessDeniedExit()
if GM.Globals[GM.CURRENT_SVCACCT_API]: if GM.Globals[GM.CURRENT_SVCACCT_API]:
_m.stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]))) _gam().stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
def APIAccessDeniedExit(): def APIAccessDeniedExit():
_m = _getMain()
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]: if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
ClientAPIAccessDeniedExit() ClientAPIAccessDeniedExit()
if GM.Globals[GM.CURRENT_SVCACCT_API]: if GM.Globals[GM.CURRENT_SVCACCT_API]:
SvcAcctAPIAccessDeniedExit() SvcAcctAPIAccessDeniedExit()
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
def checkEntityDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0): def checkEntityDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
_m = _getMain()
message = accessErrorMessage(cd) message = accessErrorMessage(cd)
if message: if message:
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message) _gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_m.entityDoesNotExistWarning(entityType, entityName, i, count) _gam().entityDoesNotExistWarning(entityType, entityName, i, count)
def checkEntityAFDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0): def checkEntityAFDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
_m = _getMain()
message = accessErrorMessage(cd) message = accessErrorMessage(cd)
if message: if message:
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message) _gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_m.entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count) _gam().entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
def checkEntityItemValueAFDNEorAccessErrorExit(cd, entityType, entityName, itemType, itemValue, i=0, count=0): def checkEntityItemValueAFDNEorAccessErrorExit(cd, entityType, entityName, itemType, itemValue, i=0, count=0):
_m = _getMain()
message = accessErrorMessage(cd) message = accessErrorMessage(cd)
if message: if message:
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message) _gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_m.entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count) _gam().entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
def entityUnknownWarning(entityType, entityName, i=0, count=0): def entityUnknownWarning(entityType, entityName, i=0, count=0):
_m = _getMain() domain = _gam().getEmailAddressDomain(entityName)
domain = _m.getEmailAddressDomain(entityName)
if (domain.endswith(GC.Values[GC.DOMAIN])) or (domain.endswith('google.com')): if (domain.endswith(GC.Values[GC.DOMAIN])) or (domain.endswith('google.com')):
_m.entityDoesNotExistWarning(entityType, entityName, i, count) _gam().entityDoesNotExistWarning(entityType, entityName, i, count)
else: else:
_m.entityServiceNotApplicableWarning(entityType, entityName, i, count) _gam().entityServiceNotApplicableWarning(entityType, entityName, i, count)
def entityOrEntityUnknownWarning(entity1Type, entity1Name, entity2Type, entity2Name, i=0, count=0): def entityOrEntityUnknownWarning(entity1Type, entity1Name, entity2Type, entity2Name, i=0, count=0):
_m = _getMain() _gam().setSysExitRC(_gam().ENTITY_DOES_NOT_EXIST_RC)
_m.setSysExitRC(_m.ENTITY_DOES_NOT_EXIST_RC) _gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
_m.writeStderr(_m.formatKeyValueList(Ind.Spaces(), [f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _gam().getPhraseDNEorSNA(entity1Name), None,
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _m.getPhraseDNEorSNA(entity1Name), None, f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _gam().getPhraseDNEorSNA(entity2Name)],
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _m.getPhraseDNEorSNA(entity2Name)], _gam().currentCountNL(i, count)))
_m.currentCountNL(i, count)))
def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0): def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
_m = _getMain()
email = entityValueList[1] email = entityValueList[1]
try: try:
result = _m.callGAPI(cd.users(), 'get', result = _gam().callGAPI(cd.users(), 'get',
throwReasons=GAPI.USER_GET_THROW_REASONS, throwReasons=GAPI.USER_GET_THROW_REASONS,
userKey=email, fields='id,primaryEmail') userKey=email, fields='id,primaryEmail')
if (result['primaryEmail'].lower() == email) or (result['id'] == email): if (result['primaryEmail'].lower() == email) or (result['id'] == email):
@@ -169,7 +154,7 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
except (GAPI.userNotFound, GAPI.badRequest, except (GAPI.userNotFound, GAPI.badRequest,
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.backendError, GAPI.systemError): GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.backendError, GAPI.systemError):
try: try:
result = _m.callGAPI(cd.groups(), 'get', result = _gam().callGAPI(cd.groups(), 'get',
throwReasons=GAPI.GROUP_GET_THROW_REASONS, throwReasons=GAPI.GROUP_GET_THROW_REASONS,
groupKey=email, fields='id,email') groupKey=email, fields='id,email')
if (result['email'].lower() == email) or (result['id'] == email): if (result['email'].lower() == email) or (result['id'] == email):
@@ -179,10 +164,10 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
except (GAPI.groupNotFound, except (GAPI.groupNotFound,
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.badRequest): GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.badRequest):
kvList = [Ent.EMAIL, email] kvList = [Ent.EMAIL, email]
_m.writeStderr(_m.formatKeyValueList(Ind.Spaces(), _gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
Ent.FormatEntityValueList(entityValueList)+ Ent.FormatEntityValueList(entityValueList)+
[Act.Failed(), Msg.DUPLICATE]+ [Act.Failed(), Msg.DUPLICATE]+
Ent.FormatEntityValueList(kvList), Ent.FormatEntityValueList(kvList),
_m.currentCountNL(i, count))) _gam().currentCountNL(i, count)))
_m.setSysExitRC(_m.ENTITY_DUPLICATE_RC) _gam().setSysExitRC(_gam().ENTITY_DUPLICATE_RC)
return kvList[0] return kvList[0]

View File

@@ -56,8 +56,7 @@ DEVELOPER_PREVIEW_DISCOVERY_URI = "https://{api}.googleapis.com/$discovery/rest?
_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds _DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
def _getEnt(): def _getEnt():
return sys.modules['gam'].Ent return sys.modules['gam'].Ent
@@ -67,13 +66,12 @@ def _getInd():
def handleServerError(e): def handleServerError(e):
m = _getMain()
errMsg = str(e) errMsg = str(e)
if 'setting tls' not in errMsg: if 'setting tls' not in errMsg:
m.systemErrorExit(m.NETWORK_ERROR_RC, errMsg) _gam().systemErrorExit(_gam().NETWORK_ERROR_RC, errMsg)
m.stderrErrorMsg(errMsg) _gam().stderrErrorMsg(errMsg)
m.writeStderr(Msg.DISABLE_TLS_MIN_MAX) _gam().writeStderr(Msg.DISABLE_TLS_MIN_MAX)
m.systemErrorExit(m.NETWORK_ERROR_RC, None) _gam().systemErrorExit(_gam().NETWORK_ERROR_RC, None)
def getHttpObj(cache=None, timeout=None, override_min_tls=None, override_max_tls=None): def getHttpObj(cache=None, timeout=None, override_min_tls=None, override_max_tls=None):
tls_minimum_version = override_min_tls if override_min_tls else GC.Values[GC.TLS_MIN_VERSION] if GC.Values[GC.TLS_MIN_VERSION] else None tls_minimum_version = override_min_tls if override_min_tls else GC.Values[GC.TLS_MIN_VERSION] if GC.Values[GC.TLS_MIN_VERSION] else None
@@ -113,7 +111,7 @@ def _force_user_agent(user_agent):
def _lazy_force_user_agent(request_method): def _lazy_force_user_agent(request_method):
"""Wraps a request method to lazily insert GAM_USER_AGENT at call time.""" """Wraps a request method to lazily insert GAM_USER_AGENT at call time."""
def wrapped_request_method(*args, **kwargs): def wrapped_request_method(*args, **kwargs):
user_agent = _getMain().GAM_USER_AGENT user_agent = _gam().GAM_USER_AGENT
if kwargs.get('headers') is not None: if kwargs.get('headers') is not None:
if kwargs['headers'].get('user-agent'): if kwargs['headers'].get('user-agent'):
if user_agent not in kwargs['headers']['user-agent']: if user_agent not in kwargs['headers']['user-agent']:
@@ -157,14 +155,13 @@ def transportCreateRequest(httpObj=None):
return transportAgentRequest(httpObj) return transportAgentRequest(httpObj)
def doGAMCheckForUpdates(forceCheck): def doGAMCheckForUpdates(forceCheck):
m = _getMain()
Ind = _getInd() Ind = _getInd()
def _gamLatestVersionNotAvailable(): def _gamLatestVersionNotAvailable():
if forceCheck: if forceCheck:
m.systemErrorExit(m.NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE) _gam().systemErrorExit(_gam().NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
try: try:
_, c = getHttpObj(timeout=10).request(m.GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'}) _, c = getHttpObj(timeout=10).request(_gam().GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
try: try:
release_data = json.loads(c) release_data = json.loads(c)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError): except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
@@ -173,17 +170,17 @@ def doGAMCheckForUpdates(forceCheck):
if not isinstance(release_data, dict) or 'tag_name' not in release_data: if not isinstance(release_data, dict) or 'tag_name' not in release_data:
_gamLatestVersionNotAvailable() _gamLatestVersionNotAvailable()
return return
current_version = m.__version__ current_version = _gam().__version__
latest_version = release_data['tag_name'] latest_version = release_data['tag_name']
if latest_version[0].lower() == 'v': if latest_version[0].lower() == 'v':
latest_version = latest_version[1:] latest_version = latest_version[1:]
m.printKeyValueList(['Version Check', None]) _gam().printKeyValueList(['Version Check', None])
Ind.Increment() Ind.Increment()
m.printKeyValueList(['Current', current_version]) _gam().printKeyValueList(['Current', current_version])
m.printKeyValueList([' Latest', latest_version]) _gam().printKeyValueList([' Latest', latest_version])
Ind.Decrement() Ind.Decrement()
if forceCheck < 0: if forceCheck < 0:
m.setSysExitRC(1 if latest_version > current_version else 0) _gam().setSysExitRC(1 if latest_version > current_version else 0)
return return
except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError, except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError,
google.auth.exceptions.TransportError, google.auth.exceptions.TransportError,
@@ -250,13 +247,12 @@ class signjwtSignJwt(google.auth.crypt.Signer):
def sign(self, message): def sign(self, message):
''' Call IAM Credentials SignJWT API to get our signed JWT ''' ''' Call IAM Credentials SignJWT API to get our signed JWT '''
m = _getMain()
request = get_adc_request() request = get_adc_request()
try: try:
credentials, _ = google.auth.default(scopes=[API.IAM_SCOPE], credentials, _ = google.auth.default(scopes=[API.IAM_SCOPE],
request=request) request=request)
except (google.auth.exceptions.DefaultCredentialsError, google.auth.exceptions.RefreshError) as e: except (google.auth.exceptions.DefaultCredentialsError, google.auth.exceptions.RefreshError) as e:
m.systemErrorExit(m.API_ACCESS_DENIED_RC, str(e)) _gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, str(e))
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj()) httpObj = transportAuthorizedHttp(credentials, http=getHttpObj())
# refresh here so we can use the proper request from above # refresh here so we can use the proper request from above
httpObj.credentials.refresh(request) httpObj.credentials.refresh(request)
@@ -267,42 +263,40 @@ class signjwtSignJwt(google.auth.crypt.Signer):
return signed_jwt return signed_jwt
def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0): def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0):
m = _getMain()
Ent = _getEnt() Ent = _getEnt()
errMsg = str(e).replace('.', '') errMsg = str(e).replace('.', '')
if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or
errMsg.startswith('Invalid response') or errMsg.startswith('Invalid response') or
errMsg.startswith('invalid_request: Invalid impersonation &quot;sub&quot; field')): errMsg.startswith('invalid_request: Invalid impersonation &quot;sub&quot; field')):
if not GM.Globals[GM.CURRENT_SVCACCT_USER]: if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
m.ClientAPIAccessDeniedExit() _gam().ClientAPIAccessDeniedExit()
# 403 Forbidden, API disabled, user not enabled # 403 Forbidden, API disabled, user not enabled
# 400 Bad Request, user not defined # 400 Bad Request, user not defined
if softErrors: if softErrors:
m.entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count) _gam().entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
return None return None
m.systemErrorExit(m.SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER])) _gam().systemErrorExit(_gam().SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
if errMsg in API.OAUTH2_UNAUTHORIZED_ERRORS: if errMsg in API.OAUTH2_UNAUTHORIZED_ERRORS:
if not GM.Globals[GM.CURRENT_SVCACCT_USER]: if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
m.ClientAPIAccessDeniedExit() _gam().ClientAPIAccessDeniedExit()
# 401 Unauthorized, API disabled, user enabled # 401 Unauthorized, API disabled, user enabled
if softErrors: if softErrors:
if displayError: if displayError:
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])) apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
m.userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count) _gam().userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
return None return None
m.SvcAcctAPIAccessDeniedExit() _gam().SvcAcctAPIAccessDeniedExit()
if errMsg in API.REFRESH_PERM_ERRORS: if errMsg in API.REFRESH_PERM_ERRORS:
if softErrors: if softErrors:
return None return None
if not GM.Globals[GM.CURRENT_SVCACCT_USER]: if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
m.expiredRevokedOauth2TxtExit() _gam().expiredRevokedOauth2TxtExit()
m.stderrErrorMsg(f'Authentication Token Error - {errMsg}') _gam().stderrErrorMsg(f'Authentication Token Error - {errMsg}')
m.APIAccessDeniedExit() _gam().APIAccessDeniedExit()
def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnly=False, noScopes=False): def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnly=False, noScopes=False):
m = _getMain()
if not noDASA and GC.Values[GC.ENABLE_DASA]: if not noDASA and GC.Values[GC.ENABLE_DASA]:
jsonData = m.readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False) jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
if jsonData: if jsonData:
try: try:
if api in API.APIS_NEEDING_ACCESS_TOKEN: if api in API.APIS_NEEDING_ACCESS_TOKEN:
@@ -320,9 +314,9 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
sjsigner = signjwtSignJwt(jsonDict) sjsigner = signjwtSignJwt(jsonDict)
return (True, signjwtJWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience)) return (True, signjwtJWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience))
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
m.invalidOauth2serviceJsonExit(str(e)) _gam().invalidOauth2serviceJsonExit(str(e))
m.invalidOauth2serviceJsonExit(Msg.NO_DATA) _gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
jsonData = m.readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False) jsonData = _gam().readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
if jsonData: if jsonData:
try: try:
jsonDict = json.loads(jsonData) jsonDict = json.loads(jsonData)
@@ -332,11 +326,11 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
if not refreshOnly: if not refreshOnly:
if set(jsonDict.get('scopes', API.REQUIRED_SCOPES)) == API.REQUIRED_SCOPES_SET: if set(jsonDict.get('scopes', API.REQUIRED_SCOPES)) == API.REQUIRED_SCOPES_SET:
if exitOnError: if exitOnError:
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED) _gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
return (False, None) return (False, None)
else: else:
GM.Globals[GM.CREDENTIALS_SCOPES] = set(jsonDict.pop('scopes', API.REQUIRED_SCOPES)) GM.Globals[GM.CREDENTIALS_SCOPES] = set(jsonDict.pop('scopes', API.REQUIRED_SCOPES))
token_expiry = jsonDict.get('token_expiry', m.REFRESH_EXPIRY) token_expiry = jsonDict.get('token_expiry', _gam().REFRESH_EXPIRY)
if GC.Values[GC.TRUNCATE_CLIENT_ID]: if GC.Values[GC.TRUNCATE_CLIENT_ID]:
# chop off .apps.googleusercontent.com suffix as it's not needed and we need to keep things short for the Auth URL. # chop off .apps.googleusercontent.com suffix as it's not needed and we need to keep things short for the Auth URL.
jsonDict['client_id'] = re.sub(r'\.apps\.googleusercontent\.com$', '', jsonDict['client_id']) jsonDict['client_id'] = re.sub(r'\.apps\.googleusercontent\.com$', '', jsonDict['client_id'])
@@ -349,19 +343,18 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
creds.token = jsonDict['access_token'] creds.token = jsonDict['access_token']
creds._id_token = jsonDict['id_token_jwt'] creds._id_token = jsonDict['id_token_jwt']
GM.Globals[GM.DECODED_ID_TOKEN] = jsonDict['id_token'] GM.Globals[GM.DECODED_ID_TOKEN] = jsonDict['id_token']
creds.expiry = arrow.Arrow.strptime(token_expiry, m.YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive creds.expiry = arrow.Arrow.strptime(token_expiry, _gam().YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
return (not noScopes, creds) return (not noScopes, creds)
if jsonDict and exitOnError: if jsonDict and exitOnError:
m.invalidOauth2TxtExit(Msg.INVALID) _gam().invalidOauth2TxtExit(Msg.INVALID)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
if exitOnError: if exitOnError:
m.invalidOauth2TxtExit(str(e)) _gam().invalidOauth2TxtExit(str(e))
if exitOnError: if exitOnError:
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED) _gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
return (False, None) return (False, None)
def _getValueFromOAuth(field, credentials=None): def _getValueFromOAuth(field, credentials=None):
m = _getMain()
if not GM.Globals[GM.DECODED_ID_TOKEN]: if not GM.Globals[GM.DECODED_ID_TOKEN]:
request = transportCreateRequest() request = transportCreateRequest()
if credentials is None: if credentials is None:
@@ -373,9 +366,9 @@ def _getValueFromOAuth(field, credentials=None):
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS]) clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
except ValueError as e: except ValueError as e:
if 'Token used too early' in str(e): if 'Token used too early' in str(e):
m.stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME) _gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
m.systemErrorExit(m.SYSTEM_ERROR_RC, str(e)) _gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, m.UNKNOWN) return GM.Globals[GM.DECODED_ID_TOKEN].get(field, _gam().UNKNOWN)
def _getAdminEmail(): def _getAdminEmail():
if GC.Values[GC.ADMIN_EMAIL]: if GC.Values[GC.ADMIN_EMAIL]:
@@ -383,7 +376,6 @@ def _getAdminEmail():
return _getValueFromOAuth('email') return _getValueFromOAuth('email')
def writeClientCredentials(creds, filename): def writeClientCredentials(creds, filename):
m = _getMain()
creds_data = { creds_data = {
'client_id': creds.client_id, 'client_id': creds.client_id,
'client_secret': creds.client_secret, 'client_secret': creds.client_secret,
@@ -391,28 +383,27 @@ def writeClientCredentials(creds, filename):
'refresh_token': creds.refresh_token, 'refresh_token': creds.refresh_token,
'scopes': sorted(creds.scopes or GM.Globals[GM.CREDENTIALS_SCOPES]), 'scopes': sorted(creds.scopes or GM.Globals[GM.CREDENTIALS_SCOPES]),
'token': creds.token, 'token': creds.token,
'token_expiry': creds.expiry.strftime(m.YYYYMMDDTHHMMSSZ_FORMAT), 'token_expiry': creds.expiry.strftime(_gam().YYYYMMDDTHHMMSSZ_FORMAT),
'token_uri': creds.token_uri, 'token_uri': creds.token_uri,
} }
expected_iss = ['https://accounts.google.com', 'accounts.google.com'] expected_iss = ['https://accounts.google.com', 'accounts.google.com']
if _getValueFromOAuth('iss', creds) not in expected_iss: if _getValueFromOAuth('iss', creds) not in expected_iss:
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}') _gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
request = transportCreateRequest() request = transportCreateRequest()
try: try:
creds_data['decoded_id_token'] = google.oauth2.id_token.verify_oauth2_token(creds.id_token, request, creds_data['decoded_id_token'] = google.oauth2.id_token.verify_oauth2_token(creds.id_token, request,
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS]) clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
except ValueError as e: except ValueError as e:
if 'Token used too early' in str(e): if 'Token used too early' in str(e):
m.stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME) _gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
m.systemErrorExit(m.SYSTEM_ERROR_RC, str(e)) _gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
GM.Globals[GM.DECODED_ID_TOKEN] = creds_data['decoded_id_token'] GM.Globals[GM.DECODED_ID_TOKEN] = creds_data['decoded_id_token']
if filename != '-': if filename != '-':
m.writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n') _gam().writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
else: else:
m.writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n') _gam().writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
def shortenURL(long_url): def shortenURL(long_url):
m = _getMain()
if GC.Values[GC.NO_SHORT_URLS]: if GC.Values[GC.NO_SHORT_URLS]:
return long_url return long_url
httpObj = getHttpObj(timeout=10) httpObj = getHttpObj(timeout=10)
@@ -421,7 +412,7 @@ def shortenURL(long_url):
resp, content = httpObj.request(URL_SHORTENER_ENDPOINT, 'POST', resp, content = httpObj.request(URL_SHORTENER_ENDPOINT, 'POST',
payload, payload,
headers={'Content-Type': 'application/json', headers={'Content-Type': 'application/json',
'User-Agent': m.GAM_USER_AGENT}) 'User-Agent': _gam().GAM_USER_AGENT})
except: except:
return long_url return long_url
if resp.status != 200: if resp.status != 200:
@@ -440,13 +431,12 @@ def runSqliteQuery(db_file, query):
return curr.fetchone()[0] return curr.fetchone()[0]
def refreshCredentialsWithReauth(credentials): def refreshCredentialsWithReauth(credentials):
m = _getMain()
def gcloudError(): def gcloudError():
m.writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup') _gam().writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
e = Msg.REAUTHENTICATION_IS_NEEDED e = Msg.REAUTHENTICATION_IS_NEEDED
handleOAuthTokenError(e, False) handleOAuthTokenError(e, False)
m.writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH) _gam().writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
if 'termios' in sys.modules: if 'termios' in sys.modules:
import termios import termios
old_settings = termios.tcgetattr(sys.stdin) old_settings = termios.tcgetattr(sys.stdin)
@@ -454,7 +444,7 @@ def refreshCredentialsWithReauth(credentials):
# First makes sure gcloud has a valid access token and thus # First makes sure gcloud has a valid access token and thus
# should also have a valid RAPT token # should also have a valid RAPT token
try: try:
devnull = open(os.devnull, 'w', encoding=m.UTF8) devnull = open(os.devnull, 'w', encoding=_gam().UTF8)
subprocess.run(['gcloud', subprocess.run(['gcloud',
'auth', 'auth',
'print-identity-token', 'print-identity-token',
@@ -472,7 +462,7 @@ def refreshCredentialsWithReauth(credentials):
if 'termios' in sys.modules: if 'termios' in sys.modules:
import termios import termios
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings) termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
m.printBlankLine() _gam().printBlankLine()
raise KeyboardInterrupt from e raise KeyboardInterrupt from e
token_path = gcloud_path_result.stdout.decode().strip() token_path = gcloud_path_result.stdout.decode().strip()
if not token_path: if not token_path:
@@ -484,19 +474,18 @@ def refreshCredentialsWithReauth(credentials):
except TypeError: except TypeError:
gcloudError() gcloudError()
if not credentials._rapt_token: if not credentials._rapt_token:
m.systemErrorExit(m.SYSTEM_ERROR_RC, _gam().systemErrorExit(_gam().SYSTEM_ERROR_RC,
'Failed to retrieve reauth token from gcloud. You may need to wait until gcloud is also prompted for reauth.') 'Failed to retrieve reauth token from gcloud. You may need to wait until gcloud is also prompted for reauth.')
def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, api=None, noDASA=False, refreshOnly=False, noScopes=False): def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, api=None, noDASA=False, refreshOnly=False, noScopes=False):
"""Gets OAuth2 credentials which are guaranteed to be fresh and valid. """Gets OAuth2 credentials which are guaranteed to be fresh and valid.
Locks during read and possible write so that only one process will Locks during read and possible write so that only one process will
attempt refresh/write when running in parallel. """ attempt refresh/write when running in parallel. """
m = _getMain()
lock = FileLock(GM.Globals[GM.OAUTH2_TXT_LOCK], mode=GC.Values[GC.OAUTH2_TXT_LOCK_MODE]) lock = FileLock(GM.Globals[GM.OAUTH2_TXT_LOCK], mode=GC.Values[GC.OAUTH2_TXT_LOCK_MODE])
with lock: with lock:
writeCreds, credentials = getOauth2TxtCredentials(api=api, noDASA=noDASA, refreshOnly=refreshOnly, noScopes=noScopes) writeCreds, credentials = getOauth2TxtCredentials(api=api, noDASA=noDASA, refreshOnly=refreshOnly, noScopes=noScopes)
if not credentials: if not credentials:
m.invalidOauth2TxtExit('') _gam().invalidOauth2TxtExit('')
if credentials.expired or forceRefresh: if credentials.expired or forceRefresh:
triesLimit = 3 triesLimit = 3
for n in range(1, triesLimit+1): for n in range(1, triesLimit+1):
@@ -507,7 +496,7 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
break break
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit: if n != triesLimit:
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e)) waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue continue
handleServerError(e) handleServerError(e)
except google.auth.exceptions.RefreshError as e: except google.auth.exceptions.RefreshError as e:
@@ -522,14 +511,13 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
return credentials return credentials
def waitOnFailure(n, triesLimit, error_code, error_message): def waitOnFailure(n, triesLimit, error_code, error_message):
m = _getMain()
delta = min(2 ** n, 60)+float(random.randint(1, 1000))/1000 delta = min(2 ** n, 60)+float(random.randint(1, 1000))/1000
if n > 3: if n > 3:
m.writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n') _gam().writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
m.flushStderr() _gam().flushStderr()
time.sleep(delta) time.sleep(delta)
if GC.Values[GC.SHOW_API_CALLS_RETRY_DATA]: if GC.Values[GC.SHOW_API_CALLS_RETRY_DATA]:
m.incrAPICallsRetryData(error_message, delta) _gam().incrAPICallsRetryData(error_message, delta)
def clearServiceCache(service): def clearServiceCache(service):
if hasattr(service._http, 'http') and hasattr(service._http.http, 'cache'): if hasattr(service._http, 'http') and hasattr(service._http.http, 'cache'):
@@ -551,7 +539,6 @@ def getAPIService(api, httpObj):
discoveryServiceUrl=DISCOVERY_URIS[v2discovery], static_discovery=False) discoveryServiceUrl=DISCOVERY_URIS[v2discovery], static_discovery=False)
def getService(api, httpObj): def getService(api, httpObj):
m = _getMain()
hasLocalJSON = API.hasLocalJSON(api) hasLocalJSON = API.hasLocalJSON(api)
api, version, v2discovery = API.getVersion(api) api, version, v2discovery = API.getVersion(api)
if api in GM.Globals[GM.CURRENT_API_SERVICES] and version in GM.Globals[GM.CURRENT_API_SERVICES][api]: if api in GM.Globals[GM.CURRENT_API_SERVICES] and version in GM.Globals[GM.CURRENT_API_SERVICES][api]:
@@ -577,22 +564,22 @@ def getService(api, httpObj):
clearServiceCache(service) clearServiceCache(service)
return service return service
except googleapiclient.errors.UnknownApiNameOrVersion as e: except googleapiclient.errors.UnknownApiNameOrVersion as e:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), m.__author__)) _gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), _gam().__author__))
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e: except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
if n != triesLimit: if n != triesLimit:
waitOnFailure(n, triesLimit, m.INVALID_JSON_RC, str(e)) waitOnFailure(n, triesLimit, _gam().INVALID_JSON_RC, str(e))
continue continue
m.systemErrorExit(m.INVALID_JSON_RC, str(e)) _gam().systemErrorExit(_gam().INVALID_JSON_RC, str(e))
except (http.client.ResponseNotReady, OSError, googleapiclient.errors.HttpError) as e: except (http.client.ResponseNotReady, OSError, googleapiclient.errors.HttpError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}' errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit: if n != triesLimit:
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg) waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
continue continue
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg) _gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit: if n != triesLimit:
httpObj.connections = {} httpObj.connections = {}
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e)) waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue continue
handleServerError(e) handleServerError(e)
disc_file, discovery = readDiscoveryFile(f'{api}-{version}') disc_file, discovery = readDiscoveryFile(f'{api}-{version}')
@@ -604,9 +591,9 @@ def getService(api, httpObj):
clearServiceCache(service) clearServiceCache(service)
return service return service
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e: except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
m.invalidDiscoveryJsonExit(disc_file, str(e)) _gam().invalidDiscoveryJsonExit(disc_file, str(e))
except IOError as e: except IOError as e:
m.systemErrorExit(m.FILE_ERROR_RC, str(e)) _gam().systemErrorExit(_gam().FILE_ERROR_RC, str(e))
def defaultSvcAcctScopes(): def defaultSvcAcctScopes():
scopesList = API.getSvcAcctScopesList(GC.Values[GC.USER_SERVICE_ACCOUNT_ACCESS_ONLY], False) scopesList = API.getSvcAcctScopesList(GC.Values[GC.USER_SERVICE_ACCOUNT_ACCESS_ONLY], False)
@@ -622,17 +609,16 @@ def defaultSvcAcctScopes():
return saScopes return saScopes
def _getSvcAcctData(): def _getSvcAcctData():
m = _getMain()
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]: if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
jsonData = m.readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True) jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
if not jsonData: if not jsonData:
m.invalidOauth2serviceJsonExit(Msg.NO_DATA) _gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
try: try:
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = json.loads(jsonData) GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = json.loads(jsonData)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
m.invalidOauth2serviceJsonExit(str(e)) _gam().invalidOauth2serviceJsonExit(str(e))
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]: if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
m.systemErrorExit(m.OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED) _gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
requiredFields = ['client_email', 'client_id', 'project_id', 'token_uri'] requiredFields = ['client_email', 'client_id', 'project_id', 'token_uri']
key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default') key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
if key_type == 'default': if key_type == 'default':
@@ -642,7 +628,7 @@ def _getSvcAcctData():
if field not in GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]: if field not in GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
missingFields.append(field) missingFields.append(field)
if missingFields: if missingFields:
m.invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields))) _gam().invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
# Some old oauth2service.json files have: 'https://accounts.google.com/o/oauth2/auth' which no longer works # Some old oauth2service.json files have: 'https://accounts.google.com/o/oauth2/auth' which no longer works
if GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] == 'https://accounts.google.com/o/oauth2/auth': if GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] == 'https://accounts.google.com/o/oauth2/auth':
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] = API.GOOGLE_OAUTH2_TOKEN_ENDPOINT GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] = API.GOOGLE_OAUTH2_TOKEN_ENDPOINT
@@ -654,7 +640,6 @@ def _getSvcAcctData():
GM.Globals[GM.SVCACCT_SCOPES] = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].pop(API.OAUTH2SA_SCOPES) GM.Globals[GM.SVCACCT_SCOPES] = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].pop(API.OAUTH2SA_SCOPES)
def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=False): def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=False):
m = _getMain()
_getSvcAcctData() _getSvcAcctData()
if isinstance(scopesOrAPI, str): if isinstance(scopesOrAPI, str):
GM.Globals[GM.CURRENT_SVCACCT_API] = scopesOrAPI GM.Globals[GM.CURRENT_SVCACCT_API] = scopesOrAPI
@@ -665,7 +650,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
if scopesOrAPI != API.CHAT_EVENTS and not GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]: if scopesOrAPI != API.CHAT_EVENTS and not GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]:
if softErrors: if softErrors:
return None return None
m.SvcAcctAPIAccessDeniedExit() _gam().SvcAcctAPIAccessDeniedExit()
if scopesOrAPI in {API.PEOPLE, API.PEOPLE_DIRECTORY, API.PEOPLE_OTHERCONTACTS}: if scopesOrAPI in {API.PEOPLE, API.PEOPLE_DIRECTORY, API.PEOPLE_OTHERCONTACTS}:
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES].append(API.USERINFO_PROFILE_SCOPE) GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES].append(API.USERINFO_PROFILE_SCOPE)
if scopesOrAPI in {API.PEOPLE_OTHERCONTACTS}: if scopesOrAPI in {API.PEOPLE_OTHERCONTACTS}:
@@ -692,7 +677,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
except (ValueError, IndexError, KeyError) as e: except (ValueError, IndexError, KeyError) as e:
if softErrors: if softErrors:
return None return None
m.invalidOauth2serviceJsonExit(str(e)) _gam().invalidOauth2serviceJsonExit(str(e))
credentials = credentials.with_scopes(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]) credentials = credentials.with_scopes(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])
else: else:
audience = f'https://{scopesOrAPI}.googleapis.com/' audience = f'https://{scopesOrAPI}.googleapis.com/'
@@ -714,7 +699,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
except (ValueError, IndexError, KeyError) as e: except (ValueError, IndexError, KeyError) as e:
if softErrors: if softErrors:
return None return None
m.invalidOauth2serviceJsonExit(str(e)) _gam().invalidOauth2serviceJsonExit(str(e))
GM.Globals[GM.CURRENT_SVCACCT_USER] = userEmail GM.Globals[GM.CURRENT_SVCACCT_USER] = userEmail
if userEmail: if userEmail:
credentials = credentials.with_subject(userEmail) credentials = credentials.with_subject(userEmail)
@@ -723,7 +708,6 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
return credentials return credentials
def getGDataOAuthToken(gdataObj, credentials=None): def getGDataOAuthToken(gdataObj, credentials=None):
m = _getMain()
if not credentials: if not credentials:
credentials = getClientCredentials(refreshOnly=True) credentials = getClientCredentials(refreshOnly=True)
try: try:
@@ -742,14 +726,13 @@ def getGDataOAuthToken(gdataObj, credentials=None):
GM.Globals[GM.ADMIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('email', 'UNKNOWN').lower() GM.Globals[GM.ADMIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('email', 'UNKNOWN').lower()
GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id
gdataObj.domain = GC.Values[GC.DOMAIN] gdataObj.domain = GC.Values[GC.DOMAIN]
gdataObj.source = m.GAM_USER_AGENT gdataObj.source = _gam().GAM_USER_AGENT
return True return True
def checkGDataError(e, service): def checkGDataError(e, service):
m = _getMain()
error = e.args error = e.args
reason = error[0].get('reason', '') reason = error[0].get('reason', '')
body = error[0].get('body', '').decode(m.UTF8) body = error[0].get('body', '').decode(_gam().UTF8)
# First check for errors that need special handling # First check for errors that need special handling
if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']: if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']:
keep_domain = service.domain keep_domain = service.domain
@@ -758,7 +741,7 @@ def checkGDataError(e, service):
return (GDATA.TOKEN_EXPIRED, reason) return (GDATA.TOKEN_EXPIRED, reason)
error_code = getattr(e, 'error_code', 600) error_code = getattr(e, 'error_code', 600)
if GC.Values[GC.DEBUG_LEVEL] > 0: if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} {error_code}: {reason}, {body}\n') _gam().writeStdout(f'{_gam().ERROR_PREFIX} {error_code}: {reason}, {body}\n')
if error_code == 600: if error_code == 600:
if (body.startswith('Quota exceeded for the current request') or if (body.startswith('Quota exceeded for the current request') or
body.startswith('Quota exceeded for quota metric') or body.startswith('Quota exceeded for quota metric') or
@@ -859,7 +842,6 @@ def callGData(service, function,
bailOnInternalServerError=False, softErrors=False, bailOnInternalServerError=False, softErrors=False,
throwErrors=None, retryErrors=None, triesLimit=0, throwErrors=None, retryErrors=None, triesLimit=0,
**kwargs): **kwargs):
m = _getMain()
if throwErrors is None: if throwErrors is None:
throwErrors = [] throwErrors = []
if retryErrors is None: if retryErrors is None:
@@ -869,7 +851,7 @@ def callGData(service, function,
allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors
method = getattr(service, function) method = getattr(service, function)
if GC.Values[GC.API_CALLS_RATE_CHECK]: if GC.Values[GC.API_CALLS_RATE_CHECK]:
m.checkAPICallsRate() _gam().checkAPICallsRate()
for n in range(1, triesLimit+1): for n in range(1, triesLimit+1):
try: try:
return method(**kwargs) return method(**kwargs)
@@ -886,14 +868,14 @@ def callGData(service, function,
raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message) raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message)
raise raise
if softErrors: if softErrors:
m.stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}') _gam().stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
return None return None
if error_code == GDATA.INSUFFICIENT_PERMISSIONS: if error_code == GDATA.INSUFFICIENT_PERMISSIONS:
m.APIAccessDeniedExit() _gam().APIAccessDeniedExit()
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}') _gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit: if n != triesLimit:
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e)) waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue continue
handleServerError(e) handleServerError(e)
except google.auth.exceptions.RefreshError as e: except google.auth.exceptions.RefreshError as e:
@@ -904,33 +886,31 @@ def callGData(service, function,
except (http.client.ResponseNotReady, OSError) as e: except (http.client.ResponseNotReady, OSError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}' errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit: if n != triesLimit:
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg) waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
continue continue
if softErrors: if softErrors:
m.writeStderr(f'\n{m.ERROR_PREFIX}{errMsg} - Giving up.\n') _gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
return None return None
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg) _gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
def writeGotMessage(msg): def writeGotMessage(msg):
m = _getMain()
if GC.Values[GC.SHOW_GETTINGS_GOT_NL]: if GC.Values[GC.SHOW_GETTINGS_GOT_NL]:
m.writeStderr(msg) _gam().writeStderr(msg)
else: else:
m.writeStderr('\r') _gam().writeStderr('\r')
msgLen = len(msg) msgLen = len(msg)
if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]: if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]:
m.writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen)) _gam().writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
else: else:
m.writeStderr(msg) _gam().writeStderr(msg)
GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen
m.flushStderr() _gam().flushStderr()
def callGDataPages(service, function, def callGDataPages(service, function,
pageMessage=None, pageMessage=None,
softErrors=False, throwErrors=None, retryErrors=None, softErrors=False, throwErrors=None, retryErrors=None,
uri=None, uri=None,
**kwargs): **kwargs):
m = _getMain()
Ent = _getEnt() Ent = _getEnt()
if throwErrors is None: if throwErrors is None:
throwErrors = [] throwErrors = []
@@ -955,32 +935,30 @@ def callGDataPages(service, function,
nextLink = None nextLink = None
pageItems = 0 pageItems = 0
if pageMessage: if pageMessage:
show_message = pageMessage.replace(m.TOTAL_ITEMS_MARKER, str(totalItems)) show_message = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems))) writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems)))
if nextLink is None: if nextLink is None:
if pageMessage and (pageMessage[-1] != '\n'): if pageMessage and (pageMessage[-1] != '\n'):
m.writeStderr('\r\n') _gam().writeStderr('\r\n')
m.flushStderr() _gam().flushStderr()
return allResults return allResults
uri = nextLink.href uri = nextLink.href
if 'url_params' in kwargs: if 'url_params' in kwargs:
kwargs['url_params'].pop('start-index', None) kwargs['url_params'].pop('start-index', None)
def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True): def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True):
m = _getMain()
def makeErrorDict(code, reason, message): def makeErrorDict(code, reason, message):
return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}} return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}}
try: try:
error = json.loads(e.content.decode(m.UTF8)) error = json.loads(e.content.decode(_gam().UTF8))
if GC.Values[GC.DEBUG_LEVEL] > 0: if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} JSON: {str(error)}\n') _gam().writeStdout(f'{_gam().ERROR_PREFIX} JSON: {str(error)}\n')
except (IndexError, KeyError, SyntaxError, TypeError, ValueError): except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
eContent = e.content.decode(m.UTF8) if isinstance(e.content, bytes) else e.content eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
lContent = eContent.lower() lContent = eContent.lower()
if GC.Values[GC.DEBUG_LEVEL] > 0: if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} HTTP: {str(eContent)}\n') _gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
if eContent[0:15] != '<!DOCTYPE html>': if eContent[0:15] != '<!DOCTYPE html>':
if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')): if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')):
return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent) return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent)
@@ -993,7 +971,7 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
if (e.resp['status'] == '504') and ('gateway timeout' in lContent): if (e.resp['status'] == '504') and ('gateway timeout' in lContent):
return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent) return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent)
else: else:
tg = m.HTML_TITLE_PATTERN.match(lContent) tg = _gam().HTML_TITLE_PATTERN.match(lContent)
lContent = tg.group(1) if tg else 'bad request' lContent = tg.group(1) if tg else 'bad request'
if (e.resp['status'] == '403') and ('invalid domain.' in lContent): if (e.resp['status'] == '403') and ('invalid domain.' in lContent):
error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found') error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found')
@@ -1020,10 +998,10 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
elif retryOnHttpError: elif retryOnHttpError:
return (-1, None, eContent) return (-1, None, eContent)
elif softErrors: elif softErrors:
m.stderrErrorMsg(eContent) _gam().stderrErrorMsg(eContent)
return (0, None, None) return (0, None, None)
else: else:
m.systemErrorExit(m.HTTP_ERROR_RC, eContent) _gam().systemErrorExit(_gam().HTTP_ERROR_RC, eContent)
requiredScopes = '' requiredScopes = ''
wwwAuthenticate = e.resp.get('www-authenticate', '') wwwAuthenticate = e.resp.get('www-authenticate', '')
if 'insufficient_scope' in wwwAuthenticate: if 'insufficient_scope' in wwwAuthenticate:
@@ -1129,9 +1107,9 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
http_status = 400 http_status = 400
error = makeErrorDict(http_status, GAPI.INVALID, message) error = makeErrorDict(http_status, GAPI.INVALID, message)
else: else:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(error)) _gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
else: else:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(error)) _gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
try: try:
reason = error['error']['errors'][0]['reason'] reason = error['error']['errors'][0]['reason']
for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []): for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []):
@@ -1157,7 +1135,6 @@ def callGAPI(service, function,
softErrors=False, mapNotFound=True, softErrors=False, mapNotFound=True,
throwReasons=None, retryReasons=None, triesLimit=0, throwReasons=None, retryReasons=None, triesLimit=0,
**kwargs): **kwargs):
m = _getMain()
if throwReasons is None: if throwReasons is None:
throwReasons = [] throwReasons = []
if retryReasons is None: if retryReasons is None:
@@ -1168,7 +1145,7 @@ def callGAPI(service, function,
method = getattr(service, function) method = getattr(service, function)
svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST]) svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST])
if GC.Values[GC.API_CALLS_RATE_CHECK]: if GC.Values[GC.API_CALLS_RATE_CHECK]:
m.checkAPICallsRate() _gam().checkAPICallsRate()
for n in range(1, triesLimit+1): for n in range(1, triesLimit+1):
try: try:
return method(**svcparms).execute() return method(**svcparms).execute()
@@ -1181,7 +1158,7 @@ def callGAPI(service, function,
# service._http.credentials.refresh(getHttpObj()) # service._http.credentials.refresh(getHttpObj())
service._http.credentials.refresh(transportCreateRequest()) service._http.credentials.refresh(transportCreateRequest())
except TypeError: except TypeError:
m.systemErrorExit(m.HTTP_ERROR_RC, message) _gam().systemErrorExit(_gam().HTTP_ERROR_RC, message)
continue continue
if http_status == 0: if http_status == 0:
return None return None
@@ -1202,17 +1179,17 @@ def callGAPI(service, function,
raise GAPI.REASON_EXCEPTION_MAP[reason](message) raise GAPI.REASON_EXCEPTION_MAP[reason](message)
raise e raise e
if softErrors: if softErrors:
m.stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}') _gam().stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
return None return None
if reason == GAPI.INSUFFICIENT_PERMISSIONS: if reason == GAPI.INSUFFICIENT_PERMISSIONS:
m.APIAccessDeniedExit() _gam().APIAccessDeniedExit()
m.systemErrorExit(m.HTTP_ERROR_RC, m.formatHTTPError(http_status, reason, message)) _gam().systemErrorExit(_gam().HTTP_ERROR_RC, _gam().formatHTTPError(http_status, reason, message))
except googleapiclient.errors.MediaUploadSizeError as e: except googleapiclient.errors.MediaUploadSizeError as e:
raise e raise e
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit: if n != triesLimit:
service._http.connections = {} service._http.connections = {}
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e)) waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue continue
handleServerError(e) handleServerError(e)
except google.auth.exceptions.RefreshError as e: except google.auth.exceptions.RefreshError as e:
@@ -1223,23 +1200,22 @@ def callGAPI(service, function,
except (http.client.ResponseNotReady, OSError) as e: except (http.client.ResponseNotReady, OSError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}' errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit: if n != triesLimit:
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg) waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
continue continue
if softErrors: if softErrors:
m.writeStderr(f'\n{m.ERROR_PREFIX}{errMsg} - Giving up.\n') _gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
return None return None
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg) _gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
except ValueError as e: except ValueError as e:
if clearServiceCache(service): if clearServiceCache(service):
continue continue
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(e)) _gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
except TypeError as e: except TypeError as e:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(e)) _gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType): def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType):
m = _getMain()
Ent = _getEnt() Ent = _getEnt()
showMessage = pageMessage.replace(m.TOTAL_ITEMS_MARKER, str(totalItems)) showMessage = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
if pageItems: if pageItems:
if messageAttribute: if messageAttribute:
firstItem = results[0] if pageItems > 0 else {} firstItem = results[0] if pageItems > 0 else {}
@@ -1253,11 +1229,11 @@ def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAtt
lastItem = lastItem.get(attr, {}) lastItem = lastItem.get(attr, {})
firstItem = str(firstItem) firstItem = str(firstItem)
lastItem = str(lastItem) lastItem = str(lastItem)
showMessage = showMessage.replace(m.FIRST_ITEM_MARKER, firstItem) showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, firstItem)
showMessage = showMessage.replace(m.LAST_ITEM_MARKER, lastItem) showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, lastItem)
else: else:
showMessage = showMessage.replace(m.FIRST_ITEM_MARKER, '') showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, '')
showMessage = showMessage.replace(m.LAST_ITEM_MARKER, '') showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, '')
writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems)))) writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems))))
def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType): def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType):
@@ -1280,10 +1256,9 @@ def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage,
return (pageToken, totalItems) return (pageToken, totalItems)
def _finalizeGAPIpagesResult(pageMessage): def _finalizeGAPIpagesResult(pageMessage):
m = _getMain()
if pageMessage and (pageMessage[-1] != '\n'): if pageMessage and (pageMessage[-1] != '\n'):
m.writeStderr('\r\n') _gam().writeStderr('\r\n')
m.flushStderr() _gam().flushStderr()
def _setMaxArgResults(maxItems, pageArgsInBody, kwargs): def _setMaxArgResults(maxItems, pageArgsInBody, kwargs):
if pageArgsInBody: if pageArgsInBody:
@@ -1403,25 +1378,23 @@ def callGAPIitems(service, function, items,
return [] return []
def readDiscoveryFile(api_version): def readDiscoveryFile(api_version):
m = _getMain()
disc_filename = f'{api_version}.json' disc_filename = f'{api_version}.json'
disc_file = os.path.join(GM.Globals[GM.GAM_PATH], disc_filename) disc_file = os.path.join(GM.Globals[GM.GAM_PATH], disc_filename)
if hasattr(sys, '_MEIPASS'): if hasattr(sys, '_MEIPASS'):
json_string = m.readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member json_string = _gam().readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
elif os.path.isfile(disc_file): elif os.path.isfile(disc_file):
json_string = m.readFile(disc_file, continueOnError=True, displayError=True) json_string = _gam().readFile(disc_file, continueOnError=True, displayError=True)
else: else:
json_string = None json_string = None
if not json_string: if not json_string:
m.invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA) _gam().invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
try: try:
discovery = json.loads(json_string) discovery = json.loads(json_string)
return (disc_file, discovery) return (disc_file, discovery)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
m.invalidDiscoveryJsonExit(disc_file, str(e)) _gam().invalidDiscoveryJsonExit(disc_file, str(e))
def buildGAPIObject(api, credentials=None): def buildGAPIObject(api, credentials=None):
m = _getMain()
if credentials is None: if credentials is None:
credentials = getClientCredentials(api=api, refreshOnly=True) credentials = getClientCredentials(api=api, refreshOnly=True)
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj(cache=GM.Globals[GM.CACHE_DIR])) httpObj = transportAuthorizedHttp(credentials, http=getHttpObj(cache=GM.Globals[GM.CACHE_DIR]))
@@ -1433,7 +1406,7 @@ def buildGAPIObject(api, credentials=None):
GM.Globals[GM.CURRENT_CLIENT_API] = api GM.Globals[GM.CURRENT_CLIENT_API] = api
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API_Scopes.intersection(GM.Globals[GM.CREDENTIALS_SCOPES]) GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API_Scopes.intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
if api not in API.SCOPELESS_APIS and not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]: if api not in API.SCOPELESS_APIS and not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
m.systemErrorExit(m.NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api))) _gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
if not GC.Values[GC.DOMAIN]: if not GC.Values[GC.DOMAIN]:
GC.Values[GC.DOMAIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('hd', 'UNKNOWN').lower() GC.Values[GC.DOMAIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('hd', 'UNKNOWN').lower()
if not GC.Values[GC.CUSTOMER_ID]: if not GC.Values[GC.CUSTOMER_ID]:
@@ -1443,10 +1416,9 @@ def buildGAPIObject(api, credentials=None):
return service return service
def getSaUser(user): def getSaUser(user):
m = _getMain()
currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API] currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API]
currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
userEmail = m.convertUIDtoEmailAddress(user) if user else None userEmail = _gam().convertUIDtoEmailAddress(user) if user else None
GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes
return userEmail return userEmail
@@ -1458,7 +1430,6 @@ def chooseSaAPI(api1, api2):
return api2 return api2
def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True): def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
m = _getMain()
userEmail = getSaUser(user) userEmail = getSaUser(user)
if GM.Globals[GM.HTTP_OBJECT] is None: if GM.Globals[GM.HTTP_OBJECT] is None:
GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR]) GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR])
@@ -1475,7 +1446,7 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit: if n != triesLimit:
httpObj.connections = {} httpObj.connections = {}
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e)) waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue continue
handleServerError(e) handleServerError(e)
except google.auth.exceptions.RefreshError as e: except google.auth.exceptions.RefreshError as e:
@@ -1485,12 +1456,12 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
if isinstance(e, str): if isinstance(e, str):
eContent = e eContent = e
else: else:
eContent = e.content.decode(m.UTF8) if isinstance(e.content, bytes) else e.content eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
if eContent[0:15] == '<!DOCTYPE html>': if eContent[0:15] == '<!DOCTYPE html>':
if GC.Values[GC.DEBUG_LEVEL] > 0: if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} HTTP: {str(eContent)}\n') _gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
lContent = eContent.lower() lContent = eContent.lower()
tg = m.HTML_TITLE_PATTERN.match(lContent) tg = _gam().HTML_TITLE_PATTERN.match(lContent)
lContent = tg.group(1) if tg else '' lContent = tg.group(1) if tg else ''
if lContent.startswith('Error 502 (Server Error)'): if lContent.startswith('Error 502 (Server Error)'):
time.sleep(30) time.sleep(30)
@@ -1504,12 +1475,11 @@ def buildGAPIObjectNoAuthentication(api):
return service return service
def initGDataObject(gdataObj, api): def initGDataObject(gdataObj, api):
m = _getMain()
GM.Globals[GM.CURRENT_CLIENT_API] = api GM.Globals[GM.CURRENT_CLIENT_API] = api
credentials = getClientCredentials(noDASA=True, refreshOnly=True) credentials = getClientCredentials(noDASA=True, refreshOnly=True)
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API.getClientScopesSet(api).intersection(GM.Globals[GM.CREDENTIALS_SCOPES]) GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API.getClientScopesSet(api).intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
if not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]: if not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
m.systemErrorExit(m.NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api))) _gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
getGDataOAuthToken(gdataObj, credentials) getGDataOAuthToken(gdataObj, credentials)
if GC.Values[GC.DEBUG_LEVEL] > 0: if GC.Values[GC.DEBUG_LEVEL] > 0:
gdataObj.debug = True gdataObj.debug = True

View File

@@ -115,8 +115,7 @@ def _getEnt():
return sys.modules['gam'].Ent return sys.modules['gam'].Ent
# Lazy accessor for main module # Lazy accessor for main module
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
# --- Constants duplicated from __init__.py --- # --- Constants duplicated from __init__.py ---
# These are simple literals that never change, duplicated to avoid # These are simple literals that never change, duplicated to avoid

View File

@@ -27,8 +27,7 @@ from gamlib import glmsgs as Msg
from util.csv_pf import CSVPrintFile from util.csv_pf import CSVPrintFile
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
Cmd = glclargs.GamCLArgs() Cmd = glclargs.GamCLArgs()
@@ -72,18 +71,18 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
def reopenSTDFile(stdtype): def reopenSTDFile(stdtype):
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null': if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_getMain().UTF8) GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-': elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][stdtype == GM.STDOUT]), GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][stdtype == GM.STDOUT]),
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING]) GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
elif stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout': elif stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
GM.Globals[stdtype][GM.REDIRECT_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_FD] GM.Globals[stdtype][GM.REDIRECT_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_FD]
else: else:
GM.Globals[stdtype][GM.REDIRECT_FD] = _getMain().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE]) GM.Globals[stdtype][GM.REDIRECT_FD] = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
if stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout': if stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD] GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
else: else:
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _getMain().StringIOobject() GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _gam().StringIOobject()
GM.Globals[GM.DATETIME_NOW] = datetimeNow GM.Globals[GM.DATETIME_NOW] = datetimeNow
GC.Values[GC.TIMEZONE] = tzinfo GC.Values[GC.TIMEZONE] = tzinfo
@@ -158,11 +157,11 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
if mpQueueStdout: if mpQueueStdout:
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue())) mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()))
else: else:
_getMain().flushStdout() _gam().flushStdout()
if mpQueueStderr and mpQueueStderr is not mpQueueStdout: if mpQueueStderr and mpQueueStderr is not mpQueueStdout:
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD].getvalue())) mpQueueStderr.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD].getvalue()))
else: else:
_getMain().flushStderr() _gam().flushStderr()
def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr): def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr):
mpQueue = mpManager.Queue() mpQueue = mpManager.Queue()
@@ -178,7 +177,7 @@ def terminateCSVFileQueueHandler(mpQueue, mpQueueHandler):
GM.Globals[GM.PARSER] = None GM.Globals[GM.PARSER] = None
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None
if multiprocessing.get_start_method() != 'fork': if multiprocessing.get_start_method() != 'fork':
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _getMain().Cmd.AllArguments())) mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _gam().Cmd.AllArguments()))
savedValues = saveNonPickleableValues() savedValues = saveNonPickleableValues()
mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals)) mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals))
restoreNonPickleableValues(savedValues) restoreNonPickleableValues(savedValues)
@@ -200,27 +199,27 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
if data[1] is not None: if data[1] is not None:
_writeData(data[1]) _writeData(data[1])
if GC.Values[GC.SHOW_MULTIPROCESS_INFO]: if GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _getMain().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd'])) _writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _gam().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
fd.flush() fd.flush()
except IOError as e: except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e)) _gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
if multiprocessing.get_start_method() != 'fork': if multiprocessing.get_start_method() != 'fork':
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
GM.Globals = gmGlobals.copy() GM.Globals = gmGlobals.copy()
GC.Values = gcValues.copy() GC.Values = gcValues.copy()
pid0DataItem = [_getMain().KEYBOARD_INTERRUPT_RC, None] pid0DataItem = [_gam().KEYBOARD_INTERRUPT_RC, None]
pidData = {} pidData = {}
if multiprocessing.get_start_method() != 'fork': if multiprocessing.get_start_method() != 'fork':
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null': if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_getMain().UTF8) fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-': elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
fd = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stdout']), fd = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stdout']),
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING]) GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout' and GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stderr': elif GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout' and GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stderr':
fd = os.fdopen(os.dup(sys.stdout.fileno()), GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING]) fd = os.fdopen(os.dup(sys.stdout.fileno()), GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
else: else:
fd = _getMain().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE]) fd = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
else: else:
fd = GM.Globals[stdtype][GM.REDIRECT_FD] fd = GM.Globals[stdtype][GM.REDIRECT_FD]
while True: while True:
@@ -230,7 +229,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
break break
if dataType == GM.REDIRECT_QUEUE_START: if dataType == GM.REDIRECT_QUEUE_START:
pidData[pid] = {'queue': GM.Globals[stdtype][GM.REDIRECT_QUEUE], pidData[pid] = {'queue': GM.Globals[stdtype][GM.REDIRECT_QUEUE],
'start': _getMain().currentISOformatTimeStamp(), 'start': _gam().currentISOformatTimeStamp(),
'cmd': Cmd.QuotedArgumentList(dataItem)} 'cmd': Cmd.QuotedArgumentList(dataItem)}
if pid == 0 and GC.Values[GC.SHOW_MULTIPROCESS_INFO]: if pid == 0 and GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
fd.write(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'Start', pidData[pid]['start'], 0, pidData[pid]['cmd'])) fd.write(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'Start', pidData[pid]['start'], 0, pidData[pid]['cmd']))
@@ -246,7 +245,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
break break
for pid in pidData: for pid in pidData:
if pid != 0: if pid != 0:
_writePidData(pid, [_getMain().KEYBOARD_INTERRUPT_RC, None]) _writePidData(pid, [_gam().KEYBOARD_INTERRUPT_RC, None])
_writePidData(0, pid0DataItem) _writePidData(0, pid0DataItem)
if fd not in [sys.stdout, sys.stderr]: if fd not in [sys.stdout, sys.stderr]:
try: try:
@@ -267,7 +266,7 @@ def batchWriteStderr(data):
sys.stderr.write(data) sys.stderr.write(data)
sys.stderr.flush() sys.stderr.flush()
except IOError as e: except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage('stderr', e)) _gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage('stderr', e))
def writeStdQueueHandler(mpQueue, item): def writeStdQueueHandler(mpQueue, item):
while True: while True:
@@ -276,7 +275,7 @@ def writeStdQueueHandler(mpQueue, item):
return return
except Exception as e: except Exception as e:
time.sleep(1) time.sleep(1)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
def terminateStdQueueHandler(mpQueue, mpQueueHandler): def terminateStdQueueHandler(mpQueue, mpQueueHandler):
mpQueue.put((0, GM.REDIRECT_QUEUE_EOF, None)) mpQueue.put((0, GM.REDIRECT_QUEUE_EOF, None))
@@ -339,7 +338,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
if mpQueueCSVFile: if mpQueueCSVFile:
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = mpQueueCSVFile GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = mpQueueCSVFile
if mpQueueStdout: if mpQueueStdout:
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _getMain().StringIOobject()} GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
if debugLevel: if debugLevel:
sys.stdout = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD] sys.stdout = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_START, args)) # mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_START, args))
@@ -348,14 +347,14 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
GM.Globals[GM.STDOUT] = {} GM.Globals[GM.STDOUT] = {}
if mpQueueStderr: if mpQueueStderr:
if mpQueueStderr is not mpQueueStdout: if mpQueueStderr is not mpQueueStdout:
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _getMain().StringIOobject()} GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
# mpQueueStderr.put((pid, GM.REDIRECT_QUEUE_START, args)) # mpQueueStderr.put((pid, GM.REDIRECT_QUEUE_START, args))
writeStdQueueHandler(mpQueueStderr, (pid, GM.REDIRECT_QUEUE_START, args)) writeStdQueueHandler(mpQueueStderr, (pid, GM.REDIRECT_QUEUE_START, args))
else: else:
GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD] GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
else: else:
GM.Globals[GM.STDERR] = {} GM.Globals[GM.STDERR] = {}
sysRC = _getMain().ProcessGAMCommand(args) sysRC = _gam().ProcessGAMCommand(args)
with mplock: with mplock:
if mpQueueStdout: if mpQueueStdout:
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_END, [sysRC, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()])) # mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_END, [sysRC, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()]))
@@ -405,9 +404,9 @@ def MultiprocessGAMCommands(items, showCmds):
def poolCallback(result): def poolCallback(result):
poolProcessResults[0] -= 1 poolProcessResults[0] -= 1
if showCmds: if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
if GM.Globals[GM.CMDLOG_LOGGER]: if GM.Globals[GM.CMDLOG_LOGGER]:
GM.Globals[GM.CMDLOG_LOGGER].info(f'{_getMain().currentISOformatTimeStamp()},{result[1]},{result[2]}') GM.Globals[GM.CMDLOG_LOGGER].info(f'{_gam().currentISOformatTimeStamp()},{result[1]},{result[2]}')
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(result[1]): if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(result[1]):
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
@@ -418,8 +417,8 @@ def MultiprocessGAMCommands(items, showCmds):
def handleControlC(source): def handleControlC(source):
nonlocal controlC nonlocal controlC
batchWriteStderr(f'Control-C (Multiprocess-{source})\n') batchWriteStderr(f'Control-C (Multiprocess-{source})\n')
_getMain().setSysExitRC(_getMain().KEYBOARD_INTERRUPT_RC) _gam().setSysExitRC(_gam().KEYBOARD_INTERRUPT_RC)
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0], numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1])) PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
pool.terminate() pool.terminate()
@@ -445,21 +444,21 @@ def MultiprocessGAMCommands(items, showCmds):
else: else:
pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200) pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
except IOError as e: except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, e) _gam().systemErrorExit(_gam().FILE_ERROR_RC, e)
except AssertionError as e: except AssertionError as e:
_getMain().Cmd.SetLocation(0) _gam().Cmd.SetLocation(0)
_getMain().usageErrorExit(str(e)) _gam().usageErrorExit(str(e))
if multiprocessing.get_start_method() != 'fork': if multiprocessing.get_start_method() != 'fork':
savedValues = saveNonPickleableValues() savedValues = saveNonPickleableValues()
if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]: if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]:
mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values) mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values)
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _getMain().Cmd.AllArguments())) mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
else: else:
mpQueueStdout = None mpQueueStdout = None
if GM.Globals[GM.STDERR][GM.REDIRECT_MULTIPROCESS]: if GM.Globals[GM.STDERR][GM.REDIRECT_MULTIPROCESS]:
if GM.Globals[GM.STDERR][GM.REDIRECT_NAME] != 'stdout': if GM.Globals[GM.STDERR][GM.REDIRECT_NAME] != 'stdout':
mpQueueStderr, mpQueueHandlerStderr = initializeStdQueueHandler(mpManager, GM.STDERR, GM.Globals, GC.Values) mpQueueStderr, mpQueueHandlerStderr = initializeStdQueueHandler(mpManager, GM.STDERR, GM.Globals, GC.Values)
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _getMain().Cmd.AllArguments())) mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
else: else:
mpQueueStderr = mpQueueStdout mpQueueStderr = mpQueueStdout
else: else:
@@ -479,7 +478,7 @@ def MultiprocessGAMCommands(items, showCmds):
# signal.signal(signal.SIGINT, origSigintHandler) # signal.signal(signal.SIGINT, origSigintHandler)
controlC = False controlC = False
signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGINT, signal_handler)
batchWriteStderr(Msg.USING_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, numPoolProcesses, numItems, numPoolProcesses,
PROCESS_PLURAL_SINGULAR[numPoolProcesses == 1])) PROCESS_PLURAL_SINGULAR[numPoolProcesses == 1]))
try: try:
@@ -491,7 +490,7 @@ def MultiprocessGAMCommands(items, showCmds):
if controlC: if controlC:
break break
if item[0] == Cmd.COMMIT_BATCH_CMD: if item[0] == Cmd.COMMIT_BATCH_CMD:
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0], numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1])) PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
while poolProcessResults[0] > 0: while poolProcessResults[0] > 0:
@@ -503,24 +502,24 @@ def MultiprocessGAMCommands(items, showCmds):
completedProcesses.append(p) completedProcesses.append(p)
for p in completedProcesses: for p in completedProcesses:
del poolProcessResults[p] del poolProcessResults[p]
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems, Msg.PROCESSES)) batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
if len(item) > 1: if len(item) > 1:
_getMain().readStdin(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}') _gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
continue continue
if item[0] == Cmd.PRINT_CMD: if item[0] == Cmd.PRINT_CMD:
batchWriteStderr(Cmd.QuotedArgumentList(item[1:])+'\n') batchWriteStderr(Cmd.QuotedArgumentList(item[1:])+'\n')
continue continue
if item[0] == Cmd.SLEEP_CMD: if item[0] == Cmd.SLEEP_CMD:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
time.sleep(int(item[1])) time.sleep(int(item[1]))
continue continue
pid += 1 pid += 1
if not showCmds and ((pid % 100 == 0) or (pid == numItems)): if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_getMain().currentISOformatTimeStamp(), pid, numItems)) batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
if showCmds or GM.Globals[GM.CMDLOG_LOGGER]: if showCmds or GM.Globals[GM.CMDLOG_LOGGER]:
logCmd = Cmd.QuotedArgumentList(item) logCmd = Cmd.QuotedArgumentList(item)
if showCmds: if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
else: else:
logCmd = '' logCmd = ''
poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti, poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti,
@@ -566,7 +565,7 @@ def MultiprocessGAMCommands(items, showCmds):
else: else:
waitRemaining = 'unlimited' waitRemaining = 'unlimited'
while poolProcessResults[0] > 0: while poolProcessResults[0] > 0:
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0], numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1], PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1],
Msg.BATCH_CSV_WAIT_LIMIT.format(waitRemaining))) Msg.BATCH_CSV_WAIT_LIMIT.format(waitRemaining)))
@@ -585,7 +584,7 @@ def MultiprocessGAMCommands(items, showCmds):
if GC.Values[GC.PROCESS_WAIT_LIMIT] > 0: if GC.Values[GC.PROCESS_WAIT_LIMIT] > 0:
delta = int(time.time()-processWaitStart) delta = int(time.time()-processWaitStart)
if delta >= GC.Values[GC.PROCESS_WAIT_LIMIT]: if delta >= GC.Values[GC.PROCESS_WAIT_LIMIT]:
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0], numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1])) PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
pool.terminate() pool.terminate()
@@ -597,7 +596,7 @@ def MultiprocessGAMCommands(items, showCmds):
except KeyboardInterrupt: except KeyboardInterrupt:
handleControlC('KBI') handleControlC('KBI')
pool.join() pool.join()
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems)) batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems))
if mpQueueCSVFile: if mpQueueCSVFile:
terminateCSVFileQueueHandler(mpQueueCSVFile, mpQueueHandlerCSVFile) terminateCSVFileQueueHandler(mpQueueCSVFile, mpQueueHandlerCSVFile)
if mpQueueStdout: if mpQueueStdout:
@@ -618,11 +617,11 @@ def threadBatchWorker(showCmds=False, numItems=0):
sysRC = subprocess.call(item, stdout=GM.Globals[GM.STDOUT].get(GM.REDIRECT_MULTI_FD, sys.stdout), sysRC = subprocess.call(item, stdout=GM.Globals[GM.STDOUT].get(GM.REDIRECT_MULTI_FD, sys.stdout),
stderr=GM.Globals[GM.STDERR].get(GM.REDIRECT_MULTI_FD, sys.stderr)) stderr=GM.Globals[GM.STDERR].get(GM.REDIRECT_MULTI_FD, sys.stderr))
if showCmds: if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(sysRC): if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(sysRC):
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
except Exception as e: except Exception as e:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
GM.Globals[GM.TBATCH_QUEUE].task_done() GM.Globals[GM.TBATCH_QUEUE].task_done()
BATCH_COMMANDS = [Cmd.GAM_CMD, Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD, Cmd.SLEEP_CMD, Cmd.DATETIME_CMD, Cmd.SET_CMD, Cmd.CLEAR_CMD] BATCH_COMMANDS = [Cmd.GAM_CMD, Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD, Cmd.SLEEP_CMD, Cmd.DATETIME_CMD, Cmd.SET_CMD, Cmd.CLEAR_CMD]
@@ -633,12 +632,12 @@ def ThreadBatchGAMCommands(items, showCmds):
return return
pythonCmd = [sys.executable] pythonCmd = [sys.executable]
if not getattr(sys, 'frozen', False): # we're not frozen if not getattr(sys, 'frozen', False): # we're not frozen
pythonCmd.append(os.path.realpath(_getMain().Cmd.Argument(0))) pythonCmd.append(os.path.realpath(_gam().Cmd.Argument(0)))
GM.Globals[GM.NUM_BATCH_ITEMS] = numItems = len(items) GM.Globals[GM.NUM_BATCH_ITEMS] = numItems = len(items)
numWorkerThreads = min(numItems, GC.Values[GC.NUM_TBATCH_THREADS]) numWorkerThreads = min(numItems, GC.Values[GC.NUM_TBATCH_THREADS])
# GM.Globals[GM.TBATCH_QUEUE].put() gets blocked when trying to create more items than there are workers # GM.Globals[GM.TBATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
GM.Globals[GM.TBATCH_QUEUE] = queue.Queue(maxsize=numWorkerThreads) GM.Globals[GM.TBATCH_QUEUE] = queue.Queue(maxsize=numWorkerThreads)
batchWriteStderr(Msg.USING_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, numWorkerThreads, numItems, numWorkerThreads,
THREAD_PLURAL_SINGULAR[numWorkerThreads == 1])) THREAD_PLURAL_SINGULAR[numWorkerThreads == 1]))
for _ in range(numWorkerThreads): for _ in range(numWorkerThreads):
@@ -651,28 +650,28 @@ def ThreadBatchGAMCommands(items, showCmds):
if GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING]: if GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING]:
break break
if item[0] == Cmd.COMMIT_BATCH_CMD: if item[0] == Cmd.COMMIT_BATCH_CMD:
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(), batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, numThreadsInUse, numItems, numThreadsInUse,
THREAD_PLURAL_SINGULAR[numThreadsInUse == 1])) THREAD_PLURAL_SINGULAR[numThreadsInUse == 1]))
GM.Globals[GM.TBATCH_QUEUE].join() GM.Globals[GM.TBATCH_QUEUE].join()
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems, Msg.THREADS)) batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.THREADS))
numThreadsInUse = 0 numThreadsInUse = 0
if len(item) > 1: if len(item) > 1:
_getMain().readStdin(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}') _gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
continue continue
if item[0] == Cmd.PRINT_CMD: if item[0] == Cmd.PRINT_CMD:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
continue continue
if item[0] == Cmd.SLEEP_CMD: if item[0] == Cmd.SLEEP_CMD:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
time.sleep(int(item[1])) time.sleep(int(item[1]))
continue continue
pid += 1 pid += 1
if not showCmds and ((pid % 100 == 0) or (pid == numItems)): if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_getMain().currentISOformatTimeStamp(), pid, numItems)) batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
if showCmds: if showCmds:
logCmd = Cmd.QuotedArgumentList(item) logCmd = Cmd.QuotedArgumentList(item)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
else: else:
logCmd = '' logCmd = ''
if item[0] == Cmd.GAM_CMD: if item[0] == Cmd.GAM_CMD:
@@ -682,43 +681,43 @@ def ThreadBatchGAMCommands(items, showCmds):
numThreadsInUse += 1 numThreadsInUse += 1
GM.Globals[GM.TBATCH_QUEUE].join() GM.Globals[GM.TBATCH_QUEUE].join()
if showCmds: if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Complete\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Complete\n')
def _getShowCommands(): def _getShowCommands():
if _getMain().checkArgumentPresent('showcmds'): if _gam().checkArgumentPresent('showcmds'):
return _getMain().getBoolean() return _gam().getBoolean()
return GC.Values[GC.SHOW_COMMANDS] return GC.Values[GC.SHOW_COMMANDS]
def _getSkipRows(): def _getSkipRows():
if _getMain().checkArgumentPresent('skiprows'): if _gam().checkArgumentPresent('skiprows'):
return _getMain().getInteger(minVal=0) return _gam().getInteger(minVal=0)
# return GC.Values[GC.CSV_INPUT_ROW_SKIP] # return GC.Values[GC.CSV_INPUT_ROW_SKIP]
return 0 return 0
def _getMaxRows(): def _getMaxRows():
if _getMain().checkArgumentPresent('maxrows'): if _gam().checkArgumentPresent('maxrows'):
return _getMain().getInteger(minVal=0) return _gam().getInteger(minVal=0)
return GC.Values[GC.CSV_INPUT_ROW_LIMIT] return GC.Values[GC.CSV_INPUT_ROW_LIMIT]
# gam batch <BatchContent> [showcmds [<Boolean>]] # gam batch <BatchContent> [showcmds [<Boolean>]]
def doBatch(threadBatch=False): def doBatch(threadBatch=False):
filename = _getMain().getString(Cmd.OB_FILE_NAME) filename = _gam().getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0): if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_getMain().Cmd.Backup() _gam().Cmd.Backup()
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD)) _gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
filenameLower = filename.lower() filenameLower = filename.lower()
if filenameLower not in {'gdoc', 'gcsdoc'}: if filenameLower not in {'gdoc', 'gcsdoc'}:
encoding = _getMain().getCharSet() encoding = _gam().getCharSet()
filename = _getMain().setFilePath(filename, GC.INPUT_DIR) filename = _gam().setFilePath(filename, GC.INPUT_DIR)
f = _getMain().openFile(filename, encoding=encoding, stripUTFBOM=True) f = _gam().openFile(filename, encoding=encoding, stripUTFBOM=True)
elif filenameLower == 'gdoc': elif filenameLower == 'gdoc':
f = _getMain().getGDocData(filenameLower) f = _gam().getGDocData(filenameLower)
_getMain().getCharSet() _gam().getCharSet()
else: #filenameLower == 'gcsdoc': else: #filenameLower == 'gcsdoc':
f = _getMain().getStorageFileData(filenameLower) f = _gam().getStorageFileData(filenameLower)
_getMain().getCharSet() _gam().getCharSet()
showCmds = _getShowCommands() showCmds = _getShowCommands()
_getMain().checkForExtraneousArguments() _gam().checkForExtraneousArguments()
validCommands = BATCH_COMMANDS if not threadBatch else TBATCH_COMMANDS validCommands = BATCH_COMMANDS if not threadBatch else TBATCH_COMMANDS
kwValues = {} kwValues = {}
items = [] items = []
@@ -733,40 +732,40 @@ def doBatch(threadBatch=False):
try: try:
argv = shlex.split(line) argv = shlex.split(line)
except ValueError as e: except ValueError as e:
_getMain().writeStderr(f'Command: >>>{line.strip()}<<<\n') _gam().writeStderr(f'Command: >>>{line.strip()}<<<\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{str(e)}\n') _gam().writeStderr(f'{_gam().ERROR_PREFIX}{str(e)}\n')
errors += 1 errors += 1
continue continue
if argv: if argv:
cmd = argv[0].strip().lower() cmd = argv[0].strip().lower()
if cmd == Cmd.DATETIME_CMD: if cmd == Cmd.DATETIME_CMD:
if len(argv) == 2: if len(argv) == 2:
kwValues['datetime'] = _getMain().todaysTime().strftime(argv[1]) kwValues['datetime'] = _gam().todaysTime().strftime(argv[1])
else: else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n') _gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n') _gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
errors += 1 errors += 1
continue continue
if cmd == Cmd.SET_CMD: if cmd == Cmd.SET_CMD:
if len(argv) == 3: if len(argv) == 3:
kwValues[argv[1]] = argv[2] kwValues[argv[1]] = argv[2]
else: else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n') _gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n') _gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
errors += 1 errors += 1
continue continue
if cmd == Cmd.CLEAR_CMD: if cmd == Cmd.CLEAR_CMD:
if len(argv) == 2: if len(argv) == 2:
kwValues.pop(argv[1], None) kwValues.pop(argv[1], None)
else: else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n') _gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n') _gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
errors += 1 errors += 1
continue continue
if cmd == Cmd.SLEEP_CMD: if cmd == Cmd.SLEEP_CMD:
if len(argv) != 2 or not argv[1].isdigit(): if len(argv) != 2 or not argv[1].isdigit():
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n') _gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n') _gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
errors += 1 errors += 1
continue continue
if (not cmd) or ((len(argv) == 1) and (cmd not in [Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD])): if (not cmd) or ((len(argv) == 1) and (cmd not in [Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD])):
@@ -774,28 +773,28 @@ def doBatch(threadBatch=False):
if cmd in validCommands: if cmd in validCommands:
items.append(argv) items.append(argv)
else: else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n') _gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_getMain().formatChoiceList(validCommands)}>\n') _gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_gam().formatChoiceList(validCommands)}>\n')
errors += 1 errors += 1
except IOError as e: except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage(filename, e)) _gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
_getMain().closeFile(f) _gam().closeFile(f)
if errors == 0: if errors == 0:
if not threadBatch: if not threadBatch:
MultiprocessGAMCommands(items, showCmds) MultiprocessGAMCommands(items, showCmds)
else: else:
ThreadBatchGAMCommands(items, showCmds) ThreadBatchGAMCommands(items, showCmds)
else: else:
_getMain().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_getMain().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1])) _gam().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_gam().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
_getMain().setSysExitRC(_getMain().USAGE_ERROR_RC) _gam().setSysExitRC(_gam().USAGE_ERROR_RC)
# gam tbatch <BatchContent> [showcmds [<Boolean>]] # gam tbatch <BatchContent> [showcmds [<Boolean>]]
def doThreadBatch(): def doThreadBatch():
_getMain().adjustRedirectedSTDFilesIfNotMultiprocessing() _gam().adjustRedirectedSTDFilesIfNotMultiprocessing()
doBatch(True) doBatch(True)
def doAutoBatch(entityType, entityList, CL_command): def doAutoBatch(entityType, entityList, CL_command):
remaining = _getMain().Cmd.Remaining() remaining = _gam().Cmd.Remaining()
items = [] items = []
initial_argv = [Cmd.GAM_CMD] initial_argv = [Cmd.GAM_CMD]
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION]: if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION]:
@@ -824,8 +823,8 @@ def getSubFields(initial_argv, fieldNames):
subFields = {} subFields = {}
GAM_argv = initial_argv[:] GAM_argv = initial_argv[:]
GAM_argvI = len(GAM_argv) GAM_argvI = len(GAM_argv)
while _getMain().Cmd.ArgumentsRemaining(): while _gam().Cmd.ArgumentsRemaining():
myarg = _getMain().Cmd.Current() myarg = _gam().Cmd.Current()
if not myarg: if not myarg:
GAM_argv.append(myarg) GAM_argv.append(myarg)
elif SUB_PATTERN.search(myarg): elif SUB_PATTERN.search(myarg):
@@ -839,17 +838,17 @@ def getSubFields(initial_argv, fieldNames):
if not rematch: if not rematch:
fieldName = submatch.group(1) fieldName = submatch.group(1)
if fieldName not in fieldNames: if fieldName not in fieldNames:
_getMain().csvFieldErrorExit(fieldName, fieldNames) _gam().csvFieldErrorExit(fieldName, fieldNames)
subFields[GAM_argvI].append((SUB_TYPE, fieldName, submatch.start(), submatch.end())) subFields[GAM_argvI].append((SUB_TYPE, fieldName, submatch.start(), submatch.end()))
else: else:
fieldName = rematch.group(1) fieldName = rematch.group(1)
if fieldName not in fieldNames: if fieldName not in fieldNames:
_getMain().csvFieldErrorExit(fieldName, fieldNames) _gam().csvFieldErrorExit(fieldName, fieldNames)
try: try:
re.compile(rematch.group(2)) re.compile(rematch.group(2))
subFields[GAM_argvI].append((RE_TYPE, fieldName, submatch.start(), submatch.end(), rematch.group(2), rematch.group(3))) subFields[GAM_argvI].append((RE_TYPE, fieldName, submatch.start(), submatch.end(), rematch.group(2), rematch.group(3)))
except re.error as e: except re.error as e:
_getMain().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}') _gam().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
pos = submatch.end() pos = submatch.end()
GAM_argv.append(myarg) GAM_argv.append(myarg)
elif myarg[0] == '~': elif myarg[0] == '~':
@@ -858,11 +857,11 @@ def getSubFields(initial_argv, fieldNames):
subFields[GAM_argvI] = [(SUB_TYPE, fieldName, 0, len(myarg))] subFields[GAM_argvI] = [(SUB_TYPE, fieldName, 0, len(myarg))]
GAM_argv.append(myarg) GAM_argv.append(myarg)
else: else:
_getMain().csvFieldErrorExit(fieldName, fieldNames) _gam().csvFieldErrorExit(fieldName, fieldNames)
else: else:
GAM_argv.append(myarg) GAM_argv.append(myarg)
GAM_argvI += 1 GAM_argvI += 1
_getMain().Cmd.Advance() _gam().Cmd.Advance()
return(GAM_argv, subFields) return(GAM_argv, subFields)
def processSubFields(GAM_argv, row, subFields): def processSubFields(GAM_argv, row, subFields):
@@ -889,28 +888,28 @@ def processSubFields(GAM_argv, row, subFields):
# [skiprows <Integer>] [maxrows <Integer>] # [skiprows <Integer>] [maxrows <Integer>]
# gam <GAM argument list> # gam <GAM argument list>
def doCSV(testMode=False): def doCSV(testMode=False):
filename = _getMain().getString(Cmd.OB_FILE_NAME) filename = _gam().getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0): if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_getMain().Cmd.Backup() _gam().Cmd.Backup()
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD)) _gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
f, csvFile, fieldnames = _getMain().openCSVFileReader(filename) f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
matchFields, skipFields = _getMain().getMatchSkipFields(fieldnames) matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
showCmds = _getShowCommands() showCmds = _getShowCommands()
skipRows = _getSkipRows() skipRows = _getSkipRows()
maxRows = _getMaxRows() maxRows = _getMaxRows()
_getMain().checkArgumentPresent(Cmd.GAM_CMD, required=True) _gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _getMain().Cmd.ArgumentsRemaining(): if not _gam().Cmd.ArgumentsRemaining():
_getMain().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST) _gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
initial_argv = [Cmd.GAM_CMD] initial_argv = [Cmd.GAM_CMD]
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _getMain().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD): if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _gam().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
initial_argv.extend([Cmd.SELECT_CMD, GM.Globals[GM.SECTION]]) initial_argv.extend([Cmd.SELECT_CMD, GM.Globals[GM.SECTION]])
GAM_argv, subFields = getSubFields(initial_argv, fieldnames) GAM_argv, subFields = getSubFields(initial_argv, fieldnames)
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]: if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
_getMain().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]) _gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
items = [] items = []
i = 0 i = 0
for row in csvFile: for row in csvFile:
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields): if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1 i += 1
if skipRows: if skipRows:
if i <= skipRows: if i <= skipRows:
@@ -920,21 +919,21 @@ def doCSV(testMode=False):
items.append(processSubFields(GAM_argv, row, subFields)) items.append(processSubFields(GAM_argv, row, subFields))
if maxRows and i >= maxRows: if maxRows and i >= maxRows:
break break
_getMain().closeFile(f) _gam().closeFile(f)
if not testMode: if not testMode:
MultiprocessGAMCommands(items, showCmds) MultiprocessGAMCommands(items, showCmds)
else: else:
numItems = min(len(items), 10) numItems = min(len(items), 10)
_getMain().writeStdout(Msg.CSV_FILE_HEADERS.format(filename)) _gam().writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
_getMain().Ind.Increment() _gam().Ind.Increment()
for field in fieldnames: for field in fieldnames:
_getMain().writeStdout(f'{_getMain().Ind.Spaces()}{field}\n') _gam().writeStdout(f'{_gam().Ind.Spaces()}{field}\n')
_getMain().Ind.Decrement() _gam().Ind.Decrement()
_getMain().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _getMain().GAM)) _gam().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _gam().GAM))
_getMain().Ind.Increment() _gam().Ind.Increment()
for i in range(numItems): for i in range(numItems):
_getMain().writeStdout(f'{_getMain().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n') _gam().writeStdout(f'{_gam().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
_getMain().Ind.Decrement() _gam().Ind.Decrement()
def doCSVTest(): def doCSVTest():
doCSV(testMode=True) doCSV(testMode=True)
@@ -945,23 +944,23 @@ def doCSVTest():
# [skiprows <Integer>] [maxrows <Integer>] # [skiprows <Integer>] [maxrows <Integer>]
# gam <GAM argument list> # gam <GAM argument list>
def doLoop(loopCmd): def doLoop(loopCmd):
filename = _getMain().getString(Cmd.OB_FILE_NAME) filename = _gam().getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0): if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_getMain().Cmd.Backup() _gam().Cmd.Backup()
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD)) _gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
f, csvFile, fieldnames = _getMain().openCSVFileReader(filename) f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
matchFields, skipFields = _getMain().getMatchSkipFields(fieldnames) matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
showCmds = _getShowCommands() showCmds = _getShowCommands()
skipRows = _getSkipRows() skipRows = _getSkipRows()
maxRows = _getMaxRows() maxRows = _getMaxRows()
_getMain().checkArgumentPresent(Cmd.GAM_CMD, required=True) _gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _getMain().Cmd.ArgumentsRemaining(): if not _gam().Cmd.ArgumentsRemaining():
_getMain().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST) _gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]: if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
_getMain().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]) _gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
choice = _getMain().Cmd.Current().strip().lower() choice = _gam().Cmd.Current().strip().lower()
if choice == Cmd.LOOP_CMD: if choice == Cmd.LOOP_CMD:
_getMain().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED) _gam().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
# gam loop ... gam redirect|select|config ... process gam.cfg on each iteration # gam loop ... gam redirect|select|config ... process gam.cfg on each iteration
# gam redirect|select|config ... loop ... gam redirect|select|config ... process gam.cfg on each iteration # gam redirect|select|config ... loop ... gam redirect|select|config ... process gam.cfg on each iteration
# gam loop ... gam !redirect|select|config ... no further processing of gam.cfg # gam loop ... gam !redirect|select|config ... no further processing of gam.cfg
@@ -981,13 +980,13 @@ def doLoop(loopCmd):
else: else:
LoopGlobals = {GM.CMDLOG_LOGGER: None, GM.CMDLOG_HANDLER: None} LoopGlobals = {GM.CMDLOG_LOGGER: None, GM.CMDLOG_HANDLER: None}
if (GM.Globals[GM.PID] > 0) and GC.Values[GC.CMDLOG]: if (GM.Globals[GM.PID] > 0) and GC.Values[GC.CMDLOG]:
_getMain().openGAMCommandLog(LoopGlobals, 'looplog') _gam().openGAMCommandLog(LoopGlobals, 'looplog')
if LoopGlobals[GM.CMDLOG_LOGGER]: if LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().writeGAMCommandLog(LoopGlobals, loopCmd, '*') _gam().writeGAMCommandLog(LoopGlobals, loopCmd, '*')
if not showCmds: if not showCmds:
i = 0 i = 0
for row in csvFile: for row in csvFile:
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields): if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1 i += 1
if skipRows: if skipRows:
if i <= skipRows: if i <= skipRows:
@@ -997,20 +996,20 @@ def doLoop(loopCmd):
item = processSubFields(GAM_argv, row, subFields) item = processSubFields(GAM_argv, row, subFields)
logCmd = Cmd.QuotedArgumentList(item) logCmd = Cmd.QuotedArgumentList(item)
if i % 100 == 0: if i % 100 == 0:
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_getMain().currentISOformatTimeStamp(), i)) batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_gam().currentISOformatTimeStamp(), i))
sysRC = _getMain().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True) sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]: if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().writeGAMCommandLog(LoopGlobals, logCmd, sysRC) _gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _getMain().HARD_ERROR_RC): if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
break break
if maxRows and i >= maxRows: if maxRows and i >= maxRows:
break break
_getMain().closeFile(f) _gam().closeFile(f)
else: else:
items = [] items = []
i = 0 i = 0
for row in csvFile: for row in csvFile:
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields): if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1 i += 1
if skipRows: if skipRows:
if i <= skipRows: if i <= skipRows:
@@ -1020,26 +1019,26 @@ def doLoop(loopCmd):
items.append(processSubFields(GAM_argv, row, subFields)) items.append(processSubFields(GAM_argv, row, subFields))
if maxRows and i >= maxRows: if maxRows and i >= maxRows:
break break
_getMain().closeFile(f) _gam().closeFile(f)
numItems = len(items) numItems = len(items)
pid = 0 pid = 0
for item in items: for item in items:
pid += 1 pid += 1
logCmd = Cmd.QuotedArgumentList(item) logCmd = Cmd.QuotedArgumentList(item)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
sysRC = _getMain().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True) sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n') batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]: if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().writeGAMCommandLog(LoopGlobals, logCmd, sysRC) _gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _getMain().HARD_ERROR_RC): if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
break break
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]: if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().closeGAMCommandLog(LoopGlobals) _gam().closeGAMCommandLog(LoopGlobals)
if multi: if multi:
terminateCSVFileQueueHandler(mpQueue, mpQueueHandler) terminateCSVFileQueueHandler(mpQueue, mpQueueHandler)
def _doList(entityList, entityType): def _doList(entityList, entityType):
_getMain().buildGAPIObject(API.DIRECTORY) _gam().buildGAPIObject(API.DIRECTORY)
if GM.Globals[GM.CSV_DATA_DICT]: if GM.Globals[GM.CSV_DATA_DICT]:
keyField = GM.Globals[GM.CSV_KEY_FIELD] keyField = GM.Globals[GM.CSV_KEY_FIELD]
dataField = GM.Globals[GM.CSV_DATA_FIELD] dataField = GM.Globals[GM.CSV_DATA_FIELD]
@@ -1047,26 +1046,26 @@ def _doList(entityList, entityType):
keyField = 'Entity' keyField = 'Entity'
dataField = 'Data' dataField = 'Data'
csvPF = CSVPrintFile(keyField) csvPF = CSVPrintFile(keyField)
if _getMain().checkArgumentPresent('todrive'): if _gam().checkArgumentPresent('todrive'):
csvPF.GetTodriveParameters() csvPF.GetTodriveParameters()
if entityList is None: if entityList is None:
entityList = _getMain().getEntityList(Cmd.OB_ENTITY) entityList = _gam().getEntityList(Cmd.OB_ENTITY)
showData = _getMain().checkArgumentPresent('data') showData = _gam().checkArgumentPresent('data')
if showData: if showData:
if not entityType: if not entityType:
itemType, itemList = _getMain().getEntityToModify(crosAllowed=True) itemType, itemList = _gam().getEntityToModify(crosAllowed=True)
else: else:
itemType = None itemType = None
itemList = _getMain().getEntityList(Cmd.OB_ENTITY) itemList = _gam().getEntityList(Cmd.OB_ENTITY)
entityItemLists = itemList if isinstance(itemList, dict) else None entityItemLists = itemList if isinstance(itemList, dict) else None
csvPF.AddTitle(dataField) csvPF.AddTitle(dataField)
else: else:
entityItemLists = None entityItemLists = None
dataDelimiter = _getMain().getDelimiter() dataDelimiter = _gam().getDelimiter()
_getMain().checkForExtraneousArguments() _gam().checkForExtraneousArguments()
_, _, entityList = _getMain().getEntityArgument(entityList) _, _, entityList = _gam().getEntityArgument(entityList)
for entity in entityList: for entity in entityList:
entityEmail = _getMain().normalizeEmailAddressOrUID(entity) entityEmail = _gam().normalizeEmailAddressOrUID(entity)
if showData: if showData:
if entityItemLists: if entityItemLists:
if entity not in entityItemLists: if entity not in entityItemLists:
@@ -1075,7 +1074,7 @@ def _doList(entityList, entityType):
itemList = entityItemLists[entity] itemList = entityItemLists[entity]
if itemType == Cmd.ENTITY_USERS: if itemType == Cmd.ENTITY_USERS:
for i, item in enumerate(itemList): for i, item in enumerate(itemList):
itemList[i] = _getMain().normalizeEmailAddressOrUID(item) itemList[i] = _gam().normalizeEmailAddressOrUID(item)
if dataDelimiter: if dataDelimiter:
csvPF.WriteRow({keyField: entityEmail, dataField: dataDelimiter.join(itemList)}) csvPF.WriteRow({keyField: entityEmail, dataField: dataDelimiter.join(itemList)})
else: else:
@@ -1098,10 +1097,10 @@ def doListUser(entityList):
_doList(entityList, Cmd.ENTITY_USERS) _doList(entityList, Cmd.ENTITY_USERS)
def _showCount(entityList, entityType): def _showCount(entityList, entityType):
_getMain().buildGAPIObject(API.DIRECTORY) _gam().buildGAPIObject(API.DIRECTORY)
_getMain().checkForExtraneousArguments() _gam().checkForExtraneousArguments()
_, count, entityList = _getMain().getEntityArgument(entityList) _, count, entityList = _gam().getEntityArgument(entityList)
_getMain().actionPerformedNumItems(count, entityType) _gam().actionPerformedNumItems(count, entityType)
# gam <CrOSTypeEntity> show count # gam <CrOSTypeEntity> show count
def showCountCrOS(entityList): def showCountCrOS(entityList):

File diff suppressed because it is too large Load Diff

View File

@@ -22,67 +22,63 @@ from gamlib import glcfg as GC
from gamlib import glgapi as GAPI from gamlib import glgapi as GAPI
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
# Add attachements to an email message # Add attachements to an email message
def _addAttachmentsToMessage(message, attachments): def _addAttachmentsToMessage(message, attachments):
gam = _getMain()
for attachment in attachments: for attachment in attachments:
try: try:
attachFilename = gam.setFilePath(attachment[0], GC.INPUT_DIR) attachFilename = _gam().setFilePath(attachment[0], GC.INPUT_DIR)
attachContentType, attachEncoding = mimetypes.guess_type(attachFilename) attachContentType, attachEncoding = mimetypes.guess_type(attachFilename)
if attachContentType is None or attachEncoding is not None: if attachContentType is None or attachEncoding is not None:
attachContentType = 'application/octet-stream' attachContentType = 'application/octet-stream'
main_type, sub_type = attachContentType.split('/', 1) main_type, sub_type = attachContentType.split('/', 1)
if main_type == 'text': if main_type == 'text':
msg = MIMEText(gam.readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=gam.UTF8) msg = MIMEText(_gam().readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=_gam().UTF8)
elif main_type == 'image': elif main_type == 'image':
msg = MIMEImage(gam.readFile(attachFilename, 'rb'), _subtype=sub_type) msg = MIMEImage(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
elif main_type == 'audio': elif main_type == 'audio':
msg = MIMEAudio(gam.readFile(attachFilename, 'rb'), _subtype=sub_type) msg = MIMEAudio(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
elif main_type == 'application': elif main_type == 'application':
msg = MIMEApplication(gam.readFile(attachFilename, 'rb'), _subtype=sub_type) msg = MIMEApplication(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
else: else:
msg = MIMEBase(main_type, sub_type) msg = MIMEBase(main_type, sub_type)
msg.set_payload(gam.readFile(attachFilename, 'rb')) msg.set_payload(_gam().readFile(attachFilename, 'rb'))
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFilename)) msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFilename))
message.attach(msg) message.attach(msg)
except (IOError, UnicodeDecodeError) as e: except (IOError, UnicodeDecodeError) as e:
gam.usageErrorExit(f'{attachFilename}: {str(e)}') _gam().usageErrorExit(f'{attachFilename}: {str(e)}')
# Add embedded images to an email message # Add embedded images to an email message
def _addEmbeddedImagesToMessage(message, embeddedImages): def _addEmbeddedImagesToMessage(message, embeddedImages):
gam = _getMain()
for embeddedImage in embeddedImages: for embeddedImage in embeddedImages:
try: try:
imageFilename = gam.setFilePath(embeddedImage[0], GC.INPUT_DIR) imageFilename = _gam().setFilePath(embeddedImage[0], GC.INPUT_DIR)
imageContentType, imageEncoding = mimetypes.guess_type(imageFilename) imageContentType, imageEncoding = mimetypes.guess_type(imageFilename)
if imageContentType is None or imageEncoding is not None: if imageContentType is None or imageEncoding is not None:
imageContentType = 'application/octet-stream' imageContentType = 'application/octet-stream'
main_type, sub_type = imageContentType.split('/', 1) main_type, sub_type = imageContentType.split('/', 1)
if main_type == 'image': if main_type == 'image':
msg = MIMEImage(gam.readFile(imageFilename, 'rb'), _subtype=sub_type) msg = MIMEImage(_gam().readFile(imageFilename, 'rb'), _subtype=sub_type)
else: else:
msg = MIMEBase(main_type, sub_type) msg = MIMEBase(main_type, sub_type)
msg.set_payload(gam.readFile(imageFilename, 'rb')) msg.set_payload(_gam().readFile(imageFilename, 'rb'))
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(imageFilename)) msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(imageFilename))
msg.add_header('Content-ID', f'<{embeddedImage[1]}>') msg.add_header('Content-ID', f'<{embeddedImage[1]}>')
message.attach(msg) message.attach(msg)
except (IOError, UnicodeDecodeError) as e: except (IOError, UnicodeDecodeError) as e:
gam.usageErrorExit(f'{imageFilename}: {str(e)}') _gam().usageErrorExit(f'{imageFilename}: {str(e)}')
# Send an email # Send an email
def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msgFrom=None, msgReplyTo=None, def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msgFrom=None, msgReplyTo=None,
html=False, charset=None, attachments=None, embeddedImages=None, html=False, charset=None, attachments=None, embeddedImages=None,
msgHeaders=None, ccRecipients=None, bccRecipients=None, mailBox=None, threadId=None, msgHeaders=None, ccRecipients=None, bccRecipients=None, mailBox=None, threadId=None,
action=None): action=None):
gam = _getMain() Act = _gam().Act
Act = gam.Act Ent = _gam().Ent
Ent = gam.Ent
if charset is None: if charset is None:
charset = gam.UTF8 charset = _gam().UTF8
if action is None: if action is None:
action = Act.SENDEMAIL action = Act.SENDEMAIL
@@ -96,26 +92,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
toSent.remove(addr) toSent.remove(addr)
toFailed[addr] = f'{err[0]}: {err[1]}' toFailed[addr] = f'{err[0]}: {err[1]}'
if toSent: if toSent:
gam.entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count) _gam().entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
for addr, errMsg in toFailed.items(): for addr, errMsg in toFailed.items():
gam.entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count) _gam().entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
def cleanAddr(emailAddr): def cleanAddr(emailAddr):
match = gam.NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr) match = _gam().NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
if match: if match:
emailName = match.group(1) emailName = match.group(1)
emailAddr = gam.normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True) emailAddr = _gam().normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
return (f'{emailName} <{emailAddr}>', emailAddr) return (f'{emailName} <{emailAddr}>', emailAddr)
emailAddr = gam.normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True) emailAddr = _gam().normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
return (emailAddr, emailAddr) return (emailAddr, emailAddr)
if msgFrom is None: if msgFrom is None:
msgFrom = gam._getAdminEmail() msgFrom = _gam()._getAdminEmail()
# Force ASCII for RFC compliance # Force ASCII for RFC compliance
# xmlcharref seems to work to display at least # xmlcharref seems to work to display at least
# some unicode in HTML body and is ignored in # some unicode in HTML body and is ignored in
# plain text body. # plain text body.
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(gam.UTF8) # msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(_gam().UTF8)
if not attachments and not embeddedImages: if not attachments and not embeddedImages:
message = MIMEText(msgBody, ['plain', 'html'][html], charset) message = MIMEText(msgBody, ['plain', 'html'][html], charset)
else: else:
@@ -145,26 +141,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
Act.Set(action) Act.Set(action)
if not GC.Values[GC.SMTP_HOST]: if not GC.Values[GC.SMTP_HOST]:
if not clientAccess: if not clientAccess:
userId, gmail = gam.buildGAPIServiceObject(API.GMAIL, mailBoxAddr) userId, gmail = _gam().buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
if not gmail: if not gmail:
Act.Set(parentAction) Act.Set(parentAction)
return return
else: else:
userId = mailBoxAddr userId = mailBoxAddr
gmail = gam.buildGAPIObject(API.GMAIL) gmail = _gam().buildGAPIObject(API.GMAIL)
message['To'] = msgTo if msgTo else userId message['To'] = msgTo if msgTo else userId
body = {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()} body = {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}
if threadId is not None: if threadId is not None:
body['threadId'] = threadId body['threadId'] = threadId
try: try:
result = gam.callGAPI(gmail.users().messages(), 'send', result = _gam().callGAPI(gmail.users().messages(), 'send',
throwReasons=[GAPI.SERVICE_NOT_AVAILABLE, GAPI.AUTH_ERROR, GAPI.DOMAIN_POLICY, throwReasons=[GAPI.SERVICE_NOT_AVAILABLE, GAPI.AUTH_ERROR, GAPI.DOMAIN_POLICY,
GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED], GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
userId=userId, body=body, fields='id') userId=userId, body=body, fields='id')
gam.entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count) _gam().entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy, except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy,
GAPI.invalid, GAPI.invalidArgument, GAPI.forbidden, GAPI.permissionDenied) as e: GAPI.invalid, GAPI.invalidArgument, GAPI.forbidden, GAPI.permissionDenied) as e:
gam.entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count) _gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
else: else:
message['To'] = msgTo if msgTo else mailBoxAddr message['To'] = msgTo if msgTo else mailBoxAddr
server = None server = None
@@ -175,7 +171,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
server.starttls(context=ssl.create_default_context(cafile=GC.Values[GC.CACERTS_PEM])) server.starttls(context=ssl.create_default_context(cafile=GC.Values[GC.CACERTS_PEM]))
if GC.Values[GC.SMTP_USERNAME] and GC.Values[GC.SMTP_PASSWORD]: if GC.Values[GC.SMTP_USERNAME] and GC.Values[GC.SMTP_PASSWORD]:
if isinstance(GC.Values[GC.SMTP_PASSWORD], bytes): if isinstance(GC.Values[GC.SMTP_PASSWORD], bytes):
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(gam.UTF8)) server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(_gam().UTF8))
else: else:
server.login(GC.Values[GC.SMTP_USERNAME], GC.Values[GC.SMTP_PASSWORD]) server.login(GC.Values[GC.SMTP_USERNAME], GC.Values[GC.SMTP_PASSWORD])
result = server.send_message(message) result = server.send_message(message)
@@ -183,7 +179,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
checkResult(Ent.RECIPIENT_CC, ccRecipients) checkResult(Ent.RECIPIENT_CC, ccRecipients)
checkResult(Ent.RECIPIENT_BCC, bccRecipients) checkResult(Ent.RECIPIENT_BCC, bccRecipients)
except smtplib.SMTPException as e: except smtplib.SMTPException as e:
gam.entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count) _gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
if server: if server:
try: try:
server.quit() server.quit()

File diff suppressed because it is too large Load Diff

View File

@@ -24,8 +24,7 @@ from gamlib import glglobals as GM
from gamlib import glmsgs as Msg from gamlib import glmsgs as Msg
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
GDOC_FORMAT_MIME_TYPES = { GDOC_FORMAT_MIME_TYPES = {
@@ -37,39 +36,39 @@ GDOC_FORMAT_MIME_TYPES = {
# gdoc <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> # gdoc <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity>
def getGDocData(gformat): def getGDocData(gformat):
mimeType = GDOC_FORMAT_MIME_TYPES[gformat] mimeType = GDOC_FORMAT_MIME_TYPES[gformat]
user = _getMain().getEmailAddress() user = _gam().getEmailAddress()
fileIdEntity = _getMain().getDriveFileEntity(queryShortcutsOK=False) fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]: if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
_, drive = _getMain().buildGAPIServiceObject(API.DRIVE3, user) _, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
else: else:
drive = _getMain().buildGAPIObject(API.DRIVE3) drive = _gam().buildGAPIObject(API.DRIVE3)
if not drive: if not drive:
sys.exit(GM.Globals[GM.SYSEXITRC]) sys.exit(GM.Globals[GM.SYSEXITRC])
_, _, jcount = _getMain()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive) _, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
if jcount == 0: if jcount == 0:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE))) _gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
if jcount > 1: if jcount > 1:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list']))) _gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
fileId = fileIdEntity['list'][0] fileId = fileIdEntity['list'][0]
f = None f = None
try: try:
result = _getMain().callGAPI(drive.files(), 'get', result = _gam().callGAPI(drive.files(), 'get',
throwReasons=GAPI.DRIVE_GET_THROW_REASONS, throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
fileId=fileId, fields='name,mimeType,exportLinks', fileId=fileId, fields='name,mimeType,exportLinks',
supportsAllDrives=True) supportsAllDrives=True)
# Google Doc # Google Doc
if 'exportLinks' in result: if 'exportLinks' in result:
if mimeType not in result['exportLinks']: if mimeType not in result['exportLinks']:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']], _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType)) Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8) f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
_, content = drive._http.request(uri=result['exportLinks'][mimeType], method='GET') _, content = drive._http.request(uri=result['exportLinks'][mimeType], method='GET')
f.write(content.decode(_getMain().UTF8_SIG)) f.write(content.decode(_gam().UTF8_SIG))
f.seek(0) f.seek(0)
return f return f
# Drive File # Drive File
if result['mimeType'] != mimeType: if result['mimeType'] != mimeType:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']], _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType)) Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
fb = TemporaryFile(mode='wb+') fb = TemporaryFile(mode='wb+')
request = drive.files().get_media(fileId=fileId) request = drive.files().get_media(fileId=fileId)
@@ -77,63 +76,63 @@ def getGDocData(gformat):
done = False done = False
while not done: while not done:
_, done = downloader.next_chunk() _, done = downloader.next_chunk()
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8) f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
fb.seek(0) fb.seek(0)
f.write(fb.read().decode(_getMain().UTF8_SIG)) f.write(fb.read().decode(_gam().UTF8_SIG))
fb.close() fb.close()
f.seek(0) f.seek(0)
return f return f
except GAPI.fileNotFound: except GAPI.fileNotFound:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
except (IOError, httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e: except (IOError, httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if f: if f:
f.close() f.close()
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e)) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e: except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
_getMain().userDriveServiceNotEnabledWarning(user, str(e)) _gam().userDriveServiceNotEnabledWarning(user, str(e))
sys.exit(GM.Globals[GM.SYSEXITRC]) sys.exit(GM.Globals[GM.SYSEXITRC])
HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>') HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>')
# gsheet <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> <SheetEntity> # gsheet <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> <SheetEntity>
def getGSheetData(): def getGSheetData():
user = _getMain().getEmailAddress() user = _gam().getEmailAddress()
fileIdEntity = _getMain().getDriveFileEntity(queryShortcutsOK=False) fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
sheetEntity = _getMain().getSheetEntity(False) sheetEntity = _gam().getSheetEntity(False)
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]: if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
user, drive = _getMain().buildGAPIServiceObject(API.DRIVE3, user) user, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
else: else:
drive = _getMain().buildGAPIObject(API.DRIVE3) drive = _gam().buildGAPIObject(API.DRIVE3)
if not drive: if not drive:
sys.exit(GM.Globals[GM.SYSEXITRC]) sys.exit(GM.Globals[GM.SYSEXITRC])
_, _, jcount = _getMain()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive) _, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
if jcount == 0: if jcount == 0:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE))) _gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
if jcount > 1: if jcount > 1:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list']))) _gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]: if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
_, sheet = _getMain().buildGAPIServiceObject(API.SHEETS, user) _, sheet = _gam().buildGAPIServiceObject(API.SHEETS, user)
else: else:
sheet = _getMain().buildGAPIObject(API.SHEETS) sheet = _gam().buildGAPIObject(API.SHEETS)
if not sheet: if not sheet:
sys.exit(GM.Globals[GM.SYSEXITRC]) sys.exit(GM.Globals[GM.SYSEXITRC])
fileId = fileIdEntity['list'][0] fileId = fileIdEntity['list'][0]
f = None f = None
try: try:
result = _getMain().callGAPI(drive.files(), 'get', result = _gam().callGAPI(drive.files(), 'get',
throwReasons=GAPI.DRIVE_GET_THROW_REASONS, throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
fileId=fileId, fields='name,mimeType', supportsAllDrives=True) fileId=fileId, fields='name,mimeType', supportsAllDrives=True)
if result['mimeType'] != _getMain().MIMETYPE_GA_SPREADSHEET: if result['mimeType'] != _gam().MIMETYPE_GA_SPREADSHEET:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']], _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], _getMain().MIMETYPE_GA_SPREADSHEET)) Msg.INVALID_MIMETYPE.format(result['mimeType'], _gam().MIMETYPE_GA_SPREADSHEET))
spreadsheet = _getMain().callGAPI(sheet.spreadsheets(), 'get', spreadsheet = _gam().callGAPI(sheet.spreadsheets(), 'get',
throwReasons=GAPI.SHEETS_ACCESS_THROW_REASONS, throwReasons=GAPI.SHEETS_ACCESS_THROW_REASONS,
spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))') spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))')
sheetId = _getMain().getSheetIdFromSheetEntity(spreadsheet, sheetEntity) sheetId = _gam().getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
if sheetId is None: if sheetId is None:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
spreadsheetUrl = f'{re.sub("/edit.*$", "/export", spreadsheet["spreadsheetUrl"])}?format=csv&id={fileId}&gid={sheetId}' spreadsheetUrl = f'{re.sub("/edit.*$", "/export", spreadsheet["spreadsheetUrl"])}?format=csv&id={fileId}&gid={sheetId}'
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8) f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
if GC.Values[GC.DEBUG_LEVEL] > 0: if GC.Values[GC.DEBUG_LEVEL] > 0:
sys.stderr.write(f'Debug: spreadsheetUrl: {spreadsheetUrl}\n') sys.stderr.write(f'Debug: spreadsheetUrl: {spreadsheetUrl}\n')
triesLimit = 3 triesLimit = 3
@@ -144,25 +143,25 @@ def getGSheetData():
break break
tg = HTML_TITLE_PATTERN.match(content[0:600].decode('utf-8')) tg = HTML_TITLE_PATTERN.match(content[0:600].decode('utf-8'))
errMsg = tg.group(1) if tg else 'Unknown error' errMsg = tg.group(1) if tg else 'Unknown error'
_getMain().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit) _gam().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
time.sleep(20) time.sleep(20)
else: else:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
f.write(content.decode(_getMain().UTF8_SIG)) f.write(content.decode(_gam().UTF8_SIG))
f.seek(0) f.seek(0)
return f return f
except GAPI.fileNotFound: except GAPI.fileNotFound:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
except (GAPI.notFound, GAPI.forbidden, GAPI.permissionDenied, except (GAPI.notFound, GAPI.forbidden, GAPI.permissionDenied,
GAPI.internalError, GAPI.insufficientFilePermissions, GAPI.badRequest, GAPI.internalError, GAPI.insufficientFilePermissions, GAPI.badRequest,
GAPI.invalid, GAPI.invalidArgument, GAPI.failedPrecondition) as e: GAPI.invalid, GAPI.invalidArgument, GAPI.failedPrecondition) as e:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e)) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
except (IOError, httplib2.HttpLib2Error) as e: except (IOError, httplib2.HttpLib2Error) as e:
if f: if f:
f.close() f.close()
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e)) _gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e: except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
_getMain().userDriveServiceNotEnabledWarning(user, str(e)) _gam().userDriveServiceNotEnabledWarning(user, str(e))
sys.exit(GM.Globals[GM.SYSEXITRC]) sys.exit(GM.Globals[GM.SYSEXITRC])
@@ -173,15 +172,15 @@ BUCKET_OBJECT_PATTERNS = [
] ]
def getBucketObjectName(): def getBucketObjectName():
Cmd = _getMain().Cmd Cmd = _gam().Cmd
uri = _getMain().getString(Cmd.OB_STRING) uri = _gam().getString(Cmd.OB_STRING)
for pattern in BUCKET_OBJECT_PATTERNS: for pattern in BUCKET_OBJECT_PATTERNS:
mg = re.search(pattern['pattern'], uri) mg = re.search(pattern['pattern'], uri)
if mg: if mg:
bucket = mg.group(1) bucket = mg.group(1)
s_object = mg.group(2) if not pattern['unquote'] else unquote(mg.group(2)) s_object = mg.group(2) if not pattern['unquote'] else unquote(mg.group(2))
return (bucket, s_object, f'{bucket}/{s_object}') return (bucket, s_object, f'{bucket}/{s_object}')
_getMain().systemErrorExit(_getMain().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}') _gam().systemErrorExit(_gam().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
GCS_FORMAT_MIME_TYPES = { GCS_FORMAT_MIME_TYPES = {
'gcscsv': 'text/csv', 'gcscsv': 'text/csv',
@@ -193,17 +192,17 @@ GCS_FORMAT_MIME_TYPES = {
def getStorageFileData(gcsformat, returnData=True): def getStorageFileData(gcsformat, returnData=True):
mimeType = GCS_FORMAT_MIME_TYPES[gcsformat] mimeType = GCS_FORMAT_MIME_TYPES[gcsformat]
bucket, s_object, bucketObject = getBucketObjectName() bucket, s_object, bucketObject = getBucketObjectName()
s = _getMain().buildGAPIObject(API.STORAGEREAD) s = _gam().buildGAPIObject(API.STORAGEREAD)
try: try:
result = _getMain().callGAPI(s.objects(), 'get', result = _gam().callGAPI(s.objects(), 'get',
throwReasons=[GAPI.NOT_FOUND, GAPI.FORBIDDEN], throwReasons=[GAPI.NOT_FOUND, GAPI.FORBIDDEN],
bucket=bucket, object=s_object, projection='noAcl', fields='contentType') bucket=bucket, object=s_object, projection='noAcl', fields='contentType')
except GAPI.notFound: except GAPI.notFound:
_getMain().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject) _gam().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
except GAPI.forbidden as e: except GAPI.forbidden as e:
_getMain().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e)) _gam().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
if result['contentType'] != mimeType: if result['contentType'] != mimeType:
_getMain().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], _gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
Msg.INVALID_MIMETYPE.format(result['contentType'], mimeType)) Msg.INVALID_MIMETYPE.format(result['contentType'], mimeType))
fb = TemporaryFile(mode='wb+') fb = TemporaryFile(mode='wb+')
try: try:
@@ -214,58 +213,58 @@ def getStorageFileData(gcsformat, returnData=True):
_, done = downloader.next_chunk() _, done = downloader.next_chunk()
fb.seek(0) fb.seek(0)
if returnData: if returnData:
data = fb.read().decode(_getMain().UTF8) data = fb.read().decode(_gam().UTF8)
fb.close() fb.close()
return data return data
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8) f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
f.write(fb.read().decode(_getMain().UTF8_SIG)) f.write(fb.read().decode(_gam().UTF8_SIG))
fb.close() fb.close()
f.seek(0) f.seek(0)
return f return f
except googleapiclient.http.HttpError as e: except googleapiclient.http.HttpError as e:
mg = _getMain().HTTP_ERROR_PATTERN.match(str(e)) mg = _gam().HTTP_ERROR_PATTERN.match(str(e))
_getMain().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e)) _gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
# <CSVFileInput> # <CSVFileInput>
def openCSVFileReader(filename, fieldnames=None): def openCSVFileReader(filename, fieldnames=None):
Cmd = _getMain().Cmd Cmd = _gam().Cmd
filenameLower = filename.lower() filenameLower = filename.lower()
if filenameLower == 'gsheet': if filenameLower == 'gsheet':
f = getGSheetData() f = getGSheetData()
_getMain().getCharSet() _gam().getCharSet()
elif filenameLower in {'gcsv', 'gdoc'}: elif filenameLower in {'gcsv', 'gdoc'}:
f = getGDocData(filenameLower) f = getGDocData(filenameLower)
_getMain().getCharSet() _gam().getCharSet()
elif filenameLower in {'gcscsv', 'gcsdoc'}: elif filenameLower in {'gcscsv', 'gcsdoc'}:
f = getStorageFileData(filenameLower, False) f = getStorageFileData(filenameLower, False)
_getMain().getCharSet() _gam().getCharSet()
else: else:
encoding = _getMain().getCharSet() encoding = _gam().getCharSet()
filename = _getMain().setFilePath(filename, GC.INPUT_DIR) filename = _gam().setFilePath(filename, GC.INPUT_DIR)
f = _getMain().openFile(filename, mode=_getMain().DEFAULT_CSV_READ_MODE, encoding=encoding) f = _gam().openFile(filename, mode=_gam().DEFAULT_CSV_READ_MODE, encoding=encoding)
if _getMain().checkArgumentPresent('warnifnodata'): if _gam().checkArgumentPresent('warnifnodata'):
loc = f.tell() loc = f.tell()
try: try:
if not f.readline() or not f.readline(): if not f.readline() or not f.readline():
_getMain().stderrWarningMsg(_getMain().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND)) _gam().stderrWarningMsg(_gam().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
sys.exit(_getMain().NO_ENTITIES_FOUND_RC) sys.exit(_gam().NO_ENTITIES_FOUND_RC)
f.seek(loc) f.seek(loc)
except (IOError, UnicodeDecodeError, UnicodeError) as e: except (IOError, UnicodeDecodeError, UnicodeError) as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage(filename, e)) _gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
if _getMain().checkArgumentPresent('columndelimiter'): if _gam().checkArgumentPresent('columndelimiter'):
columnDelimiter = _getMain().getCharacter() columnDelimiter = _gam().getCharacter()
else: else:
columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER] columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER]
if _getMain().checkArgumentPresent('noescapechar'): if _gam().checkArgumentPresent('noescapechar'):
noEscapeChar = _getMain().getBoolean() noEscapeChar = _gam().getBoolean()
else: else:
noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR] noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR]
if _getMain().checkArgumentPresent('quotechar'): if _gam().checkArgumentPresent('quotechar'):
quotechar = _getMain().getCharacter() quotechar = _gam().getCharacter()
else: else:
quotechar = GC.Values[GC.CSV_INPUT_QUOTE_CHAR] quotechar = GC.Values[GC.CSV_INPUT_QUOTE_CHAR]
if not _getMain().checkArgumentPresent('endcsv') and _getMain().checkArgumentPresent('fields'): if not _gam().checkArgumentPresent('endcsv') and _gam().checkArgumentPresent('fields'):
fieldnames = _getMain().shlexSplitList(_getMain().getString(Cmd.OB_FIELD_NAME_LIST)) fieldnames = _gam().shlexSplitList(_gam().getString(Cmd.OB_FIELD_NAME_LIST))
try: try:
csvFile = csv.DictReader(f, fieldnames=fieldnames, csvFile = csv.DictReader(f, fieldnames=fieldnames,
delimiter=columnDelimiter, delimiter=columnDelimiter,
@@ -273,4 +272,4 @@ def openCSVFileReader(filename, fieldnames=None):
quotechar=quotechar) quotechar=quotechar)
return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else []) return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else [])
except (csv.Error, UnicodeDecodeError, UnicodeError) as e: except (csv.Error, UnicodeDecodeError, UnicodeError) as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, e) _gam().systemErrorExit(_gam().FILE_ERROR_RC, e)

View File

@@ -11,12 +11,11 @@ from gamlib import glgapi as GAPI
from gamlib import glglobals as GM from gamlib import glglobals as GM
def _getMain(): _gam = lambda: sys.modules['gam']
return sys.modules['gam']
def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None): def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
Cmd = _getMain().Cmd Cmd = _gam().Cmd
if Cmd.ArgumentsRemaining(): if Cmd.ArgumentsRemaining():
path = Cmd.Current().strip() path = Cmd.Current().strip()
if path == 'root': if path == 'root':
@@ -24,40 +23,40 @@ def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
if path: if path:
if pathOnly and (path.startswith('id:') or path.startswith('uid:')) and cd is not None: if pathOnly and (path.startswith('id:') or path.startswith('uid:')) and cd is not None:
try: try:
result = _getMain().callGAPI(cd.orgunits(), 'get', result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS, throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=path, customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=path,
fields='orgUnitPath') fields='orgUnitPath')
Cmd.Advance() Cmd.Advance()
if absolutePath: if absolutePath:
return _getMain().makeOrgUnitPathAbsolute(result['orgUnitPath']) return _gam().makeOrgUnitPathAbsolute(result['orgUnitPath'])
return _getMain().makeOrgUnitPathRelative(result['orgUnitPath']) return _gam().makeOrgUnitPathRelative(result['orgUnitPath'])
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError, except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError,
GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired): GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().checkEntityAFDNEorAccessErrorExit(cd, _getMain().Ent.ORGANIZATIONAL_UNIT, path) _gam().checkEntityAFDNEorAccessErrorExit(cd, _gam().Ent.ORGANIZATIONAL_UNIT, path)
_getMain().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH) _gam().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
Cmd.Advance() Cmd.Advance()
if absolutePath: if absolutePath:
return _getMain().makeOrgUnitPathAbsolute(path) return _gam().makeOrgUnitPathAbsolute(path)
return _getMain().makeOrgUnitPathRelative(path) return _gam().makeOrgUnitPathRelative(path)
_getMain().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly]) _gam().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
def getTopLevelOrgId(cd, parentOrgUnitPath): def getTopLevelOrgId(cd, parentOrgUnitPath):
Ent = _getMain().Ent Ent = _gam().Ent
if parentOrgUnitPath != '/': if parentOrgUnitPath != '/':
try: try:
result = _getMain().callGAPI(cd.orgunits(), 'get', result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS, throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(parentOrgUnitPath)), customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentOrgUnitPath)),
fields='orgUnitId') fields='orgUnitId')
return result['orgUnitId'] return result['orgUnitId']
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError): except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
return None return None
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired): except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath) _gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
return None return None
try: try:
result = _getMain().callGAPI(cd.orgunits(), 'list', result = _gam().callGAPI(cd.orgunits(), 'list',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS, throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='allIncludingParent', customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='allIncludingParent',
fields='organizationUnits(orgUnitId,orgUnitPath)') fields='organizationUnits(orgUnitId,orgUnitPath)')
@@ -68,18 +67,18 @@ def getTopLevelOrgId(cd, parentOrgUnitPath):
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError): except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
return None return None
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired): except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath) _gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
return None return None
def getOrgUnitId(cd=None, orgUnit=None): def getOrgUnitId(cd=None, orgUnit=None):
Ent = _getMain().Ent Ent = _gam().Ent
if cd is None: if cd is None:
cd = _getMain().buildGAPIObject(API.DIRECTORY) cd = _gam().buildGAPIObject(API.DIRECTORY)
if orgUnit is None: if orgUnit is None:
orgUnit = getOrgUnitItem() orgUnit = getOrgUnitItem()
try: try:
if orgUnit == '/': if orgUnit == '/':
result = _getMain().callGAPI(cd.orgunits(), 'list', result = _gam().callGAPI(cd.orgunits(), 'list',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS, throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='children', customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='children',
fields='organizationUnits(parentOrgUnitId,parentOrgUnitPath)') fields='organizationUnits(parentOrgUnitId,parentOrgUnitPath)')
@@ -89,35 +88,35 @@ def getOrgUnitId(cd=None, orgUnit=None):
if topLevelOrgId: if topLevelOrgId:
return (orgUnit, topLevelOrgId) return (orgUnit, topLevelOrgId)
return (orgUnit, '/') #Bogus but should never happen return (orgUnit, '/') #Bogus but should never happen
result = _getMain().callGAPI(cd.orgunits(), 'get', result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS, throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(orgUnit)), customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(orgUnit)),
fields='orgUnitId,orgUnitPath') fields='orgUnitId,orgUnitPath')
return (result['orgUnitPath'], result['orgUnitId']) return (result['orgUnitPath'], result['orgUnitId'])
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError): except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
_getMain().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit) _gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired): except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().accessErrorExit(cd) _gam().accessErrorExit(cd)
def getAllParentOrgUnitsForUser(cd, user): def getAllParentOrgUnitsForUser(cd, user):
Ent = _getMain().Ent Ent = _gam().Ent
try: try:
result = _getMain().callGAPI(cd.users(), 'get', result = _gam().callGAPI(cd.users(), 'get',
throwReasons=GAPI.USER_GET_THROW_REASONS, throwReasons=GAPI.USER_GET_THROW_REASONS,
userKey=user, fields='orgUnitPath', projection='basic') userKey=user, fields='orgUnitPath', projection='basic')
except (GAPI.userNotFound, GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden): except (GAPI.userNotFound, GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden):
_getMain().entityDoesNotExistExit(Ent.USER, user) _gam().entityDoesNotExistExit(Ent.USER, user)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired): except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().accessErrorExit(cd) _gam().accessErrorExit(cd)
parentPath = result['orgUnitPath'] parentPath = result['orgUnitPath']
if parentPath == '/': if parentPath == '/':
orgUnitPath, orgUnitId = getOrgUnitId(cd, '/') orgUnitPath, orgUnitId = getOrgUnitId(cd, '/')
return {orgUnitId: orgUnitPath} return {orgUnitId: orgUnitPath}
parentPath = _getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(parentPath)) parentPath = _gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentPath))
orgUnits = {} orgUnits = {}
while True: while True:
try: try:
result = _getMain().callGAPI(cd.orgunits(), 'get', result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS, throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=parentPath, customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=parentPath,
fields='orgUnitId,orgUnitPath,parentOrgUnitId') fields='orgUnitId,orgUnitPath,parentOrgUnitId')
@@ -126,9 +125,9 @@ def getAllParentOrgUnitsForUser(cd, user):
break break
parentPath = result['parentOrgUnitId'] parentPath = result['parentOrgUnitId']
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError): except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
_getMain().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath) _gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired): except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().accessErrorExit(cd) _gam().accessErrorExit(cd)
return orgUnits return orgUnits
def _getOrgunitsOrgUnitIdPath(cd, orgUnit): def _getOrgunitsOrgUnitIdPath(cd, orgUnit):