phase 2 code complete
Some checks are pending
Build and test GAM / build (false, build, 1, Build Intel Ubuntu Jammy, ubuntu-22.04) (push) Waiting to run
Build and test GAM / build (false, build, 10, Build x86_64 macOS 15, macos-15-intel) (push) Waiting to run
Build and test GAM / build (false, build, 11, Build x86_64 macOS 26, macos-26-intel) (push) Waiting to run
Build and test GAM / build (false, build, 12, Build Arm MacOS 26, macos-26) (push) Waiting to run
Build and test GAM / build (false, build, 13, Build Intel Windows, windows-2025-vs2026) (push) Waiting to run
Build and test GAM / build (false, build, 14, Build Arm Windows, windows-11-arm) (push) Waiting to run
Build and test GAM / build (false, build, 2, Build Intel Ubuntu Noble, ubuntu-24.04) (push) Waiting to run
Build and test GAM / build (false, build, 3, Build Arm Ubuntu Noble, ubuntu-24.04-arm) (push) Waiting to run
Build and test GAM / build (false, build, 4, Build Arm Ubuntu Jammy, ubuntu-22.04-arm) (push) Waiting to run
Build and test GAM / build (false, build, 5, Build Intel StaticX Legacy, ubuntu-22.04, yes) (push) Waiting to run
Build and test GAM / build (false, build, 6, Build Arm StaticX Legacy, ubuntu-22.04-arm, yes) (push) Waiting to run
Build and test GAM / build (false, build, 8, Build Arm MacOS 14, macos-14) (push) Waiting to run
Build and test GAM / build (false, build, 9, Build Arm MacOS 15, macos-15) (push) Waiting to run
Build and test GAM / build (false, test, 16, Test Python 3.11, ubuntu-24.04, 3.11) (push) Waiting to run
Build and test GAM / build (false, test, 17, Test Python 3.12, ubuntu-24.04, 3.12) (push) Waiting to run
Build and test GAM / build (false, test, 18, Test Python 3.13, ubuntu-24.04, 3.13) (push) Waiting to run
Build and test GAM / build (false, test, 19, Test Python 3.15-dev, ubuntu-24.04, 3.15-dev) (push) Waiting to run
Build and test GAM / build (true, test, 20, Test Python 3.14 freethread, ubuntu-24.04, 3.14) (push) Waiting to run
Build and test GAM / publish (push) Blocked by required conditions
CodeQL / Analyze (python) (push) Waiting to run

This commit is contained in:
Jay Lee
2026-07-03 19:52:57 -04:00
parent 7c639d3487
commit 5b27b7b875
10 changed files with 1210 additions and 1206 deletions

View File

@@ -3207,3 +3207,62 @@ def ProcessGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=True):
# Process GAM command
def CallGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=False):
return ProcessGAMCommand(args, processGamCfg=processGamCfg, inLoop=inLoop, closeSTD=closeSTD)
# Re-export util names for _gam().X access by util/ modules.
# This block runs AFTER all modules are fully loaded (no circular import risk).
# Only names actually referenced via _gam() are included.
from gamlib import glskus as SKU # noqa: E402,F811
from tempfile import TemporaryFile # noqa: E402,F401
from util.access import ( # noqa: E402,F401
APIAccessDeniedExit, ClientAPIAccessDeniedExit, SvcAcctAPIAccessDeniedExit, accessErrorExit,
accessErrorExitNonDirectory, checkEntityAFDNEorAccessErrorExit,
checkEntityDNEorAccessErrorExit, entityUnknownWarning,
)
from util.api import ( # noqa: E402,F401,F811
_getAdminEmail, _getSvcAcctData, buildGAPIObject, buildGAPIServiceObject,
callGAPI, callGAPIitems, callGAPIpages, chooseSaAPI, yieldGAPIpages,
)
from util.args import ( # noqa: E402,F401,F811
ARCHIVED_ARGUMENTS, ISOformatTimeStamp, LOCALE_CODES_MAP, NAME_EMAIL_ADDRESS_PATTERN,
SUSPENDED_ARGUMENTS, YYYYMMDDTHHMMSSZ_FORMAT, YYYYMMDD_FORMAT, YYYYMMDD_PATTERN,
_getIsArchived, _getIsSuspended, checkDataField, checkForExtraneousArguments,
checkMatchSkipFields, checkSubkeyField, encodeOrgUnitPath, escapeCRsNLs,
formatLocalTime, formatLocalTimestamp, getBoolean, getCharSet, getCharacter,
getDelimiter, getEmailAddress, getEmailAddressDomain, getLanguageCode,
getMatchSkipFields, getPhraseDNEorSNA, getREPattern, getString,
getSheetEntity, getSheetIdFromSheetEntity, makeOrgUnitPathAbsolute,
makeOrgUnitPathRelative, orgUnitPathQuery, protectedSheetId, removeCourseIdScope,
shlexSplitList, splitEmailAddress, todaysTime, validateEmailAddressOrUID,
)
from util.csv_pf import CheckInputRowFilterHeaders # noqa: E402,F401
from util.display import ( # noqa: E402,F401
FIRST_ITEM_MARKER, LAST_ITEM_MARKER, TOTAL_ITEMS_MARKER, actionPerformedNumItems,
entityActionFailedWarning, entityActionNotPerformedWarning, entityActionPerformed,
entityActionPerformedMessage, entityDoesNotExistWarning, entityPerformActionNumItems,
entityServiceNotApplicableWarning, getPageMessage, getPageMessageForWhom, printBlankLine,
printGettingAllAccountEntities, printGettingAllEntityItemsForWhom,
printGotEntityItemsForWhom, printJSONKey, printJSONValue, printKeyValueList,
setGettingAllEntityItemsForWhom, userDriveServiceNotEnabledWarning,
userServiceNotEnabledWarning,
)
from util.email import send_email # noqa: E402,F401,F811
from util.entity import checkUserExists, convertUIDtoEmailAddress # noqa: E402,F401,F811
from util.errors import ( # noqa: E402,F401,F811
csvDataAlreadySavedErrorExit, csvFieldErrorExit, entityActionFailedExit,
entityDoesNotExistExit, expiredRevokedOauth2TxtExit, formatChoiceList,
invalidArgumentExit, invalidChoiceExit, invalidDiscoveryJsonExit,
invalidOauth2TxtExit, invalidOauth2serviceJsonExit, missingArgumentExit,
)
from util.fileio import ( # noqa: E402,F401,F811
StringIOobject, checkAPICallsRate, closeFile, fdErrorMessage, fileErrorMessage,
getGDocSheetDataFailedExit, getGDocSheetDataRetryWarning, incrAPICallsRetryData,
openFile, readFile, writeFile,
)
from util.gdoc import ( # noqa: E402,F401
getGDocData, getStorageFileData, openCSVFileReader,
)
from util.output import ( # noqa: E402,F401,F811
currentCountNL, flushStderr, flushStdout,
printWarningMessage, readStdin, stderrWarningMsg, writeStdout,
)

View File

@@ -16,8 +16,7 @@ from gamlib import glindent
from gamlib import glmsgs as Msg
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
Act = glaction.GamAction()
@@ -27,139 +26,125 @@ Ind = glindent.GamIndent()
# Something's wrong with CustomerID??
def accessErrorMessage(cd, errMsg=None):
_m = _getMain()
if cd is None:
cd = _m.buildGAPIObject(API.DIRECTORY)
cd = _gam().buildGAPIObject(API.DIRECTORY)
try:
_m.callGAPI(cd.customers(), 'get',
_gam().callGAPI(cd.customers(), 'get',
throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND,
GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
customerKey=GC.Values[GC.CUSTOMER_ID], fields='id')
except (GAPI.badRequest, GAPI.invalidInput):
return _m.formatKeyValueList('',
return _gam().formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
Msg.INVALID],
'')
except GAPI.resourceNotFound:
return _m.formatKeyValueList('',
return _gam().formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
Msg.DOES_NOT_EXIST],
'')
except (GAPI.forbidden, GAPI.permissionDenied):
return _m.formatKeyValueList('',
return _gam().formatKeyValueList('',
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
Ent.DOMAIN, GC.Values[GC.DOMAIN],
Ent.USER, GM.Globals[GM.ADMIN]])+[Msg.ACCESS_FORBIDDEN],
'')
if errMsg:
return _m.formatKeyValueList('',
return _gam().formatKeyValueList('',
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
errMsg],
'')
return None
def accessErrorExit(cd, errMsg=None):
_m = _getMain()
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, accessErrorMessage(cd or _m.buildGAPIObject(API.DIRECTORY), errMsg))
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, accessErrorMessage(cd or _gam().buildGAPIObject(API.DIRECTORY), errMsg))
def accessErrorExitNonDirectory(api, errMsg):
_m = _getMain()
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC,
_m.formatKeyValueList('',
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC,
_gam().formatKeyValueList('',
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
Ent.DOMAIN, GC.Values[GC.DOMAIN],
Ent.API, api])+[errMsg],
''))
def ClientAPIAccessDeniedExit(errMsg=None):
_m = _getMain()
if errMsg is None:
_m.stderrErrorMsg(Msg.API_ACCESS_DENIED)
_gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
missingScopes = API.getClientScopesSet(GM.Globals[GM.CURRENT_CLIENT_API])-GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
if missingScopes:
_m.writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
_gam().writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
','.join(sorted(missingScopes))))
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
else:
_m.stderrErrorMsg(errMsg)
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
_gam().stderrErrorMsg(errMsg)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
def SvcAcctAPIAccessDenied():
_m = _getMain()
_m._getSvcAcctData()
_gam()._getSvcAcctData()
if (GM.Globals[GM.CURRENT_SVCACCT_API] == API.GMAIL and
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] and
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES][0] == API.GMAIL_SEND_SCOPE):
_m.systemErrorExit(_m.OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
_m.stderrErrorMsg(Msg.API_ACCESS_DENIED)
_gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
_gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
_m.writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
_gam().writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
apiOrScopes,
GM.Globals[GM.CURRENT_SVCACCT_USER] or _m._getAdminEmail()))
GM.Globals[GM.CURRENT_SVCACCT_USER] or _gam()._getAdminEmail()))
def SvcAcctAPIAccessDeniedExit():
_m = _getMain()
SvcAcctAPIAccessDenied()
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
def SvcAcctAPIDisabledExit():
_m = _getMain()
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
ClientAPIAccessDeniedExit()
if GM.Globals[GM.CURRENT_SVCACCT_API]:
_m.stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None)
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
_gam().stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
def APIAccessDeniedExit():
_m = _getMain()
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
ClientAPIAccessDeniedExit()
if GM.Globals[GM.CURRENT_SVCACCT_API]:
SvcAcctAPIAccessDeniedExit()
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
def checkEntityDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
_m = _getMain()
message = accessErrorMessage(cd)
if message:
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message)
_m.entityDoesNotExistWarning(entityType, entityName, i, count)
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_gam().entityDoesNotExistWarning(entityType, entityName, i, count)
def checkEntityAFDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
_m = _getMain()
message = accessErrorMessage(cd)
if message:
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message)
_m.entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_gam().entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
def checkEntityItemValueAFDNEorAccessErrorExit(cd, entityType, entityName, itemType, itemValue, i=0, count=0):
_m = _getMain()
message = accessErrorMessage(cd)
if message:
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message)
_m.entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
_gam().entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
def entityUnknownWarning(entityType, entityName, i=0, count=0):
_m = _getMain()
domain = _m.getEmailAddressDomain(entityName)
domain = _gam().getEmailAddressDomain(entityName)
if (domain.endswith(GC.Values[GC.DOMAIN])) or (domain.endswith('google.com')):
_m.entityDoesNotExistWarning(entityType, entityName, i, count)
_gam().entityDoesNotExistWarning(entityType, entityName, i, count)
else:
_m.entityServiceNotApplicableWarning(entityType, entityName, i, count)
_gam().entityServiceNotApplicableWarning(entityType, entityName, i, count)
def entityOrEntityUnknownWarning(entity1Type, entity1Name, entity2Type, entity2Name, i=0, count=0):
_m = _getMain()
_m.setSysExitRC(_m.ENTITY_DOES_NOT_EXIST_RC)
_m.writeStderr(_m.formatKeyValueList(Ind.Spaces(),
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _m.getPhraseDNEorSNA(entity1Name), None,
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _m.getPhraseDNEorSNA(entity2Name)],
_m.currentCountNL(i, count)))
_gam().setSysExitRC(_gam().ENTITY_DOES_NOT_EXIST_RC)
_gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _gam().getPhraseDNEorSNA(entity1Name), None,
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _gam().getPhraseDNEorSNA(entity2Name)],
_gam().currentCountNL(i, count)))
def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
_m = _getMain()
email = entityValueList[1]
try:
result = _m.callGAPI(cd.users(), 'get',
result = _gam().callGAPI(cd.users(), 'get',
throwReasons=GAPI.USER_GET_THROW_REASONS,
userKey=email, fields='id,primaryEmail')
if (result['primaryEmail'].lower() == email) or (result['id'] == email):
@@ -169,7 +154,7 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
except (GAPI.userNotFound, GAPI.badRequest,
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.backendError, GAPI.systemError):
try:
result = _m.callGAPI(cd.groups(), 'get',
result = _gam().callGAPI(cd.groups(), 'get',
throwReasons=GAPI.GROUP_GET_THROW_REASONS,
groupKey=email, fields='id,email')
if (result['email'].lower() == email) or (result['id'] == email):
@@ -179,10 +164,10 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
except (GAPI.groupNotFound,
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.badRequest):
kvList = [Ent.EMAIL, email]
_m.writeStderr(_m.formatKeyValueList(Ind.Spaces(),
_gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
Ent.FormatEntityValueList(entityValueList)+
[Act.Failed(), Msg.DUPLICATE]+
Ent.FormatEntityValueList(kvList),
_m.currentCountNL(i, count)))
_m.setSysExitRC(_m.ENTITY_DUPLICATE_RC)
_gam().currentCountNL(i, count)))
_gam().setSysExitRC(_gam().ENTITY_DUPLICATE_RC)
return kvList[0]

View File

@@ -56,8 +56,7 @@ DEVELOPER_PREVIEW_DISCOVERY_URI = "https://{api}.googleapis.com/$discovery/rest?
_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
def _getEnt():
return sys.modules['gam'].Ent
@@ -67,13 +66,12 @@ def _getInd():
def handleServerError(e):
m = _getMain()
errMsg = str(e)
if 'setting tls' not in errMsg:
m.systemErrorExit(m.NETWORK_ERROR_RC, errMsg)
m.stderrErrorMsg(errMsg)
m.writeStderr(Msg.DISABLE_TLS_MIN_MAX)
m.systemErrorExit(m.NETWORK_ERROR_RC, None)
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, errMsg)
_gam().stderrErrorMsg(errMsg)
_gam().writeStderr(Msg.DISABLE_TLS_MIN_MAX)
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, None)
def getHttpObj(cache=None, timeout=None, override_min_tls=None, override_max_tls=None):
tls_minimum_version = override_min_tls if override_min_tls else GC.Values[GC.TLS_MIN_VERSION] if GC.Values[GC.TLS_MIN_VERSION] else None
@@ -113,7 +111,7 @@ def _force_user_agent(user_agent):
def _lazy_force_user_agent(request_method):
"""Wraps a request method to lazily insert GAM_USER_AGENT at call time."""
def wrapped_request_method(*args, **kwargs):
user_agent = _getMain().GAM_USER_AGENT
user_agent = _gam().GAM_USER_AGENT
if kwargs.get('headers') is not None:
if kwargs['headers'].get('user-agent'):
if user_agent not in kwargs['headers']['user-agent']:
@@ -157,14 +155,13 @@ def transportCreateRequest(httpObj=None):
return transportAgentRequest(httpObj)
def doGAMCheckForUpdates(forceCheck):
m = _getMain()
Ind = _getInd()
def _gamLatestVersionNotAvailable():
if forceCheck:
m.systemErrorExit(m.NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
try:
_, c = getHttpObj(timeout=10).request(m.GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
_, c = getHttpObj(timeout=10).request(_gam().GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
try:
release_data = json.loads(c)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
@@ -173,17 +170,17 @@ def doGAMCheckForUpdates(forceCheck):
if not isinstance(release_data, dict) or 'tag_name' not in release_data:
_gamLatestVersionNotAvailable()
return
current_version = m.__version__
current_version = _gam().__version__
latest_version = release_data['tag_name']
if latest_version[0].lower() == 'v':
latest_version = latest_version[1:]
m.printKeyValueList(['Version Check', None])
_gam().printKeyValueList(['Version Check', None])
Ind.Increment()
m.printKeyValueList(['Current', current_version])
m.printKeyValueList([' Latest', latest_version])
_gam().printKeyValueList(['Current', current_version])
_gam().printKeyValueList([' Latest', latest_version])
Ind.Decrement()
if forceCheck < 0:
m.setSysExitRC(1 if latest_version > current_version else 0)
_gam().setSysExitRC(1 if latest_version > current_version else 0)
return
except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError,
google.auth.exceptions.TransportError,
@@ -250,13 +247,12 @@ class signjwtSignJwt(google.auth.crypt.Signer):
def sign(self, message):
''' Call IAM Credentials SignJWT API to get our signed JWT '''
m = _getMain()
request = get_adc_request()
try:
credentials, _ = google.auth.default(scopes=[API.IAM_SCOPE],
request=request)
except (google.auth.exceptions.DefaultCredentialsError, google.auth.exceptions.RefreshError) as e:
m.systemErrorExit(m.API_ACCESS_DENIED_RC, str(e))
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, str(e))
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj())
# refresh here so we can use the proper request from above
httpObj.credentials.refresh(request)
@@ -267,42 +263,40 @@ class signjwtSignJwt(google.auth.crypt.Signer):
return signed_jwt
def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0):
m = _getMain()
Ent = _getEnt()
errMsg = str(e).replace('.', '')
if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or
errMsg.startswith('Invalid response') or
errMsg.startswith('invalid_request: Invalid impersonation &quot;sub&quot; field')):
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
m.ClientAPIAccessDeniedExit()
_gam().ClientAPIAccessDeniedExit()
# 403 Forbidden, API disabled, user not enabled
# 400 Bad Request, user not defined
if softErrors:
m.entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
_gam().entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
return None
m.systemErrorExit(m.SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
_gam().systemErrorExit(_gam().SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
if errMsg in API.OAUTH2_UNAUTHORIZED_ERRORS:
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
m.ClientAPIAccessDeniedExit()
_gam().ClientAPIAccessDeniedExit()
# 401 Unauthorized, API disabled, user enabled
if softErrors:
if displayError:
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
m.userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
_gam().userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
return None
m.SvcAcctAPIAccessDeniedExit()
_gam().SvcAcctAPIAccessDeniedExit()
if errMsg in API.REFRESH_PERM_ERRORS:
if softErrors:
return None
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
m.expiredRevokedOauth2TxtExit()
m.stderrErrorMsg(f'Authentication Token Error - {errMsg}')
m.APIAccessDeniedExit()
_gam().expiredRevokedOauth2TxtExit()
_gam().stderrErrorMsg(f'Authentication Token Error - {errMsg}')
_gam().APIAccessDeniedExit()
def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnly=False, noScopes=False):
m = _getMain()
if not noDASA and GC.Values[GC.ENABLE_DASA]:
jsonData = m.readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
if jsonData:
try:
if api in API.APIS_NEEDING_ACCESS_TOKEN:
@@ -320,9 +314,9 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
sjsigner = signjwtSignJwt(jsonDict)
return (True, signjwtJWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience))
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
m.invalidOauth2serviceJsonExit(str(e))
m.invalidOauth2serviceJsonExit(Msg.NO_DATA)
jsonData = m.readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
_gam().invalidOauth2serviceJsonExit(str(e))
_gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
jsonData = _gam().readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
if jsonData:
try:
jsonDict = json.loads(jsonData)
@@ -332,11 +326,11 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
if not refreshOnly:
if set(jsonDict.get('scopes', API.REQUIRED_SCOPES)) == API.REQUIRED_SCOPES_SET:
if exitOnError:
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
return (False, None)
else:
GM.Globals[GM.CREDENTIALS_SCOPES] = set(jsonDict.pop('scopes', API.REQUIRED_SCOPES))
token_expiry = jsonDict.get('token_expiry', m.REFRESH_EXPIRY)
token_expiry = jsonDict.get('token_expiry', _gam().REFRESH_EXPIRY)
if GC.Values[GC.TRUNCATE_CLIENT_ID]:
# chop off .apps.googleusercontent.com suffix as it's not needed and we need to keep things short for the Auth URL.
jsonDict['client_id'] = re.sub(r'\.apps\.googleusercontent\.com$', '', jsonDict['client_id'])
@@ -349,19 +343,18 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
creds.token = jsonDict['access_token']
creds._id_token = jsonDict['id_token_jwt']
GM.Globals[GM.DECODED_ID_TOKEN] = jsonDict['id_token']
creds.expiry = arrow.Arrow.strptime(token_expiry, m.YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
creds.expiry = arrow.Arrow.strptime(token_expiry, _gam().YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
return (not noScopes, creds)
if jsonDict and exitOnError:
m.invalidOauth2TxtExit(Msg.INVALID)
_gam().invalidOauth2TxtExit(Msg.INVALID)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
if exitOnError:
m.invalidOauth2TxtExit(str(e))
_gam().invalidOauth2TxtExit(str(e))
if exitOnError:
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
return (False, None)
def _getValueFromOAuth(field, credentials=None):
m = _getMain()
if not GM.Globals[GM.DECODED_ID_TOKEN]:
request = transportCreateRequest()
if credentials is None:
@@ -373,9 +366,9 @@ def _getValueFromOAuth(field, credentials=None):
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
except ValueError as e:
if 'Token used too early' in str(e):
m.stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
m.systemErrorExit(m.SYSTEM_ERROR_RC, str(e))
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, m.UNKNOWN)
_gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, _gam().UNKNOWN)
def _getAdminEmail():
if GC.Values[GC.ADMIN_EMAIL]:
@@ -383,7 +376,6 @@ def _getAdminEmail():
return _getValueFromOAuth('email')
def writeClientCredentials(creds, filename):
m = _getMain()
creds_data = {
'client_id': creds.client_id,
'client_secret': creds.client_secret,
@@ -391,28 +383,27 @@ def writeClientCredentials(creds, filename):
'refresh_token': creds.refresh_token,
'scopes': sorted(creds.scopes or GM.Globals[GM.CREDENTIALS_SCOPES]),
'token': creds.token,
'token_expiry': creds.expiry.strftime(m.YYYYMMDDTHHMMSSZ_FORMAT),
'token_expiry': creds.expiry.strftime(_gam().YYYYMMDDTHHMMSSZ_FORMAT),
'token_uri': creds.token_uri,
}
expected_iss = ['https://accounts.google.com', 'accounts.google.com']
if _getValueFromOAuth('iss', creds) not in expected_iss:
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
request = transportCreateRequest()
try:
creds_data['decoded_id_token'] = google.oauth2.id_token.verify_oauth2_token(creds.id_token, request,
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
except ValueError as e:
if 'Token used too early' in str(e):
m.stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
m.systemErrorExit(m.SYSTEM_ERROR_RC, str(e))
_gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
GM.Globals[GM.DECODED_ID_TOKEN] = creds_data['decoded_id_token']
if filename != '-':
m.writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
_gam().writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
else:
m.writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
_gam().writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
def shortenURL(long_url):
m = _getMain()
if GC.Values[GC.NO_SHORT_URLS]:
return long_url
httpObj = getHttpObj(timeout=10)
@@ -421,7 +412,7 @@ def shortenURL(long_url):
resp, content = httpObj.request(URL_SHORTENER_ENDPOINT, 'POST',
payload,
headers={'Content-Type': 'application/json',
'User-Agent': m.GAM_USER_AGENT})
'User-Agent': _gam().GAM_USER_AGENT})
except:
return long_url
if resp.status != 200:
@@ -440,13 +431,12 @@ def runSqliteQuery(db_file, query):
return curr.fetchone()[0]
def refreshCredentialsWithReauth(credentials):
m = _getMain()
def gcloudError():
m.writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
_gam().writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
e = Msg.REAUTHENTICATION_IS_NEEDED
handleOAuthTokenError(e, False)
m.writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
_gam().writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
if 'termios' in sys.modules:
import termios
old_settings = termios.tcgetattr(sys.stdin)
@@ -454,7 +444,7 @@ def refreshCredentialsWithReauth(credentials):
# First makes sure gcloud has a valid access token and thus
# should also have a valid RAPT token
try:
devnull = open(os.devnull, 'w', encoding=m.UTF8)
devnull = open(os.devnull, 'w', encoding=_gam().UTF8)
subprocess.run(['gcloud',
'auth',
'print-identity-token',
@@ -472,7 +462,7 @@ def refreshCredentialsWithReauth(credentials):
if 'termios' in sys.modules:
import termios
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
m.printBlankLine()
_gam().printBlankLine()
raise KeyboardInterrupt from e
token_path = gcloud_path_result.stdout.decode().strip()
if not token_path:
@@ -484,19 +474,18 @@ def refreshCredentialsWithReauth(credentials):
except TypeError:
gcloudError()
if not credentials._rapt_token:
m.systemErrorExit(m.SYSTEM_ERROR_RC,
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC,
'Failed to retrieve reauth token from gcloud. You may need to wait until gcloud is also prompted for reauth.')
def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, api=None, noDASA=False, refreshOnly=False, noScopes=False):
"""Gets OAuth2 credentials which are guaranteed to be fresh and valid.
Locks during read and possible write so that only one process will
attempt refresh/write when running in parallel. """
m = _getMain()
lock = FileLock(GM.Globals[GM.OAUTH2_TXT_LOCK], mode=GC.Values[GC.OAUTH2_TXT_LOCK_MODE])
with lock:
writeCreds, credentials = getOauth2TxtCredentials(api=api, noDASA=noDASA, refreshOnly=refreshOnly, noScopes=noScopes)
if not credentials:
m.invalidOauth2TxtExit('')
_gam().invalidOauth2TxtExit('')
if credentials.expired or forceRefresh:
triesLimit = 3
for n in range(1, triesLimit+1):
@@ -507,7 +496,7 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
break
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -522,14 +511,13 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
return credentials
def waitOnFailure(n, triesLimit, error_code, error_message):
m = _getMain()
delta = min(2 ** n, 60)+float(random.randint(1, 1000))/1000
if n > 3:
m.writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
m.flushStderr()
_gam().writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
_gam().flushStderr()
time.sleep(delta)
if GC.Values[GC.SHOW_API_CALLS_RETRY_DATA]:
m.incrAPICallsRetryData(error_message, delta)
_gam().incrAPICallsRetryData(error_message, delta)
def clearServiceCache(service):
if hasattr(service._http, 'http') and hasattr(service._http.http, 'cache'):
@@ -551,7 +539,6 @@ def getAPIService(api, httpObj):
discoveryServiceUrl=DISCOVERY_URIS[v2discovery], static_discovery=False)
def getService(api, httpObj):
m = _getMain()
hasLocalJSON = API.hasLocalJSON(api)
api, version, v2discovery = API.getVersion(api)
if api in GM.Globals[GM.CURRENT_API_SERVICES] and version in GM.Globals[GM.CURRENT_API_SERVICES][api]:
@@ -577,22 +564,22 @@ def getService(api, httpObj):
clearServiceCache(service)
return service
except googleapiclient.errors.UnknownApiNameOrVersion as e:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), m.__author__))
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), _gam().__author__))
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
if n != triesLimit:
waitOnFailure(n, triesLimit, m.INVALID_JSON_RC, str(e))
waitOnFailure(n, triesLimit, _gam().INVALID_JSON_RC, str(e))
continue
m.systemErrorExit(m.INVALID_JSON_RC, str(e))
_gam().systemErrorExit(_gam().INVALID_JSON_RC, str(e))
except (http.client.ResponseNotReady, OSError, googleapiclient.errors.HttpError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit:
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg)
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
continue
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg)
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
httpObj.connections = {}
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
disc_file, discovery = readDiscoveryFile(f'{api}-{version}')
@@ -604,9 +591,9 @@ def getService(api, httpObj):
clearServiceCache(service)
return service
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
m.invalidDiscoveryJsonExit(disc_file, str(e))
_gam().invalidDiscoveryJsonExit(disc_file, str(e))
except IOError as e:
m.systemErrorExit(m.FILE_ERROR_RC, str(e))
_gam().systemErrorExit(_gam().FILE_ERROR_RC, str(e))
def defaultSvcAcctScopes():
scopesList = API.getSvcAcctScopesList(GC.Values[GC.USER_SERVICE_ACCOUNT_ACCESS_ONLY], False)
@@ -622,17 +609,16 @@ def defaultSvcAcctScopes():
return saScopes
def _getSvcAcctData():
m = _getMain()
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
jsonData = m.readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
if not jsonData:
m.invalidOauth2serviceJsonExit(Msg.NO_DATA)
_gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
try:
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = json.loads(jsonData)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
m.invalidOauth2serviceJsonExit(str(e))
_gam().invalidOauth2serviceJsonExit(str(e))
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
m.systemErrorExit(m.OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
_gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
requiredFields = ['client_email', 'client_id', 'project_id', 'token_uri']
key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
if key_type == 'default':
@@ -642,7 +628,7 @@ def _getSvcAcctData():
if field not in GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
missingFields.append(field)
if missingFields:
m.invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
_gam().invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
# Some old oauth2service.json files have: 'https://accounts.google.com/o/oauth2/auth' which no longer works
if GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] == 'https://accounts.google.com/o/oauth2/auth':
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] = API.GOOGLE_OAUTH2_TOKEN_ENDPOINT
@@ -654,7 +640,6 @@ def _getSvcAcctData():
GM.Globals[GM.SVCACCT_SCOPES] = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].pop(API.OAUTH2SA_SCOPES)
def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=False):
m = _getMain()
_getSvcAcctData()
if isinstance(scopesOrAPI, str):
GM.Globals[GM.CURRENT_SVCACCT_API] = scopesOrAPI
@@ -665,7 +650,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
if scopesOrAPI != API.CHAT_EVENTS and not GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]:
if softErrors:
return None
m.SvcAcctAPIAccessDeniedExit()
_gam().SvcAcctAPIAccessDeniedExit()
if scopesOrAPI in {API.PEOPLE, API.PEOPLE_DIRECTORY, API.PEOPLE_OTHERCONTACTS}:
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES].append(API.USERINFO_PROFILE_SCOPE)
if scopesOrAPI in {API.PEOPLE_OTHERCONTACTS}:
@@ -692,7 +677,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
except (ValueError, IndexError, KeyError) as e:
if softErrors:
return None
m.invalidOauth2serviceJsonExit(str(e))
_gam().invalidOauth2serviceJsonExit(str(e))
credentials = credentials.with_scopes(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])
else:
audience = f'https://{scopesOrAPI}.googleapis.com/'
@@ -714,7 +699,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
except (ValueError, IndexError, KeyError) as e:
if softErrors:
return None
m.invalidOauth2serviceJsonExit(str(e))
_gam().invalidOauth2serviceJsonExit(str(e))
GM.Globals[GM.CURRENT_SVCACCT_USER] = userEmail
if userEmail:
credentials = credentials.with_subject(userEmail)
@@ -723,7 +708,6 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
return credentials
def getGDataOAuthToken(gdataObj, credentials=None):
m = _getMain()
if not credentials:
credentials = getClientCredentials(refreshOnly=True)
try:
@@ -742,14 +726,13 @@ def getGDataOAuthToken(gdataObj, credentials=None):
GM.Globals[GM.ADMIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('email', 'UNKNOWN').lower()
GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id
gdataObj.domain = GC.Values[GC.DOMAIN]
gdataObj.source = m.GAM_USER_AGENT
gdataObj.source = _gam().GAM_USER_AGENT
return True
def checkGDataError(e, service):
m = _getMain()
error = e.args
reason = error[0].get('reason', '')
body = error[0].get('body', '').decode(m.UTF8)
body = error[0].get('body', '').decode(_gam().UTF8)
# First check for errors that need special handling
if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']:
keep_domain = service.domain
@@ -758,7 +741,7 @@ def checkGDataError(e, service):
return (GDATA.TOKEN_EXPIRED, reason)
error_code = getattr(e, 'error_code', 600)
if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} {error_code}: {reason}, {body}\n')
_gam().writeStdout(f'{_gam().ERROR_PREFIX} {error_code}: {reason}, {body}\n')
if error_code == 600:
if (body.startswith('Quota exceeded for the current request') or
body.startswith('Quota exceeded for quota metric') or
@@ -859,7 +842,6 @@ def callGData(service, function,
bailOnInternalServerError=False, softErrors=False,
throwErrors=None, retryErrors=None, triesLimit=0,
**kwargs):
m = _getMain()
if throwErrors is None:
throwErrors = []
if retryErrors is None:
@@ -869,7 +851,7 @@ def callGData(service, function,
allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors
method = getattr(service, function)
if GC.Values[GC.API_CALLS_RATE_CHECK]:
m.checkAPICallsRate()
_gam().checkAPICallsRate()
for n in range(1, triesLimit+1):
try:
return method(**kwargs)
@@ -886,14 +868,14 @@ def callGData(service, function,
raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message)
raise
if softErrors:
m.stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
_gam().stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
return None
if error_code == GDATA.INSUFFICIENT_PERMISSIONS:
m.APIAccessDeniedExit()
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
_gam().APIAccessDeniedExit()
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -904,33 +886,31 @@ def callGData(service, function,
except (http.client.ResponseNotReady, OSError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit:
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg)
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
continue
if softErrors:
m.writeStderr(f'\n{m.ERROR_PREFIX}{errMsg} - Giving up.\n')
_gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
return None
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg)
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
def writeGotMessage(msg):
m = _getMain()
if GC.Values[GC.SHOW_GETTINGS_GOT_NL]:
m.writeStderr(msg)
_gam().writeStderr(msg)
else:
m.writeStderr('\r')
_gam().writeStderr('\r')
msgLen = len(msg)
if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]:
m.writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
_gam().writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
else:
m.writeStderr(msg)
_gam().writeStderr(msg)
GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen
m.flushStderr()
_gam().flushStderr()
def callGDataPages(service, function,
pageMessage=None,
softErrors=False, throwErrors=None, retryErrors=None,
uri=None,
**kwargs):
m = _getMain()
Ent = _getEnt()
if throwErrors is None:
throwErrors = []
@@ -955,32 +935,30 @@ def callGDataPages(service, function,
nextLink = None
pageItems = 0
if pageMessage:
show_message = pageMessage.replace(m.TOTAL_ITEMS_MARKER, str(totalItems))
show_message = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems)))
if nextLink is None:
if pageMessage and (pageMessage[-1] != '\n'):
m.writeStderr('\r\n')
m.flushStderr()
_gam().writeStderr('\r\n')
_gam().flushStderr()
return allResults
uri = nextLink.href
if 'url_params' in kwargs:
kwargs['url_params'].pop('start-index', None)
def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True):
m = _getMain()
def makeErrorDict(code, reason, message):
return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}}
try:
error = json.loads(e.content.decode(m.UTF8))
error = json.loads(e.content.decode(_gam().UTF8))
if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} JSON: {str(error)}\n')
_gam().writeStdout(f'{_gam().ERROR_PREFIX} JSON: {str(error)}\n')
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
eContent = e.content.decode(m.UTF8) if isinstance(e.content, bytes) else e.content
eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
lContent = eContent.lower()
if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} HTTP: {str(eContent)}\n')
_gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
if eContent[0:15] != '<!DOCTYPE html>':
if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')):
return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent)
@@ -993,7 +971,7 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
if (e.resp['status'] == '504') and ('gateway timeout' in lContent):
return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent)
else:
tg = m.HTML_TITLE_PATTERN.match(lContent)
tg = _gam().HTML_TITLE_PATTERN.match(lContent)
lContent = tg.group(1) if tg else 'bad request'
if (e.resp['status'] == '403') and ('invalid domain.' in lContent):
error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found')
@@ -1020,10 +998,10 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
elif retryOnHttpError:
return (-1, None, eContent)
elif softErrors:
m.stderrErrorMsg(eContent)
_gam().stderrErrorMsg(eContent)
return (0, None, None)
else:
m.systemErrorExit(m.HTTP_ERROR_RC, eContent)
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, eContent)
requiredScopes = ''
wwwAuthenticate = e.resp.get('www-authenticate', '')
if 'insufficient_scope' in wwwAuthenticate:
@@ -1129,9 +1107,9 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
http_status = 400
error = makeErrorDict(http_status, GAPI.INVALID, message)
else:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(error))
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
else:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(error))
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
try:
reason = error['error']['errors'][0]['reason']
for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []):
@@ -1157,7 +1135,6 @@ def callGAPI(service, function,
softErrors=False, mapNotFound=True,
throwReasons=None, retryReasons=None, triesLimit=0,
**kwargs):
m = _getMain()
if throwReasons is None:
throwReasons = []
if retryReasons is None:
@@ -1168,7 +1145,7 @@ def callGAPI(service, function,
method = getattr(service, function)
svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST])
if GC.Values[GC.API_CALLS_RATE_CHECK]:
m.checkAPICallsRate()
_gam().checkAPICallsRate()
for n in range(1, triesLimit+1):
try:
return method(**svcparms).execute()
@@ -1181,7 +1158,7 @@ def callGAPI(service, function,
# service._http.credentials.refresh(getHttpObj())
service._http.credentials.refresh(transportCreateRequest())
except TypeError:
m.systemErrorExit(m.HTTP_ERROR_RC, message)
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, message)
continue
if http_status == 0:
return None
@@ -1202,17 +1179,17 @@ def callGAPI(service, function,
raise GAPI.REASON_EXCEPTION_MAP[reason](message)
raise e
if softErrors:
m.stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
_gam().stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
return None
if reason == GAPI.INSUFFICIENT_PERMISSIONS:
m.APIAccessDeniedExit()
m.systemErrorExit(m.HTTP_ERROR_RC, m.formatHTTPError(http_status, reason, message))
_gam().APIAccessDeniedExit()
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, _gam().formatHTTPError(http_status, reason, message))
except googleapiclient.errors.MediaUploadSizeError as e:
raise e
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
service._http.connections = {}
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -1223,23 +1200,22 @@ def callGAPI(service, function,
except (http.client.ResponseNotReady, OSError) as e:
errMsg = f'Connection error: {str(e) or repr(e)}'
if n != triesLimit:
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg)
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
continue
if softErrors:
m.writeStderr(f'\n{m.ERROR_PREFIX}{errMsg} - Giving up.\n')
_gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
return None
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg)
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
except ValueError as e:
if clearServiceCache(service):
continue
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(e))
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
except TypeError as e:
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(e))
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType):
m = _getMain()
Ent = _getEnt()
showMessage = pageMessage.replace(m.TOTAL_ITEMS_MARKER, str(totalItems))
showMessage = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
if pageItems:
if messageAttribute:
firstItem = results[0] if pageItems > 0 else {}
@@ -1253,11 +1229,11 @@ def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAtt
lastItem = lastItem.get(attr, {})
firstItem = str(firstItem)
lastItem = str(lastItem)
showMessage = showMessage.replace(m.FIRST_ITEM_MARKER, firstItem)
showMessage = showMessage.replace(m.LAST_ITEM_MARKER, lastItem)
showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, firstItem)
showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, lastItem)
else:
showMessage = showMessage.replace(m.FIRST_ITEM_MARKER, '')
showMessage = showMessage.replace(m.LAST_ITEM_MARKER, '')
showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, '')
showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, '')
writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems))))
def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType):
@@ -1280,10 +1256,9 @@ def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage,
return (pageToken, totalItems)
def _finalizeGAPIpagesResult(pageMessage):
m = _getMain()
if pageMessage and (pageMessage[-1] != '\n'):
m.writeStderr('\r\n')
m.flushStderr()
_gam().writeStderr('\r\n')
_gam().flushStderr()
def _setMaxArgResults(maxItems, pageArgsInBody, kwargs):
if pageArgsInBody:
@@ -1403,25 +1378,23 @@ def callGAPIitems(service, function, items,
return []
def readDiscoveryFile(api_version):
m = _getMain()
disc_filename = f'{api_version}.json'
disc_file = os.path.join(GM.Globals[GM.GAM_PATH], disc_filename)
if hasattr(sys, '_MEIPASS'):
json_string = m.readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
json_string = _gam().readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
elif os.path.isfile(disc_file):
json_string = m.readFile(disc_file, continueOnError=True, displayError=True)
json_string = _gam().readFile(disc_file, continueOnError=True, displayError=True)
else:
json_string = None
if not json_string:
m.invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
_gam().invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
try:
discovery = json.loads(json_string)
return (disc_file, discovery)
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
m.invalidDiscoveryJsonExit(disc_file, str(e))
_gam().invalidDiscoveryJsonExit(disc_file, str(e))
def buildGAPIObject(api, credentials=None):
m = _getMain()
if credentials is None:
credentials = getClientCredentials(api=api, refreshOnly=True)
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj(cache=GM.Globals[GM.CACHE_DIR]))
@@ -1433,7 +1406,7 @@ def buildGAPIObject(api, credentials=None):
GM.Globals[GM.CURRENT_CLIENT_API] = api
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API_Scopes.intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
if api not in API.SCOPELESS_APIS and not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
m.systemErrorExit(m.NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
_gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
if not GC.Values[GC.DOMAIN]:
GC.Values[GC.DOMAIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('hd', 'UNKNOWN').lower()
if not GC.Values[GC.CUSTOMER_ID]:
@@ -1443,10 +1416,9 @@ def buildGAPIObject(api, credentials=None):
return service
def getSaUser(user):
m = _getMain()
currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API]
currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
userEmail = m.convertUIDtoEmailAddress(user) if user else None
userEmail = _gam().convertUIDtoEmailAddress(user) if user else None
GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes
return userEmail
@@ -1458,7 +1430,6 @@ def chooseSaAPI(api1, api2):
return api2
def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
m = _getMain()
userEmail = getSaUser(user)
if GM.Globals[GM.HTTP_OBJECT] is None:
GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR])
@@ -1475,7 +1446,7 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if n != triesLimit:
httpObj.connections = {}
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
continue
handleServerError(e)
except google.auth.exceptions.RefreshError as e:
@@ -1485,12 +1456,12 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
if isinstance(e, str):
eContent = e
else:
eContent = e.content.decode(m.UTF8) if isinstance(e.content, bytes) else e.content
eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
if eContent[0:15] == '<!DOCTYPE html>':
if GC.Values[GC.DEBUG_LEVEL] > 0:
m.writeStdout(f'{m.ERROR_PREFIX} HTTP: {str(eContent)}\n')
_gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
lContent = eContent.lower()
tg = m.HTML_TITLE_PATTERN.match(lContent)
tg = _gam().HTML_TITLE_PATTERN.match(lContent)
lContent = tg.group(1) if tg else ''
if lContent.startswith('Error 502 (Server Error)'):
time.sleep(30)
@@ -1504,12 +1475,11 @@ def buildGAPIObjectNoAuthentication(api):
return service
def initGDataObject(gdataObj, api):
m = _getMain()
GM.Globals[GM.CURRENT_CLIENT_API] = api
credentials = getClientCredentials(noDASA=True, refreshOnly=True)
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API.getClientScopesSet(api).intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
if not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
m.systemErrorExit(m.NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
_gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
getGDataOAuthToken(gdataObj, credentials)
if GC.Values[GC.DEBUG_LEVEL] > 0:
gdataObj.debug = True

View File

@@ -115,8 +115,7 @@ def _getEnt():
return sys.modules['gam'].Ent
# Lazy accessor for main module
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
# --- Constants duplicated from __init__.py ---
# These are simple literals that never change, duplicated to avoid

View File

@@ -27,8 +27,7 @@ from gamlib import glmsgs as Msg
from util.csv_pf import CSVPrintFile
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
Cmd = glclargs.GamCLArgs()
@@ -72,18 +71,18 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
def reopenSTDFile(stdtype):
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_getMain().UTF8)
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][stdtype == GM.STDOUT]),
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
elif stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
GM.Globals[stdtype][GM.REDIRECT_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_FD]
else:
GM.Globals[stdtype][GM.REDIRECT_FD] = _getMain().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
GM.Globals[stdtype][GM.REDIRECT_FD] = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
if stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
else:
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _getMain().StringIOobject()
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _gam().StringIOobject()
GM.Globals[GM.DATETIME_NOW] = datetimeNow
GC.Values[GC.TIMEZONE] = tzinfo
@@ -158,11 +157,11 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
if mpQueueStdout:
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()))
else:
_getMain().flushStdout()
_gam().flushStdout()
if mpQueueStderr and mpQueueStderr is not mpQueueStdout:
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD].getvalue()))
else:
_getMain().flushStderr()
_gam().flushStderr()
def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr):
mpQueue = mpManager.Queue()
@@ -178,7 +177,7 @@ def terminateCSVFileQueueHandler(mpQueue, mpQueueHandler):
GM.Globals[GM.PARSER] = None
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None
if multiprocessing.get_start_method() != 'fork':
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _getMain().Cmd.AllArguments()))
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _gam().Cmd.AllArguments()))
savedValues = saveNonPickleableValues()
mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals))
restoreNonPickleableValues(savedValues)
@@ -200,27 +199,27 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
if data[1] is not None:
_writeData(data[1])
if GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _getMain().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _gam().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
fd.flush()
except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
if multiprocessing.get_start_method() != 'fork':
signal.signal(signal.SIGINT, signal.SIG_IGN)
GM.Globals = gmGlobals.copy()
GC.Values = gcValues.copy()
pid0DataItem = [_getMain().KEYBOARD_INTERRUPT_RC, None]
pid0DataItem = [_gam().KEYBOARD_INTERRUPT_RC, None]
pidData = {}
if multiprocessing.get_start_method() != 'fork':
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_getMain().UTF8)
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
fd = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stdout']),
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout' and GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stderr':
fd = os.fdopen(os.dup(sys.stdout.fileno()), GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
else:
fd = _getMain().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
fd = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
else:
fd = GM.Globals[stdtype][GM.REDIRECT_FD]
while True:
@@ -230,7 +229,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
break
if dataType == GM.REDIRECT_QUEUE_START:
pidData[pid] = {'queue': GM.Globals[stdtype][GM.REDIRECT_QUEUE],
'start': _getMain().currentISOformatTimeStamp(),
'start': _gam().currentISOformatTimeStamp(),
'cmd': Cmd.QuotedArgumentList(dataItem)}
if pid == 0 and GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
fd.write(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'Start', pidData[pid]['start'], 0, pidData[pid]['cmd']))
@@ -246,7 +245,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
break
for pid in pidData:
if pid != 0:
_writePidData(pid, [_getMain().KEYBOARD_INTERRUPT_RC, None])
_writePidData(pid, [_gam().KEYBOARD_INTERRUPT_RC, None])
_writePidData(0, pid0DataItem)
if fd not in [sys.stdout, sys.stderr]:
try:
@@ -267,7 +266,7 @@ def batchWriteStderr(data):
sys.stderr.write(data)
sys.stderr.flush()
except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage('stderr', e))
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage('stderr', e))
def writeStdQueueHandler(mpQueue, item):
while True:
@@ -276,7 +275,7 @@ def writeStdQueueHandler(mpQueue, item):
return
except Exception as e:
time.sleep(1)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
def terminateStdQueueHandler(mpQueue, mpQueueHandler):
mpQueue.put((0, GM.REDIRECT_QUEUE_EOF, None))
@@ -339,7 +338,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
if mpQueueCSVFile:
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = mpQueueCSVFile
if mpQueueStdout:
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _getMain().StringIOobject()}
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
if debugLevel:
sys.stdout = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_START, args))
@@ -348,14 +347,14 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
GM.Globals[GM.STDOUT] = {}
if mpQueueStderr:
if mpQueueStderr is not mpQueueStdout:
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _getMain().StringIOobject()}
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
# mpQueueStderr.put((pid, GM.REDIRECT_QUEUE_START, args))
writeStdQueueHandler(mpQueueStderr, (pid, GM.REDIRECT_QUEUE_START, args))
else:
GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
else:
GM.Globals[GM.STDERR] = {}
sysRC = _getMain().ProcessGAMCommand(args)
sysRC = _gam().ProcessGAMCommand(args)
with mplock:
if mpQueueStdout:
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_END, [sysRC, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()]))
@@ -405,9 +404,9 @@ def MultiprocessGAMCommands(items, showCmds):
def poolCallback(result):
poolProcessResults[0] -= 1
if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
if GM.Globals[GM.CMDLOG_LOGGER]:
GM.Globals[GM.CMDLOG_LOGGER].info(f'{_getMain().currentISOformatTimeStamp()},{result[1]},{result[2]}')
GM.Globals[GM.CMDLOG_LOGGER].info(f'{_gam().currentISOformatTimeStamp()},{result[1]},{result[2]}')
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(result[1]):
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
@@ -418,8 +417,8 @@ def MultiprocessGAMCommands(items, showCmds):
def handleControlC(source):
nonlocal controlC
batchWriteStderr(f'Control-C (Multiprocess-{source})\n')
_getMain().setSysExitRC(_getMain().KEYBOARD_INTERRUPT_RC)
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
_gam().setSysExitRC(_gam().KEYBOARD_INTERRUPT_RC)
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
pool.terminate()
@@ -445,21 +444,21 @@ def MultiprocessGAMCommands(items, showCmds):
else:
pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, e)
_gam().systemErrorExit(_gam().FILE_ERROR_RC, e)
except AssertionError as e:
_getMain().Cmd.SetLocation(0)
_getMain().usageErrorExit(str(e))
_gam().Cmd.SetLocation(0)
_gam().usageErrorExit(str(e))
if multiprocessing.get_start_method() != 'fork':
savedValues = saveNonPickleableValues()
if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]:
mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values)
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _getMain().Cmd.AllArguments()))
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
else:
mpQueueStdout = None
if GM.Globals[GM.STDERR][GM.REDIRECT_MULTIPROCESS]:
if GM.Globals[GM.STDERR][GM.REDIRECT_NAME] != 'stdout':
mpQueueStderr, mpQueueHandlerStderr = initializeStdQueueHandler(mpManager, GM.STDERR, GM.Globals, GC.Values)
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _getMain().Cmd.AllArguments()))
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
else:
mpQueueStderr = mpQueueStdout
else:
@@ -479,7 +478,7 @@ def MultiprocessGAMCommands(items, showCmds):
# signal.signal(signal.SIGINT, origSigintHandler)
controlC = False
signal.signal(signal.SIGINT, signal_handler)
batchWriteStderr(Msg.USING_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, numPoolProcesses,
PROCESS_PLURAL_SINGULAR[numPoolProcesses == 1]))
try:
@@ -491,7 +490,7 @@ def MultiprocessGAMCommands(items, showCmds):
if controlC:
break
if item[0] == Cmd.COMMIT_BATCH_CMD:
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
while poolProcessResults[0] > 0:
@@ -503,24 +502,24 @@ def MultiprocessGAMCommands(items, showCmds):
completedProcesses.append(p)
for p in completedProcesses:
del poolProcessResults[p]
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
if len(item) > 1:
_getMain().readStdin(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
_gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
continue
if item[0] == Cmd.PRINT_CMD:
batchWriteStderr(Cmd.QuotedArgumentList(item[1:])+'\n')
continue
if item[0] == Cmd.SLEEP_CMD:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
time.sleep(int(item[1]))
continue
pid += 1
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_getMain().currentISOformatTimeStamp(), pid, numItems))
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
if showCmds or GM.Globals[GM.CMDLOG_LOGGER]:
logCmd = Cmd.QuotedArgumentList(item)
if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
else:
logCmd = ''
poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti,
@@ -566,7 +565,7 @@ def MultiprocessGAMCommands(items, showCmds):
else:
waitRemaining = 'unlimited'
while poolProcessResults[0] > 0:
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1],
Msg.BATCH_CSV_WAIT_LIMIT.format(waitRemaining)))
@@ -585,7 +584,7 @@ def MultiprocessGAMCommands(items, showCmds):
if GC.Values[GC.PROCESS_WAIT_LIMIT] > 0:
delta = int(time.time()-processWaitStart)
if delta >= GC.Values[GC.PROCESS_WAIT_LIMIT]:
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, poolProcessResults[0],
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
pool.terminate()
@@ -597,7 +596,7 @@ def MultiprocessGAMCommands(items, showCmds):
except KeyboardInterrupt:
handleControlC('KBI')
pool.join()
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems))
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems))
if mpQueueCSVFile:
terminateCSVFileQueueHandler(mpQueueCSVFile, mpQueueHandlerCSVFile)
if mpQueueStdout:
@@ -618,11 +617,11 @@ def threadBatchWorker(showCmds=False, numItems=0):
sysRC = subprocess.call(item, stdout=GM.Globals[GM.STDOUT].get(GM.REDIRECT_MULTI_FD, sys.stdout),
stderr=GM.Globals[GM.STDERR].get(GM.REDIRECT_MULTI_FD, sys.stderr))
if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(sysRC):
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
except Exception as e:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
GM.Globals[GM.TBATCH_QUEUE].task_done()
BATCH_COMMANDS = [Cmd.GAM_CMD, Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD, Cmd.SLEEP_CMD, Cmd.DATETIME_CMD, Cmd.SET_CMD, Cmd.CLEAR_CMD]
@@ -633,12 +632,12 @@ def ThreadBatchGAMCommands(items, showCmds):
return
pythonCmd = [sys.executable]
if not getattr(sys, 'frozen', False): # we're not frozen
pythonCmd.append(os.path.realpath(_getMain().Cmd.Argument(0)))
pythonCmd.append(os.path.realpath(_gam().Cmd.Argument(0)))
GM.Globals[GM.NUM_BATCH_ITEMS] = numItems = len(items)
numWorkerThreads = min(numItems, GC.Values[GC.NUM_TBATCH_THREADS])
# GM.Globals[GM.TBATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
GM.Globals[GM.TBATCH_QUEUE] = queue.Queue(maxsize=numWorkerThreads)
batchWriteStderr(Msg.USING_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, numWorkerThreads,
THREAD_PLURAL_SINGULAR[numWorkerThreads == 1]))
for _ in range(numWorkerThreads):
@@ -651,28 +650,28 @@ def ThreadBatchGAMCommands(items, showCmds):
if GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING]:
break
if item[0] == Cmd.COMMIT_BATCH_CMD:
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
numItems, numThreadsInUse,
THREAD_PLURAL_SINGULAR[numThreadsInUse == 1]))
GM.Globals[GM.TBATCH_QUEUE].join()
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems, Msg.THREADS))
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.THREADS))
numThreadsInUse = 0
if len(item) > 1:
_getMain().readStdin(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
_gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
continue
if item[0] == Cmd.PRINT_CMD:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
continue
if item[0] == Cmd.SLEEP_CMD:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
time.sleep(int(item[1]))
continue
pid += 1
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_getMain().currentISOformatTimeStamp(), pid, numItems))
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
if showCmds:
logCmd = Cmd.QuotedArgumentList(item)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
else:
logCmd = ''
if item[0] == Cmd.GAM_CMD:
@@ -682,43 +681,43 @@ def ThreadBatchGAMCommands(items, showCmds):
numThreadsInUse += 1
GM.Globals[GM.TBATCH_QUEUE].join()
if showCmds:
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Complete\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Complete\n')
def _getShowCommands():
if _getMain().checkArgumentPresent('showcmds'):
return _getMain().getBoolean()
if _gam().checkArgumentPresent('showcmds'):
return _gam().getBoolean()
return GC.Values[GC.SHOW_COMMANDS]
def _getSkipRows():
if _getMain().checkArgumentPresent('skiprows'):
return _getMain().getInteger(minVal=0)
if _gam().checkArgumentPresent('skiprows'):
return _gam().getInteger(minVal=0)
# return GC.Values[GC.CSV_INPUT_ROW_SKIP]
return 0
def _getMaxRows():
if _getMain().checkArgumentPresent('maxrows'):
return _getMain().getInteger(minVal=0)
if _gam().checkArgumentPresent('maxrows'):
return _gam().getInteger(minVal=0)
return GC.Values[GC.CSV_INPUT_ROW_LIMIT]
# gam batch <BatchContent> [showcmds [<Boolean>]]
def doBatch(threadBatch=False):
filename = _getMain().getString(Cmd.OB_FILE_NAME)
filename = _gam().getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_getMain().Cmd.Backup()
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
_gam().Cmd.Backup()
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
filenameLower = filename.lower()
if filenameLower not in {'gdoc', 'gcsdoc'}:
encoding = _getMain().getCharSet()
filename = _getMain().setFilePath(filename, GC.INPUT_DIR)
f = _getMain().openFile(filename, encoding=encoding, stripUTFBOM=True)
encoding = _gam().getCharSet()
filename = _gam().setFilePath(filename, GC.INPUT_DIR)
f = _gam().openFile(filename, encoding=encoding, stripUTFBOM=True)
elif filenameLower == 'gdoc':
f = _getMain().getGDocData(filenameLower)
_getMain().getCharSet()
f = _gam().getGDocData(filenameLower)
_gam().getCharSet()
else: #filenameLower == 'gcsdoc':
f = _getMain().getStorageFileData(filenameLower)
_getMain().getCharSet()
f = _gam().getStorageFileData(filenameLower)
_gam().getCharSet()
showCmds = _getShowCommands()
_getMain().checkForExtraneousArguments()
_gam().checkForExtraneousArguments()
validCommands = BATCH_COMMANDS if not threadBatch else TBATCH_COMMANDS
kwValues = {}
items = []
@@ -733,40 +732,40 @@ def doBatch(threadBatch=False):
try:
argv = shlex.split(line)
except ValueError as e:
_getMain().writeStderr(f'Command: >>>{line.strip()}<<<\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{str(e)}\n')
_gam().writeStderr(f'Command: >>>{line.strip()}<<<\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{str(e)}\n')
errors += 1
continue
if argv:
cmd = argv[0].strip().lower()
if cmd == Cmd.DATETIME_CMD:
if len(argv) == 2:
kwValues['datetime'] = _getMain().todaysTime().strftime(argv[1])
kwValues['datetime'] = _gam().todaysTime().strftime(argv[1])
else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
errors += 1
continue
if cmd == Cmd.SET_CMD:
if len(argv) == 3:
kwValues[argv[1]] = argv[2]
else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
errors += 1
continue
if cmd == Cmd.CLEAR_CMD:
if len(argv) == 2:
kwValues.pop(argv[1], None)
else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
errors += 1
continue
if cmd == Cmd.SLEEP_CMD:
if len(argv) != 2 or not argv[1].isdigit():
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
errors += 1
continue
if (not cmd) or ((len(argv) == 1) and (cmd not in [Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD])):
@@ -774,28 +773,28 @@ def doBatch(threadBatch=False):
if cmd in validCommands:
items.append(argv)
else:
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_getMain().formatChoiceList(validCommands)}>\n')
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_gam().formatChoiceList(validCommands)}>\n')
errors += 1
except IOError as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage(filename, e))
_getMain().closeFile(f)
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
_gam().closeFile(f)
if errors == 0:
if not threadBatch:
MultiprocessGAMCommands(items, showCmds)
else:
ThreadBatchGAMCommands(items, showCmds)
else:
_getMain().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_getMain().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
_getMain().setSysExitRC(_getMain().USAGE_ERROR_RC)
_gam().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_gam().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
_gam().setSysExitRC(_gam().USAGE_ERROR_RC)
# gam tbatch <BatchContent> [showcmds [<Boolean>]]
def doThreadBatch():
_getMain().adjustRedirectedSTDFilesIfNotMultiprocessing()
_gam().adjustRedirectedSTDFilesIfNotMultiprocessing()
doBatch(True)
def doAutoBatch(entityType, entityList, CL_command):
remaining = _getMain().Cmd.Remaining()
remaining = _gam().Cmd.Remaining()
items = []
initial_argv = [Cmd.GAM_CMD]
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION]:
@@ -824,8 +823,8 @@ def getSubFields(initial_argv, fieldNames):
subFields = {}
GAM_argv = initial_argv[:]
GAM_argvI = len(GAM_argv)
while _getMain().Cmd.ArgumentsRemaining():
myarg = _getMain().Cmd.Current()
while _gam().Cmd.ArgumentsRemaining():
myarg = _gam().Cmd.Current()
if not myarg:
GAM_argv.append(myarg)
elif SUB_PATTERN.search(myarg):
@@ -839,17 +838,17 @@ def getSubFields(initial_argv, fieldNames):
if not rematch:
fieldName = submatch.group(1)
if fieldName not in fieldNames:
_getMain().csvFieldErrorExit(fieldName, fieldNames)
_gam().csvFieldErrorExit(fieldName, fieldNames)
subFields[GAM_argvI].append((SUB_TYPE, fieldName, submatch.start(), submatch.end()))
else:
fieldName = rematch.group(1)
if fieldName not in fieldNames:
_getMain().csvFieldErrorExit(fieldName, fieldNames)
_gam().csvFieldErrorExit(fieldName, fieldNames)
try:
re.compile(rematch.group(2))
subFields[GAM_argvI].append((RE_TYPE, fieldName, submatch.start(), submatch.end(), rematch.group(2), rematch.group(3)))
except re.error as e:
_getMain().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
_gam().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
pos = submatch.end()
GAM_argv.append(myarg)
elif myarg[0] == '~':
@@ -858,11 +857,11 @@ def getSubFields(initial_argv, fieldNames):
subFields[GAM_argvI] = [(SUB_TYPE, fieldName, 0, len(myarg))]
GAM_argv.append(myarg)
else:
_getMain().csvFieldErrorExit(fieldName, fieldNames)
_gam().csvFieldErrorExit(fieldName, fieldNames)
else:
GAM_argv.append(myarg)
GAM_argvI += 1
_getMain().Cmd.Advance()
_gam().Cmd.Advance()
return(GAM_argv, subFields)
def processSubFields(GAM_argv, row, subFields):
@@ -889,28 +888,28 @@ def processSubFields(GAM_argv, row, subFields):
# [skiprows <Integer>] [maxrows <Integer>]
# gam <GAM argument list>
def doCSV(testMode=False):
filename = _getMain().getString(Cmd.OB_FILE_NAME)
filename = _gam().getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_getMain().Cmd.Backup()
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
f, csvFile, fieldnames = _getMain().openCSVFileReader(filename)
matchFields, skipFields = _getMain().getMatchSkipFields(fieldnames)
_gam().Cmd.Backup()
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
showCmds = _getShowCommands()
skipRows = _getSkipRows()
maxRows = _getMaxRows()
_getMain().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _getMain().Cmd.ArgumentsRemaining():
_getMain().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
_gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _gam().Cmd.ArgumentsRemaining():
_gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
initial_argv = [Cmd.GAM_CMD]
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _getMain().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _gam().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
initial_argv.extend([Cmd.SELECT_CMD, GM.Globals[GM.SECTION]])
GAM_argv, subFields = getSubFields(initial_argv, fieldnames)
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
_getMain().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
_gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
items = []
i = 0
for row in csvFile:
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1
if skipRows:
if i <= skipRows:
@@ -920,21 +919,21 @@ def doCSV(testMode=False):
items.append(processSubFields(GAM_argv, row, subFields))
if maxRows and i >= maxRows:
break
_getMain().closeFile(f)
_gam().closeFile(f)
if not testMode:
MultiprocessGAMCommands(items, showCmds)
else:
numItems = min(len(items), 10)
_getMain().writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
_getMain().Ind.Increment()
_gam().writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
_gam().Ind.Increment()
for field in fieldnames:
_getMain().writeStdout(f'{_getMain().Ind.Spaces()}{field}\n')
_getMain().Ind.Decrement()
_getMain().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _getMain().GAM))
_getMain().Ind.Increment()
_gam().writeStdout(f'{_gam().Ind.Spaces()}{field}\n')
_gam().Ind.Decrement()
_gam().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _gam().GAM))
_gam().Ind.Increment()
for i in range(numItems):
_getMain().writeStdout(f'{_getMain().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
_getMain().Ind.Decrement()
_gam().writeStdout(f'{_gam().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
_gam().Ind.Decrement()
def doCSVTest():
doCSV(testMode=True)
@@ -945,23 +944,23 @@ def doCSVTest():
# [skiprows <Integer>] [maxrows <Integer>]
# gam <GAM argument list>
def doLoop(loopCmd):
filename = _getMain().getString(Cmd.OB_FILE_NAME)
filename = _gam().getString(Cmd.OB_FILE_NAME)
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
_getMain().Cmd.Backup()
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
f, csvFile, fieldnames = _getMain().openCSVFileReader(filename)
matchFields, skipFields = _getMain().getMatchSkipFields(fieldnames)
_gam().Cmd.Backup()
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
showCmds = _getShowCommands()
skipRows = _getSkipRows()
maxRows = _getMaxRows()
_getMain().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _getMain().Cmd.ArgumentsRemaining():
_getMain().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
_gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
if not _gam().Cmd.ArgumentsRemaining():
_gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
_getMain().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
choice = _getMain().Cmd.Current().strip().lower()
_gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
choice = _gam().Cmd.Current().strip().lower()
if choice == Cmd.LOOP_CMD:
_getMain().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
_gam().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
# gam loop ... gam redirect|select|config ... process gam.cfg on each iteration
# gam redirect|select|config ... loop ... gam redirect|select|config ... process gam.cfg on each iteration
# gam loop ... gam !redirect|select|config ... no further processing of gam.cfg
@@ -981,13 +980,13 @@ def doLoop(loopCmd):
else:
LoopGlobals = {GM.CMDLOG_LOGGER: None, GM.CMDLOG_HANDLER: None}
if (GM.Globals[GM.PID] > 0) and GC.Values[GC.CMDLOG]:
_getMain().openGAMCommandLog(LoopGlobals, 'looplog')
_gam().openGAMCommandLog(LoopGlobals, 'looplog')
if LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().writeGAMCommandLog(LoopGlobals, loopCmd, '*')
_gam().writeGAMCommandLog(LoopGlobals, loopCmd, '*')
if not showCmds:
i = 0
for row in csvFile:
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1
if skipRows:
if i <= skipRows:
@@ -997,20 +996,20 @@ def doLoop(loopCmd):
item = processSubFields(GAM_argv, row, subFields)
logCmd = Cmd.QuotedArgumentList(item)
if i % 100 == 0:
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_getMain().currentISOformatTimeStamp(), i))
sysRC = _getMain().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_gam().currentISOformatTimeStamp(), i))
sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _getMain().HARD_ERROR_RC):
_gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
break
if maxRows and i >= maxRows:
break
_getMain().closeFile(f)
_gam().closeFile(f)
else:
items = []
i = 0
for row in csvFile:
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
i += 1
if skipRows:
if i <= skipRows:
@@ -1020,26 +1019,26 @@ def doLoop(loopCmd):
items.append(processSubFields(GAM_argv, row, subFields))
if maxRows and i >= maxRows:
break
_getMain().closeFile(f)
_gam().closeFile(f)
numItems = len(items)
pid = 0
for item in items:
pid += 1
logCmd = Cmd.QuotedArgumentList(item)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
sysRC = _getMain().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _getMain().HARD_ERROR_RC):
_gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
break
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
_getMain().closeGAMCommandLog(LoopGlobals)
_gam().closeGAMCommandLog(LoopGlobals)
if multi:
terminateCSVFileQueueHandler(mpQueue, mpQueueHandler)
def _doList(entityList, entityType):
_getMain().buildGAPIObject(API.DIRECTORY)
_gam().buildGAPIObject(API.DIRECTORY)
if GM.Globals[GM.CSV_DATA_DICT]:
keyField = GM.Globals[GM.CSV_KEY_FIELD]
dataField = GM.Globals[GM.CSV_DATA_FIELD]
@@ -1047,26 +1046,26 @@ def _doList(entityList, entityType):
keyField = 'Entity'
dataField = 'Data'
csvPF = CSVPrintFile(keyField)
if _getMain().checkArgumentPresent('todrive'):
if _gam().checkArgumentPresent('todrive'):
csvPF.GetTodriveParameters()
if entityList is None:
entityList = _getMain().getEntityList(Cmd.OB_ENTITY)
showData = _getMain().checkArgumentPresent('data')
entityList = _gam().getEntityList(Cmd.OB_ENTITY)
showData = _gam().checkArgumentPresent('data')
if showData:
if not entityType:
itemType, itemList = _getMain().getEntityToModify(crosAllowed=True)
itemType, itemList = _gam().getEntityToModify(crosAllowed=True)
else:
itemType = None
itemList = _getMain().getEntityList(Cmd.OB_ENTITY)
itemList = _gam().getEntityList(Cmd.OB_ENTITY)
entityItemLists = itemList if isinstance(itemList, dict) else None
csvPF.AddTitle(dataField)
else:
entityItemLists = None
dataDelimiter = _getMain().getDelimiter()
_getMain().checkForExtraneousArguments()
_, _, entityList = _getMain().getEntityArgument(entityList)
dataDelimiter = _gam().getDelimiter()
_gam().checkForExtraneousArguments()
_, _, entityList = _gam().getEntityArgument(entityList)
for entity in entityList:
entityEmail = _getMain().normalizeEmailAddressOrUID(entity)
entityEmail = _gam().normalizeEmailAddressOrUID(entity)
if showData:
if entityItemLists:
if entity not in entityItemLists:
@@ -1075,7 +1074,7 @@ def _doList(entityList, entityType):
itemList = entityItemLists[entity]
if itemType == Cmd.ENTITY_USERS:
for i, item in enumerate(itemList):
itemList[i] = _getMain().normalizeEmailAddressOrUID(item)
itemList[i] = _gam().normalizeEmailAddressOrUID(item)
if dataDelimiter:
csvPF.WriteRow({keyField: entityEmail, dataField: dataDelimiter.join(itemList)})
else:
@@ -1098,10 +1097,10 @@ def doListUser(entityList):
_doList(entityList, Cmd.ENTITY_USERS)
def _showCount(entityList, entityType):
_getMain().buildGAPIObject(API.DIRECTORY)
_getMain().checkForExtraneousArguments()
_, count, entityList = _getMain().getEntityArgument(entityList)
_getMain().actionPerformedNumItems(count, entityType)
_gam().buildGAPIObject(API.DIRECTORY)
_gam().checkForExtraneousArguments()
_, count, entityList = _gam().getEntityArgument(entityList)
_gam().actionPerformedNumItems(count, entityType)
# gam <CrOSTypeEntity> show count
def showCountCrOS(entityList):

File diff suppressed because it is too large Load Diff

View File

@@ -22,67 +22,63 @@ from gamlib import glcfg as GC
from gamlib import glgapi as GAPI
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
# Add attachements to an email message
def _addAttachmentsToMessage(message, attachments):
gam = _getMain()
for attachment in attachments:
try:
attachFilename = gam.setFilePath(attachment[0], GC.INPUT_DIR)
attachFilename = _gam().setFilePath(attachment[0], GC.INPUT_DIR)
attachContentType, attachEncoding = mimetypes.guess_type(attachFilename)
if attachContentType is None or attachEncoding is not None:
attachContentType = 'application/octet-stream'
main_type, sub_type = attachContentType.split('/', 1)
if main_type == 'text':
msg = MIMEText(gam.readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=gam.UTF8)
msg = MIMEText(_gam().readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=_gam().UTF8)
elif main_type == 'image':
msg = MIMEImage(gam.readFile(attachFilename, 'rb'), _subtype=sub_type)
msg = MIMEImage(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
elif main_type == 'audio':
msg = MIMEAudio(gam.readFile(attachFilename, 'rb'), _subtype=sub_type)
msg = MIMEAudio(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
elif main_type == 'application':
msg = MIMEApplication(gam.readFile(attachFilename, 'rb'), _subtype=sub_type)
msg = MIMEApplication(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
else:
msg = MIMEBase(main_type, sub_type)
msg.set_payload(gam.readFile(attachFilename, 'rb'))
msg.set_payload(_gam().readFile(attachFilename, 'rb'))
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFilename))
message.attach(msg)
except (IOError, UnicodeDecodeError) as e:
gam.usageErrorExit(f'{attachFilename}: {str(e)}')
_gam().usageErrorExit(f'{attachFilename}: {str(e)}')
# Add embedded images to an email message
def _addEmbeddedImagesToMessage(message, embeddedImages):
gam = _getMain()
for embeddedImage in embeddedImages:
try:
imageFilename = gam.setFilePath(embeddedImage[0], GC.INPUT_DIR)
imageFilename = _gam().setFilePath(embeddedImage[0], GC.INPUT_DIR)
imageContentType, imageEncoding = mimetypes.guess_type(imageFilename)
if imageContentType is None or imageEncoding is not None:
imageContentType = 'application/octet-stream'
main_type, sub_type = imageContentType.split('/', 1)
if main_type == 'image':
msg = MIMEImage(gam.readFile(imageFilename, 'rb'), _subtype=sub_type)
msg = MIMEImage(_gam().readFile(imageFilename, 'rb'), _subtype=sub_type)
else:
msg = MIMEBase(main_type, sub_type)
msg.set_payload(gam.readFile(imageFilename, 'rb'))
msg.set_payload(_gam().readFile(imageFilename, 'rb'))
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(imageFilename))
msg.add_header('Content-ID', f'<{embeddedImage[1]}>')
message.attach(msg)
except (IOError, UnicodeDecodeError) as e:
gam.usageErrorExit(f'{imageFilename}: {str(e)}')
_gam().usageErrorExit(f'{imageFilename}: {str(e)}')
# Send an email
def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msgFrom=None, msgReplyTo=None,
html=False, charset=None, attachments=None, embeddedImages=None,
msgHeaders=None, ccRecipients=None, bccRecipients=None, mailBox=None, threadId=None,
action=None):
gam = _getMain()
Act = gam.Act
Ent = gam.Ent
Act = _gam().Act
Ent = _gam().Ent
if charset is None:
charset = gam.UTF8
charset = _gam().UTF8
if action is None:
action = Act.SENDEMAIL
@@ -96,26 +92,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
toSent.remove(addr)
toFailed[addr] = f'{err[0]}: {err[1]}'
if toSent:
gam.entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
_gam().entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
for addr, errMsg in toFailed.items():
gam.entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
_gam().entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
def cleanAddr(emailAddr):
match = gam.NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
match = _gam().NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
if match:
emailName = match.group(1)
emailAddr = gam.normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
emailAddr = _gam().normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
return (f'{emailName} <{emailAddr}>', emailAddr)
emailAddr = gam.normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
emailAddr = _gam().normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
return (emailAddr, emailAddr)
if msgFrom is None:
msgFrom = gam._getAdminEmail()
msgFrom = _gam()._getAdminEmail()
# Force ASCII for RFC compliance
# xmlcharref seems to work to display at least
# some unicode in HTML body and is ignored in
# plain text body.
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(gam.UTF8)
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(_gam().UTF8)
if not attachments and not embeddedImages:
message = MIMEText(msgBody, ['plain', 'html'][html], charset)
else:
@@ -145,26 +141,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
Act.Set(action)
if not GC.Values[GC.SMTP_HOST]:
if not clientAccess:
userId, gmail = gam.buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
userId, gmail = _gam().buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
if not gmail:
Act.Set(parentAction)
return
else:
userId = mailBoxAddr
gmail = gam.buildGAPIObject(API.GMAIL)
gmail = _gam().buildGAPIObject(API.GMAIL)
message['To'] = msgTo if msgTo else userId
body = {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}
if threadId is not None:
body['threadId'] = threadId
try:
result = gam.callGAPI(gmail.users().messages(), 'send',
result = _gam().callGAPI(gmail.users().messages(), 'send',
throwReasons=[GAPI.SERVICE_NOT_AVAILABLE, GAPI.AUTH_ERROR, GAPI.DOMAIN_POLICY,
GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
userId=userId, body=body, fields='id')
gam.entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
_gam().entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy,
GAPI.invalid, GAPI.invalidArgument, GAPI.forbidden, GAPI.permissionDenied) as e:
gam.entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
_gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
else:
message['To'] = msgTo if msgTo else mailBoxAddr
server = None
@@ -175,7 +171,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
server.starttls(context=ssl.create_default_context(cafile=GC.Values[GC.CACERTS_PEM]))
if GC.Values[GC.SMTP_USERNAME] and GC.Values[GC.SMTP_PASSWORD]:
if isinstance(GC.Values[GC.SMTP_PASSWORD], bytes):
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(gam.UTF8))
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(_gam().UTF8))
else:
server.login(GC.Values[GC.SMTP_USERNAME], GC.Values[GC.SMTP_PASSWORD])
result = server.send_message(message)
@@ -183,7 +179,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
checkResult(Ent.RECIPIENT_CC, ccRecipients)
checkResult(Ent.RECIPIENT_BCC, bccRecipients)
except smtplib.SMTPException as e:
gam.entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
_gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
if server:
try:
server.quit()

File diff suppressed because it is too large Load Diff

View File

@@ -24,8 +24,7 @@ from gamlib import glglobals as GM
from gamlib import glmsgs as Msg
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
GDOC_FORMAT_MIME_TYPES = {
@@ -37,39 +36,39 @@ GDOC_FORMAT_MIME_TYPES = {
# gdoc <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity>
def getGDocData(gformat):
mimeType = GDOC_FORMAT_MIME_TYPES[gformat]
user = _getMain().getEmailAddress()
fileIdEntity = _getMain().getDriveFileEntity(queryShortcutsOK=False)
user = _gam().getEmailAddress()
fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
_, drive = _getMain().buildGAPIServiceObject(API.DRIVE3, user)
_, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
else:
drive = _getMain().buildGAPIObject(API.DRIVE3)
drive = _gam().buildGAPIObject(API.DRIVE3)
if not drive:
sys.exit(GM.Globals[GM.SYSEXITRC])
_, _, jcount = _getMain()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
_, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
if jcount == 0:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
if jcount > 1:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
fileId = fileIdEntity['list'][0]
f = None
try:
result = _getMain().callGAPI(drive.files(), 'get',
result = _gam().callGAPI(drive.files(), 'get',
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
fileId=fileId, fields='name,mimeType,exportLinks',
supportsAllDrives=True)
# Google Doc
if 'exportLinks' in result:
if mimeType not in result['exportLinks']:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
_, content = drive._http.request(uri=result['exportLinks'][mimeType], method='GET')
f.write(content.decode(_getMain().UTF8_SIG))
f.write(content.decode(_gam().UTF8_SIG))
f.seek(0)
return f
# Drive File
if result['mimeType'] != mimeType:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
fb = TemporaryFile(mode='wb+')
request = drive.files().get_media(fileId=fileId)
@@ -77,63 +76,63 @@ def getGDocData(gformat):
done = False
while not done:
_, done = downloader.next_chunk()
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
fb.seek(0)
f.write(fb.read().decode(_getMain().UTF8_SIG))
f.write(fb.read().decode(_gam().UTF8_SIG))
fb.close()
f.seek(0)
return f
except GAPI.fileNotFound:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
except (IOError, httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
if f:
f.close()
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
_getMain().userDriveServiceNotEnabledWarning(user, str(e))
_gam().userDriveServiceNotEnabledWarning(user, str(e))
sys.exit(GM.Globals[GM.SYSEXITRC])
HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>')
# gsheet <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> <SheetEntity>
def getGSheetData():
user = _getMain().getEmailAddress()
fileIdEntity = _getMain().getDriveFileEntity(queryShortcutsOK=False)
sheetEntity = _getMain().getSheetEntity(False)
user = _gam().getEmailAddress()
fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
sheetEntity = _gam().getSheetEntity(False)
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
user, drive = _getMain().buildGAPIServiceObject(API.DRIVE3, user)
user, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
else:
drive = _getMain().buildGAPIObject(API.DRIVE3)
drive = _gam().buildGAPIObject(API.DRIVE3)
if not drive:
sys.exit(GM.Globals[GM.SYSEXITRC])
_, _, jcount = _getMain()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
_, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
if jcount == 0:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
if jcount > 1:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
_, sheet = _getMain().buildGAPIServiceObject(API.SHEETS, user)
_, sheet = _gam().buildGAPIServiceObject(API.SHEETS, user)
else:
sheet = _getMain().buildGAPIObject(API.SHEETS)
sheet = _gam().buildGAPIObject(API.SHEETS)
if not sheet:
sys.exit(GM.Globals[GM.SYSEXITRC])
fileId = fileIdEntity['list'][0]
f = None
try:
result = _getMain().callGAPI(drive.files(), 'get',
result = _gam().callGAPI(drive.files(), 'get',
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
fileId=fileId, fields='name,mimeType', supportsAllDrives=True)
if result['mimeType'] != _getMain().MIMETYPE_GA_SPREADSHEET:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], _getMain().MIMETYPE_GA_SPREADSHEET))
spreadsheet = _getMain().callGAPI(sheet.spreadsheets(), 'get',
if result['mimeType'] != _gam().MIMETYPE_GA_SPREADSHEET:
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
Msg.INVALID_MIMETYPE.format(result['mimeType'], _gam().MIMETYPE_GA_SPREADSHEET))
spreadsheet = _gam().callGAPI(sheet.spreadsheets(), 'get',
throwReasons=GAPI.SHEETS_ACCESS_THROW_REASONS,
spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))')
sheetId = _getMain().getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
sheetId = _gam().getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
if sheetId is None:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
spreadsheetUrl = f'{re.sub("/edit.*$", "/export", spreadsheet["spreadsheetUrl"])}?format=csv&id={fileId}&gid={sheetId}'
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
if GC.Values[GC.DEBUG_LEVEL] > 0:
sys.stderr.write(f'Debug: spreadsheetUrl: {spreadsheetUrl}\n')
triesLimit = 3
@@ -144,25 +143,25 @@ def getGSheetData():
break
tg = HTML_TITLE_PATTERN.match(content[0:600].decode('utf-8'))
errMsg = tg.group(1) if tg else 'Unknown error'
_getMain().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
_gam().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
time.sleep(20)
else:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
f.write(content.decode(_getMain().UTF8_SIG))
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
f.write(content.decode(_gam().UTF8_SIG))
f.seek(0)
return f
except GAPI.fileNotFound:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
except (GAPI.notFound, GAPI.forbidden, GAPI.permissionDenied,
GAPI.internalError, GAPI.insufficientFilePermissions, GAPI.badRequest,
GAPI.invalid, GAPI.invalidArgument, GAPI.failedPrecondition) as e:
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
except (IOError, httplib2.HttpLib2Error) as e:
if f:
f.close()
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
_getMain().userDriveServiceNotEnabledWarning(user, str(e))
_gam().userDriveServiceNotEnabledWarning(user, str(e))
sys.exit(GM.Globals[GM.SYSEXITRC])
@@ -173,15 +172,15 @@ BUCKET_OBJECT_PATTERNS = [
]
def getBucketObjectName():
Cmd = _getMain().Cmd
uri = _getMain().getString(Cmd.OB_STRING)
Cmd = _gam().Cmd
uri = _gam().getString(Cmd.OB_STRING)
for pattern in BUCKET_OBJECT_PATTERNS:
mg = re.search(pattern['pattern'], uri)
if mg:
bucket = mg.group(1)
s_object = mg.group(2) if not pattern['unquote'] else unquote(mg.group(2))
return (bucket, s_object, f'{bucket}/{s_object}')
_getMain().systemErrorExit(_getMain().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
_gam().systemErrorExit(_gam().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
GCS_FORMAT_MIME_TYPES = {
'gcscsv': 'text/csv',
@@ -193,17 +192,17 @@ GCS_FORMAT_MIME_TYPES = {
def getStorageFileData(gcsformat, returnData=True):
mimeType = GCS_FORMAT_MIME_TYPES[gcsformat]
bucket, s_object, bucketObject = getBucketObjectName()
s = _getMain().buildGAPIObject(API.STORAGEREAD)
s = _gam().buildGAPIObject(API.STORAGEREAD)
try:
result = _getMain().callGAPI(s.objects(), 'get',
result = _gam().callGAPI(s.objects(), 'get',
throwReasons=[GAPI.NOT_FOUND, GAPI.FORBIDDEN],
bucket=bucket, object=s_object, projection='noAcl', fields='contentType')
except GAPI.notFound:
_getMain().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
_gam().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
except GAPI.forbidden as e:
_getMain().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
_gam().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
if result['contentType'] != mimeType:
_getMain().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
_gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
Msg.INVALID_MIMETYPE.format(result['contentType'], mimeType))
fb = TemporaryFile(mode='wb+')
try:
@@ -214,58 +213,58 @@ def getStorageFileData(gcsformat, returnData=True):
_, done = downloader.next_chunk()
fb.seek(0)
if returnData:
data = fb.read().decode(_getMain().UTF8)
data = fb.read().decode(_gam().UTF8)
fb.close()
return data
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
f.write(fb.read().decode(_getMain().UTF8_SIG))
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
f.write(fb.read().decode(_gam().UTF8_SIG))
fb.close()
f.seek(0)
return f
except googleapiclient.http.HttpError as e:
mg = _getMain().HTTP_ERROR_PATTERN.match(str(e))
_getMain().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
mg = _gam().HTTP_ERROR_PATTERN.match(str(e))
_gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
# <CSVFileInput>
def openCSVFileReader(filename, fieldnames=None):
Cmd = _getMain().Cmd
Cmd = _gam().Cmd
filenameLower = filename.lower()
if filenameLower == 'gsheet':
f = getGSheetData()
_getMain().getCharSet()
_gam().getCharSet()
elif filenameLower in {'gcsv', 'gdoc'}:
f = getGDocData(filenameLower)
_getMain().getCharSet()
_gam().getCharSet()
elif filenameLower in {'gcscsv', 'gcsdoc'}:
f = getStorageFileData(filenameLower, False)
_getMain().getCharSet()
_gam().getCharSet()
else:
encoding = _getMain().getCharSet()
filename = _getMain().setFilePath(filename, GC.INPUT_DIR)
f = _getMain().openFile(filename, mode=_getMain().DEFAULT_CSV_READ_MODE, encoding=encoding)
if _getMain().checkArgumentPresent('warnifnodata'):
encoding = _gam().getCharSet()
filename = _gam().setFilePath(filename, GC.INPUT_DIR)
f = _gam().openFile(filename, mode=_gam().DEFAULT_CSV_READ_MODE, encoding=encoding)
if _gam().checkArgumentPresent('warnifnodata'):
loc = f.tell()
try:
if not f.readline() or not f.readline():
_getMain().stderrWarningMsg(_getMain().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
sys.exit(_getMain().NO_ENTITIES_FOUND_RC)
_gam().stderrWarningMsg(_gam().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
sys.exit(_gam().NO_ENTITIES_FOUND_RC)
f.seek(loc)
except (IOError, UnicodeDecodeError, UnicodeError) as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage(filename, e))
if _getMain().checkArgumentPresent('columndelimiter'):
columnDelimiter = _getMain().getCharacter()
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
if _gam().checkArgumentPresent('columndelimiter'):
columnDelimiter = _gam().getCharacter()
else:
columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER]
if _getMain().checkArgumentPresent('noescapechar'):
noEscapeChar = _getMain().getBoolean()
if _gam().checkArgumentPresent('noescapechar'):
noEscapeChar = _gam().getBoolean()
else:
noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR]
if _getMain().checkArgumentPresent('quotechar'):
quotechar = _getMain().getCharacter()
if _gam().checkArgumentPresent('quotechar'):
quotechar = _gam().getCharacter()
else:
quotechar = GC.Values[GC.CSV_INPUT_QUOTE_CHAR]
if not _getMain().checkArgumentPresent('endcsv') and _getMain().checkArgumentPresent('fields'):
fieldnames = _getMain().shlexSplitList(_getMain().getString(Cmd.OB_FIELD_NAME_LIST))
if not _gam().checkArgumentPresent('endcsv') and _gam().checkArgumentPresent('fields'):
fieldnames = _gam().shlexSplitList(_gam().getString(Cmd.OB_FIELD_NAME_LIST))
try:
csvFile = csv.DictReader(f, fieldnames=fieldnames,
delimiter=columnDelimiter,
@@ -273,4 +272,4 @@ def openCSVFileReader(filename, fieldnames=None):
quotechar=quotechar)
return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else [])
except (csv.Error, UnicodeDecodeError, UnicodeError) as e:
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, e)
_gam().systemErrorExit(_gam().FILE_ERROR_RC, e)

View File

@@ -11,12 +11,11 @@ from gamlib import glgapi as GAPI
from gamlib import glglobals as GM
def _getMain():
return sys.modules['gam']
_gam = lambda: sys.modules['gam']
def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
Cmd = _getMain().Cmd
Cmd = _gam().Cmd
if Cmd.ArgumentsRemaining():
path = Cmd.Current().strip()
if path == 'root':
@@ -24,40 +23,40 @@ def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
if path:
if pathOnly and (path.startswith('id:') or path.startswith('uid:')) and cd is not None:
try:
result = _getMain().callGAPI(cd.orgunits(), 'get',
result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=path,
fields='orgUnitPath')
Cmd.Advance()
if absolutePath:
return _getMain().makeOrgUnitPathAbsolute(result['orgUnitPath'])
return _getMain().makeOrgUnitPathRelative(result['orgUnitPath'])
return _gam().makeOrgUnitPathAbsolute(result['orgUnitPath'])
return _gam().makeOrgUnitPathRelative(result['orgUnitPath'])
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError,
GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().checkEntityAFDNEorAccessErrorExit(cd, _getMain().Ent.ORGANIZATIONAL_UNIT, path)
_getMain().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
_gam().checkEntityAFDNEorAccessErrorExit(cd, _gam().Ent.ORGANIZATIONAL_UNIT, path)
_gam().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
Cmd.Advance()
if absolutePath:
return _getMain().makeOrgUnitPathAbsolute(path)
return _getMain().makeOrgUnitPathRelative(path)
_getMain().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
return _gam().makeOrgUnitPathAbsolute(path)
return _gam().makeOrgUnitPathRelative(path)
_gam().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
def getTopLevelOrgId(cd, parentOrgUnitPath):
Ent = _getMain().Ent
Ent = _gam().Ent
if parentOrgUnitPath != '/':
try:
result = _getMain().callGAPI(cd.orgunits(), 'get',
result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(parentOrgUnitPath)),
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentOrgUnitPath)),
fields='orgUnitId')
return result['orgUnitId']
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
return None
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
_gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
return None
try:
result = _getMain().callGAPI(cd.orgunits(), 'list',
result = _gam().callGAPI(cd.orgunits(), 'list',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='allIncludingParent',
fields='organizationUnits(orgUnitId,orgUnitPath)')
@@ -68,18 +67,18 @@ def getTopLevelOrgId(cd, parentOrgUnitPath):
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
return None
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
_gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
return None
def getOrgUnitId(cd=None, orgUnit=None):
Ent = _getMain().Ent
Ent = _gam().Ent
if cd is None:
cd = _getMain().buildGAPIObject(API.DIRECTORY)
cd = _gam().buildGAPIObject(API.DIRECTORY)
if orgUnit is None:
orgUnit = getOrgUnitItem()
try:
if orgUnit == '/':
result = _getMain().callGAPI(cd.orgunits(), 'list',
result = _gam().callGAPI(cd.orgunits(), 'list',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='children',
fields='organizationUnits(parentOrgUnitId,parentOrgUnitPath)')
@@ -89,35 +88,35 @@ def getOrgUnitId(cd=None, orgUnit=None):
if topLevelOrgId:
return (orgUnit, topLevelOrgId)
return (orgUnit, '/') #Bogus but should never happen
result = _getMain().callGAPI(cd.orgunits(), 'get',
result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(orgUnit)),
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(orgUnit)),
fields='orgUnitId,orgUnitPath')
return (result['orgUnitPath'], result['orgUnitId'])
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
_getMain().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
_gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().accessErrorExit(cd)
_gam().accessErrorExit(cd)
def getAllParentOrgUnitsForUser(cd, user):
Ent = _getMain().Ent
Ent = _gam().Ent
try:
result = _getMain().callGAPI(cd.users(), 'get',
result = _gam().callGAPI(cd.users(), 'get',
throwReasons=GAPI.USER_GET_THROW_REASONS,
userKey=user, fields='orgUnitPath', projection='basic')
except (GAPI.userNotFound, GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden):
_getMain().entityDoesNotExistExit(Ent.USER, user)
_gam().entityDoesNotExistExit(Ent.USER, user)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().accessErrorExit(cd)
_gam().accessErrorExit(cd)
parentPath = result['orgUnitPath']
if parentPath == '/':
orgUnitPath, orgUnitId = getOrgUnitId(cd, '/')
return {orgUnitId: orgUnitPath}
parentPath = _getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(parentPath))
parentPath = _gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentPath))
orgUnits = {}
while True:
try:
result = _getMain().callGAPI(cd.orgunits(), 'get',
result = _gam().callGAPI(cd.orgunits(), 'get',
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=parentPath,
fields='orgUnitId,orgUnitPath,parentOrgUnitId')
@@ -126,9 +125,9 @@ def getAllParentOrgUnitsForUser(cd, user):
break
parentPath = result['parentOrgUnitId']
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
_getMain().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
_gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
_getMain().accessErrorExit(cd)
_gam().accessErrorExit(cd)
return orgUnits
def _getOrgunitsOrgUnitIdPath(cd, orgUnit):