mirror of
https://github.com/GAM-team/GAM.git
synced 2026-07-05 21:31:35 +00:00
phase 2 code complete
Some checks failed
Build and test GAM / build (false, build, 1, Build Intel Ubuntu Jammy, ubuntu-22.04) (push) Has been cancelled
Build and test GAM / build (false, build, 10, Build x86_64 macOS 15, macos-15-intel) (push) Has been cancelled
Build and test GAM / build (false, build, 11, Build x86_64 macOS 26, macos-26-intel) (push) Has been cancelled
Build and test GAM / build (false, build, 12, Build Arm MacOS 26, macos-26) (push) Has been cancelled
Build and test GAM / build (false, build, 13, Build Intel Windows, windows-2025-vs2026) (push) Has been cancelled
Build and test GAM / build (false, build, 14, Build Arm Windows, windows-11-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 2, Build Intel Ubuntu Noble, ubuntu-24.04) (push) Has been cancelled
Build and test GAM / build (false, build, 3, Build Arm Ubuntu Noble, ubuntu-24.04-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 4, Build Arm Ubuntu Jammy, ubuntu-22.04-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 5, Build Intel StaticX Legacy, ubuntu-22.04, yes) (push) Has been cancelled
Build and test GAM / build (false, build, 6, Build Arm StaticX Legacy, ubuntu-22.04-arm, yes) (push) Has been cancelled
Build and test GAM / build (false, build, 8, Build Arm MacOS 14, macos-14) (push) Has been cancelled
Build and test GAM / build (false, build, 9, Build Arm MacOS 15, macos-15) (push) Has been cancelled
Build and test GAM / build (false, test, 16, Test Python 3.11, ubuntu-24.04, 3.11) (push) Has been cancelled
Build and test GAM / build (false, test, 17, Test Python 3.12, ubuntu-24.04, 3.12) (push) Has been cancelled
Build and test GAM / build (false, test, 18, Test Python 3.13, ubuntu-24.04, 3.13) (push) Has been cancelled
Build and test GAM / build (false, test, 19, Test Python 3.15-dev, ubuntu-24.04, 3.15-dev) (push) Has been cancelled
Build and test GAM / build (true, test, 20, Test Python 3.14 freethread, ubuntu-24.04, 3.14) (push) Has been cancelled
Build and test GAM / publish (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Some checks failed
Build and test GAM / build (false, build, 1, Build Intel Ubuntu Jammy, ubuntu-22.04) (push) Has been cancelled
Build and test GAM / build (false, build, 10, Build x86_64 macOS 15, macos-15-intel) (push) Has been cancelled
Build and test GAM / build (false, build, 11, Build x86_64 macOS 26, macos-26-intel) (push) Has been cancelled
Build and test GAM / build (false, build, 12, Build Arm MacOS 26, macos-26) (push) Has been cancelled
Build and test GAM / build (false, build, 13, Build Intel Windows, windows-2025-vs2026) (push) Has been cancelled
Build and test GAM / build (false, build, 14, Build Arm Windows, windows-11-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 2, Build Intel Ubuntu Noble, ubuntu-24.04) (push) Has been cancelled
Build and test GAM / build (false, build, 3, Build Arm Ubuntu Noble, ubuntu-24.04-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 4, Build Arm Ubuntu Jammy, ubuntu-22.04-arm) (push) Has been cancelled
Build and test GAM / build (false, build, 5, Build Intel StaticX Legacy, ubuntu-22.04, yes) (push) Has been cancelled
Build and test GAM / build (false, build, 6, Build Arm StaticX Legacy, ubuntu-22.04-arm, yes) (push) Has been cancelled
Build and test GAM / build (false, build, 8, Build Arm MacOS 14, macos-14) (push) Has been cancelled
Build and test GAM / build (false, build, 9, Build Arm MacOS 15, macos-15) (push) Has been cancelled
Build and test GAM / build (false, test, 16, Test Python 3.11, ubuntu-24.04, 3.11) (push) Has been cancelled
Build and test GAM / build (false, test, 17, Test Python 3.12, ubuntu-24.04, 3.12) (push) Has been cancelled
Build and test GAM / build (false, test, 18, Test Python 3.13, ubuntu-24.04, 3.13) (push) Has been cancelled
Build and test GAM / build (false, test, 19, Test Python 3.15-dev, ubuntu-24.04, 3.15-dev) (push) Has been cancelled
Build and test GAM / build (true, test, 20, Test Python 3.14 freethread, ubuntu-24.04, 3.14) (push) Has been cancelled
Build and test GAM / publish (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
@@ -3207,3 +3207,62 @@ def ProcessGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=True):
|
|||||||
# Process GAM command
|
# Process GAM command
|
||||||
def CallGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=False):
|
def CallGAMCommand(args, processGamCfg=True, inLoop=False, closeSTD=False):
|
||||||
return ProcessGAMCommand(args, processGamCfg=processGamCfg, inLoop=inLoop, closeSTD=closeSTD)
|
return ProcessGAMCommand(args, processGamCfg=processGamCfg, inLoop=inLoop, closeSTD=closeSTD)
|
||||||
|
|
||||||
|
# Re-export util names for _gam().X access by util/ modules.
|
||||||
|
# This block runs AFTER all modules are fully loaded (no circular import risk).
|
||||||
|
# Only names actually referenced via _gam() are included.
|
||||||
|
from gamlib import glskus as SKU # noqa: E402,F811
|
||||||
|
from tempfile import TemporaryFile # noqa: E402,F401
|
||||||
|
from util.access import ( # noqa: E402,F401
|
||||||
|
APIAccessDeniedExit, ClientAPIAccessDeniedExit, SvcAcctAPIAccessDeniedExit, accessErrorExit,
|
||||||
|
accessErrorExitNonDirectory, checkEntityAFDNEorAccessErrorExit,
|
||||||
|
checkEntityDNEorAccessErrorExit, entityUnknownWarning,
|
||||||
|
)
|
||||||
|
from util.api import ( # noqa: E402,F401,F811
|
||||||
|
_getAdminEmail, _getSvcAcctData, buildGAPIObject, buildGAPIServiceObject,
|
||||||
|
callGAPI, callGAPIitems, callGAPIpages, chooseSaAPI, yieldGAPIpages,
|
||||||
|
)
|
||||||
|
from util.args import ( # noqa: E402,F401,F811
|
||||||
|
ARCHIVED_ARGUMENTS, ISOformatTimeStamp, LOCALE_CODES_MAP, NAME_EMAIL_ADDRESS_PATTERN,
|
||||||
|
SUSPENDED_ARGUMENTS, YYYYMMDDTHHMMSSZ_FORMAT, YYYYMMDD_FORMAT, YYYYMMDD_PATTERN,
|
||||||
|
_getIsArchived, _getIsSuspended, checkDataField, checkForExtraneousArguments,
|
||||||
|
checkMatchSkipFields, checkSubkeyField, encodeOrgUnitPath, escapeCRsNLs,
|
||||||
|
formatLocalTime, formatLocalTimestamp, getBoolean, getCharSet, getCharacter,
|
||||||
|
getDelimiter, getEmailAddress, getEmailAddressDomain, getLanguageCode,
|
||||||
|
getMatchSkipFields, getPhraseDNEorSNA, getREPattern, getString,
|
||||||
|
getSheetEntity, getSheetIdFromSheetEntity, makeOrgUnitPathAbsolute,
|
||||||
|
makeOrgUnitPathRelative, orgUnitPathQuery, protectedSheetId, removeCourseIdScope,
|
||||||
|
shlexSplitList, splitEmailAddress, todaysTime, validateEmailAddressOrUID,
|
||||||
|
)
|
||||||
|
from util.csv_pf import CheckInputRowFilterHeaders # noqa: E402,F401
|
||||||
|
from util.display import ( # noqa: E402,F401
|
||||||
|
FIRST_ITEM_MARKER, LAST_ITEM_MARKER, TOTAL_ITEMS_MARKER, actionPerformedNumItems,
|
||||||
|
entityActionFailedWarning, entityActionNotPerformedWarning, entityActionPerformed,
|
||||||
|
entityActionPerformedMessage, entityDoesNotExistWarning, entityPerformActionNumItems,
|
||||||
|
entityServiceNotApplicableWarning, getPageMessage, getPageMessageForWhom, printBlankLine,
|
||||||
|
printGettingAllAccountEntities, printGettingAllEntityItemsForWhom,
|
||||||
|
printGotEntityItemsForWhom, printJSONKey, printJSONValue, printKeyValueList,
|
||||||
|
setGettingAllEntityItemsForWhom, userDriveServiceNotEnabledWarning,
|
||||||
|
userServiceNotEnabledWarning,
|
||||||
|
)
|
||||||
|
from util.email import send_email # noqa: E402,F401,F811
|
||||||
|
from util.entity import checkUserExists, convertUIDtoEmailAddress # noqa: E402,F401,F811
|
||||||
|
from util.errors import ( # noqa: E402,F401,F811
|
||||||
|
csvDataAlreadySavedErrorExit, csvFieldErrorExit, entityActionFailedExit,
|
||||||
|
entityDoesNotExistExit, expiredRevokedOauth2TxtExit, formatChoiceList,
|
||||||
|
invalidArgumentExit, invalidChoiceExit, invalidDiscoveryJsonExit,
|
||||||
|
invalidOauth2TxtExit, invalidOauth2serviceJsonExit, missingArgumentExit,
|
||||||
|
)
|
||||||
|
from util.fileio import ( # noqa: E402,F401,F811
|
||||||
|
StringIOobject, checkAPICallsRate, closeFile, fdErrorMessage, fileErrorMessage,
|
||||||
|
getGDocSheetDataFailedExit, getGDocSheetDataRetryWarning, incrAPICallsRetryData,
|
||||||
|
openFile, readFile, writeFile,
|
||||||
|
)
|
||||||
|
from util.gdoc import ( # noqa: E402,F401
|
||||||
|
getGDocData, getStorageFileData, openCSVFileReader,
|
||||||
|
)
|
||||||
|
from util.output import ( # noqa: E402,F401,F811
|
||||||
|
currentCountNL, flushStderr, flushStdout,
|
||||||
|
printWarningMessage, readStdin, stderrWarningMsg, writeStdout,
|
||||||
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,7 @@ from gamlib import glindent
|
|||||||
from gamlib import glmsgs as Msg
|
from gamlib import glmsgs as Msg
|
||||||
|
|
||||||
|
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
|
|
||||||
Act = glaction.GamAction()
|
Act = glaction.GamAction()
|
||||||
@@ -27,139 +26,125 @@ Ind = glindent.GamIndent()
|
|||||||
|
|
||||||
# Something's wrong with CustomerID??
|
# Something's wrong with CustomerID??
|
||||||
def accessErrorMessage(cd, errMsg=None):
|
def accessErrorMessage(cd, errMsg=None):
|
||||||
_m = _getMain()
|
|
||||||
if cd is None:
|
if cd is None:
|
||||||
cd = _m.buildGAPIObject(API.DIRECTORY)
|
cd = _gam().buildGAPIObject(API.DIRECTORY)
|
||||||
try:
|
try:
|
||||||
_m.callGAPI(cd.customers(), 'get',
|
_gam().callGAPI(cd.customers(), 'get',
|
||||||
throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND,
|
throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND,
|
||||||
GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
|
GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
|
||||||
customerKey=GC.Values[GC.CUSTOMER_ID], fields='id')
|
customerKey=GC.Values[GC.CUSTOMER_ID], fields='id')
|
||||||
except (GAPI.badRequest, GAPI.invalidInput):
|
except (GAPI.badRequest, GAPI.invalidInput):
|
||||||
return _m.formatKeyValueList('',
|
return _gam().formatKeyValueList('',
|
||||||
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
|
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
|
||||||
Msg.INVALID],
|
Msg.INVALID],
|
||||||
'')
|
'')
|
||||||
except GAPI.resourceNotFound:
|
except GAPI.resourceNotFound:
|
||||||
return _m.formatKeyValueList('',
|
return _gam().formatKeyValueList('',
|
||||||
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
|
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
|
||||||
Msg.DOES_NOT_EXIST],
|
Msg.DOES_NOT_EXIST],
|
||||||
'')
|
'')
|
||||||
except (GAPI.forbidden, GAPI.permissionDenied):
|
except (GAPI.forbidden, GAPI.permissionDenied):
|
||||||
return _m.formatKeyValueList('',
|
return _gam().formatKeyValueList('',
|
||||||
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
|
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
|
||||||
Ent.DOMAIN, GC.Values[GC.DOMAIN],
|
Ent.DOMAIN, GC.Values[GC.DOMAIN],
|
||||||
Ent.USER, GM.Globals[GM.ADMIN]])+[Msg.ACCESS_FORBIDDEN],
|
Ent.USER, GM.Globals[GM.ADMIN]])+[Msg.ACCESS_FORBIDDEN],
|
||||||
'')
|
'')
|
||||||
if errMsg:
|
if errMsg:
|
||||||
return _m.formatKeyValueList('',
|
return _gam().formatKeyValueList('',
|
||||||
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
|
[Ent.Singular(Ent.CUSTOMER_ID), GC.Values[GC.CUSTOMER_ID],
|
||||||
errMsg],
|
errMsg],
|
||||||
'')
|
'')
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def accessErrorExit(cd, errMsg=None):
|
def accessErrorExit(cd, errMsg=None):
|
||||||
_m = _getMain()
|
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, accessErrorMessage(cd or _gam().buildGAPIObject(API.DIRECTORY), errMsg))
|
||||||
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, accessErrorMessage(cd or _m.buildGAPIObject(API.DIRECTORY), errMsg))
|
|
||||||
|
|
||||||
def accessErrorExitNonDirectory(api, errMsg):
|
def accessErrorExitNonDirectory(api, errMsg):
|
||||||
_m = _getMain()
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC,
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC,
|
_gam().formatKeyValueList('',
|
||||||
_m.formatKeyValueList('',
|
|
||||||
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
|
Ent.FormatEntityValueList([Ent.CUSTOMER_ID, GC.Values[GC.CUSTOMER_ID],
|
||||||
Ent.DOMAIN, GC.Values[GC.DOMAIN],
|
Ent.DOMAIN, GC.Values[GC.DOMAIN],
|
||||||
Ent.API, api])+[errMsg],
|
Ent.API, api])+[errMsg],
|
||||||
''))
|
''))
|
||||||
|
|
||||||
def ClientAPIAccessDeniedExit(errMsg=None):
|
def ClientAPIAccessDeniedExit(errMsg=None):
|
||||||
_m = _getMain()
|
|
||||||
if errMsg is None:
|
if errMsg is None:
|
||||||
_m.stderrErrorMsg(Msg.API_ACCESS_DENIED)
|
_gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
|
||||||
missingScopes = API.getClientScopesSet(GM.Globals[GM.CURRENT_CLIENT_API])-GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
|
missingScopes = API.getClientScopesSet(GM.Globals[GM.CURRENT_CLIENT_API])-GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
|
||||||
if missingScopes:
|
if missingScopes:
|
||||||
_m.writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
|
_gam().writeStderr(Msg.API_CHECK_CLIENT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2_CLIENT_ID],
|
||||||
','.join(sorted(missingScopes))))
|
','.join(sorted(missingScopes))))
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None)
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
|
||||||
else:
|
else:
|
||||||
_m.stderrErrorMsg(errMsg)
|
_gam().stderrErrorMsg(errMsg)
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.REAUTHENTICATION_IS_NEEDED)
|
||||||
|
|
||||||
def SvcAcctAPIAccessDenied():
|
def SvcAcctAPIAccessDenied():
|
||||||
_m = _getMain()
|
_gam()._getSvcAcctData()
|
||||||
_m._getSvcAcctData()
|
|
||||||
if (GM.Globals[GM.CURRENT_SVCACCT_API] == API.GMAIL and
|
if (GM.Globals[GM.CURRENT_SVCACCT_API] == API.GMAIL and
|
||||||
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] and
|
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] and
|
||||||
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES][0] == API.GMAIL_SEND_SCOPE):
|
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES][0] == API.GMAIL_SEND_SCOPE):
|
||||||
_m.systemErrorExit(_m.OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
|
_gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
|
||||||
_m.stderrErrorMsg(Msg.API_ACCESS_DENIED)
|
_gam().stderrErrorMsg(Msg.API_ACCESS_DENIED)
|
||||||
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
|
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
|
||||||
_m.writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
|
_gam().writeStderr(Msg.API_CHECK_SVCACCT_AUTHORIZATION.format(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['client_id'],
|
||||||
apiOrScopes,
|
apiOrScopes,
|
||||||
GM.Globals[GM.CURRENT_SVCACCT_USER] or _m._getAdminEmail()))
|
GM.Globals[GM.CURRENT_SVCACCT_USER] or _gam()._getAdminEmail()))
|
||||||
def SvcAcctAPIAccessDeniedExit():
|
def SvcAcctAPIAccessDeniedExit():
|
||||||
_m = _getMain()
|
|
||||||
SvcAcctAPIAccessDenied()
|
SvcAcctAPIAccessDenied()
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None)
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
|
||||||
|
|
||||||
def SvcAcctAPIDisabledExit():
|
def SvcAcctAPIDisabledExit():
|
||||||
_m = _getMain()
|
|
||||||
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
|
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
|
||||||
ClientAPIAccessDeniedExit()
|
ClientAPIAccessDeniedExit()
|
||||||
if GM.Globals[GM.CURRENT_SVCACCT_API]:
|
if GM.Globals[GM.CURRENT_SVCACCT_API]:
|
||||||
_m.stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
|
_gam().stderrErrorMsg(Msg.SERVICE_ACCOUNT_API_DISABLED.format(API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API])))
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, None)
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, None)
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
|
||||||
|
|
||||||
def APIAccessDeniedExit():
|
def APIAccessDeniedExit():
|
||||||
_m = _getMain()
|
|
||||||
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
|
if not GM.Globals[GM.CURRENT_SVCACCT_USER] and GM.Globals[GM.CURRENT_CLIENT_API]:
|
||||||
ClientAPIAccessDeniedExit()
|
ClientAPIAccessDeniedExit()
|
||||||
if GM.Globals[GM.CURRENT_SVCACCT_API]:
|
if GM.Globals[GM.CURRENT_SVCACCT_API]:
|
||||||
SvcAcctAPIAccessDeniedExit()
|
SvcAcctAPIAccessDeniedExit()
|
||||||
_m.systemErrorExit(_m.API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, Msg.API_ACCESS_DENIED)
|
||||||
|
|
||||||
def checkEntityDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
|
def checkEntityDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
|
||||||
_m = _getMain()
|
|
||||||
message = accessErrorMessage(cd)
|
message = accessErrorMessage(cd)
|
||||||
if message:
|
if message:
|
||||||
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message)
|
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
|
||||||
_m.entityDoesNotExistWarning(entityType, entityName, i, count)
|
_gam().entityDoesNotExistWarning(entityType, entityName, i, count)
|
||||||
|
|
||||||
def checkEntityAFDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
|
def checkEntityAFDNEorAccessErrorExit(cd, entityType, entityName, i=0, count=0):
|
||||||
_m = _getMain()
|
|
||||||
message = accessErrorMessage(cd)
|
message = accessErrorMessage(cd)
|
||||||
if message:
|
if message:
|
||||||
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message)
|
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
|
||||||
_m.entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
|
_gam().entityActionFailedWarning([entityType, entityName], Msg.DOES_NOT_EXIST, i, count)
|
||||||
|
|
||||||
def checkEntityItemValueAFDNEorAccessErrorExit(cd, entityType, entityName, itemType, itemValue, i=0, count=0):
|
def checkEntityItemValueAFDNEorAccessErrorExit(cd, entityType, entityName, itemType, itemValue, i=0, count=0):
|
||||||
_m = _getMain()
|
|
||||||
message = accessErrorMessage(cd)
|
message = accessErrorMessage(cd)
|
||||||
if message:
|
if message:
|
||||||
_m.systemErrorExit(_m.INVALID_DOMAIN_RC, message)
|
_gam().systemErrorExit(_gam().INVALID_DOMAIN_RC, message)
|
||||||
_m.entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
|
_gam().entityActionFailedWarning([entityType, entityName, itemType, itemValue], Msg.DOES_NOT_EXIST, i, count)
|
||||||
|
|
||||||
def entityUnknownWarning(entityType, entityName, i=0, count=0):
|
def entityUnknownWarning(entityType, entityName, i=0, count=0):
|
||||||
_m = _getMain()
|
domain = _gam().getEmailAddressDomain(entityName)
|
||||||
domain = _m.getEmailAddressDomain(entityName)
|
|
||||||
if (domain.endswith(GC.Values[GC.DOMAIN])) or (domain.endswith('google.com')):
|
if (domain.endswith(GC.Values[GC.DOMAIN])) or (domain.endswith('google.com')):
|
||||||
_m.entityDoesNotExistWarning(entityType, entityName, i, count)
|
_gam().entityDoesNotExistWarning(entityType, entityName, i, count)
|
||||||
else:
|
else:
|
||||||
_m.entityServiceNotApplicableWarning(entityType, entityName, i, count)
|
_gam().entityServiceNotApplicableWarning(entityType, entityName, i, count)
|
||||||
|
|
||||||
def entityOrEntityUnknownWarning(entity1Type, entity1Name, entity2Type, entity2Name, i=0, count=0):
|
def entityOrEntityUnknownWarning(entity1Type, entity1Name, entity2Type, entity2Name, i=0, count=0):
|
||||||
_m = _getMain()
|
_gam().setSysExitRC(_gam().ENTITY_DOES_NOT_EXIST_RC)
|
||||||
_m.setSysExitRC(_m.ENTITY_DOES_NOT_EXIST_RC)
|
_gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
|
||||||
_m.writeStderr(_m.formatKeyValueList(Ind.Spaces(),
|
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _gam().getPhraseDNEorSNA(entity1Name), None,
|
||||||
[f'{Msg.EITHER} {Ent.Singular(entity1Type)}', entity1Name, _m.getPhraseDNEorSNA(entity1Name), None,
|
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _gam().getPhraseDNEorSNA(entity2Name)],
|
||||||
f'{Msg.OR} {Ent.Singular(entity2Type)}', entity2Name, _m.getPhraseDNEorSNA(entity2Name)],
|
_gam().currentCountNL(i, count)))
|
||||||
_m.currentCountNL(i, count)))
|
|
||||||
|
|
||||||
def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
|
def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
|
||||||
_m = _getMain()
|
|
||||||
email = entityValueList[1]
|
email = entityValueList[1]
|
||||||
try:
|
try:
|
||||||
result = _m.callGAPI(cd.users(), 'get',
|
result = _gam().callGAPI(cd.users(), 'get',
|
||||||
throwReasons=GAPI.USER_GET_THROW_REASONS,
|
throwReasons=GAPI.USER_GET_THROW_REASONS,
|
||||||
userKey=email, fields='id,primaryEmail')
|
userKey=email, fields='id,primaryEmail')
|
||||||
if (result['primaryEmail'].lower() == email) or (result['id'] == email):
|
if (result['primaryEmail'].lower() == email) or (result['id'] == email):
|
||||||
@@ -169,7 +154,7 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
|
|||||||
except (GAPI.userNotFound, GAPI.badRequest,
|
except (GAPI.userNotFound, GAPI.badRequest,
|
||||||
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.backendError, GAPI.systemError):
|
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.backendError, GAPI.systemError):
|
||||||
try:
|
try:
|
||||||
result = _m.callGAPI(cd.groups(), 'get',
|
result = _gam().callGAPI(cd.groups(), 'get',
|
||||||
throwReasons=GAPI.GROUP_GET_THROW_REASONS,
|
throwReasons=GAPI.GROUP_GET_THROW_REASONS,
|
||||||
groupKey=email, fields='id,email')
|
groupKey=email, fields='id,email')
|
||||||
if (result['email'].lower() == email) or (result['id'] == email):
|
if (result['email'].lower() == email) or (result['id'] == email):
|
||||||
@@ -179,10 +164,10 @@ def duplicateAliasGroupUserWarning(cd, entityValueList, i=0, count=0):
|
|||||||
except (GAPI.groupNotFound,
|
except (GAPI.groupNotFound,
|
||||||
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.badRequest):
|
GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden, GAPI.badRequest):
|
||||||
kvList = [Ent.EMAIL, email]
|
kvList = [Ent.EMAIL, email]
|
||||||
_m.writeStderr(_m.formatKeyValueList(Ind.Spaces(),
|
_gam().writeStderr(_gam().formatKeyValueList(Ind.Spaces(),
|
||||||
Ent.FormatEntityValueList(entityValueList)+
|
Ent.FormatEntityValueList(entityValueList)+
|
||||||
[Act.Failed(), Msg.DUPLICATE]+
|
[Act.Failed(), Msg.DUPLICATE]+
|
||||||
Ent.FormatEntityValueList(kvList),
|
Ent.FormatEntityValueList(kvList),
|
||||||
_m.currentCountNL(i, count)))
|
_gam().currentCountNL(i, count)))
|
||||||
_m.setSysExitRC(_m.ENTITY_DUPLICATE_RC)
|
_gam().setSysExitRC(_gam().ENTITY_DUPLICATE_RC)
|
||||||
return kvList[0]
|
return kvList[0]
|
||||||
|
|||||||
@@ -56,8 +56,7 @@ DEVELOPER_PREVIEW_DISCOVERY_URI = "https://{api}.googleapis.com/$discovery/rest?
|
|||||||
_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds
|
_DEFAULT_TOKEN_LIFETIME_SECS = 3600 # 1 hour in seconds
|
||||||
|
|
||||||
|
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
def _getEnt():
|
def _getEnt():
|
||||||
return sys.modules['gam'].Ent
|
return sys.modules['gam'].Ent
|
||||||
@@ -67,13 +66,12 @@ def _getInd():
|
|||||||
|
|
||||||
|
|
||||||
def handleServerError(e):
|
def handleServerError(e):
|
||||||
m = _getMain()
|
|
||||||
errMsg = str(e)
|
errMsg = str(e)
|
||||||
if 'setting tls' not in errMsg:
|
if 'setting tls' not in errMsg:
|
||||||
m.systemErrorExit(m.NETWORK_ERROR_RC, errMsg)
|
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, errMsg)
|
||||||
m.stderrErrorMsg(errMsg)
|
_gam().stderrErrorMsg(errMsg)
|
||||||
m.writeStderr(Msg.DISABLE_TLS_MIN_MAX)
|
_gam().writeStderr(Msg.DISABLE_TLS_MIN_MAX)
|
||||||
m.systemErrorExit(m.NETWORK_ERROR_RC, None)
|
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, None)
|
||||||
|
|
||||||
def getHttpObj(cache=None, timeout=None, override_min_tls=None, override_max_tls=None):
|
def getHttpObj(cache=None, timeout=None, override_min_tls=None, override_max_tls=None):
|
||||||
tls_minimum_version = override_min_tls if override_min_tls else GC.Values[GC.TLS_MIN_VERSION] if GC.Values[GC.TLS_MIN_VERSION] else None
|
tls_minimum_version = override_min_tls if override_min_tls else GC.Values[GC.TLS_MIN_VERSION] if GC.Values[GC.TLS_MIN_VERSION] else None
|
||||||
@@ -113,7 +111,7 @@ def _force_user_agent(user_agent):
|
|||||||
def _lazy_force_user_agent(request_method):
|
def _lazy_force_user_agent(request_method):
|
||||||
"""Wraps a request method to lazily insert GAM_USER_AGENT at call time."""
|
"""Wraps a request method to lazily insert GAM_USER_AGENT at call time."""
|
||||||
def wrapped_request_method(*args, **kwargs):
|
def wrapped_request_method(*args, **kwargs):
|
||||||
user_agent = _getMain().GAM_USER_AGENT
|
user_agent = _gam().GAM_USER_AGENT
|
||||||
if kwargs.get('headers') is not None:
|
if kwargs.get('headers') is not None:
|
||||||
if kwargs['headers'].get('user-agent'):
|
if kwargs['headers'].get('user-agent'):
|
||||||
if user_agent not in kwargs['headers']['user-agent']:
|
if user_agent not in kwargs['headers']['user-agent']:
|
||||||
@@ -157,14 +155,13 @@ def transportCreateRequest(httpObj=None):
|
|||||||
return transportAgentRequest(httpObj)
|
return transportAgentRequest(httpObj)
|
||||||
|
|
||||||
def doGAMCheckForUpdates(forceCheck):
|
def doGAMCheckForUpdates(forceCheck):
|
||||||
m = _getMain()
|
|
||||||
Ind = _getInd()
|
Ind = _getInd()
|
||||||
def _gamLatestVersionNotAvailable():
|
def _gamLatestVersionNotAvailable():
|
||||||
if forceCheck:
|
if forceCheck:
|
||||||
m.systemErrorExit(m.NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
|
_gam().systemErrorExit(_gam().NETWORK_ERROR_RC, Msg.GAM_LATEST_VERSION_NOT_AVAILABLE)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
_, c = getHttpObj(timeout=10).request(m.GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
|
_, c = getHttpObj(timeout=10).request(_gam().GAM_LATEST_RELEASE, 'GET', headers={'Accept': 'application/vnd.github.v3.text+json'})
|
||||||
try:
|
try:
|
||||||
release_data = json.loads(c)
|
release_data = json.loads(c)
|
||||||
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
|
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
|
||||||
@@ -173,17 +170,17 @@ def doGAMCheckForUpdates(forceCheck):
|
|||||||
if not isinstance(release_data, dict) or 'tag_name' not in release_data:
|
if not isinstance(release_data, dict) or 'tag_name' not in release_data:
|
||||||
_gamLatestVersionNotAvailable()
|
_gamLatestVersionNotAvailable()
|
||||||
return
|
return
|
||||||
current_version = m.__version__
|
current_version = _gam().__version__
|
||||||
latest_version = release_data['tag_name']
|
latest_version = release_data['tag_name']
|
||||||
if latest_version[0].lower() == 'v':
|
if latest_version[0].lower() == 'v':
|
||||||
latest_version = latest_version[1:]
|
latest_version = latest_version[1:]
|
||||||
m.printKeyValueList(['Version Check', None])
|
_gam().printKeyValueList(['Version Check', None])
|
||||||
Ind.Increment()
|
Ind.Increment()
|
||||||
m.printKeyValueList(['Current', current_version])
|
_gam().printKeyValueList(['Current', current_version])
|
||||||
m.printKeyValueList([' Latest', latest_version])
|
_gam().printKeyValueList([' Latest', latest_version])
|
||||||
Ind.Decrement()
|
Ind.Decrement()
|
||||||
if forceCheck < 0:
|
if forceCheck < 0:
|
||||||
m.setSysExitRC(1 if latest_version > current_version else 0)
|
_gam().setSysExitRC(1 if latest_version > current_version else 0)
|
||||||
return
|
return
|
||||||
except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError,
|
except (httplib2.HttpLib2Error, httplib2.ServerNotFoundError,
|
||||||
google.auth.exceptions.TransportError,
|
google.auth.exceptions.TransportError,
|
||||||
@@ -250,13 +247,12 @@ class signjwtSignJwt(google.auth.crypt.Signer):
|
|||||||
|
|
||||||
def sign(self, message):
|
def sign(self, message):
|
||||||
''' Call IAM Credentials SignJWT API to get our signed JWT '''
|
''' Call IAM Credentials SignJWT API to get our signed JWT '''
|
||||||
m = _getMain()
|
|
||||||
request = get_adc_request()
|
request = get_adc_request()
|
||||||
try:
|
try:
|
||||||
credentials, _ = google.auth.default(scopes=[API.IAM_SCOPE],
|
credentials, _ = google.auth.default(scopes=[API.IAM_SCOPE],
|
||||||
request=request)
|
request=request)
|
||||||
except (google.auth.exceptions.DefaultCredentialsError, google.auth.exceptions.RefreshError) as e:
|
except (google.auth.exceptions.DefaultCredentialsError, google.auth.exceptions.RefreshError) as e:
|
||||||
m.systemErrorExit(m.API_ACCESS_DENIED_RC, str(e))
|
_gam().systemErrorExit(_gam().API_ACCESS_DENIED_RC, str(e))
|
||||||
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj())
|
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj())
|
||||||
# refresh here so we can use the proper request from above
|
# refresh here so we can use the proper request from above
|
||||||
httpObj.credentials.refresh(request)
|
httpObj.credentials.refresh(request)
|
||||||
@@ -267,42 +263,40 @@ class signjwtSignJwt(google.auth.crypt.Signer):
|
|||||||
return signed_jwt
|
return signed_jwt
|
||||||
|
|
||||||
def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0):
|
def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0):
|
||||||
m = _getMain()
|
|
||||||
Ent = _getEnt()
|
Ent = _getEnt()
|
||||||
errMsg = str(e).replace('.', '')
|
errMsg = str(e).replace('.', '')
|
||||||
if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or
|
if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or
|
||||||
errMsg.startswith('Invalid response') or
|
errMsg.startswith('Invalid response') or
|
||||||
errMsg.startswith('invalid_request: Invalid impersonation "sub" field')):
|
errMsg.startswith('invalid_request: Invalid impersonation "sub" field')):
|
||||||
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
|
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
|
||||||
m.ClientAPIAccessDeniedExit()
|
_gam().ClientAPIAccessDeniedExit()
|
||||||
# 403 Forbidden, API disabled, user not enabled
|
# 403 Forbidden, API disabled, user not enabled
|
||||||
# 400 Bad Request, user not defined
|
# 400 Bad Request, user not defined
|
||||||
if softErrors:
|
if softErrors:
|
||||||
m.entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
|
_gam().entityActionFailedWarning([Ent.USER, GM.Globals[GM.CURRENT_SVCACCT_USER], Ent.USER, None], errMsg, i, count)
|
||||||
return None
|
return None
|
||||||
m.systemErrorExit(m.SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
|
_gam().systemErrorExit(_gam().SERVICE_NOT_APPLICABLE_RC, Msg.SERVICE_NOT_APPLICABLE_THIS_ADDRESS.format(GM.Globals[GM.CURRENT_SVCACCT_USER]))
|
||||||
if errMsg in API.OAUTH2_UNAUTHORIZED_ERRORS:
|
if errMsg in API.OAUTH2_UNAUTHORIZED_ERRORS:
|
||||||
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
|
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
|
||||||
m.ClientAPIAccessDeniedExit()
|
_gam().ClientAPIAccessDeniedExit()
|
||||||
# 401 Unauthorized, API disabled, user enabled
|
# 401 Unauthorized, API disabled, user enabled
|
||||||
if softErrors:
|
if softErrors:
|
||||||
if displayError:
|
if displayError:
|
||||||
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
|
apiOrScopes = API.getAPIName(GM.Globals[GM.CURRENT_SVCACCT_API]) if GM.Globals[GM.CURRENT_SVCACCT_API] else ','.join(sorted(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]))
|
||||||
m.userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
|
_gam().userServiceNotEnabledWarning(GM.Globals[GM.CURRENT_SVCACCT_USER], apiOrScopes, i, count)
|
||||||
return None
|
return None
|
||||||
m.SvcAcctAPIAccessDeniedExit()
|
_gam().SvcAcctAPIAccessDeniedExit()
|
||||||
if errMsg in API.REFRESH_PERM_ERRORS:
|
if errMsg in API.REFRESH_PERM_ERRORS:
|
||||||
if softErrors:
|
if softErrors:
|
||||||
return None
|
return None
|
||||||
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
|
if not GM.Globals[GM.CURRENT_SVCACCT_USER]:
|
||||||
m.expiredRevokedOauth2TxtExit()
|
_gam().expiredRevokedOauth2TxtExit()
|
||||||
m.stderrErrorMsg(f'Authentication Token Error - {errMsg}')
|
_gam().stderrErrorMsg(f'Authentication Token Error - {errMsg}')
|
||||||
m.APIAccessDeniedExit()
|
_gam().APIAccessDeniedExit()
|
||||||
|
|
||||||
def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnly=False, noScopes=False):
|
def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnly=False, noScopes=False):
|
||||||
m = _getMain()
|
|
||||||
if not noDASA and GC.Values[GC.ENABLE_DASA]:
|
if not noDASA and GC.Values[GC.ENABLE_DASA]:
|
||||||
jsonData = m.readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
|
jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=False)
|
||||||
if jsonData:
|
if jsonData:
|
||||||
try:
|
try:
|
||||||
if api in API.APIS_NEEDING_ACCESS_TOKEN:
|
if api in API.APIS_NEEDING_ACCESS_TOKEN:
|
||||||
@@ -320,9 +314,9 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
|
|||||||
sjsigner = signjwtSignJwt(jsonDict)
|
sjsigner = signjwtSignJwt(jsonDict)
|
||||||
return (True, signjwtJWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience))
|
return (True, signjwtJWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience))
|
||||||
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
||||||
m.invalidOauth2serviceJsonExit(str(e))
|
_gam().invalidOauth2serviceJsonExit(str(e))
|
||||||
m.invalidOauth2serviceJsonExit(Msg.NO_DATA)
|
_gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
|
||||||
jsonData = m.readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
|
jsonData = _gam().readFile(GC.Values[GC.OAUTH2_TXT], continueOnError=True, displayError=False)
|
||||||
if jsonData:
|
if jsonData:
|
||||||
try:
|
try:
|
||||||
jsonDict = json.loads(jsonData)
|
jsonDict = json.loads(jsonData)
|
||||||
@@ -332,11 +326,11 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
|
|||||||
if not refreshOnly:
|
if not refreshOnly:
|
||||||
if set(jsonDict.get('scopes', API.REQUIRED_SCOPES)) == API.REQUIRED_SCOPES_SET:
|
if set(jsonDict.get('scopes', API.REQUIRED_SCOPES)) == API.REQUIRED_SCOPES_SET:
|
||||||
if exitOnError:
|
if exitOnError:
|
||||||
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
|
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
|
||||||
return (False, None)
|
return (False, None)
|
||||||
else:
|
else:
|
||||||
GM.Globals[GM.CREDENTIALS_SCOPES] = set(jsonDict.pop('scopes', API.REQUIRED_SCOPES))
|
GM.Globals[GM.CREDENTIALS_SCOPES] = set(jsonDict.pop('scopes', API.REQUIRED_SCOPES))
|
||||||
token_expiry = jsonDict.get('token_expiry', m.REFRESH_EXPIRY)
|
token_expiry = jsonDict.get('token_expiry', _gam().REFRESH_EXPIRY)
|
||||||
if GC.Values[GC.TRUNCATE_CLIENT_ID]:
|
if GC.Values[GC.TRUNCATE_CLIENT_ID]:
|
||||||
# chop off .apps.googleusercontent.com suffix as it's not needed and we need to keep things short for the Auth URL.
|
# chop off .apps.googleusercontent.com suffix as it's not needed and we need to keep things short for the Auth URL.
|
||||||
jsonDict['client_id'] = re.sub(r'\.apps\.googleusercontent\.com$', '', jsonDict['client_id'])
|
jsonDict['client_id'] = re.sub(r'\.apps\.googleusercontent\.com$', '', jsonDict['client_id'])
|
||||||
@@ -349,19 +343,18 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl
|
|||||||
creds.token = jsonDict['access_token']
|
creds.token = jsonDict['access_token']
|
||||||
creds._id_token = jsonDict['id_token_jwt']
|
creds._id_token = jsonDict['id_token_jwt']
|
||||||
GM.Globals[GM.DECODED_ID_TOKEN] = jsonDict['id_token']
|
GM.Globals[GM.DECODED_ID_TOKEN] = jsonDict['id_token']
|
||||||
creds.expiry = arrow.Arrow.strptime(token_expiry, m.YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
|
creds.expiry = arrow.Arrow.strptime(token_expiry, _gam().YYYYMMDDTHHMMSSZ_FORMAT, tzinfo='UTC').naive
|
||||||
return (not noScopes, creds)
|
return (not noScopes, creds)
|
||||||
if jsonDict and exitOnError:
|
if jsonDict and exitOnError:
|
||||||
m.invalidOauth2TxtExit(Msg.INVALID)
|
_gam().invalidOauth2TxtExit(Msg.INVALID)
|
||||||
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
||||||
if exitOnError:
|
if exitOnError:
|
||||||
m.invalidOauth2TxtExit(str(e))
|
_gam().invalidOauth2TxtExit(str(e))
|
||||||
if exitOnError:
|
if exitOnError:
|
||||||
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
|
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, Msg.NO_CLIENT_ACCESS_ALLOWED)
|
||||||
return (False, None)
|
return (False, None)
|
||||||
|
|
||||||
def _getValueFromOAuth(field, credentials=None):
|
def _getValueFromOAuth(field, credentials=None):
|
||||||
m = _getMain()
|
|
||||||
if not GM.Globals[GM.DECODED_ID_TOKEN]:
|
if not GM.Globals[GM.DECODED_ID_TOKEN]:
|
||||||
request = transportCreateRequest()
|
request = transportCreateRequest()
|
||||||
if credentials is None:
|
if credentials is None:
|
||||||
@@ -373,9 +366,9 @@ def _getValueFromOAuth(field, credentials=None):
|
|||||||
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
|
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
if 'Token used too early' in str(e):
|
if 'Token used too early' in str(e):
|
||||||
m.stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
|
_gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
|
||||||
m.systemErrorExit(m.SYSTEM_ERROR_RC, str(e))
|
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
|
||||||
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, m.UNKNOWN)
|
return GM.Globals[GM.DECODED_ID_TOKEN].get(field, _gam().UNKNOWN)
|
||||||
|
|
||||||
def _getAdminEmail():
|
def _getAdminEmail():
|
||||||
if GC.Values[GC.ADMIN_EMAIL]:
|
if GC.Values[GC.ADMIN_EMAIL]:
|
||||||
@@ -383,7 +376,6 @@ def _getAdminEmail():
|
|||||||
return _getValueFromOAuth('email')
|
return _getValueFromOAuth('email')
|
||||||
|
|
||||||
def writeClientCredentials(creds, filename):
|
def writeClientCredentials(creds, filename):
|
||||||
m = _getMain()
|
|
||||||
creds_data = {
|
creds_data = {
|
||||||
'client_id': creds.client_id,
|
'client_id': creds.client_id,
|
||||||
'client_secret': creds.client_secret,
|
'client_secret': creds.client_secret,
|
||||||
@@ -391,28 +383,27 @@ def writeClientCredentials(creds, filename):
|
|||||||
'refresh_token': creds.refresh_token,
|
'refresh_token': creds.refresh_token,
|
||||||
'scopes': sorted(creds.scopes or GM.Globals[GM.CREDENTIALS_SCOPES]),
|
'scopes': sorted(creds.scopes or GM.Globals[GM.CREDENTIALS_SCOPES]),
|
||||||
'token': creds.token,
|
'token': creds.token,
|
||||||
'token_expiry': creds.expiry.strftime(m.YYYYMMDDTHHMMSSZ_FORMAT),
|
'token_expiry': creds.expiry.strftime(_gam().YYYYMMDDTHHMMSSZ_FORMAT),
|
||||||
'token_uri': creds.token_uri,
|
'token_uri': creds.token_uri,
|
||||||
}
|
}
|
||||||
expected_iss = ['https://accounts.google.com', 'accounts.google.com']
|
expected_iss = ['https://accounts.google.com', 'accounts.google.com']
|
||||||
if _getValueFromOAuth('iss', creds) not in expected_iss:
|
if _getValueFromOAuth('iss', creds) not in expected_iss:
|
||||||
m.systemErrorExit(m.OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
|
_gam().systemErrorExit(_gam().OAUTH2_TXT_REQUIRED_RC, f'Wrong OAuth 2.0 credentials issuer. Got {_getValueFromOAuth("iss", creds)} expected one of {", ".join(expected_iss)}')
|
||||||
request = transportCreateRequest()
|
request = transportCreateRequest()
|
||||||
try:
|
try:
|
||||||
creds_data['decoded_id_token'] = google.oauth2.id_token.verify_oauth2_token(creds.id_token, request,
|
creds_data['decoded_id_token'] = google.oauth2.id_token.verify_oauth2_token(creds.id_token, request,
|
||||||
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
|
clock_skew_in_seconds=GC.Values[GC.CLOCK_SKEW_IN_SECONDS])
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
if 'Token used too early' in str(e):
|
if 'Token used too early' in str(e):
|
||||||
m.stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
|
_gam().stderrErrorMsg(Msg.PLEASE_CORRECT_YOUR_SYSTEM_TIME)
|
||||||
m.systemErrorExit(m.SYSTEM_ERROR_RC, str(e))
|
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC, str(e))
|
||||||
GM.Globals[GM.DECODED_ID_TOKEN] = creds_data['decoded_id_token']
|
GM.Globals[GM.DECODED_ID_TOKEN] = creds_data['decoded_id_token']
|
||||||
if filename != '-':
|
if filename != '-':
|
||||||
m.writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
|
_gam().writeFile(filename, json.dumps(creds_data, indent=2, sort_keys=True)+'\n')
|
||||||
else:
|
else:
|
||||||
m.writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
|
_gam().writeStdout(json.dumps(creds_data, ensure_ascii=False, indent=2, sort_keys=True)+'\n')
|
||||||
|
|
||||||
def shortenURL(long_url):
|
def shortenURL(long_url):
|
||||||
m = _getMain()
|
|
||||||
if GC.Values[GC.NO_SHORT_URLS]:
|
if GC.Values[GC.NO_SHORT_URLS]:
|
||||||
return long_url
|
return long_url
|
||||||
httpObj = getHttpObj(timeout=10)
|
httpObj = getHttpObj(timeout=10)
|
||||||
@@ -421,7 +412,7 @@ def shortenURL(long_url):
|
|||||||
resp, content = httpObj.request(URL_SHORTENER_ENDPOINT, 'POST',
|
resp, content = httpObj.request(URL_SHORTENER_ENDPOINT, 'POST',
|
||||||
payload,
|
payload,
|
||||||
headers={'Content-Type': 'application/json',
|
headers={'Content-Type': 'application/json',
|
||||||
'User-Agent': m.GAM_USER_AGENT})
|
'User-Agent': _gam().GAM_USER_AGENT})
|
||||||
except:
|
except:
|
||||||
return long_url
|
return long_url
|
||||||
if resp.status != 200:
|
if resp.status != 200:
|
||||||
@@ -440,13 +431,12 @@ def runSqliteQuery(db_file, query):
|
|||||||
return curr.fetchone()[0]
|
return curr.fetchone()[0]
|
||||||
|
|
||||||
def refreshCredentialsWithReauth(credentials):
|
def refreshCredentialsWithReauth(credentials):
|
||||||
m = _getMain()
|
|
||||||
def gcloudError():
|
def gcloudError():
|
||||||
m.writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
|
_gam().writeStderr(f'Failed to run gcloud as {admin_email}. Please make sure it\'s setup')
|
||||||
e = Msg.REAUTHENTICATION_IS_NEEDED
|
e = Msg.REAUTHENTICATION_IS_NEEDED
|
||||||
handleOAuthTokenError(e, False)
|
handleOAuthTokenError(e, False)
|
||||||
|
|
||||||
m.writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
|
_gam().writeStderr(Msg.CALLING_GCLOUD_FOR_REAUTH)
|
||||||
if 'termios' in sys.modules:
|
if 'termios' in sys.modules:
|
||||||
import termios
|
import termios
|
||||||
old_settings = termios.tcgetattr(sys.stdin)
|
old_settings = termios.tcgetattr(sys.stdin)
|
||||||
@@ -454,7 +444,7 @@ def refreshCredentialsWithReauth(credentials):
|
|||||||
# First makes sure gcloud has a valid access token and thus
|
# First makes sure gcloud has a valid access token and thus
|
||||||
# should also have a valid RAPT token
|
# should also have a valid RAPT token
|
||||||
try:
|
try:
|
||||||
devnull = open(os.devnull, 'w', encoding=m.UTF8)
|
devnull = open(os.devnull, 'w', encoding=_gam().UTF8)
|
||||||
subprocess.run(['gcloud',
|
subprocess.run(['gcloud',
|
||||||
'auth',
|
'auth',
|
||||||
'print-identity-token',
|
'print-identity-token',
|
||||||
@@ -472,7 +462,7 @@ def refreshCredentialsWithReauth(credentials):
|
|||||||
if 'termios' in sys.modules:
|
if 'termios' in sys.modules:
|
||||||
import termios
|
import termios
|
||||||
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
|
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
|
||||||
m.printBlankLine()
|
_gam().printBlankLine()
|
||||||
raise KeyboardInterrupt from e
|
raise KeyboardInterrupt from e
|
||||||
token_path = gcloud_path_result.stdout.decode().strip()
|
token_path = gcloud_path_result.stdout.decode().strip()
|
||||||
if not token_path:
|
if not token_path:
|
||||||
@@ -484,19 +474,18 @@ def refreshCredentialsWithReauth(credentials):
|
|||||||
except TypeError:
|
except TypeError:
|
||||||
gcloudError()
|
gcloudError()
|
||||||
if not credentials._rapt_token:
|
if not credentials._rapt_token:
|
||||||
m.systemErrorExit(m.SYSTEM_ERROR_RC,
|
_gam().systemErrorExit(_gam().SYSTEM_ERROR_RC,
|
||||||
'Failed to retrieve reauth token from gcloud. You may need to wait until gcloud is also prompted for reauth.')
|
'Failed to retrieve reauth token from gcloud. You may need to wait until gcloud is also prompted for reauth.')
|
||||||
|
|
||||||
def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, api=None, noDASA=False, refreshOnly=False, noScopes=False):
|
def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, api=None, noDASA=False, refreshOnly=False, noScopes=False):
|
||||||
"""Gets OAuth2 credentials which are guaranteed to be fresh and valid.
|
"""Gets OAuth2 credentials which are guaranteed to be fresh and valid.
|
||||||
Locks during read and possible write so that only one process will
|
Locks during read and possible write so that only one process will
|
||||||
attempt refresh/write when running in parallel. """
|
attempt refresh/write when running in parallel. """
|
||||||
m = _getMain()
|
|
||||||
lock = FileLock(GM.Globals[GM.OAUTH2_TXT_LOCK], mode=GC.Values[GC.OAUTH2_TXT_LOCK_MODE])
|
lock = FileLock(GM.Globals[GM.OAUTH2_TXT_LOCK], mode=GC.Values[GC.OAUTH2_TXT_LOCK_MODE])
|
||||||
with lock:
|
with lock:
|
||||||
writeCreds, credentials = getOauth2TxtCredentials(api=api, noDASA=noDASA, refreshOnly=refreshOnly, noScopes=noScopes)
|
writeCreds, credentials = getOauth2TxtCredentials(api=api, noDASA=noDASA, refreshOnly=refreshOnly, noScopes=noScopes)
|
||||||
if not credentials:
|
if not credentials:
|
||||||
m.invalidOauth2TxtExit('')
|
_gam().invalidOauth2TxtExit('')
|
||||||
if credentials.expired or forceRefresh:
|
if credentials.expired or forceRefresh:
|
||||||
triesLimit = 3
|
triesLimit = 3
|
||||||
for n in range(1, triesLimit+1):
|
for n in range(1, triesLimit+1):
|
||||||
@@ -507,7 +496,7 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
|
|||||||
break
|
break
|
||||||
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
|
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
|
||||||
continue
|
continue
|
||||||
handleServerError(e)
|
handleServerError(e)
|
||||||
except google.auth.exceptions.RefreshError as e:
|
except google.auth.exceptions.RefreshError as e:
|
||||||
@@ -522,14 +511,13 @@ def getClientCredentials(forceRefresh=False, forceWrite=False, filename=None, ap
|
|||||||
return credentials
|
return credentials
|
||||||
|
|
||||||
def waitOnFailure(n, triesLimit, error_code, error_message):
|
def waitOnFailure(n, triesLimit, error_code, error_message):
|
||||||
m = _getMain()
|
|
||||||
delta = min(2 ** n, 60)+float(random.randint(1, 1000))/1000
|
delta = min(2 ** n, 60)+float(random.randint(1, 1000))/1000
|
||||||
if n > 3:
|
if n > 3:
|
||||||
m.writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
|
_gam().writeStderr(f'Temporary error: {error_code} - {error_message}, Backing off: {int(delta)} seconds, Retry: {n}/{triesLimit}\n')
|
||||||
m.flushStderr()
|
_gam().flushStderr()
|
||||||
time.sleep(delta)
|
time.sleep(delta)
|
||||||
if GC.Values[GC.SHOW_API_CALLS_RETRY_DATA]:
|
if GC.Values[GC.SHOW_API_CALLS_RETRY_DATA]:
|
||||||
m.incrAPICallsRetryData(error_message, delta)
|
_gam().incrAPICallsRetryData(error_message, delta)
|
||||||
|
|
||||||
def clearServiceCache(service):
|
def clearServiceCache(service):
|
||||||
if hasattr(service._http, 'http') and hasattr(service._http.http, 'cache'):
|
if hasattr(service._http, 'http') and hasattr(service._http.http, 'cache'):
|
||||||
@@ -551,7 +539,6 @@ def getAPIService(api, httpObj):
|
|||||||
discoveryServiceUrl=DISCOVERY_URIS[v2discovery], static_discovery=False)
|
discoveryServiceUrl=DISCOVERY_URIS[v2discovery], static_discovery=False)
|
||||||
|
|
||||||
def getService(api, httpObj):
|
def getService(api, httpObj):
|
||||||
m = _getMain()
|
|
||||||
hasLocalJSON = API.hasLocalJSON(api)
|
hasLocalJSON = API.hasLocalJSON(api)
|
||||||
api, version, v2discovery = API.getVersion(api)
|
api, version, v2discovery = API.getVersion(api)
|
||||||
if api in GM.Globals[GM.CURRENT_API_SERVICES] and version in GM.Globals[GM.CURRENT_API_SERVICES][api]:
|
if api in GM.Globals[GM.CURRENT_API_SERVICES] and version in GM.Globals[GM.CURRENT_API_SERVICES][api]:
|
||||||
@@ -577,22 +564,22 @@ def getService(api, httpObj):
|
|||||||
clearServiceCache(service)
|
clearServiceCache(service)
|
||||||
return service
|
return service
|
||||||
except googleapiclient.errors.UnknownApiNameOrVersion as e:
|
except googleapiclient.errors.UnknownApiNameOrVersion as e:
|
||||||
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), m.__author__))
|
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, Msg.UNKNOWN_API_OR_VERSION.format(str(e), _gam().__author__))
|
||||||
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
|
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
waitOnFailure(n, triesLimit, m.INVALID_JSON_RC, str(e))
|
waitOnFailure(n, triesLimit, _gam().INVALID_JSON_RC, str(e))
|
||||||
continue
|
continue
|
||||||
m.systemErrorExit(m.INVALID_JSON_RC, str(e))
|
_gam().systemErrorExit(_gam().INVALID_JSON_RC, str(e))
|
||||||
except (http.client.ResponseNotReady, OSError, googleapiclient.errors.HttpError) as e:
|
except (http.client.ResponseNotReady, OSError, googleapiclient.errors.HttpError) as e:
|
||||||
errMsg = f'Connection error: {str(e) or repr(e)}'
|
errMsg = f'Connection error: {str(e) or repr(e)}'
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg)
|
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
|
||||||
continue
|
continue
|
||||||
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg)
|
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
|
||||||
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
httpObj.connections = {}
|
httpObj.connections = {}
|
||||||
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
|
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
|
||||||
continue
|
continue
|
||||||
handleServerError(e)
|
handleServerError(e)
|
||||||
disc_file, discovery = readDiscoveryFile(f'{api}-{version}')
|
disc_file, discovery = readDiscoveryFile(f'{api}-{version}')
|
||||||
@@ -604,9 +591,9 @@ def getService(api, httpObj):
|
|||||||
clearServiceCache(service)
|
clearServiceCache(service)
|
||||||
return service
|
return service
|
||||||
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
|
except (googleapiclient.errors.InvalidJsonError, KeyError, ValueError) as e:
|
||||||
m.invalidDiscoveryJsonExit(disc_file, str(e))
|
_gam().invalidDiscoveryJsonExit(disc_file, str(e))
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
m.systemErrorExit(m.FILE_ERROR_RC, str(e))
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, str(e))
|
||||||
|
|
||||||
def defaultSvcAcctScopes():
|
def defaultSvcAcctScopes():
|
||||||
scopesList = API.getSvcAcctScopesList(GC.Values[GC.USER_SERVICE_ACCOUNT_ACCESS_ONLY], False)
|
scopesList = API.getSvcAcctScopesList(GC.Values[GC.USER_SERVICE_ACCOUNT_ACCESS_ONLY], False)
|
||||||
@@ -622,17 +609,16 @@ def defaultSvcAcctScopes():
|
|||||||
return saScopes
|
return saScopes
|
||||||
|
|
||||||
def _getSvcAcctData():
|
def _getSvcAcctData():
|
||||||
m = _getMain()
|
|
||||||
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
|
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
|
||||||
jsonData = m.readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
|
jsonData = _gam().readFile(GC.Values[GC.OAUTH2SERVICE_JSON], continueOnError=True, displayError=True)
|
||||||
if not jsonData:
|
if not jsonData:
|
||||||
m.invalidOauth2serviceJsonExit(Msg.NO_DATA)
|
_gam().invalidOauth2serviceJsonExit(Msg.NO_DATA)
|
||||||
try:
|
try:
|
||||||
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = json.loads(jsonData)
|
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = json.loads(jsonData)
|
||||||
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
||||||
m.invalidOauth2serviceJsonExit(str(e))
|
_gam().invalidOauth2serviceJsonExit(str(e))
|
||||||
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
|
if not GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
|
||||||
m.systemErrorExit(m.OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
|
_gam().systemErrorExit(_gam().OAUTH2SERVICE_JSON_REQUIRED_RC, Msg.NO_SVCACCT_ACCESS_ALLOWED)
|
||||||
requiredFields = ['client_email', 'client_id', 'project_id', 'token_uri']
|
requiredFields = ['client_email', 'client_id', 'project_id', 'token_uri']
|
||||||
key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
|
key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default')
|
||||||
if key_type == 'default':
|
if key_type == 'default':
|
||||||
@@ -642,7 +628,7 @@ def _getSvcAcctData():
|
|||||||
if field not in GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
|
if field not in GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]:
|
||||||
missingFields.append(field)
|
missingFields.append(field)
|
||||||
if missingFields:
|
if missingFields:
|
||||||
m.invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
|
_gam().invalidOauth2serviceJsonExit(Msg.MISSING_FIELDS.format(','.join(missingFields)))
|
||||||
# Some old oauth2service.json files have: 'https://accounts.google.com/o/oauth2/auth' which no longer works
|
# Some old oauth2service.json files have: 'https://accounts.google.com/o/oauth2/auth' which no longer works
|
||||||
if GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] == 'https://accounts.google.com/o/oauth2/auth':
|
if GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] == 'https://accounts.google.com/o/oauth2/auth':
|
||||||
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] = API.GOOGLE_OAUTH2_TOKEN_ENDPOINT
|
GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['token_uri'] = API.GOOGLE_OAUTH2_TOKEN_ENDPOINT
|
||||||
@@ -654,7 +640,6 @@ def _getSvcAcctData():
|
|||||||
GM.Globals[GM.SVCACCT_SCOPES] = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].pop(API.OAUTH2SA_SCOPES)
|
GM.Globals[GM.SVCACCT_SCOPES] = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].pop(API.OAUTH2SA_SCOPES)
|
||||||
|
|
||||||
def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=False):
|
def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=False):
|
||||||
m = _getMain()
|
|
||||||
_getSvcAcctData()
|
_getSvcAcctData()
|
||||||
if isinstance(scopesOrAPI, str):
|
if isinstance(scopesOrAPI, str):
|
||||||
GM.Globals[GM.CURRENT_SVCACCT_API] = scopesOrAPI
|
GM.Globals[GM.CURRENT_SVCACCT_API] = scopesOrAPI
|
||||||
@@ -665,7 +650,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
|
|||||||
if scopesOrAPI != API.CHAT_EVENTS and not GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]:
|
if scopesOrAPI != API.CHAT_EVENTS and not GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES]:
|
||||||
if softErrors:
|
if softErrors:
|
||||||
return None
|
return None
|
||||||
m.SvcAcctAPIAccessDeniedExit()
|
_gam().SvcAcctAPIAccessDeniedExit()
|
||||||
if scopesOrAPI in {API.PEOPLE, API.PEOPLE_DIRECTORY, API.PEOPLE_OTHERCONTACTS}:
|
if scopesOrAPI in {API.PEOPLE, API.PEOPLE_DIRECTORY, API.PEOPLE_OTHERCONTACTS}:
|
||||||
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES].append(API.USERINFO_PROFILE_SCOPE)
|
GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES].append(API.USERINFO_PROFILE_SCOPE)
|
||||||
if scopesOrAPI in {API.PEOPLE_OTHERCONTACTS}:
|
if scopesOrAPI in {API.PEOPLE_OTHERCONTACTS}:
|
||||||
@@ -692,7 +677,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
|
|||||||
except (ValueError, IndexError, KeyError) as e:
|
except (ValueError, IndexError, KeyError) as e:
|
||||||
if softErrors:
|
if softErrors:
|
||||||
return None
|
return None
|
||||||
m.invalidOauth2serviceJsonExit(str(e))
|
_gam().invalidOauth2serviceJsonExit(str(e))
|
||||||
credentials = credentials.with_scopes(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])
|
credentials = credentials.with_scopes(GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES])
|
||||||
else:
|
else:
|
||||||
audience = f'https://{scopesOrAPI}.googleapis.com/'
|
audience = f'https://{scopesOrAPI}.googleapis.com/'
|
||||||
@@ -714,7 +699,7 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
|
|||||||
except (ValueError, IndexError, KeyError) as e:
|
except (ValueError, IndexError, KeyError) as e:
|
||||||
if softErrors:
|
if softErrors:
|
||||||
return None
|
return None
|
||||||
m.invalidOauth2serviceJsonExit(str(e))
|
_gam().invalidOauth2serviceJsonExit(str(e))
|
||||||
GM.Globals[GM.CURRENT_SVCACCT_USER] = userEmail
|
GM.Globals[GM.CURRENT_SVCACCT_USER] = userEmail
|
||||||
if userEmail:
|
if userEmail:
|
||||||
credentials = credentials.with_subject(userEmail)
|
credentials = credentials.with_subject(userEmail)
|
||||||
@@ -723,7 +708,6 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F
|
|||||||
return credentials
|
return credentials
|
||||||
|
|
||||||
def getGDataOAuthToken(gdataObj, credentials=None):
|
def getGDataOAuthToken(gdataObj, credentials=None):
|
||||||
m = _getMain()
|
|
||||||
if not credentials:
|
if not credentials:
|
||||||
credentials = getClientCredentials(refreshOnly=True)
|
credentials = getClientCredentials(refreshOnly=True)
|
||||||
try:
|
try:
|
||||||
@@ -742,14 +726,13 @@ def getGDataOAuthToken(gdataObj, credentials=None):
|
|||||||
GM.Globals[GM.ADMIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('email', 'UNKNOWN').lower()
|
GM.Globals[GM.ADMIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('email', 'UNKNOWN').lower()
|
||||||
GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id
|
GM.Globals[GM.OAUTH2_CLIENT_ID] = credentials.client_id
|
||||||
gdataObj.domain = GC.Values[GC.DOMAIN]
|
gdataObj.domain = GC.Values[GC.DOMAIN]
|
||||||
gdataObj.source = m.GAM_USER_AGENT
|
gdataObj.source = _gam().GAM_USER_AGENT
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def checkGDataError(e, service):
|
def checkGDataError(e, service):
|
||||||
m = _getMain()
|
|
||||||
error = e.args
|
error = e.args
|
||||||
reason = error[0].get('reason', '')
|
reason = error[0].get('reason', '')
|
||||||
body = error[0].get('body', '').decode(m.UTF8)
|
body = error[0].get('body', '').decode(_gam().UTF8)
|
||||||
# First check for errors that need special handling
|
# First check for errors that need special handling
|
||||||
if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']:
|
if reason in ['Token invalid - Invalid token: Stateless token expired', 'Token invalid - Invalid token: Token not found', 'gone']:
|
||||||
keep_domain = service.domain
|
keep_domain = service.domain
|
||||||
@@ -758,7 +741,7 @@ def checkGDataError(e, service):
|
|||||||
return (GDATA.TOKEN_EXPIRED, reason)
|
return (GDATA.TOKEN_EXPIRED, reason)
|
||||||
error_code = getattr(e, 'error_code', 600)
|
error_code = getattr(e, 'error_code', 600)
|
||||||
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
||||||
m.writeStdout(f'{m.ERROR_PREFIX} {error_code}: {reason}, {body}\n')
|
_gam().writeStdout(f'{_gam().ERROR_PREFIX} {error_code}: {reason}, {body}\n')
|
||||||
if error_code == 600:
|
if error_code == 600:
|
||||||
if (body.startswith('Quota exceeded for the current request') or
|
if (body.startswith('Quota exceeded for the current request') or
|
||||||
body.startswith('Quota exceeded for quota metric') or
|
body.startswith('Quota exceeded for quota metric') or
|
||||||
@@ -859,7 +842,6 @@ def callGData(service, function,
|
|||||||
bailOnInternalServerError=False, softErrors=False,
|
bailOnInternalServerError=False, softErrors=False,
|
||||||
throwErrors=None, retryErrors=None, triesLimit=0,
|
throwErrors=None, retryErrors=None, triesLimit=0,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
m = _getMain()
|
|
||||||
if throwErrors is None:
|
if throwErrors is None:
|
||||||
throwErrors = []
|
throwErrors = []
|
||||||
if retryErrors is None:
|
if retryErrors is None:
|
||||||
@@ -869,7 +851,7 @@ def callGData(service, function,
|
|||||||
allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors
|
allRetryErrors = GDATA.NON_TERMINATING_ERRORS+retryErrors
|
||||||
method = getattr(service, function)
|
method = getattr(service, function)
|
||||||
if GC.Values[GC.API_CALLS_RATE_CHECK]:
|
if GC.Values[GC.API_CALLS_RATE_CHECK]:
|
||||||
m.checkAPICallsRate()
|
_gam().checkAPICallsRate()
|
||||||
for n in range(1, triesLimit+1):
|
for n in range(1, triesLimit+1):
|
||||||
try:
|
try:
|
||||||
return method(**kwargs)
|
return method(**kwargs)
|
||||||
@@ -886,14 +868,14 @@ def callGData(service, function,
|
|||||||
raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message)
|
raise GDATA.ERROR_CODE_EXCEPTION_MAP[error_code](error_message)
|
||||||
raise
|
raise
|
||||||
if softErrors:
|
if softErrors:
|
||||||
m.stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
|
_gam().stderrErrorMsg(f'{error_code} - {error_message}{["", ": Giving up."][n > 1]}')
|
||||||
return None
|
return None
|
||||||
if error_code == GDATA.INSUFFICIENT_PERMISSIONS:
|
if error_code == GDATA.INSUFFICIENT_PERMISSIONS:
|
||||||
m.APIAccessDeniedExit()
|
_gam().APIAccessDeniedExit()
|
||||||
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
|
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, f'{error_code} - {error_message}')
|
||||||
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
|
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
|
||||||
continue
|
continue
|
||||||
handleServerError(e)
|
handleServerError(e)
|
||||||
except google.auth.exceptions.RefreshError as e:
|
except google.auth.exceptions.RefreshError as e:
|
||||||
@@ -904,33 +886,31 @@ def callGData(service, function,
|
|||||||
except (http.client.ResponseNotReady, OSError) as e:
|
except (http.client.ResponseNotReady, OSError) as e:
|
||||||
errMsg = f'Connection error: {str(e) or repr(e)}'
|
errMsg = f'Connection error: {str(e) or repr(e)}'
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg)
|
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
|
||||||
continue
|
continue
|
||||||
if softErrors:
|
if softErrors:
|
||||||
m.writeStderr(f'\n{m.ERROR_PREFIX}{errMsg} - Giving up.\n')
|
_gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
|
||||||
return None
|
return None
|
||||||
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg)
|
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
|
||||||
|
|
||||||
def writeGotMessage(msg):
|
def writeGotMessage(msg):
|
||||||
m = _getMain()
|
|
||||||
if GC.Values[GC.SHOW_GETTINGS_GOT_NL]:
|
if GC.Values[GC.SHOW_GETTINGS_GOT_NL]:
|
||||||
m.writeStderr(msg)
|
_gam().writeStderr(msg)
|
||||||
else:
|
else:
|
||||||
m.writeStderr('\r')
|
_gam().writeStderr('\r')
|
||||||
msgLen = len(msg)
|
msgLen = len(msg)
|
||||||
if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]:
|
if msgLen < GM.Globals[GM.LAST_GOT_MSG_LEN]:
|
||||||
m.writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
|
_gam().writeStderr(msg+' '*(GM.Globals[GM.LAST_GOT_MSG_LEN]-msgLen))
|
||||||
else:
|
else:
|
||||||
m.writeStderr(msg)
|
_gam().writeStderr(msg)
|
||||||
GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen
|
GM.Globals[GM.LAST_GOT_MSG_LEN] = msgLen
|
||||||
m.flushStderr()
|
_gam().flushStderr()
|
||||||
|
|
||||||
def callGDataPages(service, function,
|
def callGDataPages(service, function,
|
||||||
pageMessage=None,
|
pageMessage=None,
|
||||||
softErrors=False, throwErrors=None, retryErrors=None,
|
softErrors=False, throwErrors=None, retryErrors=None,
|
||||||
uri=None,
|
uri=None,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
m = _getMain()
|
|
||||||
Ent = _getEnt()
|
Ent = _getEnt()
|
||||||
if throwErrors is None:
|
if throwErrors is None:
|
||||||
throwErrors = []
|
throwErrors = []
|
||||||
@@ -955,32 +935,30 @@ def callGDataPages(service, function,
|
|||||||
nextLink = None
|
nextLink = None
|
||||||
pageItems = 0
|
pageItems = 0
|
||||||
if pageMessage:
|
if pageMessage:
|
||||||
show_message = pageMessage.replace(m.TOTAL_ITEMS_MARKER, str(totalItems))
|
show_message = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
|
||||||
writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems)))
|
writeGotMessage(show_message.format(Ent.ChooseGetting(totalItems)))
|
||||||
if nextLink is None:
|
if nextLink is None:
|
||||||
if pageMessage and (pageMessage[-1] != '\n'):
|
if pageMessage and (pageMessage[-1] != '\n'):
|
||||||
m.writeStderr('\r\n')
|
_gam().writeStderr('\r\n')
|
||||||
m.flushStderr()
|
_gam().flushStderr()
|
||||||
return allResults
|
return allResults
|
||||||
uri = nextLink.href
|
uri = nextLink.href
|
||||||
if 'url_params' in kwargs:
|
if 'url_params' in kwargs:
|
||||||
kwargs['url_params'].pop('start-index', None)
|
kwargs['url_params'].pop('start-index', None)
|
||||||
|
|
||||||
def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True):
|
def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True):
|
||||||
m = _getMain()
|
|
||||||
|
|
||||||
def makeErrorDict(code, reason, message):
|
def makeErrorDict(code, reason, message):
|
||||||
return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}}
|
return {'error': {'code': code, 'errors': [{'reason': reason, 'message': message}]}}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
error = json.loads(e.content.decode(m.UTF8))
|
error = json.loads(e.content.decode(_gam().UTF8))
|
||||||
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
||||||
m.writeStdout(f'{m.ERROR_PREFIX} JSON: {str(error)}\n')
|
_gam().writeStdout(f'{_gam().ERROR_PREFIX} JSON: {str(error)}\n')
|
||||||
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
|
except (IndexError, KeyError, SyntaxError, TypeError, ValueError):
|
||||||
eContent = e.content.decode(m.UTF8) if isinstance(e.content, bytes) else e.content
|
eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
|
||||||
lContent = eContent.lower()
|
lContent = eContent.lower()
|
||||||
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
||||||
m.writeStdout(f'{m.ERROR_PREFIX} HTTP: {str(eContent)}\n')
|
_gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
|
||||||
if eContent[0:15] != '<!DOCTYPE html>':
|
if eContent[0:15] != '<!DOCTYPE html>':
|
||||||
if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')):
|
if (e.resp['status'] == '403') and (lContent.startswith('request rate higher than configured')):
|
||||||
return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent)
|
return (e.resp['status'], GAPI.QUOTA_EXCEEDED, eContent)
|
||||||
@@ -993,7 +971,7 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
|
|||||||
if (e.resp['status'] == '504') and ('gateway timeout' in lContent):
|
if (e.resp['status'] == '504') and ('gateway timeout' in lContent):
|
||||||
return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent)
|
return (e.resp['status'], GAPI.GATEWAY_TIMEOUT, eContent)
|
||||||
else:
|
else:
|
||||||
tg = m.HTML_TITLE_PATTERN.match(lContent)
|
tg = _gam().HTML_TITLE_PATTERN.match(lContent)
|
||||||
lContent = tg.group(1) if tg else 'bad request'
|
lContent = tg.group(1) if tg else 'bad request'
|
||||||
if (e.resp['status'] == '403') and ('invalid domain.' in lContent):
|
if (e.resp['status'] == '403') and ('invalid domain.' in lContent):
|
||||||
error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found')
|
error = makeErrorDict(403, GAPI.NOT_FOUND, 'Domain not found')
|
||||||
@@ -1020,10 +998,10 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
|
|||||||
elif retryOnHttpError:
|
elif retryOnHttpError:
|
||||||
return (-1, None, eContent)
|
return (-1, None, eContent)
|
||||||
elif softErrors:
|
elif softErrors:
|
||||||
m.stderrErrorMsg(eContent)
|
_gam().stderrErrorMsg(eContent)
|
||||||
return (0, None, None)
|
return (0, None, None)
|
||||||
else:
|
else:
|
||||||
m.systemErrorExit(m.HTTP_ERROR_RC, eContent)
|
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, eContent)
|
||||||
requiredScopes = ''
|
requiredScopes = ''
|
||||||
wwwAuthenticate = e.resp.get('www-authenticate', '')
|
wwwAuthenticate = e.resp.get('www-authenticate', '')
|
||||||
if 'insufficient_scope' in wwwAuthenticate:
|
if 'insufficient_scope' in wwwAuthenticate:
|
||||||
@@ -1129,9 +1107,9 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True
|
|||||||
http_status = 400
|
http_status = 400
|
||||||
error = makeErrorDict(http_status, GAPI.INVALID, message)
|
error = makeErrorDict(http_status, GAPI.INVALID, message)
|
||||||
else:
|
else:
|
||||||
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(error))
|
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
|
||||||
else:
|
else:
|
||||||
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(error))
|
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(error))
|
||||||
try:
|
try:
|
||||||
reason = error['error']['errors'][0]['reason']
|
reason = error['error']['errors'][0]['reason']
|
||||||
for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []):
|
for messageItem in GAPI.REASON_MESSAGE_MAP.get(reason, []):
|
||||||
@@ -1157,7 +1135,6 @@ def callGAPI(service, function,
|
|||||||
softErrors=False, mapNotFound=True,
|
softErrors=False, mapNotFound=True,
|
||||||
throwReasons=None, retryReasons=None, triesLimit=0,
|
throwReasons=None, retryReasons=None, triesLimit=0,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
m = _getMain()
|
|
||||||
if throwReasons is None:
|
if throwReasons is None:
|
||||||
throwReasons = []
|
throwReasons = []
|
||||||
if retryReasons is None:
|
if retryReasons is None:
|
||||||
@@ -1168,7 +1145,7 @@ def callGAPI(service, function,
|
|||||||
method = getattr(service, function)
|
method = getattr(service, function)
|
||||||
svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST])
|
svcparms = dict(list(kwargs.items())+GM.Globals[GM.EXTRA_ARGS_LIST])
|
||||||
if GC.Values[GC.API_CALLS_RATE_CHECK]:
|
if GC.Values[GC.API_CALLS_RATE_CHECK]:
|
||||||
m.checkAPICallsRate()
|
_gam().checkAPICallsRate()
|
||||||
for n in range(1, triesLimit+1):
|
for n in range(1, triesLimit+1):
|
||||||
try:
|
try:
|
||||||
return method(**svcparms).execute()
|
return method(**svcparms).execute()
|
||||||
@@ -1181,7 +1158,7 @@ def callGAPI(service, function,
|
|||||||
# service._http.credentials.refresh(getHttpObj())
|
# service._http.credentials.refresh(getHttpObj())
|
||||||
service._http.credentials.refresh(transportCreateRequest())
|
service._http.credentials.refresh(transportCreateRequest())
|
||||||
except TypeError:
|
except TypeError:
|
||||||
m.systemErrorExit(m.HTTP_ERROR_RC, message)
|
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, message)
|
||||||
continue
|
continue
|
||||||
if http_status == 0:
|
if http_status == 0:
|
||||||
return None
|
return None
|
||||||
@@ -1202,17 +1179,17 @@ def callGAPI(service, function,
|
|||||||
raise GAPI.REASON_EXCEPTION_MAP[reason](message)
|
raise GAPI.REASON_EXCEPTION_MAP[reason](message)
|
||||||
raise e
|
raise e
|
||||||
if softErrors:
|
if softErrors:
|
||||||
m.stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
|
_gam().stderrErrorMsg(f'{http_status}: {reason} - {message}{["", ": Giving up."][n > 1]}')
|
||||||
return None
|
return None
|
||||||
if reason == GAPI.INSUFFICIENT_PERMISSIONS:
|
if reason == GAPI.INSUFFICIENT_PERMISSIONS:
|
||||||
m.APIAccessDeniedExit()
|
_gam().APIAccessDeniedExit()
|
||||||
m.systemErrorExit(m.HTTP_ERROR_RC, m.formatHTTPError(http_status, reason, message))
|
_gam().systemErrorExit(_gam().HTTP_ERROR_RC, _gam().formatHTTPError(http_status, reason, message))
|
||||||
except googleapiclient.errors.MediaUploadSizeError as e:
|
except googleapiclient.errors.MediaUploadSizeError as e:
|
||||||
raise e
|
raise e
|
||||||
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
service._http.connections = {}
|
service._http.connections = {}
|
||||||
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
|
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
|
||||||
continue
|
continue
|
||||||
handleServerError(e)
|
handleServerError(e)
|
||||||
except google.auth.exceptions.RefreshError as e:
|
except google.auth.exceptions.RefreshError as e:
|
||||||
@@ -1223,23 +1200,22 @@ def callGAPI(service, function,
|
|||||||
except (http.client.ResponseNotReady, OSError) as e:
|
except (http.client.ResponseNotReady, OSError) as e:
|
||||||
errMsg = f'Connection error: {str(e) or repr(e)}'
|
errMsg = f'Connection error: {str(e) or repr(e)}'
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
waitOnFailure(n, triesLimit, m.SOCKET_ERROR_RC, errMsg)
|
waitOnFailure(n, triesLimit, _gam().SOCKET_ERROR_RC, errMsg)
|
||||||
continue
|
continue
|
||||||
if softErrors:
|
if softErrors:
|
||||||
m.writeStderr(f'\n{m.ERROR_PREFIX}{errMsg} - Giving up.\n')
|
_gam().writeStderr(f'\n{_gam().ERROR_PREFIX}{errMsg} - Giving up.\n')
|
||||||
return None
|
return None
|
||||||
m.systemErrorExit(m.SOCKET_ERROR_RC, errMsg)
|
_gam().systemErrorExit(_gam().SOCKET_ERROR_RC, errMsg)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
if clearServiceCache(service):
|
if clearServiceCache(service):
|
||||||
continue
|
continue
|
||||||
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(e))
|
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
|
||||||
except TypeError as e:
|
except TypeError as e:
|
||||||
m.systemErrorExit(m.GOOGLE_API_ERROR_RC, str(e))
|
_gam().systemErrorExit(_gam().GOOGLE_API_ERROR_RC, str(e))
|
||||||
|
|
||||||
def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType):
|
def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAttribute, entityType):
|
||||||
m = _getMain()
|
|
||||||
Ent = _getEnt()
|
Ent = _getEnt()
|
||||||
showMessage = pageMessage.replace(m.TOTAL_ITEMS_MARKER, str(totalItems))
|
showMessage = pageMessage.replace(_gam().TOTAL_ITEMS_MARKER, str(totalItems))
|
||||||
if pageItems:
|
if pageItems:
|
||||||
if messageAttribute:
|
if messageAttribute:
|
||||||
firstItem = results[0] if pageItems > 0 else {}
|
firstItem = results[0] if pageItems > 0 else {}
|
||||||
@@ -1253,11 +1229,11 @@ def _showGAPIpagesResult(results, pageItems, totalItems, pageMessage, messageAtt
|
|||||||
lastItem = lastItem.get(attr, {})
|
lastItem = lastItem.get(attr, {})
|
||||||
firstItem = str(firstItem)
|
firstItem = str(firstItem)
|
||||||
lastItem = str(lastItem)
|
lastItem = str(lastItem)
|
||||||
showMessage = showMessage.replace(m.FIRST_ITEM_MARKER, firstItem)
|
showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, firstItem)
|
||||||
showMessage = showMessage.replace(m.LAST_ITEM_MARKER, lastItem)
|
showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, lastItem)
|
||||||
else:
|
else:
|
||||||
showMessage = showMessage.replace(m.FIRST_ITEM_MARKER, '')
|
showMessage = showMessage.replace(_gam().FIRST_ITEM_MARKER, '')
|
||||||
showMessage = showMessage.replace(m.LAST_ITEM_MARKER, '')
|
showMessage = showMessage.replace(_gam().LAST_ITEM_MARKER, '')
|
||||||
writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems))))
|
writeGotMessage(showMessage.replace('{0}', str(Ent.Choose(entityType, totalItems))))
|
||||||
|
|
||||||
def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType):
|
def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage, messageAttribute, entityType):
|
||||||
@@ -1280,10 +1256,9 @@ def _processGAPIpagesResult(results, items, allResults, totalItems, pageMessage,
|
|||||||
return (pageToken, totalItems)
|
return (pageToken, totalItems)
|
||||||
|
|
||||||
def _finalizeGAPIpagesResult(pageMessage):
|
def _finalizeGAPIpagesResult(pageMessage):
|
||||||
m = _getMain()
|
|
||||||
if pageMessage and (pageMessage[-1] != '\n'):
|
if pageMessage and (pageMessage[-1] != '\n'):
|
||||||
m.writeStderr('\r\n')
|
_gam().writeStderr('\r\n')
|
||||||
m.flushStderr()
|
_gam().flushStderr()
|
||||||
|
|
||||||
def _setMaxArgResults(maxItems, pageArgsInBody, kwargs):
|
def _setMaxArgResults(maxItems, pageArgsInBody, kwargs):
|
||||||
if pageArgsInBody:
|
if pageArgsInBody:
|
||||||
@@ -1403,25 +1378,23 @@ def callGAPIitems(service, function, items,
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
def readDiscoveryFile(api_version):
|
def readDiscoveryFile(api_version):
|
||||||
m = _getMain()
|
|
||||||
disc_filename = f'{api_version}.json'
|
disc_filename = f'{api_version}.json'
|
||||||
disc_file = os.path.join(GM.Globals[GM.GAM_PATH], disc_filename)
|
disc_file = os.path.join(GM.Globals[GM.GAM_PATH], disc_filename)
|
||||||
if hasattr(sys, '_MEIPASS'):
|
if hasattr(sys, '_MEIPASS'):
|
||||||
json_string = m.readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
|
json_string = _gam().readFile(os.path.join(sys._MEIPASS, disc_filename), continueOnError=True, displayError=True) #pylint: disable=no-member
|
||||||
elif os.path.isfile(disc_file):
|
elif os.path.isfile(disc_file):
|
||||||
json_string = m.readFile(disc_file, continueOnError=True, displayError=True)
|
json_string = _gam().readFile(disc_file, continueOnError=True, displayError=True)
|
||||||
else:
|
else:
|
||||||
json_string = None
|
json_string = None
|
||||||
if not json_string:
|
if not json_string:
|
||||||
m.invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
|
_gam().invalidDiscoveryJsonExit(disc_file, Msg.NO_DATA)
|
||||||
try:
|
try:
|
||||||
discovery = json.loads(json_string)
|
discovery = json.loads(json_string)
|
||||||
return (disc_file, discovery)
|
return (disc_file, discovery)
|
||||||
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e:
|
||||||
m.invalidDiscoveryJsonExit(disc_file, str(e))
|
_gam().invalidDiscoveryJsonExit(disc_file, str(e))
|
||||||
|
|
||||||
def buildGAPIObject(api, credentials=None):
|
def buildGAPIObject(api, credentials=None):
|
||||||
m = _getMain()
|
|
||||||
if credentials is None:
|
if credentials is None:
|
||||||
credentials = getClientCredentials(api=api, refreshOnly=True)
|
credentials = getClientCredentials(api=api, refreshOnly=True)
|
||||||
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj(cache=GM.Globals[GM.CACHE_DIR]))
|
httpObj = transportAuthorizedHttp(credentials, http=getHttpObj(cache=GM.Globals[GM.CACHE_DIR]))
|
||||||
@@ -1433,7 +1406,7 @@ def buildGAPIObject(api, credentials=None):
|
|||||||
GM.Globals[GM.CURRENT_CLIENT_API] = api
|
GM.Globals[GM.CURRENT_CLIENT_API] = api
|
||||||
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API_Scopes.intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
|
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API_Scopes.intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
|
||||||
if api not in API.SCOPELESS_APIS and not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
|
if api not in API.SCOPELESS_APIS and not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
|
||||||
m.systemErrorExit(m.NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
|
_gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
|
||||||
if not GC.Values[GC.DOMAIN]:
|
if not GC.Values[GC.DOMAIN]:
|
||||||
GC.Values[GC.DOMAIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('hd', 'UNKNOWN').lower()
|
GC.Values[GC.DOMAIN] = GM.Globals[GM.DECODED_ID_TOKEN].get('hd', 'UNKNOWN').lower()
|
||||||
if not GC.Values[GC.CUSTOMER_ID]:
|
if not GC.Values[GC.CUSTOMER_ID]:
|
||||||
@@ -1443,10 +1416,9 @@ def buildGAPIObject(api, credentials=None):
|
|||||||
return service
|
return service
|
||||||
|
|
||||||
def getSaUser(user):
|
def getSaUser(user):
|
||||||
m = _getMain()
|
|
||||||
currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API]
|
currentClientAPI = GM.Globals[GM.CURRENT_CLIENT_API]
|
||||||
currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
|
currentClientAPIScopes = GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]
|
||||||
userEmail = m.convertUIDtoEmailAddress(user) if user else None
|
userEmail = _gam().convertUIDtoEmailAddress(user) if user else None
|
||||||
GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI
|
GM.Globals[GM.CURRENT_CLIENT_API] = currentClientAPI
|
||||||
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes
|
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = currentClientAPIScopes
|
||||||
return userEmail
|
return userEmail
|
||||||
@@ -1458,7 +1430,6 @@ def chooseSaAPI(api1, api2):
|
|||||||
return api2
|
return api2
|
||||||
|
|
||||||
def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
|
def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
|
||||||
m = _getMain()
|
|
||||||
userEmail = getSaUser(user)
|
userEmail = getSaUser(user)
|
||||||
if GM.Globals[GM.HTTP_OBJECT] is None:
|
if GM.Globals[GM.HTTP_OBJECT] is None:
|
||||||
GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR])
|
GM.Globals[GM.HTTP_OBJECT] = getHttpObj(cache=GM.Globals[GM.CACHE_DIR])
|
||||||
@@ -1475,7 +1446,7 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
|
|||||||
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
except (httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
||||||
if n != triesLimit:
|
if n != triesLimit:
|
||||||
httpObj.connections = {}
|
httpObj.connections = {}
|
||||||
waitOnFailure(n, triesLimit, m.NETWORK_ERROR_RC, str(e))
|
waitOnFailure(n, triesLimit, _gam().NETWORK_ERROR_RC, str(e))
|
||||||
continue
|
continue
|
||||||
handleServerError(e)
|
handleServerError(e)
|
||||||
except google.auth.exceptions.RefreshError as e:
|
except google.auth.exceptions.RefreshError as e:
|
||||||
@@ -1485,12 +1456,12 @@ def buildGAPIServiceObject(api, user, i=0, count=0, displayError=True):
|
|||||||
if isinstance(e, str):
|
if isinstance(e, str):
|
||||||
eContent = e
|
eContent = e
|
||||||
else:
|
else:
|
||||||
eContent = e.content.decode(m.UTF8) if isinstance(e.content, bytes) else e.content
|
eContent = e.content.decode(_gam().UTF8) if isinstance(e.content, bytes) else e.content
|
||||||
if eContent[0:15] == '<!DOCTYPE html>':
|
if eContent[0:15] == '<!DOCTYPE html>':
|
||||||
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
||||||
m.writeStdout(f'{m.ERROR_PREFIX} HTTP: {str(eContent)}\n')
|
_gam().writeStdout(f'{_gam().ERROR_PREFIX} HTTP: {str(eContent)}\n')
|
||||||
lContent = eContent.lower()
|
lContent = eContent.lower()
|
||||||
tg = m.HTML_TITLE_PATTERN.match(lContent)
|
tg = _gam().HTML_TITLE_PATTERN.match(lContent)
|
||||||
lContent = tg.group(1) if tg else ''
|
lContent = tg.group(1) if tg else ''
|
||||||
if lContent.startswith('Error 502 (Server Error)'):
|
if lContent.startswith('Error 502 (Server Error)'):
|
||||||
time.sleep(30)
|
time.sleep(30)
|
||||||
@@ -1504,12 +1475,11 @@ def buildGAPIObjectNoAuthentication(api):
|
|||||||
return service
|
return service
|
||||||
|
|
||||||
def initGDataObject(gdataObj, api):
|
def initGDataObject(gdataObj, api):
|
||||||
m = _getMain()
|
|
||||||
GM.Globals[GM.CURRENT_CLIENT_API] = api
|
GM.Globals[GM.CURRENT_CLIENT_API] = api
|
||||||
credentials = getClientCredentials(noDASA=True, refreshOnly=True)
|
credentials = getClientCredentials(noDASA=True, refreshOnly=True)
|
||||||
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API.getClientScopesSet(api).intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
|
GM.Globals[GM.CURRENT_CLIENT_API_SCOPES] = API.getClientScopesSet(api).intersection(GM.Globals[GM.CREDENTIALS_SCOPES])
|
||||||
if not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
|
if not GM.Globals[GM.CURRENT_CLIENT_API_SCOPES]:
|
||||||
m.systemErrorExit(m.NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
|
_gam().systemErrorExit(_gam().NO_SCOPES_FOR_API_RC, Msg.NO_SCOPES_FOR_API.format(API.getAPIName(api)))
|
||||||
getGDataOAuthToken(gdataObj, credentials)
|
getGDataOAuthToken(gdataObj, credentials)
|
||||||
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
||||||
gdataObj.debug = True
|
gdataObj.debug = True
|
||||||
|
|||||||
@@ -115,8 +115,7 @@ def _getEnt():
|
|||||||
return sys.modules['gam'].Ent
|
return sys.modules['gam'].Ent
|
||||||
|
|
||||||
# Lazy accessor for main module
|
# Lazy accessor for main module
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
# --- Constants duplicated from __init__.py ---
|
# --- Constants duplicated from __init__.py ---
|
||||||
# These are simple literals that never change, duplicated to avoid
|
# These are simple literals that never change, duplicated to avoid
|
||||||
|
|||||||
@@ -27,8 +27,7 @@ from gamlib import glmsgs as Msg
|
|||||||
from util.csv_pf import CSVPrintFile
|
from util.csv_pf import CSVPrintFile
|
||||||
|
|
||||||
|
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
Cmd = glclargs.GamCLArgs()
|
Cmd = glclargs.GamCLArgs()
|
||||||
|
|
||||||
@@ -72,18 +71,18 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
|
|||||||
|
|
||||||
def reopenSTDFile(stdtype):
|
def reopenSTDFile(stdtype):
|
||||||
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
|
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
|
||||||
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_getMain().UTF8)
|
GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
|
||||||
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
|
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
|
||||||
GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][stdtype == GM.STDOUT]),
|
GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][stdtype == GM.STDOUT]),
|
||||||
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
|
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
|
||||||
elif stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
|
elif stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
|
||||||
GM.Globals[stdtype][GM.REDIRECT_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_FD]
|
GM.Globals[stdtype][GM.REDIRECT_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_FD]
|
||||||
else:
|
else:
|
||||||
GM.Globals[stdtype][GM.REDIRECT_FD] = _getMain().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
|
GM.Globals[stdtype][GM.REDIRECT_FD] = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
|
||||||
if stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
|
if stdtype == GM.STDERR and GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout':
|
||||||
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
|
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
|
||||||
else:
|
else:
|
||||||
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _getMain().StringIOobject()
|
GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] else _gam().StringIOobject()
|
||||||
|
|
||||||
GM.Globals[GM.DATETIME_NOW] = datetimeNow
|
GM.Globals[GM.DATETIME_NOW] = datetimeNow
|
||||||
GC.Values[GC.TIMEZONE] = tzinfo
|
GC.Values[GC.TIMEZONE] = tzinfo
|
||||||
@@ -158,11 +157,11 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo
|
|||||||
if mpQueueStdout:
|
if mpQueueStdout:
|
||||||
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()))
|
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()))
|
||||||
else:
|
else:
|
||||||
_getMain().flushStdout()
|
_gam().flushStdout()
|
||||||
if mpQueueStderr and mpQueueStderr is not mpQueueStdout:
|
if mpQueueStderr and mpQueueStderr is not mpQueueStdout:
|
||||||
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD].getvalue()))
|
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_DATA, GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD].getvalue()))
|
||||||
else:
|
else:
|
||||||
_getMain().flushStderr()
|
_gam().flushStderr()
|
||||||
|
|
||||||
def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr):
|
def initializeCSVFileQueueHandler(mpManager, mpQueueStdout, mpQueueStderr):
|
||||||
mpQueue = mpManager.Queue()
|
mpQueue = mpManager.Queue()
|
||||||
@@ -178,7 +177,7 @@ def terminateCSVFileQueueHandler(mpQueue, mpQueueHandler):
|
|||||||
GM.Globals[GM.PARSER] = None
|
GM.Globals[GM.PARSER] = None
|
||||||
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None
|
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None
|
||||||
if multiprocessing.get_start_method() != 'fork':
|
if multiprocessing.get_start_method() != 'fork':
|
||||||
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _getMain().Cmd.AllArguments()))
|
mpQueue.put((GM.REDIRECT_QUEUE_ARGS, _gam().Cmd.AllArguments()))
|
||||||
savedValues = saveNonPickleableValues()
|
savedValues = saveNonPickleableValues()
|
||||||
mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals))
|
mpQueue.put((GM.REDIRECT_QUEUE_GLOBALS, GM.Globals))
|
||||||
restoreNonPickleableValues(savedValues)
|
restoreNonPickleableValues(savedValues)
|
||||||
@@ -200,27 +199,27 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
|
|||||||
if data[1] is not None:
|
if data[1] is not None:
|
||||||
_writeData(data[1])
|
_writeData(data[1])
|
||||||
if GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
|
if GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
|
||||||
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _getMain().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
|
_writeData(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'End', _gam().currentISOformatTimeStamp(), data[0], pidData[pid]['cmd']))
|
||||||
fd.flush()
|
fd.flush()
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fdErrorMessage(fd, GM.Globals[stdtype][GM.REDIRECT_NAME], e))
|
||||||
|
|
||||||
if multiprocessing.get_start_method() != 'fork':
|
if multiprocessing.get_start_method() != 'fork':
|
||||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||||
GM.Globals = gmGlobals.copy()
|
GM.Globals = gmGlobals.copy()
|
||||||
GC.Values = gcValues.copy()
|
GC.Values = gcValues.copy()
|
||||||
pid0DataItem = [_getMain().KEYBOARD_INTERRUPT_RC, None]
|
pid0DataItem = [_gam().KEYBOARD_INTERRUPT_RC, None]
|
||||||
pidData = {}
|
pidData = {}
|
||||||
if multiprocessing.get_start_method() != 'fork':
|
if multiprocessing.get_start_method() != 'fork':
|
||||||
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
|
if GM.Globals[stdtype][GM.REDIRECT_NAME] == 'null':
|
||||||
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_getMain().UTF8)
|
fd = open(os.devnull, GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=_gam().UTF8)
|
||||||
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
|
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == '-':
|
||||||
fd = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stdout']),
|
fd = os.fdopen(os.dup([sys.stderr.fileno(), sys.stdout.fileno()][GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stdout']),
|
||||||
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
|
GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
|
||||||
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout' and GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stderr':
|
elif GM.Globals[stdtype][GM.REDIRECT_NAME] == 'stdout' and GM.Globals[stdtype][GM.REDIRECT_QUEUE] == 'stderr':
|
||||||
fd = os.fdopen(os.dup(sys.stdout.fileno()), GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
|
fd = os.fdopen(os.dup(sys.stdout.fileno()), GM.Globals[stdtype][GM.REDIRECT_MODE], encoding=GM.Globals[GM.SYS_ENCODING])
|
||||||
else:
|
else:
|
||||||
fd = _getMain().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
|
fd = _gam().openFile(GM.Globals[stdtype][GM.REDIRECT_NAME], GM.Globals[stdtype][GM.REDIRECT_MODE])
|
||||||
else:
|
else:
|
||||||
fd = GM.Globals[stdtype][GM.REDIRECT_FD]
|
fd = GM.Globals[stdtype][GM.REDIRECT_FD]
|
||||||
while True:
|
while True:
|
||||||
@@ -230,7 +229,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
|
|||||||
break
|
break
|
||||||
if dataType == GM.REDIRECT_QUEUE_START:
|
if dataType == GM.REDIRECT_QUEUE_START:
|
||||||
pidData[pid] = {'queue': GM.Globals[stdtype][GM.REDIRECT_QUEUE],
|
pidData[pid] = {'queue': GM.Globals[stdtype][GM.REDIRECT_QUEUE],
|
||||||
'start': _getMain().currentISOformatTimeStamp(),
|
'start': _gam().currentISOformatTimeStamp(),
|
||||||
'cmd': Cmd.QuotedArgumentList(dataItem)}
|
'cmd': Cmd.QuotedArgumentList(dataItem)}
|
||||||
if pid == 0 and GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
|
if pid == 0 and GC.Values[GC.SHOW_MULTIPROCESS_INFO]:
|
||||||
fd.write(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'Start', pidData[pid]['start'], 0, pidData[pid]['cmd']))
|
fd.write(PROCESS_MSG.format(pidData[pid]['queue'], pid, 'Start', pidData[pid]['start'], 0, pidData[pid]['cmd']))
|
||||||
@@ -246,7 +245,7 @@ def StdQueueHandler(mpQueue, stdtype, gmGlobals, gcValues):
|
|||||||
break
|
break
|
||||||
for pid in pidData:
|
for pid in pidData:
|
||||||
if pid != 0:
|
if pid != 0:
|
||||||
_writePidData(pid, [_getMain().KEYBOARD_INTERRUPT_RC, None])
|
_writePidData(pid, [_gam().KEYBOARD_INTERRUPT_RC, None])
|
||||||
_writePidData(0, pid0DataItem)
|
_writePidData(0, pid0DataItem)
|
||||||
if fd not in [sys.stdout, sys.stderr]:
|
if fd not in [sys.stdout, sys.stderr]:
|
||||||
try:
|
try:
|
||||||
@@ -267,7 +266,7 @@ def batchWriteStderr(data):
|
|||||||
sys.stderr.write(data)
|
sys.stderr.write(data)
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage('stderr', e))
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage('stderr', e))
|
||||||
|
|
||||||
def writeStdQueueHandler(mpQueue, item):
|
def writeStdQueueHandler(mpQueue, item):
|
||||||
while True:
|
while True:
|
||||||
@@ -276,7 +275,7 @@ def writeStdQueueHandler(mpQueue, item):
|
|||||||
return
|
return
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{item[0]}/{GM.Globals[GM.NUM_BATCH_ITEMS]},Error,{str(e)}\n')
|
||||||
|
|
||||||
def terminateStdQueueHandler(mpQueue, mpQueueHandler):
|
def terminateStdQueueHandler(mpQueue, mpQueueHandler):
|
||||||
mpQueue.put((0, GM.REDIRECT_QUEUE_EOF, None))
|
mpQueue.put((0, GM.REDIRECT_QUEUE_EOF, None))
|
||||||
@@ -339,7 +338,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
|
|||||||
if mpQueueCSVFile:
|
if mpQueueCSVFile:
|
||||||
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = mpQueueCSVFile
|
GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = mpQueueCSVFile
|
||||||
if mpQueueStdout:
|
if mpQueueStdout:
|
||||||
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _getMain().StringIOobject()}
|
GM.Globals[GM.STDOUT] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
|
||||||
if debugLevel:
|
if debugLevel:
|
||||||
sys.stdout = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
|
sys.stdout = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
|
||||||
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_START, args))
|
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_START, args))
|
||||||
@@ -348,14 +347,14 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout,
|
|||||||
GM.Globals[GM.STDOUT] = {}
|
GM.Globals[GM.STDOUT] = {}
|
||||||
if mpQueueStderr:
|
if mpQueueStderr:
|
||||||
if mpQueueStderr is not mpQueueStdout:
|
if mpQueueStderr is not mpQueueStdout:
|
||||||
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _getMain().StringIOobject()}
|
GM.Globals[GM.STDERR] = {GM.REDIRECT_NAME: '', GM.REDIRECT_FD: None, GM.REDIRECT_MULTI_FD: _gam().StringIOobject()}
|
||||||
# mpQueueStderr.put((pid, GM.REDIRECT_QUEUE_START, args))
|
# mpQueueStderr.put((pid, GM.REDIRECT_QUEUE_START, args))
|
||||||
writeStdQueueHandler(mpQueueStderr, (pid, GM.REDIRECT_QUEUE_START, args))
|
writeStdQueueHandler(mpQueueStderr, (pid, GM.REDIRECT_QUEUE_START, args))
|
||||||
else:
|
else:
|
||||||
GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
|
GM.Globals[GM.STDERR][GM.REDIRECT_MULTI_FD] = GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]
|
||||||
else:
|
else:
|
||||||
GM.Globals[GM.STDERR] = {}
|
GM.Globals[GM.STDERR] = {}
|
||||||
sysRC = _getMain().ProcessGAMCommand(args)
|
sysRC = _gam().ProcessGAMCommand(args)
|
||||||
with mplock:
|
with mplock:
|
||||||
if mpQueueStdout:
|
if mpQueueStdout:
|
||||||
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_END, [sysRC, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()]))
|
# mpQueueStdout.put((pid, GM.REDIRECT_QUEUE_END, [sysRC, GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD].getvalue()]))
|
||||||
@@ -405,9 +404,9 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
def poolCallback(result):
|
def poolCallback(result):
|
||||||
poolProcessResults[0] -= 1
|
poolProcessResults[0] -= 1
|
||||||
if showCmds:
|
if showCmds:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{result[0]}/{numItems},End,{result[1]},{result[2]}\n')
|
||||||
if GM.Globals[GM.CMDLOG_LOGGER]:
|
if GM.Globals[GM.CMDLOG_LOGGER]:
|
||||||
GM.Globals[GM.CMDLOG_LOGGER].info(f'{_getMain().currentISOformatTimeStamp()},{result[1]},{result[2]}')
|
GM.Globals[GM.CMDLOG_LOGGER].info(f'{_gam().currentISOformatTimeStamp()},{result[1]},{result[2]}')
|
||||||
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(result[1]):
|
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(result[1]):
|
||||||
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
|
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
|
||||||
|
|
||||||
@@ -418,8 +417,8 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
def handleControlC(source):
|
def handleControlC(source):
|
||||||
nonlocal controlC
|
nonlocal controlC
|
||||||
batchWriteStderr(f'Control-C (Multiprocess-{source})\n')
|
batchWriteStderr(f'Control-C (Multiprocess-{source})\n')
|
||||||
_getMain().setSysExitRC(_getMain().KEYBOARD_INTERRUPT_RC)
|
_gam().setSysExitRC(_gam().KEYBOARD_INTERRUPT_RC)
|
||||||
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, poolProcessResults[0],
|
numItems, poolProcessResults[0],
|
||||||
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
|
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
@@ -445,21 +444,21 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
else:
|
else:
|
||||||
pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
|
pool = multiprocessing.Pool(processes=numPoolProcesses, initializer=initGamWorker, initargs=(l,), maxtasksperchild=200)
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, e)
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, e)
|
||||||
except AssertionError as e:
|
except AssertionError as e:
|
||||||
_getMain().Cmd.SetLocation(0)
|
_gam().Cmd.SetLocation(0)
|
||||||
_getMain().usageErrorExit(str(e))
|
_gam().usageErrorExit(str(e))
|
||||||
if multiprocessing.get_start_method() != 'fork':
|
if multiprocessing.get_start_method() != 'fork':
|
||||||
savedValues = saveNonPickleableValues()
|
savedValues = saveNonPickleableValues()
|
||||||
if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]:
|
if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTIPROCESS]:
|
||||||
mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values)
|
mpQueueStdout, mpQueueHandlerStdout = initializeStdQueueHandler(mpManager, GM.STDOUT, GM.Globals, GC.Values)
|
||||||
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _getMain().Cmd.AllArguments()))
|
mpQueueStdout.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
|
||||||
else:
|
else:
|
||||||
mpQueueStdout = None
|
mpQueueStdout = None
|
||||||
if GM.Globals[GM.STDERR][GM.REDIRECT_MULTIPROCESS]:
|
if GM.Globals[GM.STDERR][GM.REDIRECT_MULTIPROCESS]:
|
||||||
if GM.Globals[GM.STDERR][GM.REDIRECT_NAME] != 'stdout':
|
if GM.Globals[GM.STDERR][GM.REDIRECT_NAME] != 'stdout':
|
||||||
mpQueueStderr, mpQueueHandlerStderr = initializeStdQueueHandler(mpManager, GM.STDERR, GM.Globals, GC.Values)
|
mpQueueStderr, mpQueueHandlerStderr = initializeStdQueueHandler(mpManager, GM.STDERR, GM.Globals, GC.Values)
|
||||||
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _getMain().Cmd.AllArguments()))
|
mpQueueStderr.put((0, GM.REDIRECT_QUEUE_START, _gam().Cmd.AllArguments()))
|
||||||
else:
|
else:
|
||||||
mpQueueStderr = mpQueueStdout
|
mpQueueStderr = mpQueueStdout
|
||||||
else:
|
else:
|
||||||
@@ -479,7 +478,7 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
# signal.signal(signal.SIGINT, origSigintHandler)
|
# signal.signal(signal.SIGINT, origSigintHandler)
|
||||||
controlC = False
|
controlC = False
|
||||||
signal.signal(signal.SIGINT, signal_handler)
|
signal.signal(signal.SIGINT, signal_handler)
|
||||||
batchWriteStderr(Msg.USING_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, numPoolProcesses,
|
numItems, numPoolProcesses,
|
||||||
PROCESS_PLURAL_SINGULAR[numPoolProcesses == 1]))
|
PROCESS_PLURAL_SINGULAR[numPoolProcesses == 1]))
|
||||||
try:
|
try:
|
||||||
@@ -491,7 +490,7 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
if controlC:
|
if controlC:
|
||||||
break
|
break
|
||||||
if item[0] == Cmd.COMMIT_BATCH_CMD:
|
if item[0] == Cmd.COMMIT_BATCH_CMD:
|
||||||
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, poolProcessResults[0],
|
numItems, poolProcessResults[0],
|
||||||
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
|
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
|
||||||
while poolProcessResults[0] > 0:
|
while poolProcessResults[0] > 0:
|
||||||
@@ -503,24 +502,24 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
completedProcesses.append(p)
|
completedProcesses.append(p)
|
||||||
for p in completedProcesses:
|
for p in completedProcesses:
|
||||||
del poolProcessResults[p]
|
del poolProcessResults[p]
|
||||||
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
|
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.PROCESSES))
|
||||||
if len(item) > 1:
|
if len(item) > 1:
|
||||||
_getMain().readStdin(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
|
_gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
|
||||||
continue
|
continue
|
||||||
if item[0] == Cmd.PRINT_CMD:
|
if item[0] == Cmd.PRINT_CMD:
|
||||||
batchWriteStderr(Cmd.QuotedArgumentList(item[1:])+'\n')
|
batchWriteStderr(Cmd.QuotedArgumentList(item[1:])+'\n')
|
||||||
continue
|
continue
|
||||||
if item[0] == Cmd.SLEEP_CMD:
|
if item[0] == Cmd.SLEEP_CMD:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleepiing {item[1]} seconds\n')
|
||||||
time.sleep(int(item[1]))
|
time.sleep(int(item[1]))
|
||||||
continue
|
continue
|
||||||
pid += 1
|
pid += 1
|
||||||
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
|
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
|
||||||
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_getMain().currentISOformatTimeStamp(), pid, numItems))
|
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
|
||||||
if showCmds or GM.Globals[GM.CMDLOG_LOGGER]:
|
if showCmds or GM.Globals[GM.CMDLOG_LOGGER]:
|
||||||
logCmd = Cmd.QuotedArgumentList(item)
|
logCmd = Cmd.QuotedArgumentList(item)
|
||||||
if showCmds:
|
if showCmds:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
|
||||||
else:
|
else:
|
||||||
logCmd = ''
|
logCmd = ''
|
||||||
poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti,
|
poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti,
|
||||||
@@ -566,7 +565,7 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
else:
|
else:
|
||||||
waitRemaining = 'unlimited'
|
waitRemaining = 'unlimited'
|
||||||
while poolProcessResults[0] > 0:
|
while poolProcessResults[0] > 0:
|
||||||
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.BATCH_CSV_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, poolProcessResults[0],
|
numItems, poolProcessResults[0],
|
||||||
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1],
|
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1],
|
||||||
Msg.BATCH_CSV_WAIT_LIMIT.format(waitRemaining)))
|
Msg.BATCH_CSV_WAIT_LIMIT.format(waitRemaining)))
|
||||||
@@ -585,7 +584,7 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
if GC.Values[GC.PROCESS_WAIT_LIMIT] > 0:
|
if GC.Values[GC.PROCESS_WAIT_LIMIT] > 0:
|
||||||
delta = int(time.time()-processWaitStart)
|
delta = int(time.time()-processWaitStart)
|
||||||
if delta >= GC.Values[GC.PROCESS_WAIT_LIMIT]:
|
if delta >= GC.Values[GC.PROCESS_WAIT_LIMIT]:
|
||||||
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.BATCH_CSV_TERMINATE_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, poolProcessResults[0],
|
numItems, poolProcessResults[0],
|
||||||
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
|
PROCESS_PLURAL_SINGULAR[poolProcessResults[0] == 1]))
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
@@ -597,7 +596,7 @@ def MultiprocessGAMCommands(items, showCmds):
|
|||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
handleControlC('KBI')
|
handleControlC('KBI')
|
||||||
pool.join()
|
pool.join()
|
||||||
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems))
|
batchWriteStderr(Msg.BATCH_CSV_PROCESSING_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems))
|
||||||
if mpQueueCSVFile:
|
if mpQueueCSVFile:
|
||||||
terminateCSVFileQueueHandler(mpQueueCSVFile, mpQueueHandlerCSVFile)
|
terminateCSVFileQueueHandler(mpQueueCSVFile, mpQueueHandlerCSVFile)
|
||||||
if mpQueueStdout:
|
if mpQueueStdout:
|
||||||
@@ -618,11 +617,11 @@ def threadBatchWorker(showCmds=False, numItems=0):
|
|||||||
sysRC = subprocess.call(item, stdout=GM.Globals[GM.STDOUT].get(GM.REDIRECT_MULTI_FD, sys.stdout),
|
sysRC = subprocess.call(item, stdout=GM.Globals[GM.STDOUT].get(GM.REDIRECT_MULTI_FD, sys.stdout),
|
||||||
stderr=GM.Globals[GM.STDERR].get(GM.REDIRECT_MULTI_FD, sys.stderr))
|
stderr=GM.Globals[GM.STDERR].get(GM.REDIRECT_MULTI_FD, sys.stderr))
|
||||||
if showCmds:
|
if showCmds:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
|
||||||
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(sysRC):
|
if GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] is not None and checkChildProcessRC(sysRC):
|
||||||
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
|
GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING] = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Error,{str(e)},{logCmd}\n')
|
||||||
GM.Globals[GM.TBATCH_QUEUE].task_done()
|
GM.Globals[GM.TBATCH_QUEUE].task_done()
|
||||||
|
|
||||||
BATCH_COMMANDS = [Cmd.GAM_CMD, Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD, Cmd.SLEEP_CMD, Cmd.DATETIME_CMD, Cmd.SET_CMD, Cmd.CLEAR_CMD]
|
BATCH_COMMANDS = [Cmd.GAM_CMD, Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD, Cmd.SLEEP_CMD, Cmd.DATETIME_CMD, Cmd.SET_CMD, Cmd.CLEAR_CMD]
|
||||||
@@ -633,12 +632,12 @@ def ThreadBatchGAMCommands(items, showCmds):
|
|||||||
return
|
return
|
||||||
pythonCmd = [sys.executable]
|
pythonCmd = [sys.executable]
|
||||||
if not getattr(sys, 'frozen', False): # we're not frozen
|
if not getattr(sys, 'frozen', False): # we're not frozen
|
||||||
pythonCmd.append(os.path.realpath(_getMain().Cmd.Argument(0)))
|
pythonCmd.append(os.path.realpath(_gam().Cmd.Argument(0)))
|
||||||
GM.Globals[GM.NUM_BATCH_ITEMS] = numItems = len(items)
|
GM.Globals[GM.NUM_BATCH_ITEMS] = numItems = len(items)
|
||||||
numWorkerThreads = min(numItems, GC.Values[GC.NUM_TBATCH_THREADS])
|
numWorkerThreads = min(numItems, GC.Values[GC.NUM_TBATCH_THREADS])
|
||||||
# GM.Globals[GM.TBATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
|
# GM.Globals[GM.TBATCH_QUEUE].put() gets blocked when trying to create more items than there are workers
|
||||||
GM.Globals[GM.TBATCH_QUEUE] = queue.Queue(maxsize=numWorkerThreads)
|
GM.Globals[GM.TBATCH_QUEUE] = queue.Queue(maxsize=numWorkerThreads)
|
||||||
batchWriteStderr(Msg.USING_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.USING_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, numWorkerThreads,
|
numItems, numWorkerThreads,
|
||||||
THREAD_PLURAL_SINGULAR[numWorkerThreads == 1]))
|
THREAD_PLURAL_SINGULAR[numWorkerThreads == 1]))
|
||||||
for _ in range(numWorkerThreads):
|
for _ in range(numWorkerThreads):
|
||||||
@@ -651,28 +650,28 @@ def ThreadBatchGAMCommands(items, showCmds):
|
|||||||
if GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING]:
|
if GM.Globals[GM.MULTIPROCESS_EXIT_PROCESSING]:
|
||||||
break
|
break
|
||||||
if item[0] == Cmd.COMMIT_BATCH_CMD:
|
if item[0] == Cmd.COMMIT_BATCH_CMD:
|
||||||
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_getMain().currentISOformatTimeStamp(),
|
batchWriteStderr(Msg.COMMIT_BATCH_WAIT_N_PROCESSES.format(_gam().currentISOformatTimeStamp(),
|
||||||
numItems, numThreadsInUse,
|
numItems, numThreadsInUse,
|
||||||
THREAD_PLURAL_SINGULAR[numThreadsInUse == 1]))
|
THREAD_PLURAL_SINGULAR[numThreadsInUse == 1]))
|
||||||
GM.Globals[GM.TBATCH_QUEUE].join()
|
GM.Globals[GM.TBATCH_QUEUE].join()
|
||||||
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_getMain().currentISOformatTimeStamp(), numItems, Msg.THREADS))
|
batchWriteStderr(Msg.COMMIT_BATCH_COMPLETE.format(_gam().currentISOformatTimeStamp(), numItems, Msg.THREADS))
|
||||||
numThreadsInUse = 0
|
numThreadsInUse = 0
|
||||||
if len(item) > 1:
|
if len(item) > 1:
|
||||||
_getMain().readStdin(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
|
_gam().readStdin(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}')
|
||||||
continue
|
continue
|
||||||
if item[0] == Cmd.PRINT_CMD:
|
if item[0] == Cmd.PRINT_CMD:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},{Cmd.QuotedArgumentList(item[1:])}\n')
|
||||||
continue
|
continue
|
||||||
if item[0] == Cmd.SLEEP_CMD:
|
if item[0] == Cmd.SLEEP_CMD:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Sleeping {item[1]} seconds\n')
|
||||||
time.sleep(int(item[1]))
|
time.sleep(int(item[1]))
|
||||||
continue
|
continue
|
||||||
pid += 1
|
pid += 1
|
||||||
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
|
if not showCmds and ((pid % 100 == 0) or (pid == numItems)):
|
||||||
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_getMain().currentISOformatTimeStamp(), pid, numItems))
|
batchWriteStderr(Msg.PROCESSING_ITEM_N_OF_M.format(_gam().currentISOformatTimeStamp(), pid, numItems))
|
||||||
if showCmds:
|
if showCmds:
|
||||||
logCmd = Cmd.QuotedArgumentList(item)
|
logCmd = Cmd.QuotedArgumentList(item)
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,{Cmd.QuotedArgumentList(item)}\n')
|
||||||
else:
|
else:
|
||||||
logCmd = ''
|
logCmd = ''
|
||||||
if item[0] == Cmd.GAM_CMD:
|
if item[0] == Cmd.GAM_CMD:
|
||||||
@@ -682,43 +681,43 @@ def ThreadBatchGAMCommands(items, showCmds):
|
|||||||
numThreadsInUse += 1
|
numThreadsInUse += 1
|
||||||
GM.Globals[GM.TBATCH_QUEUE].join()
|
GM.Globals[GM.TBATCH_QUEUE].join()
|
||||||
if showCmds:
|
if showCmds:
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},0/{numItems},Complete\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},0/{numItems},Complete\n')
|
||||||
|
|
||||||
def _getShowCommands():
|
def _getShowCommands():
|
||||||
if _getMain().checkArgumentPresent('showcmds'):
|
if _gam().checkArgumentPresent('showcmds'):
|
||||||
return _getMain().getBoolean()
|
return _gam().getBoolean()
|
||||||
return GC.Values[GC.SHOW_COMMANDS]
|
return GC.Values[GC.SHOW_COMMANDS]
|
||||||
|
|
||||||
def _getSkipRows():
|
def _getSkipRows():
|
||||||
if _getMain().checkArgumentPresent('skiprows'):
|
if _gam().checkArgumentPresent('skiprows'):
|
||||||
return _getMain().getInteger(minVal=0)
|
return _gam().getInteger(minVal=0)
|
||||||
# return GC.Values[GC.CSV_INPUT_ROW_SKIP]
|
# return GC.Values[GC.CSV_INPUT_ROW_SKIP]
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
def _getMaxRows():
|
def _getMaxRows():
|
||||||
if _getMain().checkArgumentPresent('maxrows'):
|
if _gam().checkArgumentPresent('maxrows'):
|
||||||
return _getMain().getInteger(minVal=0)
|
return _gam().getInteger(minVal=0)
|
||||||
return GC.Values[GC.CSV_INPUT_ROW_LIMIT]
|
return GC.Values[GC.CSV_INPUT_ROW_LIMIT]
|
||||||
|
|
||||||
# gam batch <BatchContent> [showcmds [<Boolean>]]
|
# gam batch <BatchContent> [showcmds [<Boolean>]]
|
||||||
def doBatch(threadBatch=False):
|
def doBatch(threadBatch=False):
|
||||||
filename = _getMain().getString(Cmd.OB_FILE_NAME)
|
filename = _gam().getString(Cmd.OB_FILE_NAME)
|
||||||
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
|
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
|
||||||
_getMain().Cmd.Backup()
|
_gam().Cmd.Backup()
|
||||||
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
|
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.BATCH_CMD))
|
||||||
filenameLower = filename.lower()
|
filenameLower = filename.lower()
|
||||||
if filenameLower not in {'gdoc', 'gcsdoc'}:
|
if filenameLower not in {'gdoc', 'gcsdoc'}:
|
||||||
encoding = _getMain().getCharSet()
|
encoding = _gam().getCharSet()
|
||||||
filename = _getMain().setFilePath(filename, GC.INPUT_DIR)
|
filename = _gam().setFilePath(filename, GC.INPUT_DIR)
|
||||||
f = _getMain().openFile(filename, encoding=encoding, stripUTFBOM=True)
|
f = _gam().openFile(filename, encoding=encoding, stripUTFBOM=True)
|
||||||
elif filenameLower == 'gdoc':
|
elif filenameLower == 'gdoc':
|
||||||
f = _getMain().getGDocData(filenameLower)
|
f = _gam().getGDocData(filenameLower)
|
||||||
_getMain().getCharSet()
|
_gam().getCharSet()
|
||||||
else: #filenameLower == 'gcsdoc':
|
else: #filenameLower == 'gcsdoc':
|
||||||
f = _getMain().getStorageFileData(filenameLower)
|
f = _gam().getStorageFileData(filenameLower)
|
||||||
_getMain().getCharSet()
|
_gam().getCharSet()
|
||||||
showCmds = _getShowCommands()
|
showCmds = _getShowCommands()
|
||||||
_getMain().checkForExtraneousArguments()
|
_gam().checkForExtraneousArguments()
|
||||||
validCommands = BATCH_COMMANDS if not threadBatch else TBATCH_COMMANDS
|
validCommands = BATCH_COMMANDS if not threadBatch else TBATCH_COMMANDS
|
||||||
kwValues = {}
|
kwValues = {}
|
||||||
items = []
|
items = []
|
||||||
@@ -733,40 +732,40 @@ def doBatch(threadBatch=False):
|
|||||||
try:
|
try:
|
||||||
argv = shlex.split(line)
|
argv = shlex.split(line)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
_getMain().writeStderr(f'Command: >>>{line.strip()}<<<\n')
|
_gam().writeStderr(f'Command: >>>{line.strip()}<<<\n')
|
||||||
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{str(e)}\n')
|
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{str(e)}\n')
|
||||||
errors += 1
|
errors += 1
|
||||||
continue
|
continue
|
||||||
if argv:
|
if argv:
|
||||||
cmd = argv[0].strip().lower()
|
cmd = argv[0].strip().lower()
|
||||||
if cmd == Cmd.DATETIME_CMD:
|
if cmd == Cmd.DATETIME_CMD:
|
||||||
if len(argv) == 2:
|
if len(argv) == 2:
|
||||||
kwValues['datetime'] = _getMain().todaysTime().strftime(argv[1])
|
kwValues['datetime'] = _gam().todaysTime().strftime(argv[1])
|
||||||
else:
|
else:
|
||||||
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
||||||
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
|
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.DATETIME_CMD} DateTimeFormat>)>\n')
|
||||||
errors += 1
|
errors += 1
|
||||||
continue
|
continue
|
||||||
if cmd == Cmd.SET_CMD:
|
if cmd == Cmd.SET_CMD:
|
||||||
if len(argv) == 3:
|
if len(argv) == 3:
|
||||||
kwValues[argv[1]] = argv[2]
|
kwValues[argv[1]] = argv[2]
|
||||||
else:
|
else:
|
||||||
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
||||||
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
|
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SET_CMD} keyword value>)>\n')
|
||||||
errors += 1
|
errors += 1
|
||||||
continue
|
continue
|
||||||
if cmd == Cmd.CLEAR_CMD:
|
if cmd == Cmd.CLEAR_CMD:
|
||||||
if len(argv) == 2:
|
if len(argv) == 2:
|
||||||
kwValues.pop(argv[1], None)
|
kwValues.pop(argv[1], None)
|
||||||
else:
|
else:
|
||||||
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
||||||
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
|
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.CLEAR_CMD} keyword>)>\n')
|
||||||
errors += 1
|
errors += 1
|
||||||
continue
|
continue
|
||||||
if cmd == Cmd.SLEEP_CMD:
|
if cmd == Cmd.SLEEP_CMD:
|
||||||
if len(argv) != 2 or not argv[1].isdigit():
|
if len(argv) != 2 or not argv[1].isdigit():
|
||||||
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
||||||
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
|
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{Cmd.SLEEP_CMD} integer>)>\n')
|
||||||
errors += 1
|
errors += 1
|
||||||
continue
|
continue
|
||||||
if (not cmd) or ((len(argv) == 1) and (cmd not in [Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD])):
|
if (not cmd) or ((len(argv) == 1) and (cmd not in [Cmd.COMMIT_BATCH_CMD, Cmd.PRINT_CMD])):
|
||||||
@@ -774,28 +773,28 @@ def doBatch(threadBatch=False):
|
|||||||
if cmd in validCommands:
|
if cmd in validCommands:
|
||||||
items.append(argv)
|
items.append(argv)
|
||||||
else:
|
else:
|
||||||
_getMain().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
_gam().writeStderr(f'Command: >>>{Cmd.QuotedArgumentList([argv[0]])}<<< {Cmd.QuotedArgumentList(argv[1:])}\n')
|
||||||
_getMain().writeStderr(f'{_getMain().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_getMain().formatChoiceList(validCommands)}>\n')
|
_gam().writeStderr(f'{_gam().ERROR_PREFIX}{Cmd.ARGUMENT_ERROR_NAMES[Cmd.ARGUMENT_INVALID][1]}: {Msg.EXPECTED} <{_gam().formatChoiceList(validCommands)}>\n')
|
||||||
errors += 1
|
errors += 1
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage(filename, e))
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
|
||||||
_getMain().closeFile(f)
|
_gam().closeFile(f)
|
||||||
if errors == 0:
|
if errors == 0:
|
||||||
if not threadBatch:
|
if not threadBatch:
|
||||||
MultiprocessGAMCommands(items, showCmds)
|
MultiprocessGAMCommands(items, showCmds)
|
||||||
else:
|
else:
|
||||||
ThreadBatchGAMCommands(items, showCmds)
|
ThreadBatchGAMCommands(items, showCmds)
|
||||||
else:
|
else:
|
||||||
_getMain().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_getMain().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
|
_gam().writeStderr(Msg.BATCH_NOT_PROCESSED_ERRORS.format(_gam().ERROR_PREFIX, filename, errors, ERROR_PLURAL_SINGULAR[errors == 1]))
|
||||||
_getMain().setSysExitRC(_getMain().USAGE_ERROR_RC)
|
_gam().setSysExitRC(_gam().USAGE_ERROR_RC)
|
||||||
|
|
||||||
# gam tbatch <BatchContent> [showcmds [<Boolean>]]
|
# gam tbatch <BatchContent> [showcmds [<Boolean>]]
|
||||||
def doThreadBatch():
|
def doThreadBatch():
|
||||||
_getMain().adjustRedirectedSTDFilesIfNotMultiprocessing()
|
_gam().adjustRedirectedSTDFilesIfNotMultiprocessing()
|
||||||
doBatch(True)
|
doBatch(True)
|
||||||
|
|
||||||
def doAutoBatch(entityType, entityList, CL_command):
|
def doAutoBatch(entityType, entityList, CL_command):
|
||||||
remaining = _getMain().Cmd.Remaining()
|
remaining = _gam().Cmd.Remaining()
|
||||||
items = []
|
items = []
|
||||||
initial_argv = [Cmd.GAM_CMD]
|
initial_argv = [Cmd.GAM_CMD]
|
||||||
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION]:
|
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION]:
|
||||||
@@ -824,8 +823,8 @@ def getSubFields(initial_argv, fieldNames):
|
|||||||
subFields = {}
|
subFields = {}
|
||||||
GAM_argv = initial_argv[:]
|
GAM_argv = initial_argv[:]
|
||||||
GAM_argvI = len(GAM_argv)
|
GAM_argvI = len(GAM_argv)
|
||||||
while _getMain().Cmd.ArgumentsRemaining():
|
while _gam().Cmd.ArgumentsRemaining():
|
||||||
myarg = _getMain().Cmd.Current()
|
myarg = _gam().Cmd.Current()
|
||||||
if not myarg:
|
if not myarg:
|
||||||
GAM_argv.append(myarg)
|
GAM_argv.append(myarg)
|
||||||
elif SUB_PATTERN.search(myarg):
|
elif SUB_PATTERN.search(myarg):
|
||||||
@@ -839,17 +838,17 @@ def getSubFields(initial_argv, fieldNames):
|
|||||||
if not rematch:
|
if not rematch:
|
||||||
fieldName = submatch.group(1)
|
fieldName = submatch.group(1)
|
||||||
if fieldName not in fieldNames:
|
if fieldName not in fieldNames:
|
||||||
_getMain().csvFieldErrorExit(fieldName, fieldNames)
|
_gam().csvFieldErrorExit(fieldName, fieldNames)
|
||||||
subFields[GAM_argvI].append((SUB_TYPE, fieldName, submatch.start(), submatch.end()))
|
subFields[GAM_argvI].append((SUB_TYPE, fieldName, submatch.start(), submatch.end()))
|
||||||
else:
|
else:
|
||||||
fieldName = rematch.group(1)
|
fieldName = rematch.group(1)
|
||||||
if fieldName not in fieldNames:
|
if fieldName not in fieldNames:
|
||||||
_getMain().csvFieldErrorExit(fieldName, fieldNames)
|
_gam().csvFieldErrorExit(fieldName, fieldNames)
|
||||||
try:
|
try:
|
||||||
re.compile(rematch.group(2))
|
re.compile(rematch.group(2))
|
||||||
subFields[GAM_argvI].append((RE_TYPE, fieldName, submatch.start(), submatch.end(), rematch.group(2), rematch.group(3)))
|
subFields[GAM_argvI].append((RE_TYPE, fieldName, submatch.start(), submatch.end(), rematch.group(2), rematch.group(3)))
|
||||||
except re.error as e:
|
except re.error as e:
|
||||||
_getMain().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
|
_gam().usageErrorExit(f'{Cmd.OB_RE_PATTERN} {Msg.ERROR}: {e}')
|
||||||
pos = submatch.end()
|
pos = submatch.end()
|
||||||
GAM_argv.append(myarg)
|
GAM_argv.append(myarg)
|
||||||
elif myarg[0] == '~':
|
elif myarg[0] == '~':
|
||||||
@@ -858,11 +857,11 @@ def getSubFields(initial_argv, fieldNames):
|
|||||||
subFields[GAM_argvI] = [(SUB_TYPE, fieldName, 0, len(myarg))]
|
subFields[GAM_argvI] = [(SUB_TYPE, fieldName, 0, len(myarg))]
|
||||||
GAM_argv.append(myarg)
|
GAM_argv.append(myarg)
|
||||||
else:
|
else:
|
||||||
_getMain().csvFieldErrorExit(fieldName, fieldNames)
|
_gam().csvFieldErrorExit(fieldName, fieldNames)
|
||||||
else:
|
else:
|
||||||
GAM_argv.append(myarg)
|
GAM_argv.append(myarg)
|
||||||
GAM_argvI += 1
|
GAM_argvI += 1
|
||||||
_getMain().Cmd.Advance()
|
_gam().Cmd.Advance()
|
||||||
return(GAM_argv, subFields)
|
return(GAM_argv, subFields)
|
||||||
|
|
||||||
def processSubFields(GAM_argv, row, subFields):
|
def processSubFields(GAM_argv, row, subFields):
|
||||||
@@ -889,28 +888,28 @@ def processSubFields(GAM_argv, row, subFields):
|
|||||||
# [skiprows <Integer>] [maxrows <Integer>]
|
# [skiprows <Integer>] [maxrows <Integer>]
|
||||||
# gam <GAM argument list>
|
# gam <GAM argument list>
|
||||||
def doCSV(testMode=False):
|
def doCSV(testMode=False):
|
||||||
filename = _getMain().getString(Cmd.OB_FILE_NAME)
|
filename = _gam().getString(Cmd.OB_FILE_NAME)
|
||||||
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
|
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
|
||||||
_getMain().Cmd.Backup()
|
_gam().Cmd.Backup()
|
||||||
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
|
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.CSV_CMD))
|
||||||
f, csvFile, fieldnames = _getMain().openCSVFileReader(filename)
|
f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
|
||||||
matchFields, skipFields = _getMain().getMatchSkipFields(fieldnames)
|
matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
|
||||||
showCmds = _getShowCommands()
|
showCmds = _getShowCommands()
|
||||||
skipRows = _getSkipRows()
|
skipRows = _getSkipRows()
|
||||||
maxRows = _getMaxRows()
|
maxRows = _getMaxRows()
|
||||||
_getMain().checkArgumentPresent(Cmd.GAM_CMD, required=True)
|
_gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
|
||||||
if not _getMain().Cmd.ArgumentsRemaining():
|
if not _gam().Cmd.ArgumentsRemaining():
|
||||||
_getMain().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
|
_gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
|
||||||
initial_argv = [Cmd.GAM_CMD]
|
initial_argv = [Cmd.GAM_CMD]
|
||||||
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _getMain().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
|
if GM.Globals[GM.SECTION] and not GM.Globals[GM.GAM_CFG_SECTION] and not _gam().Cmd.PeekArgumentPresent(Cmd.SELECT_CMD):
|
||||||
initial_argv.extend([Cmd.SELECT_CMD, GM.Globals[GM.SECTION]])
|
initial_argv.extend([Cmd.SELECT_CMD, GM.Globals[GM.SECTION]])
|
||||||
GAM_argv, subFields = getSubFields(initial_argv, fieldnames)
|
GAM_argv, subFields = getSubFields(initial_argv, fieldnames)
|
||||||
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
|
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
|
||||||
_getMain().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
|
_gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
|
||||||
items = []
|
items = []
|
||||||
i = 0
|
i = 0
|
||||||
for row in csvFile:
|
for row in csvFile:
|
||||||
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
|
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
|
||||||
i += 1
|
i += 1
|
||||||
if skipRows:
|
if skipRows:
|
||||||
if i <= skipRows:
|
if i <= skipRows:
|
||||||
@@ -920,21 +919,21 @@ def doCSV(testMode=False):
|
|||||||
items.append(processSubFields(GAM_argv, row, subFields))
|
items.append(processSubFields(GAM_argv, row, subFields))
|
||||||
if maxRows and i >= maxRows:
|
if maxRows and i >= maxRows:
|
||||||
break
|
break
|
||||||
_getMain().closeFile(f)
|
_gam().closeFile(f)
|
||||||
if not testMode:
|
if not testMode:
|
||||||
MultiprocessGAMCommands(items, showCmds)
|
MultiprocessGAMCommands(items, showCmds)
|
||||||
else:
|
else:
|
||||||
numItems = min(len(items), 10)
|
numItems = min(len(items), 10)
|
||||||
_getMain().writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
|
_gam().writeStdout(Msg.CSV_FILE_HEADERS.format(filename))
|
||||||
_getMain().Ind.Increment()
|
_gam().Ind.Increment()
|
||||||
for field in fieldnames:
|
for field in fieldnames:
|
||||||
_getMain().writeStdout(f'{_getMain().Ind.Spaces()}{field}\n')
|
_gam().writeStdout(f'{_gam().Ind.Spaces()}{field}\n')
|
||||||
_getMain().Ind.Decrement()
|
_gam().Ind.Decrement()
|
||||||
_getMain().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _getMain().GAM))
|
_gam().writeStdout(Msg.CSV_SAMPLE_COMMANDS.format(numItems, _gam().GAM))
|
||||||
_getMain().Ind.Increment()
|
_gam().Ind.Increment()
|
||||||
for i in range(numItems):
|
for i in range(numItems):
|
||||||
_getMain().writeStdout(f'{_getMain().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
|
_gam().writeStdout(f'{_gam().Ind.Spaces()}{Cmd.QuotedArgumentList(items[i])}\n')
|
||||||
_getMain().Ind.Decrement()
|
_gam().Ind.Decrement()
|
||||||
|
|
||||||
def doCSVTest():
|
def doCSVTest():
|
||||||
doCSV(testMode=True)
|
doCSV(testMode=True)
|
||||||
@@ -945,23 +944,23 @@ def doCSVTest():
|
|||||||
# [skiprows <Integer>] [maxrows <Integer>]
|
# [skiprows <Integer>] [maxrows <Integer>]
|
||||||
# gam <GAM argument list>
|
# gam <GAM argument list>
|
||||||
def doLoop(loopCmd):
|
def doLoop(loopCmd):
|
||||||
filename = _getMain().getString(Cmd.OB_FILE_NAME)
|
filename = _gam().getString(Cmd.OB_FILE_NAME)
|
||||||
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
|
if (filename == '-') and (GC.Values[GC.DEBUG_LEVEL] > 0):
|
||||||
_getMain().Cmd.Backup()
|
_gam().Cmd.Backup()
|
||||||
_getMain().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
|
_gam().usageErrorExit(Msg.BATCH_CSV_LOOP_DASH_DEBUG_INCOMPATIBLE.format(Cmd.LOOP_CMD))
|
||||||
f, csvFile, fieldnames = _getMain().openCSVFileReader(filename)
|
f, csvFile, fieldnames = _gam().openCSVFileReader(filename)
|
||||||
matchFields, skipFields = _getMain().getMatchSkipFields(fieldnames)
|
matchFields, skipFields = _gam().getMatchSkipFields(fieldnames)
|
||||||
showCmds = _getShowCommands()
|
showCmds = _getShowCommands()
|
||||||
skipRows = _getSkipRows()
|
skipRows = _getSkipRows()
|
||||||
maxRows = _getMaxRows()
|
maxRows = _getMaxRows()
|
||||||
_getMain().checkArgumentPresent(Cmd.GAM_CMD, required=True)
|
_gam().checkArgumentPresent(Cmd.GAM_CMD, required=True)
|
||||||
if not _getMain().Cmd.ArgumentsRemaining():
|
if not _gam().Cmd.ArgumentsRemaining():
|
||||||
_getMain().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
|
_gam().missingArgumentExit(Cmd.OB_GAM_ARGUMENT_LIST)
|
||||||
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
|
if GC.Values[GC.CSV_INPUT_ROW_FILTER] or GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER]:
|
||||||
_getMain().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
|
_gam().CheckInputRowFilterHeaders(fieldnames, GC.Values[GC.CSV_INPUT_ROW_FILTER], GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER])
|
||||||
choice = _getMain().Cmd.Current().strip().lower()
|
choice = _gam().Cmd.Current().strip().lower()
|
||||||
if choice == Cmd.LOOP_CMD:
|
if choice == Cmd.LOOP_CMD:
|
||||||
_getMain().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
|
_gam().usageErrorExit(Msg.NESTED_LOOP_CMD_NOT_ALLOWED)
|
||||||
# gam loop ... gam redirect|select|config ... process gam.cfg on each iteration
|
# gam loop ... gam redirect|select|config ... process gam.cfg on each iteration
|
||||||
# gam redirect|select|config ... loop ... gam redirect|select|config ... process gam.cfg on each iteration
|
# gam redirect|select|config ... loop ... gam redirect|select|config ... process gam.cfg on each iteration
|
||||||
# gam loop ... gam !redirect|select|config ... no further processing of gam.cfg
|
# gam loop ... gam !redirect|select|config ... no further processing of gam.cfg
|
||||||
@@ -981,13 +980,13 @@ def doLoop(loopCmd):
|
|||||||
else:
|
else:
|
||||||
LoopGlobals = {GM.CMDLOG_LOGGER: None, GM.CMDLOG_HANDLER: None}
|
LoopGlobals = {GM.CMDLOG_LOGGER: None, GM.CMDLOG_HANDLER: None}
|
||||||
if (GM.Globals[GM.PID] > 0) and GC.Values[GC.CMDLOG]:
|
if (GM.Globals[GM.PID] > 0) and GC.Values[GC.CMDLOG]:
|
||||||
_getMain().openGAMCommandLog(LoopGlobals, 'looplog')
|
_gam().openGAMCommandLog(LoopGlobals, 'looplog')
|
||||||
if LoopGlobals[GM.CMDLOG_LOGGER]:
|
if LoopGlobals[GM.CMDLOG_LOGGER]:
|
||||||
_getMain().writeGAMCommandLog(LoopGlobals, loopCmd, '*')
|
_gam().writeGAMCommandLog(LoopGlobals, loopCmd, '*')
|
||||||
if not showCmds:
|
if not showCmds:
|
||||||
i = 0
|
i = 0
|
||||||
for row in csvFile:
|
for row in csvFile:
|
||||||
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
|
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
|
||||||
i += 1
|
i += 1
|
||||||
if skipRows:
|
if skipRows:
|
||||||
if i <= skipRows:
|
if i <= skipRows:
|
||||||
@@ -997,20 +996,20 @@ def doLoop(loopCmd):
|
|||||||
item = processSubFields(GAM_argv, row, subFields)
|
item = processSubFields(GAM_argv, row, subFields)
|
||||||
logCmd = Cmd.QuotedArgumentList(item)
|
logCmd = Cmd.QuotedArgumentList(item)
|
||||||
if i % 100 == 0:
|
if i % 100 == 0:
|
||||||
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_getMain().currentISOformatTimeStamp(), i))
|
batchWriteStderr(Msg.PROCESSING_ITEM_N.format(_gam().currentISOformatTimeStamp(), i))
|
||||||
sysRC = _getMain().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
|
sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
|
||||||
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
|
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
|
||||||
_getMain().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
|
_gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
|
||||||
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _getMain().HARD_ERROR_RC):
|
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
|
||||||
break
|
break
|
||||||
if maxRows and i >= maxRows:
|
if maxRows and i >= maxRows:
|
||||||
break
|
break
|
||||||
_getMain().closeFile(f)
|
_gam().closeFile(f)
|
||||||
else:
|
else:
|
||||||
items = []
|
items = []
|
||||||
i = 0
|
i = 0
|
||||||
for row in csvFile:
|
for row in csvFile:
|
||||||
if _getMain().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
|
if _gam().checkMatchSkipFields(row, fieldnames, matchFields, skipFields):
|
||||||
i += 1
|
i += 1
|
||||||
if skipRows:
|
if skipRows:
|
||||||
if i <= skipRows:
|
if i <= skipRows:
|
||||||
@@ -1020,26 +1019,26 @@ def doLoop(loopCmd):
|
|||||||
items.append(processSubFields(GAM_argv, row, subFields))
|
items.append(processSubFields(GAM_argv, row, subFields))
|
||||||
if maxRows and i >= maxRows:
|
if maxRows and i >= maxRows:
|
||||||
break
|
break
|
||||||
_getMain().closeFile(f)
|
_gam().closeFile(f)
|
||||||
numItems = len(items)
|
numItems = len(items)
|
||||||
pid = 0
|
pid = 0
|
||||||
for item in items:
|
for item in items:
|
||||||
pid += 1
|
pid += 1
|
||||||
logCmd = Cmd.QuotedArgumentList(item)
|
logCmd = Cmd.QuotedArgumentList(item)
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},Start,0,{logCmd}\n')
|
||||||
sysRC = _getMain().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
|
sysRC = _gam().ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True)
|
||||||
batchWriteStderr(f'{_getMain().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
|
batchWriteStderr(f'{_gam().currentISOformatTimeStamp()},{pid}/{numItems},End,{sysRC},{logCmd}\n')
|
||||||
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
|
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
|
||||||
_getMain().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
|
_gam().writeGAMCommandLog(LoopGlobals, logCmd, sysRC)
|
||||||
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _getMain().HARD_ERROR_RC):
|
if (sysRC > 0) and (GM.Globals[GM.SYSEXITRC] <= _gam().HARD_ERROR_RC):
|
||||||
break
|
break
|
||||||
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
|
if (GM.Globals[GM.PID] > 0) and LoopGlobals[GM.CMDLOG_LOGGER]:
|
||||||
_getMain().closeGAMCommandLog(LoopGlobals)
|
_gam().closeGAMCommandLog(LoopGlobals)
|
||||||
if multi:
|
if multi:
|
||||||
terminateCSVFileQueueHandler(mpQueue, mpQueueHandler)
|
terminateCSVFileQueueHandler(mpQueue, mpQueueHandler)
|
||||||
|
|
||||||
def _doList(entityList, entityType):
|
def _doList(entityList, entityType):
|
||||||
_getMain().buildGAPIObject(API.DIRECTORY)
|
_gam().buildGAPIObject(API.DIRECTORY)
|
||||||
if GM.Globals[GM.CSV_DATA_DICT]:
|
if GM.Globals[GM.CSV_DATA_DICT]:
|
||||||
keyField = GM.Globals[GM.CSV_KEY_FIELD]
|
keyField = GM.Globals[GM.CSV_KEY_FIELD]
|
||||||
dataField = GM.Globals[GM.CSV_DATA_FIELD]
|
dataField = GM.Globals[GM.CSV_DATA_FIELD]
|
||||||
@@ -1047,26 +1046,26 @@ def _doList(entityList, entityType):
|
|||||||
keyField = 'Entity'
|
keyField = 'Entity'
|
||||||
dataField = 'Data'
|
dataField = 'Data'
|
||||||
csvPF = CSVPrintFile(keyField)
|
csvPF = CSVPrintFile(keyField)
|
||||||
if _getMain().checkArgumentPresent('todrive'):
|
if _gam().checkArgumentPresent('todrive'):
|
||||||
csvPF.GetTodriveParameters()
|
csvPF.GetTodriveParameters()
|
||||||
if entityList is None:
|
if entityList is None:
|
||||||
entityList = _getMain().getEntityList(Cmd.OB_ENTITY)
|
entityList = _gam().getEntityList(Cmd.OB_ENTITY)
|
||||||
showData = _getMain().checkArgumentPresent('data')
|
showData = _gam().checkArgumentPresent('data')
|
||||||
if showData:
|
if showData:
|
||||||
if not entityType:
|
if not entityType:
|
||||||
itemType, itemList = _getMain().getEntityToModify(crosAllowed=True)
|
itemType, itemList = _gam().getEntityToModify(crosAllowed=True)
|
||||||
else:
|
else:
|
||||||
itemType = None
|
itemType = None
|
||||||
itemList = _getMain().getEntityList(Cmd.OB_ENTITY)
|
itemList = _gam().getEntityList(Cmd.OB_ENTITY)
|
||||||
entityItemLists = itemList if isinstance(itemList, dict) else None
|
entityItemLists = itemList if isinstance(itemList, dict) else None
|
||||||
csvPF.AddTitle(dataField)
|
csvPF.AddTitle(dataField)
|
||||||
else:
|
else:
|
||||||
entityItemLists = None
|
entityItemLists = None
|
||||||
dataDelimiter = _getMain().getDelimiter()
|
dataDelimiter = _gam().getDelimiter()
|
||||||
_getMain().checkForExtraneousArguments()
|
_gam().checkForExtraneousArguments()
|
||||||
_, _, entityList = _getMain().getEntityArgument(entityList)
|
_, _, entityList = _gam().getEntityArgument(entityList)
|
||||||
for entity in entityList:
|
for entity in entityList:
|
||||||
entityEmail = _getMain().normalizeEmailAddressOrUID(entity)
|
entityEmail = _gam().normalizeEmailAddressOrUID(entity)
|
||||||
if showData:
|
if showData:
|
||||||
if entityItemLists:
|
if entityItemLists:
|
||||||
if entity not in entityItemLists:
|
if entity not in entityItemLists:
|
||||||
@@ -1075,7 +1074,7 @@ def _doList(entityList, entityType):
|
|||||||
itemList = entityItemLists[entity]
|
itemList = entityItemLists[entity]
|
||||||
if itemType == Cmd.ENTITY_USERS:
|
if itemType == Cmd.ENTITY_USERS:
|
||||||
for i, item in enumerate(itemList):
|
for i, item in enumerate(itemList):
|
||||||
itemList[i] = _getMain().normalizeEmailAddressOrUID(item)
|
itemList[i] = _gam().normalizeEmailAddressOrUID(item)
|
||||||
if dataDelimiter:
|
if dataDelimiter:
|
||||||
csvPF.WriteRow({keyField: entityEmail, dataField: dataDelimiter.join(itemList)})
|
csvPF.WriteRow({keyField: entityEmail, dataField: dataDelimiter.join(itemList)})
|
||||||
else:
|
else:
|
||||||
@@ -1098,10 +1097,10 @@ def doListUser(entityList):
|
|||||||
_doList(entityList, Cmd.ENTITY_USERS)
|
_doList(entityList, Cmd.ENTITY_USERS)
|
||||||
|
|
||||||
def _showCount(entityList, entityType):
|
def _showCount(entityList, entityType):
|
||||||
_getMain().buildGAPIObject(API.DIRECTORY)
|
_gam().buildGAPIObject(API.DIRECTORY)
|
||||||
_getMain().checkForExtraneousArguments()
|
_gam().checkForExtraneousArguments()
|
||||||
_, count, entityList = _getMain().getEntityArgument(entityList)
|
_, count, entityList = _gam().getEntityArgument(entityList)
|
||||||
_getMain().actionPerformedNumItems(count, entityType)
|
_gam().actionPerformedNumItems(count, entityType)
|
||||||
|
|
||||||
# gam <CrOSTypeEntity> show count
|
# gam <CrOSTypeEntity> show count
|
||||||
def showCountCrOS(entityList):
|
def showCountCrOS(entityList):
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -22,67 +22,63 @@ from gamlib import glcfg as GC
|
|||||||
from gamlib import glgapi as GAPI
|
from gamlib import glgapi as GAPI
|
||||||
|
|
||||||
|
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
|
|
||||||
# Add attachements to an email message
|
# Add attachements to an email message
|
||||||
def _addAttachmentsToMessage(message, attachments):
|
def _addAttachmentsToMessage(message, attachments):
|
||||||
gam = _getMain()
|
|
||||||
for attachment in attachments:
|
for attachment in attachments:
|
||||||
try:
|
try:
|
||||||
attachFilename = gam.setFilePath(attachment[0], GC.INPUT_DIR)
|
attachFilename = _gam().setFilePath(attachment[0], GC.INPUT_DIR)
|
||||||
attachContentType, attachEncoding = mimetypes.guess_type(attachFilename)
|
attachContentType, attachEncoding = mimetypes.guess_type(attachFilename)
|
||||||
if attachContentType is None or attachEncoding is not None:
|
if attachContentType is None or attachEncoding is not None:
|
||||||
attachContentType = 'application/octet-stream'
|
attachContentType = 'application/octet-stream'
|
||||||
main_type, sub_type = attachContentType.split('/', 1)
|
main_type, sub_type = attachContentType.split('/', 1)
|
||||||
if main_type == 'text':
|
if main_type == 'text':
|
||||||
msg = MIMEText(gam.readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=gam.UTF8)
|
msg = MIMEText(_gam().readFile(attachFilename, 'r', attachment[1]), _subtype=sub_type, _charset=_gam().UTF8)
|
||||||
elif main_type == 'image':
|
elif main_type == 'image':
|
||||||
msg = MIMEImage(gam.readFile(attachFilename, 'rb'), _subtype=sub_type)
|
msg = MIMEImage(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
|
||||||
elif main_type == 'audio':
|
elif main_type == 'audio':
|
||||||
msg = MIMEAudio(gam.readFile(attachFilename, 'rb'), _subtype=sub_type)
|
msg = MIMEAudio(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
|
||||||
elif main_type == 'application':
|
elif main_type == 'application':
|
||||||
msg = MIMEApplication(gam.readFile(attachFilename, 'rb'), _subtype=sub_type)
|
msg = MIMEApplication(_gam().readFile(attachFilename, 'rb'), _subtype=sub_type)
|
||||||
else:
|
else:
|
||||||
msg = MIMEBase(main_type, sub_type)
|
msg = MIMEBase(main_type, sub_type)
|
||||||
msg.set_payload(gam.readFile(attachFilename, 'rb'))
|
msg.set_payload(_gam().readFile(attachFilename, 'rb'))
|
||||||
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFilename))
|
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFilename))
|
||||||
message.attach(msg)
|
message.attach(msg)
|
||||||
except (IOError, UnicodeDecodeError) as e:
|
except (IOError, UnicodeDecodeError) as e:
|
||||||
gam.usageErrorExit(f'{attachFilename}: {str(e)}')
|
_gam().usageErrorExit(f'{attachFilename}: {str(e)}')
|
||||||
|
|
||||||
# Add embedded images to an email message
|
# Add embedded images to an email message
|
||||||
def _addEmbeddedImagesToMessage(message, embeddedImages):
|
def _addEmbeddedImagesToMessage(message, embeddedImages):
|
||||||
gam = _getMain()
|
|
||||||
for embeddedImage in embeddedImages:
|
for embeddedImage in embeddedImages:
|
||||||
try:
|
try:
|
||||||
imageFilename = gam.setFilePath(embeddedImage[0], GC.INPUT_DIR)
|
imageFilename = _gam().setFilePath(embeddedImage[0], GC.INPUT_DIR)
|
||||||
imageContentType, imageEncoding = mimetypes.guess_type(imageFilename)
|
imageContentType, imageEncoding = mimetypes.guess_type(imageFilename)
|
||||||
if imageContentType is None or imageEncoding is not None:
|
if imageContentType is None or imageEncoding is not None:
|
||||||
imageContentType = 'application/octet-stream'
|
imageContentType = 'application/octet-stream'
|
||||||
main_type, sub_type = imageContentType.split('/', 1)
|
main_type, sub_type = imageContentType.split('/', 1)
|
||||||
if main_type == 'image':
|
if main_type == 'image':
|
||||||
msg = MIMEImage(gam.readFile(imageFilename, 'rb'), _subtype=sub_type)
|
msg = MIMEImage(_gam().readFile(imageFilename, 'rb'), _subtype=sub_type)
|
||||||
else:
|
else:
|
||||||
msg = MIMEBase(main_type, sub_type)
|
msg = MIMEBase(main_type, sub_type)
|
||||||
msg.set_payload(gam.readFile(imageFilename, 'rb'))
|
msg.set_payload(_gam().readFile(imageFilename, 'rb'))
|
||||||
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(imageFilename))
|
msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(imageFilename))
|
||||||
msg.add_header('Content-ID', f'<{embeddedImage[1]}>')
|
msg.add_header('Content-ID', f'<{embeddedImage[1]}>')
|
||||||
message.attach(msg)
|
message.attach(msg)
|
||||||
except (IOError, UnicodeDecodeError) as e:
|
except (IOError, UnicodeDecodeError) as e:
|
||||||
gam.usageErrorExit(f'{imageFilename}: {str(e)}')
|
_gam().usageErrorExit(f'{imageFilename}: {str(e)}')
|
||||||
|
|
||||||
# Send an email
|
# Send an email
|
||||||
def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msgFrom=None, msgReplyTo=None,
|
def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msgFrom=None, msgReplyTo=None,
|
||||||
html=False, charset=None, attachments=None, embeddedImages=None,
|
html=False, charset=None, attachments=None, embeddedImages=None,
|
||||||
msgHeaders=None, ccRecipients=None, bccRecipients=None, mailBox=None, threadId=None,
|
msgHeaders=None, ccRecipients=None, bccRecipients=None, mailBox=None, threadId=None,
|
||||||
action=None):
|
action=None):
|
||||||
gam = _getMain()
|
Act = _gam().Act
|
||||||
Act = gam.Act
|
Ent = _gam().Ent
|
||||||
Ent = gam.Ent
|
|
||||||
if charset is None:
|
if charset is None:
|
||||||
charset = gam.UTF8
|
charset = _gam().UTF8
|
||||||
if action is None:
|
if action is None:
|
||||||
action = Act.SENDEMAIL
|
action = Act.SENDEMAIL
|
||||||
|
|
||||||
@@ -96,26 +92,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
|
|||||||
toSent.remove(addr)
|
toSent.remove(addr)
|
||||||
toFailed[addr] = f'{err[0]}: {err[1]}'
|
toFailed[addr] = f'{err[0]}: {err[1]}'
|
||||||
if toSent:
|
if toSent:
|
||||||
gam.entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
|
_gam().entityActionPerformed([entityType, ','.join(toSent), Ent.MESSAGE, msgSubject], i, count)
|
||||||
for addr, errMsg in toFailed.items():
|
for addr, errMsg in toFailed.items():
|
||||||
gam.entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
|
_gam().entityActionFailedWarning([entityType, addr, Ent.MESSAGE, msgSubject], errMsg, i, count)
|
||||||
|
|
||||||
def cleanAddr(emailAddr):
|
def cleanAddr(emailAddr):
|
||||||
match = gam.NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
|
match = _gam().NAME_EMAIL_ADDRESS_PATTERN.match(emailAddr)
|
||||||
if match:
|
if match:
|
||||||
emailName = match.group(1)
|
emailName = match.group(1)
|
||||||
emailAddr = gam.normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
|
emailAddr = _gam().normalizeEmailAddressOrUID(match.group(2), noUid=True, noLower=True)
|
||||||
return (f'{emailName} <{emailAddr}>', emailAddr)
|
return (f'{emailName} <{emailAddr}>', emailAddr)
|
||||||
emailAddr = gam.normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
|
emailAddr = _gam().normalizeEmailAddressOrUID(emailAddr, noUid=True, noLower=True)
|
||||||
return (emailAddr, emailAddr)
|
return (emailAddr, emailAddr)
|
||||||
|
|
||||||
if msgFrom is None:
|
if msgFrom is None:
|
||||||
msgFrom = gam._getAdminEmail()
|
msgFrom = _gam()._getAdminEmail()
|
||||||
# Force ASCII for RFC compliance
|
# Force ASCII for RFC compliance
|
||||||
# xmlcharref seems to work to display at least
|
# xmlcharref seems to work to display at least
|
||||||
# some unicode in HTML body and is ignored in
|
# some unicode in HTML body and is ignored in
|
||||||
# plain text body.
|
# plain text body.
|
||||||
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(gam.UTF8)
|
# msgBody = msgBody.encode('ascii', 'xmlcharrefreplace').decode(_gam().UTF8)
|
||||||
if not attachments and not embeddedImages:
|
if not attachments and not embeddedImages:
|
||||||
message = MIMEText(msgBody, ['plain', 'html'][html], charset)
|
message = MIMEText(msgBody, ['plain', 'html'][html], charset)
|
||||||
else:
|
else:
|
||||||
@@ -145,26 +141,26 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
|
|||||||
Act.Set(action)
|
Act.Set(action)
|
||||||
if not GC.Values[GC.SMTP_HOST]:
|
if not GC.Values[GC.SMTP_HOST]:
|
||||||
if not clientAccess:
|
if not clientAccess:
|
||||||
userId, gmail = gam.buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
|
userId, gmail = _gam().buildGAPIServiceObject(API.GMAIL, mailBoxAddr)
|
||||||
if not gmail:
|
if not gmail:
|
||||||
Act.Set(parentAction)
|
Act.Set(parentAction)
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
userId = mailBoxAddr
|
userId = mailBoxAddr
|
||||||
gmail = gam.buildGAPIObject(API.GMAIL)
|
gmail = _gam().buildGAPIObject(API.GMAIL)
|
||||||
message['To'] = msgTo if msgTo else userId
|
message['To'] = msgTo if msgTo else userId
|
||||||
body = {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}
|
body = {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}
|
||||||
if threadId is not None:
|
if threadId is not None:
|
||||||
body['threadId'] = threadId
|
body['threadId'] = threadId
|
||||||
try:
|
try:
|
||||||
result = gam.callGAPI(gmail.users().messages(), 'send',
|
result = _gam().callGAPI(gmail.users().messages(), 'send',
|
||||||
throwReasons=[GAPI.SERVICE_NOT_AVAILABLE, GAPI.AUTH_ERROR, GAPI.DOMAIN_POLICY,
|
throwReasons=[GAPI.SERVICE_NOT_AVAILABLE, GAPI.AUTH_ERROR, GAPI.DOMAIN_POLICY,
|
||||||
GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
|
GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.FORBIDDEN, GAPI.PERMISSION_DENIED],
|
||||||
userId=userId, body=body, fields='id')
|
userId=userId, body=body, fields='id')
|
||||||
gam.entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
|
_gam().entityActionPerformedMessage([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], f"{result['id']}", i, count)
|
||||||
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy,
|
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy,
|
||||||
GAPI.invalid, GAPI.invalidArgument, GAPI.forbidden, GAPI.permissionDenied) as e:
|
GAPI.invalid, GAPI.invalidArgument, GAPI.forbidden, GAPI.permissionDenied) as e:
|
||||||
gam.entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
|
_gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
|
||||||
else:
|
else:
|
||||||
message['To'] = msgTo if msgTo else mailBoxAddr
|
message['To'] = msgTo if msgTo else mailBoxAddr
|
||||||
server = None
|
server = None
|
||||||
@@ -175,7 +171,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
|
|||||||
server.starttls(context=ssl.create_default_context(cafile=GC.Values[GC.CACERTS_PEM]))
|
server.starttls(context=ssl.create_default_context(cafile=GC.Values[GC.CACERTS_PEM]))
|
||||||
if GC.Values[GC.SMTP_USERNAME] and GC.Values[GC.SMTP_PASSWORD]:
|
if GC.Values[GC.SMTP_USERNAME] and GC.Values[GC.SMTP_PASSWORD]:
|
||||||
if isinstance(GC.Values[GC.SMTP_PASSWORD], bytes):
|
if isinstance(GC.Values[GC.SMTP_PASSWORD], bytes):
|
||||||
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(gam.UTF8))
|
server.login(GC.Values[GC.SMTP_USERNAME], base64.b64decode(GC.Values[GC.SMTP_PASSWORD]).decode(_gam().UTF8))
|
||||||
else:
|
else:
|
||||||
server.login(GC.Values[GC.SMTP_USERNAME], GC.Values[GC.SMTP_PASSWORD])
|
server.login(GC.Values[GC.SMTP_USERNAME], GC.Values[GC.SMTP_PASSWORD])
|
||||||
result = server.send_message(message)
|
result = server.send_message(message)
|
||||||
@@ -183,7 +179,7 @@ def send_email(msgSubject, msgBody, msgTo, i=0, count=0, clientAccess=False, msg
|
|||||||
checkResult(Ent.RECIPIENT_CC, ccRecipients)
|
checkResult(Ent.RECIPIENT_CC, ccRecipients)
|
||||||
checkResult(Ent.RECIPIENT_BCC, bccRecipients)
|
checkResult(Ent.RECIPIENT_BCC, bccRecipients)
|
||||||
except smtplib.SMTPException as e:
|
except smtplib.SMTPException as e:
|
||||||
gam.entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
|
_gam().entityActionFailedWarning([Ent.RECIPIENT, msgTo, Ent.MESSAGE, msgSubject], str(e), i, count)
|
||||||
if server:
|
if server:
|
||||||
try:
|
try:
|
||||||
server.quit()
|
server.quit()
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -24,8 +24,7 @@ from gamlib import glglobals as GM
|
|||||||
from gamlib import glmsgs as Msg
|
from gamlib import glmsgs as Msg
|
||||||
|
|
||||||
|
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
|
|
||||||
GDOC_FORMAT_MIME_TYPES = {
|
GDOC_FORMAT_MIME_TYPES = {
|
||||||
@@ -37,39 +36,39 @@ GDOC_FORMAT_MIME_TYPES = {
|
|||||||
# gdoc <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity>
|
# gdoc <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity>
|
||||||
def getGDocData(gformat):
|
def getGDocData(gformat):
|
||||||
mimeType = GDOC_FORMAT_MIME_TYPES[gformat]
|
mimeType = GDOC_FORMAT_MIME_TYPES[gformat]
|
||||||
user = _getMain().getEmailAddress()
|
user = _gam().getEmailAddress()
|
||||||
fileIdEntity = _getMain().getDriveFileEntity(queryShortcutsOK=False)
|
fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
|
||||||
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
|
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
|
||||||
_, drive = _getMain().buildGAPIServiceObject(API.DRIVE3, user)
|
_, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
|
||||||
else:
|
else:
|
||||||
drive = _getMain().buildGAPIObject(API.DRIVE3)
|
drive = _gam().buildGAPIObject(API.DRIVE3)
|
||||||
if not drive:
|
if not drive:
|
||||||
sys.exit(GM.Globals[GM.SYSEXITRC])
|
sys.exit(GM.Globals[GM.SYSEXITRC])
|
||||||
_, _, jcount = _getMain()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
|
_, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
|
||||||
if jcount == 0:
|
if jcount == 0:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
|
||||||
if jcount > 1:
|
if jcount > 1:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
|
||||||
fileId = fileIdEntity['list'][0]
|
fileId = fileIdEntity['list'][0]
|
||||||
f = None
|
f = None
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(drive.files(), 'get',
|
result = _gam().callGAPI(drive.files(), 'get',
|
||||||
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
|
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
|
||||||
fileId=fileId, fields='name,mimeType,exportLinks',
|
fileId=fileId, fields='name,mimeType,exportLinks',
|
||||||
supportsAllDrives=True)
|
supportsAllDrives=True)
|
||||||
# Google Doc
|
# Google Doc
|
||||||
if 'exportLinks' in result:
|
if 'exportLinks' in result:
|
||||||
if mimeType not in result['exportLinks']:
|
if mimeType not in result['exportLinks']:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
|
||||||
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
|
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
|
||||||
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
|
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
|
||||||
_, content = drive._http.request(uri=result['exportLinks'][mimeType], method='GET')
|
_, content = drive._http.request(uri=result['exportLinks'][mimeType], method='GET')
|
||||||
f.write(content.decode(_getMain().UTF8_SIG))
|
f.write(content.decode(_gam().UTF8_SIG))
|
||||||
f.seek(0)
|
f.seek(0)
|
||||||
return f
|
return f
|
||||||
# Drive File
|
# Drive File
|
||||||
if result['mimeType'] != mimeType:
|
if result['mimeType'] != mimeType:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
|
||||||
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
|
Msg.INVALID_MIMETYPE.format(result['mimeType'], mimeType))
|
||||||
fb = TemporaryFile(mode='wb+')
|
fb = TemporaryFile(mode='wb+')
|
||||||
request = drive.files().get_media(fileId=fileId)
|
request = drive.files().get_media(fileId=fileId)
|
||||||
@@ -77,63 +76,63 @@ def getGDocData(gformat):
|
|||||||
done = False
|
done = False
|
||||||
while not done:
|
while not done:
|
||||||
_, done = downloader.next_chunk()
|
_, done = downloader.next_chunk()
|
||||||
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
|
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
|
||||||
fb.seek(0)
|
fb.seek(0)
|
||||||
f.write(fb.read().decode(_getMain().UTF8_SIG))
|
f.write(fb.read().decode(_gam().UTF8_SIG))
|
||||||
fb.close()
|
fb.close()
|
||||||
f.seek(0)
|
f.seek(0)
|
||||||
return f
|
return f
|
||||||
except GAPI.fileNotFound:
|
except GAPI.fileNotFound:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], Msg.DOES_NOT_EXIST)
|
||||||
except (IOError, httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
except (IOError, httplib2.HttpLib2Error, google.auth.exceptions.TransportError, RuntimeError) as e:
|
||||||
if f:
|
if f:
|
||||||
f.close()
|
f.close()
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DOCUMENT, fileId], str(e))
|
||||||
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
|
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
|
||||||
_getMain().userDriveServiceNotEnabledWarning(user, str(e))
|
_gam().userDriveServiceNotEnabledWarning(user, str(e))
|
||||||
sys.exit(GM.Globals[GM.SYSEXITRC])
|
sys.exit(GM.Globals[GM.SYSEXITRC])
|
||||||
|
|
||||||
HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>')
|
HTML_TITLE_PATTERN = re.compile(r'.*<title>(.+)</title>')
|
||||||
|
|
||||||
# gsheet <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> <SheetEntity>
|
# gsheet <EmailAddress> <DriveFileIDEntity>|<DriveFileNameEntity> <SheetEntity>
|
||||||
def getGSheetData():
|
def getGSheetData():
|
||||||
user = _getMain().getEmailAddress()
|
user = _gam().getEmailAddress()
|
||||||
fileIdEntity = _getMain().getDriveFileEntity(queryShortcutsOK=False)
|
fileIdEntity = _gam().getDriveFileEntity(queryShortcutsOK=False)
|
||||||
sheetEntity = _getMain().getSheetEntity(False)
|
sheetEntity = _gam().getSheetEntity(False)
|
||||||
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
|
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
|
||||||
user, drive = _getMain().buildGAPIServiceObject(API.DRIVE3, user)
|
user, drive = _gam().buildGAPIServiceObject(API.DRIVE3, user)
|
||||||
else:
|
else:
|
||||||
drive = _getMain().buildGAPIObject(API.DRIVE3)
|
drive = _gam().buildGAPIObject(API.DRIVE3)
|
||||||
if not drive:
|
if not drive:
|
||||||
sys.exit(GM.Globals[GM.SYSEXITRC])
|
sys.exit(GM.Globals[GM.SYSEXITRC])
|
||||||
_, _, jcount = _getMain()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
|
_, _, jcount = _gam()._validateUserGetFileIDs(user, 0, 0, fileIdEntity, drive=drive)
|
||||||
if jcount == 0:
|
if jcount == 0:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.NO_ENTITIES_FOUND.format(Ent.Singular(Ent.DRIVE_FILE)))
|
||||||
if jcount > 1:
|
if jcount > 1:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user], Msg.MULTIPLE_ENTITIES_FOUND.format(Ent.Plural(Ent.DRIVE_FILE), jcount, ','.join(fileIdEntity['list'])))
|
||||||
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
|
if not GC.Values[GC.COMMANDDATA_CLIENTACCESS]:
|
||||||
_, sheet = _getMain().buildGAPIServiceObject(API.SHEETS, user)
|
_, sheet = _gam().buildGAPIServiceObject(API.SHEETS, user)
|
||||||
else:
|
else:
|
||||||
sheet = _getMain().buildGAPIObject(API.SHEETS)
|
sheet = _gam().buildGAPIObject(API.SHEETS)
|
||||||
if not sheet:
|
if not sheet:
|
||||||
sys.exit(GM.Globals[GM.SYSEXITRC])
|
sys.exit(GM.Globals[GM.SYSEXITRC])
|
||||||
fileId = fileIdEntity['list'][0]
|
fileId = fileIdEntity['list'][0]
|
||||||
f = None
|
f = None
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(drive.files(), 'get',
|
result = _gam().callGAPI(drive.files(), 'get',
|
||||||
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
|
throwReasons=GAPI.DRIVE_GET_THROW_REASONS,
|
||||||
fileId=fileId, fields='name,mimeType', supportsAllDrives=True)
|
fileId=fileId, fields='name,mimeType', supportsAllDrives=True)
|
||||||
if result['mimeType'] != _getMain().MIMETYPE_GA_SPREADSHEET:
|
if result['mimeType'] != _gam().MIMETYPE_GA_SPREADSHEET:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.DRIVE_FILE, result['name']],
|
||||||
Msg.INVALID_MIMETYPE.format(result['mimeType'], _getMain().MIMETYPE_GA_SPREADSHEET))
|
Msg.INVALID_MIMETYPE.format(result['mimeType'], _gam().MIMETYPE_GA_SPREADSHEET))
|
||||||
spreadsheet = _getMain().callGAPI(sheet.spreadsheets(), 'get',
|
spreadsheet = _gam().callGAPI(sheet.spreadsheets(), 'get',
|
||||||
throwReasons=GAPI.SHEETS_ACCESS_THROW_REASONS,
|
throwReasons=GAPI.SHEETS_ACCESS_THROW_REASONS,
|
||||||
spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))')
|
spreadsheetId=fileId, fields='spreadsheetUrl,sheets(properties(sheetId,title))')
|
||||||
sheetId = _getMain().getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
|
sheetId = _gam().getSheetIdFromSheetEntity(spreadsheet, sheetEntity)
|
||||||
if sheetId is None:
|
if sheetId is None:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], Msg.NOT_FOUND)
|
||||||
spreadsheetUrl = f'{re.sub("/edit.*$", "/export", spreadsheet["spreadsheetUrl"])}?format=csv&id={fileId}&gid={sheetId}'
|
spreadsheetUrl = f'{re.sub("/edit.*$", "/export", spreadsheet["spreadsheetUrl"])}?format=csv&id={fileId}&gid={sheetId}'
|
||||||
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
|
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
|
||||||
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
if GC.Values[GC.DEBUG_LEVEL] > 0:
|
||||||
sys.stderr.write(f'Debug: spreadsheetUrl: {spreadsheetUrl}\n')
|
sys.stderr.write(f'Debug: spreadsheetUrl: {spreadsheetUrl}\n')
|
||||||
triesLimit = 3
|
triesLimit = 3
|
||||||
@@ -144,25 +143,25 @@ def getGSheetData():
|
|||||||
break
|
break
|
||||||
tg = HTML_TITLE_PATTERN.match(content[0:600].decode('utf-8'))
|
tg = HTML_TITLE_PATTERN.match(content[0:600].decode('utf-8'))
|
||||||
errMsg = tg.group(1) if tg else 'Unknown error'
|
errMsg = tg.group(1) if tg else 'Unknown error'
|
||||||
_getMain().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
|
_gam().getGDocSheetDataRetryWarning([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg, n, triesLimit)
|
||||||
time.sleep(20)
|
time.sleep(20)
|
||||||
else:
|
else:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, result['name'], sheetEntity['sheetType'], sheetEntity['sheetValue']], errMsg)
|
||||||
f.write(content.decode(_getMain().UTF8_SIG))
|
f.write(content.decode(_gam().UTF8_SIG))
|
||||||
f.seek(0)
|
f.seek(0)
|
||||||
return f
|
return f
|
||||||
except GAPI.fileNotFound:
|
except GAPI.fileNotFound:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId], Msg.DOES_NOT_EXIST)
|
||||||
except (GAPI.notFound, GAPI.forbidden, GAPI.permissionDenied,
|
except (GAPI.notFound, GAPI.forbidden, GAPI.permissionDenied,
|
||||||
GAPI.internalError, GAPI.insufficientFilePermissions, GAPI.badRequest,
|
GAPI.internalError, GAPI.insufficientFilePermissions, GAPI.badRequest,
|
||||||
GAPI.invalid, GAPI.invalidArgument, GAPI.failedPrecondition) as e:
|
GAPI.invalid, GAPI.invalidArgument, GAPI.failedPrecondition) as e:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
|
||||||
except (IOError, httplib2.HttpLib2Error) as e:
|
except (IOError, httplib2.HttpLib2Error) as e:
|
||||||
if f:
|
if f:
|
||||||
f.close()
|
f.close()
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
|
_gam().getGDocSheetDataFailedExit([Ent.USER, user, Ent.SPREADSHEET, fileId, sheetEntity['sheetType'], sheetEntity['sheetValue']], str(e))
|
||||||
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
|
except (GAPI.serviceNotAvailable, GAPI.authError, GAPI.domainPolicy) as e:
|
||||||
_getMain().userDriveServiceNotEnabledWarning(user, str(e))
|
_gam().userDriveServiceNotEnabledWarning(user, str(e))
|
||||||
sys.exit(GM.Globals[GM.SYSEXITRC])
|
sys.exit(GM.Globals[GM.SYSEXITRC])
|
||||||
|
|
||||||
|
|
||||||
@@ -173,15 +172,15 @@ BUCKET_OBJECT_PATTERNS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
def getBucketObjectName():
|
def getBucketObjectName():
|
||||||
Cmd = _getMain().Cmd
|
Cmd = _gam().Cmd
|
||||||
uri = _getMain().getString(Cmd.OB_STRING)
|
uri = _gam().getString(Cmd.OB_STRING)
|
||||||
for pattern in BUCKET_OBJECT_PATTERNS:
|
for pattern in BUCKET_OBJECT_PATTERNS:
|
||||||
mg = re.search(pattern['pattern'], uri)
|
mg = re.search(pattern['pattern'], uri)
|
||||||
if mg:
|
if mg:
|
||||||
bucket = mg.group(1)
|
bucket = mg.group(1)
|
||||||
s_object = mg.group(2) if not pattern['unquote'] else unquote(mg.group(2))
|
s_object = mg.group(2) if not pattern['unquote'] else unquote(mg.group(2))
|
||||||
return (bucket, s_object, f'{bucket}/{s_object}')
|
return (bucket, s_object, f'{bucket}/{s_object}')
|
||||||
_getMain().systemErrorExit(_getMain().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
|
_gam().systemErrorExit(_gam().ACTION_NOT_PERFORMED_RC, f'Invalid <StorageBucketObjectName>: {uri}')
|
||||||
|
|
||||||
GCS_FORMAT_MIME_TYPES = {
|
GCS_FORMAT_MIME_TYPES = {
|
||||||
'gcscsv': 'text/csv',
|
'gcscsv': 'text/csv',
|
||||||
@@ -193,17 +192,17 @@ GCS_FORMAT_MIME_TYPES = {
|
|||||||
def getStorageFileData(gcsformat, returnData=True):
|
def getStorageFileData(gcsformat, returnData=True):
|
||||||
mimeType = GCS_FORMAT_MIME_TYPES[gcsformat]
|
mimeType = GCS_FORMAT_MIME_TYPES[gcsformat]
|
||||||
bucket, s_object, bucketObject = getBucketObjectName()
|
bucket, s_object, bucketObject = getBucketObjectName()
|
||||||
s = _getMain().buildGAPIObject(API.STORAGEREAD)
|
s = _gam().buildGAPIObject(API.STORAGEREAD)
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(s.objects(), 'get',
|
result = _gam().callGAPI(s.objects(), 'get',
|
||||||
throwReasons=[GAPI.NOT_FOUND, GAPI.FORBIDDEN],
|
throwReasons=[GAPI.NOT_FOUND, GAPI.FORBIDDEN],
|
||||||
bucket=bucket, object=s_object, projection='noAcl', fields='contentType')
|
bucket=bucket, object=s_object, projection='noAcl', fields='contentType')
|
||||||
except GAPI.notFound:
|
except GAPI.notFound:
|
||||||
_getMain().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
|
_gam().entityDoesNotExistExit(Ent.CLOUD_STORAGE_FILE, bucketObject)
|
||||||
except GAPI.forbidden as e:
|
except GAPI.forbidden as e:
|
||||||
_getMain().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
|
_gam().entityActionFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], str(e))
|
||||||
if result['contentType'] != mimeType:
|
if result['contentType'] != mimeType:
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
|
_gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject],
|
||||||
Msg.INVALID_MIMETYPE.format(result['contentType'], mimeType))
|
Msg.INVALID_MIMETYPE.format(result['contentType'], mimeType))
|
||||||
fb = TemporaryFile(mode='wb+')
|
fb = TemporaryFile(mode='wb+')
|
||||||
try:
|
try:
|
||||||
@@ -214,58 +213,58 @@ def getStorageFileData(gcsformat, returnData=True):
|
|||||||
_, done = downloader.next_chunk()
|
_, done = downloader.next_chunk()
|
||||||
fb.seek(0)
|
fb.seek(0)
|
||||||
if returnData:
|
if returnData:
|
||||||
data = fb.read().decode(_getMain().UTF8)
|
data = fb.read().decode(_gam().UTF8)
|
||||||
fb.close()
|
fb.close()
|
||||||
return data
|
return data
|
||||||
f = TemporaryFile(mode='w+', encoding=_getMain().UTF8)
|
f = TemporaryFile(mode='w+', encoding=_gam().UTF8)
|
||||||
f.write(fb.read().decode(_getMain().UTF8_SIG))
|
f.write(fb.read().decode(_gam().UTF8_SIG))
|
||||||
fb.close()
|
fb.close()
|
||||||
f.seek(0)
|
f.seek(0)
|
||||||
return f
|
return f
|
||||||
except googleapiclient.http.HttpError as e:
|
except googleapiclient.http.HttpError as e:
|
||||||
mg = _getMain().HTTP_ERROR_PATTERN.match(str(e))
|
mg = _gam().HTTP_ERROR_PATTERN.match(str(e))
|
||||||
_getMain().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
|
_gam().getGDocSheetDataFailedExit([Ent.CLOUD_STORAGE_FILE, bucketObject], mg.group(1) if mg else str(e))
|
||||||
|
|
||||||
# <CSVFileInput>
|
# <CSVFileInput>
|
||||||
def openCSVFileReader(filename, fieldnames=None):
|
def openCSVFileReader(filename, fieldnames=None):
|
||||||
Cmd = _getMain().Cmd
|
Cmd = _gam().Cmd
|
||||||
filenameLower = filename.lower()
|
filenameLower = filename.lower()
|
||||||
if filenameLower == 'gsheet':
|
if filenameLower == 'gsheet':
|
||||||
f = getGSheetData()
|
f = getGSheetData()
|
||||||
_getMain().getCharSet()
|
_gam().getCharSet()
|
||||||
elif filenameLower in {'gcsv', 'gdoc'}:
|
elif filenameLower in {'gcsv', 'gdoc'}:
|
||||||
f = getGDocData(filenameLower)
|
f = getGDocData(filenameLower)
|
||||||
_getMain().getCharSet()
|
_gam().getCharSet()
|
||||||
elif filenameLower in {'gcscsv', 'gcsdoc'}:
|
elif filenameLower in {'gcscsv', 'gcsdoc'}:
|
||||||
f = getStorageFileData(filenameLower, False)
|
f = getStorageFileData(filenameLower, False)
|
||||||
_getMain().getCharSet()
|
_gam().getCharSet()
|
||||||
else:
|
else:
|
||||||
encoding = _getMain().getCharSet()
|
encoding = _gam().getCharSet()
|
||||||
filename = _getMain().setFilePath(filename, GC.INPUT_DIR)
|
filename = _gam().setFilePath(filename, GC.INPUT_DIR)
|
||||||
f = _getMain().openFile(filename, mode=_getMain().DEFAULT_CSV_READ_MODE, encoding=encoding)
|
f = _gam().openFile(filename, mode=_gam().DEFAULT_CSV_READ_MODE, encoding=encoding)
|
||||||
if _getMain().checkArgumentPresent('warnifnodata'):
|
if _gam().checkArgumentPresent('warnifnodata'):
|
||||||
loc = f.tell()
|
loc = f.tell()
|
||||||
try:
|
try:
|
||||||
if not f.readline() or not f.readline():
|
if not f.readline() or not f.readline():
|
||||||
_getMain().stderrWarningMsg(_getMain().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
|
_gam().stderrWarningMsg(_gam().fileErrorMessage(filename, Msg.NO_CSV_FILE_DATA_FOUND))
|
||||||
sys.exit(_getMain().NO_ENTITIES_FOUND_RC)
|
sys.exit(_gam().NO_ENTITIES_FOUND_RC)
|
||||||
f.seek(loc)
|
f.seek(loc)
|
||||||
except (IOError, UnicodeDecodeError, UnicodeError) as e:
|
except (IOError, UnicodeDecodeError, UnicodeError) as e:
|
||||||
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, _getMain().fileErrorMessage(filename, e))
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, _gam().fileErrorMessage(filename, e))
|
||||||
if _getMain().checkArgumentPresent('columndelimiter'):
|
if _gam().checkArgumentPresent('columndelimiter'):
|
||||||
columnDelimiter = _getMain().getCharacter()
|
columnDelimiter = _gam().getCharacter()
|
||||||
else:
|
else:
|
||||||
columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER]
|
columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER]
|
||||||
if _getMain().checkArgumentPresent('noescapechar'):
|
if _gam().checkArgumentPresent('noescapechar'):
|
||||||
noEscapeChar = _getMain().getBoolean()
|
noEscapeChar = _gam().getBoolean()
|
||||||
else:
|
else:
|
||||||
noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR]
|
noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR]
|
||||||
if _getMain().checkArgumentPresent('quotechar'):
|
if _gam().checkArgumentPresent('quotechar'):
|
||||||
quotechar = _getMain().getCharacter()
|
quotechar = _gam().getCharacter()
|
||||||
else:
|
else:
|
||||||
quotechar = GC.Values[GC.CSV_INPUT_QUOTE_CHAR]
|
quotechar = GC.Values[GC.CSV_INPUT_QUOTE_CHAR]
|
||||||
if not _getMain().checkArgumentPresent('endcsv') and _getMain().checkArgumentPresent('fields'):
|
if not _gam().checkArgumentPresent('endcsv') and _gam().checkArgumentPresent('fields'):
|
||||||
fieldnames = _getMain().shlexSplitList(_getMain().getString(Cmd.OB_FIELD_NAME_LIST))
|
fieldnames = _gam().shlexSplitList(_gam().getString(Cmd.OB_FIELD_NAME_LIST))
|
||||||
try:
|
try:
|
||||||
csvFile = csv.DictReader(f, fieldnames=fieldnames,
|
csvFile = csv.DictReader(f, fieldnames=fieldnames,
|
||||||
delimiter=columnDelimiter,
|
delimiter=columnDelimiter,
|
||||||
@@ -273,4 +272,4 @@ def openCSVFileReader(filename, fieldnames=None):
|
|||||||
quotechar=quotechar)
|
quotechar=quotechar)
|
||||||
return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else [])
|
return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else [])
|
||||||
except (csv.Error, UnicodeDecodeError, UnicodeError) as e:
|
except (csv.Error, UnicodeDecodeError, UnicodeError) as e:
|
||||||
_getMain().systemErrorExit(_getMain().FILE_ERROR_RC, e)
|
_gam().systemErrorExit(_gam().FILE_ERROR_RC, e)
|
||||||
|
|||||||
@@ -11,12 +11,11 @@ from gamlib import glgapi as GAPI
|
|||||||
from gamlib import glglobals as GM
|
from gamlib import glglobals as GM
|
||||||
|
|
||||||
|
|
||||||
def _getMain():
|
_gam = lambda: sys.modules['gam']
|
||||||
return sys.modules['gam']
|
|
||||||
|
|
||||||
|
|
||||||
def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
|
def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
|
||||||
Cmd = _getMain().Cmd
|
Cmd = _gam().Cmd
|
||||||
if Cmd.ArgumentsRemaining():
|
if Cmd.ArgumentsRemaining():
|
||||||
path = Cmd.Current().strip()
|
path = Cmd.Current().strip()
|
||||||
if path == 'root':
|
if path == 'root':
|
||||||
@@ -24,40 +23,40 @@ def getOrgUnitItem(pathOnly=False, absolutePath=True, cd=None):
|
|||||||
if path:
|
if path:
|
||||||
if pathOnly and (path.startswith('id:') or path.startswith('uid:')) and cd is not None:
|
if pathOnly and (path.startswith('id:') or path.startswith('uid:')) and cd is not None:
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(cd.orgunits(), 'get',
|
result = _gam().callGAPI(cd.orgunits(), 'get',
|
||||||
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
||||||
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=path,
|
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=path,
|
||||||
fields='orgUnitPath')
|
fields='orgUnitPath')
|
||||||
Cmd.Advance()
|
Cmd.Advance()
|
||||||
if absolutePath:
|
if absolutePath:
|
||||||
return _getMain().makeOrgUnitPathAbsolute(result['orgUnitPath'])
|
return _gam().makeOrgUnitPathAbsolute(result['orgUnitPath'])
|
||||||
return _getMain().makeOrgUnitPathRelative(result['orgUnitPath'])
|
return _gam().makeOrgUnitPathRelative(result['orgUnitPath'])
|
||||||
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError,
|
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError,
|
||||||
GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
||||||
_getMain().checkEntityAFDNEorAccessErrorExit(cd, _getMain().Ent.ORGANIZATIONAL_UNIT, path)
|
_gam().checkEntityAFDNEorAccessErrorExit(cd, _gam().Ent.ORGANIZATIONAL_UNIT, path)
|
||||||
_getMain().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
|
_gam().invalidArgumentExit(Cmd.OB_ORGUNIT_PATH)
|
||||||
Cmd.Advance()
|
Cmd.Advance()
|
||||||
if absolutePath:
|
if absolutePath:
|
||||||
return _getMain().makeOrgUnitPathAbsolute(path)
|
return _gam().makeOrgUnitPathAbsolute(path)
|
||||||
return _getMain().makeOrgUnitPathRelative(path)
|
return _gam().makeOrgUnitPathRelative(path)
|
||||||
_getMain().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
|
_gam().missingArgumentExit([Cmd.OB_ORGUNIT_ITEM, Cmd.OB_ORGUNIT_PATH][pathOnly])
|
||||||
|
|
||||||
def getTopLevelOrgId(cd, parentOrgUnitPath):
|
def getTopLevelOrgId(cd, parentOrgUnitPath):
|
||||||
Ent = _getMain().Ent
|
Ent = _gam().Ent
|
||||||
if parentOrgUnitPath != '/':
|
if parentOrgUnitPath != '/':
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(cd.orgunits(), 'get',
|
result = _gam().callGAPI(cd.orgunits(), 'get',
|
||||||
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
||||||
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(parentOrgUnitPath)),
|
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentOrgUnitPath)),
|
||||||
fields='orgUnitId')
|
fields='orgUnitId')
|
||||||
return result['orgUnitId']
|
return result['orgUnitId']
|
||||||
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
||||||
return None
|
return None
|
||||||
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
||||||
_getMain().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
|
_gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(cd.orgunits(), 'list',
|
result = _gam().callGAPI(cd.orgunits(), 'list',
|
||||||
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
||||||
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='allIncludingParent',
|
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='allIncludingParent',
|
||||||
fields='organizationUnits(orgUnitId,orgUnitPath)')
|
fields='organizationUnits(orgUnitId,orgUnitPath)')
|
||||||
@@ -68,18 +67,18 @@ def getTopLevelOrgId(cd, parentOrgUnitPath):
|
|||||||
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
||||||
return None
|
return None
|
||||||
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
||||||
_getMain().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
|
_gam().checkEntityAFDNEorAccessErrorExit(cd, Ent.ORGANIZATIONAL_UNIT, parentOrgUnitPath)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def getOrgUnitId(cd=None, orgUnit=None):
|
def getOrgUnitId(cd=None, orgUnit=None):
|
||||||
Ent = _getMain().Ent
|
Ent = _gam().Ent
|
||||||
if cd is None:
|
if cd is None:
|
||||||
cd = _getMain().buildGAPIObject(API.DIRECTORY)
|
cd = _gam().buildGAPIObject(API.DIRECTORY)
|
||||||
if orgUnit is None:
|
if orgUnit is None:
|
||||||
orgUnit = getOrgUnitItem()
|
orgUnit = getOrgUnitItem()
|
||||||
try:
|
try:
|
||||||
if orgUnit == '/':
|
if orgUnit == '/':
|
||||||
result = _getMain().callGAPI(cd.orgunits(), 'list',
|
result = _gam().callGAPI(cd.orgunits(), 'list',
|
||||||
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
||||||
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='children',
|
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath='/', type='children',
|
||||||
fields='organizationUnits(parentOrgUnitId,parentOrgUnitPath)')
|
fields='organizationUnits(parentOrgUnitId,parentOrgUnitPath)')
|
||||||
@@ -89,35 +88,35 @@ def getOrgUnitId(cd=None, orgUnit=None):
|
|||||||
if topLevelOrgId:
|
if topLevelOrgId:
|
||||||
return (orgUnit, topLevelOrgId)
|
return (orgUnit, topLevelOrgId)
|
||||||
return (orgUnit, '/') #Bogus but should never happen
|
return (orgUnit, '/') #Bogus but should never happen
|
||||||
result = _getMain().callGAPI(cd.orgunits(), 'get',
|
result = _gam().callGAPI(cd.orgunits(), 'get',
|
||||||
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
||||||
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(orgUnit)),
|
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=_gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(orgUnit)),
|
||||||
fields='orgUnitId,orgUnitPath')
|
fields='orgUnitId,orgUnitPath')
|
||||||
return (result['orgUnitPath'], result['orgUnitId'])
|
return (result['orgUnitPath'], result['orgUnitId'])
|
||||||
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
||||||
_getMain().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
|
_gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, orgUnit)
|
||||||
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
||||||
_getMain().accessErrorExit(cd)
|
_gam().accessErrorExit(cd)
|
||||||
|
|
||||||
def getAllParentOrgUnitsForUser(cd, user):
|
def getAllParentOrgUnitsForUser(cd, user):
|
||||||
Ent = _getMain().Ent
|
Ent = _gam().Ent
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(cd.users(), 'get',
|
result = _gam().callGAPI(cd.users(), 'get',
|
||||||
throwReasons=GAPI.USER_GET_THROW_REASONS,
|
throwReasons=GAPI.USER_GET_THROW_REASONS,
|
||||||
userKey=user, fields='orgUnitPath', projection='basic')
|
userKey=user, fields='orgUnitPath', projection='basic')
|
||||||
except (GAPI.userNotFound, GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden):
|
except (GAPI.userNotFound, GAPI.domainNotFound, GAPI.domainCannotUseApis, GAPI.forbidden):
|
||||||
_getMain().entityDoesNotExistExit(Ent.USER, user)
|
_gam().entityDoesNotExistExit(Ent.USER, user)
|
||||||
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
||||||
_getMain().accessErrorExit(cd)
|
_gam().accessErrorExit(cd)
|
||||||
parentPath = result['orgUnitPath']
|
parentPath = result['orgUnitPath']
|
||||||
if parentPath == '/':
|
if parentPath == '/':
|
||||||
orgUnitPath, orgUnitId = getOrgUnitId(cd, '/')
|
orgUnitPath, orgUnitId = getOrgUnitId(cd, '/')
|
||||||
return {orgUnitId: orgUnitPath}
|
return {orgUnitId: orgUnitPath}
|
||||||
parentPath = _getMain().encodeOrgUnitPath(_getMain().makeOrgUnitPathRelative(parentPath))
|
parentPath = _gam().encodeOrgUnitPath(_gam().makeOrgUnitPathRelative(parentPath))
|
||||||
orgUnits = {}
|
orgUnits = {}
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
result = _getMain().callGAPI(cd.orgunits(), 'get',
|
result = _gam().callGAPI(cd.orgunits(), 'get',
|
||||||
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
throwReasons=GAPI.ORGUNIT_GET_THROW_REASONS,
|
||||||
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=parentPath,
|
customerId=GC.Values[GC.CUSTOMER_ID], orgUnitPath=parentPath,
|
||||||
fields='orgUnitId,orgUnitPath,parentOrgUnitId')
|
fields='orgUnitId,orgUnitPath,parentOrgUnitId')
|
||||||
@@ -126,9 +125,9 @@ def getAllParentOrgUnitsForUser(cd, user):
|
|||||||
break
|
break
|
||||||
parentPath = result['parentOrgUnitId']
|
parentPath = result['parentOrgUnitId']
|
||||||
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
except (GAPI.invalidOrgunit, GAPI.orgunitNotFound, GAPI.backendError):
|
||||||
_getMain().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
|
_gam().entityDoesNotExistExit(Ent.ORGANIZATIONAL_UNIT, parentPath)
|
||||||
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
except (GAPI.badRequest, GAPI.invalidCustomerId, GAPI.loginRequired):
|
||||||
_getMain().accessErrorExit(cd)
|
_gam().accessErrorExit(cd)
|
||||||
return orgUnits
|
return orgUnits
|
||||||
|
|
||||||
def _getOrgunitsOrgUnitIdPath(cd, orgUnit):
|
def _getOrgunitsOrgUnitIdPath(cd, orgUnit):
|
||||||
|
|||||||
Reference in New Issue
Block a user