diff --git a/src/gam/util/api.py b/src/gam/util/api.py index ff5a501b..838a5843 100644 --- a/src/gam/util/api.py +++ b/src/gam/util/api.py @@ -217,6 +217,23 @@ def _getIAMSigner(service_account_info): return google.auth.iam.Signer(request, credentials, service_account_info['client_email']) +def _getSigner(service_account_info): + '''Return a signer for the given key_type, or None for default keys. + + key_type is read from service_account_info: + - "default": Returns None (caller should use from_service_account_info) + - "yubikey": Returns a YubiKey hardware signer + - "signjwt": Returns an IAM signBlob signer via ADC + ''' + key_type = service_account_info.get('key_type', 'default') + if key_type == 'default': + return None + if key_type == 'yubikey': + return yubikey.YubiKey(service_account_info) + if key_type == 'signjwt': + return _getIAMSigner(service_account_info) + return None + def handleOAuthTokenError(e, softErrors, displayError=False, i=0, count=0): errMsg = str(e).replace('.', '') if ((errMsg in API.OAUTH2_TOKEN_ERRORS) or @@ -258,15 +275,10 @@ def getOauth2TxtCredentials(exitOnError=True, api=None, noDASA=False, refreshOnl jsonDict = json.loads(jsonData) api, _, _ = API.getVersion(api) audience = f'https://{api}.googleapis.com/' - key_type = jsonDict.get('key_type', 'default') - if key_type == 'default': + signer = _getSigner(jsonDict) + if signer is None: return (True, JWTCredentials.from_service_account_info(jsonDict, audience=audience)) - if key_type == 'yubikey': - yksigner = yubikey.YubiKey(jsonDict) - return (True, JWTCredentials._from_signer_and_info(yksigner, jsonDict, audience=audience)) - if key_type == 'signjwt': - sjsigner = _getIAMSigner(jsonDict) - return (True, JWTCredentials._from_signer_and_info(sjsigner, jsonDict, audience=audience)) + return (True, JWTCredentials._from_signer_and_info(signer, jsonDict, audience=audience)) except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: invalidOauth2serviceJsonExit(str(e)) invalidOauth2serviceJsonExit(Msg.NO_DATA) @@ -615,19 +627,14 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F else: GM.Globals[GM.CURRENT_SVCACCT_API] = '' GM.Globals[GM.CURRENT_SVCACCT_API_SCOPES] = scopesOrAPI - key_type = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA].get('key_type', 'default') + svcacct_info = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] + signer = _getSigner(svcacct_info) if not GM.Globals[GM.CURRENT_SVCACCT_API] or scopesOrAPI not in API.JWT_APIS or forceOauth: try: - if key_type == 'default': - credentials = google.oauth2.service_account.Credentials.from_service_account_info(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) - elif key_type == 'yubikey': - yksigner = yubikey.YubiKey(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) - credentials = google.oauth2.service_account.Credentials._from_signer_and_info(yksigner, - GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) - elif key_type == 'signjwt': - sjsigner = _getIAMSigner(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) - credentials = google.oauth2.service_account.Credentials._from_signer_and_info(sjsigner, - GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) + if signer is None: + credentials = google.oauth2.service_account.Credentials.from_service_account_info(svcacct_info) + else: + credentials = google.oauth2.service_account.Credentials._from_signer_and_info(signer, svcacct_info) except (ValueError, IndexError, KeyError) as e: if softErrors: return None @@ -636,19 +643,10 @@ def getSvcAcctCredentials(scopesOrAPI, userEmail, softErrors=False, forceOauth=F else: audience = f'https://{scopesOrAPI}.googleapis.com/' try: - if key_type == 'default': - credentials = JWTCredentials.from_service_account_info(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA], - audience=audience) - elif key_type == 'yubikey': - yksigner = yubikey.YubiKey(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) - credentials = JWTCredentials._from_signer_and_info(yksigner, - GM.Globals[GM.OAUTH2SERVICE_JSON_DATA], - audience=audience) - elif key_type == 'signjwt': - sjsigner = _getIAMSigner(GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]) - credentials = JWTCredentials._from_signer_and_info(sjsigner, - GM.Globals[GM.OAUTH2SERVICE_JSON_DATA], - audience=audience) + if signer is None: + credentials = JWTCredentials.from_service_account_info(svcacct_info, audience=audience) + else: + credentials = JWTCredentials._from_signer_and_info(signer, svcacct_info, audience=audience) credentials.project_id = GM.Globals[GM.OAUTH2SERVICE_JSON_DATA]['project_id'] except (ValueError, IndexError, KeyError) as e: if softErrors: diff --git a/src/gam/util/config.py b/src/gam/util/config.py index 7cbe9c3a..06a0b3a3 100644 --- a/src/gam/util/config.py +++ b/src/gam/util/config.py @@ -60,609 +60,626 @@ from gam.constants import ( FN_GAM_CFG, GAM, ) - -def SetGlobalVariables(): +REGEX_CHARS = '^$*+|$[{(' +ROW_FILTER_ANY_ALL_PATTERN = re.compile(r'^(any:|all:)(.+)$', re.IGNORECASE) +ROW_FILTER_COMP_PATTERN = re.compile(r'^(date|time|count|length|number)\s*([<>]=?|=|!=)(.+)$', re.IGNORECASE) +ROW_FILTER_RANGE_PATTERN = re.compile(r'^(daterange|timerange|countrange|lengthrange|numberrange)(=|!=)(\S+)/(\S+)$', re.IGNORECASE) +ROW_FILTER_TIMEOFDAYRANGE_PATTERN = re.compile(r'^(timeofdayrange)(=|!=)(\d\d):(\d\d)/(\d\d):(\d\d)$', re.IGNORECASE) +ROW_FILTER_BOOL_PATTERN = re.compile(r'^(boolean):(.+)$', re.IGNORECASE) +ROW_FILTER_TEXT_PATTERN = re.compile(r'^(text)([<>]=?|=|!=)(.*)$', re.IGNORECASE) +ROW_FILTER_TEXTRANGE_PATTERN = re.compile(r'^(textrange)(=|!=)(.*)/(.*)$', re.IGNORECASE) +ROW_FILTER_RE_PATTERN = re.compile(r'^(regex|regexcs|notregex|notregexcs):(.*)$', re.IGNORECASE) +ROW_FILTER_DATA_PATTERN = re.compile(r'^(data|notdata):(list|file|csvfile) +(.+)$', re.IGNORECASE) +MULTIPROCESS_EXIT_COMP_PATTERN = re.compile(r'^rc([<>]=?|=|!=)(.+)$', re.IGNORECASE) +MULTIPROCESS_EXIT_RANGE_PATTERN = re.compile(r'^rcrange(=|!=)(\S+)/(\S+)$', re.IGNORECASE) - def _stringInQuotes(value): - return (len(value) > 1) and (((value.startswith('"') and value.endswith('"'))) or ((value.startswith("'") and value.endswith("'")))) +def _stringInQuotes(value): + return (len(value) > 1) and (((value.startswith('"') and value.endswith('"'))) or ((value.startswith("'") and value.endswith("'")))) - def _stripStringQuotes(value): - if _stringInQuotes(value): - return value[1:-1] +def _stripStringQuotes(value): + if _stringInQuotes(value): + return value[1:-1] + return value + +def _quoteStringIfLeadingTrailingBlanks(value): + if not value: + return "''" + if _stringInQuotes(value): return value + if (value[0] != ' ') and (value[-1] != ' '): + return value + return f"'{value}'" - def _quoteStringIfLeadingTrailingBlanks(value): - if not value: - return "''" - if _stringInQuotes(value): - return value - if (value[0] != ' ') and (value[-1] != ' '): - return value - return f"'{value}'" - - def _getDefault(itemName, itemEntry, oldGamPath): - if GC.VAR_SIGFILE in itemEntry: - GC.Defaults[itemName] = itemEntry[GC.VAR_SFFT][os.path.isfile(os.path.join(oldGamPath, itemEntry[GC.VAR_SIGFILE]))] - elif GC.VAR_ENVVAR in itemEntry: - value = os.environ.get(itemEntry[GC.VAR_ENVVAR], GC.Defaults[itemName]) - if itemEntry[GC.VAR_TYPE] in [GC.TYPE_INTEGER, GC.TYPE_FLOAT]: - try: - number = int(value) if itemEntry[GC.VAR_TYPE] == GC.TYPE_INTEGER else float(value) - minVal, maxVal = itemEntry[GC.VAR_LIMITS] - if (minVal is not None) and (number < minVal): - number = minVal - elif (maxVal is not None) and (number > maxVal): - number = maxVal - except ValueError: - number = GC.Defaults[itemName] - value = str(number) - elif itemEntry[GC.VAR_TYPE] == GC.TYPE_STRING: - value = _quoteStringIfLeadingTrailingBlanks(value) - GC.Defaults[itemName] = value - - def _selectSection(): - value = getString(Cmd.OB_SECTION_NAME, minLen=0) - if (not value) or (value.upper() == configparser.DEFAULTSECT): - return configparser.DEFAULTSECT - if GM.Globals[GM.PARSER].has_section(value): - return value - Cmd.Backup() - usageErrorExit(formatKeyValueList('', [Ent.Singular(Ent.SECTION), value, Msg.NOT_FOUND], '')) - - def _showSections(): - printKeyValueList([Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE]]) - Ind.Increment() - for section in [configparser.DEFAULTSECT]+sorted(GM.Globals[GM.PARSER].sections()): - printKeyValueList([f'{section}{" *" if section == sectionName else ""}']) - Ind.Decrement() - - def _checkMakeDir(itemName): - if not os.path.isdir(GC.Defaults[itemName]): +def _getDefault(itemName, itemEntry, oldGamPath): + if GC.VAR_SIGFILE in itemEntry: + GC.Defaults[itemName] = itemEntry[GC.VAR_SFFT][os.path.isfile(os.path.join(oldGamPath, itemEntry[GC.VAR_SIGFILE]))] + elif GC.VAR_ENVVAR in itemEntry: + value = os.environ.get(itemEntry[GC.VAR_ENVVAR], GC.Defaults[itemName]) + if itemEntry[GC.VAR_TYPE] in [GC.TYPE_INTEGER, GC.TYPE_FLOAT]: try: - os.makedirs(GC.Defaults[itemName]) - printKeyValueList([Act.PerformedName(Act.CREATE), GC.Defaults[itemName]]) - except OSError as e: - if not os.path.isdir(GC.Defaults[itemName]): - systemErrorExit(FILE_ERROR_RC, e) + number = int(value) if itemEntry[GC.VAR_TYPE] == GC.TYPE_INTEGER else float(value) + minVal, maxVal = itemEntry[GC.VAR_LIMITS] + if (minVal is not None) and (number < minVal): + number = minVal + elif (maxVal is not None) and (number > maxVal): + number = maxVal + except ValueError: + number = GC.Defaults[itemName] + value = str(number) + elif itemEntry[GC.VAR_TYPE] == GC.TYPE_STRING: + value = _quoteStringIfLeadingTrailingBlanks(value) + GC.Defaults[itemName] = value - def _copyCfgFile(srcFile, targetDir, oldGamPath): - if (not srcFile) or os.path.isabs(srcFile): - return - dstFile = os.path.join(GC.Defaults[targetDir], srcFile) - if os.path.isfile(dstFile): - return - srcFile = os.path.join(oldGamPath, srcFile) - if not os.path.isfile(srcFile): - return - data = readFile(srcFile, continueOnError=True, displayError=False) - if (data is not None) and writeFile(dstFile, data, continueOnError=True): - printKeyValueList([Act.PerformedName(Act.COPY), srcFile, Msg.TO, dstFile]) - - def _printValueError(sectionName, itemName, value, errMessage, sysRC=CONFIG_ERROR_RC): - kvlMsg = formatKeyValueList('', - [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], - Ent.Singular(Ent.SECTION), sectionName, - Ent.Singular(Ent.ITEM), itemName, - Ent.Singular(Ent.VALUE), value, - errMessage], - '') - if sysRC != 0: - status['errors'] = True - printErrorMessage(sysRC, kvlMsg) - else: - writeStderr(formatKeyValueList(Ind.Spaces(), [WARNING, kvlMsg], '\n')) - - def _getCfgBoolean(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName).lower() - if value in TRUE_VALUES: - return True - if value in FALSE_VALUES: - return False - _printValueError(sectionName, itemName, value, f'{Msg.EXPECTED}: {formatChoiceList(TRUE_FALSE)}') - return False - - def _getCfgCharacter(sectionName, itemName): - value = codecs.escape_decode(bytes(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)), UTF8))[0].decode(UTF8) - if not value and (itemName == 'csv_output_field_delimiter'): - return ' ' - if not value and (itemName in {'csv_input_escape_char', 'csv_output_escape_char'}): - return None - if len(value) == 1: - return value - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {integerLimits(1, 1, Msg.STRING_LENGTH)}') - return '' - - def _getCfgChoice(sectionName, itemName): - value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)).lower() - choices = GC.VAR_INFO[itemName][GC.VAR_CHOICES] - if value in choices: - return choices[value] - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {",".join(choices)}') - return '' - - def _getCfgLocale(sectionName, itemName): - value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)).lower().replace('_', '-') - if value in LOCALE_CODES_MAP: - return LOCALE_CODES_MAP[value] - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {",".join(LOCALE_CODES_MAP)}') - return '' - - def _getCfgNumber(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName) - minVal, maxVal = GC.VAR_INFO[itemName][GC.VAR_LIMITS] - try: - number = int(value) if GC.VAR_INFO[itemName][GC.VAR_TYPE] == GC.TYPE_INTEGER else float(value) - if ((minVal is None) or (number >= minVal)) and ((maxVal is None) or (number <= maxVal)): - return number - if (minVal is not None) and (number < minVal): - number = minVal - else: - number = maxVal - _printValueError(sectionName, itemName, value, f'{Msg.EXPECTED}: {integerLimits(minVal, maxVal)}, {Msg.USED}: {number}', sysRC=0) - return number - except ValueError: - pass - _printValueError(sectionName, itemName, value, f'{Msg.EXPECTED}: {integerLimits(minVal, maxVal)}') - return 0 - - def _getCfgHeaderFilter(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName) - headerFilters = [] - if not value or (len(value) == 2 and _stringInQuotes(value)): - return headerFilters - splitStatus, filters = shlexSplitListStatus(value) - if splitStatus: - for filterStr in filters: - try: - headerFilters.append(re.compile(filterStr, re.IGNORECASE)) - except re.error as e: - _printValueError(sectionName, itemName, f'"{filterStr}"', f'{Msg.INVALID_RE}: {e}') - else: - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {filters}') - return headerFilters - - def _getCfgHeaderFilterFromForce(sectionName, itemName): - headerFilters = [] - for filterStr in GC.Values[itemName]: - try: - headerFilters.append(re.compile(fr'^{filterStr}$')) - except re.error as e: - _printValueError(sectionName, itemName, f'"{filterStr}"', f'{Msg.INVALID_RE}: {e}') - return headerFilters - - ROW_FILTER_ANY_ALL_PATTERN = re.compile(r'^(any:|all:)(.+)$', re.IGNORECASE) - ROW_FILTER_COMP_PATTERN = re.compile(r'^(date|time|count|length|number)\s*([<>]=?|=|!=)(.+)$', re.IGNORECASE) - ROW_FILTER_RANGE_PATTERN = re.compile(r'^(daterange|timerange|countrange|lengthrange|numberrange)(=|!=)(\S+)/(\S+)$', re.IGNORECASE) - ROW_FILTER_TIMEOFDAYRANGE_PATTERN = re.compile(r'^(timeofdayrange)(=|!=)(\d\d):(\d\d)/(\d\d):(\d\d)$', re.IGNORECASE) - ROW_FILTER_BOOL_PATTERN = re.compile(r'^(boolean):(.+)$', re.IGNORECASE) - ROW_FILTER_TEXT_PATTERN = re.compile(r'^(text)([<>]=?|=|!=)(.*)$', re.IGNORECASE) - ROW_FILTER_TEXTRANGE_PATTERN = re.compile(r'^(textrange)(=|!=)(.*)/(.*)$', re.IGNORECASE) - ROW_FILTER_RE_PATTERN = re.compile(r'^(regex|regexcs|notregex|notregexcs):(.*)$', re.IGNORECASE) - ROW_FILTER_DATA_PATTERN = re.compile(r'^(data|notdata):(list|file|csvfile) +(.+)$', re.IGNORECASE) - REGEX_CHARS = '^$*+|$[{(' - - def _getCfgRowFilter(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName) - rowFilters = [] - if not value: - return rowFilters - if value.startswith('{'): - try: - filterDict = json.loads(value.encode('unicode-escape').decode(UTF8)) - except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.FAILED_TO_PARSE_AS_JSON}: {str(e)}') - return rowFilters - else: - filterDict = {} - status, filterList = shlexSplitListStatus(value) - if not status: - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.FAILED_TO_PARSE_AS_LIST}: {str(filterList)}') - return rowFilters - for filterVal in filterList: - if not filterVal: - continue - try: - filterTokens = shlexSplitList(filterVal, ':') - column = filterTokens[0] - filterStr = ':'.join(filterTokens[1:]) - except ValueError: - _printValueError(sectionName, itemName, f'"{filterVal}"', f'{Msg.EXPECTED}: column:filter') - continue - filterDict[column] = filterStr - for column, filterStr in filterDict.items(): - for c in REGEX_CHARS: - if c in column: - columnPat = column - break - else: - columnPat = f'^{column}$' - try: - columnPat = re.compile(columnPat, re.IGNORECASE) - except re.error as e: - _printValueError(sectionName, itemName, f'"{column}"', f'{Msg.INVALID_RE}: {e}') - continue - anyMatch = True - mg = ROW_FILTER_ANY_ALL_PATTERN.match(filterStr) - if mg: - anyMatch = mg.group(1).lower() == 'any:' - filterStr = mg.group(2) - mg = ROW_FILTER_COMP_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - if filterType in {'date', 'time'}: - if filterType == 'date': - valid, filterValue = getRowFilterDateOrDeltaFromNow(mg.group(3)) - else: - valid, filterValue = getRowFilterTimeOrDeltaFromNow(mg.group(3)) - if valid: - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), filterValue)) - else: - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: {filterValue}') - else: # filterType in {'count', 'length', 'number'}: - if mg.group(3).isdigit(): - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), int(mg.group(3)))) - else: - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: ') - continue - mg = ROW_FILTER_TEXT_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), mg.group(3))) - continue - mg = ROW_FILTER_TEXTRANGE_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), mg.group(3), mg.group(4))) - continue - mg = ROW_FILTER_RANGE_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - if filterType in {'daterange', 'timerange'}: - if filterType == 'daterange': - valid1, filterValue1 = getRowFilterDateOrDeltaFromNow(mg.group(3)) - valid2, filterValue2 = getRowFilterDateOrDeltaFromNow(mg.group(4)) - else: - valid1, filterValue1 = getRowFilterTimeOrDeltaFromNow(mg.group(3)) - valid2, filterValue2 = getRowFilterTimeOrDeltaFromNow(mg.group(4)) - if valid1 and valid2: - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), filterValue1, filterValue2)) - else: - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: {filterValue1}/{filterValue2}') - else: #countrange|lengthrange|numberrange - if mg.group(3).isdigit() and mg.group(4).isdigit(): - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), int(mg.group(3)), int(mg.group(4)))) - else: - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: /') - continue - mg = ROW_FILTER_TIMEOFDAYRANGE_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - startHour = int(mg.group(3)) - startMinute = int(mg.group(4)) - endHour = int(mg.group(5)) - endMinute = int(mg.group(6)) - if startHour > 23 or startMinute > 59 or endHour > 23 or endMinute > 59 or \ - endHour < startHour or (endHour == startHour and endMinute < startMinute): - Cmd.Backup() - usageErrorExit(Msg.INVALID_TIMEOFDAY_RANGE.format(f'{startHour:02d}:{startMinute:02d}', f'{endHour:02d}:{endMinute:02d}')) - rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), f'{startHour:02d}:{startMinute:02d}', f'{endHour:02d}:{endMinute:02d}')) - continue - mg = ROW_FILTER_BOOL_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - filterValue = mg.group(2).lower() - if filterValue in TRUE_VALUES: - rowFilters.append((columnPat, anyMatch, filterType, True)) - elif filterValue in FALSE_VALUES: - rowFilters.append((columnPat, anyMatch, filterType, False)) - else: - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: ') - continue - mg = ROW_FILTER_RE_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - try: - if filterType.endswith('cs'): - filterType = filterType[0:-2] - flags = 0 - else: - flags = re.IGNORECASE - rowFilters.append((columnPat, anyMatch, filterType, re.compile(mg.group(2), flags))) - except re.error as e: - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.INVALID_RE}: {e}') - continue - mg = ROW_FILTER_DATA_PATTERN.match(filterStr) - if mg: - filterType = mg.group(1).lower() - filterSubType = mg.group(2).lower() - if filterSubType == 'list': - rowFilters.append((columnPat, anyMatch, filterType, set(shlexSplitList(mg.group(3))))) - continue - Cmd.MergeArguments(shlexSplitList(mg.group(3), ' ')) - if filterSubType == 'file': - rowFilters.append((columnPat, anyMatch, filterType, getEntitiesFromFile(False, returnSet=True))) - else: #elif filterSubType == 'csvfile': - rowFilters.append((columnPat, anyMatch, filterType, getEntitiesFromCSVFile(False, returnSet=True))) - Cmd.RestoreArguments() - continue - _printValueError(sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: ') - return rowFilters - - def _getCfgSection(sectionName, itemName): - value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)) - if (not value) or (value.upper() == configparser.DEFAULTSECT): - return configparser.DEFAULTSECT - if GM.Globals[GM.PARSER].has_section(value): - return value - _printValueError(sectionName, itemName, value, Msg.NOT_FOUND) +def _selectSection(): + value = getString(Cmd.OB_SECTION_NAME, minLen=0) + if (not value) or (value.upper() == configparser.DEFAULTSECT): return configparser.DEFAULTSECT - - def _getCfgPassword(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName) - if isinstance(value, bytes): - return value - value = _stripStringQuotes(value) - if value.startswith("b'") and value.endswith("'"): - return bytes(value[2:-1], UTF8) - if value: - return value - return '' - - def _validateLicenseSKUs(sectionName, itemName, skuList): - GM.Globals[GM.LICENSE_SKUS] = [] - for sku in skuList.split(','): - if '/' not in sku: - productId, sku = SKU.getProductAndSKU(sku) - if not productId: - _printValueError(sectionName, itemName, sku, f'{Msg.EXPECTED}: {",".join(SKU.getSortedSKUList())}') - else: - (productId, sku) = sku.split('/') - if (productId, sku) not in GM.Globals[GM.LICENSE_SKUS]: - GM.Globals[GM.LICENSE_SKUS].append((productId, sku)) - - def _validateDeveloperPreviewAPIs(sectionName, itemName, apiList): - GM.Globals[GM.DEVELOPER_PREVIEW_APIS] = set() - validAPIs = API.getAPIsList() - for api in apiList.split(','): - if api in validAPIs: - GM.Globals[GM.DEVELOPER_PREVIEW_APIS].add(api) - else: - _printValueError(sectionName, itemName, api, f'{Msg.EXPECTED}: {",".join(sorted(validAPIs))}') - - def _validateGCPOrgId(sectionName, itemName, gcpOrgId): - mg = re.match(r'organizations/\d+', gcpOrgId) - if not mg: - _printValueError(sectionName, itemName, gcpOrgId, f'{Msg.EXPECTED}: "organizations/"') - - def _getCfgString(sectionName, itemName): - value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)) - if itemName == GC.DOMAIN: - value = value.strip() - minLen, maxLen = GC.VAR_INFO[itemName].get(GC.VAR_LIMITS, (None, None)) - if ((minLen is None) or (len(value) >= minLen)) and ((maxLen is None) or (len(value) <= maxLen)): - if itemName == GC.LICENSE_SKUS and value: - _validateLicenseSKUs(sectionName, itemName, value) - elif itemName == GC.DEVELOPER_PREVIEW_APIS and value: - _validateDeveloperPreviewAPIs(sectionName, itemName, value.lower()) - elif itemName == GC.GCP_ORG_ID and value: - _validateGCPOrgId(sectionName, itemName, value) - return value - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {integerLimits(minLen, maxLen, Msg.STRING_LENGTH)}') - return '' - - def _getCfgStringList(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName) - stringlist = [] - if not value or (len(value) == 2 and _stringInQuotes(value)): - return stringlist - splitStatus, stringlist = shlexSplitListStatus(value) - if not splitStatus: - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {stringlist}') - return stringlist - - def _getCfgTimezone(sectionName, itemName): - value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)) - if value.lower() in {'utc', 'z'}: - GM.Globals[GM.CONVERT_TO_LOCAL_TIME] = False - return arrow.now('utc').tzinfo - GM.Globals[GM.CONVERT_TO_LOCAL_TIME] = True - if value.lower() == 'local': - return arrow.now(value).tzinfo - try: - return arrow.now(value).tzinfo - except (arrow.parser.ParserError, OverflowError): - _printValueError(sectionName, itemName, value, f'{Msg.EXPECTED}: {TIMEZONE_FORMAT_REQUIRED}') - GM.Globals[GM.CONVERT_TO_LOCAL_TIME] = False - return arrow.now('utc').tzinfo - - def _getCfgDirectory(sectionName, itemName): - dirPath = os.path.expanduser(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName))) - if (not dirPath) and (itemName in {GC.GMAIL_CSE_INCERT_DIR, GC.GMAIL_CSE_INKEY_DIR, GC.INPUT_DIR}): - return dirPath - if (not dirPath) or (not os.path.isabs(dirPath) and dirPath != '.'): - if (sectionName != configparser.DEFAULTSECT) and (GM.Globals[GM.PARSER].has_option(sectionName, itemName)): - dirPath = os.path.join(os.path.expanduser(_stripStringQuotes(GM.Globals[GM.PARSER].get(configparser.DEFAULTSECT, itemName))), dirPath) - if not os.path.isabs(dirPath): - dirPath = os.path.join(GM.Globals[GM.GAM_CFG_PATH], dirPath) - return dirPath - - def _getCfgFile(sectionName, itemName): - value = os.path.expanduser(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName))) - if value and not os.path.isabs(value): - value = os.path.expanduser(os.path.join(_getCfgDirectory(sectionName, GC.CONFIG_DIR), value)) - elif not value and itemName == GC.CACERTS_PEM: - value = os.path.join(GM.Globals[GM.GAM_PATH], GC.FN_CACERTS_PEM) + if GM.Globals[GM.PARSER].has_section(value): return value + Cmd.Backup() + usageErrorExit(formatKeyValueList('', [Ent.Singular(Ent.SECTION), value, Msg.NOT_FOUND], '')) - def _readGamCfgFile(config, fileName): +def _showSections(sectionName): + printKeyValueList([Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE]]) + Ind.Increment() + for section in [configparser.DEFAULTSECT]+sorted(GM.Globals[GM.PARSER].sections()): + printKeyValueList([f'{section}{" *" if section == sectionName else ""}']) + Ind.Decrement() + +def _checkMakeDir(itemName): + if not os.path.isdir(GC.Defaults[itemName]): try: - with open(fileName, DEFAULT_FILE_READ_MODE, encoding=GM.Globals[GM.SYS_ENCODING]) as f: - config.read_file(f) - except (configparser.DuplicateOptionError, configparser.DuplicateSectionError, - configparser.MissingSectionHeaderError, configparser.ParsingError) as e: - systemErrorExit(CONFIG_ERROR_RC, formatKeyValueList('', - [Ent.Singular(Ent.CONFIG_FILE), fileName, - Msg.INVALID, str(e)], - '')) - except IOError as e: - systemErrorExit(FILE_ERROR_RC, fileErrorMessage(fileName, e, Ent.CONFIG_FILE)) + os.makedirs(GC.Defaults[itemName]) + printKeyValueList([Act.PerformedName(Act.CREATE), GC.Defaults[itemName]]) + except OSError as e: + if not os.path.isdir(GC.Defaults[itemName]): + systemErrorExit(FILE_ERROR_RC, e) - def _writeGamCfgFile(config, fileName, action): - GM.Globals[GM.SECTION] = None # No need to save section for inner gams - try: - with open(fileName, DEFAULT_FILE_WRITE_MODE, encoding=GM.Globals[GM.SYS_ENCODING]) as f: - config.write(f) - printKeyValueList([Ent.Singular(Ent.CONFIG_FILE), fileName, Act.PerformedName(action)]) - except IOError as e: - stderrErrorMsg(fileErrorMessage(fileName, e, Ent.CONFIG_FILE)) +def _copyCfgFile(srcFile, targetDir, oldGamPath): + if (not srcFile) or os.path.isabs(srcFile): + return + dstFile = os.path.join(GC.Defaults[targetDir], srcFile) + if os.path.isfile(dstFile): + return + srcFile = os.path.join(oldGamPath, srcFile) + if not os.path.isfile(srcFile): + return + data = readFile(srcFile, continueOnError=True, displayError=False) + if (data is not None) and writeFile(dstFile, data, continueOnError=True): + printKeyValueList([Act.PerformedName(Act.COPY), srcFile, Msg.TO, dstFile]) - def _verifyValues(sectionName, inputFilterSectionName, outputFilterSectionName): - itemNamePattern = getREPattern() if checkArgumentPresent('variables') else None - printKeyValueList([Ent.Singular(Ent.SECTION), sectionName]) # Do not use printEntity - Ind.Increment() - for itemName, itemEntry in GC.VAR_INFO.items(): - if itemNamePattern and not itemNamePattern.search(itemName): - continue - sectName = sectionName - if itemName in GC.CSV_INPUT_ROW_FILTER_ITEMS: - if inputFilterSectionName: - sectName = inputFilterSectionName - elif itemName in GC.CSV_OUTPUT_ROW_FILTER_ITEMS: - if outputFilterSectionName: - sectName = outputFilterSectionName - cfgValue = GM.Globals[GM.PARSER].get(sectName, itemName) - varType = itemEntry[GC.VAR_TYPE] - if varType == GC.TYPE_CHOICE: - for choice, value in itemEntry[GC.VAR_CHOICES].items(): - if cfgValue == value: - cfgValue = choice - break - elif varType not in [GC.TYPE_BOOLEAN, GC.TYPE_INTEGER, GC.TYPE_FLOAT, GC.TYPE_PASSWORD]: - cfgValue = _quoteStringIfLeadingTrailingBlanks(cfgValue) - if varType == GC.TYPE_FILE: - expdValue = _getCfgFile(sectName, itemName) - if cfgValue not in ("''", expdValue): - cfgValue = f'{cfgValue} ; {expdValue}' - elif varType == GC.TYPE_DIRECTORY: - expdValue = _getCfgDirectory(sectName, itemName) - if cfgValue not in ("''", expdValue): - cfgValue = f'{cfgValue} ; {expdValue}' - elif (itemName == GC.SECTION) and (sectName != configparser.DEFAULTSECT): - continue - printLine(f'{Ind.Spaces()}{itemName} = {cfgValue}') - Ind.Decrement() +def _printValueError(status, sectionName, itemName, value, errMessage, sysRC=CONFIG_ERROR_RC): + kvlMsg = formatKeyValueList('', + [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], + Ent.Singular(Ent.SECTION), sectionName, + Ent.Singular(Ent.ITEM), itemName, + Ent.Singular(Ent.VALUE), value, + errMessage], + '') + if sysRC != 0: + status['errors'] = True + printErrorMessage(sysRC, kvlMsg) + else: + writeStderr(formatKeyValueList(Ind.Spaces(), [WARNING, kvlMsg], '\n')) - def _chkCfgDirectories(sectionName): - for itemName, itemEntry in GC.VAR_INFO.items(): - if itemEntry[GC.VAR_TYPE] == GC.TYPE_DIRECTORY: - dirPath = GC.Values[itemName] - if (not dirPath) and (itemName in {GC.GMAIL_CSE_INCERT_DIR, GC.GMAIL_CSE_INKEY_DIR, GC.INPUT_DIR}): - return - if (itemName != GC.CACHE_DIR or not GC.Values[GC.NO_CACHE]) and not os.path.isdir(dirPath): - writeStderr(formatKeyValueList(WARNING_PREFIX, - [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], - Ent.Singular(Ent.SECTION), sectionName, - Ent.Singular(Ent.ITEM), itemName, - Ent.Singular(Ent.VALUE), dirPath, - Msg.INVALID_PATH], - '\n')) +def _getCfgBoolean(status, sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName).lower() + if value in TRUE_VALUES: + return True + if value in FALSE_VALUES: + return False + _printValueError(status, sectionName, itemName, value, f'{Msg.EXPECTED}: {formatChoiceList(TRUE_FALSE)}') + return False - def _chkCfgFiles(sectionName): - for itemName, itemEntry in GC.VAR_INFO.items(): - if itemEntry[GC.VAR_TYPE] == GC.TYPE_FILE: - fileName = GC.Values[itemName] - if (not fileName) and (itemName in {GC.EXTRA_ARGS, GC.CMDLOG}): - continue - if itemName == GC.CLIENT_SECRETS_JSON: # Added 6.57.01 - continue - if GC.Values[GC.ENABLE_DASA] and itemName == GC.OAUTH2_TXT: - continue - if not os.path.isfile(fileName): - writeStderr(formatKeyValueList([WARNING_PREFIX, ERROR_PREFIX][itemName == GC.CACERTS_PEM], - [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], - Ent.Singular(Ent.SECTION), sectionName, - Ent.Singular(Ent.ITEM), itemName, - Ent.Singular(Ent.VALUE), fileName, - Msg.NOT_FOUND], - '\n')) - if itemName == GC.CACERTS_PEM: - status['errors'] = True - elif not os.access(fileName, itemEntry[GC.VAR_ACCESS]): - if itemEntry[GC.VAR_ACCESS] == os.R_OK | os.W_OK: - accessMsg = Msg.NEED_READ_WRITE_ACCESS - elif itemEntry[GC.VAR_ACCESS] == os.R_OK: - accessMsg = Msg.NEED_READ_ACCESS - else: - accessMsg = Msg.NEED_WRITE_ACCESS - writeStderr(formatKeyValueList(ERROR_PREFIX, - [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], - Ent.Singular(Ent.SECTION), sectionName, - Ent.Singular(Ent.ITEM), itemName, - Ent.Singular(Ent.VALUE), fileName, - accessMsg], - '\n')) - status['errors'] = True +def _getCfgCharacter(status, sectionName, itemName): + value = codecs.escape_decode(bytes(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)), UTF8))[0].decode(UTF8) + if not value and (itemName == 'csv_output_field_delimiter'): + return ' ' + if not value and (itemName in {'csv_input_escape_char', 'csv_output_escape_char'}): + return None + if len(value) == 1: + return value + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {integerLimits(1, 1, Msg.STRING_LENGTH)}') + return '' - def _setCSVFile(fileName, mode, encoding, writeHeader, multi, delayOpen): - if fileName != '-': - fileName = setFilePath(fileName, GC.DRIVE_DIR) - GM.Globals[GM.CSVFILE][GM.REDIRECT_NAME] = fileName - GM.Globals[GM.CSVFILE][GM.REDIRECT_MODE] = mode - GM.Globals[GM.CSVFILE][GM.REDIRECT_ENCODING] = encoding - GM.Globals[GM.CSVFILE][GM.REDIRECT_WRITE_HEADER] = writeHeader - GM.Globals[GM.CSVFILE][GM.REDIRECT_MULTIPROCESS] = multi - GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None - if not delayOpen and fileName != '-': - GM.Globals[GM.CSVFILE][GM.REDIRECT_FD] = openFile(fileName, mode, newline='', - encoding=encoding, errors='backslashreplace') +def _getCfgChoice(status, sectionName, itemName): + value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)).lower() + choices = GC.VAR_INFO[itemName][GC.VAR_CHOICES] + if value in choices: + return choices[value] + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {",".join(choices)}') + return '' - def _setSTDFile(stdtype, fileName, mode, multi): - if stdtype == GM.STDOUT: - GM.Globals[GM.SAVED_STDOUT] = None - GM.Globals[stdtype][GM.REDIRECT_STD] = False - if fileName == 'null': - GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, mode, encoding=UTF8) - elif fileName == '-': - GM.Globals[stdtype][GM.REDIRECT_STD] = True - if stdtype == GM.STDOUT: - GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup(sys.stdout.fileno()), mode, encoding=GM.Globals[GM.SYS_ENCODING]) - else: - GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup(sys.stderr.fileno()), mode, encoding=GM.Globals[GM.SYS_ENCODING]) +def _getCfgLocale(status, sectionName, itemName): + value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)).lower().replace('_', '-') + if value in LOCALE_CODES_MAP: + return LOCALE_CODES_MAP[value] + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {",".join(LOCALE_CODES_MAP)}') + return '' + +def _getCfgNumber(status, sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName) + minVal, maxVal = GC.VAR_INFO[itemName][GC.VAR_LIMITS] + try: + number = int(value) if GC.VAR_INFO[itemName][GC.VAR_TYPE] == GC.TYPE_INTEGER else float(value) + if ((minVal is None) or (number >= minVal)) and ((maxVal is None) or (number <= maxVal)): + return number + if (minVal is not None) and (number < minVal): + number = minVal else: - fileName = setFilePath(fileName, GC.DRIVE_DIR) - if multi and mode == DEFAULT_FILE_WRITE_MODE: - deleteFile(fileName) - mode = DEFAULT_FILE_APPEND_MODE - GM.Globals[stdtype][GM.REDIRECT_FD] = openFile(fileName, mode) - GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not multi else StringIOobject() - if (stdtype == GM.STDOUT) and (GC.Values[GC.DEBUG_LEVEL] > 0): - GM.Globals[GM.SAVED_STDOUT] = sys.stdout - sys.stdout = GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] - GM.Globals[stdtype][GM.REDIRECT_NAME] = fileName - GM.Globals[stdtype][GM.REDIRECT_MODE] = mode - GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] = multi - GM.Globals[stdtype][GM.REDIRECT_QUEUE] = 'stdout' if stdtype == GM.STDOUT else 'stderr' + number = maxVal + _printValueError(status, sectionName, itemName, value, f'{Msg.EXPECTED}: {integerLimits(minVal, maxVal)}, {Msg.USED}: {number}', sysRC=0) + return number + except ValueError: + pass + _printValueError(status, sectionName, itemName, value, f'{Msg.EXPECTED}: {integerLimits(minVal, maxVal)}') + return 0 - MULTIPROCESS_EXIT_COMP_PATTERN = re.compile(r'^rc([<>]=?|=|!=)(.+)$', re.IGNORECASE) - MULTIPROCESS_EXIT_RANGE_PATTERN = re.compile(r'^rcrange(=|!=)(\S+)/(\S+)$', re.IGNORECASE) +def _getCfgHeaderFilter(status, sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName) + headerFilters = [] + if not value or (len(value) == 2 and _stringInQuotes(value)): + return headerFilters + splitStatus, filters = shlexSplitListStatus(value) + if splitStatus: + for filterStr in filters: + try: + headerFilters.append(re.compile(filterStr, re.IGNORECASE)) + except re.error as e: + _printValueError(status, sectionName, itemName, f'"{filterStr}"', f'{Msg.INVALID_RE}: {e}') + else: + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {filters}') + return headerFilters - def _setMultiprocessExit(): - rcStr = getString(Cmd.OB_STRING) - mg = MULTIPROCESS_EXIT_COMP_PATTERN.match(rcStr) +def _getCfgHeaderFilterFromForce(status, sectionName, itemName): + headerFilters = [] + for filterStr in GC.Values[itemName]: + try: + headerFilters.append(re.compile(fr'^{filterStr}$')) + except re.error as e: + _printValueError(status, sectionName, itemName, f'"{filterStr}"', f'{Msg.INVALID_RE}: {e}') + return headerFilters + + +def _getCfgRowFilter(status, sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName) + rowFilters = [] + if not value: + return rowFilters + if value.startswith('{'): + try: + filterDict = json.loads(value.encode('unicode-escape').decode(UTF8)) + except (IndexError, KeyError, SyntaxError, TypeError, ValueError) as e: + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.FAILED_TO_PARSE_AS_JSON}: {str(e)}') + return rowFilters + else: + filterDict = {} + splitOk, filterList = shlexSplitListStatus(value) + if not splitOk: + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.FAILED_TO_PARSE_AS_LIST}: {str(filterList)}') + return rowFilters + for filterVal in filterList: + if not filterVal: + continue + try: + filterTokens = shlexSplitList(filterVal, ':') + column = filterTokens[0] + filterStr = ':'.join(filterTokens[1:]) + except ValueError: + _printValueError(status, sectionName, itemName, f'"{filterVal}"', f'{Msg.EXPECTED}: column:filter') + continue + filterDict[column] = filterStr + for column, filterStr in filterDict.items(): + for c in REGEX_CHARS: + if c in column: + columnPat = column + break + else: + columnPat = f'^{column}$' + try: + columnPat = re.compile(columnPat, re.IGNORECASE) + except re.error as e: + _printValueError(status, sectionName, itemName, f'"{column}"', f'{Msg.INVALID_RE}: {e}') + continue + anyMatch = True + mg = ROW_FILTER_ANY_ALL_PATTERN.match(filterStr) if mg: - if not mg.group(2).isdigit(): - usageErrorExit(f'{Msg.EXPECTED}: rc') - GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] = {'comp': mg.group(1), 'value': int(mg.group(2))} - return - mg = MULTIPROCESS_EXIT_RANGE_PATTERN.match(rcStr) + anyMatch = mg.group(1).lower() == 'any:' + filterStr = mg.group(2) + mg = ROW_FILTER_COMP_PATTERN.match(filterStr) if mg: - if not mg.group(2).isdigit() or not mg.group(3).isdigit(): - usageErrorExit(f'{Msg.EXPECTED}: rcrange/Value>') - GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] = {'range': mg.group(1), 'low': int(mg.group(2)), 'high': int(mg.group(3))} - return - usageErrorExit(f'{Msg.EXPECTED}: (rc)|(rcrange/Value>)') + filterType = mg.group(1).lower() + if filterType in {'date', 'time'}: + if filterType == 'date': + valid, filterValue = getRowFilterDateOrDeltaFromNow(mg.group(3)) + else: + valid, filterValue = getRowFilterTimeOrDeltaFromNow(mg.group(3)) + if valid: + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), filterValue)) + else: + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: {filterValue}') + else: # filterType in {'count', 'length', 'number'}: + if mg.group(3).isdigit(): + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), int(mg.group(3)))) + else: + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: ') + continue + mg = ROW_FILTER_TEXT_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), mg.group(3))) + continue + mg = ROW_FILTER_TEXTRANGE_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), mg.group(3), mg.group(4))) + continue + mg = ROW_FILTER_RANGE_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + if filterType in {'daterange', 'timerange'}: + if filterType == 'daterange': + valid1, filterValue1 = getRowFilterDateOrDeltaFromNow(mg.group(3)) + valid2, filterValue2 = getRowFilterDateOrDeltaFromNow(mg.group(4)) + else: + valid1, filterValue1 = getRowFilterTimeOrDeltaFromNow(mg.group(3)) + valid2, filterValue2 = getRowFilterTimeOrDeltaFromNow(mg.group(4)) + if valid1 and valid2: + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), filterValue1, filterValue2)) + else: + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: {filterValue1}/{filterValue2}') + else: #countrange|lengthrange|numberrange + if mg.group(3).isdigit() and mg.group(4).isdigit(): + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), int(mg.group(3)), int(mg.group(4)))) + else: + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: /') + continue + mg = ROW_FILTER_TIMEOFDAYRANGE_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + startHour = int(mg.group(3)) + startMinute = int(mg.group(4)) + endHour = int(mg.group(5)) + endMinute = int(mg.group(6)) + if startHour > 23 or startMinute > 59 or endHour > 23 or endMinute > 59 or \ + endHour < startHour or (endHour == startHour and endMinute < startMinute): + Cmd.Backup() + usageErrorExit(Msg.INVALID_TIMEOFDAY_RANGE.format(f'{startHour:02d}:{startMinute:02d}', f'{endHour:02d}:{endMinute:02d}')) + rowFilters.append((columnPat, anyMatch, filterType, mg.group(2), f'{startHour:02d}:{startMinute:02d}', f'{endHour:02d}:{endMinute:02d}')) + continue + mg = ROW_FILTER_BOOL_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + filterValue = mg.group(2).lower() + if filterValue in TRUE_VALUES: + rowFilters.append((columnPat, anyMatch, filterType, True)) + elif filterValue in FALSE_VALUES: + rowFilters.append((columnPat, anyMatch, filterType, False)) + else: + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: ') + continue + mg = ROW_FILTER_RE_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + try: + if filterType.endswith('cs'): + filterType = filterType[0:-2] + flags = 0 + else: + flags = re.IGNORECASE + rowFilters.append((columnPat, anyMatch, filterType, re.compile(mg.group(2), flags))) + except re.error as e: + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.INVALID_RE}: {e}') + continue + mg = ROW_FILTER_DATA_PATTERN.match(filterStr) + if mg: + filterType = mg.group(1).lower() + filterSubType = mg.group(2).lower() + if filterSubType == 'list': + rowFilters.append((columnPat, anyMatch, filterType, set(shlexSplitList(mg.group(3))))) + continue + Cmd.MergeArguments(shlexSplitList(mg.group(3), ' ')) + if filterSubType == 'file': + rowFilters.append((columnPat, anyMatch, filterType, getEntitiesFromFile(False, returnSet=True))) + else: #elif filterSubType == 'csvfile': + rowFilters.append((columnPat, anyMatch, filterType, getEntitiesFromCSVFile(False, returnSet=True))) + Cmd.RestoreArguments() + continue + _printValueError(status, sectionName, itemName, f'"{column}": "{filterStr}"', f'{Msg.EXPECTED}: ') + return rowFilters +def _getCfgSection(status, sectionName, itemName): + value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)) + if (not value) or (value.upper() == configparser.DEFAULTSECT): + return configparser.DEFAULTSECT + if GM.Globals[GM.PARSER].has_section(value): + return value + _printValueError(status, sectionName, itemName, value, Msg.NOT_FOUND) + return configparser.DEFAULTSECT + +def _getCfgPassword(status, sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName) + if isinstance(value, bytes): + return value + value = _stripStringQuotes(value) + if value.startswith("b'") and value.endswith("'"): + return bytes(value[2:-1], UTF8) + if value: + return value + return '' + +def _validateLicenseSKUs(status, sectionName, itemName, skuList): + GM.Globals[GM.LICENSE_SKUS] = [] + for sku in skuList.split(','): + if '/' not in sku: + productId, sku = SKU.getProductAndSKU(sku) + if not productId: + _printValueError(status, sectionName, itemName, sku, f'{Msg.EXPECTED}: {",".join(SKU.getSortedSKUList())}') + else: + (productId, sku) = sku.split('/') + if (productId, sku) not in GM.Globals[GM.LICENSE_SKUS]: + GM.Globals[GM.LICENSE_SKUS].append((productId, sku)) + +def _validateDeveloperPreviewAPIs(status, sectionName, itemName, apiList): + GM.Globals[GM.DEVELOPER_PREVIEW_APIS] = set() + validAPIs = API.getAPIsList() + for api in apiList.split(','): + if api in validAPIs: + GM.Globals[GM.DEVELOPER_PREVIEW_APIS].add(api) + else: + _printValueError(status, sectionName, itemName, api, f'{Msg.EXPECTED}: {",".join(sorted(validAPIs))}') + +def _validateGCPOrgId(status, sectionName, itemName, gcpOrgId): + mg = re.match(r'organizations/\d+', gcpOrgId) + if not mg: + _printValueError(status, sectionName, itemName, gcpOrgId, f'{Msg.EXPECTED}: "organizations/"') + +def _getCfgString(status, sectionName, itemName): + value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)) + if itemName == GC.DOMAIN: + value = value.strip() + minLen, maxLen = GC.VAR_INFO[itemName].get(GC.VAR_LIMITS, (None, None)) + if ((minLen is None) or (len(value) >= minLen)) and ((maxLen is None) or (len(value) <= maxLen)): + if itemName == GC.LICENSE_SKUS and value: + _validateLicenseSKUs(status, sectionName, itemName, value) + elif itemName == GC.DEVELOPER_PREVIEW_APIS and value: + _validateDeveloperPreviewAPIs(status, sectionName, itemName, value.lower()) + elif itemName == GC.GCP_ORG_ID and value: + _validateGCPOrgId(status, sectionName, itemName, value) + return value + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {integerLimits(minLen, maxLen, Msg.STRING_LENGTH)}') + return '' + +def _getCfgStringList(status, sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName) + stringlist = [] + if not value or (len(value) == 2 and _stringInQuotes(value)): + return stringlist + splitStatus, stringlist = shlexSplitListStatus(value) + if not splitStatus: + _printValueError(status, sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {stringlist}') + return stringlist + +def _getCfgTimezone(status, sectionName, itemName): + value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)) + if value.lower() in {'utc', 'z'}: + GM.Globals[GM.CONVERT_TO_LOCAL_TIME] = False + return arrow.now('utc').tzinfo + GM.Globals[GM.CONVERT_TO_LOCAL_TIME] = True + if value.lower() == 'local': + return arrow.now(value).tzinfo + try: + return arrow.now(value).tzinfo + except (arrow.parser.ParserError, OverflowError): + _printValueError(status, sectionName, itemName, value, f'{Msg.EXPECTED}: {TIMEZONE_FORMAT_REQUIRED}') + GM.Globals[GM.CONVERT_TO_LOCAL_TIME] = False + return arrow.now('utc').tzinfo + +def _getCfgDirectory(status, sectionName, itemName): + dirPath = os.path.expanduser(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName))) + if (not dirPath) and (itemName in {GC.GMAIL_CSE_INCERT_DIR, GC.GMAIL_CSE_INKEY_DIR, GC.INPUT_DIR}): + return dirPath + if (not dirPath) or (not os.path.isabs(dirPath) and dirPath != '.'): + if (sectionName != configparser.DEFAULTSECT) and (GM.Globals[GM.PARSER].has_option(sectionName, itemName)): + dirPath = os.path.join(os.path.expanduser(_stripStringQuotes(GM.Globals[GM.PARSER].get(configparser.DEFAULTSECT, itemName))), dirPath) + if not os.path.isabs(dirPath): + dirPath = os.path.join(GM.Globals[GM.GAM_CFG_PATH], dirPath) + return dirPath + +def _getCfgFile(status, sectionName, itemName): + value = os.path.expanduser(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName))) + if value and not os.path.isabs(value): + value = os.path.expanduser(os.path.join(_getCfgDirectory(status, sectionName, GC.CONFIG_DIR), value)) + elif not value and itemName == GC.CACERTS_PEM: + value = os.path.join(GM.Globals[GM.GAM_PATH], GC.FN_CACERTS_PEM) + return value + +def _readGamCfgFile(config, fileName): + try: + with open(fileName, DEFAULT_FILE_READ_MODE, encoding=GM.Globals[GM.SYS_ENCODING]) as f: + config.read_file(f) + except (configparser.DuplicateOptionError, configparser.DuplicateSectionError, + configparser.MissingSectionHeaderError, configparser.ParsingError) as e: + systemErrorExit(CONFIG_ERROR_RC, formatKeyValueList('', + [Ent.Singular(Ent.CONFIG_FILE), fileName, + Msg.INVALID, str(e)], + '')) + except IOError as e: + systemErrorExit(FILE_ERROR_RC, fileErrorMessage(fileName, e, Ent.CONFIG_FILE)) + +def _writeGamCfgFile(config, fileName, action): + GM.Globals[GM.SECTION] = None # No need to save section for inner gams + try: + with open(fileName, DEFAULT_FILE_WRITE_MODE, encoding=GM.Globals[GM.SYS_ENCODING]) as f: + config.write(f) + printKeyValueList([Ent.Singular(Ent.CONFIG_FILE), fileName, Act.PerformedName(action)]) + except IOError as e: + stderrErrorMsg(fileErrorMessage(fileName, e, Ent.CONFIG_FILE)) + +def _verifyValues(sectionName, inputFilterSectionName, outputFilterSectionName): + itemNamePattern = getREPattern() if checkArgumentPresent('variables') else None + printKeyValueList([Ent.Singular(Ent.SECTION), sectionName]) # Do not use printEntity + Ind.Increment() + for itemName, itemEntry in GC.VAR_INFO.items(): + if itemNamePattern and not itemNamePattern.search(itemName): + continue + sectName = sectionName + if itemName in GC.CSV_INPUT_ROW_FILTER_ITEMS: + if inputFilterSectionName: + sectName = inputFilterSectionName + elif itemName in GC.CSV_OUTPUT_ROW_FILTER_ITEMS: + if outputFilterSectionName: + sectName = outputFilterSectionName + cfgValue = GM.Globals[GM.PARSER].get(sectName, itemName) + varType = itemEntry[GC.VAR_TYPE] + if varType == GC.TYPE_CHOICE: + for choice, value in itemEntry[GC.VAR_CHOICES].items(): + if cfgValue == value: + cfgValue = choice + break + elif varType not in [GC.TYPE_BOOLEAN, GC.TYPE_INTEGER, GC.TYPE_FLOAT, GC.TYPE_PASSWORD]: + cfgValue = _quoteStringIfLeadingTrailingBlanks(cfgValue) + if varType == GC.TYPE_FILE: + expdValue = _getCfgFile(None, sectName, itemName) + if cfgValue not in ("''", expdValue): + cfgValue = f'{cfgValue} ; {expdValue}' + elif varType == GC.TYPE_DIRECTORY: + expdValue = _getCfgDirectory(None, sectName, itemName) + if cfgValue not in ("''", expdValue): + cfgValue = f'{cfgValue} ; {expdValue}' + elif (itemName == GC.SECTION) and (sectName != configparser.DEFAULTSECT): + continue + printLine(f'{Ind.Spaces()}{itemName} = {cfgValue}') + Ind.Decrement() + +def _chkCfgDirectories(sectionName): + for itemName, itemEntry in GC.VAR_INFO.items(): + if itemEntry[GC.VAR_TYPE] == GC.TYPE_DIRECTORY: + dirPath = GC.Values[itemName] + if (not dirPath) and (itemName in {GC.GMAIL_CSE_INCERT_DIR, GC.GMAIL_CSE_INKEY_DIR, GC.INPUT_DIR}): + return + if (itemName != GC.CACHE_DIR or not GC.Values[GC.NO_CACHE]) and not os.path.isdir(dirPath): + writeStderr(formatKeyValueList(WARNING_PREFIX, + [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], + Ent.Singular(Ent.SECTION), sectionName, + Ent.Singular(Ent.ITEM), itemName, + Ent.Singular(Ent.VALUE), dirPath, + Msg.INVALID_PATH], + '\n')) + +def _chkCfgFiles(status, sectionName): + for itemName, itemEntry in GC.VAR_INFO.items(): + if itemEntry[GC.VAR_TYPE] == GC.TYPE_FILE: + fileName = GC.Values[itemName] + if (not fileName) and (itemName in {GC.EXTRA_ARGS, GC.CMDLOG}): + continue + if itemName == GC.CLIENT_SECRETS_JSON: # Added 6.57.01 + continue + if GC.Values[GC.ENABLE_DASA] and itemName == GC.OAUTH2_TXT: + continue + if not os.path.isfile(fileName): + writeStderr(formatKeyValueList([WARNING_PREFIX, ERROR_PREFIX][itemName == GC.CACERTS_PEM], + [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], + Ent.Singular(Ent.SECTION), sectionName, + Ent.Singular(Ent.ITEM), itemName, + Ent.Singular(Ent.VALUE), fileName, + Msg.NOT_FOUND], + '\n')) + if itemName == GC.CACERTS_PEM: + status['errors'] = True + elif not os.access(fileName, itemEntry[GC.VAR_ACCESS]): + if itemEntry[GC.VAR_ACCESS] == os.R_OK | os.W_OK: + accessMsg = Msg.NEED_READ_WRITE_ACCESS + elif itemEntry[GC.VAR_ACCESS] == os.R_OK: + accessMsg = Msg.NEED_READ_ACCESS + else: + accessMsg = Msg.NEED_WRITE_ACCESS + writeStderr(formatKeyValueList(ERROR_PREFIX, + [Ent.Singular(Ent.CONFIG_FILE), GM.Globals[GM.GAM_CFG_FILE], + Ent.Singular(Ent.SECTION), sectionName, + Ent.Singular(Ent.ITEM), itemName, + Ent.Singular(Ent.VALUE), fileName, + accessMsg], + '\n')) + status['errors'] = True + +def _setCSVFile(fileName, mode, encoding, writeHeader, multi, delayOpen): + if fileName != '-': + fileName = setFilePath(fileName, GC.DRIVE_DIR) + GM.Globals[GM.CSVFILE][GM.REDIRECT_NAME] = fileName + GM.Globals[GM.CSVFILE][GM.REDIRECT_MODE] = mode + GM.Globals[GM.CSVFILE][GM.REDIRECT_ENCODING] = encoding + GM.Globals[GM.CSVFILE][GM.REDIRECT_WRITE_HEADER] = writeHeader + GM.Globals[GM.CSVFILE][GM.REDIRECT_MULTIPROCESS] = multi + GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE] = None + if not delayOpen and fileName != '-': + GM.Globals[GM.CSVFILE][GM.REDIRECT_FD] = openFile(fileName, mode, newline='', + encoding=encoding, errors='backslashreplace') + +def _setSTDFile(stdtype, fileName, mode, multi): + if stdtype == GM.STDOUT: + GM.Globals[GM.SAVED_STDOUT] = None + GM.Globals[stdtype][GM.REDIRECT_STD] = False + if fileName == 'null': + GM.Globals[stdtype][GM.REDIRECT_FD] = open(os.devnull, mode, encoding=UTF8) + elif fileName == '-': + GM.Globals[stdtype][GM.REDIRECT_STD] = True + if stdtype == GM.STDOUT: + GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup(sys.stdout.fileno()), mode, encoding=GM.Globals[GM.SYS_ENCODING]) + else: + GM.Globals[stdtype][GM.REDIRECT_FD] = os.fdopen(os.dup(sys.stderr.fileno()), mode, encoding=GM.Globals[GM.SYS_ENCODING]) + else: + fileName = setFilePath(fileName, GC.DRIVE_DIR) + if multi and mode == DEFAULT_FILE_WRITE_MODE: + deleteFile(fileName) + mode = DEFAULT_FILE_APPEND_MODE + GM.Globals[stdtype][GM.REDIRECT_FD] = openFile(fileName, mode) + GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] = GM.Globals[stdtype][GM.REDIRECT_FD] if not multi else StringIOobject() + if (stdtype == GM.STDOUT) and (GC.Values[GC.DEBUG_LEVEL] > 0): + GM.Globals[GM.SAVED_STDOUT] = sys.stdout + sys.stdout = GM.Globals[stdtype][GM.REDIRECT_MULTI_FD] + GM.Globals[stdtype][GM.REDIRECT_NAME] = fileName + GM.Globals[stdtype][GM.REDIRECT_MODE] = mode + GM.Globals[stdtype][GM.REDIRECT_MULTIPROCESS] = multi + GM.Globals[stdtype][GM.REDIRECT_QUEUE] = 'stdout' if stdtype == GM.STDOUT else 'stderr' + +def _setMultiprocessExit(): + rcStr = getString(Cmd.OB_STRING) + mg = MULTIPROCESS_EXIT_COMP_PATTERN.match(rcStr) + if mg: + if not mg.group(2).isdigit(): + usageErrorExit(f'{Msg.EXPECTED}: rc') + GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] = {'comp': mg.group(1), 'value': int(mg.group(2))} + return + mg = MULTIPROCESS_EXIT_RANGE_PATTERN.match(rcStr) + if mg: + if not mg.group(2).isdigit() or not mg.group(3).isdigit(): + usageErrorExit(f'{Msg.EXPECTED}: rcrange/Value>') + GM.Globals[GM.MULTIPROCESS_EXIT_CONDITION] = {'range': mg.group(1), 'low': int(mg.group(2)), 'high': int(mg.group(3))} + return + usageErrorExit(f'{Msg.EXPECTED}: (rc)|(rcrange/Value>)') + + +_CFG_TYPE_HANDLERS = { + GC.TYPE_BOOLEAN: _getCfgBoolean, + GC.TYPE_CHARACTER: _getCfgCharacter, + GC.TYPE_CHOICE: _getCfgChoice, + GC.TYPE_INTEGER: _getCfgNumber, + GC.TYPE_FLOAT: _getCfgNumber, + GC.TYPE_HEADERFILTER: _getCfgHeaderFilter, + GC.TYPE_LOCALE: _getCfgLocale, + GC.TYPE_PASSWORD: _getCfgPassword, + GC.TYPE_STRING: _getCfgString, + GC.TYPE_STRINGLIST: _getCfgStringList, + GC.TYPE_HEADERFORCEREQUIRED: _getCfgStringList, + GC.TYPE_HEADERORDER: _getCfgStringList, + GC.TYPE_FILE: _getCfgFile, +} + + +def _initConfigParser(): + """Initialize config parser if not already done.""" if not GM.Globals[GM.PARSER]: homePath = os.path.expanduser('~') GM.Globals[GM.GAM_CFG_PATH] = os.environ.get(EV_GAMCFGDIR, None) @@ -694,7 +711,13 @@ def SetGlobalVariables(): else: GM.Globals[GM.PARSER] = configparser.RawConfigParser(defaults=collections.OrderedDict(sorted(list(GC.Defaults.items()), key=lambda t: t[0]))) _readGamCfgFile(GM.Globals[GM.PARSER], GM.Globals[GM.GAM_CFG_FILE]) - status = {'errors': False} + + +def _selectConfigSection(status): + """Determine which config section to use and process select/filter commands. + + Returns (sectionName, inputFilterSectionName, outputFilterSectionName). + """ inputFilterSectionName = outputFilterSectionName = None GM.Globals[GM.GAM_CFG_SECTION] = os.environ.get(EV_GAMCFGSECTION, None) if GM.Globals[GM.GAM_CFG_SECTION]: @@ -706,7 +729,7 @@ def SetGlobalVariables(): Cmd.Backup() usageErrorExit(formatKeyValueList('', [EV_GAMCFGSECTION, sectionName, 'select', Msg.NOT_ALLOWED], '')) else: - sectionName = _getCfgSection(configparser.DEFAULTSECT, GC.SECTION) + sectionName = _getCfgSection(status, configparser.DEFAULTSECT, GC.SECTION) # select [save] [verify [variables ]] if checkArgumentPresent(Cmd.SELECT_CMD): sectionName = _selectSection() @@ -728,7 +751,7 @@ def SetGlobalVariables(): GM.Globals[GM.GAM_CFG_SECTION_NAME] = sectionName # showsections if checkArgumentPresent(Cmd.SHOWSECTIONS_CMD): - _showSections() + _showSections(sectionName) # selectfilter|selectoutputfilter|selectinputfilter while True: filterCommand = getChoice([Cmd.SELECTFILTER_CMD, Cmd.SELECTOUTPUTFILTER_CMD, Cmd.SELECTINPUTFILTER_CMD], defaultChoice=None) @@ -738,21 +761,26 @@ def SetGlobalVariables(): outputFilterSectionName = _selectSection() else: inputFilterSectionName = _selectSection() + return sectionName, inputFilterSectionName, outputFilterSectionName + + +def _fixupTodriveAndConfig(status, sectionName, inputFilterSectionName, outputFilterSectionName): + """Handle todrive defaults, fix mistyped keywords, and process config command.""" # Handle todrive_nobrowser and todrive_noemail if not present value = GM.Globals[GM.PARSER].get(configparser.DEFAULTSECT, GC.TODRIVE_NOBROWSER) if value == '': - GM.Globals[GM.PARSER].set(configparser.DEFAULTSECT, GC.TODRIVE_NOBROWSER, str(_getCfgBoolean(configparser.DEFAULTSECT, GC.NO_BROWSER)).lower()) + GM.Globals[GM.PARSER].set(configparser.DEFAULTSECT, GC.TODRIVE_NOBROWSER, str(_getCfgBoolean(status, configparser.DEFAULTSECT, GC.NO_BROWSER)).lower()) value = GM.Globals[GM.PARSER].get(configparser.DEFAULTSECT, GC.TODRIVE_NOEMAIL) if value == '': - GM.Globals[GM.PARSER].set(configparser.DEFAULTSECT, GC.TODRIVE_NOEMAIL, str(not _getCfgBoolean(configparser.DEFAULTSECT, GC.NO_BROWSER)).lower()) + GM.Globals[GM.PARSER].set(configparser.DEFAULTSECT, GC.TODRIVE_NOEMAIL, str(not _getCfgBoolean(status, configparser.DEFAULTSECT, GC.NO_BROWSER)).lower()) # Handle todrive_sheet_timestamp and todrive_sheet_timeformat if not present for section in [sectionName, configparser.DEFAULTSECT]: value = GM.Globals[GM.PARSER].get(section, GC.TODRIVE_SHEET_TIMESTAMP) if value == 'copy': - GM.Globals[GM.PARSER].set(section, GC.TODRIVE_SHEET_TIMESTAMP, str(_getCfgBoolean(section, GC.TODRIVE_TIMESTAMP)).lower()) + GM.Globals[GM.PARSER].set(section, GC.TODRIVE_SHEET_TIMESTAMP, str(_getCfgBoolean(status, section, GC.TODRIVE_TIMESTAMP)).lower()) value = GM.Globals[GM.PARSER].get(section, GC.TODRIVE_SHEET_TIMEFORMAT) if value == 'copy': - GM.Globals[GM.PARSER].set(section, GC.TODRIVE_SHEET_TIMEFORMAT, _getCfgString(section, GC.TODRIVE_TIMEFORMAT)) + GM.Globals[GM.PARSER].set(section, GC.TODRIVE_SHEET_TIMEFORMAT, _getCfgString(status, section, GC.TODRIVE_TIMEFORMAT)) # Fix mistyped keyword cmdlog_max__backups for section in [configparser.DEFAULTSECT, sectionName]: if GM.Globals[GM.PARSER].has_option(section, GC.CMDLOG_MAX__BACKUPS): @@ -771,96 +799,88 @@ def SetGlobalVariables(): break itemEntry = GC.VAR_INFO[itemName] checkArgumentPresent('=') - varType = itemEntry[GC.VAR_TYPE] - if varType == GC.TYPE_BOOLEAN: - value = TRUE if getBoolean(None) else FALSE - elif varType == GC.TYPE_CHARACTER: - value = getCharacter() - elif varType == GC.TYPE_CHOICE: - value = getChoice(itemEntry[GC.VAR_CHOICES]) - elif varType == GC.TYPE_INTEGER: - minVal, maxVal = itemEntry[GC.VAR_LIMITS] - value = str(getInteger(minVal=minVal, maxVal=maxVal)) - elif varType == GC.TYPE_FLOAT: - minVal, maxVal = itemEntry[GC.VAR_LIMITS] - value = str(getFloat(minVal=minVal, maxVal=maxVal)) - elif varType == GC.TYPE_LOCALE: - value = getLanguageCode(LOCALE_CODES_MAP) - elif varType == GC.TYPE_PASSWORD: - minLen, maxLen = itemEntry[GC.VAR_LIMITS] - value = getString(Cmd.OB_STRING, checkBlank=True, minLen=minLen, maxLen=maxLen) - if value and value.startswith("b'") and value.endswith("'"): - value = bytes(value[2:-1], UTF8) - elif varType == GC.TYPE_TIMEZONE: - value = getString(Cmd.OB_STRING, checkBlank=True) - else: # GC.TYPE_STRING, GC.TYPE_STRINGLIST - minLen, maxLen = itemEntry.get(GC.VAR_LIMITS, (0, None)) - value = _quoteStringIfLeadingTrailingBlanks(getString(Cmd.OB_STRING, minLen=minLen, maxLen=maxLen)) + match varType: + case GC.TYPE_BOOLEAN: + value = TRUE if getBoolean(None) else FALSE + case GC.TYPE_CHARACTER: + value = getCharacter() + case GC.TYPE_CHOICE: + value = getChoice(itemEntry[GC.VAR_CHOICES]) + case GC.TYPE_INTEGER: + minVal, maxVal = itemEntry[GC.VAR_LIMITS] + value = str(getInteger(minVal=minVal, maxVal=maxVal)) + case GC.TYPE_FLOAT: + minVal, maxVal = itemEntry[GC.VAR_LIMITS] + value = str(getFloat(minVal=minVal, maxVal=maxVal)) + case GC.TYPE_LOCALE: + value = getLanguageCode(LOCALE_CODES_MAP) + case GC.TYPE_PASSWORD: + minLen, maxLen = itemEntry[GC.VAR_LIMITS] + value = getString(Cmd.OB_STRING, checkBlank=True, minLen=minLen, maxLen=maxLen) + if value and value.startswith("b'") and value.endswith("'"): + value = bytes(value[2:-1], UTF8) + case GC.TYPE_TIMEZONE: + value = getString(Cmd.OB_STRING, checkBlank=True) + case _: # GC.TYPE_STRING, GC.TYPE_STRINGLIST + minLen, maxLen = itemEntry.get(GC.VAR_LIMITS, (0, None)) + value = _quoteStringIfLeadingTrailingBlanks(getString(Cmd.OB_STRING, minLen=minLen, maxLen=maxLen)) GM.Globals[GM.PARSER].set(sectionName, itemName, value) + + +def _assignConfigValues(status, sectionName, inputFilterSectionName, outputFilterSectionName): + """Assign all GC.Values from config: directories, timezone, types, row filters, filter overrides.""" prevExtraArgsTxt = GC.Values.get(GC.EXTRA_ARGS, None) prevOauth2serviceJson = GC.Values.get(GC.OAUTH2SERVICE_JSON, None) # Assign global variables, directories, timezone first as other variables depend on them for itemName, itemEntry in sorted(GC.VAR_INFO.items()): varType = itemEntry[GC.VAR_TYPE] if varType == GC.TYPE_DIRECTORY: - GC.Values[itemName] = _getCfgDirectory(sectionName, itemName) + GC.Values[itemName] = _getCfgDirectory(status, sectionName, itemName) elif varType == GC.TYPE_TIMEZONE: - GC.Values[itemName] = _getCfgTimezone(sectionName, itemName) + GC.Values[itemName] = _getCfgTimezone(status, sectionName, itemName) GM.Globals[GM.DATETIME_NOW] = arrow.now(GC.Values[GC.TIMEZONE]) # Everything else except row filters for itemName, itemEntry in sorted(GC.VAR_INFO.items()): varType = itemEntry[GC.VAR_TYPE] - if varType == GC.TYPE_BOOLEAN: - GC.Values[itemName] = _getCfgBoolean(sectionName, itemName) - elif varType == GC.TYPE_CHARACTER: - GC.Values[itemName] = _getCfgCharacter(sectionName, itemName) - elif varType == GC.TYPE_CHOICE: - GC.Values[itemName] = _getCfgChoice(sectionName, itemName) - elif varType in [GC.TYPE_INTEGER, GC.TYPE_FLOAT]: - GC.Values[itemName] = _getCfgNumber(sectionName, itemName) - elif varType == GC.TYPE_HEADERFILTER: - GC.Values[itemName] = _getCfgHeaderFilter(sectionName, itemName) - elif varType == GC.TYPE_LOCALE: - GC.Values[itemName] = _getCfgLocale(sectionName, itemName) - elif varType == GC.TYPE_PASSWORD: - GC.Values[itemName] = _getCfgPassword(sectionName, itemName) - elif varType == GC.TYPE_STRING: - GC.Values[itemName] = _getCfgString(sectionName, itemName) - elif varType in {GC.TYPE_STRINGLIST, GC.TYPE_HEADERFORCEREQUIRED, GC.TYPE_HEADERORDER}: - GC.Values[itemName] = _getCfgStringList(sectionName, itemName) - elif varType == GC.TYPE_FILE: - GC.Values[itemName] = _getCfgFile(sectionName, itemName) + handler = _CFG_TYPE_HANDLERS.get(varType) + if handler: + GC.Values[itemName] = handler(status, sectionName, itemName) # Row filters for itemName, itemEntry in sorted(GC.VAR_INFO.items()): varType = itemEntry[GC.VAR_TYPE] if varType == GC.TYPE_ROWFILTER: - GC.Values[itemName] = _getCfgRowFilter(sectionName, itemName) + GC.Values[itemName] = _getCfgRowFilter(status, sectionName, itemName) # Process selectfilter|selectoutputfilter|selectinputfilter if inputFilterSectionName: - GC.Values[GC.CSV_INPUT_ROW_FILTER] = _getCfgRowFilter(inputFilterSectionName, GC.CSV_INPUT_ROW_FILTER) - GC.Values[GC.CSV_INPUT_ROW_FILTER_MODE] = _getCfgChoice(inputFilterSectionName, GC.CSV_INPUT_ROW_FILTER_MODE) - GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER] = _getCfgRowFilter(inputFilterSectionName, GC.CSV_INPUT_ROW_DROP_FILTER) - GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER_MODE] = _getCfgChoice(inputFilterSectionName, GC.CSV_INPUT_ROW_DROP_FILTER_MODE) - GC.Values[GC.CSV_INPUT_ROW_LIMIT] = _getCfgNumber(inputFilterSectionName, GC.CSV_INPUT_ROW_LIMIT) + GC.Values[GC.CSV_INPUT_ROW_FILTER] = _getCfgRowFilter(status, inputFilterSectionName, GC.CSV_INPUT_ROW_FILTER) + GC.Values[GC.CSV_INPUT_ROW_FILTER_MODE] = _getCfgChoice(status, inputFilterSectionName, GC.CSV_INPUT_ROW_FILTER_MODE) + GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER] = _getCfgRowFilter(status, inputFilterSectionName, GC.CSV_INPUT_ROW_DROP_FILTER) + GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER_MODE] = _getCfgChoice(status, inputFilterSectionName, GC.CSV_INPUT_ROW_DROP_FILTER_MODE) + GC.Values[GC.CSV_INPUT_ROW_LIMIT] = _getCfgNumber(status, inputFilterSectionName, GC.CSV_INPUT_ROW_LIMIT) if outputFilterSectionName: - GC.Values[GC.CSV_OUTPUT_HEADER_FORCE] = _getCfgStringList(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) + GC.Values[GC.CSV_OUTPUT_HEADER_FORCE] = _getCfgStringList(status, outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) if GC.Values[GC.CSV_OUTPUT_HEADER_FORCE]: - GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilterFromForce(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) + GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilterFromForce(status, outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) else: - GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilter(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FILTER) - GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER] = _getCfgHeaderFilter(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_DROP_FILTER) - GC.Values[GC.CSV_OUTPUT_HEADER_ORDER] = _getCfgStringList(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_ORDER) - GC.Values[GC.CSV_OUTPUT_HEADER_REQUIRED] = _getCfgStringList(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_REQUIRED) - GC.Values[GC.CSV_OUTPUT_ROW_FILTER] = _getCfgRowFilter(outputFilterSectionName, GC.CSV_OUTPUT_ROW_FILTER) - GC.Values[GC.CSV_OUTPUT_ROW_FILTER_MODE] = _getCfgChoice(outputFilterSectionName, GC.CSV_OUTPUT_ROW_FILTER_MODE) - GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER] = _getCfgRowFilter(outputFilterSectionName, GC.CSV_OUTPUT_ROW_DROP_FILTER) - GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE] = _getCfgChoice(outputFilterSectionName, GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE) - GC.Values[GC.CSV_OUTPUT_ROW_LIMIT] = _getCfgNumber(outputFilterSectionName, GC.CSV_OUTPUT_ROW_LIMIT) - GC.Values[GC.CSV_OUTPUT_SORT_HEADERS] = _getCfgStringList(outputFilterSectionName, GC.CSV_OUTPUT_SORT_HEADERS) + GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilter(status, outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FILTER) + GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER] = _getCfgHeaderFilter(status, outputFilterSectionName, GC.CSV_OUTPUT_HEADER_DROP_FILTER) + GC.Values[GC.CSV_OUTPUT_HEADER_ORDER] = _getCfgStringList(status, outputFilterSectionName, GC.CSV_OUTPUT_HEADER_ORDER) + GC.Values[GC.CSV_OUTPUT_HEADER_REQUIRED] = _getCfgStringList(status, outputFilterSectionName, GC.CSV_OUTPUT_HEADER_REQUIRED) + GC.Values[GC.CSV_OUTPUT_ROW_FILTER] = _getCfgRowFilter(status, outputFilterSectionName, GC.CSV_OUTPUT_ROW_FILTER) + GC.Values[GC.CSV_OUTPUT_ROW_FILTER_MODE] = _getCfgChoice(status, outputFilterSectionName, GC.CSV_OUTPUT_ROW_FILTER_MODE) + GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER] = _getCfgRowFilter(status, outputFilterSectionName, GC.CSV_OUTPUT_ROW_DROP_FILTER) + GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE] = _getCfgChoice(status, outputFilterSectionName, GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE) + GC.Values[GC.CSV_OUTPUT_ROW_LIMIT] = _getCfgNumber(status, outputFilterSectionName, GC.CSV_OUTPUT_ROW_LIMIT) + GC.Values[GC.CSV_OUTPUT_SORT_HEADERS] = _getCfgStringList(status, outputFilterSectionName, GC.CSV_OUTPUT_SORT_HEADERS) elif GC.Values[GC.CSV_OUTPUT_HEADER_FORCE]: - GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilterFromForce(sectionName, GC.CSV_OUTPUT_HEADER_FORCE) + GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilterFromForce(status, sectionName, GC.CSV_OUTPUT_HEADER_FORCE) if status['errors']: sys.exit(CONFIG_ERROR_RC) + return prevExtraArgsTxt, prevOauth2serviceJson + + +def _applyRuntimeDefaults(prevExtraArgsTxt, prevOauth2serviceJson): + """Apply runtime defaults: domain cleanup, inheritance, lockfile, debug, extra args.""" # Global values cleanup GC.Values[GC.DOMAIN] = GC.Values[GC.DOMAIN].lower() if not GC.Values[GC.SMTP_FQDN]: @@ -894,6 +914,10 @@ def SetGlobalVariables(): GM.Globals[GM.OAUTH2SERVICE_JSON_DATA] = {} GM.Globals[GM.OAUTH2SERVICE_CLIENT_ID] = None Cmd.SetEncoding(GM.Globals[GM.SYS_ENCODING]) + + +def _processRedirects(sectionName): + """Process multiprocessexit and redirect csv/stdout/stderr commands.""" # multiprocessexit (rc)|(rcrange=/)|(rcrange!=/) if checkArgumentPresent(Cmd.MULTIPROCESSEXIT_CMD): _setMultiprocessExit() @@ -909,66 +933,67 @@ def SetGlobalVariables(): while checkArgumentPresent(Cmd.REDIRECT_CMD): myarg = getChoice(['csv', 'stdout', 'stderr']) filename = re.sub(r'{{Section}}', sectionName, getString(Cmd.OB_FILE_NAME, checkBlank=True)) - if myarg == 'csv': - multi = False - mode = DEFAULT_FILE_WRITE_MODE - writeHeader = True - encoding = GC.Values[GC.CHARSET] - delayOpen = False - while Cmd.ArgumentsRemaining(): - myarg = getArgument() - if myarg == 'multiprocess': - multi = True - elif myarg == 'append': - mode = DEFAULT_FILE_APPEND_MODE - elif myarg == 'noheader': - writeHeader = False - elif myarg == 'charset': - encoding = getString(Cmd.OB_CHAR_SET) - elif myarg == 'delayopen': - delayOpen = True - elif myarg == 'columndelimiter': - GM.Globals[GM.CSV_OUTPUT_COLUMN_DELIMITER] = GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER] = getCharacter() - elif myarg == 'quotechar': - GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR] = GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR] = getCharacter() - elif myarg == 'noescapechar': - GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR] = getBoolean() - elif myarg == 'sortheaders': - GM.Globals[GM.CSV_OUTPUT_SORT_HEADERS] = GC.Values[GC.CSV_OUTPUT_SORT_HEADERS] = getString(Cmd.OB_STRING_LIST, minLen=0).replace(',', ' ').split() - elif myarg == 'timestampcolumn': - GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN] = getString(Cmd.OB_STRING, minLen=0) - elif myarg == 'transpose': - GM.Globals[GM.CSV_OUTPUT_TRANSPOSE] = getBoolean() + match myarg: + case 'csv': + multi = False + mode = DEFAULT_FILE_WRITE_MODE + writeHeader = True + encoding = GC.Values[GC.CHARSET] + delayOpen = False + while Cmd.ArgumentsRemaining(): + myarg = getArgument() + if myarg == 'multiprocess': + multi = True + elif myarg == 'append': + mode = DEFAULT_FILE_APPEND_MODE + elif myarg == 'noheader': + writeHeader = False + elif myarg == 'charset': + encoding = getString(Cmd.OB_CHAR_SET) + elif myarg == 'delayopen': + delayOpen = True + elif myarg == 'columndelimiter': + GM.Globals[GM.CSV_OUTPUT_COLUMN_DELIMITER] = GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER] = getCharacter() + elif myarg == 'quotechar': + GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR] = GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR] = getCharacter() + elif myarg == 'noescapechar': + GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR] = getBoolean() + elif myarg == 'sortheaders': + GM.Globals[GM.CSV_OUTPUT_SORT_HEADERS] = GC.Values[GC.CSV_OUTPUT_SORT_HEADERS] = getString(Cmd.OB_STRING_LIST, minLen=0).replace(',', ' ').split() + elif myarg == 'timestampcolumn': + GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN] = getString(Cmd.OB_STRING, minLen=0) + elif myarg == 'transpose': + GM.Globals[GM.CSV_OUTPUT_TRANSPOSE] = getBoolean() + else: + Cmd.Backup() + break + _setCSVFile(filename, mode, encoding, writeHeader, multi, delayOpen) + GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE_CSVPF] = CSVPrintFile() + if checkArgumentPresent('todrive'): + GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE_CSVPF].GetTodriveParameters() + GM.Globals[GM.CSV_TODRIVE] = GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE_CSVPF].todrive.copy() + case 'stdout': + if filename.lower() == 'null': + multi = checkArgumentPresent('multiprocess') + _setSTDFile(GM.STDOUT, 'null', DEFAULT_FILE_WRITE_MODE, multi) else: - Cmd.Backup() - break - _setCSVFile(filename, mode, encoding, writeHeader, multi, delayOpen) - GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE_CSVPF] = CSVPrintFile() - if checkArgumentPresent('todrive'): - GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE_CSVPF].GetTodriveParameters() - GM.Globals[GM.CSV_TODRIVE] = GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE_CSVPF].todrive.copy() - elif myarg == 'stdout': - if filename.lower() == 'null': - multi = checkArgumentPresent('multiprocess') - _setSTDFile(GM.STDOUT, 'null', DEFAULT_FILE_WRITE_MODE, multi) - else: - multi = checkArgumentPresent('multiprocess') - mode = DEFAULT_FILE_APPEND_MODE if checkArgumentPresent('append') else DEFAULT_FILE_WRITE_MODE - _setSTDFile(GM.STDOUT, filename, mode, multi) - else: # myarg == 'stderr' - if filename.lower() == 'null': - multi = checkArgumentPresent('multiprocess') - _setSTDFile(GM.STDERR, 'null', DEFAULT_FILE_WRITE_MODE, multi) - elif filename.lower() != 'stdout': - multi = checkArgumentPresent('multiprocess') - mode = DEFAULT_FILE_APPEND_MODE if checkArgumentPresent('append') else DEFAULT_FILE_WRITE_MODE - _setSTDFile(GM.STDERR, filename, mode, multi) - else: - multi = checkArgumentPresent('multiprocess') - if not GM.Globals[GM.STDOUT]: - _setSTDFile(GM.STDOUT, '-', DEFAULT_FILE_WRITE_MODE, multi) - GM.Globals[GM.STDERR] = GM.Globals[GM.STDOUT].copy() - GM.Globals[GM.STDERR][GM.REDIRECT_NAME] = 'stdout' + multi = checkArgumentPresent('multiprocess') + mode = DEFAULT_FILE_APPEND_MODE if checkArgumentPresent('append') else DEFAULT_FILE_WRITE_MODE + _setSTDFile(GM.STDOUT, filename, mode, multi) + case 'stderr': + if filename.lower() == 'null': + multi = checkArgumentPresent('multiprocess') + _setSTDFile(GM.STDERR, 'null', DEFAULT_FILE_WRITE_MODE, multi) + elif filename.lower() != 'stdout': + multi = checkArgumentPresent('multiprocess') + mode = DEFAULT_FILE_APPEND_MODE if checkArgumentPresent('append') else DEFAULT_FILE_WRITE_MODE + _setSTDFile(GM.STDERR, filename, mode, multi) + else: + multi = checkArgumentPresent('multiprocess') + if not GM.Globals[GM.STDOUT]: + _setSTDFile(GM.STDOUT, '-', DEFAULT_FILE_WRITE_MODE, multi) + GM.Globals[GM.STDERR] = GM.Globals[GM.STDOUT].copy() + GM.Globals[GM.STDERR][GM.REDIRECT_NAME] = 'stdout' if not GM.Globals[GM.STDOUT]: _setSTDFile(GM.STDOUT, '-', DEFAULT_FILE_WRITE_MODE, False) if not GM.Globals[GM.STDERR]: @@ -983,6 +1008,10 @@ def SetGlobalVariables(): elif not GM.Globals[GM.CSVFILE]: _setCSVFile('-', GM.Globals[GM.STDOUT].get(GM.REDIRECT_MODE, DEFAULT_FILE_WRITE_MODE), GC.Values[GC.CHARSET], True, False, False) initAPICallsRateCheck() + + +def _finalizeConfig(status, sectionName): + """Finalize config: filter inheritance, DASA validation, env vars, return.""" # Main process # Clear input row filters/limit from parser, children can define but shouldn't inherit global value # Clear output header/row filters/limit from parser, children can define or they will inherit global value if not defined @@ -996,16 +1025,16 @@ def SetGlobalVariables(): # Child process # Inherit main process output header/row filters/limit, print defaults if not locally defined else: - if not GC.Values[GC.CSV_OUTPUT_HEADER_FILTER]: - GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = GM.Globals[GM.CSV_OUTPUT_HEADER_FILTER][:] - if not GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER]: - GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER] = GM.Globals[GM.CSV_OUTPUT_HEADER_DROP_FILTER][:] - if not GC.Values[GC.CSV_OUTPUT_HEADER_FORCE]: - GC.Values[GC.CSV_OUTPUT_HEADER_FORCE] = GM.Globals[GM.CSV_OUTPUT_HEADER_FORCE][:] - if not GC.Values[GC.CSV_OUTPUT_HEADER_ORDER]: - GC.Values[GC.CSV_OUTPUT_HEADER_ORDER] = GM.Globals[GM.CSV_OUTPUT_HEADER_ORDER][:] - if not GC.Values[GC.CSV_OUTPUT_HEADER_REQUIRED]: - GC.Values[GC.CSV_OUTPUT_HEADER_REQUIRED] = GM.Globals[GM.CSV_OUTPUT_HEADER_REQUIRED][:] + _CHILD_INHERIT_LIST_ITEMS = [ + (GC.CSV_OUTPUT_HEADER_FILTER, GM.CSV_OUTPUT_HEADER_FILTER), + (GC.CSV_OUTPUT_HEADER_DROP_FILTER, GM.CSV_OUTPUT_HEADER_DROP_FILTER), + (GC.CSV_OUTPUT_HEADER_FORCE, GM.CSV_OUTPUT_HEADER_FORCE), + (GC.CSV_OUTPUT_HEADER_ORDER, GM.CSV_OUTPUT_HEADER_ORDER), + (GC.CSV_OUTPUT_HEADER_REQUIRED, GM.CSV_OUTPUT_HEADER_REQUIRED), + ] + for gcItem, gmItem in _CHILD_INHERIT_LIST_ITEMS: + if not GC.Values[gcItem]: + GC.Values[gcItem] = GM.Globals[gmItem][:] if not GC.Values[GC.CSV_OUTPUT_ROW_FILTER]: GC.Values[GC.CSV_OUTPUT_ROW_FILTER] = GM.Globals[GM.CSV_OUTPUT_ROW_FILTER][:] GC.Values[GC.CSV_OUTPUT_ROW_FILTER_MODE] = GM.Globals[GM.CSV_OUTPUT_ROW_FILTER_MODE] @@ -1014,12 +1043,14 @@ def SetGlobalVariables(): GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE] = GM.Globals[GM.CSV_OUTPUT_ROW_DROP_FILTER_MODE] if not GC.Values[GC.CSV_OUTPUT_ROW_LIMIT]: GC.Values[GC.CSV_OUTPUT_ROW_LIMIT] = GM.Globals[GM.CSV_OUTPUT_ROW_LIMIT] - if not GC.Values[GC.PRINT_AGU_DOMAINS]: - GC.Values[GC.PRINT_AGU_DOMAINS] = GM.Globals[GM.PRINT_AGU_DOMAINS] - if not GC.Values[GC.PRINT_CROS_OUS]: - GC.Values[GC.PRINT_CROS_OUS] = GM.Globals[GM.PRINT_CROS_OUS] - if not GC.Values[GC.PRINT_CROS_OUS_AND_CHILDREN]: - GC.Values[GC.PRINT_CROS_OUS_AND_CHILDREN] = GM.Globals[GM.PRINT_CROS_OUS_AND_CHILDREN] + _CHILD_INHERIT_SCALAR_ITEMS = [ + (GC.PRINT_AGU_DOMAINS, GM.PRINT_AGU_DOMAINS), + (GC.PRINT_CROS_OUS, GM.PRINT_CROS_OUS), + (GC.PRINT_CROS_OUS_AND_CHILDREN, GM.PRINT_CROS_OUS_AND_CHILDREN), + ] + for gcItem, gmItem in _CHILD_INHERIT_SCALAR_ITEMS: + if not GC.Values[gcItem]: + GC.Values[gcItem] = GM.Globals[gmItem] GC.Values[GC.SHOW_GETTINGS] = GM.Globals[GM.SHOW_GETTINGS] GC.Values[GC.SHOW_GETTINGS_GOT_NL] = GM.Globals[GM.SHOW_GETTINGS_GOT_NL] # customer_id, domain and admin_email must be set when enable_dasa = true @@ -1042,7 +1073,7 @@ def SetGlobalVariables(): if (Cmd.Location() == 1) or (Cmd.ArgumentsRemaining()): _chkCfgDirectories(sectionName) if not Cmd.PeekArgumentPresent(['checkconn', 'checkconnection', 'comment', 'oauth', 'oauth2', 'version']): - _chkCfgFiles(sectionName) + _chkCfgFiles(status, sectionName) if status['errors']: sys.exit(CONFIG_ERROR_RC) if GC.Values[GC.NO_CACHE]: @@ -1066,3 +1097,15 @@ def SetGlobalVariables(): # We're done, nothing else to do return False + +def SetGlobalVariables(): + _initConfigParser() + status = {'errors': False} + sectionName, inputFilterSectionName, outputFilterSectionName = _selectConfigSection(status) + _fixupTodriveAndConfig(status, sectionName, inputFilterSectionName, outputFilterSectionName) + prevExtraArgsTxt, prevOauth2serviceJson = _assignConfigValues(status, sectionName, inputFilterSectionName, outputFilterSectionName) + _applyRuntimeDefaults(prevExtraArgsTxt, prevOauth2serviceJson) + _processRedirects(sectionName) + return _finalizeConfig(status, sectionName) + +