From c9cda88f7f2c595e84a959490eae0e46169d5b05 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Mon, 7 Aug 2023 12:19:10 -0700 Subject: [PATCH] Added `output_dateformat` and `output_timeformat` variables to `gam.cfg` --- src/GamUpdate.txt | 8 +++++++ src/gam/__init__.py | 48 ++++++++++++++++++++++++++++--------- src/gam/gamlib/glcfg.py | 8 +++++++ src/gam/gamlib/glglobals.py | 6 +++++ 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/GamUpdate.txt b/src/GamUpdate.txt index 1cdb13d1..4d9a40cc 100644 --- a/src/GamUpdate.txt +++ b/src/GamUpdate.txt @@ -2,6 +2,14 @@ Merged GAM-Team version +6.62.00 + +Added `output_dateformat` and `output_timeformat` variables to `gam.cfg` that provide alternate +output date and time formats that may be required by programs that will be processing the data. +GAM will not accept alternate date/time formats as input. + +* See: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes + 6.61.21 Updated `gam empty drivetrash ` to use new Drive API v3 diff --git a/src/gam/__init__.py b/src/gam/__init__.py index ceb374d4..b207ebb9 100755 --- a/src/gam/__init__.py +++ b/src/gam/__init__.py @@ -2309,20 +2309,33 @@ def formatLocalTime(dateTimeStr): return GC.Values[GC.NEVER_TIME] try: timestamp, _ = iso8601.parse_date(dateTimeStr) + if not GC.Values[GC.OUTPUT_TIMEFORMAT]: + if GM.Globals[GM.CONVERT_TO_LOCAL_TIME]: + return ISOformatTimeStamp(timestamp.astimezone(GC.Values[GC.TIMEZONE])) + return timestamp.strftime(YYYYMMDDTHHMMSSZ_FORMAT) if GM.Globals[GM.CONVERT_TO_LOCAL_TIME]: - return ISOformatTimeStamp(timestamp.astimezone(GC.Values[GC.TIMEZONE])) - return timestamp.strftime(YYYYMMDDTHHMMSSZ_FORMAT) + return timestamp.astimezone(GC.Values[GC.TIMEZONE]).strftime(GC.Values[GC.OUTPUT_TIMEFORMAT]) + return timestamp.strftime(GC.Values[GC.OUTPUT_TIMEFORMAT]) except (iso8601.ParseError, OverflowError): return dateTimeStr def formatLocalSecondsTimestamp(timestamp): - return ISOformatTimeStamp(datetime.datetime.fromtimestamp(int(timestamp), GC.Values[GC.TIMEZONE])) + if not GC.Values[GC.OUTPUT_TIMEFORMAT]: + return ISOformatTimeStamp(datetime.datetime.fromtimestamp(int(timestamp), GC.Values[GC.TIMEZONE])) + return datetime.datetime.fromtimestamp(int(timestamp), GC.Values[GC.TIMEZONE]).strftime(GC.Values[GC.OUTPUT_TIMEFORMAT]) def formatLocalTimestamp(timestamp): - return ISOformatTimeStamp(datetime.datetime.fromtimestamp(int(timestamp)//1000, GC.Values[GC.TIMEZONE])) + if not GC.Values[GC.OUTPUT_TIMEFORMAT]: + return ISOformatTimeStamp(datetime.datetime.fromtimestamp(int(timestamp)//1000, GC.Values[GC.TIMEZONE])) + return datetime.datetime.fromtimestamp(int(timestamp)//1000, GC.Values[GC.TIMEZONE]).strftime(GC.Values[GC.OUTPUT_TIMEFORMAT]) + +def formatLocalTimestampUTC(timestamp): + return ISOformatTimeStamp(datetime.datetime.fromtimestamp(int(timestamp)//1000, iso8601.UTC)) def formatLocalDatestamp(timestamp): - return datetime.datetime.fromtimestamp(int(timestamp)//1000, GC.Values[GC.TIMEZONE]).strftime(YYYYMMDD_FORMAT) + if not GC.Values[GC.OUTPUT_DATEFORMAT]: + return datetime.datetime.fromtimestamp(int(timestamp)//1000, GC.Values[GC.TIMEZONE]).strftime(YYYYMMDD_FORMAT) + return datetime.datetime.fromtimestamp(int(timestamp)//1000, GC.Values[GC.TIMEZONE]).strftime(GC.Values[GC.OUTPUT_DATEFORMAT]) def formatMaxMessageBytes(maxMessageBytes, oneKiloBytes, oneMegaBytes): if maxMessageBytes < oneKiloBytes: @@ -3979,9 +3992,14 @@ def SetGlobalVariables(): GC.Values[GC.DOMAIN] = GC.Values[GC.DOMAIN].lower() if not GC.Values[GC.SMTP_FQDN]: GC.Values[GC.SMTP_FQDN] = None -# Inherit debug_level if not locally defined - if GM.Globals[GM.PID] != 0 and GC.Values[GC.DEBUG_LEVEL] == 0: - GC.Values[GC.DEBUG_LEVEL] = GM.Globals[GM.DEBUG_LEVEL] +# Inherit debug_level, output_dateformat, output_timeformat if not locally defined + if GM.Globals[GM.PID] != 0: + if GC.Values[GC.DEBUG_LEVEL] == 0: + GC.Values[GC.DEBUG_LEVEL] = GM.Globals[GM.DEBUG_LEVEL] + if not GC.Values[GC.OUTPUT_DATEFORMAT]: + GC.Values[GC.OUTPUT_DATEFORMAT] = GM.Globals[GM.OUTPUT_DATEFORMAT] + if not GC.Values[GC.OUTPUT_TIMEFORMAT]: + GC.Values[GC.OUTPUT_TIMEFORMAT] = GM.Globals[GM.OUTPUT_TIMEFORMAT] # Create/set mode for oauth2.txt.lock if not GM.Globals[GM.OAUTH2_TXT_LOCK]: fileName = f'{GC.Values[GC.OAUTH2_TXT]}.lock' @@ -9251,6 +9269,7 @@ def terminateStdQueueHandler(mpQueue, mpQueueHandler): def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, mpQueueStderr, debugLevel, todrive, + output_dateformat, output_timeformat, csvColumnDelimiter, csvQuoteChar, csvTimestampColumn, csvHeaderFilter, csvHeaderDropFilter, @@ -9284,6 +9303,8 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = csvTimestampColumn GM.Globals[GM.CSV_TODRIVE] = todrive.copy() GM.Globals[GM.DEBUG_LEVEL] = debugLevel + GM.Globals[GM.OUTPUT_DATEFORMAT] = output_dateformat + GM.Globals[GM.OUTPUT_TIMEFORMAT] = output_timeformat GM.Globals[GM.NUM_BATCH_ITEMS] = numItems GM.Globals[GM.PID] = pid GM.Globals[GM.SAVED_STDOUT] = None @@ -9455,6 +9476,7 @@ def MultiprocessGAMCommands(items, showCmds): poolProcessResults[pid] = pool.apply_async(ProcessGAMCommandMulti, [pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, mpQueueStderr, GC.Values[GC.DEBUG_LEVEL], GM.Globals[GM.CSV_TODRIVE], + GC.Values[GC.OUTPUT_DATEFORMAT], GC.Values[GC.OUTPUT_TIMEFORMAT], GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER], GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR], GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN], @@ -15315,6 +15337,10 @@ def doInfoCustomer(returnCustomerInfo=None, FJQC=None): customerInfo = callGAPI(cd.customers(), 'get', throwReasons=[GAPI.BAD_REQUEST, GAPI.INVALID_INPUT, GAPI.RESOURCE_NOT_FOUND, GAPI.FORBIDDEN], customerKey=customerId) + if 'customerCreationTime' in customerInfo: + customerInfo['customerCreationTime'] = formatLocalTime(customerInfo['customerCreationTime']) + else: + customerInfo['customerCreationTime'] = UNKNOWN primaryDomain = {'domainName': UNKNOWN, 'verified': UNKNOWN} try: domains = callGAPIitems(cd.domains(), 'list', 'domains', @@ -15329,10 +15355,10 @@ def doInfoCustomer(returnCustomerInfo=None, FJQC=None): # We should get all domains and use oldest date customerCreationTime = UNKNOWN for domain in domains: - domainCreationTime = formatLocalTimestamp(domain['creationTime']) + domainCreationTime = formatLocalTimestampUTC(domain['creationTime']) if customerCreationTime == UNKNOWN or domainCreationTime < customerCreationTime: customerCreationTime = domainCreationTime - customerInfo['customerCreationTime'] = customerCreationTime + customerInfo['customerCreationTime'] = formatLocalTime(customerCreationTime) except (GAPI.badRequest, GAPI.notFound): pass customerInfo['customerDomain'] = primaryDomain['domainName'] @@ -15347,7 +15373,7 @@ def doInfoCustomer(returnCustomerInfo=None, FJQC=None): printKeyValueList(['Customer ID', customerInfo['id']]) printKeyValueList(['Primary Domain', customerInfo['customerDomain']]) printKeyValueList(['Primary Domain Verified', customerInfo['verified']]) - printKeyValueList(['Customer Creation Time', customerInfo.get('customerCreationTime', UNKNOWN)]) + printKeyValueList(['Customer Creation Time', customerInfo['customerCreationTime']]) printKeyValueList(['Default Language', customerInfo.get('language', 'Unset or Unknown (defaults to en)')]) _showCustomerAddressPhoneNumber(customerInfo) printKeyValueList(['Admin Secondary Email', customerInfo.get('alternateEmail', UNKNOWN)]) diff --git a/src/gam/gamlib/glcfg.py b/src/gam/gamlib/glcfg.py index 6d9af228..6dc09b7e 100644 --- a/src/gam/gamlib/glcfg.py +++ b/src/gam/gamlib/glcfg.py @@ -192,6 +192,10 @@ NUM_THREADS = 'num_threads' OAUTH2_TXT = 'oauth2_txt' # Path to oauth2service.json OAUTH2SERVICE_JSON = 'oauth2service_json' +# Output date format, empty defalts to ISOFormat +OUTPUT_DATEFORMAT = 'output_dateformat' +# Output time format, empty defalts to ISOFormat +OUTPUT_TIMEFORMAT = 'output_timeformat' # When retrieving lists of people from API, how many should be retrieved in each chunk PEOPLE_MAX_RESULTS = 'people_max_results' # Use quick method to move Chromebooks to OU @@ -354,6 +358,8 @@ Defaults = { NUM_THREADS: '5', OAUTH2_TXT: FN_OAUTH2_TXT, OAUTH2SERVICE_JSON: FN_OAUTH2SERVICE_JSON, + OUTPUT_DATEFORMAT: '', + OUTPUT_TIMEFORMAT: '', PEOPLE_MAX_RESULTS: '100', QUICK_CROS_MOVE: FALSE, QUICK_INFO_USER: FALSE, @@ -499,6 +505,8 @@ VAR_INFO = { NUM_THREADS: {VAR_TYPE: TYPE_INTEGER, VAR_ENVVAR: 'GAM_THREADS', VAR_LIMITS: (1, 1000)}, OAUTH2_TXT: {VAR_TYPE: TYPE_FILE, VAR_ENVVAR: 'OAUTHFILE', VAR_ACCESS: os.R_OK | os.W_OK}, OAUTH2SERVICE_JSON: {VAR_TYPE: TYPE_FILE, VAR_ENVVAR: 'OAUTHSERVICEFILE', VAR_ACCESS: os.R_OK | os.W_OK}, + OUTPUT_DATEFORMAT: {VAR_TYPE: TYPE_STRING, VAR_LIMITS: (0, None)}, + OUTPUT_TIMEFORMAT: {VAR_TYPE: TYPE_STRING, VAR_LIMITS: (0, None)}, PEOPLE_MAX_RESULTS: {VAR_TYPE: TYPE_INTEGER, VAR_LIMITS: (0, 1000)}, QUICK_CROS_MOVE: {VAR_TYPE: TYPE_BOOLEAN}, QUICK_INFO_USER: {VAR_TYPE: TYPE_BOOLEAN}, diff --git a/src/gam/gamlib/glglobals.py b/src/gam/gamlib/glglobals.py index 19118312..5b163df7 100644 --- a/src/gam/gamlib/glglobals.py +++ b/src/gam/gamlib/glglobals.py @@ -148,6 +148,10 @@ OAUTH2SERVICE_JSON_DATA = 'osjd' OAUTH2_CLIENT_ID = 'oaci' # oauth2.txt lock file OAUTH2_TXT_LOCK = 'oatl' +# Output date format, empty defalts to ISOFormat +OUTPUT_DATEFORMAT = 'oudf' +# Output time format, empty defalts to ISOFormat +OUTPUT_TIMEFORMAT = 'outf' # gam.cfg parser PARSER = 'pars' # Process ID @@ -261,6 +265,8 @@ Globals = { OAUTH2SERVICE_JSON_DATA: {}, OAUTH2_CLIENT_ID: None, OAUTH2_TXT_LOCK: None, + OUTPUT_DATEFORMAT: '', + OUTPUT_TIMEFORMAT: '', PARSER: None, PID: 0, RATE_CHECK_COUNT: 0,