From 435388aa0ba8bae3b3428f955d0d14ec66648ecb Mon Sep 17 00:00:00 2001
From: Ross Scroggs <ross.scroggs@gmail.com>
Date: Thu, 27 Jun 2024 10:11:06 -0700
Subject: [PATCH] gam report updates

---
 docs/GamUpdates.md  | 11 +++++++++++
 docs/Reports.md     | 16 ++++++++++++----
 src/GamCommands.txt |  4 ++--
 src/GamUpdate.txt   | 11 +++++++++++
 src/gam/__init__.py | 46 +++++++++++++++++++++++++++++----------------
 5 files changed, 66 insertions(+), 22 deletions(-)

diff --git a/docs/GamUpdates.md b/docs/GamUpdates.md
index 8b315b73..82d31aa5 100644
--- a/docs/GamUpdates.md
+++ b/docs/GamUpdates.md
@@ -21,6 +21,17 @@ Thanks to jay, added the following Colab License SKUs:
 Thanks to Jay, updated `gam print|show admins` to properly display addresses
 of service accounts with admin role assignments.
 
+Added option `limitdatechanges <Integer>` to `gam report users|customers`.
+
+If no report is available for the specified date, can an earlier date be used?
+* `limitdatechanges -1' - Back up to earlier dates to find report data; this is the default.
+* `limitdatechanges 0 | nodatechange' - Do not report on an earlier date if no report data is available for the specified date.
+* `limitdatechanges N' - Back up to earlier dates to find report data; do not back up more than N times.
+
+By default, when `gam report user user <UserItem>` is specified and no report data is available, there is no output.
+If `csv_output_users_audit = true` in `gam.cfg`, then a row with columns `email,date` will be displayed
+where `date` is the earliest date for which report data was requested.
+
 ### 6.77.02
 
 Cleaned up problems with some of the new Chat API asadmin commands.
diff --git a/docs/Reports.md b/docs/Reports.md
index a8e5f407..def6e4b6 100644
--- a/docs/Reports.md
+++ b/docs/Reports.md
@@ -234,7 +234,7 @@ Customer reports are generally available up to two days before the current date.
 gam report customers|customer|domain [todrive <ToDriveAttributes>*]
         [(date <Date>)|(range <Date> <Date>)|
          yesterday|today|thismonth|(previousmonths <Integer>)]
-        [nodatechange|(fulldatarequired all|<CustomerServiceNameList>)]
+        [(nodatechange | limitdatechanges <Integer>) | (fulldatarequired all|<CustomerServiceNameList>)]
         [(fields|parameters <String>)|(services <CustomerServiceNameList>)]
         [noauthorizedapps]
 ```
@@ -247,7 +247,9 @@ Specify the report date; the default is today's date.
 * `previousmonths <Integer>` - A number in the range 1 to 6 indicating calendar months previous to the current month; there is an API call per date
 
 If no report is available for the specified date, can an earlier date be used?
-* `nodatechange` - Do not report on an earlier date if no report is available for the specified date.
+* `limitdatechanges -1' - Back up to earlier dates to find report data; this is the default.
+* `limitdatechanges 0 | nodatechange' - Do not report on an earlier date if no report data is available for the specified date.
+* `limitdatechanges N' - Back up to earlier dates to find report data; do not back up more than N times.
 
 If only partial report data is available for the specified date and applications, can an earlier date be used?
 * `fulldatarequired all` - Back up to an earlier date to get complete data until all applications have full report data
@@ -328,7 +330,7 @@ gam report users|user [todrive <ToDriveAttributes>*]
         [allverifyuser <UserItem>]
         [(date <Date>)|(range <Date> <Date>)|
          yesterday|today|thismonth|(previousmonths <Integer>)]
-        [nodatechange|(fulldatarequired all|<UserServiceNameList>)]
+        [(nodatechange | limitdatechanges <Integer>) | (fulldatarequired all|<UserServiceNameList>)]
         [filtertime.* <Time>] [filter|filters <String>]
         [(fields|parameters <String>)|(services <UserServiceNameList>)]
         [aggregatebydate|aggregatebyuser [Boolean]]
@@ -353,12 +355,18 @@ Specify the report date; the default is today's date.
 * `previousmonths <Integer>` - A number in the range 1 to 6 indicating calendar months previous to the current month; there is an API call per date
 
 If no report is available for the specified date, can an earlier date be used?
-* `nodatechange` - Do not report on an earlier date if no report is available for the specified date.
+* `limitdatechanges -1' - Back up to earlier dates to find report data; this is the default.
+* `limitdatechanges 0 | nodatechange' - Do not report on an earlier date if no report data is available for the specified date.
+* `limitdatechanges N' - Back up to earlier dates to find report data; do not back up more than N times.
 
 If only partial report data is available for the specified date and applications, can an earlier date be used?
 * `fulldatarequired all` - Back up to an earlier date to get complete data until all applications have full report data
 * `fulldatarequired <UserServiceNameList>` - Back up to an earlier date to get complete data until all applications in `<UserServiceNameList>` have full report data
 
+By default, when `user <UserItem>` is specified and no report data is available, there is no output.
+If `csv_output_users_audit = true` in `gam.cfg`, then a row with columns `email,date` will be displayed
+where `date` is the earliest date for which report data was requested.
+
 Apply filters.
 * `filter|filters <String>` - `<String>` is a comma separated list of filter expressions.
 
diff --git a/src/GamCommands.txt b/src/GamCommands.txt
index a6beec5c..9fe418c8 100644
--- a/src/GamCommands.txt
+++ b/src/GamCommands.txt
@@ -4331,7 +4331,7 @@ gam report <ActivityApplicationName> [todrive <ToDriveAttribute>*]
 gam report customers|customer|domain [todrive <ToDriveAttribute>*]
         [(date <Date>)|(range <Date> <Date>)|
           yesterday|today|thismonth|(previousmonths <Integer>)]
-        [nodatechange | (fulldatarequired all|<CustomerServiceNameList>)]
+        [(nodatechange | limitdatechanges <Integer>) | (fulldatarequired all|<CustomerServiceNameList>)]
         [(fields|parameters <String>)|(services <CustomerServiceNameList>)] [noauthorizedapps]
 
 <UserServiceName> ::=
@@ -4348,7 +4348,7 @@ gam report users|user [todrive <ToDriveAttribute>*]
         [allverifyuser <UserItem>]
         [(date <Date>)|(range <Date> <Date>)|
           yesterday|today|thismonth|(previousmonths <Integer>)]
-        [nodatechange | (fulldatarequired all|<UserServiceNameList>)]
+        [(nodatechange | limitdatechanges <Integer>) | (fulldatarequired all|<UserServiceNameList>)]
         [filtertime.* <Time>] [filter|filters <String>]
         [(fields|parameters <String>)|(services <UserServiceNameList>)]
         [aggregatebydate|aggregatebyuser [Boolean]]
diff --git a/src/GamUpdate.txt b/src/GamUpdate.txt
index 723c5bdf..358e6de5 100644
--- a/src/GamUpdate.txt
+++ b/src/GamUpdate.txt
@@ -1,3 +1,14 @@
+Added option `limitdatechanges <Integer>` to `gam report users|customers`.
+
+If no report is available for the specified date, can an earlier date be used?
+* `limitdatechanges -1' - Back up to earlier dates to find report data; this is the default.
+* `limitdatechanges 0 | nodatechange' - Do not report on an earlier date if no report data is available for the specified date.
+* `limitdatechanges N' - Back up to earlier dates to find report data; do not back up more than N times.
+
+By default, when `gam report user user <UserItem>` is specified and no report data is available, there is no output.
+If `csv_output_users_audit = true` in `gam.cfg`, then a row with columns `email,date` will be displayed
+where `date` is the earliest date for which report data was requested.
+
 7.00.00
 
 Merged GAM-Team version
diff --git a/src/gam/__init__.py b/src/gam/__init__.py
index 86299858..4e350110 100755
--- a/src/gam/__init__.py
+++ b/src/gam/__init__.py
@@ -11735,8 +11735,8 @@ def doCreateProject():
 #	 (yubikey yubikey_pin yubikey_slot AUTHENTICATION yubikey_serialnumber <String>)]
 def doUseProject():
   _checkForExistingProjectFiles([GC.Values[GC.OAUTH2SERVICE_JSON], GC.Values[GC.CLIENT_SECRETS_JSON]])
-  _, httpObj, login_hint, _, projectInfo, svcAcctInfo = _getLoginHintProjectInfo(False)
-  _createClientSecretsOauth2service(httpObj, login_hint, {}, projectInfo, svcAcctInfo)
+  _, httpObj, login_hint, _, projectInfo, svcAcctInfo, create_key = _getLoginHintProjectInfo(False)
+  _createClientSecretsOauth2service(httpObj, login_hint, {}, projectInfo, svcAcctInfo, create_key)
 
 # gam update project [[admin] <EmailAddress>] [<ProjectIDEntity>]
 def doUpdateProject():
@@ -12910,7 +12910,7 @@ def doWhatIs():
     entityUnknownWarning(Ent.EMAIL, email)
     setSysExitRC(ENTITY_IS_UKNOWN_RC)
 
-def _adjustTryDate(errMsg, noDateChange, prevTryDate):
+def _adjustTryDate(errMsg, numDateChanges, limitDateChanges, prevTryDate):
   match_date = re.match('Data for dates later than (.*) is not yet available. Please check back later', errMsg)
   if match_date:
     tryDate = match_date.group(1)
@@ -12923,7 +12923,7 @@ def _adjustTryDate(errMsg, noDateChange, prevTryDate):
       if match_date:
         tryDateTime = datetime.datetime.strptime(prevTryDate, YYYYMMDD_FORMAT)-datetime.timedelta(days=1)
         tryDate = tryDateTime.strftime(YYYYMMDD_FORMAT)
-  if (not match_date) or noDateChange:
+  if (not match_date) or (numDateChanges > limitDateChanges >= 0):
     printWarningMessage(DATA_NOT_AVALIABLE_RC, errMsg)
     return None
   return tryDate
@@ -13034,7 +13034,7 @@ def doReportUsageParameters():
       printErrorMessage(BAD_REQUEST_RC, Msg.BAD_REQUEST)
       return
     except GAPI.invalid as e:
-      tryDate = _adjustTryDate(str(e), False, tryDate)
+      tryDate = _adjustTryDate(str(e), 0, -1, tryDate)
       if not tryDate:
         break
   for parameter in sorted(allParameters):
@@ -13610,8 +13610,9 @@ def doReport():
   filterTimes = {}
   maxActivities = maxEvents = 0
   maxResults = 1000
-  aggregateByDate = aggregateByUser = countsOnly = eventRowFilter = exitUserLoop = noAuthorizedApps = noDateChange = \
+  aggregateByDate = aggregateByUser = countsOnly = eventRowFilter = exitUserLoop = noAuthorizedApps = \
     normalizeUsers = select = summary = userCustomerRange = False
+  limitDateChanges = -1
   allVerifyUser = userKey = 'all'
   cd = orgUnit = orgUnitId = None
   userOrgUnits = {}
@@ -13651,9 +13652,12 @@ def doReport():
       startEndTime.Get('start' if myarg == 'date' else myarg)
       startEndTime.endDateTime = startEndTime.startDateTime
       userCustomerRange = False
-    elif usageReports and myarg == 'nodatechange':
-      noDateChange = True
-      if (startEndTime.startDateTime is not None) and (startEndTime.endDateTime == startEndTime.startDateTime):
+    elif usageReports and myarg in {'nodatechange', 'limitdatechanges'}:
+      if myarg == 'nodatechange':
+        limitDateChanges = 0
+      else:
+        limitDateChanges = getInteger(minVal=-1)
+      if (limitDateChanges == 0) and (startEndTime.startDateTime is not None) and (startEndTime.endDateTime == startEndTime.startDateTime):
         userCustomerRange = True
     elif usageReports and myarg in {'fields', 'parameters'}:
       for field in getString(Cmd.OB_STRING).replace(',', ' ').split():
@@ -13787,6 +13791,7 @@ def doReport():
       startDateTime = startEndTime.startDateTime
       endDateTime = startEndTime.endDateTime
       lastDate = None
+      numDateChanges = 0
       while startDateTime <= endDateTime:
         tryDate = startDateTime.strftime(YYYYMMDD_FORMAT)
         try:
@@ -13796,13 +13801,15 @@ def doReport():
                               userKey=verifyUser, date=tryDate, customerId=customerId,
                               orgUnitID=orgUnitId, parameters=parameters,
                               fields='warnings,usageReports', maxResults=1)
+            prevTryDate = tryDate
             fullData, tryDate, usageReports = _checkDataRequiredServices(result, tryDate,
                                                                          dataRequiredServices, parameterServices, True)
             if fullData < 0:
               printWarningMessage(DATA_NOT_AVALIABLE_RC, Msg.NO_REPORT_AVAILABLE.format(report))
               break
+            numDateChanges += 1
             if fullData == 0:
-              if noDateChange:
+              if numDateChanges > limitDateChanges >= 0:
                 break
               startDateTime = endDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
               continue
@@ -13810,6 +13817,7 @@ def doReport():
             pageMessage = getPageMessageForWhom(forWhom, showDate=tryDate)
           else:
             pageMessage = getPageMessageForWhom(user, showDate=tryDate)
+          prevTryDate = tryDate
           usage = callGAPIpages(service, 'get', 'usageReports',
                                 pageMessage=pageMessage,
                                 throwReasons=[GAPI.INVALID, GAPI.INVALID_INPUT, GAPI.BAD_REQUEST, GAPI.FORBIDDEN],
@@ -13825,7 +13833,8 @@ def doReport():
           if not status:
             break
         except GAPI.invalid as e:
-          tryDate = _adjustTryDate(str(e), noDateChange, tryDate)
+          numDateChanges += 1
+          tryDate = _adjustTryDate(str(e), numDateChanges, limitDateChanges, tryDate)
           if not tryDate:
             break
           startDateTime = endDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
@@ -13844,6 +13853,8 @@ def doReport():
         startDateTime += oneDay
       if exitUserLoop:
         break
+      if user != 'all' and lastDate is None and GC.Values[GC.CSV_OUTPUT_USERS_AUDIT]:
+        csvPF.WriteRowNoFilter({'date': prevTryDate, 'email': user})
     if aggregateByDate:
       for usageDate, events in iter(eventCounts.items()):
         row = {'date': usageDate}
@@ -13875,6 +13886,7 @@ def doReport():
     startDateTime = startEndTime.startDateTime
     endDateTime = startEndTime.endDateTime
     lastDate = None
+    numDateChanges = 0
     while startDateTime <= endDateTime:
       tryDate = startDateTime.strftime(YYYYMMDD_FORMAT)
       try:
@@ -13887,8 +13899,9 @@ def doReport():
           if fullData < 0:
             printWarningMessage(DATA_NOT_AVALIABLE_RC, Msg.NO_REPORT_AVAILABLE.format(report))
             break
+          numDateChanges += 1
           if fullData == 0:
-            if noDateChange:
+            if numDateChanges > limitDateChanges >= 0:
               break
             startDateTime = endDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
             continue
@@ -13902,7 +13915,8 @@ def doReport():
         if not status:
           break
       except GAPI.invalid as e:
-        tryDate = _adjustTryDate(str(e), noDateChange, tryDate)
+        numDateChanges += 1
+        tryDate = _adjustTryDate(str(e), numDateChanges, limitDateChanges, tryDate)
         if not tryDate:
           break
         startDateTime = endDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
@@ -14016,7 +14030,7 @@ def doReport():
                 if addCSVData:
                   row.update(addCSVData)
                 csvPF.WriteRowTitles(row)
-                if maxEvents > 0 and numEvents >= maxEvents:
+                if numEvents >= maxEvents > 0:
                   break
               elif csvPF.CheckRowTitles(row):
                 if not summary:
@@ -15836,7 +15850,7 @@ def _showCustomerLicenseInfo(customerInfo, FJQC):
         continue
       break
     except GAPI.invalid as e:
-      tryDate = _adjustTryDate(str(e), False, tryDate)
+      tryDate = _adjustTryDate(str(e), 0, -1, tryDate)
       if not tryDate:
         return
       continue
@@ -45495,7 +45509,7 @@ class CourseAttributes():
             entityModifierItemValueListActionNotPerformedWarning([Ent.COURSE, newCourseId, Ent.COURSE_WORK, f'{body.get("title", courseWorkId)}'], Act.MODIFIER_FROM,
                                                                  [Ent.COURSE, self.courseId], 'individualStudentAssignments delete', j, jcount)
             continue
-          elif self.individualStudentAssignments == 'maptoall':
+          if self.individualStudentAssignments == 'maptoall':
             body['assigneeMode'] = 'ALL_STUDENTS'
             body.pop('individualStudentsOptions', None)
         if self.copyMaterialsFiles: