GAM 5.06

Add date range to skipdates

Make report name match dates; indicate if no data

If only enddate is specified change time period
current: now-1 month to enddate
improved: enddate-1 month to enddate

.travis.yml

@@ -7,8 +7,8 @@ env:
   global:
     - BUILD_PYTHON_VERSION=3.8.2
     - MIN_PYTHON_VERSION=3.8.2
-    - BUILD_OPENSSL_VERSION=1.1.1f
-    - MIN_OPENSSL_VERSION=1.1.1f
+    - BUILD_OPENSSL_VERSION=1.1.1g
+    - MIN_OPENSSL_VERSION=1.1.1g
     - PATCHELF_VERSION=0.10
     - PYINSTALLER_VERSION=3.5
     - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
@@ -185,12 +185,12 @@ script:
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user addevent summary "Travis test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
-- if [ "$e2e" = true ]; then $gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser; fi
-- if [ "$e2e" = true ]; then $gam create vaulthold matter "Travis matter $newbase" name "Travis hold $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then matterid=uid:$($gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser | head -1 | cut -d ' ' -f 3); fi
+- if [ "$e2e" = true ]; then $gam create vaulthold matter $matterid name "Travis hold $newbase" corpus mail accounts $newuser; fi
 - if [ "$e2e" = true ]; then $gam print vaultmatters matterstate open; fi
-- if [ "$e2e" = true ]; then $gam print vaultholds matter "Travis matter $newbase"; fi
-- if [ "$e2e" = true ]; then $gam create vaultexport matter "Travis matter $newbase" name "Travis export $newbase" corpus mail accounts $newuser; fi
-- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam info export id:~~matterId~~ id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam print vaultholds matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam create vaultexport matter $matterid name "Travis export $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add calendar id:$newresource; fi
 - if [ "$e2e" = true ]; then $gam delete resource $newresource; fi
 - if [ "$e2e" = true ]; then $gam delete feature Whiteboard-$newbase; fi
@@ -200,16 +200,18 @@ script:
 - if [ "$e2e" = true ]; then $gam create alias $newalias user $newuser; fi
 - if [ "$e2e" = true ]; then $gam whatis $newuser; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
-- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam download export id:~~matterId~~ id:~~id~~; fi
-- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter "Travis matter $newbase"; fi
-- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action close; fi
-- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action delete; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action close; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action delete; fi
 - if [ "$e2e" = true ]; then $gam delete user $newuser; fi
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
 - if [ "$e2e" = true ]; then $gam print mobile; fi
 - if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$e2e" = true ]; then $gam report usageparameters customer; fi
+- if [ "$e2e" = true ]; then $gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins; fi
 - if [ "$e2e" = true ]; then $gam report customer todrive; fi
-- if [ "$e2e" = true ]; then $gam report users fulldatarequired accounts,gmail fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
+- if [ "$e2e" = true ]; then $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
 - if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
 
 before_deploy:

src/GamCommands.txt

@@ -38,6 +38,7 @@ If an item contains spaces, it should be surrounded by ".
 	papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|
 	saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|
 	tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen
+<DayOfWeek> ::= mon|tue|wed|thu|fri|sat|sun
 <FileFormat> ::=
 	csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
 <LabelColorHex> ::=
@@ -914,12 +915,28 @@ gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 	sites
 <ReportsAppList> ::= "<ReportsApp>(,<ReportsApp>)*"
 
-gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
-	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)] [filter|filters <String>] [fields|parameters <String>]
-gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
+gam report usageparameters customer|user [todrive]
+gam report usage user [todrive]
+	[<UserTypeItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+gam report usage customer [todrive]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+
+gam report users|user [todrive]
+	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[filter|filters <String>] [fields|parameters <String>]
+gam report customers|customer|domain [todrive]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
 	[fields|parameters <String>]
 gam report <ActivityApplicationName> [todrive]
-	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
+	[user all|<UserItem>]
+	[start <Time>] [end <Time>]
+	[filter|filters <String>] [event <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
 gam delete admin <RoleAssignmentId>
@@ -1282,7 +1299,7 @@ gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
 gam <UserTypeEntity> show filerevisions <DriveFileID>
 gam <UserTypeEntity> show filetree [anyowner] (orderby <DriveOrderByFieldName> [ascending|descending])*
 
-gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive]
+gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive] [returnidonly]
 gam <UserTypeEntity> update drivefile (id <DriveFileID)|(drivefilename <DriveFileName>)|(query <QueryDriveFile) [copy] [newfilename <DriveFileName>] <DriveFileUpdateAttributes>*
 gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>)
 	[revision <Number>] [(format <FileFormatList>)|(csvsheet <String>)]

src/auth/__init__.py

@@ -1,10 +1,10 @@
 """Authentication/Credentials general purpose and convenience methods."""
 
-from . import oauth
+import transport
 from var import _FN_OAUTH2_TXT
 from var import GC_OAUTH2_TXT
 from var import GC_Values
-
+from . import oauth
 # TODO: Move logic that determines file name into this module. We should be able
 # to discover the file location without accessing a private member or waiting
 # for a global initialization.
@@ -12,16 +12,15 @@ DEFAULT_OAUTH_STORAGE_FILE = _FN_OAUTH2_TXT
 
 
 def get_admin_credentials_filename():
-  """Gets the name of the file that stores the admin account credentials."""
-  # If the environment globals are loaded, use the set global value. It may have
-  # some custom name in it. Otherwise, just use the default name.
-  if GC_Values[GC_OAUTH2_TXT]:
-    return GC_Values[GC_OAUTH2_TXT]
-  else:
+    """Gets the name of the file that stores the admin account credentials."""
+    # If the environment globals are loaded, use the set global value. It may have
+    # some custom name in it. Otherwise, just use the default name.
+    if GC_Values[GC_OAUTH2_TXT]:
+        return GC_Values[GC_OAUTH2_TXT]
     return DEFAULT_OAUTH_STORAGE_FILE
 
 
 def get_admin_credentials():
-  """Gets oauth.Credentials that are authenticated as the domain's admin user."""
-  credential_file = get_admin_credentials_filename()
-  return oauth.Credentials.from_credentials_file(credential_file)
+    """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+    credential_file = get_admin_credentials_filename()
+    return oauth.Credentials.from_credentials_file(credential_file)

src/auth/oauth.py

@@ -382,7 +382,12 @@ class Credentials(google.oauth2.credentials.Credentials):
     """
     if not self.id_token:
       raise CredentialsError('Failed to fetch token data. No id_token present.')
+
     request = transport.create_request()
+    if self.expired:
+      # The id_token needs to be unexpired, in order to request data about it.
+      self.refresh(request)
+
     self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
         self.id_token, request)
 

src/auth/oauth_test.py

@@ -336,6 +336,26 @@ class CredentialsTest(unittest.TestCase):
         id_token_data=self.fake_token_data)
     self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
 
+  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+  def test_get_token_value_credentials_expired(self, mock_verify_oauth2_token):
+    mock_verify_oauth2_token.return_value = {'fetched-field': 'fetched-value'}
+    time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
+        minutes=5)
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        expiry=time_earlier_than_now,
+        id_token=self.fake_id_token,
+        id_token_data=None)
+    self.assertTrue(creds.expired)
+    creds.refresh = MagicMock()
+
+    token_value = creds.get_token_value('fetched-field')
+
+    self.assertEqual('fetched-value', token_value)
+    self.assertTrue(creds.refresh.called)
+
   def test_to_json_contains_all_required_fields(self):
     creds = oauth.Credentials(
         token=self.fake_token,
@@ -585,7 +605,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
         self.fake_client_config, scopes=self.fake_scopes)
@@ -609,7 +629,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_request_error(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
@@ -625,7 +645,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
@@ -644,7 +664,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
@@ -663,7 +683,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(

src/display.py

@@ -130,8 +130,8 @@ def write_csv_file(csvRows, titles, list_type, todrive):
       return False
     return rowBoolean == filterBoolean
 
-  def headerFilterMatch(title):
-    for filterStr in GC_Values[GC_CSV_HEADER_FILTER]:
+  def headerFilterMatch(filters, title):
+    for filterStr in filters:
       if filterStr.match(title):
         return True
     return False
@@ -151,10 +151,13 @@ def write_csv_file(csvRows, titles, list_type, todrive):
         csvRows = [row for row in csvRows if rowCountFilterMatch(row.get(column, 0), filterVal[1], filterVal[2])]
       else: #boolean
         csvRows = [row for row in csvRows if rowBooleanFilterMatch(row.get(column, False), filterVal[1])]
-  if GC_Values[GC_CSV_HEADER_FILTER]:
-    titles = [t for t in titles if headerFilterMatch(t)]
+  if GC_Values[GC_CSV_HEADER_FILTER] or GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+    if GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+      titles = [t for t in titles if not headerFilterMatch(GC_Values[GC_CSV_HEADER_DROP_FILTER], t)]
+    if GC_Values[GC_CSV_HEADER_FILTER]:
+      titles = [t for t in titles if headerFilterMatch(GC_Values[GC_CSV_HEADER_FILTER], t)]
     if not titles:
-      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER\n')
+      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER and GAM_CSV_HEADER_DROP_FILTER\n')
       return
   csv.register_dialect('nixstdout', lineterminator='\n')
   if todrive:

src/gam-install.sh

@@ -238,9 +238,11 @@ fi
 if [ "$update_profile" = true ]; then
   alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
   if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0
+    update_profile "$HOME/.zshrc" 0
   elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.zshrc" 0
   fi
 else
   echo_yellow "skipping profile update."

src/gam.py

@@ -33,6 +33,7 @@ import io
 import json
 import mimetypes
 import os
+import pkg_resources
 import platform
 import random
 from secrets import SystemRandom
@@ -431,6 +432,7 @@ def SetGlobalVariables():
   _getOldEnvVar(GC_AUTO_BATCH_MIN, 'GAM_AUTOBATCH')
   _getOldEnvVar(GC_BATCH_SIZE, 'GAM_BATCH_SIZE')
   _getOldEnvVar(GC_CSV_HEADER_FILTER, 'GAM_CSV_HEADER_FILTER')
+  _getOldEnvVar(GC_CSV_HEADER_DROP_FILTER, 'GAM_CSV_HEADER_DROP_FILTER')
   _getOldEnvVar(GC_CSV_ROW_FILTER, 'GAM_CSV_ROW_FILTER')
   _getOldEnvVar(GC_TLS_MIN_VERSION, 'GAM_TLS_MIN_VERSION')
   _getOldEnvVar(GC_TLS_MAX_VERSION, 'GAM_TLS_MAX_VERSION')
@@ -602,10 +604,11 @@ def doGAMVersion(checkForArgs=True):
     return
   pyversion = platform.python_version()
   cpu_bits = struct.calcsize('P') * 8
+  api_client_ver = pkg_resources.get_distribution("google-api-python-client").version
   print((f'GAM {gam_version} - {GAM_URL} - {GM_Globals[GM_GAM_TYPE]}\n'
          f'{gam_author}\n'
          f'Python {pyversion} {cpu_bits}-bit {sys.version_info.releaselevel}\n'
-         f'google-api-python-client {googleapiclient.__version__}\n'
+         f'google-api-python-client {api_client_ver}\n'
          f'{getOSPlatform()} {platform.machine()}\n'
          f'Path: {GM_Globals[GM_GAM_PATH]}'))
   if sys.platform.startswith('win') and \
@@ -3523,7 +3526,7 @@ def doUpdateDriveFile(users):
         print(f'Successfully copied {fileId} to {result["id"]}')
 
 def createDriveFile(users):
-  csv_output = to_drive = False
+  csv_output = return_id_only = to_drive = False
   csv_rows = []
   csv_titles = ['User', 'title', 'id']
   media_body = None
@@ -3540,6 +3543,9 @@ def createDriveFile(users):
     elif myarg == 'todrive':
       to_drive = True
       i += 1
+    elif myarg == 'returnidonly':
+      return_id_only = True
+      i += 1
     else:
       i = getDriveFileAttribute(i, body, parameters, myarg, False)
   for user in users:
@@ -3558,10 +3564,12 @@ def createDriveFile(users):
                        ocrLanguage=parameters[DFA_OCRLANGUAGE],
                        media_body=media_body, body=body, fields='id,title,mimeType',
                        supportsAllDrives=True)
-    titleInfo = f'{result["title"]}({result["id"]})'
-    if csv_output:
+    if return_id_only:
+      sys.stdout.write(f"{result['id']}\n")
+    elif csv_output:
       csv_rows.append({'User': user, 'title': result['title'], 'id': result['id']})
     else:
+      titleInfo = f'{result["title"]}({result["id"]})'
       if parameters[DFA_LOCALFILENAME]:
         print(f'Successfully uploaded {parameters[DFA_LOCALFILENAME]} to Drive File {titleInfo}')
       else:
@@ -8748,6 +8756,8 @@ def send_email(subject, body, recipient=None, sender=None, user=None, method='se
   if not user:
     user = _getValueFromOAuth('email')
   userId, gmail = buildGmailGAPIObject(user)
+  if not gmail:
+    return
   resource = gmail.users().messages()
   if labels:
     api_body['labelIds'] = labelsToLabelIds(gmail, labels)
@@ -8757,6 +8767,11 @@ def send_email(subject, body, recipient=None, sender=None, user=None, method='se
   if not recipient:
     recipient = userId
     default_recipient = True
+  # Force ASCII for RFC compliance
+  # xmlcharref seems to work to display at least
+  # some unicode in HTML body and is ignored in
+  # plain text body.
+  body = body.encode('ascii', 'xmlcharrefreplace').decode()
   msg = message_from_string(body)
   for header, value in msgHeaders.items():
     msg.__delitem__(header) # can remove multiple case-insensitive matching headers

src/gam.spec

@@ -2,23 +2,31 @@
 
 import sys
 
+import importlib
+from PyInstaller.utils.hooks import copy_metadata
+
 sys.modules['FixTk'] = None
 
+extra_files = [('cloudprint-v2.json', 'cloudprint-v2.json')]
+
+# dynamically determine where httplib2/cacerts.txt lives
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+extra_files += [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+
+extra_files += copy_metadata('google-api-python-client')
+
 a = Analysis(['gam.py'],
              hiddenimports=[],
              hookspath=None,
              excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             datas=extra_files,
              runtime_hooks=None)
+
 for d in a.datas:
     if 'pyconfig' in d[0]:
         a.datas.remove(d)
         break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
 
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
 
 pyz = PYZ(a.pure)
 exe = EXE(pyz,

src/gapi/__init__.py

@@ -63,7 +63,7 @@ def call(service,
       if http_status == -1:
         # The error detail indicated that we should retry this request
         # We'll refresh credentials and make another pass
-        service._http.request.credentials.refresh(transport.create_http())
+        service._http.credentials.refresh(transport.create_http())
         continue
       if http_status == 0:
         return None

src/gapi/__init___test.py

@@ -108,7 +108,7 @@ class GapiTest(unittest.TestCase):
         self.mock_service, self.mock_method_name, soft_errors=True)
     self.assertEqual(response, fake_200_response)
     self.assertEqual(
-        self.mock_service._http.request.credentials.refresh.call_count, 1)
+        self.mock_service._http.credentials.refresh.call_count, 1)
     self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
 
   def test_call_throws_for_provided_reason(self):

src/gapi/reports.py

@@ -1,6 +1,10 @@
+import calendar
 import datetime
+import re
 import sys
 
+from dateutil.relativedelta import relativedelta
+
 import __main__
 from var import *
 import controlflow
@@ -29,16 +33,238 @@ REPORT_CHOICE_MAP = {
     'logins': 'login',
     'oauthtoken': 'token',
     'tokens': 'token',
+    'usage': 'usage',
+    'usageparameters': 'usageparameters',
     'users': 'user',
     'useraccounts': 'user_accounts',
 }
 
 
+def showUsageParameters():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID,
+                     gapi.errors.ErrorReason.BAD_REQUEST]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit(
+            'user or customer', 'report usageparameters')
+    report = sys.argv[3].lower()
+    titles = ['parameter']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = {}
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = {'userKey': __main__._getValueFromOAuth('email')}
+    else:
+        controlflow.expected_argument_exit(
+            'usageparameters', ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    partial_apps = []
+    all_parameters = []
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam report usageparameters")
+    while True:
+        try:
+            response = gapi.call(endpoint, 'get',
+                                 throw_reasons=throw_reasons,
+                                 date=tryDate,
+                                 customerId=customerId,
+                                 **kwargs)
+            partial_on_thisday = []
+            for warning in response.get('warnings', []):
+                for data in warning.get('data', []):
+                    if data.get('key') == 'application':
+                        partial_on_thisday.append(data['value'])
+            if partial_apps:
+                partial_apps = [app for app in partial_apps if app in partial_on_thisday]
+            else:
+                partial_apps = partial_on_thisday
+            for parameter in response['usageReports'][0]['parameters']:
+                name = parameter.get('name')
+                if name and name not in all_parameters:
+                    all_parameters.append(name)
+            if not partial_apps:
+                break
+            tryDate = (utils.get_yyyymmdd(tryDate, returnDateTime=True) - \
+                    one_day).strftime(YYYYMMDD_FORMAT)
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = _adjust_date(str(e))
+    all_parameters.sort()
+    csvRows = []
+    for parameter in all_parameters:
+        csvRows.append({'parameter': parameter})
+    display.write_csv_file(
+        csvRows, titles, f'{report.capitalize()} Report Usage Parameters', todrive)
+
+REPORTS_PARAMETERS_SIMPLE_TYPES = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
+
+def showUsage():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID,
+                     gapi.errors.ErrorReason.BAD_REQUEST]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit(
+            'user or customer', 'report usage')
+    report = sys.argv[3].lower()
+    titles = ['date']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = [{}]
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = [{'userKey': 'all'}]
+        titles.append('user')
+    else:
+        controlflow.expected_argument_exit(
+            'usage', ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    parameters = []
+    start_date = end_date = orgUnitId = None
+    skip_day_numbers = []
+    skip_dates = set()
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'startdate':
+            start_date = utils.get_yyyymmdd(sys.argv[i+1], returnDateTime=True)
+            i += 2
+        elif myarg == 'enddate':
+            end_date = utils.get_yyyymmdd(sys.argv[i+1], returnDateTime=True)
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg == 'skipdates':
+            for skip in sys.argv[i+1].split(','):
+                if skip.find(':') == -1:
+                    skip_dates.add(utils.get_yyyymmdd(skip, returnDateTime=True))
+                else:
+                    skip_start, skip_end = skip.split(':', 1)
+                    skip_start = utils.get_yyyymmdd(skip_start, returnDateTime=True)
+                    skip_end = utils.get_yyyymmdd(skip_end, returnDateTime=True)
+                    while skip_start <= skip_end:
+                        skip_dates.add(skip_start)
+                        skip_start += one_day
+            i += 2
+        elif myarg == 'skipdaysofweek':
+            skipdaynames = sys.argv[i+1].split(',')
+            dow = [d.lower() for d in calendar.day_abbr]
+            skip_day_numbers = [dow.index(d) for d in skipdaynames if d in dow]
+            i += 2
+        elif report == 'user' and myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            i += 2
+        elif report == 'user' and myarg in usergroup_types:
+            users = __main__.getUsersToModify(myarg, sys.argv[i+1])
+            kwargs = [{'userKey': user} for user in users]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], f'gam report usage {report}')
+    if parameters:
+        titles.extend(parameters)
+        parameters = ','.join(parameters)
+    else:
+        parameters = None
+    if not end_date:
+        end_date = datetime.datetime.now()
+    if not start_date:
+        start_date = end_date + relativedelta(months=-1)
+    if orgUnitId:
+        for kw in kwargs:
+            kw['orgUnitID'] = orgUnitId
+    usage_on_date = start_date
+    start_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+    usage_end_date = end_date
+    end_date = end_date.strftime(YYYYMMDD_FORMAT)
+    start_use_date = end_use_date = None
+    csvRows = []
+    while usage_on_date <= usage_end_date:
+        if usage_on_date.weekday() in skip_day_numbers or \
+           usage_on_date in skip_dates:
+            usage_on_date += one_day
+            continue
+        use_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+        usage_on_date += one_day
+        try:
+            for kwarg in kwargs:
+                try:
+                    usage = gapi.get_all_pages(endpoint, 'get',
+                                               'usageReports',
+                                               throw_reasons=throw_reasons,
+                                               customerId=customerId,
+                                               date=use_date,
+                                               parameters=parameters,
+                                               **kwarg)
+                except gapi.errors.GapiBadRequestError:
+                    continue
+                for entity in usage:
+                    row = {'date': use_date}
+                    if 'userEmail' in entity['entity']:
+                        row['user'] = entity['entity']['userEmail']
+                    for item in entity.get('parameters', []):
+                        if 'name' not in item:
+                            continue
+                        name = item['name']
+                        if name == 'cros:device_version_distribution':
+                            for cros_ver in item['msgValue']:
+                                v = cros_ver['version_number']
+                                column_name = f'cros:num_devices_chrome_{v}'
+                                if column_name not in titles:
+                                    titles.append(column_name)
+                                row[column_name] = cros_ver['num_devices']
+                        else:
+                            if not name in titles:
+                                titles.append(name)
+                            for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                                if ptype in item:
+                                    row[name] = item[ptype]
+                                    break
+                            else:
+                                row[name] = ''
+                    if not start_use_date:
+                        start_use_date = use_date
+                    end_use_date = use_date
+                    csvRows.append(row)
+        except gapi.errors.GapiInvalidError as e:
+            display.print_warning(str(e))
+            break
+    if start_use_date:
+        report_name = f'{report.capitalize()} Usage Report - {start_use_date}:{end_use_date}'
+    else:
+        report_name = f'{report.capitalize()} Usage Report - {start_date}:{end_date} - No Data'
+    display.write_csv_file(
+        csvRows, titles, report_name, todrive)
+
+
 def showReport():
     rep = buildGAPIObject()
     throw_reasons = [gapi.errors.ErrorReason.INVALID]
     report = sys.argv[2].lower()
     report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    if report == 'usage':
+      showUsage()
+      return
+    if report == 'usageparameters':
+      showUsageParameters()
+      return
     valid_apps = gapi.get_enum_values_minus_unspecified(
         rep._rootDesc['resources']['activities']['methods']['list'][
             'parameters']['applicationName']['enum'])+['customer', 'user']
@@ -130,7 +356,6 @@ def showReport():
             sys.exit(1)
         titles = ['email', 'date']
         csvRows = []
-        ptypes = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
         for user_report in usage:
             if 'entity' not in user_report:
                 continue
@@ -142,7 +367,7 @@ def showReport():
                 name = item['name']
                 if not name in titles:
                     titles.append(name)
-                for ptype in ptypes:
+                for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
                     if ptype in item:
                         row[name] = item[ptype]
                         break

src/macos-gam.spec

@@ -1,32 +0,0 @@
-# -*- mode: python -*-
-import sys
-
-sys.modules['FixTk'] = None
-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )

src/transport.py

@@ -27,13 +27,16 @@ def create_http(cache=None,
   Returns:
     httplib2.Http with the specified options.
   """
-  tls_minimum_version = override_min_tls if override_min_tls else GC_Values[GC_TLS_MIN_VERSION]
-  tls_maximum_version = override_max_tls if override_max_tls else GC_Values[GC_TLS_MAX_VERSION]
-  httpObj = httplib2.Http(ca_certs=GC_Values[GC_CA_FILE],
-                          tls_maximum_version=tls_maximum_version,
-                          tls_minimum_version=tls_minimum_version,
-                          cache=cache,
-                          timeout=timeout)
+  tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
+      GC_TLS_MIN_VERSION)
+  tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
+      GC_TLS_MAX_VERSION)
+  httpObj = httplib2.Http(
+      ca_certs=GC_Values.get(GC_CA_FILE),
+      tls_maximum_version=tls_maximum_version,
+      tls_minimum_version=tls_minimum_version,
+      cache=cache,
+      timeout=timeout)
   httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
   return httpObj
 
@@ -68,7 +71,9 @@ def _force_user_agent(user_agent):
         if kwargs['headers'].get('user-agent'):
           if user_agent not in kwargs['headers']['user-agent']:
             # Save the existing user-agent header and tack on our own.
-            kwargs['headers']['user-agent'] = f'{user_agent} {kwargs["headers"]["user-agent"]}'
+            kwargs['headers']['user-agent'] = (
+                f'{user_agent} '
+                f'{kwargs["headers"]["user-agent"]}')
         else:
           kwargs['headers']['user-agent'] = user_agent
       else:

src/travis/linux-before-install.sh

@@ -5,7 +5,7 @@ if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
   echo "running tests with this version"
 else
   export whereibelong=$(pwd)
-  echo "We are running on Ubuntu $TRAVIS_DSIT $PLATFORM"
+  echo "We are running on Ubuntu $TRAVIS_DIST $PLATFORM"
   export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   cpucount=$(nproc --all)
   echo "This device has $cpucount CPUs for compiling..."
@@ -104,10 +104,11 @@ else
     $pip install staticx
   fi
 
+  $pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+
   cd $whereibelong
 fi
 
 echo "Upgrading pip packages..."
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz

src/travis/linux-install.sh

@@ -3,7 +3,7 @@ if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
   export gam="$python gam.py"
   export gampath=$(readlink -e .)
 else
-  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam gam.spec
   export gam="gam/gam"
   export gampath=$(readlink -e gam)
   export GAMVERSION=`$gam version simple`

src/travis/osx-install.sh

@@ -1,7 +1,7 @@
 cd src
 echo "MacOS Version Info According to Python:"
 python -c "import platform; print(platform.mac_ver())"
-$python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+$python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam gam.spec
 export gam="gam/gam"
 export gampath=gam
 $gam version extended

src/travis/windows-install.sh

@@ -1,6 +1,6 @@
 cd src
 echo "compiling GAM with pyinstaller..."
-pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
+pyinstaller --clean --noupx -F --distpath=gam gam.spec
 export gam="gam/gam"
 export gampath=$(readlink -e gam)
 echo "running compiled GAM..."

src/var.py

@@ -6,13 +6,13 @@ import platform
 import re
 
 gam_author = 'Jay Lee <jay0lee@gmail.com>'
-gam_version = '5.03'
+gam_version = '5.06'
 gam_license = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
 GAM_INFO = (f'GAM {gam_version} - {GAM_URL} / {gam_author} / '
-           f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
-           f'{platform.platform()} {platform.machine()}')
+            f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
+            f'{platform.platform()} {platform.machine()}')
 
 GAM_RELEASES = 'https://github.com/jay0lee/GAM/releases'
 GAM_WIKI = 'https://github.com/jay0lee/GAM/wiki'
@@ -843,6 +843,8 @@ GC_SHOW_GETTINGS = 'show_gettings'
 GC_SITE_DIR = 'site_dir'
 # CSV Columns GAM should show on CSV output
 GC_CSV_HEADER_FILTER = 'csv_header_filter'
+# CSV Columns GAM should not show on CSV output
+GC_CSV_HEADER_DROP_FILTER = 'csv_header_drop_filter'
 # CSV Rows GAM should filter
 GC_CSV_ROW_FILTER = 'csv_row_filter'
 # Minimum TLS Version required for HTTPS connections
@@ -878,6 +880,7 @@ GC_Defaults = {
   GC_SHOW_GETTINGS: True,
   GC_SITE_DIR: '',
   GC_CSV_HEADER_FILTER: '',
+  GC_CSV_HEADER_DROP_FILTER: '',
   GC_CSV_ROW_FILTER: '',
   GC_TLS_MIN_VERSION: tls_min,
   GC_TLS_MAX_VERSION: None,
@@ -925,6 +928,7 @@ GC_VAR_INFO = {
   GC_SHOW_GETTINGS: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_SITE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
   GC_CSV_HEADER_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
+  GC_CSV_HEADER_DROP_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
   GC_CSV_ROW_FILTER: {GC_VAR_TYPE: GC_TYPE_ROWFILTER},
   GC_TLS_MIN_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_TLS_MAX_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},

src/windows-gam.spec

@@ -1,33 +0,0 @@
-# -*- mode: python -*-
-import sys
-
-sys.modules['FixTk'] = None
-
-a = Analysis(['gam.py'],
-             pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam.exe',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )