GAM 5.09

My apologies. I fixed my version, then added the code in your __init__.py to loop and wait for the not found error to disappear. By then, however, the error stopped occurring and I didn't notice that errors.py wasn't recognizing the error.

.github/ISSUE_TEMPLATE.txt

@@ -5,7 +5,7 @@ Please confirm the following:
 * I am typing the command as described in the GAM Wiki at https://github.com/jay0lee/gam/wiki
 
 Full steps to reproduce the issue:
-1. 
+1.
 2.
 3.
 

.github/stale.yml

@@ -33,7 +33,7 @@ staleLabel: wontfix
 markComment: >
   This issue has been automatically marked as stale because it has not had
   recent activity. It will be closed if no further activity occurs.
-  
+
 # Comment to post when removing the stale label.
 # unmarkComment: >
 #   Your comment here.

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+    python: python3.7
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: double-quote-string-fixer
+    -   id: check-yaml
+    -   id: check-docstring-first
+    -   id: name-tests-test
+    -   id: requirements-txt-fixer
+    -   id: check-merge-conflict
+
+-   repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.30.0
+    hooks:
+    -   id: yapf
+        args: [--style=google, --in-place]
+
+-   repo: https://github.com/PyCQA/pylint
+    rev: pylint-2.5.0
+    hooks:
+    -   id: pylint
+        args: [--output-format=colorized]

.travis.yml

@@ -7,10 +7,10 @@ env:
   global:
     - BUILD_PYTHON_VERSION=3.8.2
     - MIN_PYTHON_VERSION=3.8.2
-    - BUILD_OPENSSL_VERSION=1.1.1f
-    - MIN_OPENSSL_VERSION=1.1.1f
+    - BUILD_OPENSSL_VERSION=1.1.1g
+    - MIN_OPENSSL_VERSION=1.1.1g
     - PATCHELF_VERSION=0.10
-    - PYINSTALLER_VERSION=3.5
+    - PYINSTALLER_COMMIT=3010fdfaa037e9b19e936711d0c0be9b314b03c6
     - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
     - secure: "J9380tGLOZWa7dSH1y5Il8T5JQpN6ad81gI6VR1HIU0svpRdjgikyDA7ca2MKYDUYYY9yVSkTV6gCl6iIU/9+SKaYugpP+tkvdGYkC2moJdcTgYM/WOnIK9ExQ3BPhN1neGxJjPTwKo1ft27mtZ2I5vuCiBwIcnKWLnKPyW3PD+mWpfqiLuEzkHoAh6G3jC4qbcCrZDeX/knE+PzqESUEi+8k1G8gYcSDWujba9ypSsqZ8T/MXagGla6l7y2Rz+/KZTJmFHwKAA10V+xPLVqxoiqi4ar66yUqy0BamwRXPcseI+ns3Q+4lUpMqVQ5GlRy7LF1xC8myjmcAexXk0F9hg+CMzewKI8UgmQH/ZJvQZEh8s6mW26+CqA4d3zMQkWaR0WtEtpiuH7AGHCflIqvEQ6UiG7ia3B8iZfW2wl0j/kqx4OuHkS3r0pWKVVIIvCj9Ow2BHP7SpiV1AcUGsVxzwbgTh67fitna3Z3c6Uj8ccQlNr7ZIt1az6Wf3w5njijkLOiBpQSLKunTTCTSge/JzBTKUcie3RE9vzirl58gUxAt36nDtPWnory+RttMZrOkBVbTeSxp+IUe8pNwLFPHABsafXsjkfzBOtFmm+0ZXWt2Rlog5NvlemJfQUWDlsL4g+BSakzN+4sIPKzSauWDHyaEeULY7Uprkil6c5zwo="
     - secure: "szcjWHPr0Bf1KCkyTrV5Fu3ADhWk+pg8YWucjXHdybmhaQIKG7iBNg8LJ5d0OBTwAg31wK4ZgyLVSa2gKrAZ3UeDjykJFsR711xDSQOod51Wrgqu4FbXDewE817DUk3Cwe1l5DCu3/fjEw4vbm8B/qb7iMTRKCq6hJd97FwT5oauP0QHNPer9JjrW4F0Hk9ttkgEU2dXWvBMsTJsDOGNI3ddABE2HskxV4T4thelDYGKBDHhUOAsRwSjXgWy77Tvz98psPIvd+6+WPYNRdRWcPDyAR3Z1O/fNjUymrQI6eMaHoSFrmhDS5lbhjINRfdUmECyfCfIFeLWWiw4g4bq7l+4HBORbei55tAIjhEsxJQoqHi0Q5dD5TFh8IiWqowkFbpvNonMSIpKtB0cyT5jU1G/jRA7MPcIvSrdzHaDkoDNHJgAeZfgjOhzTGYYD19lGIljz5BQBcNFZY2dJbja+Jr4He2CMAOBOdERa4Zn1VyNfOmd8Bn5hu0C9D2ybnSCxjXXq5TRiktR8X7WycVZYfqMZXAwP9FEHVitJ4MZEGUc7S92K5gX4wmjcJjLS+Xo/0nsduQm8PuiMjbcPM7/oGx8Xm1KuSfHdKWMBoaesPaDvRX+YcuiNstXf1DkCWl72TsFABzddlNUMl/s2YSKkCSHAJ5ILqrB28Gx89kzVlg="
@@ -185,12 +185,12 @@ script:
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user addevent summary "Travis test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
-- if [ "$e2e" = true ]; then $gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser; fi
-- if [ "$e2e" = true ]; then $gam create vaulthold matter "Travis matter $newbase" name "Travis hold $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then matterid=uid:$($gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser | head -1 | cut -d ' ' -f 3); fi
+- if [ "$e2e" = true ]; then $gam create vaulthold matter $matterid name "Travis hold $newbase" corpus mail accounts $newuser; fi
 - if [ "$e2e" = true ]; then $gam print vaultmatters matterstate open; fi
-- if [ "$e2e" = true ]; then $gam print vaultholds matter "Travis matter $newbase"; fi
-- if [ "$e2e" = true ]; then $gam create vaultexport matter "Travis matter $newbase" name "Travis export $newbase" corpus mail accounts $newuser; fi
-- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam info export id:~~matterId~~ id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam print vaultholds matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam create vaultexport matter $matterid name "Travis export $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add calendar id:$newresource; fi
 - if [ "$e2e" = true ]; then $gam delete resource $newresource; fi
 - if [ "$e2e" = true ]; then $gam delete feature Whiteboard-$newbase; fi
@@ -200,10 +200,10 @@ script:
 - if [ "$e2e" = true ]; then $gam create alias $newalias user $newuser; fi
 - if [ "$e2e" = true ]; then $gam whatis $newuser; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
-- if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam download export id:~~matterId~~ id:~~id~~; fi
-- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter "Travis matter $newbase"; fi
-- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action close; fi
-- if [ "$e2e" = true ]; then $gam update matter "Travis matter $newbase" action delete; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action close; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action delete; fi
 - if [ "$e2e" = true ]; then $gam delete user $newuser; fi
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
 - if [ "$e2e" = true ]; then $gam print mobile; fi
@@ -213,6 +213,12 @@ script:
 - if [ "$e2e" = true ]; then $gam report customer todrive; fi
 - if [ "$e2e" = true ]; then $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
 - if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
+- if ([ "$e2e" = true ] && [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]); then
+    for gamfile in gam-$GAMVERSION-*; do
+      fileid=$($gam user $gam_user add drivefile localfile $gamfile drivefilename $GAMVERSION-$TRAVIS_COMMIT-$gamfile parentid 1N2zbO33qzUQFsGM49-m9AQC1ijzd_ru1 returnidonly);
+      $gam user $gam_user add drivefileacl $fileid anyone role reader withlink;
+    done;
+  fi
 
 before_deploy:
 - export TRAVIS_TAG="preview"

src/.gitignore

@@ -64,7 +64,7 @@ nobrowser.txt
 nocache.txt
 noverifyssl.txt
 gamcache/
-gam/
+dist/
 gam-64/
 *.zip
 *.msi

src/GamCommands.txt

@@ -38,6 +38,7 @@ If an item contains spaces, it should be surrounded by ".
 	papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|
 	saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|
 	tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen
+<DayOfWeek> ::= mon|tue|wed|thu|fri|sat|sun
 <FileFormat> ::=
 	csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
 <LabelColorHex> ::=
@@ -223,7 +224,7 @@ If an item contains spaces, it should be surrounded by ".
 <Section> ::= <String>
 <SerialNumber> ::= <String>
 <ServiceAccountKey> ::= <String>
-<S/MIMEID> ::= <String>	
+<S/MIMEID> ::= <String>
 <SMTPHostName> ::= <String>
 <StudentItem> ::= <EmailAddress>|<UniqueID>|<String>
 <TeamDriveID> ::= <String>
@@ -772,7 +773,7 @@ Specify a collection of Users by directly specifying them or by specifiying item
 	(name <String>)|
 	(type <String>)|
 	(uservisibledescription <String>)
-	
+
 <SchemaFieldDefinition> ::=
 	field <FieldName> (type bool|date|double|email|int64|phone|string) [multivalued|multivalue] [indexed] [restricted] [range <Number> <Number>] endfield
 
@@ -844,6 +845,7 @@ gam delete sakey|sakeys <ServiceAccountKeyList>+ [doit]
 gam show sakey|sakeys [all|system|user]
 
 gam oauth|oauth2 create|request [<EmailAddress>]
+gam oauth|oauth2 create|request [admin <EmailAddress>] [scope|scopes <APIScopeURLList>]
 gam oauth|oauth2 delete|revoke
 gam oauth|oauth2 info|verify [accesstoken <AccessToken>] [idtoken <IDToken>] [showsecret]
 gam oauth|oauth2 refresh
@@ -914,12 +916,28 @@ gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 	sites
 <ReportsAppList> ::= "<ReportsApp>(,<ReportsApp>)*"
 
-gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
-	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)] [filter|filters <String>] [fields|parameters <String>]
-gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
+gam report usageparameters customer|user [todrive]
+gam report usage user [todrive]
+	[<UserTypeItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+gam report usage customer [todrive]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+
+gam report users|user [todrive]
+	[(user all|<UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[filter|filters <String>] [fields|parameters <String>]
+gam report customers|customer|domain [todrive]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
 	[fields|parameters <String>]
 gam report <ActivityApplicationName> [todrive]
-	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
+	[(user all|<UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[start <Time>] [end <Time>]
+	[filter|filters <String>] [event <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
 gam delete admin <RoleAssignmentId>
@@ -1050,11 +1068,11 @@ gam calendar <CalendarItem> printevents <EventSelectProperty>* <EventDisplayProp
 gam calendar <CalendarItem> modify <CalendarSettings>+
 
 gam update cros <CrOSEntity> (<CrOSAttributes>+)|(action deprovision_same_model_replace|deprovision_different_model_replace|deprovision_retiring_device|disable|reenable [acknowledge_device_touch_requirement])
-gam info cros <CrOSEntity> [guessaue] [nolists] [listlimit <Number>] [start <Date>] [end <Date>]
+gam info cros <CrOSEntity> [nolists] [listlimit <Number>] [start <Date>] [end <Date>]
 	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>] [downloadfile latest|<Time>] [targetfolder <FilePath>]
 
 gam print cros [todrive] [(query <QueryCrOS>)|(queries <QueryCrOSList>)] [limittoou <OrgUnitItem>]
-	[orderby <CrOSOrderByFieldName> [ascending|descending]] [guessaue] [nolists|<CrOSListFieldName>*] [listlimit <Number>] [start <Date>] [end <Date>]
+	[orderby <CrOSOrderByFieldName> [ascending|descending]] [nolists|<CrOSListFieldName>*] [listlimit <Number>] [start <Date>] [end <Date>]
 	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>] [sortheaders]
 gam <CrOSTypeEntity> print
 
@@ -1235,10 +1253,10 @@ gam download export <MatterItem> <ExportItem> [noverify] [noextract] [targetfold
 
 gam create vaulthold|hold corpus drive|groups|mail matter <MatterItem> [name <String>] [query <QueryVaultCorpus>]
 	[(accounts|groups|users <EmailItemList>) | (orgunit|ou <OrgUnit>)]
-	[start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>] 
+	[start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>]
 gam update vaulthold|hold <HoldItem> matter <MatterItem> [query <QueryVaultCorpus>]
 	[([addaccounts|addgroups|addusers <EmailItemList>] [removeaccounts|removegroups|removeusers <EmailItemList>]) | (orgunit|ou <OrgUnit>)]
-	[start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>] 
+	[start|starttime <Date>|<Time>] [end|endtime <Date>|<Time>]
 gam delete vaulthold|hold <HoldItem> matter <MatterItem>
 gam info vaulthold|hold <HoldItem> matter <MatterItem>
 gam print vaultholds|holds [todrive] [matters <MatterItemList>]
@@ -1282,7 +1300,7 @@ gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
 gam <UserTypeEntity> show filerevisions <DriveFileID>
 gam <UserTypeEntity> show filetree [anyowner] (orderby <DriveOrderByFieldName> [ascending|descending])*
 
-gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive]
+gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive] [returnidonly]
 gam <UserTypeEntity> update drivefile (id <DriveFileID)|(drivefilename <DriveFileName>)|(query <QueryDriveFile) [copy] [newfilename <DriveFileName>] <DriveFileUpdateAttributes>*
 gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>)
 	[revision <Number>] [(format <FileFormatList>)|(csvsheet <String>)]
@@ -1351,7 +1369,7 @@ gam <UserTypeEntity> insertemail [recipient|to <EmailAddress>] [from <EmailAddre
 	[deleted] [date <Time>]
 gam <UserTypeEntity> sendemail [recipient|to <EmailAddress>] [from <EmailAddress>]
 	[subject <String>] [(message <String>)|(file <FileName> [charset <Charset>])]
-	(header <String> <String>)* 
+	(header <String> <String>)*
 
 gam <UserTypeEntity> create|add delegate|delegates <EmailAddress>
 gam <UserTypeEntity> delegate|delegates to <EmailAddress>

src/LICENSE

@@ -202,12 +202,12 @@
 
 
 
-APACHE HTTP SERVER SUBCOMPONENTS: 
+APACHE HTTP SERVER SUBCOMPONENTS:
 
 The Apache HTTP Server includes a number of subcomponents with
 separate copyright notices and license terms. Your use of the source
 code for the these subcomponents is subject to the terms and
-conditions of the following licenses. 
+conditions of the following licenses.
 
 For the mod_mime_magic component:
 
@@ -273,7 +273,7 @@ For the  server\util_md5.c component:
  *  Original Code Copyright (C) 1994, Jeff Hostetler, Spyglass, Inc.
  *  Portions of Content-MD5 code Copyright (C) 1993, 1994 by Carnegie Mellon
  *     University (see Copyright below).
- *  Portions of Content-MD5 code Copyright (C) 1991 Bell Communications 
+ *  Portions of Content-MD5 code Copyright (C) 1991 Bell Communications
  *     Research, Inc. (Bellcore) (see Copyright below).
  *  Portions extracted from mpack, John G. Myers - jgm+@cmu.edu
  *  Content-MD5 Code contributed by Martin Hamilton (martin@net.lut.ac.uk)
@@ -319,10 +319,10 @@ For the  server\util_md5.c component:
  * of an authorized representative of Bellcore.  BELLCORE
  * MAKES NO REPRESENTATIONS ABOUT THE ACCURACY OR SUITABILITY
  * OF THIS MATERIAL FOR ANY PURPOSE.  IT IS PROVIDED "AS IS",
- * WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.  
+ * WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
  */
 
-For the  srclib\apr\include\apr_md5.h component: 
+For the  srclib\apr\include\apr_md5.h component:
 /*
  * This is work is derived from material Copyright RSA Data Security, Inc.
  *
@@ -501,21 +501,21 @@ This program is Copyright (C) Zeus Technology Limited 1996.
 This program may be used and copied freely providing this copyright notice
 is not removed.
 
-This software is provided "as is" and any express or implied waranties, 
+This software is provided "as is" and any express or implied waranties,
 including but not limited to, the implied warranties of merchantability and
-fitness for a particular purpose are disclaimed.  In no event shall 
-Zeus Technology Ltd. be liable for any direct, indirect, incidental, special, 
-exemplary, or consequential damaged (including, but not limited to, 
+fitness for a particular purpose are disclaimed.  In no event shall
+Zeus Technology Ltd. be liable for any direct, indirect, incidental, special,
+exemplary, or consequential damaged (including, but not limited to,
 procurement of substitute good or services; loss of use, data, or profits;
 or business interruption) however caused and on theory of liability.  Whether
-in contract, strict liability or tort (including negligence or otherwise) 
+in contract, strict liability or tort (including negligence or otherwise)
 arising in any way out of the use of this software, even if advised of the
 possibility of such damage.
 
      Written by Adam Twiss (adam@zeus.co.uk).  March 1996
 
 Thanks to the following people for their input:
-  Mike Belshe (mbelshe@netscape.com) 
+  Mike Belshe (mbelshe@netscape.com)
   Michael Campanella (campanella@stevms.enet.dec.com)
 
 */
@@ -532,10 +532,10 @@ without limitation the rights to use, copy, modify, merge, publish,
 distribute, sublicense, and/or sell copies of the Software, and to
 permit persons to whom the Software is furnished to do so, subject to
 the following conditions:
-	
+
 The above copyright notice and this permission notice shall be included
 in all copies or substantial portions of the Software.
-	
+
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

src/auth/oauth.py

@@ -1,553 +0,0 @@
-"""OAuth2.0 user credentials."""
-
-import datetime
-import json
-import os
-import re
-import threading
-from urllib.parse import urlencode
-
-from filelock import FileLock
-import google_auth_oauthlib.flow
-import google.oauth2.credentials
-import google.oauth2.id_token
-
-import fileutils
-import transport
-from var import GAM_INFO
-from var import GM_Globals
-from var import GM_WINDOWS
-import utils
-
-MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
-                                        'browser:\n\n\t{url}\n')
-MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
-MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
-                                             ' visit:\n\n\t{url}\n\nIf your '
-                                             'browser is on a different machine'
-                                             ' then press CTRL+C and create a '
-                                             'file called nobrowser.txt in the '
-                                             'same folder as GAM.\n')
-MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
-                                ' close this browser window and return to GAM.')
-
-
-class CredentialsError(Exception):
-  """Base error class."""
-  pass
-
-
-class InvalidCredentialsFileError(CredentialsError):
-  """Error raised when a file cannot be opened into a credentials object."""
-  pass
-
-
-class EmptyCredentialsFileError(InvalidCredentialsFileError):
-  """Error raised when a credentials file contains no content."""
-  pass
-
-
-class InvalidClientSecretsFileFormatError(CredentialsError):
-  """Error raised when a client secrets file format is invalid."""
-  pass
-
-
-class InvalidClientSecretsFileError(CredentialsError):
-  """Error raised when client secrets file cannot be read."""
-  pass
-
-
-class Credentials(google.oauth2.credentials.Credentials):
-  """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
-
-  DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
-
-  def __init__(self,
-               token,
-               refresh_token=None,
-               id_token=None,
-               token_uri=None,
-               client_id=None,
-               client_secret=None,
-               scopes=None,
-               quota_project_id=None,
-               expiry=None,
-               id_token_data=None,
-               filename=None):
-    """A thread-safe OAuth2.0 credentials object.
-
-    Credentials adds additional utility properties and methods to a
-    standard OAuth2.0 credentials object. When used to store credentials on
-    disk, it implements a file lock to avoid collision during writes.
-
-    Args:
-      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
-        information is provided.
-      refresh_token: String, The OAuth 2.0 refresh token. If specified,
-        credentials can be refreshed.
-      id_token: String, The Open ID Connect ID Token.
-      token_uri: String, The OAuth 2.0 authorization server's token endpoint
-        URI. Must be specified for refresh, can be left as None if the token can
-        not be refreshed.
-      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
-        can be left as None if the token can not be refreshed.
-      client_secret: String, The OAuth 2.0 client secret. Must be specified for
-        refresh, can be left as None if the token can not be refreshed.
-      scopes: Sequence[str], The scopes used to obtain authorization.
-        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
-          not request additional scopes after authorization. The scopes must be
-          derivable from the refresh token if refresh information is provided
-          (e.g. The refresh token scopes are a superset of this or contain a
-          wild card scope like
-            'https://www.googleapis.com/auth/any-api').
-      quota_project_id: String, The project ID used for quota and billing. This
-        project may be different from the project used to create the
-        credentials.
-      expiry: datetime.datetime, The time at which the provided token will
-        expire.
-      id_token_data: Oauth2.0 ID Token data which was previously fetched for
-        this access token against the google.oauth2.id_token library.
-      filename: String, Path to a file that will be used to store the
-        credentials. If provided, a lock file of the same name and a ".lock"
-          extension will be created for concurrency controls. Note: New
-            credentials are not saved to disk until write() or refresh() are
-            called.
-
-    Raises:
-      TypeError: If id_token_data is not the required dict type.
-    """
-    super(Credentials, self).__init__(
-        token=token,
-        refresh_token=refresh_token,
-        id_token=id_token,
-        token_uri=token_uri,
-        client_id=client_id,
-        client_secret=client_secret,
-        scopes=scopes,
-        quota_project_id=quota_project_id)
-
-    # Load data not restored by the super class
-    self.expiry = expiry
-    if id_token_data and not isinstance(id_token_data, dict):
-      raise TypeError(f'Expected type id_token_data dict but received '
-                      f'{type(id_token_data)}')
-    self._id_token_data = id_token_data.copy() if id_token_data else None
-
-    # If a filename is provided, use a lock file to control concurrent access
-    # to the resource. If no filename is provided, use a thread lock that has
-    # the same interface as FileLock in order to simplify the implementation.
-    if filename:
-      # Convert relative paths into absolute
-      self._filename = os.path.abspath(filename)
-      lock_file = os.path.abspath(f'{self._filename}.lock')
-      self._lock = FileLock(lock_file)
-    else:
-      self._filename = None
-      self._lock = _FileLikeThreadLock()
-
-  # Use a property to prevent external mutation of the filename.
-  @property
-  def filename(self):
-    return self._filename
-
-  @classmethod
-  def from_authorized_user_info(cls, info, filename=None):
-    """Generates Credentials from JSON containing authorized user info.
-
-    Args:
-      info: Dict, authorized user info in Google format.
-      filename: String, the filename used to store these credentials on disk. If
-        no filename is provided, the credentials will not be saved to disk.
-
-    Raises:
-      ValueError: If missing fields are detected in the info.
-    """
-    # We need all of these keys
-    keys_needed = set(('client_id', 'client_secret'))
-    # We need 1 or more of these keys
-    keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
-    missing = keys_needed.difference(info.keys())
-    has_one_of = set(info) & keys_need_one_of
-    if missing or not has_one_of:
-      raise ValueError(
-          'Authorized user info was not in the expected format, missing '
-          f'fields {", ".join(missing)} and one of '
-          f'{", ".join(keys_need_one_of)}.')
-
-    expiry = info.get('token_expiry')
-    if expiry:
-      # Convert the raw expiry to datetime
-      expiry = datetime.datetime.strptime(expiry, Credentials.DATETIME_FORMAT)
-    id_token_data = info.get('decoded_id_token')
-
-    # Provide backwards compatibility with field names when loading from JSON.
-    # Some field names may be different, depending on when/how the credentials
-    # were pickled.
-    return cls(
-        token=info.get('token', info.get('auth_token', '')),
-        refresh_token=info.get('refresh_token', ''),
-        id_token=info.get('id_token_jwt', info.get('id_token')),
-        token_uri=info.get('token_uri'),
-        client_id=info['client_id'],
-        client_secret=info['client_secret'],
-        scopes=info.get('scopes'),
-        quota_project_id=info.get('quota_project_id'),
-        expiry=expiry,
-        id_token_data=id_token_data,
-        filename=filename)
-
-  @classmethod
-  def from_google_oauth2_credentials(cls, credentials, filename=None):
-    """Generates Credentials from a google.oauth2.Credentials object."""
-    info = json.loads(credentials.to_json())
-    # Add properties which are not exported with the native to_json() output.
-    info['id_token'] = credentials.id_token
-    if credentials.expiry:
-      info['token_expiry'] = credentials.expiry.strftime(
-          Credentials.DATETIME_FORMAT)
-    info['quota_project_id'] = credentials.quota_project_id
-
-    return cls.from_authorized_user_info(info, filename=filename)
-
-  @classmethod
-  def from_credentials_file(cls, filename):
-    """Generates Credentials from a stored Credentials file.
-
-    The same file will be used to save the credentials when the access token is
-    refreshed.
-
-    Args:
-      filename: String, the name of a file containing JSON credentials to load.
-        The same filename will be used to save credentials back to disk.
-
-    Returns:
-      The credentials loaded from disk.
-
-    Raises:
-      InvalidCredentialsFileError: When the credentials file cannot be opened.
-      EmptyCredentialsFileError: When the provided file contains no credentials.
-    """
-    file_content = fileutils.read_file(
-        filename, continue_on_error=True, display_errors=False)
-    if file_content is None:
-      raise InvalidCredentialsFileError(f'File {filename} could not be opened')
-    info = json.loads(file_content)
-    if not info:
-      raise EmptyCredentialsFileError(
-          f'File {filename} contains no credential data')
-
-    try:
-      # We read the existing data from the passed in file, but we also want to
-      # save future data/tokens in the same place.
-      return cls.from_authorized_user_info(info, filename=filename)
-    except ValueError as e:
-      raise InvalidCredentialsFileError(str(e))
-
-  @classmethod
-  def from_client_secrets(cls,
-                          client_id,
-                          client_secret,
-                          scopes,
-                          access_type='offline',
-                          login_hint=None,
-                          filename=None,
-                          use_console_flow=False):
-    """Runs an OAuth Flow from client secrets to generate credentials.
-
-    Args:
-      client_id: String, The OAuth2.0 Client ID.
-      client_secret: String, The OAuth2.0 Client Secret.
-      scopes: Sequence[str], A list of scopes to include in the credentials.
-      access_type: String, 'offline' or 'online'.  Indicates whether your
-        application can refresh access tokens when the user is not present at
-        the browser. Valid parameter values are online, which is the default
-        value, and offline.  Set the value to offline if your application needs
-        to refresh access tokens when the user is not present at the browser.
-        This is the method of refreshing access tokens described later in this
-        document. This value instructs the Google authorization server to return
-        a refresh token and an access token the first time that your application
-        exchanges an authorization code for tokens.
-      login_hint: String, The email address that will be displayed on the Google
-        login page as a hint for the user to login to the correct account.
-      filename: String, the path to a file to use to save the credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
-
-    Returns:
-      Credentials
-    """
-    client_config = {
-        'installed': {
-            'client_id': client_id,
-            'client_secret': client_secret,
-            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
-            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
-            'token_uri': 'https://oauth2.googleapis.com/token',
-        }
-    }
-
-    flow = _ShortURLFlow.from_client_config(
-        client_config, scopes, autogenerate_code_verifier=True)
-    flow_kwargs = {'access_type': access_type}
-    if login_hint:
-      flow_kwargs['login_hint'] = login_hint
-
-    # TODO: Move code for browser detection somewhere in this file so that the
-    # messaging about `nobrowser.txt` is co-located with the logic that uses it.
-    if use_console_flow:
-      flow.run_console(
-          authorization_prompt_message=MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
-          authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
-          **flow_kwargs)
-    else:
-      flow.run_local_server(
-          authorization_prompt_message=MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
-          success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
-          **flow_kwargs)
-    return cls.from_google_oauth2_credentials(
-        flow.credentials, filename=filename)
-
-  @classmethod
-  def from_client_secrets_file(cls,
-                               client_secrets_file,
-                               scopes,
-                               access_type='offline',
-                               login_hint=None,
-                               credentials_file=None,
-                               use_console_flow=False):
-    """Runs an OAuth Flow from secrets stored on disk to generate credentials.
-
-    Args:
-      client_secrets_file: String, path to a file containing a client ID and
-        secret.
-      scopes: Sequence[str], A list of scopes to include in the credentials.
-      access_type: String, 'offline' or 'online'.  Indicates whether your
-        application can refresh access tokens when the user is not present at
-        the browser. Valid parameter values are online, which is the default
-        value, and offline.  Set the value to offline if your application needs
-        to refresh access tokens when the user is not present at the browser.
-        This is the method of refreshing access tokens described later in this
-        document. This value instructs the Google authorization server to return
-        a refresh token and an access token the first time that your application
-        exchanges an authorization code for tokens.
-      login_hint: String, The email address that will be displayed on the Google
-        login page as a hint for the user to login to the correct account.
-      credentials_file: String, the path to a file to use to save the
-        credentials.
-      use_console_flow: Boolean, True if the authentication flow should be run
-        strictly from a console; False to launch a browser for authentication.
-
-    Raises:
-      InvalidClientSecretsFileError: If the client secrets file cannot be
-        opened.
-      InvalidClientSecretsFileFormatError: If the client secrets file does not
-        contain the required data or the data is malformed.
-
-    Returns:
-      Credentials
-    """
-    cs_data = fileutils.read_file(
-        client_secrets_file, continue_on_error=True, display_errors=False)
-    if not cs_data:
-      raise InvalidClientSecretsFileError(
-          f'File {client_secrets_file} could not be opened')
-    try:
-      cs_json = json.loads(cs_data)
-      client_id = cs_json['installed']['client_id']
-      # Chop off .apps.googleusercontent.com suffix as it's not needed
-      # and we need to keep things short for the Auth URL.
-      client_id = re.sub(r'\.apps\.googleusercontent\.com$', '', client_id)
-      client_secret = cs_json['installed']['client_secret']
-    except (ValueError, IndexError, KeyError):
-      raise InvalidClientSecretsFileFormatError(
-          f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
-      )
-
-    return cls.from_client_secrets(
-        client_id,
-        client_secret,
-        scopes,
-        access_type=access_type,
-        login_hint=login_hint,
-        filename=credentials_file,
-        use_console_flow=use_console_flow)
-
-  def _fetch_id_token_data(self):
-    """Fetches verification details from Google for the OAuth2.0 token.
-
-    See more: https://developers.google.com/identity/sign-in/web/backend-auth
-
-    Raises:
-      CredentialsError: If no id_token is present.
-    """
-    if not self.id_token:
-      raise CredentialsError('Failed to fetch token data. No id_token present.')
-
-    request = transport.create_request()
-    if self.expired:
-      # The id_token needs to be unexpired, in order to request data about it.
-      self.refresh(request)
-
-    self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
-        self.id_token, request)
-
-  def get_token_value(self, field):
-    """Retrieves data from the OAuth ID token.
-
-    See more: https://developers.google.com/identity/sign-in/web/backend-auth
-
-    Args:
-      field: The name of the key/field to fetch
-
-    Returns:
-      The value associated with the given key or 'Unknown' if the key data can
-      not be found in the access token data.
-    """
-    if not self._id_token_data:
-      self._fetch_id_token_data()
-    # Maintain legacy GAM behavior here to return "Unknown" if the field is
-    # otherwise unpopulated.
-    return self._id_token_data.get(field, 'Unknown')
-
-  def to_json(self, strip=None):
-    """Creates a JSON representation of a Credentials.
-
-    Args:
-        strip: Sequence[str], Optional list of members to exclude from the
-          generated JSON.
-
-    Returns:
-        str: A JSON representation of this instance, suitable to pass to
-             from_json().
-    """
-    expiry = self.expiry.strftime(
-        Credentials.DATETIME_FORMAT) if self.expiry else None
-    prep = {
-        'token': self.token,
-        'refresh_token': self.refresh_token,
-        'token_uri': self.token_uri,
-        'client_id': self.client_id,
-        'client_secret': self.client_secret,
-        'id_token': self.id_token,
-        # Google auth doesn't currently give us scopes back on refresh.
-        # 'scopes': sorted(self.scopes),
-        'token_expiry': expiry,
-        'decoded_id_token': self._id_token_data,
-    }
-
-    # Remove empty entries
-    prep = {k: v for k, v in prep.items() if v is not None}
-
-    # Remove entries that explicitly need to be removed
-    if strip is not None:
-      prep = {k: v for k, v in prep.items() if k not in strip}
-
-    return json.dumps(prep, indent=2, sort_keys=True)
-
-  def refresh(self, request=None):
-    """Refreshes the credential's access token.
-
-    Args:
-      request: google.auth.transport.Request, The object used to make HTTP
-        requests. If not provided, a default request will be used.
-
-    Raises:
-      google.auth.exceptions.RefreshError: If the credentials could not be
-      refreshed.
-    """
-    with self._lock:
-      if request is None:
-        request = transport.create_request()
-      self._locked_refresh(request)
-      # Save the new tokens back to disk, if these credentials are disk-backed.
-      if self._filename:
-        self._locked_write()
-
-  def _locked_refresh(self, request):
-    """Refreshes the credential's access token while the file lock is held."""
-    assert self._lock.is_locked
-    super(Credentials, self).refresh(request)
-
-  def write(self):
-    """Writes credentials to disk."""
-    with self._lock:
-      self._locked_write()
-
-  def _locked_write(self):
-    """Writes credentials to disk while the file lock is held."""
-    assert self._lock.is_locked
-    if not self.filename:
-      # If no filename was provided to the constructor, these credentials cannot
-      # be saved to disk.
-      raise CredentialsError(
-          'The credentials have no associated filename and cannot be saved '
-          'to disk.')
-    fileutils.write_file(self._filename, self.to_json())
-
-  def delete(self):
-    """Deletes all files on disk related to these credentials."""
-    with self._lock:
-      # Only attempt to remove the file if the lock we're using is a FileLock.
-      if isinstance(self._lock, FileLock):
-        os.remove(self._filename)
-        if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
-          os.remove(self._lock.lock_file)
-
-  _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
-
-  def revoke(self, http=None):
-    """Revokes this credential's access token with the server.
-
-    Args:
-      http: httplib2.Http compatible object for use as a transport. If no http
-        is provided, a default will be used.
-    """
-    with self._lock:
-      if http is None:
-        http = transport.create_http()
-      params = urlencode({'token': self.refresh_token})
-      revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
-      http.request(revoke_uri, 'GET')
-
-
-class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
-  """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
-
-  URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
-
-  def authorization_url(self, http=None, **kwargs):
-    """Gets a shortened authorization URL."""
-    long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
-    short_url = utils.shorten_url(long_url)
-    return short_url, state
-
-class _FileLikeThreadLock(object):
-  """A threading.lock which has the same interface as filelock.Filelock."""
-
-  def __init__(self):
-    """A shell object that holds a threading.Lock.
-
-    Since we cannot inherit from built-in classes such as threading.Lock, we
-    just use a shell object and maintain a lock inside of it.
-    """
-    self._lock = threading.Lock()
-
-  def __enter__(self, *args, **kwargs):
-    return self._lock.__enter__(*args, **kwargs)
-
-  def __exit__(self, *args, **kwargs):
-    return self._lock.__exit__(*args, **kwargs)
-
-  def acquire(self, **kwargs):
-    return self._lock.acquire(**kwargs)
-
-  def release(self):
-    return self._lock.release()
-
-  @property
-  def is_locked(self):
-    return self._lock.locked()
-
-  @property
-  def lock_file(self):
-    return None

src/auth/oauth_test.py

@@ -1,705 +0,0 @@
-"""Tests for oauth."""
-
-import datetime
-import json
-import os
-import platform
-import unittest
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-import google.oauth2.credentials
-
-from auth import oauth
-
-
-class CredentialsTest(unittest.TestCase):
-
-  def setUp(self):
-    self.fake_token = 'fake_token'
-    self.fake_refresh_token = 'fake_refresh_token'
-    self.fake_id_token = 'fake_id_token'
-    self.fake_token_uri = 'https://fake.token.uri'
-    self.fake_client_id = 'fake_client_id'
-    self.fake_client_secret = 'fake_client_secret'
-    self.fake_scopes = [
-        'fake_api.readonly',
-        'fake_other_api.write',
-    ]
-    self.fake_quota_project_id = 'fake_quota_project_id'
-    self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
-    self.fake_filename = 'fake_filename'
-    self.fake_token_data = {
-        'field': 'value',
-        'another-field': 'another-value',
-    }
-    self.info_with_only_required_fields = {
-        'refresh_token': self.fake_refresh_token,
-        'client_id': self.fake_client_id,
-        'client_secret': self.fake_client_secret,
-    }
-    super(CredentialsTest, self).setUp()
-
-  def tearDown(self):
-    # Remove any credential files that may have been created.
-    if os.path.exists(self.fake_filename):
-      os.remove(self.fake_filename)
-    if os.path.exists('%s.lock' % self.fake_filename):
-      os.remove('%s.lock' % self.fake_filename)
-    super(CredentialsTest, self).tearDown()
-
-  def test_from_authorized_user_info_only_required_info(self):
-    creds = oauth.Credentials.from_authorized_user_info(
-        self.info_with_only_required_fields)
-    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
-    self.assertEqual(self.fake_client_id, creds.client_id)
-    self.assertEqual(self.fake_client_secret, creds.client_secret)
-    self.assertIsNone(creds.id_token)
-    self.assertIsNone(creds.expiry)
-    self.assertIsNone(creds.filename)
-
-  def test_from_authorized_user_info_all_info_provided(self):
-    info = {
-        'token':
-            self.fake_token,
-        'refresh_token':
-            self.fake_refresh_token,
-        'id_token':
-            self.fake_id_token,
-        'token_uri':
-            self.fake_token_uri,
-        'client_id':
-            self.fake_client_id,
-        'client_secret':
-            self.fake_client_secret,
-        'token_expiry':
-            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
-        'id_token_data':
-            self.fake_token_data,
-    }
-    creds = oauth.Credentials.from_authorized_user_info(info)
-    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
-    self.assertEqual(self.fake_client_id, creds.client_id)
-    self.assertEqual(self.fake_client_secret, creds.client_secret)
-    self.assertEqual(self.fake_id_token, creds.id_token)
-    self.assertEqual(self.fake_token_uri, creds.token_uri)
-    self.assertEqual(self.fake_token_expiry, creds.expiry)
-    self.assertIsNone(creds.filename)
-
-  def test_from_authorized_user_info_missing_required_info(self):
-    info_with_missing_fields = {'token': self.fake_token}
-    with self.assertRaises(ValueError):
-      oauth.Credentials.from_authorized_user_info(info_with_missing_fields)
-
-  def test_from_authorized_user_info_no_expiry_in_info(self):
-    info_with_no_token_expiry = self.info_with_only_required_fields.copy()
-    self.assertIsNone(info_with_no_token_expiry.get('expiry'))
-    creds = oauth.Credentials.from_authorized_user_info(
-        info_with_no_token_expiry)
-    self.assertIsNone(creds.expiry)
-
-  def test_init_saves_filename(self):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
-
-  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
-  def test_init_loads_decoded_id_token_data(self, mock_verify_token):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token=self.fake_id_token,
-        id_token_data=self.fake_token_data)
-    self.assertEqual(
-        self.fake_token_data.get('field'), creds.get_token_value('field'))
-    # Verify the fetching method was not called, since the token
-    # data was supposed to be loaded from the passed in info.
-    self.assertEqual(mock_verify_token.call_count, 0)
-
-  def test_credentials_uses_file_lock_when_filename_provided(self):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-    self.assertIsInstance(creds._lock, oauth.FileLock)
-    self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
-
-  def test_credentials_uses_thread_lock_when_filename_not_provided(self):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=None)
-    self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
-    self.assertIsNone(creds.filename)
-
-  def test_from_oauth2credentials(self):
-    google_creds = google.oauth2.credentials.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token=self.fake_id_token)
-    creds = oauth.Credentials.from_google_oauth2_credentials(
-        google_creds, filename=self.fake_filename)
-    self.assertEqual(google_creds.token, creds.token)
-    self.assertEqual(google_creds.refresh_token, creds.refresh_token)
-    self.assertEqual(google_creds.client_id, creds.client_id)
-    self.assertEqual(google_creds.client_secret, creds.client_secret)
-    self.assertEqual(google_creds.id_token, creds.id_token)
-    self.assertEqual(google_creds.expiry, creds.expiry)
-    self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
-    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
-
-  def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
-    self.assertFalse(os.path.exists(self.fake_filename))
-    with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
-      oauth.Credentials.from_credentials_file(self.fake_filename)
-    self.assertIn('could not be opened', str(e.exception))
-
-  @patch.object(oauth.fileutils, 'read_file')
-  def test_from_credentials_file_no_serialized_data_in_file_raises_error(
-      self, mock_read_file):
-    mock_read_file.return_value = json.dumps({})
-    with self.assertRaises(oauth.EmptyCredentialsFileError):
-      oauth.Credentials.from_credentials_file(self.fake_filename)
-
-  @patch.object(oauth.fileutils, 'read_file')
-  def test_from_credentials_file_missing_any_token_raises_error(
-      self, mock_read_file):
-    mock_read_file.return_value = json.dumps({
-        # This data is missing a token key/value pair
-        'client_id': self.fake_client_id,
-        'client_secret': self.fake_client_secret,
-    })
-    with self.assertRaises(oauth.InvalidCredentialsFileError):
-      oauth.Credentials.from_credentials_file(self.fake_filename)
-
-  @patch.object(oauth.fileutils, 'read_file')
-  def test_from_credentials_file_missing_required_raises_error(
-      self, mock_read_file):
-    mock_read_file.return_value = json.dumps({
-        # This data is missing a client_secret key/value pair
-        'client_id': self.fake_client_id,
-        'refresh_token': self.fake_refresh_token,
-    })
-    with self.assertRaises(oauth.InvalidCredentialsFileError):
-      oauth.Credentials.from_credentials_file(self.fake_filename)
-
-  @patch.object(oauth._ShortURLFlow, 'from_client_config')
-  def test_from_client_secrets_console_flow(self, mock_flow):
-    flow_creds = google.oauth2.credentials.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token=self.fake_id_token)
-    mock_flow.return_value.credentials = flow_creds
-
-    creds = oauth.Credentials.from_client_secrets(
-        self.fake_client_id,
-        self.fake_client_secret,
-        self.fake_scopes,
-        use_console_flow=True)
-    self.assertTrue(mock_flow.return_value.run_console.called)
-    self.assertFalse(mock_flow.return_value.run_local_server.called)
-    self.assertEqual(flow_creds.token, creds.token)
-    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
-    self.assertEqual(flow_creds.client_id, creds.client_id)
-    self.assertEqual(flow_creds.client_secret, creds.client_secret)
-    self.assertEqual(flow_creds.id_token, creds.id_token)
-
-  @patch.object(oauth._ShortURLFlow, 'from_client_config')
-  def test_from_client_secrets_local_server_flow(self, mock_flow):
-    flow_creds = google.oauth2.credentials.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token=self.fake_id_token)
-    mock_flow.return_value.credentials = flow_creds
-
-    creds = oauth.Credentials.from_client_secrets(
-        self.fake_client_id,
-        self.fake_client_secret,
-        self.fake_scopes,
-        use_console_flow=False)
-    self.assertFalse(mock_flow.return_value.run_console.called)
-    self.assertTrue(mock_flow.return_value.run_local_server.called)
-    self.assertEqual(flow_creds.token, creds.token)
-    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
-    self.assertEqual(flow_creds.client_id, creds.client_id)
-    self.assertEqual(flow_creds.client_secret, creds.client_secret)
-    self.assertEqual(flow_creds.id_token, creds.id_token)
-
-  @patch.object(oauth._ShortURLFlow, 'from_client_config')
-  def test_from_client_secrets_uses_login_hint(self, mock_flow):
-    flow_creds = google.oauth2.credentials.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token=self.fake_id_token)
-    mock_flow.return_value.credentials = flow_creds
-
-    oauth.Credentials.from_client_secrets(
-        self.fake_client_id,
-        self.fake_client_secret,
-        self.fake_scopes,
-        login_hint='someone@domain.com')
-
-    run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
-    self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
-
-  def test_from_client_secrets_uses_shortened_url_flow(self):
-    with patch.object(oauth._ShortURLFlow, 'from_client_config') as mock_flow:
-      flow_creds = google.oauth2.credentials.Credentials(
-          token=self.fake_token,
-          refresh_token=self.fake_refresh_token,
-          client_id=self.fake_client_id,
-          client_secret=self.fake_client_secret,
-          id_token=self.fake_id_token)
-      mock_flow.return_value.credentials = flow_creds
-      oauth.Credentials.from_client_secrets(self.fake_client_id,
-                                            self.fake_client_secret,
-                                            self.fake_scopes)
-    self.assertTrue(mock_flow.called)
-
-  @patch.object(oauth._ShortURLFlow, 'from_client_config')
-  def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
-    flow_creds = google.oauth2.credentials.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token=self.fake_id_token)
-    mock_flow.return_value.credentials = flow_creds
-
-    creds = oauth.Credentials.from_client_secrets(
-        self.fake_client_id,
-        self.fake_client_secret,
-        self.fake_scopes,
-        filename=self.fake_filename)
-    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
-
-  def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(self):
-    self.assertFalse(os.path.exists(self.fake_filename))
-    with self.assertRaises(oauth.InvalidClientSecretsFileError):
-      oauth.Credentials.from_client_secrets_file(self.fake_filename,
-                                                 self.fake_scopes)
-
-  @patch.object(oauth.fileutils, 'read_file')
-  def test_from_client_secrets_file_missing_required_json_raises_error(
-      self, mock_read_file):
-    mock_read_file.return_value = json.dumps({})
-    with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
-      oauth.Credentials.from_client_secrets_file(self.fake_filename,
-                                                 self.fake_scopes)
-    self.assertIn('Could not extract Client ID or Client Secret',
-                  str(e.exception))
-
-  @patch.object(oauth.Credentials, 'from_client_secrets')
-  @patch.object(oauth.fileutils, 'read_file')
-  def test_from_client_secrets_file_strips_domain_from_client_id(
-      self, mock_read_file, mock_creds_from_client_secrets):
-    mock_read_file.return_value = json.dumps({
-        'installed': {
-            'client_id': self.fake_client_id + '.apps.googleusercontent.com',
-            'client_secret': self.fake_client_secret,
-        }
-    })
-
-    oauth.Credentials.from_client_secrets_file(self.fake_filename,
-                                               self.fake_scopes)
-    self.assertEqual(self.fake_client_id,
-                     mock_creds_from_client_secrets.call_args[0][0])
-
-  def test_get_token_value_known_token_field(self):
-    token_data = {'known-field': 'known-value'}
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token_data=token_data)
-    self.assertEqual('known-value', creds.get_token_value('known-field'))
-
-  def test_get_token_value_unknown_field_returns_unknown(self):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        id_token_data=self.fake_token_data)
-    self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
-
-  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
-  def test_get_token_value_credentials_expired(self, mock_verify_oauth2_token):
-    mock_verify_oauth2_token.return_value = {'fetched-field': 'fetched-value'}
-    time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
-        minutes=5)
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        expiry=time_earlier_than_now,
-        id_token=self.fake_id_token,
-        id_token_data=None)
-    self.assertTrue(creds.expired)
-    creds.refresh = MagicMock()
-
-    token_value = creds.get_token_value('fetched-field')
-
-    self.assertEqual('fetched-value', token_value)
-    self.assertTrue(creds.refresh.called)
-
-  def test_to_json_contains_all_required_fields(self):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        id_token=self.fake_id_token,
-        id_token_data=self.fake_token_data,
-        token_uri=self.fake_token_uri,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        scopes=self.fake_scopes,
-        quota_project_id=self.fake_quota_project_id,
-        expiry=self.fake_token_expiry)
-    json_string = creds.to_json()
-    json_data = json.loads(json_string)
-    keys = json_data.keys()
-    self.assertIn('token', keys)
-    self.assertEqual(self.fake_token, json_data['token'])
-    self.assertIn('refresh_token', keys)
-    self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
-    self.assertIn('id_token', keys)
-    self.assertEqual(self.fake_id_token, json_data['id_token'])
-    self.assertIn('token_uri', keys)
-    self.assertEqual(self.fake_token_uri, json_data['token_uri'])
-    self.assertIn('client_id', keys)
-    self.assertEqual(self.fake_client_id, json_data['client_id'])
-    self.assertIn('client_secret', keys)
-    self.assertEqual(self.fake_client_secret, json_data['client_secret'])
-    self.assertNotIn('scopes', keys)  # Scopes are not currently saved
-    self.assertIn('token_expiry', keys)
-    self.assertEqual(
-        self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
-        json_data['token_expiry'])
-    self.assertIn('decoded_id_token', keys)
-    self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
-
-  def test_credentials_to_json_and_back(self):
-    original_creds = oauth.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        id_token=self.fake_id_token,
-        id_token_data=self.fake_token_data,
-        token_uri=self.fake_token_uri,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        scopes=self.fake_scopes,
-        quota_project_id=self.fake_quota_project_id,
-        expiry=self.fake_token_expiry)
-    pickled_creds = original_creds.to_json()
-    serialized_json = json.loads(pickled_creds)
-    unpickled_creds = oauth.Credentials.from_authorized_user_info(
-        serialized_json)
-    self.assertEqual(original_creds.token, unpickled_creds.token)
-    self.assertEqual(original_creds.refresh_token,
-                     unpickled_creds.refresh_token)
-    self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
-    self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
-    self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
-    self.assertEqual(original_creds.client_secret,
-                     unpickled_creds.client_secret)
-    self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
-
-  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
-  def test_refresh_calls_super_refresh(self, mock_super_refresh):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret)
-    request = MagicMock()
-
-    creds.refresh(request)
-    self.assertTrue(mock_super_refresh.called)
-    self.assertEqual(request, mock_super_refresh.call_args[0][0])
-
-  def test_refresh_locks_resource_during_refresh(self):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret)
-    lock = creds._lock
-
-    def check_lock_is_locked(*unused_args, **unused_kwargs):
-      self.assertTrue(lock.is_locked)
-
-    # We need to mock the superclass refresh so it doesn't actually try to
-    # refresh our fake token.
-    # At the same time, we'll make sure the lock is held during the refresh.
-    with patch.object(oauth.google.oauth2.credentials.Credentials,
-                      'refresh') as mock_refresh:
-      mock_refresh.side_effect = check_lock_is_locked
-      creds.refresh(request=MagicMock())
-
-    # Make sure our side effect was actually performed.
-    self.assertTrue(mock_refresh.called)
-    # The lock should be released after refresh
-    self.assertFalse(lock.is_locked)
-
-  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
-  @patch.object(oauth.fileutils, 'write_file')
-  def test_refresh_writes_new_credentials_to_disk_after_refresh(
-      self, mock_write_file, mock_super_refresh):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-
-    def update_access_token(unused_request):
-      creds.token = 'refreshed_access_token'
-
-    mock_super_refresh.side_effect = update_access_token
-
-    self.assertIsNone(creds.token)
-    creds.refresh(request=MagicMock())
-    self.assertEqual('refreshed_access_token', creds.token,
-                     'Access token was not refreshed')
-    text_written_to_file = mock_write_file.call_args[0][1]
-    self.assertIsNotNone(text_written_to_file, 'Nothing was written to file')
-    saved_json = json.loads(text_written_to_file)
-    self.assertEqual('refreshed_access_token', saved_json['token'],
-                     'Refreshed access token was not saved to disk')
-
-  def test_write_writes_credentials_to_disk(self):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-
-    self.assertFalse(os.path.exists(self.fake_filename))
-    creds.write()
-    self.assertTrue(os.path.exists(self.fake_filename))
-
-  def test_write_raises_error_when_no_credentials_file_is_set(self):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret)
-
-    self.assertIsNone(creds.filename)
-    with self.assertRaises(oauth.CredentialsError):
-      creds.write()
-
-  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
-  @patch.object(oauth.fileutils, 'write_file')
-  def test_write_locks_resource_during_write(self, mock_write_file,
-                                             unused_mock_super_refresh):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-    lock = creds._lock
-
-    def check_lock_is_locked(*unused_args, **unused_kwargs):
-      self.assertTrue(creds._lock.is_locked)
-
-    mock_write_file.side_effect = check_lock_is_locked
-
-    self.assertFalse(lock.is_locked)
-    creds.refresh(request=MagicMock())
-    self.assertFalse(lock.is_locked)
-    self.assertTrue(mock_write_file.called)
-
-  def test_delete_removes_credentials_file(self):
-    self.assertFalse(os.path.exists(self.fake_filename))
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-    creds.write()
-    self.assertTrue(os.path.exists(self.fake_filename))
-    creds.delete()
-    self.assertFalse(os.path.exists(self.fake_filename))
-
-  @unittest.skipIf(
-      platform.system() == 'Windows',
-      reason=('On Windows, Filelock deletes the lock file each time the lock '
-              'is released. Delete does not remove it.'))
-  def test_delete_removes_lock_file(self):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret,
-        filename=self.fake_filename)
-    lock_file = '%s.lock' % creds.filename
-    creds.write()
-    self.assertTrue(os.path.exists(lock_file))
-    creds.delete()
-    self.assertFalse(os.path.exists(lock_file))
-
-  def test_delete_is_noop_when_not_using_filelock(self):
-    creds = oauth.Credentials(
-        token=None,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret)
-    self.assertIsNone(creds.filename)
-    creds.delete()  # This should not raise an exception.
-
-  def test_revoke_requests_credential_revoke(self):
-    creds = oauth.Credentials(
-        token=self.fake_token,
-        refresh_token=self.fake_refresh_token,
-        client_id=self.fake_client_id,
-        client_secret=self.fake_client_secret)
-    mock_http = MagicMock()
-
-    creds.revoke(http=mock_http)
-
-    uri = mock_http.request.call_args[0][0]
-    self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
-    params = uri[uri.index('?'):]
-    self.assertIn('token=%s' % creds.refresh_token, params)
-    self.assertEqual('GET', mock_http.request.call_args[0][1])
-
-
-class ShortUrlFlowTest(unittest.TestCase):
-
-  def setUp(self):
-    self.fake_client_id = 'fake_client_id'
-    self.fake_client_secret = 'fake_client_secret'
-    self.fake_scopes = [
-        'fake_api.readonly',
-        'fake_other_api.write',
-    ]
-    self.fake_client_config = {
-        'installed': {
-            'client_id': self.fake_client_id,
-            'client_secret': self.fake_client_secret,
-            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
-            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
-            'token_uri': 'https://oauth2.googleapis.com/token',
-        }
-    }
-    self.long_url = 'http://example.com/some/long/url'
-    self.short_url = 'http://ex.co/short'
-    super(ShortUrlFlowTest, self).setUp()
-
-  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
-                'authorization_url')
-  @unittest.skip('disable short url tests temporarily.')
-  def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
-    url_flow = oauth._ShortURLFlow.from_client_config(
-        self.fake_client_config, scopes=self.fake_scopes)
-    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
-
-    mock_http = MagicMock()
-    mock_response = MagicMock()
-    mock_response.status = 200
-    content = json.dumps({'short_url': self.short_url})
-    mock_http.request.return_value = (mock_response, content)
-
-    url, state = url_flow.authorization_url(http=mock_http)
-    self.assertEqual(self.short_url, url)
-    self.assertEqual('fake_state', state)
-
-    # Verify request() was called with the expected arguments.
-    self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
-                     mock_http.request.call_args[0][0])
-    self.assertEqual('POST', mock_http.request.call_args[0][1])
-    self.assertIn(self.long_url, mock_http.request.call_args[0][2])
-
-  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
-                'authorization_url')
-  @unittest.skip('disable short url tests temporarily.')
-  def test_shorturlflow_falls_back_to_long_url_on_request_error(
-      self, mock_super_auth_url):
-    url_flow = oauth._ShortURLFlow.from_client_config(
-        self.fake_client_config, scopes=self.fake_scopes)
-    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
-
-    mock_http = MagicMock()
-    mock_http.request.side_effect = Exception()
-
-    url, state = url_flow.authorization_url(http=mock_http)
-    self.assertEqual(self.long_url, url)
-    self.assertEqual('fake_state', state)
-
-  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
-                'authorization_url')
-  @unittest.skip('disable short url tests temporarily.')
-  def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
-      self, mock_super_auth_url):
-    url_flow = oauth._ShortURLFlow.from_client_config(
-        self.fake_client_config, scopes=self.fake_scopes)
-    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
-
-    mock_http = MagicMock()
-    mock_response = MagicMock()
-    mock_response.status = 404  # Use a status that is not 200
-    content = json.dumps({'short_url': self.short_url})
-    mock_http.request.return_value = (mock_response, content)
-
-    url, state = url_flow.authorization_url(http=mock_http)
-    self.assertEqual(self.long_url, url)
-    self.assertEqual('fake_state', state)
-
-  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
-                'authorization_url')
-  @unittest.skip('disable short url tests temporarily.')
-  def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
-      self, mock_super_auth_url):
-    url_flow = oauth._ShortURLFlow.from_client_config(
-        self.fake_client_config, scopes=self.fake_scopes)
-    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
-
-    mock_http = MagicMock()
-    mock_response = MagicMock()
-    mock_response.status = 200
-    content = None
-    mock_http.request.return_value = (mock_response, content)
-
-    url, state = url_flow.authorization_url(http=mock_http)
-    self.assertEqual(self.long_url, url)
-    self.assertEqual('fake_state', state)
-
-  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
-                'authorization_url')
-  @unittest.skip('disable short url tests temporarily.')
-  def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
-      self, mock_super_auth_url):
-    url_flow = oauth._ShortURLFlow.from_client_config(
-        self.fake_client_config, scopes=self.fake_scopes)
-    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
-
-    mock_http = MagicMock()
-    mock_response = MagicMock()
-    mock_response.status = 200
-    content = json.dumps({})  # This json content contains no "short-url" key
-    mock_http.request.return_value = (mock_response, content)
-
-    url, state = url_flow.authorization_url(http=mock_http)
-    self.assertEqual(self.long_url, url)
-    self.assertEqual('fake_state', state)
-
-
-if __name__ == '__main__':
-  unittest.main()

src/cloudprint-v2.json

@@ -142,7 +142,7 @@
        "location": "query"
       }
      }
-    }, 
+    },
     "list": {
      "id": "cloudprint.jobs.list",
      "path": "jobs",
@@ -347,7 +347,7 @@
        "type": "boolean",
        "location": "query"
       }
-     } 
+     }
     },
     "unshare": {
      "id": "cloudprint.printers.unshare",
@@ -479,7 +479,7 @@
        "location": "query"
       }
      }
-    } 
+    }
    }
   }
  }

src/controlflow_test.py

@@ -1,106 +0,0 @@
-"""Tests for controlflow."""
-
-import unittest
-from unittest.mock import patch
-
-import controlflow
-
-
-class ControlFlowTest(unittest.TestCase):
-
-  def test_system_error_exit_raises_systemexit_error(self):
-    with self.assertRaises(SystemExit):
-      controlflow.system_error_exit(1, 'exit message')
-
-  def test_system_error_exit_raises_systemexit_with_return_code(self):
-    with self.assertRaises(SystemExit) as context_manager:
-      controlflow.system_error_exit(100, 'exit message')
-    self.assertEqual(context_manager.exception.code, 100)
-
-  @patch.object(controlflow.display, 'print_error')
-  def test_system_error_exit_prints_error_before_exiting(self, mock_print_err):
-    with self.assertRaises(SystemExit):
-      controlflow.system_error_exit(100, 'exit message')
-    self.assertIn('exit message', mock_print_err.call_args[0][0])
-
-  def test_csv_field_error_exit_raises_systemexit_error(self):
-    with self.assertRaises(SystemExit):
-      controlflow.csv_field_error_exit('aField',
-                                       ['unusedField1', 'unusedField2'])
-
-  def test_csv_field_error_exit_exits_code_2(self):
-    with self.assertRaises(SystemExit) as context_manager:
-      controlflow.csv_field_error_exit('aField',
-                                       ['unusedField1', 'unusedField2'])
-    self.assertEqual(context_manager.exception.code, 2)
-
-  @patch.object(controlflow.display, 'print_error')
-  def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
-    with self.assertRaises(SystemExit):
-      controlflow.csv_field_error_exit('aField',
-                                       ['unusedField1', 'unusedField2'])
-    printed_message = mock_print_err.call_args[0][0]
-    self.assertIn('aField', printed_message)
-    self.assertIn('unusedField1', printed_message)
-    self.assertIn('unusedField2', printed_message)
-
-  def test_invalid_json_exit_raises_systemexit_error(self):
-    with self.assertRaises(SystemExit):
-      controlflow.invalid_json_exit('filename')
-
-  def test_invalid_json_exit_exit_exits_code_17(self):
-    with self.assertRaises(SystemExit) as context_manager:
-      controlflow.invalid_json_exit('filename')
-    self.assertEqual(context_manager.exception.code, 17)
-
-  @patch.object(controlflow.display, 'print_error')
-  def test_invalid_json_exit_prints_error_details(self, mock_print_err):
-    with self.assertRaises(SystemExit):
-      controlflow.invalid_json_exit('filename')
-    printed_message = mock_print_err.call_args[0][0]
-    self.assertIn('filename', printed_message)
-
-  @patch.object(controlflow.time, 'sleep')
-  def test_wait_on_failure_waits_exponentially(self, mock_sleep):
-    controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
-    controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
-    controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
-
-    sleep_calls = mock_sleep.call_args_list
-    self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
-    self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
-    self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
-
-  @patch.object(controlflow.time, 'sleep')
-  def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
-    total_attempts = 20
-    for attempt in range(1, total_attempts + 1):
-      controlflow.wait_on_failure(
-          attempt,
-          total_attempts,
-          'Attempt #%s' % attempt,
-          # Suppress messages while we make a lot of attempts.
-          error_print_threshold=total_attempts + 1)
-      # Wait time may be between 60 and 61 secs, due to rand addition.
-      self.assertLessEqual(mock_sleep.call_args[0][0], 61)
-
-  # Prevent the system from actually sleeping and thus slowing down the test.
-  @patch.object(controlflow.time, 'sleep')
-  def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
-    message = 'An error message to display'
-    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
-      controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
-    self.assertIn(message, mock_stderr_write.call_args[0][0])
-
-  @patch.object(controlflow.time, 'sleep')
-  def test_wait_on_failure_only_prints_after_threshold(self, unused_mock_sleep):
-    total_attempts = 5
-    threshold = 3
-    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
-      for attempt in range(1, total_attempts + 1):
-        controlflow.wait_on_failure(
-            attempt,
-            total_attempts,
-            'Attempt #%s' % attempt,
-            error_print_threshold=threshold)
-    self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

src/display.py

@@ -1,235 +0,0 @@
-"""Methods related to display of information to the user."""
-
-import csv
-import datetime
-import io
-import sys
-import webbrowser
-
-import dateutil
-import googleapiclient.http
-
-#TODO: get rid of these hacks
-import __main__
-from var import *
-import controlflow
-import gapi
-
-
-def current_count(i, count):
-  return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
-
-def current_count_nl(i, count):
-  return f' ({i}/{count})\n' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
-
-def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
-  fields = fieldsChoiceMap[fieldName.lower()]
-  if isinstance(fields, list):
-    fieldsList.extend(fields)
-  else:
-    fieldsList.append(fields)
-
-# Write a CSV file
-def add_titles_to_csv_file(addTitles, titles):
-  for title in addTitles:
-    if title not in titles:
-      titles.append(title)
-
-def add_row_titles_to_csv_file(row, csvRows, titles):
-  csvRows.append(row)
-  for title in row:
-    if title not in titles:
-      titles.append(title)
-
-# fieldName is command line argument
-# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
-#ARGUMENT_TO_PROPERTY_MAP = {
-#  u'admincreated': [u'adminCreated'],
-#  u'aliases': [u'aliases', u'nonEditableAliases'],
-#  }
-# fieldsList is the list of API fields
-# fieldsTitles maps the API field name to the CSV file header
-def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles, titles):
-  for ftList in fieldNameMap[fieldName]:
-    if ftList not in fieldsTitles:
-      fieldsList.append(ftList)
-      fieldsTitles[ftList] = ftList
-      add_titles_to_csv_file([ftList], titles)
-
-# fieldName is command line argument
-# fieldNameTitleMap maps fieldName to API field name and CSV file header
-#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
-#  u'admincreated': [u'adminCreated', u'Admin_Created'],
-#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
-#  }
-# fieldsList is the list of API fields
-# fieldsTitles maps the API field name to the CSV file header
-def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList, fieldsTitles, titles):
-  ftList = fieldNameTitleMap[fieldName]
-  for i in range(0, len(ftList), 2):
-    if ftList[i] not in fieldsTitles:
-      fieldsList.append(ftList[i])
-      fieldsTitles[ftList[i]] = ftList[i+1]
-      add_titles_to_csv_file([ftList[i+1]], titles)
-
-def sort_csv_titles(firstTitle, titles):
-  restoreTitles = []
-  for title in firstTitle:
-    if title in titles:
-      titles.remove(title)
-      restoreTitles.append(title)
-  titles.sort()
-  for title in restoreTitles[::-1]:
-    titles.insert(0, title)
-
-def QuotedArgumentList(items):
-  return ' '.join([item if item and (item.find(' ') == -1) and (item.find(',') == -1) else '"'+item+'"' for item in items])
-
-def write_csv_file(csvRows, titles, list_type, todrive):
-  def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
-    if not rowDate or not isinstance(rowDate, str):
-      return False
-    try:
-      rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
-      if dateMode:
-        rowDate = datetime.datetime(rowTime.year, rowTime.month, rowTime.day).isoformat()+'Z'
-    except ValueError:
-      rowDate = NEVER_TIME
-    if op == '<':
-      return rowDate < filterDate
-    if op == '<=':
-      return rowDate <= filterDate
-    if op == '>':
-      return rowDate > filterDate
-    if op == '>=':
-      return rowDate >= filterDate
-    if op == '!=':
-      return rowDate != filterDate
-    return rowDate == filterDate
-
-  def rowCountFilterMatch(rowCount, op, filterCount):
-    if isinstance(rowCount, str):
-      if not rowCount.isdigit():
-        return False
-      rowCount = int(rowCount)
-    elif not isinstance(rowCount, int):
-      return False
-    if op == '<':
-      return rowCount < filterCount
-    if op == '<=':
-      return rowCount <= filterCount
-    if op == '>':
-      return rowCount > filterCount
-    if op == '>=':
-      return rowCount >= filterCount
-    if op == '!=':
-      return rowCount != filterCount
-    return rowCount == filterCount
-  def rowBooleanFilterMatch(rowBoolean, filterBoolean):
-    if not isinstance(rowBoolean, bool):
-      return False
-    return rowBoolean == filterBoolean
-
-  def headerFilterMatch(title):
-    for filterStr in GC_Values[GC_CSV_HEADER_FILTER]:
-      if filterStr.match(title):
-        return True
-    return False
-
-  if GC_Values[GC_CSV_ROW_FILTER]:
-    for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
-      if column not in titles:
-        sys.stderr.write(f'WARNING: Row filter column "{column}" is not in output columns\n')
-        continue
-      if filterVal[0] == 'regex':
-        csvRows = [row for row in csvRows if filterVal[1].search(str(row.get(column, '')))]
-      elif filterVal[0] == 'notregex':
-        csvRows = [row for row in csvRows if not filterVal[1].search(str(row.get(column, '')))]
-      elif filterVal[0] in ['date', 'time']:
-        csvRows = [row for row in csvRows if rowDateTimeFilterMatch(filterVal[0] == 'date', row.get(column, ''), filterVal[1], filterVal[2])]
-      elif filterVal[0] == 'count':
-        csvRows = [row for row in csvRows if rowCountFilterMatch(row.get(column, 0), filterVal[1], filterVal[2])]
-      else: #boolean
-        csvRows = [row for row in csvRows if rowBooleanFilterMatch(row.get(column, False), filterVal[1])]
-  if GC_Values[GC_CSV_HEADER_FILTER]:
-    titles = [t for t in titles if headerFilterMatch(t)]
-    if not titles:
-      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER\n')
-      return
-  csv.register_dialect('nixstdout', lineterminator='\n')
-  if todrive:
-    write_to = io.StringIO()
-  else:
-    write_to = sys.stdout
-  writer = csv.DictWriter(write_to, fieldnames=titles, dialect='nixstdout', extrasaction='ignore', quoting=csv.QUOTE_MINIMAL)
-  try:
-    writer.writerow(dict((item, item) for item in writer.fieldnames))
-    writer.writerows(csvRows)
-  except IOError as e:
-    controlflow.system_error_exit(6, e)
-  if todrive:
-    admin_email = __main__._getValueFromOAuth('email')
-    _, drive = __main__.buildDrive3GAPIObject(admin_email)
-    if not drive:
-      print(f'''\nGAM is not authorized to create Drive files. Please run:
-gam user {admin_email} check serviceaccount
-and follow recommend steps to authorize GAM for Drive access.''')
-      sys.exit(5)
-    result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
-    columns = len(titles)
-    rows = len(csvRows)
-    cell_count = rows * columns
-    data_size = len(write_to.getvalue())
-    max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
-    if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
-      print(f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}')
-      mimeType = 'text/csv'
-    else:
-      mimeType = MIMETYPE_GA_SPREADSHEET
-    body = {'description': QuotedArgumentList(sys.argv),
-            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
-            'mimeType': mimeType}
-    result = gapi.call(drive.files(), 'create', fields='webViewLink',
-                       body=body,
-                       media_body=googleapiclient.http.MediaInMemoryUpload(write_to.getvalue().encode(),
-                                                                           mimetype='text/csv'))
-    file_url = result['webViewLink']
-    if GC_Values[GC_NO_BROWSER]:
-      msg_txt = f'Drive file uploaded to:\n {file_url}'
-      msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
-      __main__.send_email(msg_subj, msg_txt)
-      print(msg_txt)
-    else:
-      webbrowser.open(file_url)
-
-def print_error(message):
-  """Prints a one-line error message to stderr in a standard format."""
-  sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
-
-
-def print_warning(message):
-  """Prints a one-line warning message to stderr in a standard format."""
-  sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
-
-def print_json(object_value, spacing=''):
-  """Prints Dict or Array to screen in clean human-readable format.."""
-  if isinstance(object_value, list):
-    if len(object_value) == 1 and isinstance(object_value[0], (str, int, bool)):
-      sys.stdout.write(f'{object_value[0]}\n')
-      return
-    if spacing:
-      sys.stdout.write('\n')
-    for i, a_value in enumerate(object_value):
-      if isinstance(a_value, (str, int, bool)):
-        sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
-      else:
-        sys.stdout.write(f' {spacing}{i+1}) ')
-        print_json(a_value, f' {spacing}')
-  elif isinstance(object_value, dict):
-    for key in ['kind', 'etag', 'etags']:
-      object_value.pop(key, None)
-    for another_object, another_value in object_value.items():
-      sys.stdout.write(f' {spacing}{another_object}: ')
-      print_json(another_value, f' {spacing}')
-  else:
-    sys.stdout.write(f'{object_value}\n')

src/display_test.py

@@ -1,59 +0,0 @@
-"""Tests for display."""
-
-import unittest
-from unittest.mock import patch
-
-import display
-from var import ERROR_PREFIX
-from var import WARNING_PREFIX
-
-
-class DisplayTest(unittest.TestCase):
-
-  def test_print_error_prints_to_stderr(self):
-    message = 'test error'
-    with patch.object(display.sys.stderr, 'write') as mock_write:
-      display.print_error(message)
-    printed_message = mock_write.call_args[0][0]
-    self.assertIn(message, printed_message)
-
-  def test_print_error_prints_error_prefix(self):
-    message = 'test error'
-    with patch.object(display.sys.stderr, 'write') as mock_write:
-      display.print_error(message)
-    printed_message = mock_write.call_args[0][0]
-    self.assertLess(
-        printed_message.find(ERROR_PREFIX), printed_message.find(message),
-        'The error prefix does not appear before the error message')
-
-  def test_print_error_ends_message_with_newline(self):
-    message = 'test error'
-    with patch.object(display.sys.stderr, 'write') as mock_write:
-      display.print_error(message)
-    printed_message = mock_write.call_args[0][0]
-    self.assertRegex(printed_message, '\n$',
-                     'The error message does not end in a newline.')
-
-  def test_print_warning_prints_to_stderr(self):
-    message = 'test warning'
-    with patch.object(display.sys.stderr, 'write') as mock_write:
-      display.print_error(message)
-    printed_message = mock_write.call_args[0][0]
-    self.assertIn(message, printed_message)
-
-  def test_print_warning_prints_error_prefix(self):
-    message = 'test warning'
-    with patch.object(display.sys.stderr, 'write') as mock_write:
-      display.print_error(message)
-    printed_message = mock_write.call_args[0][0]
-    self.assertLess(
-        printed_message.find(WARNING_PREFIX), printed_message.find(message),
-        'The warning prefix does not appear before the error message')
-
-  def test_print_warning_ends_message_with_newline(self):
-    message = 'test warning'
-    with patch.object(display.sys.stderr, 'write') as mock_write:
-      display.print_error(message)
-    printed_message = mock_write.call_args[0][0]
-    self.assertRegex(printed_message, '\n$',
-                     'The warning message does not end in a newline.')

src/fileutils_test.py

@@ -1,234 +0,0 @@
-"""Tests for fileutils."""
-
-import io
-import os
-import unittest
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-import fileutils
-
-
-class FileutilsTest(unittest.TestCase):
-
-  def setUp(self):
-    self.fake_path = '/some/path/to/file'
-    super(FileutilsTest, self).setUp()
-
-  @patch.object(fileutils.sys, 'stdin')
-  def test_open_file_stdin(self, mock_stdin):
-    mock_stdin.read.return_value = 'some stdin content'
-    f = fileutils.open_file('-', mode='r')
-    self.assertIsInstance(f, fileutils.io.StringIO)
-    self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
-
-  def test_open_file_stdout(self):
-    f = fileutils.open_file('-', mode='w')
-    self.assertEqual(fileutils.sys.stdout, f)
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_opens_correct_path(self, mock_open):
-    f = fileutils.open_file(self.fake_path)
-    self.assertEqual(self.fake_path, mock_open.call_args[0][0])
-    self.assertEqual(mock_open.return_value, f)
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_expands_user_file_path(self, mock_open):
-    file_path = '~/some/path/containing/tilde/shortcut/to/home'
-    fileutils.open_file(file_path)
-    opened_path = mock_open.call_args[0][0]
-    home_path = os.environ.get('HOME')
-    self.assertIsNotNone(home_path)
-    self.assertIn(home_path, opened_path)
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_opens_correct_mode(self, mock_open):
-    fileutils.open_file(self.fake_path)
-    self.assertEqual('r', mock_open.call_args[0][1])
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_encoding_for_binary(self, mock_open):
-    fileutils.open_file(self.fake_path, mode='b')
-    self.assertIsNone(mock_open.call_args[1]['encoding'])
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_default_system_encoding(self, mock_open):
-    fileutils.open_file(self.fake_path)
-    self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
-                     mock_open.call_args[1]['encoding'])
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_utf8_encoding_specified(self, mock_open):
-    fileutils.open_file(self.fake_path, encoding='UTF-8')
-    self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
-
-  def test_open_file_strips_utf_bom_in_utf(self):
-    bom_prefixed_data = u'\ufefffoobar'
-    fake_file = io.StringIO(bom_prefixed_data)
-    mock_open = MagicMock(spec=open, return_value=fake_file)
-    with patch.object(fileutils, 'open', mock_open):
-      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
-      self.assertEqual('foobar', f.read())
-
-  def test_open_file_strips_utf_bom_in_non_utf(self):
-    bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
-
-    # We need to trick the method under test into believing that a StringIO
-    # instance is a file with an encoding. Since StringIO does not usually have,
-    # an encoding, we'll mock it and add our own encoding, but send the other
-    # methods in use (read and seek) back to the real StringIO object.
-    real_stringio = io.StringIO(bom_prefixed_data)
-    mock_file = MagicMock(spec=io.StringIO)
-    mock_file.read.side_effect = real_stringio.read
-    mock_file.seek.side_effect = real_stringio.seek
-    mock_file.encoding = 'iso-8859-1'
-
-    mock_open = MagicMock(spec=open, return_value=mock_file)
-    with patch.object(fileutils, 'open', mock_open):
-      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
-      self.assertEqual('foobar', f.read())
-
-  def test_open_file_strips_utf_bom_in_binary(self):
-    bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
-    fake_file = io.BytesIO(bom_prefixed_data)
-    mock_open = MagicMock(spec=open, return_value=fake_file)
-    with patch.object(fileutils, 'open', mock_open):
-      f = fileutils.open_file(self.fake_path, mode='rb', strip_utf_bom=True)
-      self.assertEqual(b'foobar', f.read())
-
-  def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
-    no_bom_data = 'This data has no BOM'
-    fake_file = io.StringIO(no_bom_data)
-    mock_open = MagicMock(spec=open, return_value=fake_file)
-
-    with patch.object(fileutils, 'open', mock_open):
-      f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
-      # Since there was no opening BOM, we should be back at the beginning of
-      # the file.
-      self.assertEqual(fake_file.tell(), 0)
-      self.assertEqual(f.read(), no_bom_data)
-
-  @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
-  def test_open_file_exits_on_io_error(self, mock_open):
-    mock_open.side_effect = IOError('Fake IOError')
-    with self.assertRaises(SystemExit) as context:
-      fileutils.open_file(self.fake_path)
-    self.assertEqual(context.exception.code, 6)
-
-  def test_close_file_closes_file_successfully(self):
-    mock_file = MagicMock()
-    self.assertTrue(fileutils.close_file(mock_file))
-    self.assertEqual(mock_file.close.call_count, 1)
-
-  def test_close_file_with_error(self):
-    mock_file = MagicMock()
-    mock_file.close.side_effect = IOError()
-    self.assertFalse(fileutils.close_file(mock_file))
-    self.assertEqual(mock_file.close.call_count, 1)
-
-  @patch.object(fileutils.sys, 'stdin')
-  def test_read_file_from_stdin(self, mock_stdin):
-    mock_stdin.read.return_value = 'some stdin content'
-    self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
-
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_default_params(self, mock_open_file):
-    fake_content = 'some fake content'
-    mock_open_file.return_value.__enter__().read.return_value = fake_content
-    self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
-    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
-    self.assertEqual(mock_open_file.call_args[0][1], 'r')
-    self.assertIsNone(mock_open_file.call_args[1]['newline'])
-
-  @patch.object(fileutils.display, 'print_warning')
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_continues_on_errors_without_displaying(
-      self, mock_open_file, mock_print_warning):
-    mock_open_file.side_effect = IOError()
-    contents = fileutils.read_file(
-        self.fake_path, continue_on_error=True, display_errors=False)
-    self.assertIsNone(contents)
-    self.assertFalse(mock_print_warning.called)
-
-  @patch.object(fileutils.display, 'print_warning')
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_displays_errors(self, mock_open_file, mock_print_warning):
-    mock_open_file.side_effect = IOError()
-    fileutils.read_file(
-        self.fake_path, continue_on_error=True, display_errors=True)
-    self.assertTrue(mock_print_warning.called)
-
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_exits_code_6_when_continue_on_error_is_false(
-      self, mock_open_file):
-    mock_open_file.side_effect = IOError()
-    with self.assertRaises(SystemExit) as context:
-      fileutils.read_file(self.fake_path, continue_on_error=False)
-    self.assertEqual(context.exception.code, 6)
-
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
-    mock_open_file.return_value.__enter__().read.side_effect = LookupError()
-    with self.assertRaises(SystemExit) as context:
-      fileutils.read_file(self.fake_path)
-    self.assertEqual(context.exception.code, 2)
-
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
-    mock_open_file.return_value.__enter__().read.side_effect = UnicodeError()
-    with self.assertRaises(SystemExit) as context:
-      fileutils.read_file(self.fake_path)
-    self.assertEqual(context.exception.code, 2)
-
-  @patch.object(fileutils, '_open_file')
-  def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
-    fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0, 1,
-                                           'testing only')
-    mock_open_file.return_value.__enter__().read.side_effect = fake_decode_error
-    with self.assertRaises(SystemExit) as context:
-      fileutils.read_file(self.fake_path)
-    self.assertEqual(context.exception.code, 2)
-
-  @patch.object(fileutils, '_open_file')
-  def test_write_file_writes_data_to_file(self, mock_open_file):
-    fake_data = 'some fake data'
-    fileutils.write_file(self.fake_path, fake_data)
-    self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
-    self.assertEqual(mock_open_file.call_args[0][1], 'w')
-
-    opened_file = mock_open_file.return_value.__enter__()
-    self.assertTrue(opened_file.write.called)
-    self.assertEqual(opened_file.write.call_args[0][0], fake_data)
-
-  @patch.object(fileutils.display, 'print_error')
-  @patch.object(fileutils, '_open_file')
-  def test_write_file_continues_on_errors_without_displaying(
-      self, mock_open_file, mock_print_error):
-    mock_open_file.side_effect = IOError()
-    status = fileutils.write_file(
-        self.fake_path,
-        'foo data',
-        continue_on_error=True,
-        display_errors=False)
-    self.assertFalse(status)
-    self.assertFalse(mock_print_error.called)
-
-  @patch.object(fileutils.display, 'print_error')
-  @patch.object(fileutils, '_open_file')
-  def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
-    mock_open_file.side_effect = IOError()
-    fileutils.write_file(
-        self.fake_path, 'foo data', continue_on_error=True, display_errors=True)
-    self.assertTrue(mock_print_error.called)
-
-  @patch.object(fileutils, '_open_file')
-  def test_write_file_exits_code_6_when_continue_on_error_is_false(
-      self, mock_open_file):
-    mock_open_file.side_effect = IOError()
-    with self.assertRaises(SystemExit) as context:
-      fileutils.write_file(self.fake_path, 'foo data', continue_on_error=False)
-    self.assertEqual(context.exception.code, 6)
-
-
-if __name__ == '__main__':
-  unittest.main()

src/gam-install.sh

@@ -238,9 +238,11 @@ fi
 if [ "$update_profile" = true ]; then
   alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
   if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0
+    update_profile "$HOME/.zshrc" 0
   elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.zshrc" 1
   fi
 else
   echo_yellow "skipping profile update."

src/gam.spec

@@ -2,23 +2,31 @@
 
 import sys
 
+import importlib
+from PyInstaller.utils.hooks import copy_metadata
+
 sys.modules['FixTk'] = None
 
-a = Analysis(['gam.py'],
+extra_files = [('cloudprint-v2.json', 'cloudprint-v2.json')]
+
+# dynamically determine where httplib2/cacerts.txt lives
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+extra_files += [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+
+extra_files += copy_metadata('google-api-python-client')
+
+a = Analysis(['gam/__main__.py'],
              hiddenimports=[],
              hookspath=None,
              excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             datas=extra_files,
              runtime_hooks=None)
+
 for d in a.datas:
     if 'pyconfig' in d[0]:
         a.datas.remove(d)
         break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
 
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
 
 pyz = PYZ(a.pure)
 exe = EXE(pyz,

src/gam.wxs

@@ -41,7 +41,7 @@
     <ComponentGroup
         Id="ProductComponents"
         Directory="INSTALLFOLDER"
-        Source="gam-64">
+	Source="dist/gam">
       <Component Id="gam_exe" Guid="886abc07-73c5-4acc-9f71-58daf62aabc1">
         <File Name="gam.exe" KeyPath="yes" />
         <Environment Id="PATH" Name="PATH" Value="[INSTALLFOLDER]" Permanent="yes" Part="last" Action="set" System="yes" />
@@ -49,9 +49,6 @@
       <Component Id="license" Guid="7a15de2e-fb91-4d0a-b8bf-c8b19c68f569">
         <File Name="LICENSE" KeyPath="yes" />
       </Component>
-      <Component Id="whatsnew_txt" Guid="6aa9863c-90d9-412f-9b73-fda82549a950">
-        <File Name="whatsnew.txt" KeyPath="yes" />
-      </Component>
       <Component Id="gam_setup_bat" Guid="ef01f93a-4b50-488a-9c04-ec5e13e66218">
         <File Name="gam-setup.bat" KeyPath="yes" />
       </Component>

src/gam/__main__.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# GAM
+#
+# Copyright 2019, LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GAM is a command line tool which allows Administrators to control their G Suite domain and accounts.
+
+With GAM you can programmatically create users, turn on/off services for users like POP and Forwarding and much more.
+For more information, see https://git.io/gam
+"""
+
+import sys
+from multiprocessing import freeze_support
+from multiprocessing import set_start_method
+
+from gam import controlflow
+import gam
+
+
+def main(argv):
+    freeze_support()
+    if sys.platform == 'darwin':
+        # https://bugs.python.org/issue33725 in Python 3.8.0 seems
+        # to break parallel operations with errors about extra -b
+        # command line arguments
+        set_start_method('fork')
+    if sys.version_info[0] < 3 or sys.version_info[1] < 6:
+        controlflow.system_error_exit(
+            5,
+            f'GAM requires Python 3.6 or newer. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.'
+            % sys.version_info[:3])
+    sys.exit(gam.ProcessGAMCommand(sys.argv))
+
+
+# Run from command line
+if __name__ == '__main__':
+    main(sys.argv)

src/gam/auth/__init__.py

@@ -1,10 +1,10 @@
 """Authentication/Credentials general purpose and convenience methods."""
 
-import transport
-from var import _FN_OAUTH2_TXT
-from var import GC_OAUTH2_TXT
-from var import GC_Values
-from . import oauth
+from gam.auth import oauth
+from gam.var import _FN_OAUTH2_TXT
+from gam.var import GC_OAUTH2_TXT
+from gam.var import GC_Values
+
 # TODO: Move logic that determines file name into this module. We should be able
 # to discover the file location without accessing a private member or waiting
 # for a global initialization.

src/gam/auth/oauth.py

@@ -0,0 +1,560 @@
+"""OAuth2.0 user credentials."""
+
+import datetime
+import json
+import os
+import re
+import threading
+from urllib.parse import urlencode
+
+from filelock import FileLock
+import google_auth_oauthlib.flow
+import google.oauth2.credentials
+import google.oauth2.id_token
+
+from gam import fileutils
+from gam import transport
+from gam.var import GM_Globals
+from gam.var import GM_WINDOWS
+from gam import utils
+
+MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
+                                        'browser:\n\n\t{url}\n')
+MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
+MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
+                                             ' visit:\n\n\t{url}\n\nIf your '
+                                             'browser is on a different machine'
+                                             ' then press CTRL+C and create a '
+                                             'file called nobrowser.txt in the '
+                                             'same folder as GAM.\n')
+MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
+                                ' close this browser window and return to GAM.')
+
+
+class CredentialsError(Exception):
+    """Base error class."""
+    pass
+
+
+class InvalidCredentialsFileError(CredentialsError):
+    """Error raised when a file cannot be opened into a credentials object."""
+    pass
+
+
+class EmptyCredentialsFileError(InvalidCredentialsFileError):
+    """Error raised when a credentials file contains no content."""
+    pass
+
+
+class InvalidClientSecretsFileFormatError(CredentialsError):
+    """Error raised when a client secrets file format is invalid."""
+    pass
+
+
+class InvalidClientSecretsFileError(CredentialsError):
+    """Error raised when client secrets file cannot be read."""
+    pass
+
+
+class Credentials(google.oauth2.credentials.Credentials):
+    """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
+
+    DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+    def __init__(self,
+                 token,
+                 refresh_token=None,
+                 id_token=None,
+                 token_uri=None,
+                 client_id=None,
+                 client_secret=None,
+                 scopes=None,
+                 quota_project_id=None,
+                 expiry=None,
+                 id_token_data=None,
+                 filename=None):
+        """A thread-safe OAuth2.0 credentials object.
+
+    Credentials adds additional utility properties and methods to a
+    standard OAuth2.0 credentials object. When used to store credentials on
+    disk, it implements a file lock to avoid collision during writes.
+
+    Args:
+      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
+        information is provided.
+      refresh_token: String, The OAuth 2.0 refresh token. If specified,
+        credentials can be refreshed.
+      id_token: String, The Open ID Connect ID Token.
+      token_uri: String, The OAuth 2.0 authorization server's token endpoint
+        URI. Must be specified for refresh, can be left as None if the token can
+        not be refreshed.
+      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
+        can be left as None if the token can not be refreshed.
+      client_secret: String, The OAuth 2.0 client secret. Must be specified for
+        refresh, can be left as None if the token can not be refreshed.
+      scopes: Sequence[str], The scopes used to obtain authorization.
+        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
+          not request additional scopes after authorization. The scopes must be
+          derivable from the refresh token if refresh information is provided
+          (e.g. The refresh token scopes are a superset of this or contain a
+          wild card scope like
+            'https://www.googleapis.com/auth/any-api').
+      quota_project_id: String, The project ID used for quota and billing. This
+        project may be different from the project used to create the
+        credentials.
+      expiry: datetime.datetime, The time at which the provided token will
+        expire.
+      id_token_data: Oauth2.0 ID Token data which was previously fetched for
+        this access token against the google.oauth2.id_token library.
+      filename: String, Path to a file that will be used to store the
+        credentials. If provided, a lock file of the same name and a ".lock"
+          extension will be created for concurrency controls. Note: New
+            credentials are not saved to disk until write() or refresh() are
+            called.
+
+    Raises:
+      TypeError: If id_token_data is not the required dict type.
+    """
+        super(Credentials, self).__init__(token=token,
+                                          refresh_token=refresh_token,
+                                          id_token=id_token,
+                                          token_uri=token_uri,
+                                          client_id=client_id,
+                                          client_secret=client_secret,
+                                          scopes=scopes,
+                                          quota_project_id=quota_project_id)
+
+        # Load data not restored by the super class
+        self.expiry = expiry
+        if id_token_data and not isinstance(id_token_data, dict):
+            raise TypeError(f'Expected type id_token_data dict but received '
+                            f'{type(id_token_data)}')
+        self._id_token_data = id_token_data.copy() if id_token_data else None
+
+        # If a filename is provided, use a lock file to control concurrent access
+        # to the resource. If no filename is provided, use a thread lock that has
+        # the same interface as FileLock in order to simplify the implementation.
+        if filename:
+            # Convert relative paths into absolute
+            self._filename = os.path.abspath(filename)
+            lock_file = os.path.abspath(f'{self._filename}.lock')
+            self._lock = FileLock(lock_file)
+        else:
+            self._filename = None
+            self._lock = _FileLikeThreadLock()
+
+    # Use a property to prevent external mutation of the filename.
+    @property
+    def filename(self):
+        return self._filename
+
+    @classmethod
+    def from_authorized_user_info(cls, info, filename=None):
+        """Generates Credentials from JSON containing authorized user info.
+
+    Args:
+      info: Dict, authorized user info in Google format.
+      filename: String, the filename used to store these credentials on disk. If
+        no filename is provided, the credentials will not be saved to disk.
+
+    Raises:
+      ValueError: If missing fields are detected in the info.
+    """
+        # We need all of these keys
+        keys_needed = set(('client_id', 'client_secret'))
+        # We need 1 or more of these keys
+        keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
+        missing = keys_needed.difference(info.keys())
+        has_one_of = set(info) & keys_need_one_of
+        if missing or not has_one_of:
+            raise ValueError(
+                'Authorized user info was not in the expected format, missing '
+                f'fields {", ".join(missing)} and one of '
+                f'{", ".join(keys_need_one_of)}.')
+
+        expiry = info.get('token_expiry')
+        if expiry:
+            # Convert the raw expiry to datetime
+            expiry = datetime.datetime.strptime(expiry,
+                                                Credentials.DATETIME_FORMAT)
+        id_token_data = info.get('decoded_id_token')
+
+        # Provide backwards compatibility with field names when loading from JSON.
+        # Some field names may be different, depending on when/how the credentials
+        # were pickled.
+        return cls(token=info.get('token', info.get('auth_token', '')),
+                   refresh_token=info.get('refresh_token', ''),
+                   id_token=info.get('id_token_jwt', info.get('id_token')),
+                   token_uri=info.get('token_uri'),
+                   client_id=info['client_id'],
+                   client_secret=info['client_secret'],
+                   scopes=info.get('scopes'),
+                   quota_project_id=info.get('quota_project_id'),
+                   expiry=expiry,
+                   id_token_data=id_token_data,
+                   filename=filename)
+
+    @classmethod
+    def from_google_oauth2_credentials(cls, credentials, filename=None):
+        """Generates Credentials from a google.oauth2.Credentials object."""
+        info = json.loads(credentials.to_json())
+        # Add properties which are not exported with the native to_json() output.
+        info['id_token'] = credentials.id_token
+        if credentials.expiry:
+            info['token_expiry'] = credentials.expiry.strftime(
+                Credentials.DATETIME_FORMAT)
+        info['quota_project_id'] = credentials.quota_project_id
+
+        return cls.from_authorized_user_info(info, filename=filename)
+
+    @classmethod
+    def from_credentials_file(cls, filename):
+        """Generates Credentials from a stored Credentials file.
+
+    The same file will be used to save the credentials when the access token is
+    refreshed.
+
+    Args:
+      filename: String, the name of a file containing JSON credentials to load.
+        The same filename will be used to save credentials back to disk.
+
+    Returns:
+      The credentials loaded from disk.
+
+    Raises:
+      InvalidCredentialsFileError: When the credentials file cannot be opened.
+      EmptyCredentialsFileError: When the provided file contains no credentials.
+    """
+        file_content = fileutils.read_file(filename,
+                                           continue_on_error=True,
+                                           display_errors=False)
+        if file_content is None:
+            raise InvalidCredentialsFileError(
+                f'File {filename} could not be opened')
+        info = json.loads(file_content)
+        if not info:
+            raise EmptyCredentialsFileError(
+                f'File {filename} contains no credential data')
+
+        try:
+            # We read the existing data from the passed in file, but we also want to
+            # save future data/tokens in the same place.
+            return cls.from_authorized_user_info(info, filename=filename)
+        except ValueError as e:
+            raise InvalidCredentialsFileError(str(e))
+
+    @classmethod
+    def from_client_secrets(cls,
+                            client_id,
+                            client_secret,
+                            scopes,
+                            access_type='offline',
+                            login_hint=None,
+                            filename=None,
+                            use_console_flow=False):
+        """Runs an OAuth Flow from client secrets to generate credentials.
+
+    Args:
+      client_id: String, The OAuth2.0 Client ID.
+      client_secret: String, The OAuth2.0 Client Secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      filename: String, the path to a file to use to save the credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Returns:
+      Credentials
+    """
+        client_config = {
+            'installed': {
+                'client_id': client_id,
+                'client_secret': client_secret,
+                'redirect_uris': [
+                    'http://localhost', 'urn:ietf:wg:oauth:2.0:oob'
+                ],
+                'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+                'token_uri': 'https://oauth2.googleapis.com/token',
+            }
+        }
+
+        flow = _ShortURLFlow.from_client_config(client_config,
+                                                scopes,
+                                                autogenerate_code_verifier=True)
+        flow_kwargs = {'access_type': access_type}
+        if login_hint:
+            flow_kwargs['login_hint'] = login_hint
+
+        # TODO: Move code for browser detection somewhere in this file so that the
+        # messaging about `nobrowser.txt` is co-located with the logic that uses it.
+        if use_console_flow:
+            flow.run_console(
+                authorization_prompt_message=
+                MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
+                authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
+                **flow_kwargs)
+        else:
+            flow.run_local_server(authorization_prompt_message=
+                                  MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
+                                  success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
+                                  **flow_kwargs)
+        return cls.from_google_oauth2_credentials(flow.credentials,
+                                                  filename=filename)
+
+    @classmethod
+    def from_client_secrets_file(cls,
+                                 client_secrets_file,
+                                 scopes,
+                                 access_type='offline',
+                                 login_hint=None,
+                                 credentials_file=None,
+                                 use_console_flow=False):
+        """Runs an OAuth Flow from secrets stored on disk to generate credentials.
+
+    Args:
+      client_secrets_file: String, path to a file containing a client ID and
+        secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      credentials_file: String, the path to a file to use to save the
+        credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Raises:
+      InvalidClientSecretsFileError: If the client secrets file cannot be
+        opened.
+      InvalidClientSecretsFileFormatError: If the client secrets file does not
+        contain the required data or the data is malformed.
+
+    Returns:
+      Credentials
+    """
+        cs_data = fileutils.read_file(client_secrets_file,
+                                      continue_on_error=True,
+                                      display_errors=False)
+        if not cs_data:
+            raise InvalidClientSecretsFileError(
+                f'File {client_secrets_file} could not be opened')
+        try:
+            cs_json = json.loads(cs_data)
+            client_id = cs_json['installed']['client_id']
+            # Chop off .apps.googleusercontent.com suffix as it's not needed
+            # and we need to keep things short for the Auth URL.
+            client_id = re.sub(r'\.apps\.googleusercontent\.com$', '',
+                               client_id)
+            client_secret = cs_json['installed']['client_secret']
+        except (ValueError, IndexError, KeyError):
+            raise InvalidClientSecretsFileFormatError(
+                f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
+            )
+
+        return cls.from_client_secrets(client_id,
+                                       client_secret,
+                                       scopes,
+                                       access_type=access_type,
+                                       login_hint=login_hint,
+                                       filename=credentials_file,
+                                       use_console_flow=use_console_flow)
+
+    def _fetch_id_token_data(self):
+        """Fetches verification details from Google for the OAuth2.0 token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Raises:
+      CredentialsError: If no id_token is present.
+    """
+        if not self.id_token:
+            raise CredentialsError(
+                'Failed to fetch token data. No id_token present.')
+
+        request = transport.create_request()
+        if self.expired:
+            # The id_token needs to be unexpired, in order to request data about it.
+            self.refresh(request)
+
+        self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
+            self.id_token, request)
+
+    def get_token_value(self, field):
+        """Retrieves data from the OAuth ID token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Args:
+      field: The name of the key/field to fetch
+
+    Returns:
+      The value associated with the given key or 'Unknown' if the key data can
+      not be found in the access token data.
+    """
+        if not self._id_token_data:
+            self._fetch_id_token_data()
+        # Maintain legacy GAM behavior here to return "Unknown" if the field is
+        # otherwise unpopulated.
+        return self._id_token_data.get(field, 'Unknown')
+
+    def to_json(self, strip=None):
+        """Creates a JSON representation of a Credentials.
+
+    Args:
+        strip: Sequence[str], Optional list of members to exclude from the
+          generated JSON.
+
+    Returns:
+        str: A JSON representation of this instance, suitable to pass to
+             from_json().
+    """
+        expiry = self.expiry.strftime(
+            Credentials.DATETIME_FORMAT) if self.expiry else None
+        prep = {
+            'token': self.token,
+            'refresh_token': self.refresh_token,
+            'token_uri': self.token_uri,
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'id_token': self.id_token,
+            # Google auth doesn't currently give us scopes back on refresh.
+            # 'scopes': sorted(self.scopes),
+            'token_expiry': expiry,
+            'decoded_id_token': self._id_token_data,
+        }
+
+        # Remove empty entries
+        prep = {k: v for k, v in prep.items() if v is not None}
+
+        # Remove entries that explicitly need to be removed
+        if strip is not None:
+            prep = {k: v for k, v in prep.items() if k not in strip}
+
+        return json.dumps(prep, indent=2, sort_keys=True)
+
+    def refresh(self, request=None):
+        """Refreshes the credential's access token.
+
+    Args:
+      request: google.auth.transport.Request, The object used to make HTTP
+        requests. If not provided, a default request will be used.
+
+    Raises:
+      google.auth.exceptions.RefreshError: If the credentials could not be
+      refreshed.
+    """
+        with self._lock:
+            if request is None:
+                request = transport.create_request()
+            self._locked_refresh(request)
+            # Save the new tokens back to disk, if these credentials are disk-backed.
+            if self._filename:
+                self._locked_write()
+
+    def _locked_refresh(self, request):
+        """Refreshes the credential's access token while the file lock is held."""
+        assert self._lock.is_locked
+        super(Credentials, self).refresh(request)
+
+    def write(self):
+        """Writes credentials to disk."""
+        with self._lock:
+            self._locked_write()
+
+    def _locked_write(self):
+        """Writes credentials to disk while the file lock is held."""
+        assert self._lock.is_locked
+        if not self.filename:
+            # If no filename was provided to the constructor, these credentials cannot
+            # be saved to disk.
+            raise CredentialsError(
+                'The credentials have no associated filename and cannot be saved '
+                'to disk.')
+        fileutils.write_file(self._filename, self.to_json())
+
+    def delete(self):
+        """Deletes all files on disk related to these credentials."""
+        with self._lock:
+            # Only attempt to remove the file if the lock we're using is a FileLock.
+            if isinstance(self._lock, FileLock):
+                os.remove(self._filename)
+                if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
+                    os.remove(self._lock.lock_file)
+
+    _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+
+    def revoke(self, http=None):
+        """Revokes this credential's access token with the server.
+
+    Args:
+      http: httplib2.Http compatible object for use as a transport. If no http
+        is provided, a default will be used.
+    """
+        with self._lock:
+            if http is None:
+                http = transport.create_http()
+            params = urlencode({'token': self.refresh_token})
+            revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
+            http.request(revoke_uri, 'GET')
+
+
+class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
+    """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
+
+    URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+    def authorization_url(self, http=None, **kwargs):
+        """Gets a shortened authorization URL."""
+        long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
+        short_url = utils.shorten_url(long_url)
+        return short_url, state
+
+
+class _FileLikeThreadLock(object):
+    """A threading.lock which has the same interface as filelock.Filelock."""
+
+    def __init__(self):
+        """A shell object that holds a threading.Lock.
+
+    Since we cannot inherit from built-in classes such as threading.Lock, we
+    just use a shell object and maintain a lock inside of it.
+    """
+        self._lock = threading.Lock()
+
+    def __enter__(self, *args, **kwargs):
+        return self._lock.__enter__(*args, **kwargs)
+
+    def __exit__(self, *args, **kwargs):
+        return self._lock.__exit__(*args, **kwargs)
+
+    def acquire(self, **kwargs):
+        return self._lock.acquire(**kwargs)
+
+    def release(self):
+        return self._lock.release()
+
+    @property
+    def is_locked(self):
+        return self._lock.locked()
+
+    @property
+    def lock_file(self):
+        return None

src/gam/auth/oauth_test.py

@@ -0,0 +1,697 @@
+"""Tests for oauth."""
+
+import datetime
+import json
+import os
+import platform
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import google.oauth2.credentials
+
+from gam.auth import oauth
+
+
+class CredentialsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_token = 'fake_token'
+        self.fake_refresh_token = 'fake_refresh_token'
+        self.fake_id_token = 'fake_id_token'
+        self.fake_token_uri = 'https://fake.token.uri'
+        self.fake_client_id = 'fake_client_id'
+        self.fake_client_secret = 'fake_client_secret'
+        self.fake_scopes = [
+            'fake_api.readonly',
+            'fake_other_api.write',
+        ]
+        self.fake_quota_project_id = 'fake_quota_project_id'
+        self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
+        self.fake_filename = 'fake_filename'
+        self.fake_token_data = {
+            'field': 'value',
+            'another-field': 'another-value',
+        }
+        self.info_with_only_required_fields = {
+            'refresh_token': self.fake_refresh_token,
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+        }
+        super(CredentialsTest, self).setUp()
+
+    def tearDown(self):
+        # Remove any credential files that may have been created.
+        if os.path.exists(self.fake_filename):
+            os.remove(self.fake_filename)
+        if os.path.exists('%s.lock' % self.fake_filename):
+            os.remove('%s.lock' % self.fake_filename)
+        super(CredentialsTest, self).tearDown()
+
+    def test_from_authorized_user_info_only_required_info(self):
+        creds = oauth.Credentials.from_authorized_user_info(
+            self.info_with_only_required_fields)
+        self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+        self.assertEqual(self.fake_client_id, creds.client_id)
+        self.assertEqual(self.fake_client_secret, creds.client_secret)
+        self.assertIsNone(creds.id_token)
+        self.assertIsNone(creds.expiry)
+        self.assertIsNone(creds.filename)
+
+    def test_from_authorized_user_info_all_info_provided(self):
+        info = {
+            'token':
+                self.fake_token,
+            'refresh_token':
+                self.fake_refresh_token,
+            'id_token':
+                self.fake_id_token,
+            'token_uri':
+                self.fake_token_uri,
+            'client_id':
+                self.fake_client_id,
+            'client_secret':
+                self.fake_client_secret,
+            'token_expiry':
+                self.fake_token_expiry.strftime(
+                    oauth.Credentials.DATETIME_FORMAT),
+            'id_token_data':
+                self.fake_token_data,
+        }
+        creds = oauth.Credentials.from_authorized_user_info(info)
+        self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+        self.assertEqual(self.fake_client_id, creds.client_id)
+        self.assertEqual(self.fake_client_secret, creds.client_secret)
+        self.assertEqual(self.fake_id_token, creds.id_token)
+        self.assertEqual(self.fake_token_uri, creds.token_uri)
+        self.assertEqual(self.fake_token_expiry, creds.expiry)
+        self.assertIsNone(creds.filename)
+
+    def test_from_authorized_user_info_missing_required_info(self):
+        info_with_missing_fields = {'token': self.fake_token}
+        with self.assertRaises(ValueError):
+            oauth.Credentials.from_authorized_user_info(
+                info_with_missing_fields)
+
+    def test_from_authorized_user_info_no_expiry_in_info(self):
+        info_with_no_token_expiry = self.info_with_only_required_fields.copy()
+        self.assertIsNone(info_with_no_token_expiry.get('expiry'))
+        creds = oauth.Credentials.from_authorized_user_info(
+            info_with_no_token_expiry)
+        self.assertIsNone(creds.expiry)
+
+    def test_init_saves_filename(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+    def test_init_loads_decoded_id_token_data(self, mock_verify_token):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=self.fake_token_data)
+        self.assertEqual(self.fake_token_data.get('field'),
+                         creds.get_token_value('field'))
+        # Verify the fetching method was not called, since the token
+        # data was supposed to be loaded from the passed in info.
+        self.assertEqual(mock_verify_token.call_count, 0)
+
+    def test_credentials_uses_file_lock_when_filename_provided(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        self.assertIsInstance(creds._lock, oauth.FileLock)
+        self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
+
+    def test_credentials_uses_thread_lock_when_filename_not_provided(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=None)
+        self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
+        self.assertIsNone(creds.filename)
+
+    def test_from_oauth2credentials(self):
+        google_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        creds = oauth.Credentials.from_google_oauth2_credentials(
+            google_creds, filename=self.fake_filename)
+        self.assertEqual(google_creds.token, creds.token)
+        self.assertEqual(google_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(google_creds.client_id, creds.client_id)
+        self.assertEqual(google_creds.client_secret, creds.client_secret)
+        self.assertEqual(google_creds.id_token, creds.id_token)
+        self.assertEqual(google_creds.expiry, creds.expiry)
+        self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+        self.assertIn('could not be opened', str(e.exception))
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_no_serialized_data_in_file_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({})
+        with self.assertRaises(oauth.EmptyCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_missing_any_token_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({
+            # This data is missing a token key/value pair
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+        })
+        with self.assertRaises(oauth.InvalidCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_credentials_file_missing_required_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({
+            # This data is missing a client_secret key/value pair
+            'client_id': self.fake_client_id,
+            'refresh_token': self.fake_refresh_token,
+        })
+        with self.assertRaises(oauth.InvalidCredentialsFileError):
+            oauth.Credentials.from_credentials_file(self.fake_filename)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_console_flow(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                      self.fake_client_secret,
+                                                      self.fake_scopes,
+                                                      use_console_flow=True)
+        self.assertTrue(mock_flow.return_value.run_console.called)
+        self.assertFalse(mock_flow.return_value.run_local_server.called)
+        self.assertEqual(flow_creds.token, creds.token)
+        self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(flow_creds.client_id, creds.client_id)
+        self.assertEqual(flow_creds.client_secret, creds.client_secret)
+        self.assertEqual(flow_creds.id_token, creds.id_token)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_local_server_flow(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                      self.fake_client_secret,
+                                                      self.fake_scopes,
+                                                      use_console_flow=False)
+        self.assertFalse(mock_flow.return_value.run_console.called)
+        self.assertTrue(mock_flow.return_value.run_local_server.called)
+        self.assertEqual(flow_creds.token, creds.token)
+        self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+        self.assertEqual(flow_creds.client_id, creds.client_id)
+        self.assertEqual(flow_creds.client_secret, creds.client_secret)
+        self.assertEqual(flow_creds.id_token, creds.id_token)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_uses_login_hint(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                              self.fake_client_secret,
+                                              self.fake_scopes,
+                                              login_hint='someone@domain.com')
+
+        run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
+        self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
+
+    def test_from_client_secrets_uses_shortened_url_flow(self):
+        with patch.object(oauth._ShortURLFlow,
+                          'from_client_config') as mock_flow:
+            flow_creds = google.oauth2.credentials.Credentials(
+                token=self.fake_token,
+                refresh_token=self.fake_refresh_token,
+                client_id=self.fake_client_id,
+                client_secret=self.fake_client_secret,
+                id_token=self.fake_id_token)
+            mock_flow.return_value.credentials = flow_creds
+            oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                                  self.fake_client_secret,
+                                                  self.fake_scopes)
+        self.assertTrue(mock_flow.called)
+
+    @patch.object(oauth._ShortURLFlow, 'from_client_config')
+    def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
+        flow_creds = google.oauth2.credentials.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            id_token=self.fake_id_token)
+        mock_flow.return_value.credentials = flow_creds
+
+        creds = oauth.Credentials.from_client_secrets(
+            self.fake_client_id,
+            self.fake_client_secret,
+            self.fake_scopes,
+            filename=self.fake_filename)
+        self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+    def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(
+        self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        with self.assertRaises(oauth.InvalidClientSecretsFileError):
+            oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                       self.fake_scopes)
+
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_client_secrets_file_missing_required_json_raises_error(
+        self, mock_read_file):
+        mock_read_file.return_value = json.dumps({})
+        with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
+            oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                       self.fake_scopes)
+        self.assertIn('Could not extract Client ID or Client Secret',
+                      str(e.exception))
+
+    @patch.object(oauth.Credentials, 'from_client_secrets')
+    @patch.object(oauth.fileutils, 'read_file')
+    def test_from_client_secrets_file_strips_domain_from_client_id(
+        self, mock_read_file, mock_creds_from_client_secrets):
+        mock_read_file.return_value = json.dumps({
+            'installed': {
+                'client_id':
+                    self.fake_client_id + '.apps.googleusercontent.com',
+                'client_secret':
+                    self.fake_client_secret,
+            }
+        })
+
+        oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                   self.fake_scopes)
+        self.assertEqual(self.fake_client_id,
+                         mock_creds_from_client_secrets.call_args[0][0])
+
+    def test_get_token_value_known_token_field(self):
+        token_data = {'known-field': 'known-value'}
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token_data=token_data)
+        self.assertEqual('known-value', creds.get_token_value('known-field'))
+
+    def test_get_token_value_unknown_field_returns_unknown(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  id_token_data=self.fake_token_data)
+        self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
+
+    @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+    def test_get_token_value_credentials_expired(self,
+                                                 mock_verify_oauth2_token):
+        mock_verify_oauth2_token.return_value = {
+            'fetched-field': 'fetched-value'
+        }
+        time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
+            minutes=5)
+        creds = oauth.Credentials(token=self.fake_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  expiry=time_earlier_than_now,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=None)
+        self.assertTrue(creds.expired)
+        creds.refresh = MagicMock()
+
+        token_value = creds.get_token_value('fetched-field')
+
+        self.assertEqual('fetched-value', token_value)
+        self.assertTrue(creds.refresh.called)
+
+    def test_to_json_contains_all_required_fields(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  refresh_token=self.fake_refresh_token,
+                                  id_token=self.fake_id_token,
+                                  id_token_data=self.fake_token_data,
+                                  token_uri=self.fake_token_uri,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  scopes=self.fake_scopes,
+                                  quota_project_id=self.fake_quota_project_id,
+                                  expiry=self.fake_token_expiry)
+        json_string = creds.to_json()
+        json_data = json.loads(json_string)
+        keys = json_data.keys()
+        self.assertIn('token', keys)
+        self.assertEqual(self.fake_token, json_data['token'])
+        self.assertIn('refresh_token', keys)
+        self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
+        self.assertIn('id_token', keys)
+        self.assertEqual(self.fake_id_token, json_data['id_token'])
+        self.assertIn('token_uri', keys)
+        self.assertEqual(self.fake_token_uri, json_data['token_uri'])
+        self.assertIn('client_id', keys)
+        self.assertEqual(self.fake_client_id, json_data['client_id'])
+        self.assertIn('client_secret', keys)
+        self.assertEqual(self.fake_client_secret, json_data['client_secret'])
+        self.assertNotIn('scopes', keys)  # Scopes are not currently saved
+        self.assertIn('token_expiry', keys)
+        self.assertEqual(
+            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+            json_data['token_expiry'])
+        self.assertIn('decoded_id_token', keys)
+        self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
+
+    def test_credentials_to_json_and_back(self):
+        original_creds = oauth.Credentials(
+            token=self.fake_token,
+            refresh_token=self.fake_refresh_token,
+            id_token=self.fake_id_token,
+            id_token_data=self.fake_token_data,
+            token_uri=self.fake_token_uri,
+            client_id=self.fake_client_id,
+            client_secret=self.fake_client_secret,
+            scopes=self.fake_scopes,
+            quota_project_id=self.fake_quota_project_id,
+            expiry=self.fake_token_expiry)
+        pickled_creds = original_creds.to_json()
+        serialized_json = json.loads(pickled_creds)
+        unpickled_creds = oauth.Credentials.from_authorized_user_info(
+            serialized_json)
+        self.assertEqual(original_creds.token, unpickled_creds.token)
+        self.assertEqual(original_creds.refresh_token,
+                         unpickled_creds.refresh_token)
+        self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
+        self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
+        self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
+        self.assertEqual(original_creds.client_secret,
+                         unpickled_creds.client_secret)
+        self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    def test_refresh_calls_super_refresh(self, mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        request = MagicMock()
+
+        creds.refresh(request)
+        self.assertTrue(mock_super_refresh.called)
+        self.assertEqual(request, mock_super_refresh.call_args[0][0])
+
+    def test_refresh_locks_resource_during_refresh(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        lock = creds._lock
+
+        def check_lock_is_locked(*unused_args, **unused_kwargs):
+            self.assertTrue(lock.is_locked)
+
+        # We need to mock the superclass refresh so it doesn't actually try to
+        # refresh our fake token.
+        # At the same time, we'll make sure the lock is held during the refresh.
+        with patch.object(oauth.google.oauth2.credentials.Credentials,
+                          'refresh') as mock_refresh:
+            mock_refresh.side_effect = check_lock_is_locked
+            creds.refresh(request=MagicMock())
+
+        # Make sure our side effect was actually performed.
+        self.assertTrue(mock_refresh.called)
+        # The lock should be released after refresh
+        self.assertFalse(lock.is_locked)
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    @patch.object(oauth.fileutils, 'write_file')
+    def test_refresh_writes_new_credentials_to_disk_after_refresh(
+        self, mock_write_file, mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+
+        def update_access_token(unused_request):
+            creds.token = 'refreshed_access_token'
+
+        mock_super_refresh.side_effect = update_access_token
+
+        self.assertIsNone(creds.token)
+        creds.refresh(request=MagicMock())
+        self.assertEqual('refreshed_access_token', creds.token,
+                         'Access token was not refreshed')
+        text_written_to_file = mock_write_file.call_args[0][1]
+        self.assertIsNotNone(text_written_to_file,
+                             'Nothing was written to file')
+        saved_json = json.loads(text_written_to_file)
+        self.assertEqual('refreshed_access_token', saved_json['token'],
+                         'Refreshed access token was not saved to disk')
+
+    def test_write_writes_credentials_to_disk(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+
+        self.assertFalse(os.path.exists(self.fake_filename))
+        creds.write()
+        self.assertTrue(os.path.exists(self.fake_filename))
+
+    def test_write_raises_error_when_no_credentials_file_is_set(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+
+        self.assertIsNone(creds.filename)
+        with self.assertRaises(oauth.CredentialsError):
+            creds.write()
+
+    @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+    @patch.object(oauth.fileutils, 'write_file')
+    def test_write_locks_resource_during_write(self, mock_write_file,
+                                               unused_mock_super_refresh):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        lock = creds._lock
+
+        def check_lock_is_locked(*unused_args, **unused_kwargs):
+            self.assertTrue(creds._lock.is_locked)
+
+        mock_write_file.side_effect = check_lock_is_locked
+
+        self.assertFalse(lock.is_locked)
+        creds.refresh(request=MagicMock())
+        self.assertFalse(lock.is_locked)
+        self.assertTrue(mock_write_file.called)
+
+    def test_delete_removes_credentials_file(self):
+        self.assertFalse(os.path.exists(self.fake_filename))
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        creds.write()
+        self.assertTrue(os.path.exists(self.fake_filename))
+        creds.delete()
+        self.assertFalse(os.path.exists(self.fake_filename))
+
+    @unittest.skipIf(
+        platform.system() == 'Windows',
+        reason=('On Windows, Filelock deletes the lock file each time the lock '
+                'is released. Delete does not remove it.'))
+    def test_delete_removes_lock_file(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret,
+                                  filename=self.fake_filename)
+        lock_file = '%s.lock' % creds.filename
+        creds.write()
+        self.assertTrue(os.path.exists(lock_file))
+        creds.delete()
+        self.assertFalse(os.path.exists(lock_file))
+
+    def test_delete_is_noop_when_not_using_filelock(self):
+        creds = oauth.Credentials(token=None,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        self.assertIsNone(creds.filename)
+        creds.delete()  # This should not raise an exception.
+
+    def test_revoke_requests_credential_revoke(self):
+        creds = oauth.Credentials(token=self.fake_token,
+                                  refresh_token=self.fake_refresh_token,
+                                  client_id=self.fake_client_id,
+                                  client_secret=self.fake_client_secret)
+        mock_http = MagicMock()
+
+        creds.revoke(http=mock_http)
+
+        uri = mock_http.request.call_args[0][0]
+        self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
+        params = uri[uri.index('?'):]
+        self.assertIn('token=%s' % creds.refresh_token, params)
+        self.assertEqual('GET', mock_http.request.call_args[0][1])
+
+
+class ShortUrlFlowTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_client_id = 'fake_client_id'
+        self.fake_client_secret = 'fake_client_secret'
+        self.fake_scopes = [
+            'fake_api.readonly',
+            'fake_other_api.write',
+        ]
+        self.fake_client_config = {
+            'installed': {
+                'client_id': self.fake_client_id,
+                'client_secret': self.fake_client_secret,
+                'redirect_uris': [
+                    'http://localhost', 'urn:ietf:wg:oauth:2.0:oob'
+                ],
+                'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+                'token_uri': 'https://oauth2.googleapis.com/token',
+            }
+        }
+        self.long_url = 'http://example.com/some/long/url'
+        self.short_url = 'http://ex.co/short'
+        super(ShortUrlFlowTest, self).setUp()
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = json.dumps({'short_url': self.short_url})
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.short_url, url)
+        self.assertEqual('fake_state', state)
+
+        # Verify request() was called with the expected arguments.
+        self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
+                         mock_http.request.call_args[0][0])
+        self.assertEqual('POST', mock_http.request.call_args[0][1])
+        self.assertIn(self.long_url, mock_http.request.call_args[0][2])
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_request_error(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_http.request.side_effect = Exception()
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 404  # Use a status that is not 200
+        content = json.dumps({'short_url': self.short_url})
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = None
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+    @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                  'authorization_url')
+    @unittest.skip('disable short url tests temporarily.')
+    def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
+        self, mock_super_auth_url):
+        url_flow = oauth._ShortURLFlow.from_client_config(
+            self.fake_client_config, scopes=self.fake_scopes)
+        mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+        mock_http = MagicMock()
+        mock_response = MagicMock()
+        mock_response.status = 200
+        content = json.dumps(
+            {})  # This json content contains no "short-url" key
+        mock_http.request.return_value = (mock_response, content)
+
+        url, state = url_flow.authorization_url(http=mock_http)
+        self.assertEqual(self.long_url, url)
+        self.assertEqual('fake_state', state)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/controlflow.py

@@ -3,81 +3,78 @@ import random
 import sys
 import time
 
-import display  # TODO: Change to relative import when gam is setup as a package
-from var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
-from var import MESSAGE_INVALID_JSON
+from gam import display
+from gam.var import MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS
+from gam.var import MESSAGE_INVALID_JSON
 
 
 def system_error_exit(return_code, message):
-  """Raises a system exit with the given return code and message.
+    """Raises a system exit with the given return code and message.
 
   Args:
     return_code: Int, the return code to yield when the system exits.
     message: An error message to print before the system exits.
   """
-  if message:
-    display.print_error(message)
-  sys.exit(return_code)
+    if message:
+        display.print_error(message)
+    sys.exit(return_code)
 
 
 def invalid_argument_exit(argument, command):
-  '''Indicate that the argument is not valid for the command.
+    """Indicate that the argument is not valid for the command.
 
   Args:
     argument: the invalid argument
     command: the base GAM command
-  '''
-  system_error_exit(
-      2,
-      f'{argument} is not a valid argument for "{command}"')
+  """
+    system_error_exit(2, f'{argument} is not a valid argument for "{command}"')
+
 
 def missing_argument_exit(argument, command):
-  '''Indicate that the argument is missing for the command.
+    """Indicate that the argument is missing for the command.
 
   Args:
     argument: the missingagrument
     command: the base GAM command
-  '''
-  system_error_exit(
-      2,
-      f'missing argument {argument} for "{command}"')
+  """
+    system_error_exit(2, f'missing argument {argument} for "{command}"')
+
 
 def expected_argument_exit(name, expected, argument):
-  '''Indicate that the argument does not have an expected value for the command.
+    """Indicate that the argument does not have an expected value for the command.
 
   Args:
     name: the field name
     expected: the expected values
     argument: the invalid argument
-  '''
-  system_error_exit(
-      2,
-      f'{name} must be one of {expected}; got {argument}')
+  """
+    system_error_exit(2, f'{name} must be one of {expected}; got {argument}')
+
 
 def csv_field_error_exit(field_name, field_names):
-  """Raises a system exit when a CSV field is malformed.
+    """Raises a system exit when a CSV field is malformed.
 
   Args:
     field_name: The CSV field name for which a header does not exist in the
       existing CSV headers.
     field_names: The known list of CSV headers.
   """
-  system_error_exit(
-      2,
-      MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
-                                                     ','.join(field_names)))
+    system_error_exit(
+        2,
+        MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS.format(field_name,
+                                                       ','.join(field_names)))
 
 
 def invalid_json_exit(file_name):
-  """Raises a sysyem exit when invalid JSON content is encountered."""
-  system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
+    """Raises a sysyem exit when invalid JSON content is encountered."""
+    system_error_exit(17, MESSAGE_INVALID_JSON.format(file_name))
 
 
 def wait_on_failure(current_attempt_num,
                     total_num_retries,
                     error_message,
                     error_print_threshold=3):
-  """Executes an exponential backoff-style system sleep.
+    """Executes an exponential backoff-style system sleep.
 
   Args:
     current_attempt_num: Int, the current number of retries.
@@ -89,11 +86,11 @@ def wait_on_failure(current_attempt_num,
       error messages suppressed. Any current_attempt_num greater than
       error_print_threshold will print the prescribed error.
   """
-  wait_on_fail = min(2**current_attempt_num,
-                     60) + float(random.randint(1, 1000)) / 1000
-  if current_attempt_num > error_print_threshold:
-    sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
-        f'{int(wait_on_fail)} seconds, Retry: '
-        f'{current_attempt_num}/{total_num_retries}\n'))
-    sys.stderr.flush()
-  time.sleep(wait_on_fail)
+    wait_on_fail = min(2**current_attempt_num,
+                       60) + float(random.randint(1, 1000)) / 1000
+    if current_attempt_num > error_print_threshold:
+        sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+                          f'{int(wait_on_fail)} seconds, Retry: '
+                          f'{current_attempt_num}/{total_num_retries}\n'))
+        sys.stderr.flush()
+    time.sleep(wait_on_fail)

src/gam/controlflow_test.py

@@ -0,0 +1,108 @@
+"""Tests for controlflow."""
+
+import unittest
+from unittest.mock import patch
+
+from gam import controlflow
+
+
+class ControlFlowTest(unittest.TestCase):
+
+    def test_system_error_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.system_error_exit(1, 'exit message')
+
+    def test_system_error_exit_raises_systemexit_with_return_code(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.system_error_exit(100, 'exit message')
+        self.assertEqual(context_manager.exception.code, 100)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_system_error_exit_prints_error_before_exiting(
+        self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.system_error_exit(100, 'exit message')
+        self.assertIn('exit message', mock_print_err.call_args[0][0])
+
+    def test_csv_field_error_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+
+    def test_csv_field_error_exit_exits_code_2(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+        self.assertEqual(context_manager.exception.code, 2)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_csv_field_error_exit_prints_error_details(self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.csv_field_error_exit('aField',
+                                             ['unusedField1', 'unusedField2'])
+        printed_message = mock_print_err.call_args[0][0]
+        self.assertIn('aField', printed_message)
+        self.assertIn('unusedField1', printed_message)
+        self.assertIn('unusedField2', printed_message)
+
+    def test_invalid_json_exit_raises_systemexit_error(self):
+        with self.assertRaises(SystemExit):
+            controlflow.invalid_json_exit('filename')
+
+    def test_invalid_json_exit_exit_exits_code_17(self):
+        with self.assertRaises(SystemExit) as context_manager:
+            controlflow.invalid_json_exit('filename')
+        self.assertEqual(context_manager.exception.code, 17)
+
+    @patch.object(controlflow.display, 'print_error')
+    def test_invalid_json_exit_prints_error_details(self, mock_print_err):
+        with self.assertRaises(SystemExit):
+            controlflow.invalid_json_exit('filename')
+        printed_message = mock_print_err.call_args[0][0]
+        self.assertIn('filename', printed_message)
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_waits_exponentially(self, mock_sleep):
+        controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
+        controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
+        controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
+
+        sleep_calls = mock_sleep.call_args_list
+        self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
+        self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
+        self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
+        total_attempts = 20
+        for attempt in range(1, total_attempts + 1):
+            controlflow.wait_on_failure(
+                attempt,
+                total_attempts,
+                'Attempt #%s' % attempt,
+                # Suppress messages while we make a lot of attempts.
+                error_print_threshold=total_attempts + 1)
+            # Wait time may be between 60 and 61 secs, due to rand addition.
+            self.assertLessEqual(mock_sleep.call_args[0][0], 61)
+
+    # Prevent the system from actually sleeping and thus slowing down the test.
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
+        message = 'An error message to display'
+        with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+            controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+        self.assertIn(message, mock_stderr_write.call_args[0][0])
+
+    @patch.object(controlflow.time, 'sleep')
+    def test_wait_on_failure_only_prints_after_threshold(
+        self, unused_mock_sleep):
+        total_attempts = 5
+        threshold = 3
+        with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+            for attempt in range(1, total_attempts + 1):
+                controlflow.wait_on_failure(attempt,
+                                            total_attempts,
+                                            'Attempt #%s' % attempt,
+                                            error_print_threshold=threshold)
+        self.assertEqual(total_attempts - threshold,
+                         mock_stderr_write.call_count)

src/gam/display.py

@@ -0,0 +1,297 @@
+"""Methods related to display of information to the user."""
+
+import csv
+import datetime
+import io
+import sys
+import webbrowser
+
+import dateutil
+import googleapiclient.http
+
+#TODO: get rid of these hacks
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+
+
+def current_count(i, count):
+    return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
+
+
+def current_count_nl(i, count):
+    return f' ({i}/{count})\n' if (
+        count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
+
+
+def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
+    fields = fieldsChoiceMap[fieldName.lower()]
+    if isinstance(fields, list):
+        fieldsList.extend(fields)
+    else:
+        fieldsList.append(fields)
+
+
+# Write a CSV file
+def add_titles_to_csv_file(addTitles, titles):
+    for title in addTitles:
+        if title not in titles:
+            titles.append(title)
+
+
+def add_row_titles_to_csv_file(row, csvRows, titles):
+    csvRows.append(row)
+    for title in row:
+        if title not in titles:
+            titles.append(title)
+
+
+# fieldName is command line argument
+# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
+#ARGUMENT_TO_PROPERTY_MAP = {
+#  u'admincreated': [u'adminCreated'],
+#  u'aliases': [u'aliases', u'nonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles,
+                          titles):
+    for ftList in fieldNameMap[fieldName]:
+        if ftList not in fieldsTitles:
+            fieldsList.append(ftList)
+            fieldsTitles[ftList] = ftList
+            add_titles_to_csv_file([ftList], titles)
+
+
+# fieldName is command line argument
+# fieldNameTitleMap maps fieldName to API field name and CSV file header
+#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
+#  u'admincreated': [u'adminCreated', u'Admin_Created'],
+#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList,
+                                fieldsTitles, titles):
+    ftList = fieldNameTitleMap[fieldName]
+    for i in range(0, len(ftList), 2):
+        if ftList[i] not in fieldsTitles:
+            fieldsList.append(ftList[i])
+            fieldsTitles[ftList[i]] = ftList[i + 1]
+            add_titles_to_csv_file([ftList[i + 1]], titles)
+
+
+def sort_csv_titles(firstTitle, titles):
+    restoreTitles = []
+    for title in firstTitle:
+        if title in titles:
+            titles.remove(title)
+            restoreTitles.append(title)
+    titles.sort()
+    for title in restoreTitles[::-1]:
+        titles.insert(0, title)
+
+
+def QuotedArgumentList(items):
+    return ' '.join([
+        item if item and (item.find(' ') == -1) and
+        (item.find(',') == -1) else '"' + item + '"' for item in items
+    ])
+
+
+def write_csv_file(csvRows, titles, list_type, todrive):
+
+    def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
+        if not rowDate or not isinstance(rowDate, str):
+            return False
+        try:
+            rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
+            if dateMode:
+                rowDate = datetime.datetime(rowTime.year, rowTime.month,
+                                            rowTime.day).isoformat() + 'Z'
+        except ValueError:
+            rowDate = NEVER_TIME
+        if op == '<':
+            return rowDate < filterDate
+        if op == '<=':
+            return rowDate <= filterDate
+        if op == '>':
+            return rowDate > filterDate
+        if op == '>=':
+            return rowDate >= filterDate
+        if op == '!=':
+            return rowDate != filterDate
+        return rowDate == filterDate
+
+    def rowCountFilterMatch(rowCount, op, filterCount):
+        if isinstance(rowCount, str):
+            if not rowCount.isdigit():
+                return False
+            rowCount = int(rowCount)
+        elif not isinstance(rowCount, int):
+            return False
+        if op == '<':
+            return rowCount < filterCount
+        if op == '<=':
+            return rowCount <= filterCount
+        if op == '>':
+            return rowCount > filterCount
+        if op == '>=':
+            return rowCount >= filterCount
+        if op == '!=':
+            return rowCount != filterCount
+        return rowCount == filterCount
+
+    def rowBooleanFilterMatch(rowBoolean, filterBoolean):
+        if not isinstance(rowBoolean, bool):
+            return False
+        return rowBoolean == filterBoolean
+
+    def headerFilterMatch(filters, title):
+        for filterStr in filters:
+            if filterStr.match(title):
+                return True
+        return False
+
+    if GC_Values[GC_CSV_ROW_FILTER]:
+        for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
+            if column not in titles:
+                sys.stderr.write(
+                    f'WARNING: Row filter column "{column}" is not in output columns\n'
+                )
+                continue
+            if filterVal[0] == 'regex':
+                csvRows = [
+                    row for row in csvRows
+                    if filterVal[1].search(str(row.get(column, '')))
+                ]
+            elif filterVal[0] == 'notregex':
+                csvRows = [
+                    row for row in csvRows
+                    if not filterVal[1].search(str(row.get(column, '')))
+                ]
+            elif filterVal[0] in ['date', 'time']:
+                csvRows = [
+                    row for row in csvRows if rowDateTimeFilterMatch(
+                        filterVal[0] == 'date', row.get(column, ''),
+                        filterVal[1], filterVal[2])
+                ]
+            elif filterVal[0] == 'count':
+                csvRows = [
+                    row for row in csvRows if rowCountFilterMatch(
+                        row.get(column, 0), filterVal[1], filterVal[2])
+                ]
+            else:  #boolean
+                csvRows = [
+                    row for row in csvRows if rowBooleanFilterMatch(
+                        row.get(column, False), filterVal[1])
+                ]
+    if GC_Values[GC_CSV_HEADER_FILTER] or GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+        if GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+            titles = [
+                t for t in titles if
+                not headerFilterMatch(GC_Values[GC_CSV_HEADER_DROP_FILTER], t)
+            ]
+        if GC_Values[GC_CSV_HEADER_FILTER]:
+            titles = [
+                t for t in titles
+                if headerFilterMatch(GC_Values[GC_CSV_HEADER_FILTER], t)
+            ]
+        if not titles:
+            controlflow.system_error_exit(
+                3,
+                'No columns selected with GAM_CSV_HEADER_FILTER and GAM_CSV_HEADER_DROP_FILTER\n'
+            )
+            return
+    csv.register_dialect('nixstdout', lineterminator='\n')
+    if todrive:
+        write_to = io.StringIO()
+    else:
+        write_to = sys.stdout
+    writer = csv.DictWriter(write_to,
+                            fieldnames=titles,
+                            dialect='nixstdout',
+                            extrasaction='ignore',
+                            quoting=csv.QUOTE_MINIMAL)
+    try:
+        writer.writerow(dict((item, item) for item in writer.fieldnames))
+        writer.writerows(csvRows)
+    except IOError as e:
+        controlflow.system_error_exit(6, e)
+    if todrive:
+        admin_email = gam._getValueFromOAuth('email')
+        _, drive = gam.buildDrive3GAPIObject(admin_email)
+        if not drive:
+            print(f'''\nGAM is not authorized to create Drive files. Please run:
+gam user {admin_email} check serviceaccount
+and follow recommend steps to authorize GAM for Drive access.''')
+            sys.exit(5)
+        result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
+        columns = len(titles)
+        rows = len(csvRows)
+        cell_count = rows * columns
+        data_size = len(write_to.getvalue())
+        max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
+        if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
+            print(
+                f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}'
+            )
+            mimeType = 'text/csv'
+        else:
+            mimeType = MIMETYPE_GA_SPREADSHEET
+        body = {
+            'description': QuotedArgumentList(sys.argv),
+            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
+            'mimeType': mimeType
+        }
+        result = gapi.call(drive.files(),
+                           'create',
+                           fields='webViewLink',
+                           body=body,
+                           media_body=googleapiclient.http.MediaInMemoryUpload(
+                               write_to.getvalue().encode(),
+                               mimetype='text/csv'))
+        file_url = result['webViewLink']
+        if GC_Values[GC_NO_BROWSER]:
+            msg_txt = f'Drive file uploaded to:\n {file_url}'
+            msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
+            gam.send_email(msg_subj, msg_txt)
+            print(msg_txt)
+        else:
+            webbrowser.open(file_url)
+
+
+def print_error(message):
+    """Prints a one-line error message to stderr in a standard format."""
+    sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
+
+
+def print_warning(message):
+    """Prints a one-line warning message to stderr in a standard format."""
+    sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
+
+
+def print_json(object_value, spacing=''):
+    """Prints Dict or Array to screen in clean human-readable format.."""
+    if isinstance(object_value, list):
+        if len(object_value) == 1 and isinstance(object_value[0],
+                                                 (str, int, bool)):
+            sys.stdout.write(f'{object_value[0]}\n')
+            return
+        if spacing:
+            sys.stdout.write('\n')
+        for i, a_value in enumerate(object_value):
+            if isinstance(a_value, (str, int, bool)):
+                sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
+            else:
+                sys.stdout.write(f' {spacing}{i+1}) ')
+                print_json(a_value, f' {spacing}')
+    elif isinstance(object_value, dict):
+        for key in ['kind', 'etag', 'etags']:
+            object_value.pop(key, None)
+        for another_object, another_value in object_value.items():
+            sys.stdout.write(f' {spacing}{another_object}: ')
+            print_json(another_value, f' {spacing}')
+    else:
+        sys.stdout.write(f'{object_value}\n')

src/gam/display_test.py

@@ -0,0 +1,59 @@
+"""Tests for display."""
+
+import unittest
+from unittest.mock import patch
+
+from gam import display
+from gam.var import ERROR_PREFIX
+from gam.var import WARNING_PREFIX
+
+
+class DisplayTest(unittest.TestCase):
+
+    def test_print_error_prints_to_stderr(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertIn(message, printed_message)
+
+    def test_print_error_prints_error_prefix(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertLess(
+            printed_message.find(ERROR_PREFIX), printed_message.find(message),
+            'The error prefix does not appear before the error message')
+
+    def test_print_error_ends_message_with_newline(self):
+        message = 'test error'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertRegex(printed_message, '\n$',
+                         'The error message does not end in a newline.')
+
+    def test_print_warning_prints_to_stderr(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertIn(message, printed_message)
+
+    def test_print_warning_prints_error_prefix(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertLess(
+            printed_message.find(WARNING_PREFIX), printed_message.find(message),
+            'The warning prefix does not appear before the error message')
+
+    def test_print_warning_ends_message_with_newline(self):
+        message = 'test warning'
+        with patch.object(display.sys.stderr, 'write') as mock_write:
+            display.print_error(message)
+        printed_message = mock_write.call_args[0][0]
+        self.assertRegex(printed_message, '\n$',
+                         'The warning message does not end in a newline.')

src/gam/fileutils.py

@@ -4,25 +4,27 @@ import io
 import os
 import sys
 
-import controlflow
-import display
-from var import GM_Globals
-from var import GM_SYS_ENCODING
-from var import UTF8_SIG
+from gam import controlflow
+from gam import display
+from gam.var import GM_Globals
+from gam.var import GM_SYS_ENCODING
+from gam.var import UTF8_SIG
 
 
 def _open_file(filename, mode, encoding=None, newline=None):
-  """Opens a file with no error handling."""
-  # Determine which encoding to use
-  if 'b' in mode:
-    encoding = None
-  elif not encoding:
-    encoding = GM_Globals[GM_SYS_ENCODING]
-  elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
-    encoding = UTF8_SIG
+    """Opens a file with no error handling."""
+    # Determine which encoding to use
+    if 'b' in mode:
+        encoding = None
+    elif not encoding:
+        encoding = GM_Globals[GM_SYS_ENCODING]
+    elif 'r' in mode and encoding.lower().replace('-', '') == 'utf8':
+        encoding = UTF8_SIG
 
-  return open(
-      os.path.expanduser(filename), mode, newline=newline, encoding=encoding)
+    return open(os.path.expanduser(filename),
+                mode,
+                newline=newline,
+                encoding=encoding)
 
 
 def open_file(filename,
@@ -30,7 +32,7 @@ def open_file(filename,
               encoding=None,
               newline=None,
               strip_utf_bom=False):
-  """Opens a file.
+    """Opens a file.
 
   Args:
     filename: String, the name of the file to open, or '-' to use stdin/stdout,
@@ -47,41 +49,42 @@ def open_file(filename,
   Returns:
     The opened file.
   """
-  try:
-    if filename == '-':
-      # Read from stdin, rather than a file
-      if 'r' in mode:
-        return io.StringIO(str(sys.stdin.read()))
-      return sys.stdout
+    try:
+        if filename == '-':
+            # Read from stdin, rather than a file
+            if 'r' in mode:
+                return io.StringIO(str(sys.stdin.read()))
+            return sys.stdout
 
-    # Open a file on disk
-    f = _open_file(filename, mode, newline=newline, encoding=encoding)
-    if strip_utf_bom:
-      utf_bom = u'\ufeff'
-      has_bom = False
+        # Open a file on disk
+        f = _open_file(filename, mode, newline=newline, encoding=encoding)
+        if strip_utf_bom:
+            utf_bom = u'\ufeff'
+            has_bom = False
 
-      if 'b' in mode:
-        has_bom = f.read(3).decode('UTF-8') == utf_bom
-      elif f.encoding and not f.encoding.lower().startswith('utf'):
-        # Convert UTF BOM into ISO-8859-1 via Bytes
-        utf8_bom_bytes = utf_bom.encode('UTF-8')
-        iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
-            'iso-8859-1')
-        has_bom = f.read(3).encode('iso-8859-1', 'replace') == iso_8859_1_bom
-      else:
-        has_bom = f.read(1) == utf_bom
+            if 'b' in mode:
+                has_bom = f.read(3).decode('UTF-8') == utf_bom
+            elif f.encoding and not f.encoding.lower().startswith('utf'):
+                # Convert UTF BOM into ISO-8859-1 via Bytes
+                utf8_bom_bytes = utf_bom.encode('UTF-8')
+                iso_8859_1_bom = utf8_bom_bytes.decode('iso-8859-1').encode(
+                    'iso-8859-1')
+                has_bom = f.read(3).encode('iso-8859-1',
+                                           'replace') == iso_8859_1_bom
+            else:
+                has_bom = f.read(1) == utf_bom
 
-      if not has_bom:
-        f.seek(0)
+            if not has_bom:
+                f.seek(0)
 
-    return f
+        return f
 
-  except IOError as e:
-    controlflow.system_error_exit(6, e)
+    except IOError as e:
+        controlflow.system_error_exit(6, e)
 
 
 def close_file(f, force_flush=False):
-  """Closes a file.
+    """Closes a file.
 
   Args:
     f: The file to close
@@ -92,15 +95,15 @@ def close_file(f, force_flush=False):
     Boolean, True if the file was successfully closed. False if an error
         was encountered while closing.
   """
-  if force_flush:
-    f.flush()
-    os.fsync(f.fileno())
-  try:
-    f.close()
-    return True
-  except IOError as e:
-    display.print_error(e)
-    return False
+    if force_flush:
+        f.flush()
+        os.fsync(f.fileno())
+    try:
+        f.close()
+        return True
+    except IOError as e:
+        display.print_error(e)
+        return False
 
 
 def read_file(filename,
@@ -109,7 +112,7 @@ def read_file(filename,
               newline=None,
               continue_on_error=False,
               display_errors=True):
-  """Reads a file from disk.
+    """Reads a file from disk.
 
   Args:
     filename: String, the path of the file to open from disk, or "-" to read
@@ -128,22 +131,23 @@ def read_file(filename,
     The contents of the file, or stdin if filename == "-". Returns None if
     an error is encountered and continue_on_errors is True.
   """
-  try:
-    if filename == '-':
-      # Read from stdin, rather than a file.
-      return str(sys.stdin.read())
+    try:
+        if filename == '-':
+            # Read from stdin, rather than a file.
+            return str(sys.stdin.read())
 
-    with _open_file(filename, mode, newline=newline, encoding=encoding) as f:
-      return f.read()
+        with _open_file(filename, mode, newline=newline,
+                        encoding=encoding) as f:
+            return f.read()
 
-  except IOError as e:
-    if continue_on_error:
-      if display_errors:
-        display.print_warning(e)
-      return None
-    controlflow.system_error_exit(6, e)
-  except (LookupError, UnicodeDecodeError, UnicodeError) as e:
-    controlflow.system_error_exit(2, str(e))
+    except IOError as e:
+        if continue_on_error:
+            if display_errors:
+                display.print_warning(e)
+            return None
+        controlflow.system_error_exit(6, e)
+    except (LookupError, UnicodeDecodeError, UnicodeError) as e:
+        controlflow.system_error_exit(2, str(e))
 
 
 def write_file(filename,
@@ -151,7 +155,7 @@ def write_file(filename,
                mode='w',
                continue_on_error=False,
                display_errors=True):
-  """Writes data to a file.
+    """Writes data to a file.
 
   Args:
     filename: String, the path of the file to write to disk.
@@ -165,15 +169,15 @@ def write_file(filename,
   Returns:
     Boolean, True if the write operation succeeded, or False if not.
   """
-  try:
-    with _open_file(filename, mode) as f:
-      f.write(data)
-    return True
+    try:
+        with _open_file(filename, mode) as f:
+            f.write(data)
+        return True
 
-  except IOError as e:
-    if continue_on_error:
-      if display_errors:
-        display.print_error(e)
-      return False
-    else:
-      controlflow.system_error_exit(6, e)
+    except IOError as e:
+        if continue_on_error:
+            if display_errors:
+                display.print_error(e)
+            return False
+        else:
+            controlflow.system_error_exit(6, e)

src/gam/fileutils_test.py

@@ -0,0 +1,244 @@
+"""Tests for fileutils."""
+
+import io
+import os
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import fileutils
+
+
+class FileutilsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.fake_path = '/some/path/to/file'
+        super(FileutilsTest, self).setUp()
+
+    @patch.object(fileutils.sys, 'stdin')
+    def test_open_file_stdin(self, mock_stdin):
+        mock_stdin.read.return_value = 'some stdin content'
+        f = fileutils.open_file('-', mode='r')
+        self.assertIsInstance(f, fileutils.io.StringIO)
+        self.assertEqual(f.getvalue(), mock_stdin.read.return_value)
+
+    def test_open_file_stdout(self):
+        f = fileutils.open_file('-', mode='w')
+        self.assertEqual(fileutils.sys.stdout, f)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_opens_correct_path(self, mock_open):
+        f = fileutils.open_file(self.fake_path)
+        self.assertEqual(self.fake_path, mock_open.call_args[0][0])
+        self.assertEqual(mock_open.return_value, f)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_expands_user_file_path(self, mock_open):
+        file_path = '~/some/path/containing/tilde/shortcut/to/home'
+        fileutils.open_file(file_path)
+        opened_path = mock_open.call_args[0][0]
+        home_path = os.environ.get('HOME')
+        self.assertIsNotNone(home_path)
+        self.assertIn(home_path, opened_path)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_opens_correct_mode(self, mock_open):
+        fileutils.open_file(self.fake_path)
+        self.assertEqual('r', mock_open.call_args[0][1])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_encoding_for_binary(self, mock_open):
+        fileutils.open_file(self.fake_path, mode='b')
+        self.assertIsNone(mock_open.call_args[1]['encoding'])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_default_system_encoding(self, mock_open):
+        fileutils.open_file(self.fake_path)
+        self.assertEqual(fileutils.GM_Globals[fileutils.GM_SYS_ENCODING],
+                         mock_open.call_args[1]['encoding'])
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_utf8_encoding_specified(self, mock_open):
+        fileutils.open_file(self.fake_path, encoding='UTF-8')
+        self.assertEqual(fileutils.UTF8_SIG, mock_open.call_args[1]['encoding'])
+
+    def test_open_file_strips_utf_bom_in_utf(self):
+        bom_prefixed_data = u'\ufefffoobar'
+        fake_file = io.StringIO(bom_prefixed_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            self.assertEqual('foobar', f.read())
+
+    def test_open_file_strips_utf_bom_in_non_utf(self):
+        bom_prefixed_data = b'\xef\xbb\xbffoobar'.decode('iso-8859-1')
+
+        # We need to trick the method under test into believing that a StringIO
+        # instance is a file with an encoding. Since StringIO does not usually have,
+        # an encoding, we'll mock it and add our own encoding, but send the other
+        # methods in use (read and seek) back to the real StringIO object.
+        real_stringio = io.StringIO(bom_prefixed_data)
+        mock_file = MagicMock(spec=io.StringIO)
+        mock_file.read.side_effect = real_stringio.read
+        mock_file.seek.side_effect = real_stringio.seek
+        mock_file.encoding = 'iso-8859-1'
+
+        mock_open = MagicMock(spec=open, return_value=mock_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            self.assertEqual('foobar', f.read())
+
+    def test_open_file_strips_utf_bom_in_binary(self):
+        bom_prefixed_data = u'\ufefffoobar'.encode('UTF-8')
+        fake_file = io.BytesIO(bom_prefixed_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path,
+                                    mode='rb',
+                                    strip_utf_bom=True)
+            self.assertEqual(b'foobar', f.read())
+
+    def test_open_file_strip_utf_bom_when_no_bom_in_data(self):
+        no_bom_data = 'This data has no BOM'
+        fake_file = io.StringIO(no_bom_data)
+        mock_open = MagicMock(spec=open, return_value=fake_file)
+
+        with patch.object(fileutils, 'open', mock_open):
+            f = fileutils.open_file(self.fake_path, strip_utf_bom=True)
+            # Since there was no opening BOM, we should be back at the beginning of
+            # the file.
+            self.assertEqual(fake_file.tell(), 0)
+            self.assertEqual(f.read(), no_bom_data)
+
+    @patch.object(fileutils, 'open', new_callable=unittest.mock.mock_open)
+    def test_open_file_exits_on_io_error(self, mock_open):
+        mock_open.side_effect = IOError('Fake IOError')
+        with self.assertRaises(SystemExit) as context:
+            fileutils.open_file(self.fake_path)
+        self.assertEqual(context.exception.code, 6)
+
+    def test_close_file_closes_file_successfully(self):
+        mock_file = MagicMock()
+        self.assertTrue(fileutils.close_file(mock_file))
+        self.assertEqual(mock_file.close.call_count, 1)
+
+    def test_close_file_with_error(self):
+        mock_file = MagicMock()
+        mock_file.close.side_effect = IOError()
+        self.assertFalse(fileutils.close_file(mock_file))
+        self.assertEqual(mock_file.close.call_count, 1)
+
+    @patch.object(fileutils.sys, 'stdin')
+    def test_read_file_from_stdin(self, mock_stdin):
+        mock_stdin.read.return_value = 'some stdin content'
+        self.assertEqual(fileutils.read_file('-'), mock_stdin.read.return_value)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_default_params(self, mock_open_file):
+        fake_content = 'some fake content'
+        mock_open_file.return_value.__enter__().read.return_value = fake_content
+        self.assertEqual(fileutils.read_file(self.fake_path), fake_content)
+        self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+        self.assertEqual(mock_open_file.call_args[0][1], 'r')
+        self.assertIsNone(mock_open_file.call_args[1]['newline'])
+
+    @patch.object(fileutils.display, 'print_warning')
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_continues_on_errors_without_displaying(
+        self, mock_open_file, mock_print_warning):
+        mock_open_file.side_effect = IOError()
+        contents = fileutils.read_file(self.fake_path,
+                                       continue_on_error=True,
+                                       display_errors=False)
+        self.assertIsNone(contents)
+        self.assertFalse(mock_print_warning.called)
+
+    @patch.object(fileutils.display, 'print_warning')
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_displays_errors(self, mock_open_file,
+                                       mock_print_warning):
+        mock_open_file.side_effect = IOError()
+        fileutils.read_file(self.fake_path,
+                            continue_on_error=True,
+                            display_errors=True)
+        self.assertTrue(mock_print_warning.called)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_6_when_continue_on_error_is_false(
+        self, mock_open_file):
+        mock_open_file.side_effect = IOError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path, continue_on_error=False)
+        self.assertEqual(context.exception.code, 6)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_lookuperror(self, mock_open_file):
+        mock_open_file.return_value.__enter__().read.side_effect = LookupError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_unicodeerror(self, mock_open_file):
+        mock_open_file.return_value.__enter__().read.side_effect = UnicodeError(
+        )
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_read_file_exits_code_2_on_unicodedecodeerror(self, mock_open_file):
+        fake_decode_error = UnicodeDecodeError('fake-encoding', b'fakebytes', 0,
+                                               1, 'testing only')
+        mock_open_file.return_value.__enter__(
+        ).read.side_effect = fake_decode_error
+        with self.assertRaises(SystemExit) as context:
+            fileutils.read_file(self.fake_path)
+        self.assertEqual(context.exception.code, 2)
+
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_writes_data_to_file(self, mock_open_file):
+        fake_data = 'some fake data'
+        fileutils.write_file(self.fake_path, fake_data)
+        self.assertEqual(mock_open_file.call_args[0][0], self.fake_path)
+        self.assertEqual(mock_open_file.call_args[0][1], 'w')
+
+        opened_file = mock_open_file.return_value.__enter__()
+        self.assertTrue(opened_file.write.called)
+        self.assertEqual(opened_file.write.call_args[0][0], fake_data)
+
+    @patch.object(fileutils.display, 'print_error')
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_continues_on_errors_without_displaying(
+        self, mock_open_file, mock_print_error):
+        mock_open_file.side_effect = IOError()
+        status = fileutils.write_file(self.fake_path,
+                                      'foo data',
+                                      continue_on_error=True,
+                                      display_errors=False)
+        self.assertFalse(status)
+        self.assertFalse(mock_print_error.called)
+
+    @patch.object(fileutils.display, 'print_error')
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_displays_errors(self, mock_open_file, mock_print_error):
+        mock_open_file.side_effect = IOError()
+        fileutils.write_file(self.fake_path,
+                             'foo data',
+                             continue_on_error=True,
+                             display_errors=True)
+        self.assertTrue(mock_print_error.called)
+
+    @patch.object(fileutils, '_open_file')
+    def test_write_file_exits_code_6_when_continue_on_error_is_false(
+        self, mock_open_file):
+        mock_open_file.side_effect = IOError()
+        with self.assertRaises(SystemExit) as context:
+            fileutils.write_file(self.fake_path,
+                                 'foo data',
+                                 continue_on_error=False)
+        self.assertEqual(context.exception.code, 6)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/__init__.py

@@ -0,0 +1,343 @@
+"""Methods related to execution of GAPI requests."""
+
+import sys
+
+import googleapiclient.errors
+import google.auth.exceptions
+import httplib2
+
+from gam import controlflow
+from gam import display
+from gam.gapi import errors
+from gam import transport
+from gam.var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+                     GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
+                     MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
+                     MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
+
+
+def call(service,
+         function,
+         silent_errors=False,
+         soft_errors=False,
+         throw_reasons=None,
+         retry_reasons=None,
+         **kwargs):
+    """Executes a single request on a Google service function.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    silent_errors: Bool, If True, error messages are suppressed when
+      encountered.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A response object for the corresponding Google API call.
+  """
+    if throw_reasons is None:
+        throw_reasons = []
+    if retry_reasons is None:
+        retry_reasons = []
+
+    method = getattr(service, function)
+    retries = 10
+    parameters = dict(
+        list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+    for n in range(1, retries + 1):
+        try:
+            return method(**parameters).execute()
+        except googleapiclient.errors.HttpError as e:
+            http_status, reason, message = errors.get_gapi_error_detail(
+                e,
+                soft_errors=soft_errors,
+                silent_errors=silent_errors,
+                retry_on_http_error=n < 3)
+            if http_status == -1:
+                # The error detail indicated that we should retry this request
+                # We'll refresh credentials and make another pass
+                service._http.credentials.refresh(transport.create_http())
+                continue
+            if http_status == 0:
+                return None
+
+            is_known_error_reason = reason in [
+                r.value for r in errors.ErrorReason
+            ]
+            if is_known_error_reason and errors.ErrorReason(
+                    reason) in throw_reasons:
+                if errors.ErrorReason(
+                        reason) in errors.ERROR_REASON_TO_EXCEPTION:
+                    raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(
+                        reason)](message)
+                raise e
+            if (n != retries) and (is_known_error_reason and errors.ErrorReason(
+                    reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
+                controlflow.wait_on_failure(n, retries, reason)
+                continue
+            if soft_errors:
+                display.print_error(
+                    f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}'
+                )
+                return None
+            controlflow.system_error_exit(
+                int(http_status), f'{http_status}: {message} - {reason}')
+        except google.auth.exceptions.RefreshError as e:
+            handle_oauth_token_error(
+                e, soft_errors or
+                errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+            if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
+                raise errors.GapiServiceNotAvailableError(str(e))
+            display.print_error(
+                f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
+            return None
+        except ValueError as e:
+            if hasattr(service._http,
+                       'cache') and service._http.cache is not None:
+                service._http.cache = None
+                continue
+            controlflow.system_error_exit(4, str(e))
+        except (httplib2.ServerNotFoundError, RuntimeError) as e:
+            if n != retries:
+                service._http.connections = {}
+                controlflow.wait_on_failure(n, retries, str(e))
+                continue
+            controlflow.system_error_exit(4, str(e))
+        except TypeError as e:
+            controlflow.system_error_exit(4, str(e))
+
+
+def get_items(service,
+              function,
+              items='items',
+              throw_reasons=None,
+              retry_reasons=None,
+              **kwargs):
+    """Gets a single page of items from a Google service function that is paged.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the service
+      method's response object.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    The list of items in the first page of a response.
+  """
+    results = call(service,
+                   function,
+                   throw_reasons=throw_reasons,
+                   retry_reasons=retry_reasons,
+                   **kwargs)
+    if results:
+        return results.get(items, [])
+    return []
+
+
+def _get_max_page_size_for_api_call(service, function, **kwargs):
+    """Gets the maximum number of results supported for a single API call.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of the service method to check for max page size.
+    **kwargs: Additional params that will be passed to the request method.
+
+  Returns:
+    Int, A value from discovery if it exists, otherwise value from
+        MAX_RESULTS_API_EXCEPTIONS, otherwise None
+  """
+    method = getattr(service, function)
+    api_id = method(**kwargs).methodId
+    for resource in service._rootDesc.get('resources', {}).values():
+        for a_method in resource.get('methods', {}).values():
+            if a_method.get('id') == api_id:
+                if not a_method.get('parameters') or a_method['parameters'].get(
+                        'pageSize'
+                ) or not a_method['parameters'].get('maxResults'):
+                    # Make sure API call supports maxResults. For now we don't care to
+                    # set pageSize since all known pageSize API calls have
+                    # default pageSize == max pageSize.
+                    return None
+                known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
+                max_results = a_method['parameters']['maxResults'].get(
+                    'maximum', known_api_max)
+                return {'maxResults': max_results}
+
+    return None
+
+
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+
+def got_total_items_msg(items, eol):
+    """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+    return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+
+def got_total_items_first_last_msg(items):
+    """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+    return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}' + '\n'
+
+
+def get_all_pages(service,
+                  function,
+                  items='items',
+                  page_message=None,
+                  message_attribute=None,
+                  soft_errors=False,
+                  throw_reasons=None,
+                  retry_reasons=None,
+                  **kwargs):
+    """Aggregates and returns all pages of a Google service function response.
+
+  All pages of items are aggregated and returned as a single list.
+
+  Args:
+    service: A Google service object for the desired API.
+    function: String, The name of a service request method to execute.
+    items: String, the name of the resulting "items" field within the method's
+      response object. The items in this field will be aggregated across all
+      pages and returned.
+    page_message: String, a message to be displayed to the user during paging.
+      Template strings allow for dynamic content to be inserted during paging.
+        Supported template strings:
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
+            pages.
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
+            display a unique property of the first item in the current page.
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
+            display a unique property of the last item in the current page.
+    message_attribute: String, the name of a signature field within a single
+      returned item which identifies that unique item. This field is used with
+      `page_message` to templatize a paging status message.
+    soft_errors: Bool, If True, writes non-fatal errors to stderr.
+    throw_reasons: A list of Google HTTP error reason strings indicating the
+      errors generated by this request should be re-thrown. All other HTTP
+      errors are consumed.
+    retry_reasons: A list of Google HTTP error reason strings indicating which
+      error should be retried, using exponential backoff techniques, when the
+      error reason is encountered.
+    **kwargs: Additional params to pass to the request method.
+
+  Returns:
+    A list of all items received from all paged responses.
+  """
+    if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
+        page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
+        if page_key:
+            kwargs.update(page_key)
+    all_items = []
+    page_token = None
+    total_items = 0
+    while True:
+        page = call(service,
+                    function,
+                    soft_errors=soft_errors,
+                    throw_reasons=throw_reasons,
+                    retry_reasons=retry_reasons,
+                    pageToken=page_token,
+                    **kwargs)
+        if page:
+            page_token = page.get('nextPageToken')
+            page_items = page.get(items, [])
+            num_page_items = len(page_items)
+            total_items += num_page_items
+            all_items.extend(page_items)
+        else:
+            page_token = None
+            num_page_items = 0
+
+        # Show a paging message to the user that indicates paging progress
+        if page_message:
+            show_message = page_message.replace(TOTAL_ITEMS_MARKER,
+                                                str(total_items))
+            if message_attribute:
+                first_item = page_items[0] if num_page_items > 0 else {}
+                last_item = page_items[-1] if num_page_items > 1 else first_item
+                show_message = show_message.replace(
+                    FIRST_ITEM_MARKER,
+                    str(first_item.get(message_attribute, '')))
+                show_message = show_message.replace(
+                    LAST_ITEM_MARKER, str(last_item.get(message_attribute, '')))
+            sys.stderr.write('\r')
+            sys.stderr.flush()
+            sys.stderr.write(show_message)
+
+        if not page_token:
+            # End the paging status message and return all items.
+            if page_message and (page_message[-1] != '\n'):
+                sys.stderr.write('\r\n')
+                sys.stderr.flush()
+            return all_items
+
+
+# TODO: Make this private once all execution related items that use this method
+# have been brought into this file
+def handle_oauth_token_error(e, soft_errors):
+    """On a token error, exits the application and writes a message to stderr.
+
+  Args:
+    e: google.auth.exceptions.RefreshError, The error to handle.
+    soft_errors: Boolean, if True, suppresses any applicable errors and instead
+      returns to the caller.
+  """
+    token_error = str(e).replace('.', '')
+    if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
+            'Invalid response'):
+        if soft_errors:
+            return
+        if not GM_Globals[GM_CURRENT_API_USER]:
+            display.print_error(
+                MESSAGE_API_ACCESS_DENIED.format(
+                    GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+                    ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+            controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
+        else:
+            controlflow.system_error_exit(
+                19,
+                MESSAGE_SERVICE_NOT_APPLICABLE.format(
+                    GM_Globals[GM_CURRENT_API_USER]))
+    controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
+
+
+def get_enum_values_minus_unspecified(values):
+    return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gam/gapi/__init___test.py

@@ -0,0 +1,519 @@
+"""Tests for gapi."""
+
+import json
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import gam.gapi as gapi
+from gam.gapi import errors
+
+
+def create_http_error(status, reason, message):
+    """Creates a HttpError object similar to most Google API Errors.
+
+  Args:
+    status: Int, the error's HTTP response status number.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    googleapiclient.errors.HttpError
+  """
+    response = {
+        'status': status,
+        'content-type': 'application/json',
+    }
+    content = {
+        'error': {
+            'code': status,
+            'errors': [{
+                'reason': str(reason),
+                'message': message,
+            }]
+        }
+    }
+    content_bytes = json.dumps(content).encode('UTF-8')
+    return gapi.googleapiclient.errors.HttpError(response, content_bytes)
+
+
+class GapiTest(unittest.TestCase):
+
+    def setUp(self):
+        SetGlobalVariables()
+        self.mock_service = MagicMock()
+        self.mock_method_name = 'mock_method'
+        self.mock_method = getattr(self.mock_service, self.mock_method_name)
+
+        self.simple_3_page_response = [
+            {
+                'items': [{
+                    'position': 'page1,item1'
+                }, {
+                    'position': 'page1,item2'
+                }, {
+                    'position': 'page1,item3'
+                }],
+                'nextPageToken': 'page2'
+            },
+            {
+                'items': [{
+                    'position': 'page2,item1'
+                }, {
+                    'position': 'page2,item2'
+                }, {
+                    'position': 'page2,item3'
+                }],
+                'nextPageToken': 'page3'
+            },
+            {
+                'items': [{
+                    'position': 'page3,item1'
+                }, {
+                    'position': 'page3,item2'
+                }, {
+                    'position': 'page3,item3'
+                }],
+            },
+        ]
+        self.empty_items_response = {'items': []}
+
+        super(GapiTest, self).setUp()
+
+    def test_call_returns_basic_200_response(self):
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, self.mock_method().execute.return_value)
+
+    def test_call_passes_target_method_params(self):
+        gapi.call(self.mock_service,
+                  self.mock_method_name,
+                  my_param_1=1,
+                  my_param_2=2)
+        self.assertEqual(self.mock_method.call_count, 1)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(method_kwargs.get('my_param_1'), 1)
+        self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+    @patch.object(gapi.errors, 'get_gapi_error_detail')
+    def test_call_retries_with_soft_errors(self, mock_error_detail):
+        mock_error_detail.return_value = (-1, 'aReason', 'some message')
+
+        # Make the request fail first, then return the proper response on the retry.
+        fake_http_error = create_http_error(403, 'aReason', 'unused message')
+        fake_200_response = MagicMock()
+        self.mock_method.return_value.execute.side_effect = [
+            fake_http_error, fake_200_response
+        ]
+
+        response = gapi.call(self.mock_service,
+                             self.mock_method_name,
+                             soft_errors=True)
+        self.assertEqual(response, fake_200_response)
+        self.assertEqual(self.mock_service._http.credentials.refresh.call_count,
+                         1)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+
+    def test_call_throws_for_provided_reason(self):
+        throw_reason = errors.ErrorReason.USER_NOT_FOUND
+        fake_http_error = create_http_error(404, throw_reason, 'forced throw')
+        self.mock_method.return_value.execute.side_effect = fake_http_error
+
+        gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
+        with self.assertRaises(gam_exception):
+            gapi.call(self.mock_service,
+                      self.mock_method_name,
+                      throw_reasons=[throw_reason])
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_request_for_default_retry_reasons(
+        self, mock_wait_on_failure):
+
+        # Test using one of the default retry reasons
+        default_throw_reason = errors.ErrorReason.BACKEND_ERROR
+        self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
+
+        fake_http_error = create_http_error(404, default_throw_reason,
+                                            'message')
+        fake_200_response = MagicMock()
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_http_error, fake_200_response
+        ]
+
+        response = gapi.call(self.mock_service,
+                             self.mock_method_name,
+                             retry_reasons=[])
+        self.assertEqual(response, fake_200_response)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+        # Make sure a backoff technique was used for retry.
+        self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_requests_for_provided_retry_reasons(
+        self, unused_mock_wait_on_failure):
+
+        retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
+        fake_retrieable_error1 = create_http_error(400, retry_reason1,
+                                                   'Forced Error 1')
+        retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
+        fake_retrieable_error2 = create_http_error(400, retry_reason2,
+                                                   'Forced Error 2')
+        non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
+        fake_non_retriable_error = create_http_error(
+            400, non_retriable_reason,
+            'This error should not cause the request to be retried')
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_retrieable_error1, fake_retrieable_error2,
+            fake_non_retriable_error
+        ]
+
+        with self.assertRaises(SystemExit):
+            # The third call should raise the SystemExit when non_retriable_error is
+            # raised.
+            gapi.call(self.mock_service,
+                      self.mock_method_name,
+                      retry_reasons=[retry_reason1, retry_reason2])
+
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
+
+    def test_call_exits_on_oauth_token_error(self):
+        # An error with any OAUTH2_TOKEN_ERROR
+        fake_token_error = gapi.google.auth.exceptions.RefreshError(
+            errors.OAUTH2_TOKEN_ERRORS[0])
+        self.mock_method.return_value.execute.side_effect = fake_token_error
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_exits_on_nonretriable_error(self):
+        error_reason = 'unknownReason'
+        fake_http_error = create_http_error(500, error_reason,
+                                            'Testing unretriable errors')
+        self.mock_method.return_value.execute.side_effect = fake_http_error
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_exits_on_request_valueerror(self):
+        self.mock_method.return_value.execute.side_effect = ValueError()
+
+        with self.assertRaises(SystemExit):
+            gapi.call(self.mock_service, self.mock_method_name)
+
+    def test_call_clears_bad_http_cache_on_request_failure(self):
+        self.mock_service._http.cache = 'something that is not None'
+        fake_200_response = MagicMock()
+        self.mock_method.return_value.execute.side_effect = [
+            ValueError(), fake_200_response
+        ]
+
+        self.assertIsNotNone(self.mock_service._http.cache)
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, fake_200_response)
+        # Assert the cache was cleared
+        self.assertIsNone(self.mock_service._http.cache)
+
+    # Prevent wait_on_failure from performing actual backoff unnecessarily, since
+    # we're not actually testing over a network connection
+    @patch.object(gapi.controlflow, 'wait_on_failure')
+    def test_call_retries_requests_with_backoff_on_servernotfounderror(
+        self, mock_wait_on_failure):
+        fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
+        fake_200_response = MagicMock()
+        # Fail once, then succeed on retry
+        self.mock_method.return_value.execute.side_effect = [
+            fake_servernotfounderror, fake_200_response
+        ]
+
+        http_connections = self.mock_service._http.connections
+        response = gapi.call(self.mock_service, self.mock_method_name)
+        self.assertEqual(response, fake_200_response)
+        # HTTP cached connections should be cleared on receiving this error
+        self.assertNotEqual(http_connections,
+                            self.mock_service._http.connections)
+        self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
+        # Make sure a backoff technique was used for retry.
+        self.assertEqual(mock_wait_on_failure.call_count, 1)
+
+    def test_get_items_calls_correct_service_function(self):
+        gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertTrue(self.mock_method.called)
+
+    def test_get_items_returns_one_page(self):
+        fake_response = {'items': [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertEqual(page, fake_response['items'])
+
+    def test_get_items_non_default_page_field_name(self):
+        field_name = 'things'
+        fake_response = {field_name: [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_items(self.mock_service,
+                              self.mock_method_name,
+                              items=field_name)
+        self.assertEqual(page, fake_response[field_name])
+
+    def test_get_items_passes_additional_kwargs_to_service(self):
+        gapi.get_items(self.mock_service,
+                       self.mock_method_name,
+                       my_param_1=1,
+                       my_param_2=2)
+        self.assertEqual(self.mock_method.call_count, 1)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(1, method_kwargs.get('my_param_1'))
+        self.assertEqual(2, method_kwargs.get('my_param_2'))
+
+    def test_get_items_returns_empty_list_when_no_items_returned(self):
+        non_items_response = {'noItemsInThisResponse': {}}
+        self.mock_method.return_value.execute.return_value = non_items_response
+        page = gapi.get_items(self.mock_service, self.mock_method_name)
+        self.assertIsInstance(page, list)
+        self.assertEqual(0, len(page))
+
+    def test_get_all_pages_returns_all_items(self):
+        page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
+        page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
+        page_3 = {'items': ['3-1', '3-2', '3-3']}
+        self.mock_method.return_value.execute.side_effect = [
+            page_1, page_2, page_3
+        ]
+        response_items = gapi.get_all_pages(self.mock_service,
+                                            self.mock_method_name)
+        self.assertListEqual(
+            response_items, page_1['items'] + page_2['items'] + page_3['items'])
+
+    def test_get_all_pages_includes_next_pagetoken_in_request(self):
+        page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
+        page_2 = {'items': ['2-1', '2-2', '2-3']}
+        self.mock_method.return_value.execute.side_effect = [page_1, page_2]
+
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           pageSize=100)
+        self.assertEqual(self.mock_method.call_count, 2)
+        call_2_kwargs = self.mock_method.call_args_list[1][1]
+        self.assertIn('pageToken', call_2_kwargs)
+        self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
+
+    def test_get_all_pages_uses_default_max_page_size(self):
+        sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
+        sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
+        self.mock_method.return_value.methodId = sample_api_id
+        self.mock_service._rootDesc = {
+            'resources': {
+                'someResource': {
+                    'methods': {
+                        'someMethod': {
+                            'id': sample_api_id,
+                            'parameters': {
+                                'maxResults': {
+                                    'maximum': sample_api_max_results
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+        gapi.get_all_pages(self.mock_service, self.mock_method_name)
+        request_method_kwargs = self.mock_method.call_args[1]
+        self.assertIn('maxResults', request_method_kwargs)
+        self.assertEqual(request_method_kwargs['maxResults'],
+                         gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
+
+    def test_get_all_pages_max_page_size_overrided(self):
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           pageSize=123456)
+        request_method_kwargs = self.mock_method.call_args[1]
+        self.assertIn('pageSize', request_method_kwargs)
+        self.assertEqual(123456, request_method_kwargs['pageSize'])
+
+    def test_get_all_pages_prints_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        self.assertIn(paging_message, messages_written)
+
+    def test_get_all_pages_prints_paging_message_inline(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+
+        # Make sure a return carriage was written between two pages
+        paging_message_call_positions = [
+            i for i, message in enumerate(messages_written)
+            if message == paging_message
+        ]
+        self.assertGreater(len(paging_message_call_positions), 1)
+        printed_between_page_messages = messages_written[
+            paging_message_call_positions[0]:paging_message_call_positions[1]]
+        self.assertIn('\r', printed_between_page_messages)
+
+    def test_get_all_pages_ends_paging_message_with_newline(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'A simple string displayed during paging'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        last_page_message_index = len(
+            messages_written) - messages_written[::-1].index(paging_message)
+        last_carriage_return_index = len(
+            messages_written) - messages_written[::-1].index('\r\n')
+        self.assertGreater(last_carriage_return_index, last_page_message_index)
+
+    def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'Total number of items discovered: %%total_items%%'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message)
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_item_count = len(self.simple_3_page_response[0]['items'])
+        page_1_message = paging_message.replace('%%total_items%%',
+                                                str(page_1_item_count))
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_item_count = len(self.simple_3_page_response[1]['items'])
+        page_2_message = paging_message.replace(
+            '%%total_items%%', str(page_1_item_count + page_2_item_count))
+        self.assertIn(page_2_message, messages_written)
+
+        page_3_item_count = len(self.simple_3_page_response[2]['items'])
+        page_3_message = paging_message.replace(
+            '%%total_items%%',
+            str(page_1_item_count + page_2_item_count + page_3_item_count))
+        self.assertIn(page_3_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%total_items', message)
+
+    def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'First item in page: %%first_item%%'
+
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message,
+                               message_attribute='position')
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_message = paging_message.replace(
+            '%%first_item%%',
+            self.simple_3_page_response[0]['items'][0]['position'])
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_message = paging_message.replace(
+            '%%first_item%%',
+            self.simple_3_page_response[1]['items'][0]['position'])
+        self.assertIn(page_2_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%first_item', message)
+
+    def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
+        self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
+
+        paging_message = 'Last item in page: %%last_item%%'
+        with patch.object(gapi.sys.stderr, 'write') as mock_write:
+            gapi.get_all_pages(self.mock_service,
+                               self.mock_method_name,
+                               page_message=paging_message,
+                               message_attribute='position')
+
+        messages_written = [
+            call_args[0][0] for call_args in mock_write.call_args_list
+        ]
+        page_1_message = paging_message.replace(
+            '%%last_item%%',
+            self.simple_3_page_response[0]['items'][-1]['position'])
+        self.assertIn(page_1_message, messages_written)
+
+        page_2_message = paging_message.replace(
+            '%%last_item%%',
+            self.simple_3_page_response[1]['items'][-1]['position'])
+        self.assertIn(page_2_message, messages_written)
+
+        # Assert that the template text is always replaced.
+        for message in messages_written:
+            self.assertNotIn('%%last_item', message)
+
+    def test_get_all_pages_prints_all_attributes_in_paging_message(self):
+        pass
+
+    def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
+        self.mock_method.return_value.execute.return_value = self.empty_items_response
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           my_param_1=1,
+                           my_param_2=2)
+        method_kwargs = self.mock_method.call_args[1]
+        self.assertEqual(method_kwargs.get('my_param_1'), 1)
+        self.assertEqual(method_kwargs.get('my_param_2'), 2)
+
+    @patch.object(gapi, 'call')
+    def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
+        throw_for = MagicMock()
+        retry_for = MagicMock()
+        mock_call.return_value = self.empty_items_response
+        gapi.get_all_pages(self.mock_service,
+                           self.mock_method_name,
+                           throw_reasons=throw_for,
+                           retry_reasons=retry_for)
+        method_kwargs = mock_call.call_args[1]
+        self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
+        self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
+
+    def test_get_all_pages_non_default_items_field_name(self):
+        field_name = 'things'
+        fake_response = {field_name: [{}, {}, {}]}
+        self.mock_method.return_value.execute.return_value = fake_response
+        page = gapi.get_all_pages(self.mock_service,
+                                  self.mock_method_name,
+                                  items=field_name)
+        self.assertEqual(page, fake_response[field_name])
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/calendar.py

@@ -3,29 +3,28 @@ import sys
 import uuid
 
 # TODO: get rid of these hacks
-import __main__
-from var import *
+import gam
+from gam.var import *
 
-import controlflow
-import display
-import fileutils
-import gapi
-import utils
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam import utils
 
 
 def normalizeCalendarId(calname, checkPrimary=False):
     if checkPrimary and calname.lower() == 'primary':
         return calname
     if not GC_Values[GC_DOMAIN]:
-        GC_Values[GC_DOMAIN] = __main__._getValueFromOAuth('hd')
-    return __main__.convertUIDtoEmailAddress(calname,
-                                             email_types=['user', 'resource'])
+        GC_Values[GC_DOMAIN] = gam._getValueFromOAuth('hd')
+    return gam.convertUIDtoEmailAddress(calname,
+                                        email_types=['user', 'resource'])
 
 
 def buildCalendarGAPIObject(calname):
     calendarId = normalizeCalendarId(calname)
-    return (calendarId, __main__.buildGAPIServiceObject('calendar',
-                                                        calendarId))
+    return (calendarId, gam.buildGAPIServiceObject('calendar', calendarId))
 
 
 def buildCalendarDataGAPIObject(calname):
@@ -36,11 +35,12 @@ def buildCalendarDataGAPIObject(calname):
     # so we need to access them as the admin.
     cal = None
     if not calname.endswith('.calendar.google.com'):
-        cal = __main__.buildGAPIServiceObject('calendar', calendarId, False)
+        cal = gam.buildGAPIServiceObject('calendar', calendarId, False)
     if cal is None:
-        _, cal = buildCalendarGAPIObject(__main__._getValueFromOAuth('email'))
+        _, cal = buildCalendarGAPIObject(gam._getValueFromOAuth('email'))
     return (calendarId, cal)
 
+
 def printShowACLs(csvFormat):
     calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
     if not cal:
@@ -54,10 +54,9 @@ def printShowACLs(csvFormat):
             i += 1
         else:
             action = ['showacl', 'printacl'][csvFormat]
-            message = f"gam calendar <email> {action}"
+            message = f'gam calendar <email> {action}'
             controlflow.invalid_argument_exit(sys.argv[i], message)
-    acls = gapi.get_all_pages(
-        cal.acl(), 'list', 'items', calendarId=calendarId)
+    acls = gapi.get_all_pages(cal.acl(), 'list', 'items', calendarId=calendarId)
     i = 0
     if csvFormat:
         titles = []
@@ -75,10 +74,11 @@ def printShowACLs(csvFormat):
         else:
             formatted_acl = formatACLRule(rule)
             current_count = display.current_count(i, count)
-            print(f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
+            print(
+                f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
     if csvFormat:
-        display.write_csv_file(
-            rows, titles, f'{calendarId} Calendar ACLs', toDrive)
+        display.write_csv_file(rows, titles, f'{calendarId} Calendar ACLs',
+                               toDrive)
 
 
 def _getCalendarACLScope(i, body):
@@ -87,8 +87,8 @@ def _getCalendarACLScope(i, body):
     body['scope']['type'] = myarg
     i += 1
     if myarg in ['user', 'group']:
-        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
-            sys.argv[i], noUid=True)
+        body['scope']['value'] = gam.normalizeEmailAddressOrUID(sys.argv[i],
+                                                                noUid=True)
         i += 1
     elif myarg == 'domain':
         if i < len(sys.argv) and \
@@ -99,8 +99,8 @@ def _getCalendarACLScope(i, body):
             body['scope']['value'] = GC_Values[GC_DOMAIN]
     elif myarg != 'default':
         body['scope']['type'] = 'user'
-        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
-            myarg, noUid=True)
+        body['scope']['value'] = gam.normalizeEmailAddressOrUID(myarg,
+                                                                noUid=True)
     return i
 
 
@@ -122,22 +122,26 @@ def addACL(function):
         return
     myarg = sys.argv[4].lower().replace('_', '')
     if myarg not in CALENDAR_ACL_ROLES_MAP:
-        controlflow.expected_argument_exit(
-            "Role", ", ".join(CALENDAR_ACL_ROLES_MAP), myarg)
+        controlflow.expected_argument_exit('Role',
+                                           ', '.join(CALENDAR_ACL_ROLES_MAP),
+                                           myarg)
     body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]}
     i = _getCalendarACLScope(5, body)
     sendNotifications = True
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'sendnotifications':
-            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            sendNotifications = gam.getBoolean(sys.argv[i + 1], myarg)
             i += 2
         else:
             controlflow.invalid_argument_exit(
-                sys.argv[i], f"gam calendar <email> {function.lower()}")
+                sys.argv[i], f'gam calendar <email> {function.lower()}')
     print(f'Calendar: {calendarId}, {function} ACL: {formatACLRule(body)}')
-    gapi.call(cal.acl(), 'insert', calendarId=calendarId,
-              body=body, sendNotifications=sendNotifications)
+    gapi.call(cal.acl(),
+              'insert',
+              calendarId=calendarId,
+              body=body,
+              sendNotifications=sendNotifications)
 
 
 def delACL():
@@ -152,8 +156,11 @@ def delACL():
         body = {'role': 'none'}
         _getCalendarACLScope(5, body)
         print(f'Calendar: {calendarId}, Delete ACL: {formatACLScope(body)}')
-        gapi.call(cal.acl(), 'insert', calendarId=calendarId,
-                  body=body, sendNotifications=False)
+        gapi.call(cal.acl(),
+                  'insert',
+                  calendarId=calendarId,
+                  body=body,
+                  sendNotifications=False)
 
 
 def wipeData():
@@ -176,7 +183,7 @@ def printEvents():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'query':
-            q = sys.argv[i+1]
+            q = sys.argv[i + 1]
             i += 2
         elif myarg == 'includedeleted':
             showDeleted = True
@@ -185,30 +192,34 @@ def printEvents():
             showHiddenInvitations = True
             i += 1
         elif myarg == 'after':
-            timeMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            timeMin = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'before':
-            timeMax = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            timeMax = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'timezone':
-            timeZone = sys.argv[i+1]
+            timeZone = sys.argv[i + 1]
             i += 2
         elif myarg == 'updated':
-            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'todrive':
             toDrive = True
             i += 1
         else:
             controlflow.invalid_argument_exit(
-                sys.argv[i], "gam calendar <email> printevents")
+                sys.argv[i], 'gam calendar <email> printevents')
     page_message = gapi.got_total_items_msg(f'Events for {calendarId}', '')
-    results = gapi.get_all_pages(cal.events(), 'list', 'items',
+    results = gapi.get_all_pages(cal.events(),
+                                 'list',
+                                 'items',
                                  page_message=page_message,
-                                 calendarId=calendarId, q=q,
+                                 calendarId=calendarId,
+                                 q=q,
                                  showDeleted=showDeleted,
                                  showHiddenInvitations=showHiddenInvitations,
-                                 timeMin=timeMin, timeMax=timeMax,
+                                 timeMin=timeMin,
+                                 timeMax=timeMax,
                                  timeZone=timeZone,
                                  updatedMin=updatedMin)
     for result in results:
@@ -237,17 +248,19 @@ def getSendUpdates(myarg, i, cal):
         sendUpdates = 'all'
         i += 1
     elif myarg == 'sendnotifications':
-        sendUpdates = 'all' if __main__.getBoolean(sys.argv[i+1], myarg) else 'none'
+        sendUpdates = 'all' if gam.getBoolean(sys.argv[i +
+                                                       1], myarg) else 'none'
         i += 2
     else:  # 'sendupdates':
         sendUpdatesMap = {}
         for val in cal._rootDesc['resources']['events']['methods']['delete'][
                 'parameters']['sendUpdates']['enum']:
             sendUpdatesMap[val.lower()] = val
-        sendUpdates = sendUpdatesMap.get(sys.argv[i+1].lower(), False)
+        sendUpdates = sendUpdatesMap.get(sys.argv[i + 1].lower(), False)
         if not sendUpdates:
-            controlflow.expected_argument_exit(
-                "sendupdates", ", ".join(sendUpdatesMap), sys.argv[i+1])
+            controlflow.expected_argument_exit('sendupdates',
+                                               ', '.join(sendUpdatesMap),
+                                               sys.argv[i + 1])
         i += 2
     return (sendUpdates, i)
 
@@ -265,7 +278,7 @@ def moveOrDeleteEvent(moveOrDelete):
         if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
             sendUpdates, i = getSendUpdates(myarg, i, cal)
         elif myarg in ['id', 'eventid']:
-            eventId = sys.argv[i+1]
+            eventId = sys.argv[i + 1]
             i += 2
         elif myarg in ['query', 'eventquery']:
             controlflow.system_error_exit(
@@ -276,15 +289,19 @@ def moveOrDeleteEvent(moveOrDelete):
             doit = True
             i += 1
         elif moveOrDelete == 'move' and myarg == 'destination':
-            kwargs['destination'] = sys.argv[i+1]
+            kwargs['destination'] = sys.argv[i + 1]
             i += 2
         else:
             controlflow.invalid_argument_exit(
-                sys.argv[i], f"gam calendar <email> {moveOrDelete}event")
+                sys.argv[i], f'gam calendar <email> {moveOrDelete}event')
     if doit:
         print(f' going to {moveOrDelete} eventId {eventId}')
-        gapi.call(cal.events(), moveOrDelete, calendarId=calendarId,
-                  eventId=eventId, sendUpdates=sendUpdates, **kwargs)
+        gapi.call(cal.events(),
+                  moveOrDelete,
+                  calendarId=calendarId,
+                  eventId=eventId,
+                  sendUpdates=sendUpdates,
+                  **kwargs)
     else:
         print(
             f' would {moveOrDelete} eventId {eventId}. Add doit to command ' \
@@ -296,8 +313,10 @@ def infoEvent():
     if not cal:
         return
     eventId = sys.argv[4]
-    result = gapi.call(cal.events(), 'get',
-                       calendarId=calendarId, eventId=eventId)
+    result = gapi.call(cal.events(),
+                       'get',
+                       calendarId=calendarId,
+                       eventId=eventId)
     display.print_json(result)
 
 
@@ -316,25 +335,36 @@ def addOrUpdateEvent(action):
         kwargs = {'eventId': eventId}
         i = 5
         func = 'patch'
-        requires_full_update = ['attendee', 'optionalattendee',
-                                'removeattendee', 'replacedescription']
+        requires_full_update = [
+            'attendee', 'optionalattendee', 'removeattendee',
+            'replacedescription'
+        ]
         for arg in sys.argv[i:]:
             if arg.replace('_', '').lower() in requires_full_update:
                 func = 'update'
-                body = gapi.call(cal.events(), 'get',
-                                 calendarId=calendarId, eventId=eventId)
+                body = gapi.call(cal.events(),
+                                 'get',
+                                 calendarId=calendarId,
+                                 eventId=eventId)
                 break
     sendUpdates, body = getEventAttributes(i, calendarId, cal, body, action)
-    result = gapi.call(cal.events(), func, conferenceDataVersion=1,
-                       supportsAttachments=True, calendarId=calendarId,
-                       sendUpdates=sendUpdates, body=body, fields='id',
+    result = gapi.call(cal.events(),
+                       func,
+                       conferenceDataVersion=1,
+                       supportsAttachments=True,
+                       calendarId=calendarId,
+                       sendUpdates=sendUpdates,
+                       body=body,
+                       fields='id',
                        **kwargs)
     print(f'Event {result["id"]} {action} finished')
 
 
 def _remove_attendee(attendees, remove_email):
-    return [attendee for attendee in attendees
-            if not attendee['email'].lower() == remove_email]
+    return [
+        attendee for attendee in attendees
+        if not attendee['email'].lower() == remove_email
+    ]
 
 
 def getEventAttributes(i, calendarId, cal, body, action):
@@ -348,155 +378,165 @@ def getEventAttributes(i, calendarId, cal, body, action):
             sendUpdates, i = getSendUpdates(myarg, i, cal)
         elif myarg == 'attendee':
             body.setdefault('attendees', [])
-            body['attendees'].append({'email': sys.argv[i+1]})
+            body['attendees'].append({'email': sys.argv[i + 1]})
             i += 2
         elif myarg == 'removeattendee' and action == 'update':
-            remove_email = sys.argv[i+1].lower()
+            remove_email = sys.argv[i + 1].lower()
             if 'attendees' in body:
                 body['attendees'] = _remove_attendee(body['attendees'],
                                                      remove_email)
             i += 2
         elif myarg == 'optionalattendee':
             body.setdefault('attendees', [])
-            body['attendees'].append(
-                {'email': sys.argv[i+1], 'optional': True})
+            body['attendees'].append({
+                'email': sys.argv[i + 1],
+                'optional': True
+            })
             i += 2
         elif myarg == 'anyonecanaddself':
             body['anyoneCanAddSelf'] = True
             i += 1
         elif myarg == 'description':
-            body['description'] = sys.argv[i+1].replace('\\n', '\n')
+            body['description'] = sys.argv[i + 1].replace('\\n', '\n')
             i += 2
         elif myarg == 'replacedescription' and action == 'update':
-            search = sys.argv[i+1]
-            replace = sys.argv[i+2]
+            search = sys.argv[i + 1]
+            replace = sys.argv[i + 2]
             if 'description' in body:
-                body['description'] = re.sub(search, replace, body['description'])
+                body['description'] = re.sub(search, replace,
+                                             body['description'])
             i += 3
         elif myarg == 'start':
-            if sys.argv[i+1].lower() == 'allday':
-                body['start'] = {'date': utils.get_yyyymmdd(sys.argv[i+2])}
+            if sys.argv[i + 1].lower() == 'allday':
+                body['start'] = {'date': utils.get_yyyymmdd(sys.argv[i + 2])}
                 i += 3
             else:
-                start_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                start_time = utils.get_time_or_delta_from_now(sys.argv[i + 1])
                 body['start'] = {'dateTime': start_time}
                 i += 2
         elif myarg == 'end':
-            if sys.argv[i+1].lower() == 'allday':
-                body['end'] = {'date': utils.get_yyyymmdd(sys.argv[i+2])}
+            if sys.argv[i + 1].lower() == 'allday':
+                body['end'] = {'date': utils.get_yyyymmdd(sys.argv[i + 2])}
                 i += 3
             else:
-                end_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                end_time = utils.get_time_or_delta_from_now(sys.argv[i + 1])
                 body['end'] = {'dateTime': end_time}
                 i += 2
         elif myarg == 'guestscantinviteothers':
             body['guestsCanInviteOthers'] = False
             i += 1
         elif myarg == 'guestscaninviteothers':
-            body['guestsCanInviteTohters'] = __main__.getBoolean(
-                sys.argv[i+1], 'guestscaninviteothers')
+            body['guestsCanInviteTohters'] = gam.getBoolean(
+                sys.argv[i + 1], 'guestscaninviteothers')
             i += 2
         elif myarg == 'guestscantseeothers':
             body['guestsCanSeeOtherGuests'] = False
             i += 1
         elif myarg == 'guestscanseeothers':
-            body['guestsCanSeeOtherGuests'] = __main__.getBoolean(
-                sys.argv[i+1], 'guestscanseeothers')
+            body['guestsCanSeeOtherGuests'] = gam.getBoolean(
+                sys.argv[i + 1], 'guestscanseeothers')
             i += 2
         elif myarg == 'guestscanmodify':
-            body['guestsCanModify'] = __main__.getBoolean(
-                sys.argv[i+1], 'guestscanmodify')
+            body['guestsCanModify'] = gam.getBoolean(sys.argv[i + 1],
+                                                     'guestscanmodify')
             i += 2
         elif myarg == 'id':
             if action == 'update':
                 controlflow.invalid_argument_exit(
                     'id', 'gam calendar <calendar> updateevent')
-            body['id'] = sys.argv[i+1]
+            body['id'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'summary':
-            body['summary'] = sys.argv[i+1]
+            body['summary'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'location':
-            body['location'] = sys.argv[i+1]
+            body['location'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'available':
             body['transparency'] = 'transparent'
             i += 1
         elif myarg == 'transparency':
             validTransparency = ['opaque', 'transparent']
-            if sys.argv[i+1].lower() in validTransparency:
-                body['transparency'] = sys.argv[i+1].lower()
+            if sys.argv[i + 1].lower() in validTransparency:
+                body['transparency'] = sys.argv[i + 1].lower()
             else:
-                controlflow.expected_argument_exit(
-                    'transparency',
-                    ", ".join(validTransparency), sys.argv[i+1])
+                controlflow.expected_argument_exit('transparency',
+                                                   ', '.join(validTransparency),
+                                                   sys.argv[i + 1])
             i += 2
         elif myarg == 'visibility':
             validVisibility = ['default', 'public', 'private']
-            if sys.argv[i+1].lower() in validVisibility:
-                body['visibility'] = sys.argv[i+1].lower()
+            if sys.argv[i + 1].lower() in validVisibility:
+                body['visibility'] = sys.argv[i + 1].lower()
             else:
-                controlflow.expected_argument_exit(
-                    "visibility", ", ".join(validVisibility), sys.argv[i+1])
+                controlflow.expected_argument_exit('visibility',
+                                                   ', '.join(validVisibility),
+                                                   sys.argv[i + 1])
             i += 2
         elif myarg == 'tentative':
             body['status'] = 'tentative'
             i += 1
         elif myarg == 'status':
             validStatus = ['confirmed', 'tentative', 'cancelled']
-            if sys.argv[i+1].lower() in validStatus:
-                body['status'] = sys.argv[i+1].lower()
+            if sys.argv[i + 1].lower() in validStatus:
+                body['status'] = sys.argv[i + 1].lower()
             else:
-                controlflow.expected_argument_exit(
-                    'visibility', ', '.join(validStatus), sys.argv[i+1])
+                controlflow.expected_argument_exit('visibility',
+                                                   ', '.join(validStatus),
+                                                   sys.argv[i + 1])
             i += 2
         elif myarg == 'source':
-            body['source'] = {'title': sys.argv[i+1], 'url': sys.argv[i+2]}
+            body['source'] = {'title': sys.argv[i + 1], 'url': sys.argv[i + 2]}
             i += 3
         elif myarg == 'noreminders':
             body['reminders'] = {'useDefault': False}
             i += 1
         elif myarg == 'reminder':
             minutes = \
-            __main__.getInteger(sys.argv[i+1], myarg, minVal=0,
+            gam.getInteger(sys.argv[i+1], myarg, minVal=0,
                                 maxVal=CALENDAR_REMINDER_MAX_MINUTES)
-            reminder = {'minutes': minutes, 'method': sys.argv[i+2]}
-            body.setdefault(
-                'reminders', {'overrides': [], 'useDefault': False})
+            reminder = {'minutes': minutes, 'method': sys.argv[i + 2]}
+            body.setdefault('reminders', {'overrides': [], 'useDefault': False})
             body['reminders']['overrides'].append(reminder)
             i += 3
         elif myarg == 'recurrence':
             body.setdefault('recurrence', [])
-            body['recurrence'].append(sys.argv[i+1])
+            body['recurrence'].append(sys.argv[i + 1])
             i += 2
         elif myarg == 'timezone':
-            timeZone = sys.argv[i+1]
+            timeZone = sys.argv[i + 1]
             i += 2
         elif myarg == 'privateproperty':
             if 'extendedProperties' not in body:
                 body['extendedProperties'] = {'private': {}, 'shared': {}}
-            body['extendedProperties']['private'][sys.argv[i+1]] = sys.argv[i+2]
+            body['extendedProperties']['private'][sys.argv[i +
+                                                           1]] = sys.argv[i + 2]
             i += 3
         elif myarg == 'sharedproperty':
             if 'extendedProperties' not in body:
                 body['extendedProperties'] = {'private': {}, 'shared': {}}
-            body['extendedProperties']['shared'][sys.argv[i+1]] = sys.argv[i+2]
+            body['extendedProperties']['shared'][sys.argv[i + 1]] = sys.argv[i +
+                                                                             2]
             i += 3
         elif myarg == 'colorindex':
-            body['colorId'] = __main__.getInteger(
-                sys.argv[i+1], myarg, CALENDAR_EVENT_MIN_COLOR_INDEX,
-                CALENDAR_EVENT_MAX_COLOR_INDEX)
+            body['colorId'] = gam.getInteger(sys.argv[i + 1], myarg,
+                                             CALENDAR_EVENT_MIN_COLOR_INDEX,
+                                             CALENDAR_EVENT_MAX_COLOR_INDEX)
             i += 2
         elif myarg == 'hangoutsmeet':
-            body['conferenceData'] = {'createRequest': {
-                'requestId': f'{str(uuid.uuid4())}'}}
+            body['conferenceData'] = {
+                'createRequest': {
+                    'requestId': f'{str(uuid.uuid4())}'
+                }
+            }
             i += 1
         else:
             controlflow.invalid_argument_exit(
                 sys.argv[i], f'gam calendar <email> {action}event')
     if ('recurrence' in body) and (('start' in body) or ('end' in body)):
         if not timeZone:
-            timeZone = gapi.call(cal.calendars(), 'get',
+            timeZone = gapi.call(cal.calendars(),
+                                 'get',
                                  calendarId=calendarId,
                                  fields='timeZone')['timeZone']
         if 'start' in body:
@@ -515,20 +555,20 @@ def modifySettings():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'description':
-            body['description'] = sys.argv[i+1]
+            body['description'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'location':
-            body['location'] = sys.argv[i+1]
+            body['location'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'summary':
-            body['summary'] = sys.argv[i+1]
+            body['summary'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'timezone':
-            body['timeZone'] = sys.argv[i+1]
+            body['timeZone'] = sys.argv[i + 1]
             i += 2
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam calendar <email> modify")
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam calendar <email> modify')
     gapi.call(cal.calendars(), 'patch', calendarId=calendarId, body=body)
 
 
@@ -540,23 +580,23 @@ def changeAttendees(users):
     while len(sys.argv) > i:
         myarg = sys.argv[i].lower()
         if myarg == 'csv':
-            csv_file = sys.argv[i+1]
+            csv_file = sys.argv[i + 1]
             i += 2
         elif myarg == 'dryrun':
             do_it = False
             i += 1
         elif myarg == 'start':
-            start_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            start_date = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'end':
-            end_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            end_date = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'allevents':
             allevents = True
             i += 1
         else:
             controlflow.invalid_argument_exit(
-                sys.argv[i], "gam <users> update calattendees")
+                sys.argv[i], 'gam <users> update calattendees')
     attendee_map = {}
     f = fileutils.open_file(csv_file)
     csvFile = csv.reader(f)
@@ -570,9 +610,13 @@ def changeAttendees(users):
             continue
         page_token = None
         while True:
-            events_page = gapi.call(cal.events(), 'list', calendarId=user,
-                                    pageToken=page_token, timeMin=start_date,
-                                    timeMax=end_date, showDeleted=False,
+            events_page = gapi.call(cal.events(),
+                                    'list',
+                                    calendarId=user,
+                                    pageToken=page_token,
+                                    timeMin=start_date,
+                                    timeMax=end_date,
+                                    showDeleted=False,
                                     showHiddenInvitations=False)
             print(f'Got {len(events_page.get("items", []))}')
             for event in events_page.get('items', []):
@@ -596,8 +640,8 @@ def changeAttendees(users):
                         try:
                             if attendee['email'].lower() in attendee_map:
                                 old_email = attendee['email'].lower()
-                                new_email = attendee_map[attendee['email'].lower(
-                                )]
+                                new_email = attendee_map[
+                                    attendee['email'].lower()]
                                 print(f' SWITCHING attendee {old_email} to ' \
                                     f'{new_email} for {event_summary}')
                                 event['attendees'].remove(attendee)
@@ -612,9 +656,12 @@ def changeAttendees(users):
                     body['attendees'] = event['attendees']
                     print(f'UPDATING {event_summary}')
                     if do_it:
-                        gapi.call(cal.events(), 'patch', calendarId=user,
+                        gapi.call(cal.events(),
+                                  'patch',
+                                  calendarId=user,
                                   eventId=event['id'],
-                                  sendNotifications=False, body=body)
+                                  sendNotifications=False,
+                                  body=body)
                     else:
                         print(' not pulling the trigger.')
                 # else:
@@ -631,8 +678,10 @@ def deleteCalendar(users):
         user, cal = buildCalendarGAPIObject(user)
         if not cal:
             continue
-        gapi.call(cal.calendarList(), 'delete',
-                  soft_errors=True, calendarId=calendarId)
+        gapi.call(cal.calendarList(),
+                  'delete',
+                  soft_errors=True,
+                  calendarId=calendarId)
 
 
 CALENDAR_REMINDER_MAX_MINUTES = 40320
@@ -649,62 +698,71 @@ def getCalendarAttributes(i, body, function):
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'selected':
-            body['selected'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            body['selected'] = gam.getBoolean(sys.argv[i + 1], myarg)
             i += 2
         elif myarg == 'hidden':
-            body['hidden'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            body['hidden'] = gam.getBoolean(sys.argv[i + 1], myarg)
             i += 2
         elif myarg == 'summary':
-            body['summaryOverride'] = sys.argv[i+1]
+            body['summaryOverride'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'colorindex':
-            body['colorId'] = __main__.getInteger(
-                sys.argv[i+1], myarg, minVal=CALENDAR_MIN_COLOR_INDEX,
-                maxVal=CALENDAR_MAX_COLOR_INDEX)
+            body['colorId'] = gam.getInteger(sys.argv[i + 1],
+                                             myarg,
+                                             minVal=CALENDAR_MIN_COLOR_INDEX,
+                                             maxVal=CALENDAR_MAX_COLOR_INDEX)
             i += 2
         elif myarg == 'backgroundcolor':
-            body['backgroundColor'] = __main__.getColor(sys.argv[i+1])
+            body['backgroundColor'] = gam.getColor(sys.argv[i + 1])
             colorRgbFormat = True
             i += 2
         elif myarg == 'foregroundcolor':
-            body['foregroundColor'] = __main__.getColor(sys.argv[i+1])
+            body['foregroundColor'] = gam.getColor(sys.argv[i + 1])
             colorRgbFormat = True
             i += 2
         elif myarg == 'reminder':
             body.setdefault('defaultReminders', [])
-            method = sys.argv[i+1].lower()
+            method = sys.argv[i + 1].lower()
             if method not in CLEAR_NONE_ARGUMENT:
                 if method not in CALENDAR_REMINDER_METHODS:
-                    controlflow.expected_argument_exit("Method", ", ".join(
-                        CALENDAR_REMINDER_METHODS+CLEAR_NONE_ARGUMENT), method)
-                minutes = __main__.getInteger(
-                    sys.argv[i+2], myarg, minVal=0,
-                    maxVal=CALENDAR_REMINDER_MAX_MINUTES)
-                body['defaultReminders'].append(
-                    {'method': method, 'minutes': minutes})
+                    controlflow.expected_argument_exit(
+                        'Method', ', '.join(CALENDAR_REMINDER_METHODS +
+                                            CLEAR_NONE_ARGUMENT), method)
+                minutes = gam.getInteger(sys.argv[i + 2],
+                                         myarg,
+                                         minVal=0,
+                                         maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+                body['defaultReminders'].append({
+                    'method': method,
+                    'minutes': minutes
+                })
                 i += 3
             else:
                 i += 2
         elif myarg == 'notification':
             body.setdefault('notificationSettings', {'notifications': []})
-            method = sys.argv[i+1].lower()
+            method = sys.argv[i + 1].lower()
             if method not in CLEAR_NONE_ARGUMENT:
                 if method not in CALENDAR_NOTIFICATION_METHODS:
-                    controlflow.expected_argument_exit("Method", ", ".join(
-                        CALENDAR_NOTIFICATION_METHODS+CLEAR_NONE_ARGUMENT), method)
-                eventType = sys.argv[i+2].lower()
+                    controlflow.expected_argument_exit(
+                        'Method', ', '.join(CALENDAR_NOTIFICATION_METHODS +
+                                            CLEAR_NONE_ARGUMENT), method)
+                eventType = sys.argv[i + 2].lower()
                 if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP:
-                    controlflow.expected_argument_exit("Event", ", ".join(
-                        CALENDAR_NOTIFICATION_TYPES_MAP), eventType)
-                notice = {'method': method,
-                          'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]}
+                    controlflow.expected_argument_exit(
+                        'Event', ', '.join(CALENDAR_NOTIFICATION_TYPES_MAP),
+                        eventType)
+                notice = {
+                    'method': method,
+                    'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]
+                }
                 body['notificationSettings']['notifications'].append(notice)
                 i += 3
             else:
                 i += 2
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], f"gam {function} calendar")
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              f'gam {function} calendar')
     return colorRgbFormat
 
 
@@ -721,8 +779,11 @@ def addCalendar(users):
             continue
         current_count = display.current_count(i, count)
         print(f'Subscribing {user} to calendar {calendarId}{current_count}')
-        gapi.call(cal.calendarList(), 'insert', soft_errors=True,
-                  body=body, colorRgbFormat=colorRgbFormat)
+        gapi.call(cal.calendarList(),
+                  'insert',
+                  soft_errors=True,
+                  body=body,
+                  colorRgbFormat=colorRgbFormat)
 
 
 def updateCalendar(users):
@@ -740,13 +801,17 @@ def updateCalendar(users):
         print(f"Updating {user}'s subscription to calendar ' \
               f'{calendarId}{current_count}")
         calId = calendarId if calendarId != 'primary' else user
-        gapi.call(cal.calendarList(), 'patch', soft_errors=True,
-                  calendarId=calId, body=body, colorRgbFormat=colorRgbFormat)
+        gapi.call(cal.calendarList(),
+                  'patch',
+                  soft_errors=True,
+                  calendarId=calId,
+                  body=body,
+                  colorRgbFormat=colorRgbFormat)
 
 
 def _showCalendar(userCalendar, j, jcount):
     current_count = display.current_count(j, jcount)
-    summary = userCalendar.get("summaryOverride", userCalendar["summary"])
+    summary = userCalendar.get('summaryOverride', userCalendar['summary'])
     print(f'  Calendar: {userCalendar["id"]}{current_count}')
     print(f'    Summary: {summary}')
     print(f'    Description: {userCalendar.get("description", "")}')
@@ -780,7 +845,8 @@ def infoCalendar(users):
         user, cal = buildCalendarGAPIObject(user)
         if not cal:
             continue
-        result = gapi.call(cal.calendarList(), 'get',
+        result = gapi.call(cal.calendarList(),
+                           'get',
                            soft_errors=True,
                            calendarId=calendarId)
         if result:
@@ -809,8 +875,10 @@ def printShowCalendars(users, csvFormat):
         user, cal = buildCalendarGAPIObject(user)
         if not cal:
             continue
-        result = gapi.get_all_pages(
-            cal.calendarList(), 'list', 'items', soft_errors=True)
+        result = gapi.get_all_pages(cal.calendarList(),
+                                    'list',
+                                    'items',
+                                    soft_errors=True)
         jcount = len(result)
         if not csvFormat:
             print(f'User: {user}, Calendars:{display.current_count(i, count)}')
@@ -825,8 +893,9 @@ def printShowCalendars(users, csvFormat):
                 continue
             for userCalendar in result:
                 row = {'primaryEmail': user}
-                display.add_row_titles_to_csv_file(utils.flatten_json(
-                    userCalendar, flattened=row), csvRows, titles)
+                display.add_row_titles_to_csv_file(
+                    utils.flatten_json(userCalendar, flattened=row), csvRows,
+                    titles)
     if csvFormat:
         display.sort_csv_titles(['primaryEmail', 'id'], titles)
         display.write_csv_file(csvRows, titles, 'Calendars', todrive)
@@ -840,8 +909,10 @@ def showCalSettings(users):
         user, cal = buildCalendarGAPIObject(user)
         if not cal:
             continue
-        feed = gapi.get_all_pages(
-            cal.settings(), 'list', 'items', soft_errors=True)
+        feed = gapi.get_all_pages(cal.settings(),
+                                  'list',
+                                  'items',
+                                  soft_errors=True)
         if feed:
             current_count = display.current_count(i, count)
             print(f'User: {user}, Calendar Settings:{current_count}')
@@ -862,11 +933,11 @@ def transferSecCals(users):
             remove_source_user = False
             i += 1
         elif myarg == 'sendnotifications':
-            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            sendNotifications = gam.getBoolean(sys.argv[i + 1], myarg)
             i += 2
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam <users> transfer seccals")
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam <users> transfer seccals')
     if remove_source_user:
         target_user, target_cal = buildCalendarGAPIObject(target_user)
         if not target_cal:
@@ -875,20 +946,38 @@ def transferSecCals(users):
         user, source_cal = buildCalendarGAPIObject(user)
         if not source_cal:
             continue
-        calendars = gapi.get_all_pages(source_cal.calendarList(), 'list',
-                                       'items', soft_errors=True,
-                                       minAccessRole='owner', showHidden=True,
+        calendars = gapi.get_all_pages(source_cal.calendarList(),
+                                       'list',
+                                       'items',
+                                       soft_errors=True,
+                                       minAccessRole='owner',
+                                       showHidden=True,
                                        fields='items(id),nextPageToken')
         for calendar in calendars:
             calendarId = calendar['id']
             if calendarId.find('@group.calendar.google.com') != -1:
-                body = {'role': 'owner',
-                        'scope': {'type': 'user', 'value': target_user}}
-                gapi.call(source_cal.acl(), 'insert', calendarId=calendarId,
-                          body=body, sendNotifications=sendNotifications)
+                body = {
+                    'role': 'owner',
+                    'scope': {
+                        'type': 'user',
+                        'value': target_user
+                    }
+                }
+                gapi.call(source_cal.acl(),
+                          'insert',
+                          calendarId=calendarId,
+                          body=body,
+                          sendNotifications=sendNotifications)
                 if remove_source_user:
-                    body = {'role': 'none',
-                            'scope': {'type': 'user', 'value': user}}
-                    gapi.call(target_cal.acl(), 'insert',
-                              calendarId=calendarId, body=body,
+                    body = {
+                        'role': 'none',
+                        'scope': {
+                            'type': 'user',
+                            'value': user
+                        }
+                    }
+                    gapi.call(target_cal.acl(),
+                              'insert',
+                              calendarId=calendarId,
+                              body=body,
                               sendNotifications=sendNotifications)

src/gam/gapi/directory/__init__.py

@@ -0,0 +1,5 @@
+import gam
+
+
+def buildGAPIObject():
+    return gam.buildGAPIObject('directory')

src/gam/gapi/directory/cros.py

@@ -1,17 +1,17 @@
 import datetime
 
-from var import *
-import __main__
-import controlflow
-import display
-import fileutils
-import gapi
-import gapi.directory
-import utils
+from gam.var import *
+import gam
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
 
 
 def doUpdateCros():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     i, devices = getCrOSDeviceEntity(3, cd)
     update_body = {}
     action_body = {}
@@ -20,29 +20,33 @@ def doUpdateCros():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'user':
-            update_body['annotatedUser'] = sys.argv[i+1]
+            update_body['annotatedUser'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'location':
-            update_body['annotatedLocation'] = sys.argv[i+1]
+            update_body['annotatedLocation'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'notes':
-            update_body['notes'] = sys.argv[i+1].replace('\\n', '\n')
+            update_body['notes'] = sys.argv[i + 1].replace('\\n', '\n')
             i += 2
         elif myarg in ['tag', 'asset', 'assetid']:
-            update_body['annotatedAssetId'] = sys.argv[i+1]
+            update_body['annotatedAssetId'] = sys.argv[i + 1]
             i += 2
         elif myarg in ['ou', 'org']:
-            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            orgUnitPath = gam.getOrgUnitItem(sys.argv[i + 1])
             i += 2
         elif myarg == 'action':
-            action = sys.argv[i+1].lower().replace('_', '').replace('-', '')
+            action = sys.argv[i + 1].lower().replace('_', '').replace('-', '')
             deprovisionReason = None
-            if action in ['deprovisionsamemodelreplace',
-                          'deprovisionsamemodelreplacement']:
+            if action in [
+                    'deprovisionsamemodelreplace',
+                    'deprovisionsamemodelreplacement'
+            ]:
                 action = 'deprovision'
                 deprovisionReason = 'same_model_replacement'
-            elif action in ['deprovisiondifferentmodelreplace',
-                            'deprovisiondifferentmodelreplacement']:
+            elif action in [
+                    'deprovisiondifferentmodelreplace',
+                    'deprovisiondifferentmodelreplacement'
+            ]:
                 action = 'deprovision'
                 deprovisionReason = 'different_model_replacement'
             elif action in ['deprovisionretiringdevice']:
@@ -62,7 +66,7 @@ def doUpdateCros():
             ack_wipe = True
             i += 1
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam update cros")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update cros')
     i = 0
     count = len(devices)
     if action_body:
@@ -84,33 +88,39 @@ def doUpdateCros():
             sys.exit(3)
         for deviceId in devices:
             i += 1
-            cur_count = __main__.currentCount(i, count)
+            cur_count = gam.currentCount(i, count)
             print(f' performing action {action} for {deviceId}{cur_count}')
-            gapi.call(cd.chromeosdevices(), function='action',
+            gapi.call(cd.chromeosdevices(),
+                      function='action',
                       customerId=GC_Values[GC_CUSTOMER_ID],
-                      resourceId=deviceId, body=action_body)
+                      resourceId=deviceId,
+                      body=action_body)
     else:
         if update_body:
             for deviceId in devices:
                 i += 1
-                current_count = __main__.currentCount(i, count)
+                current_count = gam.currentCount(i, count)
                 print(f' updating {deviceId}{current_count}')
-                gapi.call(cd.chromeosdevices(), 'update',
+                gapi.call(cd.chromeosdevices(),
+                          'update',
                           customerId=GC_Values[GC_CUSTOMER_ID],
-                          deviceId=deviceId, body=update_body)
+                          deviceId=deviceId,
+                          body=update_body)
         if orgUnitPath:
             # split moves into max 50 devices per batch
             for l in range(0, len(devices), 50):
-                move_body = {'deviceIds': devices[l:l+50]}
+                move_body = {'deviceIds': devices[l:l + 50]}
                 print(f' moving {len(move_body["deviceIds"])} devices to ' \
                       f'{orgUnitPath}')
-                gapi.call(cd.chromeosdevices(), 'moveDevicesToOu',
+                gapi.call(cd.chromeosdevices(),
+                          'moveDevicesToOu',
                           customerId=GC_Values[GC_CUSTOMER_ID],
-                          orgUnitPath=orgUnitPath, body=move_body)
+                          orgUnitPath=orgUnitPath,
+                          body=move_body)
 
 
 def doGetCrosInfo():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     i, devices = getCrOSDeviceEntity(3, cd)
     downloadfile = None
     targetFolder = GC_Values[GC_DRIVE_DIR]
@@ -125,13 +135,13 @@ def doGetCrosInfo():
             noLists = True
             i += 1
         elif myarg == 'listlimit':
-            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=-1)
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=-1)
             i += 2
         elif myarg in CROS_START_ARGUMENTS:
-            startDate = _getFilterDate(sys.argv[i+1])
+            startDate = _getFilterDate(sys.argv[i + 1])
             i += 2
         elif myarg in CROS_END_ARGUMENTS:
-            endDate = _getFilterDate(sys.argv[i+1])
+            endDate = _getFilterDate(sys.argv[i + 1])
             i += 2
         elif myarg == 'allfields':
             projection = 'FULL'
@@ -148,7 +158,7 @@ def doGetCrosInfo():
             fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg])
             i += 1
         elif myarg == 'fields':
-            fieldNameList = sys.argv[i+1]
+            fieldNameList = sys.argv[i + 1]
             for field in fieldNameList.lower().replace(',', ' ').split():
                 if field in CROS_ARGUMENT_TO_PROPERTY_MAP:
                     fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field])
@@ -158,21 +168,21 @@ def doGetCrosInfo():
                         projection = 'FULL'
                         noLists = False
                 else:
-                    controlflow.invalid_argument_exit(
-                        field, "gam info cros fields")
+                    controlflow.invalid_argument_exit(field,
+                                                      'gam info cros fields')
             i += 2
         elif myarg == 'downloadfile':
-            downloadfile = sys.argv[i+1]
+            downloadfile = sys.argv[i + 1]
             if downloadfile.lower() == 'latest':
                 downloadfile = downloadfile.lower()
             i += 2
         elif myarg == 'targetfolder':
-            targetFolder = os.path.expanduser(sys.argv[i+1])
+            targetFolder = os.path.expanduser(sys.argv[i + 1])
             if not os.path.isdir(targetFolder):
                 os.makedirs(targetFolder)
             i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam info cros")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam info cros')
     if fieldsList:
         fieldsList.append('deviceId')
         fields = ','.join(set(fieldsList)).replace('.', '/')
@@ -182,9 +192,11 @@ def doGetCrosInfo():
     device_count = len(devices)
     for deviceId in devices:
         i += 1
-        cros = gapi.call(cd.chromeosdevices(), 'get',
+        cros = gapi.call(cd.chromeosdevices(),
+                         'get',
                          customerId=GC_Values[GC_CUSTOMER_ID],
-                         deviceId=deviceId, projection=projection,
+                         deviceId=deviceId,
+                         projection=projection,
                          fields=fields)
         print(f'CrOS Device: {deviceId} ({i} of {device_count})')
         if 'notes' in cros:
@@ -208,8 +220,8 @@ def doGetCrosInfo():
                 print('  activeTimeRanges')
                 num_ranges = min(lenATR, listLimit or lenATR)
                 for activeTimeRange in activeTimeRanges[:num_ranges]:
-                    active_date = activeTimeRange["date"]
-                    active_time = activeTimeRange["activeTime"]
+                    active_date = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
                     duration = utils.formatMilliSeconds(active_time)
                     minutes = active_time // 60000
                     print(f'    date: {active_date}')
@@ -222,16 +234,17 @@ def doGetCrosInfo():
                 print('  recentUsers')
                 num_ranges = min(lenRU, listLimit or lenRU)
                 for recentUser in recentUsers[:num_ranges]:
-                    useremail = recentUser.get("email")
+                    useremail = recentUser.get('email')
                     if not useremail:
-                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                        if recentUser['type'] == 'USER_TYPE_UNMANAGED':
                             useremail = 'UnmanagedUser'
                         else:
                             useremail = 'Unknown'
                     print(f'    type: {recentUser["type"]}')
                     print(f'      email: {useremail}')
-            deviceFiles = _filterCreateReportTime(
-                cros.get('deviceFiles', []), 'createTime', startDate, endDate)
+            deviceFiles = _filterCreateReportTime(cros.get('deviceFiles',
+                                                           []), 'createTime',
+                                                  startDate, endDate)
             lenDF = len(deviceFiles)
             if lenDF:
                 num_ranges = min(lenDF, listLimit or lenDF)
@@ -255,22 +268,21 @@ def doGetCrosInfo():
                                   f'available to download.')
                             deviceFile = None
                     if deviceFile:
-                        created = deviceFile["createTime"]
+                        created = deviceFile['createTime']
                         downloadfile = f'cros-logs-{deviceId}-{created}.zip'
                         downloadfilename = os.path.join(targetFolder,
                                                         downloadfile)
                         dl_url = deviceFile['downloadUrl']
                         _, content = cd._http.request(dl_url)
-                        fileutils.write_file(downloadfilename, content,
+                        fileutils.write_file(downloadfilename,
+                                             content,
                                              mode='wb',
                                              continue_on_error=True)
                         print(f'Downloaded: {downloadfilename}')
                 elif downloadfile:
                     print('ERROR: no files to download.')
             cpuStatusReports = _filterCreateReportTime(
-                cros.get('cpuStatusReports', []),
-                'reportTime',
-                startDate,
+                cros.get('cpuStatusReports', []), 'reportTime', startDate,
                 endDate)
             lenCSR = len(cpuStatusReports)
             if lenCSR:
@@ -284,8 +296,8 @@ def doGetCrosInfo():
                         temp_label = tempInfo['label'].strip()
                         temperature = tempInfo['temperature']
                         print(f'        {temp_label}: {temperature}')
-                    pct_info = cpuStatusReport["cpuUtilizationPercentageInfo"]
-                    util = ",".join([str(x) for x in pct_info])
+                    pct_info = cpuStatusReport['cpuUtilizationPercentageInfo']
+                    util = ','.join([str(x) for x in pct_info])
                     print(f'      cpuUtilizationPercentageInfo: {util}')
             diskVolumeReports = cros.get('diskVolumeReports', [])
             lenDVR = len(diskVolumeReports)
@@ -303,28 +315,32 @@ def doGetCrosInfo():
                         print(f'        storageFree: {vstorage_free}')
                         print(f'        storageTotal: {vstorage_total}')
             systemRamFreeReports = _filterCreateReportTime(
-                cros.get('systemRamFreeReports', []),
-                'reportTime', startDate, endDate)
+                cros.get('systemRamFreeReports', []), 'reportTime', startDate,
+                endDate)
             lenSRFR = len(systemRamFreeReports)
             if lenSRFR:
                 print('  systemRamFreeReports')
                 num_ranges = min(lenSRFR, listLimit or lenSRFR)
                 for systemRamFreeReport in systemRamFreeReports[:num_ranges]:
-                    report_time = systemRamFreeReport["reportTime"]
-                    free_info = systemRamFreeReport["systemRamFreeInfo"]
-                    free_ram = ",".join(free_info)
+                    report_time = systemRamFreeReport['reportTime']
+                    free_info = systemRamFreeReport['systemRamFreeInfo']
+                    free_ram = ','.join(free_info)
                     print(f'    reportTime: {report_time}')
                     print(f'      systemRamFreeInfo: {free_ram}')
 
 
 def doPrintCrosActivity():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     todrive = False
-    titles = ['deviceId', 'annotatedAssetId',
-              'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    titles = [
+        'deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber',
+        'orgUnitPath'
+    ]
     csvRows = []
-    fieldsList = ['deviceId', 'annotatedAssetId',
-                  'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    fieldsList = [
+        'deviceId', 'annotatedAssetId', 'annotatedLocation', 'serialNumber',
+        'orgUnitPath'
+    ]
     startDate = endDate = None
     selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False
     listLimit = 0
@@ -335,10 +351,10 @@ def doPrintCrosActivity():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg in ['query', 'queries']:
-            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
             i += 2
         elif myarg == 'limittoou':
-            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            orgUnitPath = gam.getOrgUnitItem(sys.argv[i + 1])
             i += 2
         elif myarg == 'todrive':
             todrive = True
@@ -360,32 +376,35 @@ def doPrintCrosActivity():
             selectRecentUsers = True
             i += 1
         elif myarg in CROS_START_ARGUMENTS:
-            startDate = _getFilterDate(sys.argv[i+1])
+            startDate = _getFilterDate(sys.argv[i + 1])
             i += 2
         elif myarg in CROS_END_ARGUMENTS:
-            endDate = _getFilterDate(sys.argv[i+1])
+            endDate = _getFilterDate(sys.argv[i + 1])
             i += 2
         elif myarg == 'listlimit':
-            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=0)
             i += 2
         elif myarg == 'delimiter':
-            delimiter = sys.argv[i+1]
+            delimiter = sys.argv[i + 1]
             i += 2
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam print crosactivity")
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print crosactivity')
     if not selectActiveTimeRanges and \
        not selectDeviceFiles and \
        not selectRecentUsers:
         selectActiveTimeRanges = selectRecentUsers = True
     if selectRecentUsers:
         fieldsList.append('recentUsers')
-        display.add_titles_to_csv_file(['recentUsers.email', ], titles)
+        display.add_titles_to_csv_file([
+            'recentUsers.email',
+        ], titles)
     if selectActiveTimeRanges:
         fieldsList.append('activeTimeRanges')
-        titles_to_add = ['activeTimeRanges.date',
-                         'activeTimeRanges.duration',
-                         'activeTimeRanges.minutes']
+        titles_to_add = [
+            'activeTimeRanges.date', 'activeTimeRanges.duration',
+            'activeTimeRanges.minutes'
+        ]
         display.add_titles_to_csv_file(titles_to_add, titles)
     if selectDeviceFiles:
         fieldsList.append('deviceFiles')
@@ -393,15 +412,17 @@ def doPrintCrosActivity():
         display.add_titles_to_csv_file(titles_to_add, titles)
     fields = f'nextPageToken,chromeosdevices({",".join(fieldsList)})'
     for query in queries:
-        __main__.printGettingAllItems('CrOS Devices', query)
+        gam.printGettingAllItems('CrOS Devices', query)
         page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
-        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(),
+                                      'list',
                                       'chromeosdevices',
                                       page_message=page_message,
                                       query=query,
                                       customerId=GC_Values[GC_CUSTOMER_ID],
                                       projection='FULL',
-                                      fields=fields, orgUnitPath=orgUnitPath)
+                                      fields=fields,
+                                      orgUnitPath=orgUnitPath)
         for cros in all_cros:
             row = {}
             skip_attribs = ['recentUsers', 'activeTimeRanges', 'deviceFiles']
@@ -428,9 +449,9 @@ def doPrintCrosActivity():
                 num_ranges = min(lenRU, listLimit or lenRU)
                 recent_users = []
                 for recentUser in recentUsers[:num_ranges]:
-                    useremail = recentUser.get("email")
+                    useremail = recentUser.get('email')
                     if not useremail:
-                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                        if recentUser['type'] == 'USER_TYPE_UNMANAGED':
                             useremail = 'UnmanagedUser'
                         else:
                             useremail = 'Unknown'
@@ -439,8 +460,8 @@ def doPrintCrosActivity():
                 csvRows.append(row)
             if selectDeviceFiles:
                 deviceFiles = _filterCreateReportTime(
-                    cros.get('deviceFiles', []),
-                    'createTime', startDate, endDate)
+                    cros.get('deviceFiles', []), 'createTime', startDate,
+                    endDate)
                 lenDF = len(deviceFiles)
                 num_ranges = min(lenDF, listLimit or lenDF)
                 for deviceFile in deviceFiles[:num_ranges]:
@@ -465,6 +486,7 @@ def _checkTPMVulnerability(cros):
 
 
 def doPrintCrosDevices():
+
     def _getSelectedLists(myarg):
         if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
             selectedLists['activeTimeRanges'] = True
@@ -479,14 +501,14 @@ def doPrintCrosDevices():
         elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS:
             selectedLists['systemRamFreeReports'] = True
 
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     todrive = False
     fieldsList = []
     fieldsTitles = {}
     titles = []
     csvRows = []
-    display.add_field_to_csv_file(
-        'deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles)
+    display.add_field_to_csv_file('deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                  fieldsList, fieldsTitles, titles)
     projection = orderBy = sortOrder = orgUnitPath = None
     queries = [None]
     noLists = sortHeaders = False
@@ -497,10 +519,10 @@ def doPrintCrosDevices():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg in ['query', 'queries']:
-            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            queries = gam.getQueries(myarg, sys.argv[i + 1])
             i += 2
         elif myarg == 'limittoou':
-            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            orgUnitPath = gam.getOrgUnitItem(sys.argv[i + 1])
             i += 2
         elif myarg == 'todrive':
             todrive = True
@@ -510,21 +532,24 @@ def doPrintCrosDevices():
             selectedLists = {}
             i += 1
         elif myarg == 'listlimit':
-            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            listLimit = gam.getInteger(sys.argv[i + 1], myarg, minVal=0)
             i += 2
         elif myarg in CROS_START_ARGUMENTS:
-            startDate = _getFilterDate(sys.argv[i+1])
+            startDate = _getFilterDate(sys.argv[i + 1])
             i += 2
         elif myarg in CROS_END_ARGUMENTS:
-            endDate = _getFilterDate(sys.argv[i+1])
+            endDate = _getFilterDate(sys.argv[i + 1])
             i += 2
         elif myarg == 'orderby':
-            orderBy = sys.argv[i+1].lower().replace('_', '')
-            validOrderBy = ['location', 'user', 'lastsync',
-                            'notes', 'serialnumber', 'status', 'supportenddate']
+            orderBy = sys.argv[i + 1].lower().replace('_', '')
+            validOrderBy = [
+                'location', 'user', 'lastsync', 'notes', 'serialnumber',
+                'status', 'supportenddate'
+            ]
             if orderBy not in validOrderBy:
-                controlflow.expected_argument_exit(
-                    "orderby", ", ".join(validOrderBy), orderBy)
+                controlflow.expected_argument_exit('orderby',
+                                                   ', '.join(validOrderBy),
+                                                   orderBy)
             if orderBy == 'location':
                 orderBy = 'annotatedLocation'
             elif orderBy == 'user':
@@ -559,11 +584,12 @@ def doPrintCrosDevices():
             _getSelectedLists(myarg)
             i += 1
         elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
-            display.add_field_to_fields_list(
-                myarg, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+            display.add_field_to_fields_list(myarg,
+                                             CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                             fieldsList)
             i += 1
         elif myarg == 'fields':
-            fieldNameList = sys.argv[i+1]
+            fieldNameList = sys.argv[i + 1]
             for field in fieldNameList.lower().replace(',', ' ').split():
                 if field in CROS_LISTS_ARGUMENTS:
                     _getSelectedLists(field)
@@ -571,17 +597,18 @@ def doPrintCrosDevices():
                     display.add_field_to_fields_list(
                         field, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
                 else:
-                    controlflow.invalid_argument_exit(
-                        field, "gam print cros fields")
+                    controlflow.invalid_argument_exit(field,
+                                                      'gam print cros fields')
             i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam print cros")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print cros')
     if selectedLists:
         noLists = False
         projection = 'FULL'
         for selectList in selectedLists:
-            display.add_field_to_fields_list(
-                selectList, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+            display.add_field_to_fields_list(selectList,
+                                             CROS_ARGUMENT_TO_PROPERTY_MAP,
+                                             fieldsList)
     if fieldsList:
         fieldsList.append('deviceId')
         fields = f'nextPageToken,chromeosdevices({",".join(set(fieldsList))})'.replace(
@@ -589,15 +616,18 @@ def doPrintCrosDevices():
     else:
         fields = None
     for query in queries:
-        __main__.printGettingAllItems('CrOS Devices', query)
+        gam.printGettingAllItems('CrOS Devices', query)
         page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
-        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(),
+                                      'list',
                                       'chromeosdevices',
-                                      page_message=page_message, query=query,
+                                      page_message=page_message,
+                                      query=query,
                                       customerId=GC_Values[GC_CUSTOMER_ID],
                                       projection=projection,
                                       orgUnitPath=orgUnitPath,
-                                      orderBy=orderBy, sortOrder=sortOrder,
+                                      orderBy=orderBy,
+                                      sortOrder=sortOrder,
                                       fields=fields)
         for cros in all_cros:
             _checkTPMVulnerability(cros)
@@ -612,8 +642,9 @@ def doPrintCrosDevices():
                     tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
                     for tempInfo in tempInfos:
                         tempInfo['label'] = tempInfo['label'].strip()
-                display.add_row_titles_to_csv_file(utils.flatten_json(
-                    cros, listLimit=listLimit), csvRows, titles)
+                display.add_row_titles_to_csv_file(
+                    utils.flatten_json(cros, listLimit=listLimit), csvRows,
+                    titles)
             continue
         for cros in all_cros:
             if 'notes' in cros:
@@ -623,11 +654,11 @@ def doPrintCrosDevices():
                     cros['autoUpdateExpiration'])
             row = {}
             for attrib in cros:
-                if attrib not in set(['kind', 'etag', 'tpmVersionInfo',
-                                      'recentUsers', 'activeTimeRanges',
-                                      'deviceFiles', 'cpuStatusReports',
-                                      'diskVolumeReports',
-                                      'systemRamFreeReports']):
+                if attrib not in set([
+                        'kind', 'etag', 'tpmVersionInfo', 'recentUsers',
+                        'activeTimeRanges', 'deviceFiles', 'cpuStatusReports',
+                        'diskVolumeReports', 'systemRamFreeReports'
+                ]):
                     row[attrib] = cros[attrib]
             if selectedLists.get('activeTimeRanges'):
                 timergs = cros.get('activeTimeRanges', [])
@@ -649,8 +680,8 @@ def doPrintCrosDevices():
             else:
                 cpu_reports = []
             cpuStatusReports = _filterCreateReportTime(cpu_reports,
-                                                       'reportTime',
-                                                       startDate, endDate)
+                                                       'reportTime', startDate,
+                                                       endDate)
             if selectedLists.get('diskVolumeReports'):
                 diskVolumeReports = cros.get('diskVolumeReports', [])
             else:
@@ -659,10 +690,8 @@ def doPrintCrosDevices():
                 ram_reports = cros.get('systemRamFreeReports', [])
             else:
                 ram_reports = []
-            systemRamFreeReports = _filterCreateReportTime(ram_reports,
-                                                           'reportTime',
-                                                           startDate,
-                                                           endDate)
+            systemRamFreeReports = _filterCreateReportTime(
+                ram_reports, 'reportTime', startDate, endDate)
             if noLists or (not activeTimeRanges and \
                            not recentUsers and \
                            not deviceFiles and \
@@ -707,7 +736,7 @@ def doPrintCrosDevices():
                     tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo',
                                                         [])
                     for tempInfo in tempInfos:
-                        label = tempInfo["label"].strip()
+                        label = tempInfo['label'].strip()
                         base = 'cpuStatusReports.cpuTemperatureInfo.'
                         nrow[f'{base}{label}'] = tempInfo['temperature']
                     cpu_field = 'cpuUtilizationPercentageInfo'
@@ -735,16 +764,18 @@ def doPrintCrosDevices():
                         ','.join(ram_info)
                 display.add_row_titles_to_csv_file(nrow, csvRows, titles)
     if sortHeaders:
-        display.sort_csv_titles(['deviceId', ], titles)
+        display.sort_csv_titles([
+            'deviceId',
+        ], titles)
     display.write_csv_file(csvRows, titles, 'CrOS', todrive)
 
 
 def getCrOSDeviceEntity(i, cd):
     myarg = sys.argv[i].lower()
     if myarg == 'cros_sn':
-        return i+2, __main__.getUsersToModify('cros_sn', sys.argv[i+1])
+        return i + 2, gam.getUsersToModify('cros_sn', sys.argv[i + 1])
     if myarg == 'query':
-        return i+2, __main__.getUsersToModify('crosquery', sys.argv[i+1])
+        return i + 2, gam.getUsersToModify('crosquery', sys.argv[i + 1])
     if myarg[:6] == 'query:':
         query = sys.argv[i][6:]
         if query[:12].lower() == 'orgunitpath:':
@@ -752,12 +783,14 @@ def getCrOSDeviceEntity(i, cd):
         else:
             kwargs = {'query': query}
         fields = 'nextPageToken,chromeosdevices(deviceId)'
-        devices = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+        devices = gapi.get_all_pages(cd.chromeosdevices(),
+                                     'list',
                                      'chromeosdevices',
                                      customerId=GC_Values[GC_CUSTOMER_ID],
-                                     fields=fields, **kwargs)
-        return i+1, [device['deviceId'] for device in devices]
-    return i+1, sys.argv[i].replace(',', ' ').split()
+                                     fields=fields,
+                                     **kwargs)
+        return i + 1, [device['deviceId'] for device in devices]
+    return i + 1, sys.argv[i].replace(',', ' ').split()
 
 
 def _getFilterDate(dateStr):
@@ -769,8 +802,8 @@ def _filterTimeRanges(activeTimeRanges, startDate, endDate):
         return activeTimeRanges
     filteredTimeRanges = []
     for timeRange in activeTimeRanges:
-        activityDate = datetime.datetime.strptime(
-            timeRange['date'], YYYYMMDD_FORMAT)
+        activityDate = datetime.datetime.strptime(timeRange['date'],
+                                                  YYYYMMDD_FORMAT)
         if ((startDate is None) or \
             (activityDate >= startDate)) and \
            ((endDate is None) or \

src/gam/gapi/directory/customer.py

@@ -1,19 +1,22 @@
 import datetime
 
-from var import *
-import controlflow
-import gapi
-import gapi.directory
-import gapi.reports
+from gam.var import *
+from gam import controlflow
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam.gapi import reports as gapi_reports
 
 
 def doGetCustomerInfo():
-    cd = gapi.directory.buildGAPIObject()
-    customer_info = gapi.call(cd.customers(), 'get',
+    cd = gapi_directory.buildGAPIObject()
+    customer_info = gapi.call(cd.customers(),
+                              'get',
                               customerKey=GC_Values[GC_CUSTOMER_ID])
     print(f'Customer ID: {customer_info["id"]}')
     print(f'Primary Domain: {customer_info["customerDomain"]}')
-    result = gapi.call(cd.domains(), 'get', customer=customer_info['id'],
+    result = gapi.call(cd.domains(),
+                       'get',
+                       customer=customer_info['id'],
                        domainName=customer_info['customerDomain'],
                        fields='verified')
     print(f'Primary Domain Verified: {result["verified"]}')
@@ -23,11 +26,13 @@ def doGetCustomerInfo():
     customer_creation = customer_info['customerCreationTime']
     date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
     oldest = datetime.datetime.strptime(customer_creation, date_format)
-    domains = gapi.get_items(cd.domains(), 'list', 'domains',
+    domains = gapi.get_items(cd.domains(),
+                             'list',
+                             'domains',
                              customer=GC_Values[GC_CUSTOMER_ID],
                              fields='domains(creationTime)')
     for domain in domains:
-        creation_timestamp = int(domain['creationTime'])/1000
+        creation_timestamp = int(domain['creationTime']) / 1000
         domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
         if domain_creation < oldest:
             oldest = domain_creation
@@ -59,19 +64,21 @@ def doGetCustomerInfo():
     customerId = GC_Values[GC_CUSTOMER_ID]
     if customerId == MY_CUSTOMER:
         customerId = None
-    rep = gapi.reports.buildGAPIObject()
+    rep = gapi_reports.buildGAPIObject()
     usage = None
     throw_reasons = [gapi.errors.ErrorReason.INVALID]
     while True:
         try:
-            usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+            usage = gapi.get_all_pages(rep.customerUsageReports(),
+                                       'get',
                                        'usageReports',
                                        throw_reasons=throw_reasons,
-                                       customerId=customerId, date=tryDate,
+                                       customerId=customerId,
+                                       date=tryDate,
                                        parameters=parameters)
             break
         except gapi.errors.GapiInvalidError as e:
-            tryDate = gapi.reports._adjust_date(str(e))
+            tryDate = gapi_reports._adjust_date(str(e))
     if not usage:
         print('No user count data available.')
         return
@@ -84,7 +91,7 @@ def doGetCustomerInfo():
 
 
 def doUpdateCustomer():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     body = {}
     i = 3
     while i < len(sys.argv):
@@ -92,22 +99,25 @@ def doUpdateCustomer():
         if myarg in ADDRESS_FIELDS_ARGUMENT_MAP:
             body.setdefault('postalAddress', {})
             arg = ADDRESS_FIELDS_ARGUMENT_MAP[myarg]
-            body['postalAddress'][arg] = sys.argv[i+1]
+            body['postalAddress'][arg] = sys.argv[i + 1]
             i += 2
         elif myarg in ['adminsecondaryemail', 'alternateemail']:
-            body['alternateEmail'] = sys.argv[i+1]
+            body['alternateEmail'] = sys.argv[i + 1]
             i += 2
         elif myarg in ['phone', 'phonenumber']:
-            body['phoneNumber'] = sys.argv[i+1]
+            body['phoneNumber'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'language':
-            body['language'] = sys.argv[i+1]
+            body['language'] = sys.argv[i + 1]
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam update customer")
+            controlflow.invalid_argument_exit(myarg, 'gam update customer')
     if not body:
-        controlflow.system_error_exit(2, 'no arguments specified for "gam '
-                                         'update customer"')
-    gapi.call(cd.customers(), 'patch', customerKey=GC_Values[GC_CUSTOMER_ID],
+        controlflow.system_error_exit(
+            2, 'no arguments specified for "gam '
+            'update customer"')
+    gapi.call(cd.customers(),
+              'patch',
+              customerKey=GC_Values[GC_CUSTOMER_ID],
               body=body)
     print('Updated customer')

src/gam/gapi/directory/resource.py

@@ -1,17 +1,18 @@
 import sys
 import uuid
 
-import __main__
-from var import *
-import controlflow
-import display
-import gapi.directory
-import utils
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam.gapi import directory as gapi_directory
+from gam import utils
 
 
 def printBuildings():
     to_drive = False
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     titles = []
     csvRows = []
     fieldsList = ['buildingId']
@@ -35,15 +36,16 @@ def printBuildings():
             fieldsList.append(possible_fields[myarg])
             i += 1
         # Allows shorter arguments like "name" instead of "buildingname"
-        elif 'building'+myarg in possible_fields:
-            fieldsList.append(possible_fields['building'+myarg])
+        elif 'building' + myarg in possible_fields:
+            fieldsList.append(possible_fields['building' + myarg])
             i += 1
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam print buildings")
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print buildings')
     if fields:
         fields = fields % ','.join(fieldsList)
-    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+    buildings = gapi.get_all_pages(cd.resources().buildings(),
+                                   'list',
                                    'buildings',
                                    customer=GC_Values[GC_CUSTOMER_ID],
                                    fields=fields)
@@ -65,7 +67,7 @@ def printBuildings():
 
 
 def printResourceCalendars():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     todrive = False
     fieldsList = []
     fieldsTitles = {}
@@ -79,7 +81,7 @@ def printResourceCalendars():
             todrive = True
             i += 1
         elif myarg == 'query':
-            query = sys.argv[i+1]
+            query = sys.argv[i + 1]
             i += 2
         elif myarg == 'allfields':
             fieldsList = []
@@ -88,8 +90,7 @@ def printResourceCalendars():
             for field in RESCAL_ALLFIELDS:
                 display.add_field_to_csv_file(field,
                                               RESCAL_ARGUMENT_TO_PROPERTY_MAP,
-                                              fieldsList, fieldsTitles,
-                                              titles)
+                                              fieldsList, fieldsTitles, titles)
             i += 1
         elif myarg in RESCAL_ARGUMENT_TO_PROPERTY_MAP:
             display.add_field_to_csv_file(myarg,
@@ -97,8 +98,8 @@ def printResourceCalendars():
                                           fieldsList, fieldsTitles, titles)
             i += 1
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam print resources")
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam print resources')
     if not fieldsList:
         for field in RESCAL_DFLTFIELDS:
             display.add_field_to_csv_file(field,
@@ -106,15 +107,19 @@ def printResourceCalendars():
                                           fieldsList, fieldsTitles, titles)
     fields = f'nextPageToken,items({",".join(set(fieldsList))})'
     if 'buildingId' in fieldsList:
-        display.add_field_to_csv_file('buildingName', {'buildingName': [
-            'buildingName', ]}, fieldsList, fieldsTitles, titles)
-    __main__.printGettingAllItems('Resource Calendars', None)
+        display.add_field_to_csv_file('buildingName',
+                                      {'buildingName': ['buildingName',]},
+                                      fieldsList, fieldsTitles, titles)
+    gam.printGettingAllItems('Resource Calendars', None)
     page_message = gapi.got_total_items_first_last_msg('Resource Calendars')
-    resources = gapi.get_all_pages(cd.resources().calendars(), 'list',
-                                   'items', page_message=page_message,
+    resources = gapi.get_all_pages(cd.resources().calendars(),
+                                   'list',
+                                   'items',
+                                   page_message=page_message,
                                    message_attribute='resourceId',
                                    customer=GC_Values[GC_CUSTOMER_ID],
-                                   query=query, fields=fields)
+                                   query=query,
+                                   fields=fields)
     for resource in resources:
         if 'featureInstances' in resource:
             features = [a_feature['feature']['name'] for \
@@ -128,41 +133,56 @@ def printResourceCalendars():
         for field in fieldsList:
             resUnit[fieldsTitles[field]] = resource.get(field, '')
         csvRows.append(resUnit)
-    display.sort_csv_titles(
-        ['resourceId', 'resourceName', 'resourceEmail'], titles)
+    display.sort_csv_titles(['resourceId', 'resourceName', 'resourceEmail'],
+                            titles)
     display.write_csv_file(csvRows, titles, 'Resources', todrive)
 
 
-RESCAL_DFLTFIELDS = ['id', 'name', 'email',]
-RESCAL_ALLFIELDS = ['id', 'name', 'email', 'description', 'type',
-                    'buildingid', 'category', 'capacity', 'features', 'floor',
-                    'floorsection', 'generatedresourcename',
-                    'uservisibledescription',]
+RESCAL_DFLTFIELDS = [
+    'id',
+    'name',
+    'email',
+]
+RESCAL_ALLFIELDS = [
+    'id',
+    'name',
+    'email',
+    'description',
+    'type',
+    'buildingid',
+    'category',
+    'capacity',
+    'features',
+    'floor',
+    'floorsection',
+    'generatedresourcename',
+    'uservisibledescription',
+]
 
 RESCAL_ARGUMENT_TO_PROPERTY_MAP = {
     'description': ['resourceDescription'],
-    'building': ['buildingId', ],
-    'buildingid': ['buildingId', ],
-    'capacity': ['capacity', ],
-    'category': ['resourceCategory', ],
+    'building': ['buildingId',],
+    'buildingid': ['buildingId',],
+    'capacity': ['capacity',],
+    'category': ['resourceCategory',],
     'email': ['resourceEmail'],
-    'feature': ['featureInstances', ],
-    'features': ['featureInstances', ],
-    'floor': ['floorName', ],
-    'floorname': ['floorName', ],
-    'floorsection': ['floorSection', ],
-    'generatedresourcename': ['generatedResourceName', ],
+    'feature': ['featureInstances',],
+    'features': ['featureInstances',],
+    'floor': ['floorName',],
+    'floorname': ['floorName',],
+    'floorsection': ['floorSection',],
+    'generatedresourcename': ['generatedResourceName',],
     'id': ['resourceId'],
     'name': ['resourceName'],
     'type': ['resourceType'],
-    'userdescription': ['userVisibleDescription', ],
-    'uservisibledescription': ['userVisibleDescription', ],
+    'userdescription': ['userVisibleDescription',],
+    'uservisibledescription': ['userVisibleDescription',],
 }
 
 
 def printFeatures():
     to_drive = False
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     titles = []
     csvRows = []
     fieldsList = ['name']
@@ -182,15 +202,15 @@ def printFeatures():
         elif myarg in possible_fields:
             fieldsList.append(possible_fields[myarg])
             i += 1
-        elif 'feature'+myarg in possible_fields:
-            fieldsList.append(possible_fields['feature'+myarg])
+        elif 'feature' + myarg in possible_fields:
+            fieldsList.append(possible_fields['feature' + myarg])
             i += 1
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam print features")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam print features')
     if fields:
         fields = fields % ','.join(fieldsList)
-    features = gapi.get_all_pages(cd.resources().features(), 'list',
+    features = gapi.get_all_pages(cd.resources().features(),
+                                  'list',
                                   'features',
                                   customer=GC_Values[GC_CUSTOMER_ID],
                                   fields=fields)
@@ -212,57 +232,62 @@ def _getBuildingAttributes(args, body={}):
     while i < len(args):
         myarg = args[i].lower().replace('_', '')
         if myarg == 'id':
-            body['buildingId'] = args[i+1]
+            body['buildingId'] = args[i + 1]
             i += 2
         elif myarg == 'name':
-            body['buildingName'] = args[i+1]
+            body['buildingName'] = args[i + 1]
             i += 2
         elif myarg in ['lat', 'latitude']:
             if 'coordinates' not in body:
                 body['coordinates'] = {}
-            body['coordinates']['latitude'] = args[i+1]
+            body['coordinates']['latitude'] = args[i + 1]
             i += 2
         elif myarg in ['long', 'lng', 'longitude']:
             if 'coordinates' not in body:
                 body['coordinates'] = {}
-            body['coordinates']['longitude'] = args[i+1]
+            body['coordinates']['longitude'] = args[i + 1]
             i += 2
         elif myarg == 'description':
-            body['description'] = args[i+1]
+            body['description'] = args[i + 1]
             i += 2
         elif myarg == 'floors':
-            body['floorNames'] = args[i+1].split(',')
+            body['floorNames'] = args[i + 1].split(',')
             i += 2
         else:
-            controlflow.invalid_argument_exit(
-                myarg, "gam create|update building")
+            controlflow.invalid_argument_exit(myarg,
+                                              'gam create|update building')
     return body
 
 
 def createBuilding():
-    cd = gapi.directory.buildGAPIObject()
-    body = {'floorNames': ['1'],
-            'buildingId': str(uuid.uuid4()),
-            'buildingName': sys.argv[3]}
+    cd = gapi_directory.buildGAPIObject()
+    body = {
+        'floorNames': ['1'],
+        'buildingId': str(uuid.uuid4()),
+        'buildingName': sys.argv[3]
+    }
     body = _getBuildingAttributes(sys.argv[4:], body)
     print(f'Creating building {body["buildingId"]}...')
-    gapi.call(cd.resources().buildings(), 'insert',
-              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+    gapi.call(cd.resources().buildings(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
 
 
 def _makeBuildingIdNameMap(cd):
     fields = 'nextPageToken,buildings(buildingId,buildingName)'
-    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+    buildings = gapi.get_all_pages(cd.resources().buildings(),
+                                   'list',
                                    'buildings',
                                    customer=GC_Values[GC_CUSTOMER_ID],
                                    fields=fields)
     GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {}
     GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {}
     for building in buildings:
-        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][building['buildingId']
-                                               ] = building['buildingName']
-        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building['buildingName']
-                                               ] = building['buildingId']
+        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][
+            building['buildingId']] = building['buildingName']
+        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][
+            building['buildingName']] = building['buildingId']
 
 
 def getBuildingByNameOrId(cd, which_building, minLen=1):
@@ -282,10 +307,13 @@ def getBuildingByNameOrId(cd, which_building, minLen=1):
     # No exact name match, check for case insensitive name matches
     which_building_lower = which_building.lower()
     ci_matches = []
-    for buildingName, buildingId in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID].items():
+    for buildingName, buildingId in GM_Globals[
+            GM_MAP_BUILDING_NAME_TO_ID].items():
         if buildingName.lower() == which_building_lower:
-            ci_matches.append(
-                {'buildingName': buildingName, 'buildingId': buildingId})
+            ci_matches.append({
+                'buildingName': buildingName,
+                'buildingId': buildingId
+            })
     # One match, return ID
     if len(ci_matches) == 1:
         return ci_matches[0]['buildingId']
@@ -318,19 +346,22 @@ def getBuildingNameById(cd, buildingId):
 
 
 def updateBuilding():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     buildingId = getBuildingByNameOrId(cd, sys.argv[3])
     body = _getBuildingAttributes(sys.argv[4:])
     print(f'Updating building {buildingId}...')
-    gapi.call(cd.resources().buildings(), 'patch',
-              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId,
+    gapi.call(cd.resources().buildings(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              buildingId=buildingId,
               body=body)
 
 
 def getBuildingInfo():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     buildingId = getBuildingByNameOrId(cd, sys.argv[3])
-    building = gapi.call(cd.resources().buildings(), 'get',
+    building = gapi.call(cd.resources().buildings(),
+                         'get',
                          customer=GC_Values[GC_CUSTOMER_ID],
                          buildingId=buildingId)
     if 'buildingId' in building:
@@ -343,11 +374,13 @@ def getBuildingInfo():
 
 
 def deleteBuilding():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     buildingId = getBuildingByNameOrId(cd, sys.argv[3])
     print(f'Deleting building {buildingId}...')
-    gapi.call(cd.resources().buildings(), 'delete',
-              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId)
+    gapi.call(cd.resources().buildings(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              buildingId=buildingId)
 
 
 def _getFeatureAttributes(args, body={}):
@@ -355,41 +388,47 @@ def _getFeatureAttributes(args, body={}):
     while i < len(args):
         myarg = args[i].lower().replace('_', '')
         if myarg == 'name':
-            body['name'] = args[i+1]
+            body['name'] = args[i + 1]
             i += 2
         else:
-            controlflow.invalid_argument_exit(
-                myarg, "gam create|update feature")
+            controlflow.invalid_argument_exit(myarg,
+                                              'gam create|update feature')
     return body
 
 
 def createFeature():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     body = _getFeatureAttributes(sys.argv[3:])
     print(f'Creating feature {body["name"]}...')
-    gapi.call(cd.resources().features(), 'insert',
-              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+    gapi.call(cd.resources().features(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
 
 
 def updateFeature():
     # update does not work for name and name is only field to be updated
     # if additional writable fields are added to feature in the future
     # we'll add support for update as well as rename
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     oldName = sys.argv[3]
     body = {'newName': sys.argv[5:]}
     print(f'Updating feature {oldName}...')
-    gapi.call(cd.resources().features(), 'rename',
-              customer=GC_Values[GC_CUSTOMER_ID], oldName=oldName,
+    gapi.call(cd.resources().features(),
+              'rename',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              oldName=oldName,
               body=body)
 
 
 def deleteFeature():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     featureKey = sys.argv[3]
     print(f'Deleting feature {featureKey}...')
-    gapi.call(cd.resources().features(), 'delete',
-              customer=GC_Values[GC_CUSTOMER_ID], featureKey=featureKey)
+    gapi.call(cd.resources().features(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              featureKey=featureKey)
 
 
 def _getResourceCalendarAttributes(cd, args, body={}):
@@ -397,74 +436,80 @@ def _getResourceCalendarAttributes(cd, args, body={}):
     while i < len(args):
         myarg = args[i].lower().replace('_', '')
         if myarg == 'name':
-            body['resourceName'] = args[i+1]
+            body['resourceName'] = args[i + 1]
             i += 2
         elif myarg == 'description':
-            body['resourceDescription'] = args[i+1].replace('\\n', '\n')
+            body['resourceDescription'] = args[i + 1].replace('\\n', '\n')
             i += 2
         elif myarg == 'type':
-            body['resourceType'] = args[i+1]
+            body['resourceType'] = args[i + 1]
             i += 2
         elif myarg in ['building', 'buildingid']:
-            body['buildingId'] = getBuildingByNameOrId(
-                cd, args[i+1], minLen=0)
+            body['buildingId'] = getBuildingByNameOrId(cd,
+                                                       args[i + 1],
+                                                       minLen=0)
             i += 2
         elif myarg in ['capacity']:
-            body['capacity'] = __main__.getInteger(args[i+1], myarg, minVal=0)
+            body['capacity'] = gam.getInteger(args[i + 1], myarg, minVal=0)
             i += 2
         elif myarg in ['feature', 'features']:
-            features = args[i+1].split(',')
+            features = args[i + 1].split(',')
             body['featureInstances'] = []
             for feature in features:
                 instance = {'feature': {'name': feature}}
                 body['featureInstances'].append(instance)
             i += 2
         elif myarg in ['floor', 'floorname']:
-            body['floorName'] = args[i+1]
+            body['floorName'] = args[i + 1]
             i += 2
         elif myarg in ['floorsection']:
-            body['floorSection'] = args[i+1]
+            body['floorSection'] = args[i + 1]
             i += 2
         elif myarg in ['category']:
-            body['resourceCategory'] = args[i+1].upper()
+            body['resourceCategory'] = args[i + 1].upper()
             if body['resourceCategory'] == 'ROOM':
                 body['resourceCategory'] = 'CONFERENCE_ROOM'
             i += 2
         elif myarg in ['uservisibledescription', 'userdescription']:
-            body['userVisibleDescription'] = args[i+1]
+            body['userVisibleDescription'] = args[i + 1]
             i += 2
         else:
-            controlflow.invalid_argument_exit(
-                args[i], "gam create|update resource")
+            controlflow.invalid_argument_exit(args[i],
+                                              'gam create|update resource')
     return body
 
 
 def createResourceCalendar():
-    cd = gapi.directory.buildGAPIObject()
-    body = {'resourceId': sys.argv[3],
-            'resourceName': sys.argv[4]}
+    cd = gapi_directory.buildGAPIObject()
+    body = {'resourceId': sys.argv[3], 'resourceName': sys.argv[4]}
     body = _getResourceCalendarAttributes(cd, sys.argv[5:], body)
     print(f'Creating resource {body["resourceId"]}...')
-    gapi.call(cd.resources().calendars(), 'insert',
-              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+    gapi.call(cd.resources().calendars(),
+              'insert',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              body=body)
 
 
 def updateResourceCalendar():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     resId = sys.argv[3]
     body = _getResourceCalendarAttributes(cd, sys.argv[4:])
     # Use patch since it seems to work better.
     # update requires name to be set.
-    gapi.call(cd.resources().calendars(), 'patch',
-              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId,
-              body=body, fields='')
+    gapi.call(cd.resources().calendars(),
+              'patch',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              calendarResourceId=resId,
+              body=body,
+              fields='')
     print(f'updated resource {resId}')
 
 
 def getResourceCalendarInfo():
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     resId = sys.argv[3]
-    resource = gapi.call(cd.resources().calendars(), 'get',
+    resource = gapi.call(cd.resources().calendars(),
+                         'get',
                          customer=GC_Values[GC_CUSTOMER_ID],
                          calendarResourceId=resId)
     if 'featureInstances' in resource:
@@ -473,15 +518,17 @@ def getResourceCalendarInfo():
             features.append(a_feature['feature']['name'])
         resource['features'] = ', '.join(features)
     if 'buildingId' in resource:
-        resource['buildingName'] = getBuildingNameById(
-            cd, resource['buildingId'])
+        resource['buildingName'] = getBuildingNameById(cd,
+                                                       resource['buildingId'])
         resource['buildingId'] = f'id:{resource["buildingId"]}'
     display.print_json(resource)
 
 
 def deleteResourceCalendar():
     resId = sys.argv[3]
-    cd = gapi.directory.buildGAPIObject()
+    cd = gapi_directory.buildGAPIObject()
     print(f'Deleting resource calendar {resId}')
-    gapi.call(cd.resources().calendars(), 'delete',
-              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId)
+    gapi.call(cd.resources().calendars(),
+              'delete',
+              customer=GC_Values[GC_CUSTOMER_ID],
+              calendarResourceId=resId)

src/gam/gapi/errors.py

@@ -0,0 +1,379 @@
+"""GAPI and OAuth Token related errors methods."""
+
+from enum import Enum
+import json
+
+from gam import controlflow
+from gam import display
+from gam.var import UTF8
+
+
+class GapiAbortedError(Exception):
+    pass
+
+
+class GapiAuthErrorError(Exception):
+    pass
+
+
+class GapiBadGatewayError(Exception):
+    pass
+
+
+class GapiBadRequestError(Exception):
+    pass
+
+
+class GapiConditionNotMetError(Exception):
+    pass
+
+
+class GapiCyclicMembershipsNotAllowedError(Exception):
+    pass
+
+
+class GapiDomainCannotUseApisError(Exception):
+    pass
+
+
+class GapiDomainNotFoundError(Exception):
+    pass
+
+
+class GapiDuplicateError(Exception):
+    pass
+
+
+class GapiFailedPreconditionError(Exception):
+    pass
+
+
+class GapiForbiddenError(Exception):
+    pass
+
+
+class GapiGatewayTimeoutError(Exception):
+    pass
+
+
+class GapiGroupNotFoundError(Exception):
+    pass
+
+
+class GapiInvalidError(Exception):
+    pass
+
+
+class GapiInvalidArgumentError(Exception):
+    pass
+
+
+class GapiInvalidMemberError(Exception):
+    pass
+
+
+class GapiMemberNotFoundError(Exception):
+    pass
+
+
+class GapiNotFoundError(Exception):
+    pass
+
+
+class GapiNotImplementedError(Exception):
+    pass
+
+
+class GapiPermissionDeniedError(Exception):
+    pass
+
+
+class GapiResourceNotFoundError(Exception):
+    pass
+
+
+class GapiServiceNotAvailableError(Exception):
+    pass
+
+
+class GapiUserNotFoundError(Exception):
+    pass
+
+
+# GAPI Error Reasons
+class ErrorReason(Enum):
+    """The reason why a non-200 HTTP response was returned from a GAPI."""
+    ABORTED = 'aborted'
+    AUTH_ERROR = 'authError'
+    BACKEND_ERROR = 'backendError'
+    BAD_GATEWAY = 'badGateway'
+    BAD_REQUEST = 'badRequest'
+    CONDITION_NOT_MET = 'conditionNotMet'
+    CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
+    DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
+    DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
+    DOMAIN_NOT_FOUND = 'domainNotFound'
+    DUPLICATE = 'duplicate'
+    FAILED_PRECONDITION = 'failedPrecondition'
+    FORBIDDEN = 'forbidden'
+    FOUR_O_NINE = '409'
+    FOUR_O_THREE = '403'
+    FOUR_TWO_NINE = '429'
+    GATEWAY_TIMEOUT = 'gatewayTimeout'
+    GROUP_NOT_FOUND = 'groupNotFound'
+    INTERNAL_ERROR = 'internalError'
+    INVALID = 'invalid'
+    INVALID_ARGUMENT = 'invalidArgument'
+    INVALID_MEMBER = 'invalidMember'
+    MEMBER_NOT_FOUND = 'memberNotFound'
+    NOT_FOUND = 'notFound'
+    NOT_IMPLEMENTED = 'notImplemented'
+    PERMISSION_DENIED = 'permissionDenied'
+    QUOTA_EXCEEDED = 'quotaExceeded'
+    RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
+    RESOURCE_NOT_FOUND = 'resourceNotFound'
+    SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+    SERVICE_LIMIT = 'serviceLimit'
+    SYSTEM_ERROR = 'systemError'
+    USER_NOT_FOUND = 'userNotFound'
+    USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+
+    def __str__(self):
+        return str(self.value)
+
+
+# Common sets of GAPI error reasons
+DEFAULT_RETRY_REASONS = [
+    ErrorReason.QUOTA_EXCEEDED,
+    ErrorReason.RATE_LIMIT_EXCEEDED,
+    ErrorReason.USER_RATE_LIMIT_EXCEEDED,
+    ErrorReason.BACKEND_ERROR,
+    ErrorReason.BAD_GATEWAY,
+    ErrorReason.GATEWAY_TIMEOUT,
+    ErrorReason.INTERNAL_ERROR,
+    ErrorReason.FOUR_TWO_NINE,
+]
+GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
+GROUP_GET_THROW_REASONS = [
+    ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
+    ErrorReason.BAD_REQUEST
+]
+GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
+MEMBERS_THROW_REASONS = [
+    ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
+    ErrorReason.FORBIDDEN
+]
+MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
+
+# A map of GAPI error reasons to the corresponding GAM Python Exception
+ERROR_REASON_TO_EXCEPTION = {
+    ErrorReason.ABORTED:
+        GapiAbortedError,
+    ErrorReason.AUTH_ERROR:
+        GapiAuthErrorError,
+    ErrorReason.BAD_GATEWAY:
+        GapiBadGatewayError,
+    ErrorReason.BAD_REQUEST:
+        GapiBadRequestError,
+    ErrorReason.CONDITION_NOT_MET:
+        GapiConditionNotMetError,
+    ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
+        GapiCyclicMembershipsNotAllowedError,
+    ErrorReason.DOMAIN_CANNOT_USE_APIS:
+        GapiDomainCannotUseApisError,
+    ErrorReason.DOMAIN_NOT_FOUND:
+        GapiDomainNotFoundError,
+    ErrorReason.DUPLICATE:
+        GapiDuplicateError,
+    ErrorReason.FAILED_PRECONDITION:
+        GapiFailedPreconditionError,
+    ErrorReason.FORBIDDEN:
+        GapiForbiddenError,
+    ErrorReason.GATEWAY_TIMEOUT:
+        GapiGatewayTimeoutError,
+    ErrorReason.GROUP_NOT_FOUND:
+        GapiGroupNotFoundError,
+    ErrorReason.INVALID:
+        GapiInvalidError,
+    ErrorReason.INVALID_ARGUMENT:
+        GapiInvalidArgumentError,
+    ErrorReason.INVALID_MEMBER:
+        GapiInvalidMemberError,
+    ErrorReason.MEMBER_NOT_FOUND:
+        GapiMemberNotFoundError,
+    ErrorReason.NOT_FOUND:
+        GapiNotFoundError,
+    ErrorReason.NOT_IMPLEMENTED:
+        GapiNotImplementedError,
+    ErrorReason.PERMISSION_DENIED:
+        GapiPermissionDeniedError,
+    ErrorReason.RESOURCE_NOT_FOUND:
+        GapiResourceNotFoundError,
+    ErrorReason.SERVICE_NOT_AVAILABLE:
+        GapiServiceNotAvailableError,
+    ErrorReason.USER_NOT_FOUND:
+        GapiUserNotFoundError,
+}
+
+# OAuth Token Errors
+OAUTH2_TOKEN_ERRORS = [
+    'access_denied',
+    'access_denied: Requested client not authorized',
+    'internal_failure: Backend Error',
+    'internal_failure: None',
+    'invalid_grant',
+    'invalid_grant: Bad Request',
+    'invalid_grant: Invalid email or User ID',
+    'invalid_grant: Not a valid email',
+    'invalid_grant: Invalid JWT: No valid verifier found for issuer',
+    'invalid_grant: The account has been deleted',
+    'invalid_grant: reauth related error (invalid_rapt)',
+    'invalid_request: Invalid impersonation prn email address',
+    'invalid_request: Invalid impersonation "sub" field',
+    'unauthorized_client: Client is unauthorized to retrieve access tokens '
+    'using this method',
+    'unauthorized_client: Client is unauthorized to retrieve access tokens '
+    'using this method, or client not authorized for any of the scopes '
+    'requested',
+    'unauthorized_client: Unauthorized client or scope in request',
+]
+
+
+def _create_http_error_dict(status_code, reason, message):
+    """Creates a basic error dict similar to most Google API Errors.
+
+  Args:
+    status_code: Int, the error's HTTP response status code.
+    reason: String, a camelCase reason for the HttpError being given.
+    message: String, a general error message describing the error that occurred.
+
+  Returns:
+    dict
+  """
+    return {
+        'error': {
+            'code': status_code,
+            'errors': [{
+                'reason': str(reason),
+                'message': message,
+            }]
+        }
+    }
+
+
+def get_gapi_error_detail(e,
+                          soft_errors=False,
+                          silent_errors=False,
+                          retry_on_http_error=False):
+    """Extracts error detail from a non-200 GAPI Response.
+
+  Args:
+    e: googleapiclient.HttpError, The HTTP Error received.
+    soft_errors: Boolean, If true, causes error messages to be surpressed,
+      rather than sending them to stderr.
+    silent_errors: Boolean, If true, suppresses and ignores any errors from
+      being displayed
+    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
+      code, indicating that the request can be retried. TODO: Remove this param,
+        as it seems to be outside the scope of this method.
+
+  Returns:
+    A tuple containing the HTTP Response code, GAPI error reason, and error
+        message.
+  """
+    try:
+        error = json.loads(e.content.decode(UTF8))
+    except ValueError:
+        error_content = e.content.decode(UTF8) if isinstance(
+            e.content, bytes) else e.content
+        if (e.resp['status'] == '503') and (
+                error_content == 'Quota exceeded for the current request'):
+            return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value,
+                    error_content)
+        if (e.resp['status'] == '403') and (error_content.startswith(
+                'Request rate higher than configured')):
+            return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value,
+                    error_content)
+        if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
+            return (e.resp['status'], ErrorReason.BAD_GATEWAY.value,
+                    error_content)
+        if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
+            return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value,
+                    error_content)
+        if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
+            error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
+                                            'Domain not found')
+        elif (e.resp['status'] == '400') and (
+                'InvalidSsoSigningKey' in error_content):
+            error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                            'InvalidSsoSigningKey')
+        elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
+            error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                            'UnknownError')
+        elif retry_on_http_error:
+            return (-1, None, None)
+        elif soft_errors:
+            if not silent_errors:
+                display.print_error(error_content)
+            return (0, None, None)
+        else:
+            controlflow.system_error_exit(5, error_content)
+        # END: ValueError catch
+
+    if 'error' in error:
+        http_status = error['error']['code']
+        try:
+            message = error['error']['errors'][0]['message']
+        except KeyError:
+            message = error['error']['message']
+        if http_status == 404:
+            if 'Requested entity was not found' in message or 'does not exist' in message:
+                error = _create_http_error_dict(404, ErrorReason.NOT_FOUND.value,
+                                                message)
+    else:
+        if 'error_description' in error:
+            if error['error_description'] == 'Invalid Value':
+                message = error['error_description']
+                http_status = 400
+                error = _create_http_error_dict(400, ErrorReason.INVALID.value,
+                                                message)
+            else:
+                controlflow.system_error_exit(4, str(error))
+        else:
+            controlflow.system_error_exit(4, str(error))
+
+    # Extract the error reason
+    try:
+        reason = error['error']['errors'][0]['reason']
+        if reason == 'notFound':
+            if 'userKey' in message:
+                reason = ErrorReason.USER_NOT_FOUND.value
+            elif 'groupKey' in message:
+                reason = ErrorReason.GROUP_NOT_FOUND.value
+            elif 'memberKey' in message:
+                reason = ErrorReason.MEMBER_NOT_FOUND.value
+            elif 'Domain not found' in message:
+                reason = ErrorReason.DOMAIN_NOT_FOUND.value
+            elif 'Resource Not Found' in message:
+                reason = ErrorReason.RESOURCE_NOT_FOUND.value
+        elif reason == 'invalid':
+            if 'userId' in message:
+                reason = ErrorReason.USER_NOT_FOUND.value
+            elif 'memberKey' in message:
+                reason = ErrorReason.INVALID_MEMBER.value
+        elif reason == 'failedPrecondition':
+            if 'Bad Request' in message:
+                reason = ErrorReason.BAD_REQUEST.value
+            elif 'Mail service not enabled' in message:
+                reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
+        elif reason == 'required':
+            if 'memberKey' in message:
+                reason = ErrorReason.MEMBER_NOT_FOUND.value
+        elif reason == 'conditionNotMet':
+            if 'Cyclic memberships not allowed' in message:
+                reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
+    except KeyError:
+        reason = f'{http_status}'
+    return (http_status, reason, message)

src/gam/gapi/errors_test.py

@@ -0,0 +1,210 @@
+"""Python unit tests for gapi.errors"""
+
+import json
+
+import unittest
+from unittest.mock import patch
+
+import googleapiclient.errors
+from gam.gapi import errors
+
+
+def create_simple_http_error(status, reason, message):
+    content = errors._create_http_error_dict(status, reason, message)
+    return create_http_error(status, content)
+
+
+def create_http_error(status, content):
+    response = {
+        'status': status,
+        'content-type': 'application/json',
+    }
+    content_as_bytes = json.dumps(content).encode('UTF-8')
+    return googleapiclient.errors.HttpError(response, content_as_bytes)
+
+
+class ErrorsTest(unittest.TestCase):
+
+    def test_get_gapi_error_detail_quota_exceeded(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_invalid_domain(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_invalid_signing_key(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_detail_unknown_error(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_retry_http_error(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_prints_soft_errors(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_exits_on_unrecoverable_errors(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_quota_exceeded_for_current_request(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_quota_exceeded_high_request_rate(self):
+        # TODO: Add test logic once the opening ValueError exception case has a
+        # repro case (i.e. an Exception type/format that will cause it to raise).
+        pass
+
+    def test_get_gapi_error_extracts_user_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: userKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: userKey.')
+
+    def test_get_gapi_error_extracts_group_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: groupKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: groupKey.')
+
+    def test_get_gapi_error_extracts_member_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: memberKey.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: memberKey.')
+
+    def test_get_gapi_error_extracts_domain_not_found(self):
+        err = create_simple_http_error(404, 'notFound', 'Domain not found.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
+        self.assertEqual(message, 'Domain not found.')
+
+    def test_get_gapi_error_extracts_generic_resource_not_found(self):
+        err = create_simple_http_error(404, 'notFound',
+                                       'Resource Not Found: unknownResource.')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 404)
+        self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
+        self.assertEqual(message, 'Resource Not Found: unknownResource.')
+
+    def test_get_gapi_error_extracts_invalid_userid(self):
+        err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
+        self.assertEqual(message, 'Invalid Input: userId')
+
+    def test_get_gapi_error_extracts_invalid_member(self):
+        err = create_simple_http_error(400, 'invalid',
+                                       'Invalid Input: memberKey')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
+        self.assertEqual(message, 'Invalid Input: memberKey')
+
+    def test_get_gapi_error_extracts_bad_request(self):
+        err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
+        self.assertEqual(message, 'Bad Request')
+
+    def test_get_gapi_error_extracts_service_not_available(self):
+        err = create_simple_http_error(400, 'failedPrecondition',
+                                       'Mail service not enabled')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
+        self.assertEqual(message, 'Mail service not enabled')
+
+    def test_get_gapi_error_extracts_required_member_not_found(self):
+        err = create_simple_http_error(400, 'required',
+                                       'Missing required field: memberKey')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
+        self.assertEqual(message, 'Missing required field: memberKey')
+
+    def test_get_gapi_error_extracts_cyclic_memberships_error(self):
+        err = create_simple_http_error(400, 'conditionNotMet',
+                                       'Cyclic memberships not allowed')
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, 400)
+        self.assertEqual(
+            reason, errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
+        self.assertEqual(message, 'Cyclic memberships not allowed')
+
+    def test_get_gapi_error_extracts_single_error_with_message(self):
+        status_code = 999
+        response = {'status': status_code}
+        # This error does not have an "errors" key describing each error.
+        content = {'error': {'code': status_code, 'message': 'unknown error'}}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, status_code)
+        self.assertEqual(reason, str(status_code))
+        self.assertEqual(message, content['error']['message'])
+
+    def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
+        self):
+        status_code = 999
+        response = {'status': status_code}
+        # This error only has an error_description_field and an unknown description.
+        content = {'error_description': 'something errored'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        with self.assertRaises(SystemExit) as context:
+            errors.get_gapi_error_detail(err)
+        self.assertEqual(4, context.exception.code)
+
+    def test_get_gapi_error_exits_on_invalid_error_description(self):
+        status_code = 400
+        response = {'status': status_code}
+        content = {'error_description': 'Invalid Value'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        http_status, reason, message = errors.get_gapi_error_detail(err)
+        self.assertEqual(http_status, status_code)
+        self.assertEqual(reason, errors.ErrorReason.INVALID.value)
+        self.assertEqual(message, 'Invalid Value')
+
+    def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
+        status_code = 900
+        response = {'status': status_code}
+        content = {'notErrorContentThatIsExpected': 'foo'}
+        content_as_bytes = json.dumps(content).encode('UTF-8')
+        err = googleapiclient.errors.HttpError(response, content_as_bytes)
+
+        with self.assertRaises(SystemExit) as context:
+            errors.get_gapi_error_detail(err)
+        self.assertEqual(4, context.exception.code)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/gam/gapi/reports.py

@@ -1,20 +1,20 @@
 import calendar
 import datetime
+import re
 import sys
 
-from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
 
-import __main__
-from var import *
-import controlflow
-import display
-import gapi
-import utils
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import gapi
+from gam import utils
 
 
 def buildGAPIObject():
-    return __main__.buildGAPIObject('reports')
+    return gam.buildGAPIObject('reports')
 
 
 REPORT_CHOICE_MAP = {
@@ -42,28 +42,44 @@ REPORT_CHOICE_MAP = {
 
 def showUsageParameters():
     rep = buildGAPIObject()
-    throw_reasons = [gapi.errors.ErrorReason.INVALID,
-                     gapi.errors.ErrorReason.BAD_REQUEST]
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.BAD_REQUEST
+    ]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit('user or customer',
+                                          'report usageparameters')
     report = sys.argv[3].lower()
+    titles = ['parameter']
     if report == 'customer':
         endpoint = rep.customerUsageReports()
         kwargs = {}
     elif report == 'user':
         endpoint = rep.userUsageReport()
-        kwargs = {'userKey': __main__._getValueFromOAuth('email')}
+        kwargs = {'userKey': gam._getValueFromOAuth('email')}
     else:
-        controlflow.expected_argument_exit(
-            'usageparameters', ['user', 'customer'], report)
+        controlflow.expected_argument_exit('usageparameters',
+                                           ['user', 'customer'], report)
     customerId = GC_Values[GC_CUSTOMER_ID]
     if customerId == MY_CUSTOMER:
         customerId = None
     tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
-    partial_apps = False
+    partial_apps = []
     all_parameters = []
     one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam report usageparameters')
     while True:
         try:
-            response = gapi.call(endpoint, 'get',
+            response = gapi.call(endpoint,
+                                 'get',
                                  throw_reasons=throw_reasons,
                                  date=tryDate,
                                  customerId=customerId,
@@ -74,7 +90,9 @@ def showUsageParameters():
                     if data.get('key') == 'application':
                         partial_on_thisday.append(data['value'])
             if partial_apps:
-                partial_apps = [app for app in partial_apps if app in partial_on_thisday]
+                partial_apps = [
+                    app for app in partial_apps if app in partial_on_thisday
+                ]
             else:
                 partial_apps = partial_on_thisday
             for parameter in response['usageReports'][0]['parameters']:
@@ -88,15 +106,27 @@ def showUsageParameters():
         except gapi.errors.GapiInvalidError as e:
             tryDate = _adjust_date(str(e))
     all_parameters.sort()
+    csvRows = []
     for parameter in all_parameters:
-        print(parameter)
+        csvRows.append({'parameter': parameter})
+    display.write_csv_file(csvRows, titles,
+                           f'{report.capitalize()} Report Usage Parameters',
+                           todrive)
+
+
+REPORTS_PARAMETERS_SIMPLE_TYPES = [
+    'intValue', 'boolValue', 'datetimeValue', 'stringValue'
+]
 
 
 def showUsage():
     rep = buildGAPIObject()
-    throw_reasons = [gapi.errors.ErrorReason.INVALID,
-                     gapi.errors.ErrorReason.BAD_REQUEST]
+    throw_reasons = [
+        gapi.errors.ErrorReason.INVALID, gapi.errors.ErrorReason.BAD_REQUEST
+    ]
     todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit('user or customer', 'report usage')
     report = sys.argv[3].lower()
     titles = ['date']
     if report == 'customer':
@@ -107,84 +137,96 @@ def showUsage():
         kwargs = [{'userKey': 'all'}]
         titles.append('user')
     else:
-        controlflow.expected_argument_exit(
-            'usage', ['user', 'customer'], report)
+        controlflow.expected_argument_exit('usage', ['user', 'customer'],
+                                           report)
     customerId = GC_Values[GC_CUSTOMER_ID]
     if customerId == MY_CUSTOMER:
         customerId = None
     parameters = []
-    filters = None
     start_date = end_date = orgUnitId = None
     skip_day_numbers = []
-    skip_dates = []
+    skip_dates = set()
+    one_day = datetime.timedelta(days=1)
     i = 4
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'startdate':
-            start_date = parse(sys.argv[i+1])
+            start_date = utils.get_yyyymmdd(sys.argv[i + 1],
+                                            returnDateTime=True)
             i += 2
         elif myarg == 'enddate':
-            end_date = parse(sys.argv[i+1])
+            end_date = utils.get_yyyymmdd(sys.argv[i + 1], returnDateTime=True)
             i += 2
         elif myarg == 'todrive':
             todrive = True
             i += 1
-        elif myarg in ['orgunit', 'org', 'ou']:
-            if report != 'user':
-                controlflow.invalid_argument_exit(myarg, f'gam usage {report}')
-            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
-            i += 2
-        elif myarg == 'parameters':
-            parameters = sys.argv[i+1].split(',')
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i + 1].split(',')
             i += 2
         elif myarg == 'skipdates':
-            skips = sys.argv[i+1].split(',')
-            skip_dates = [utils.get_yyyymmdd(d) for d in skips]
+            for skip in sys.argv[i + 1].split(','):
+                if skip.find(':') == -1:
+                    skip_dates.add(utils.get_yyyymmdd(skip,
+                                                      returnDateTime=True))
+                else:
+                    skip_start, skip_end = skip.split(':', 1)
+                    skip_start = utils.get_yyyymmdd(skip_start,
+                                                    returnDateTime=True)
+                    skip_end = utils.get_yyyymmdd(skip_end, returnDateTime=True)
+                    while skip_start <= skip_end:
+                        skip_dates.add(skip_start)
+                        skip_start += one_day
             i += 2
         elif myarg == 'skipdaysofweek':
-            skipdaynames = sys.argv[i+1].split(',')
+            skipdaynames = sys.argv[i + 1].split(',')
             dow = [d.lower() for d in calendar.day_abbr]
             skip_day_numbers = [dow.index(d) for d in skipdaynames if d in dow]
             i += 2
-        elif myarg in usergroup_types:
-            if report != 'user':
-                controlflow.invalid_argument_exit(myarg, f'gam usage {report}')
-            entity_type = myarg
-            entity = sys.argv[i+1]
-            users = __main__.getUsersToModify(entity_type, entity)
+        elif report == 'user' and myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = gam.getOrgUnitId(sys.argv[i + 1])
+            i += 2
+        elif report == 'user' and myarg in usergroup_types:
+            users = gam.getUsersToModify(myarg, sys.argv[i + 1])
             kwargs = [{'userKey': user} for user in users]
-            i += 3
+            i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam usage")
-    if not start_date:
-        start_date = datetime.datetime.now() + relativedelta(months=-1)
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              f'gam report usage {report}')
+    if parameters:
+        titles.extend(parameters)
+        parameters = ','.join(parameters)
+    else:
+        parameters = None
     if not end_date:
         end_date = datetime.datetime.now()
+    if not start_date:
+        start_date = end_date + relativedelta(months=-1)
     if orgUnitId:
-        for i in range(len(kwargs)):
-            kwargs[i-1]['orgUnitID'] = orgUnitId
-    one_day = datetime.timedelta(days=1)
+        for kw in kwargs:
+            kw['orgUnitID'] = orgUnitId
     usage_on_date = start_date
-    titles.extend(parameters)
+    start_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+    usage_end_date = end_date
+    end_date = end_date.strftime(YYYYMMDD_FORMAT)
+    start_use_date = end_use_date = None
     csvRows = []
-    vtypes = ['intValue', 'stringValue', 'intValue',
-              'boolValue', 'datetimeValue']
-    while usage_on_date <= end_date:
-        use_date = usage_on_date.strftime('%Y-%m-%d')
+    while usage_on_date <= usage_end_date:
         if usage_on_date.weekday() in skip_day_numbers or \
-           use_date in skip_dates:
+           usage_on_date in skip_dates:
             usage_on_date += one_day
             continue
+        use_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
         usage_on_date += one_day
         try:
             for kwarg in kwargs:
                 try:
-                    usage = gapi.get_all_pages(endpoint, 'get',
+                    usage = gapi.get_all_pages(endpoint,
+                                               'get',
                                                'usageReports',
                                                throw_reasons=throw_reasons,
                                                customerId=customerId,
                                                date=use_date,
-                                               parameters=','.join(parameters),
+                                               parameters=parameters,
                                                **kwarg)
                 except gapi.errors.GapiBadRequestError:
                     continue
@@ -192,7 +234,7 @@ def showUsage():
                     row = {'date': use_date}
                     if 'userEmail' in entity['entity']:
                         row['user'] = entity['entity']['userEmail']
-                    for item in entity['parameters']:
+                    for item in entity.get('parameters', []):
                         if 'name' not in item:
                             continue
                         name = item['name']
@@ -204,16 +246,26 @@ def showUsage():
                                     titles.append(column_name)
                                 row[column_name] = cros_ver['num_devices']
                         else:
-                            for vtype in vtypes:
-                                if vtype in item:
-                                    value = item[vtype]
+                            if not name in titles:
+                                titles.append(name)
+                            for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                                if ptype in item:
+                                    row[name] = item[ptype]
                                     break
-                            row[name] = value
+                            else:
+                                row[name] = ''
+                    if not start_use_date:
+                        start_use_date = use_date
+                    end_use_date = use_date
                     csvRows.append(row)
-        except gapi.errors.GapiInvalidError:
-            continue
-    display.write_csv_file(
-        csvRows, titles, f'Usage Reports', todrive)
+        except gapi.errors.GapiInvalidError as e:
+            display.print_warning(str(e))
+            break
+    if start_use_date:
+        report_name = f'{report.capitalize()} Usage Report - {start_use_date}:{end_use_date}'
+    else:
+        report_name = f'{report.capitalize()} Usage Report - {start_date}:{end_date} - No Data'
+    display.write_csv_file(csvRows, titles, report_name, todrive)
 
 
 def showReport():
@@ -222,17 +274,18 @@ def showReport():
     report = sys.argv[2].lower()
     report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
     if report == 'usage':
-      showUsage()
-      return
+        showUsage()
+        return
     if report == 'usageparameters':
-      showUsageParameters()
-      return
+        showUsageParameters()
+        return
     valid_apps = gapi.get_enum_values_minus_unspecified(
-        rep._rootDesc['resources']['activities']['methods']['list'][
-            'parameters']['applicationName']['enum'])+['customer', 'user']
+        rep._rootDesc['resources']['activities']['methods']['list']
+        ['parameters']['applicationName']['enum']) + ['customer', 'user']
     if report not in valid_apps:
-        controlflow.expected_argument_exit(
-            "report", ", ".join(sorted(valid_apps)), report)
+        controlflow.expected_argument_exit('report',
+                                           ', '.join(sorted(valid_apps)),
+                                           report)
     customerId = GC_Values[GC_CUSTOMER_ID]
     if customerId == MY_CUSTOMER:
         customerId = None
@@ -245,67 +298,74 @@ def showReport():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower()
         if myarg == 'date':
-            tryDate = utils.get_yyyymmdd(sys.argv[i+1])
+            tryDate = utils.get_yyyymmdd(sys.argv[i + 1])
             i += 2
         elif myarg in ['orgunit', 'org', 'ou']:
-            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            _, orgUnitId = gam.getOrgUnitId(sys.argv[i + 1])
             i += 2
         elif myarg == 'fulldatarequired':
             fullDataRequired = []
-            fdr = sys.argv[i+1].lower()
-            if fdr and fdr != 'all':
+            fdr = sys.argv[i + 1].lower()
+            if fdr and fdr == 'all':
+                fullDataRequired = 'all'
+            else:
                 fullDataRequired = fdr.replace(',', ' ').split()
             i += 2
         elif myarg == 'start':
-            startTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            startTime = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'end':
-            endTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            endTime = utils.get_time_or_delta_from_now(sys.argv[i + 1])
             i += 2
         elif myarg == 'event':
-            eventName = sys.argv[i+1]
+            eventName = sys.argv[i + 1]
             i += 2
         elif myarg == 'user':
-            userKey = __main__.normalizeEmailAddressOrUID(sys.argv[i+1])
+            userKey = gam.normalizeEmailAddressOrUID(sys.argv[i + 1])
             i += 2
         elif myarg in ['filter', 'filters']:
-            filters = sys.argv[i+1]
+            filters = sys.argv[i + 1]
             i += 2
         elif myarg in ['fields', 'parameters']:
-            parameters = sys.argv[i+1]
+            parameters = sys.argv[i + 1]
             i += 2
         elif myarg == 'ip':
-            actorIpAddress = sys.argv[i+1]
+            actorIpAddress = sys.argv[i + 1]
             i += 2
         elif myarg == 'todrive':
             to_drive = True
             i += 1
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam report")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam report')
     if report == 'user':
         while True:
             try:
-                if fullDataRequired is not None:
-                    warnings = gapi.get_items(rep.userUsageReport(), 'get',
-                                              'warnings',
-                                              throw_reasons=throw_reasons,
-                                              date=tryDate, userKey=userKey,
-                                              customerId=customerId,
-                                              orgUnitID=orgUnitId,
-                                              fields='warnings')
-                    fullData, tryDate = _check_full_data_available(
-                        warnings, tryDate, fullDataRequired)
-                    if fullData < 0:
-                        print('No user report available.')
-                        sys.exit(1)
-                    if fullData == 0:
-                        continue
+                one_page = gapi.call(rep.userUsageReport(),
+                                     'get',
+                                     throw_reasons=throw_reasons,
+                                     date=tryDate,
+                                     userKey=userKey,
+                                     customerId=customerId,
+                                     orgUnitID=orgUnitId,
+                                     fields='warnings,usageReports',
+                                     maxResults=1)
+                warnings = one_page.get('warnings', [])
+                has_reports = bool(one_page.get('usageReports'))
+                fullData, tryDate = _check_full_data_available(
+                    warnings, tryDate, fullDataRequired, has_reports)
+                if fullData < 0:
+                    print('No user report available.')
+                    sys.exit(1)
+                if fullData == 0:
+                    continue
                 page_message = gapi.got_total_items_msg('Users', '...\n')
-                usage = gapi.get_all_pages(rep.userUsageReport(), 'get',
+                usage = gapi.get_all_pages(rep.userUsageReport(),
+                                           'get',
                                            'usageReports',
                                            page_message=page_message,
                                            throw_reasons=throw_reasons,
-                                           date=tryDate, userKey=userKey,
+                                           date=tryDate,
+                                           userKey=userKey,
                                            customerId=customerId,
                                            orgUnitID=orgUnitId,
                                            filters=filters,
@@ -318,45 +378,45 @@ def showReport():
             sys.exit(1)
         titles = ['email', 'date']
         csvRows = []
-        ptypes = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
         for user_report in usage:
             if 'entity' not in user_report:
                 continue
-            row = {'email': user_report['entity']
-                            ['userEmail'], 'date': tryDate}
+            row = {'email': user_report['entity']['userEmail'], 'date': tryDate}
             for item in user_report.get('parameters', []):
                 if 'name' not in item:
                     continue
                 name = item['name']
                 if not name in titles:
                     titles.append(name)
-                for ptype in ptypes:
+                for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
                     if ptype in item:
                         row[name] = item[ptype]
                         break
                 else:
                     row[name] = ''
             csvRows.append(row)
-        display.write_csv_file(
-            csvRows, titles, f'User Reports - {tryDate}', to_drive)
+        display.write_csv_file(csvRows, titles, f'User Reports - {tryDate}',
+                               to_drive)
     elif report == 'customer':
         while True:
             try:
-                if fullDataRequired is not None:
-                    warnings = gapi.get_items(rep.customerUsageReports(),
-                                              'get', 'warnings',
-                                              throw_reasons=throw_reasons,
-                                              customerId=customerId,
-                                              date=tryDate,
-                                              fields='warnings')
-                    fullData, tryDate = _check_full_data_available(
-                        warnings, tryDate, fullDataRequired)
-                    if fullData < 0:
-                        print('No customer report available.')
-                        sys.exit(1)
-                    if fullData == 0:
-                        continue
-                usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                first_page = gapi.call(rep.customerUsageReports(),
+                                       'get',
+                                       throw_reasons=throw_reasons,
+                                       customerId=customerId,
+                                       date=tryDate,
+                                       fields='warnings,usageReports')
+                warnings = first_page.get('warnings', [])
+                has_reports = bool(first_page.get('usageReports'))
+                fullData, tryDate = _check_full_data_available(
+                    warnings, tryDate, fullDataRequired, has_reports)
+                if fullData < 0:
+                    print('No customer report available.')
+                    sys.exit(1)
+                if fullData == 0:
+                    continue
+                usage = gapi.get_all_pages(rep.customerUsageReports(),
+                                           'get',
                                            'usageReports',
                                            throw_reasons=throw_reasons,
                                            customerId=customerId,
@@ -405,27 +465,32 @@ def showReport():
                         value = ' '.join(values)
                     elif 'version_number' in subitem \
                          and 'num_devices' in subitem:
-                        values.append(
-                            f'{subitem["version_number"]}:'
-                            f'{subitem["num_devices"]}')
+                        values.append(f'{subitem["version_number"]}:'
+                                      f'{subitem["num_devices"]}')
                     else:
                         continue
                     value = ' '.join(sorted(values, reverse=True))
             csvRows.append({'name': name, 'value': value})
         for app in auth_apps:  # put apps at bottom
             csvRows.append(app)
-        display.write_csv_file(
-            csvRows, titles, f'Customer Report - {tryDate}', todrive=to_drive)
+        display.write_csv_file(csvRows,
+                               titles,
+                               f'Customer Report - {tryDate}',
+                               todrive=to_drive)
     else:
         page_message = gapi.got_total_items_msg('Activities', '...\n')
-        activities = gapi.get_all_pages(rep.activities(), 'list', 'items',
+        activities = gapi.get_all_pages(rep.activities(),
+                                        'list',
+                                        'items',
                                         page_message=page_message,
                                         applicationName=report,
                                         userKey=userKey,
                                         customerId=customerId,
                                         actorIpAddress=actorIpAddress,
-                                        startTime=startTime, endTime=endTime,
-                                        eventName=eventName, filters=filters,
+                                        startTime=startTime,
+                                        endTime=endTime,
+                                        eventName=eventName,
+                                        filters=filters,
                                         orgUnitID=orgUnitId)
         if activities:
             titles = ['name']
@@ -458,10 +523,11 @@ def showReport():
                             parts = {}
                             for message in item['multiMessageValue']:
                                 for mess in message['parameter']:
-                                    value = mess.get('value', ' '.join(
-                                        mess.get('multiValue', [])))
+                                    value = mess.get(
+                                        'value',
+                                        ' '.join(mess.get('multiValue', [])))
                                     parts[mess['name']] = parts.get(
-                                        mess['name'], [])+[value]
+                                        mess['name'], []) + [value]
                             for part, v in parts.items():
                                 if part == 'scope_name':
                                     part = 'scope'
@@ -476,15 +542,18 @@ def showReport():
                         if item not in titles:
                             titles.append(item)
                     csvRows.append(row)
-            display.sort_csv_titles(['name', ], titles)
-            display.write_csv_file(
-                csvRows, titles, f'{report.capitalize()} Activity Report',
-                to_drive)
+            display.sort_csv_titles([
+                'name',
+            ], titles)
+            display.write_csv_file(csvRows, titles,
+                                   f'{report.capitalize()} Activity Report',
+                                   to_drive)
 
 
 def _adjust_date(errMsg):
-    match_date = re.match('Data for dates later than (.*) is not yet '
-                          'available. Please check back later', errMsg)
+    match_date = re.match(
+        'Data for dates later than (.*) is not yet '
+        'available. Please check back later', errMsg)
     if not match_date:
         match_date = re.match('Start date can not be later than (.*)', errMsg)
     if not match_date:
@@ -492,16 +561,21 @@ def _adjust_date(errMsg):
     return str(match_date.group(1))
 
 
-def _check_full_data_available(warnings, tryDate, fullDataRequired):
+def _check_full_data_available(warnings, tryDate, fullDataRequired,
+                               has_reports):
     one_day = datetime.timedelta(days=1)
+    tryDateTime = datetime.datetime.strptime(tryDate, YYYYMMDD_FORMAT)
+    # move to day before if we don't have at least one usageReport
+    if not has_reports:
+        tryDateTime -= one_day
+        return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
     for warning in warnings:
         if warning['code'] == 'PARTIAL_DATA_AVAILABLE':
             for app in warning['data']:
                 if app['key'] == 'application' and \
                    app['value'] != 'docs' and \
-                   (not fullDataRequired or app['value'] in fullDataRequired):
-                    tryDateTime = datetime.datetime.strptime(
-                        tryDate, YYYYMMDD_FORMAT)
+                   fullDataRequired is not None and \
+                   (fullDataRequired == 'all' or app['value'] in fullDataRequired):
                     tryDateTime -= one_day
                     return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
         elif warning['code'] == 'DATA_NOT_AVAILABLE':

src/gam/gapi/storage.py

@@ -5,19 +5,21 @@ import sys
 
 import googleapiclient
 
-import __main__
-from var import *
-import controlflow
-import fileutils
-import gapi
-import utils
+import gam
+from gam.var import *
+from gam import fileutils
+from gam import gapi
+from gam import utils
 
 
 def build_gapi():
-    return __main__.buildGAPIObject('storage')
+    return gam.buildGAPIObject('storage')
 
 
-def get_cloud_storage_object(s, bucket, object_, local_file=None,
+def get_cloud_storage_object(s,
+                             bucket,
+                             object_,
+                             local_file=None,
                              expectedMd5=None):
     if not local_file:
         local_file = object_
@@ -61,13 +63,19 @@ def download_bucket():
     s = build_gapi()
     page_message = gapi.got_total_items_msg('Files', '...')
     fields = 'nextPageToken,items(name,id,md5Hash)'
-    objects = gapi.get_all_pages(s.objects(), 'list', 'items',
-                                 page_message=page_message, bucket=bucket,
-                                 projection='noAcl', fields=fields)
+    objects = gapi.get_all_pages(s.objects(),
+                                 'list',
+                                 'items',
+                                 page_message=page_message,
+                                 bucket=bucket,
+                                 projection='noAcl',
+                                 fields=fields)
     i = 1
     for object_ in objects:
         print(f'{i}/{len(objects)}')
         expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
-        get_cloud_storage_object(
-            s, bucket, object_['name'], expectedMd5=expectedMd5)
+        get_cloud_storage_object(s,
+                                 bucket,
+                                 object_['name'],
+                                 expectedMd5=expectedMd5)
         i += 1

src/gam/gapi/vault.py

@@ -4,35 +4,36 @@ import sys
 
 import googleapiclient.http
 
-import __main__
-from var import *
-import controlflow
-import display
-import fileutils
-import gapi
-import gapi.storage
-import utils
+import gam
+from gam.var import *
+from gam import controlflow
+from gam import display
+from gam import fileutils
+from gam import gapi
+from gam.gapi import storage as gapi_storage
+from gam import utils
 
 
 def buildGAPIObject():
-    return __main__.buildGAPIObject('vault')
+    return gam.buildGAPIObject('vault')
 
 
 def validateCollaborators(collaboratorList, cd):
     collaborators = []
     for collaborator in collaboratorList.split(','):
-        collaborator_id = __main__.convertEmailAddressToUID(collaborator, cd)
+        collaborator_id = gam.convertEmailAddressToUID(collaborator, cd)
         if not collaborator_id:
-            controlflow.system_error_exit(4, f'failed to get a UID for '
-                                             f'{collaborator}. Please make '
-                                             f'sure this is a real user.')
+            controlflow.system_error_exit(
+                4, f'failed to get a UID for '
+                f'{collaborator}. Please make '
+                f'sure this is a real user.')
         collaborators.append({'email': collaborator, 'id': collaborator_id})
     return collaborators
 
 
 def createMatter():
     v = buildGAPIObject()
-    matter_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    matter_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
     body = {'name': f'New Matter - {matter_time}'}
     collaborators = []
     cd = None
@@ -40,26 +41,29 @@ def createMatter():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'name':
-            body['name'] = sys.argv[i+1]
+            body['name'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'description':
-            body['description'] = sys.argv[i+1]
+            body['description'] = sys.argv[i + 1]
             i += 2
         elif myarg in ['collaborator', 'collaborators']:
             if not cd:
-                cd = __main__.buildGAPIObject('directory')
-            collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+                cd = gam.buildGAPIObject('directory')
+            collaborators.extend(validateCollaborators(sys.argv[i + 1], cd))
             i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam create matter")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create matter')
     matterId = gapi.call(v.matters(), 'create', body=body,
                          fields='matterId')['matterId']
     print(f'Created matter {matterId}')
     for collaborator in collaborators:
         print(f' adding collaborator {collaborator["email"]}')
-        body = {'matterPermission': {
-            'role': 'COLLABORATOR',
-            'accountId': collaborator['id']}}
+        body = {
+            'matterPermission': {
+                'role': 'COLLABORATOR',
+                'accountId': collaborator['id']
+            }
+        }
         gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
 
 
@@ -77,8 +81,9 @@ VAULT_SEARCH_METHODS_MAP = {
     'teamdrive': 'SHARED_DRIVE',
     'teamdrives': 'SHARED_DRIVE',
 }
-VAULT_SEARCH_METHODS_LIST = ['accounts',
-                             'orgunit', 'shareddrives', 'rooms', 'everyone']
+VAULT_SEARCH_METHODS_LIST = [
+    'accounts', 'orgunit', 'shareddrives', 'rooms', 'everyone'
+]
 
 
 def createExport():
@@ -98,17 +103,18 @@ def createExport():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'matter':
-            matterId = getMatterItem(v, sys.argv[i+1])
+            matterId = getMatterItem(v, sys.argv[i + 1])
             body['matterId'] = matterId
             i += 2
         elif myarg == 'name':
-            body['name'] = sys.argv[i+1]
+            body['name'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'corpus':
-            body['query']['corpus'] = sys.argv[i+1].upper()
+            body['query']['corpus'] = sys.argv[i + 1].upper()
             if body['query']['corpus'] not in allowed_corpuses:
-                controlflow.expected_argument_exit(
-                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+                controlflow.expected_argument_exit('corpus',
+                                                   ', '.join(allowed_corpuses),
+                                                   sys.argv[i + 1])
             i += 2
         elif myarg in VAULT_SEARCH_METHODS_MAP:
             if body['query'].get('searchMethod'):
@@ -120,82 +126,93 @@ def createExport():
             body['query']['searchMethod'] = searchMethod
             if searchMethod == 'ACCOUNT':
                 body['query']['accountInfo'] = {
-                    'emails': sys.argv[i+1].split(',')}
+                    'emails': sys.argv[i + 1].split(',')
+                }
                 i += 2
             elif searchMethod == 'ORG_UNIT':
                 body['query']['orgUnitInfo'] = {
-                    'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+                    'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+                }
                 i += 2
             elif searchMethod == 'SHARED_DRIVE':
                 body['query']['sharedDriveInfo'] = {
-                    'sharedDriveIds': sys.argv[i+1].split(',')}
+                    'sharedDriveIds': sys.argv[i + 1].split(',')
+                }
                 i += 2
             elif searchMethod == 'ROOM':
                 body['query']['hangoutsChatInfo'] = {
-                    'roomId': sys.argv[i+1].split(',')}
+                    'roomId': sys.argv[i + 1].split(',')
+                }
                 i += 2
             else:
                 i += 1
         elif myarg == 'scope':
-            body['query']['dataScope'] = sys.argv[i+1].upper()
+            body['query']['dataScope'] = sys.argv[i + 1].upper()
             if body['query']['dataScope'] not in allowed_scopes:
-                controlflow.expected_argument_exit(
-                    "scope", ", ".join(allowed_scopes), sys.argv[i+1])
+                controlflow.expected_argument_exit('scope',
+                                                   ', '.join(allowed_scopes),
+                                                   sys.argv[i + 1])
             i += 2
         elif myarg in ['terms']:
-            body['query']['terms'] = sys.argv[i+1]
+            body['query']['terms'] = sys.argv[i + 1]
             i += 2
         elif myarg in ['start', 'starttime']:
             body['query']['startTime'] = utils.get_date_zero_time_or_full_time(
-                sys.argv[i+1])
+                sys.argv[i + 1])
             i += 2
         elif myarg in ['end', 'endtime']:
             body['query']['endTime'] = utils.get_date_zero_time_or_full_time(
-                sys.argv[i+1])
+                sys.argv[i + 1])
             i += 2
         elif myarg in ['timezone']:
-            body['query']['timeZone'] = sys.argv[i+1]
+            body['query']['timeZone'] = sys.argv[i + 1]
             i += 2
         elif myarg in ['excludedrafts']:
             body['query']['mailOptions'] = {
-                'excludeDrafts': __main__.getBoolean(sys.argv[i+1], myarg)}
+                'excludeDrafts': gam.getBoolean(sys.argv[i + 1], myarg)
+            }
             i += 2
         elif myarg in ['driveversiondate']:
             body['query'].setdefault('driveOptions', {})['versionDate'] = \
                 utils.get_date_zero_time_or_full_time(sys.argv[i+1])
             i += 2
         elif myarg in ['includeshareddrives', 'includeteamdrives']:
-            body['query'].setdefault('driveOptions', {})[
-                'includeSharedDrives'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            body['query'].setdefault(
+                'driveOptions', {})['includeSharedDrives'] = gam.getBoolean(
+                    sys.argv[i + 1], myarg)
             i += 2
         elif myarg in ['includerooms']:
             body['query']['hangoutsChatOptions'] = {
-                'includeRooms': __main__.getBoolean(sys.argv[i+1], myarg)}
+                'includeRooms': gam.getBoolean(sys.argv[i + 1], myarg)
+            }
             i += 2
         elif myarg in ['format']:
-            export_format = sys.argv[i+1].upper()
+            export_format = sys.argv[i + 1].upper()
             if export_format not in allowed_formats:
-                controlflow.expected_argument_exit(
-                    "export format", ", ".join(allowed_formats), export_format)
+                controlflow.expected_argument_exit('export format',
+                                                   ', '.join(allowed_formats),
+                                                   export_format)
             i += 2
         elif myarg in ['showconfidentialmodecontent']:
-            showConfidentialModeContent = __main__.getBoolean(sys.argv[i+1], myarg)
+            showConfidentialModeContent = gam.getBoolean(sys.argv[i + 1], myarg)
             i += 2
         elif myarg in ['region']:
             allowed_regions = gapi.get_enum_values_minus_unspecified(
-                v._rootDesc['schemas']['ExportOptions']['properties'][
-                    'region']['enum'])
-            body['exportOptions']['region'] = sys.argv[i+1].upper()
+                v._rootDesc['schemas']['ExportOptions']['properties']['region']
+                ['enum'])
+            body['exportOptions']['region'] = sys.argv[i + 1].upper()
             if body['exportOptions']['region'] not in allowed_regions:
-                controlflow.expected_argument_exit("region", ", ".join(
-                    allowed_regions), body['exportOptions']['region'])
+                controlflow.expected_argument_exit(
+                    'region', ', '.join(allowed_regions),
+                    body['exportOptions']['region'])
             i += 2
         elif myarg in ['includeaccessinfo']:
-            body['exportOptions'].setdefault('driveOptions', {})[
-                'includeAccessInfo'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            body['exportOptions'].setdefault(
+                'driveOptions', {})['includeAccessInfo'] = gam.getBoolean(
+                    sys.argv[i + 1], myarg)
             i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam create export")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create export')
     if not matterId:
         controlflow.system_error_exit(
             3, 'you must specify a matter for the new export.')
@@ -207,7 +224,7 @@ def createExport():
           'for the new export. Choose one of ' \
           f'{", ".join(VAULT_SEARCH_METHODS_LIST)}')
     if 'name' not in body:
-        corpus_name = body["query"]["corpus"]
+        corpus_name = body['query']['corpus']
         corpus_date = datetime.datetime.now()
         body['name'] = f'GAM {corpus_name} export - {corpus_date}'
     options_field = None
@@ -223,8 +240,10 @@ def createExport():
         if showConfidentialModeContent is not None:
             body['exportOptions'][options_field][
                 'showConfidentialModeContent'] = showConfidentialModeContent
-    results = gapi.call(v.matters().exports(), 'create',
-                        matterId=matterId, body=body)
+    results = gapi.call(v.matters().exports(),
+                        'create',
+                        matterId=matterId,
+                        body=body)
     print(f'Created export {results["id"]}')
     display.print_json(results)
 
@@ -234,16 +253,20 @@ def deleteExport():
     matterId = getMatterItem(v, sys.argv[3])
     exportId = convertExportNameToID(v, sys.argv[4], matterId)
     print(f'Deleting export {sys.argv[4]} / {exportId}')
-    gapi.call(v.matters().exports(), 'delete',
-              matterId=matterId, exportId=exportId)
+    gapi.call(v.matters().exports(),
+              'delete',
+              matterId=matterId,
+              exportId=exportId)
 
 
 def getExportInfo():
     v = buildGAPIObject()
     matterId = getMatterItem(v, sys.argv[3])
     exportId = convertExportNameToID(v, sys.argv[4], matterId)
-    export = gapi.call(v.matters().exports(), 'get',
-                       matterId=matterId, exportId=exportId)
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
     display.print_json(export)
 
 
@@ -261,35 +284,37 @@ def createHold():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'name':
-            body['name'] = sys.argv[i+1]
+            body['name'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'query':
-            query = sys.argv[i+1]
+            query = sys.argv[i + 1]
             i += 2
         elif myarg == 'corpus':
-            body['corpus'] = sys.argv[i+1].upper()
+            body['corpus'] = sys.argv[i + 1].upper()
             if body['corpus'] not in allowed_corpuses:
-                controlflow.expected_argument_exit(
-                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+                controlflow.expected_argument_exit('corpus',
+                                                   ', '.join(allowed_corpuses),
+                                                   sys.argv[i + 1])
             i += 2
         elif myarg in ['accounts', 'users', 'groups']:
-            accounts = sys.argv[i+1].split(',')
+            accounts = sys.argv[i + 1].split(',')
             i += 2
         elif myarg in ['orgunit', 'ou']:
             body['orgUnit'] = {
-                'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+                'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+            }
             i += 2
         elif myarg in ['start', 'starttime']:
-            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
             i += 2
         elif myarg in ['end', 'endtime']:
-            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
             i += 2
         elif myarg == 'matter':
-            matterId = getMatterItem(v, sys.argv[i+1])
+            matterId = getMatterItem(v, sys.argv[i + 1])
             i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam create hold")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam create hold')
     if not matterId:
         controlflow.system_error_exit(
             3, 'you must specify a matter for the new hold.')
@@ -319,16 +344,18 @@ def createHold():
             body['query'][query_type]['endTime'] = end_time
     if accounts:
         body['accounts'] = []
-        cd = __main__.buildGAPIObject('directory')
+        cd = gam.buildGAPIObject('directory')
         account_type = 'group' if body['corpus'] == 'GROUPS' else 'user'
         for account in accounts:
-            body['accounts'].append(
-                {'accountId': __main__.convertEmailAddressToUID(account,
-                                                                cd,
-                                                                account_type)}
-            )
-    holdId = gapi.call(v.matters().holds(), 'create',
-                       matterId=matterId, body=body, fields='holdId')
+            body['accounts'].append({
+                'accountId':
+                    gam.convertEmailAddressToUID(account, cd, account_type)
+            })
+    holdId = gapi.call(v.matters().holds(),
+                       'create',
+                       matterId=matterId,
+                       body=body,
+                       fields='holdId')
     print(f'Created hold {holdId["holdId"]}')
 
 
@@ -340,11 +367,11 @@ def deleteHold():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'matter':
-            matterId = getMatterItem(v, sys.argv[i+1])
+            matterId = getMatterItem(v, sys.argv[i + 1])
             holdId = convertHoldNameToID(v, hold, matterId)
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam delete hold")
+            controlflow.invalid_argument_exit(myarg, 'gam delete hold')
     if not matterId:
         controlflow.system_error_exit(
             3, 'you must specify a matter for the hold.')
@@ -360,26 +387,27 @@ def getHoldInfo():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'matter':
-            matterId = getMatterItem(v, sys.argv[i+1])
+            matterId = getMatterItem(v, sys.argv[i + 1])
             holdId = convertHoldNameToID(v, hold, matterId)
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam info hold")
+            controlflow.invalid_argument_exit(myarg, 'gam info hold')
     if not matterId:
         controlflow.system_error_exit(
             3, 'you must specify a matter for the hold.')
-    results = gapi.call(v.matters().holds(), 'get',
-                        matterId=matterId, holdId=holdId)
-    cd = __main__.buildGAPIObject('directory')
+    results = gapi.call(v.matters().holds(),
+                        'get',
+                        matterId=matterId,
+                        holdId=holdId)
+    cd = gam.buildGAPIObject('directory')
     if 'accounts' in results:
         account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
         for i in range(0, len(results['accounts'])):
             uid = f'uid:{results["accounts"][i]["accountId"]}'
-            acct_email = __main__.convertUIDtoEmailAddress(
-                uid, cd, [account_type])
+            acct_email = gam.convertUIDtoEmailAddress(uid, cd, [account_type])
             results['accounts'][i]['email'] = acct_email
     if 'orgUnit' in results:
-        results['orgUnit']['orgUnitPath'] = __main__.doGetOrgInfo(
+        results['orgUnit']['orgUnitPath'] = gam.doGetOrgInfo(
             results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
     display.print_json(results)
 
@@ -390,13 +418,17 @@ def convertExportNameToID(v, nameOrID, matterId):
     if cg:
         return cg.group(1)
     fields = 'exports(id,name),nextPageToken'
-    exports = gapi.get_all_pages(v.matters().exports(
-    ), 'list', 'exports', matterId=matterId, fields=fields)
+    exports = gapi.get_all_pages(v.matters().exports(),
+                                 'list',
+                                 'exports',
+                                 matterId=matterId,
+                                 fields=fields)
     for export in exports:
         if export['name'].lower() == nameOrID:
             return export['id']
-    controlflow.system_error_exit(4, f'could not find export name {nameOrID} '
-                                     f'in matter {matterId}')
+    controlflow.system_error_exit(
+        4, f'could not find export name {nameOrID} '
+        f'in matter {matterId}')
 
 
 def convertHoldNameToID(v, nameOrID, matterId):
@@ -405,13 +437,17 @@ def convertHoldNameToID(v, nameOrID, matterId):
     if cg:
         return cg.group(1)
     fields = 'holds(holdId,name),nextPageToken'
-    holds = gapi.get_all_pages(v.matters().holds(
-    ), 'list', 'holds', matterId=matterId, fields=fields)
+    holds = gapi.get_all_pages(v.matters().holds(),
+                               'list',
+                               'holds',
+                               matterId=matterId,
+                               fields=fields)
     for hold in holds:
         if hold['name'].lower() == nameOrID:
             return hold['holdId']
-    controlflow.system_error_exit(4, f'could not find hold name {nameOrID} '
-                                     f'in matter {matterId}')
+    controlflow.system_error_exit(
+        4, f'could not find hold name {nameOrID} '
+        f'in matter {matterId}')
 
 
 def convertMatterNameToID(v, nameOrID):
@@ -420,8 +456,11 @@ def convertMatterNameToID(v, nameOrID):
     if cg:
         return cg.group(1)
     fields = 'matters(matterId,name),nextPageToken'
-    matters = gapi.get_all_pages(v.matters(
-    ), 'list', 'matters', view='BASIC', fields=fields)
+    matters = gapi.get_all_pages(v.matters(),
+                                 'list',
+                                 'matters',
+                                 view='BASIC',
+                                 fields=fields)
     for matter in matters:
         if matter['name'].lower() == nameOrID:
             return matter['matterId']
@@ -449,36 +488,41 @@ def updateHold():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'matter':
-            matterId = getMatterItem(v, sys.argv[i+1])
+            matterId = getMatterItem(v, sys.argv[i + 1])
             holdId = convertHoldNameToID(v, hold, matterId)
             i += 2
         elif myarg == 'query':
-            query = sys.argv[i+1]
+            query = sys.argv[i + 1]
             i += 2
         elif myarg in ['orgunit', 'ou']:
-            body['orgUnit'] = {'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            body['orgUnit'] = {
+                'orgUnitId': gam.getOrgUnitId(sys.argv[i + 1])[1]
+            }
             i += 2
         elif myarg in ['start', 'starttime']:
-            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
             i += 2
         elif myarg in ['end', 'endtime']:
-            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i + 1])
             i += 2
         elif myarg in ['addusers', 'addaccounts', 'addgroups']:
-            add_accounts = sys.argv[i+1].split(',')
+            add_accounts = sys.argv[i + 1].split(',')
             i += 2
         elif myarg in ['removeusers', 'removeaccounts', 'removegroups']:
-            del_accounts = sys.argv[i+1].split(',')
+            del_accounts = sys.argv[i + 1].split(',')
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam update hold")
+            controlflow.invalid_argument_exit(myarg, 'gam update hold')
     if not matterId:
         controlflow.system_error_exit(
             3, 'you must specify a matter for the hold.')
     if query or start_time or end_time or body.get('orgUnit'):
         fields = 'corpus,query,orgUnit'
-        old_body = gapi.call(v.matters().holds(
-        ), 'get', matterId=matterId, holdId=holdId, fields=fields)
+        old_body = gapi.call(v.matters().holds(),
+                             'get',
+                             matterId=matterId,
+                             holdId=holdId,
+                             fields=fields)
         body['query'] = old_body['query']
         body['corpus'] = old_body['corpus']
         if 'orgUnit' in old_body and 'orgUnit' not in body:
@@ -502,20 +546,29 @@ def updateHold():
                 body['query'][query_type]['endTime'] = end_time
     if body:
         print(f'Updating hold {hold} / {holdId}')
-        gapi.call(v.matters().holds(), 'update',
-                  matterId=matterId, holdId=holdId, body=body)
+        gapi.call(v.matters().holds(),
+                  'update',
+                  matterId=matterId,
+                  holdId=holdId,
+                  body=body)
     if add_accounts or del_accounts:
-        cd = __main__.buildGAPIObject('directory')
+        cd = gam.buildGAPIObject('directory')
         for account in add_accounts:
             print(f'adding {account} to hold.')
-            add_body = {'accountId': __main__.convertEmailAddressToUID(account, cd)}
-            gapi.call(v.matters().holds().accounts(), 'create',
-                      matterId=matterId, holdId=holdId, body=add_body)
+            add_body = {'accountId': gam.convertEmailAddressToUID(account, cd)}
+            gapi.call(v.matters().holds().accounts(),
+                      'create',
+                      matterId=matterId,
+                      holdId=holdId,
+                      body=add_body)
         for account in del_accounts:
             print(f'removing {account} from hold.')
-            accountId = __main__.convertEmailAddressToUID(account, cd)
-            gapi.call(v.matters().holds().accounts(), 'delete',
-                      matterId=matterId, holdId=holdId, accountId=accountId)
+            accountId = gam.convertEmailAddressToUID(account, cd)
+            gapi.call(v.matters().holds().accounts(),
+                      'delete',
+                      matterId=matterId,
+                      holdId=holdId,
+                      accountId=accountId)
 
 
 def updateMatter(action=None):
@@ -530,30 +583,30 @@ def updateMatter(action=None):
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'action':
-            action = sys.argv[i+1].lower()
+            action = sys.argv[i + 1].lower()
             if action not in VAULT_MATTER_ACTIONS:
                 controlflow.system_error_exit(3, f'allowed actions are ' \
                     f'{", ".join(VAULT_MATTER_ACTIONS)}, got {action}')
             i += 2
         elif myarg == 'name':
-            body['name'] = sys.argv[i+1]
+            body['name'] = sys.argv[i + 1]
             i += 2
         elif myarg == 'description':
-            body['description'] = sys.argv[i+1]
+            body['description'] = sys.argv[i + 1]
             i += 2
         elif myarg in ['addcollaborator', 'addcollaborators']:
             if not cd:
-                cd = __main__.buildGAPIObject('directory')
-            add_collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+                cd = gam.buildGAPIObject('directory')
+            add_collaborators.extend(validateCollaborators(sys.argv[i + 1], cd))
             i += 2
         elif myarg in ['removecollaborator', 'removecollaborators']:
             if not cd:
-                cd = __main__.buildGAPIObject('directory')
+                cd = gam.buildGAPIObject('directory')
             remove_collaborators.extend(
-                validateCollaborators(sys.argv[i+1], cd))
+                validateCollaborators(sys.argv[i + 1], cd))
             i += 2
         else:
-            controlflow.invalid_argument_exit(sys.argv[i], "gam update matter")
+            controlflow.invalid_argument_exit(sys.argv[i], 'gam update matter')
     if action == 'delete':
         action_kwargs = {}
     if body:
@@ -561,8 +614,10 @@ def updateMatter(action=None):
         if 'name' not in body or 'description' not in body:
             # bah, API requires name/description to be sent
             # on update even when it's not changing
-            result = gapi.call(v.matters(), 'get',
-                               matterId=matterId, view='BASIC')
+            result = gapi.call(v.matters(),
+                               'get',
+                               matterId=matterId,
+                               view='BASIC')
             body.setdefault('name', result['name'])
             body.setdefault('description', result.get('description'))
         gapi.call(v.matters(), 'update', body=body, matterId=matterId)
@@ -571,12 +626,18 @@ def updateMatter(action=None):
         gapi.call(v.matters(), action, matterId=matterId, **action_kwargs)
     for collaborator in add_collaborators:
         print(f' adding collaborator {collaborator["email"]}')
-        body = {'matterPermission': {'role': 'COLLABORATOR',
-                                     'accountId': collaborator['id']}}
+        body = {
+            'matterPermission': {
+                'role': 'COLLABORATOR',
+                'accountId': collaborator['id']
+            }
+        }
         gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
     for collaborator in remove_collaborators:
         print(f' removing collaborator {collaborator["email"]}')
-        gapi.call(v.matters(), 'removePermissions', matterId=matterId,
+        gapi.call(v.matters(),
+                  'removePermissions',
+                  matterId=matterId,
                   body={'accountId': collaborator['id']})
 
 
@@ -585,10 +646,10 @@ def getMatterInfo():
     matterId = getMatterItem(v, sys.argv[3])
     result = gapi.call(v.matters(), 'get', matterId=matterId, view='FULL')
     if 'matterPermissions' in result:
-        cd = __main__.buildGAPIObject('directory')
+        cd = gam.buildGAPIObject('directory')
         for i in range(0, len(result['matterPermissions'])):
             uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
-            user_email = __main__.convertUIDtoEmailAddress(uid, cd)
+            user_email = gam.convertUIDtoEmailAddress(uid, cd)
             result['matterPermissions'][i]['email'] = user_email
     display.print_json(result)
 
@@ -597,7 +658,7 @@ def downloadExport():
     verifyFiles = True
     extractFiles = True
     v = buildGAPIObject()
-    s = gapi.storage.build_gapi()
+    s = gapi_storage.build_gapi()
     matterId = getMatterItem(v, sys.argv[3])
     exportId = convertExportNameToID(v, sys.argv[4], matterId)
     targetFolder = GC_Values[GC_DRIVE_DIR]
@@ -605,7 +666,7 @@ def downloadExport():
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
         if myarg == 'targetfolder':
-            targetFolder = os.path.expanduser(sys.argv[i+1])
+            targetFolder = os.path.expanduser(sys.argv[i + 1])
             if not os.path.isdir(targetFolder):
                 os.makedirs(targetFolder)
             i += 2
@@ -616,10 +677,12 @@ def downloadExport():
             extractFiles = False
             i += 1
         else:
-            controlflow.invalid_argument_exit(
-                sys.argv[i], "gam download export")
-    export = gapi.call(v.matters().exports(), 'get',
-                       matterId=matterId, exportId=exportId)
+            controlflow.invalid_argument_exit(sys.argv[i],
+                                              'gam download export')
+    export = gapi.call(v.matters().exports(),
+                       'get',
+                       matterId=matterId,
+                       exportId=exportId)
     for s_file in export['cloudStorageSink']['files']:
         bucket = s_file['bucketName']
         s_object = s_file['objectName']
@@ -631,8 +694,8 @@ def downloadExport():
         done = False
         while not done:
             status, done = downloader.next_chunk()
-            sys.stdout.write(
-                ' Downloaded: {0:>7.2%}\r'.format(status.progress()))
+            sys.stdout.write(' Downloaded: {0:>7.2%}\r'.format(
+                status.progress()))
             sys.stdout.flush()
         sys.stdout.write('\n Download complete. Flushing to disk...\n')
         fileutils.close_file(f, True)
@@ -643,7 +706,7 @@ def downloadExport():
             utils.md5_matches_file(filename, expected_hash, True)
             print('VERIFIED')
         if extractFiles and re.search(r'\.zip$', filename):
-            __main__.extract_nested_zip(filename, targetFolder)
+            gam.extract_nested_zip(filename, targetFolder)
 
 
 def printMatters():
@@ -665,23 +728,26 @@ def printMatters():
             i += 1
         elif myarg == 'matterstate':
             valid_states = gapi.get_enum_values_minus_unspecified(
-                v._rootDesc['schemas']['Matter']['properties']['state'][
-                    'enum'])
-            state = sys.argv[i+1].upper()
+                v._rootDesc['schemas']['Matter']['properties']['state']['enum'])
+            state = sys.argv[i + 1].upper()
             if state not in valid_states:
-                controlflow.expected_argument_exit(
-                    'state', ', '.join(valid_states), state)
+                controlflow.expected_argument_exit('state',
+                                                   ', '.join(valid_states),
+                                                   state)
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam print matters")
-    __main__.printGettingAllItems('Vault Matters', None)
+            controlflow.invalid_argument_exit(myarg, 'gam print matters')
+    gam.printGettingAllItems('Vault Matters', None)
     page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
-    matters = gapi.get_all_pages(
-        v.matters(), 'list', 'matters', page_message=page_message, view=view,
-        state=state)
+    matters = gapi.get_all_pages(v.matters(),
+                                 'list',
+                                 'matters',
+                                 page_message=page_message,
+                                 view=view,
+                                 state=state)
     for matter in matters:
-        display.add_row_titles_to_csv_file(
-            utils.flatten_json(matter), csvRows, titles)
+        display.add_row_titles_to_csv_file(utils.flatten_json(matter), csvRows,
+                                           titles)
     display.sort_csv_titles(initialTitles, titles)
     display.write_csv_file(csvRows, titles, 'Vault Matters', todrive)
 
@@ -701,14 +767,18 @@ def printExports():
             todrive = True
             i += 1
         elif myarg in ['matter', 'matters']:
-            matters = sys.argv[i+1].split(',')
+            matters = sys.argv[i + 1].split(',')
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam print exports")
+            controlflow.invalid_argument_exit(myarg, 'gam print exports')
     if not matters:
         fields = 'matters(matterId),nextPageToken'
-        matters_results = gapi.get_all_pages(v.matters(
-        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        matters_results = gapi.get_all_pages(v.matters(),
+                                             'list',
+                                             'matters',
+                                             view='BASIC',
+                                             state='OPEN',
+                                             fields=fields)
         for matter in matters_results:
             matterIds.append(matter['matterId'])
     else:
@@ -716,11 +786,14 @@ def printExports():
             matterIds.append(getMatterItem(v, matter))
     for matterId in matterIds:
         sys.stderr.write(f'Retrieving exports for matter {matterId}\n')
-        exports = gapi.get_all_pages(
-            v.matters().exports(), 'list', 'exports', matterId=matterId)
+        exports = gapi.get_all_pages(v.matters().exports(),
+                                     'list',
+                                     'exports',
+                                     matterId=matterId)
         for export in exports:
-            display.add_row_titles_to_csv_file(utils.flatten_json(
-                export, flattened={'matterId': matterId}), csvRows, titles)
+            display.add_row_titles_to_csv_file(
+                utils.flatten_json(export, flattened={'matterId': matterId}),
+                csvRows, titles)
     display.sort_csv_titles(initialTitles, titles)
     display.write_csv_file(csvRows, titles, 'Vault Exports', todrive)
 
@@ -740,14 +813,18 @@ def printHolds():
             todrive = True
             i += 1
         elif myarg in ['matter', 'matters']:
-            matters = sys.argv[i+1].split(',')
+            matters = sys.argv[i + 1].split(',')
             i += 2
         else:
-            controlflow.invalid_argument_exit(myarg, "gam print holds")
+            controlflow.invalid_argument_exit(myarg, 'gam print holds')
     if not matters:
         fields = 'matters(matterId),nextPageToken'
-        matters_results = gapi.get_all_pages(v.matters(
-        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        matters_results = gapi.get_all_pages(v.matters(),
+                                             'list',
+                                             'matters',
+                                             view='BASIC',
+                                             state='OPEN',
+                                             fields=fields)
         for matter in matters_results:
             matterIds.append(matter['matterId'])
     else:
@@ -755,10 +832,13 @@ def printHolds():
             matterIds.append(getMatterItem(v, matter))
     for matterId in matterIds:
         sys.stderr.write(f'Retrieving holds for matter {matterId}\n')
-        holds = gapi.get_all_pages(
-            v.matters().holds(), 'list', 'holds', matterId=matterId)
+        holds = gapi.get_all_pages(v.matters().holds(),
+                                   'list',
+                                   'holds',
+                                   matterId=matterId)
         for hold in holds:
-            display.add_row_titles_to_csv_file(utils.flatten_json(
-                hold, flattened={'matterId': matterId}), csvRows, titles)
+            display.add_row_titles_to_csv_file(
+                utils.flatten_json(hold, flattened={'matterId': matterId}),
+                csvRows, titles)
     display.sort_csv_titles(initialTitles, titles)
     display.write_csv_file(csvRows, titles, 'Vault Holds', todrive)

src/gam/transport.py

@@ -0,0 +1,102 @@
+"""Methods related to network transport."""
+
+import google_auth_httplib2
+import httplib2
+
+from gam.var import GAM_INFO
+from gam.var import GC_CA_FILE
+from gam.var import GC_TLS_MAX_VERSION
+from gam.var import GC_TLS_MIN_VERSION
+from gam.var import GC_Values
+
+
+def create_http(cache=None,
+                timeout=None,
+                override_min_tls=None,
+                override_max_tls=None):
+    """Creates a uniform HTTP transport object.
+
+  Args:
+    cache: The HTTP cache to use.
+    timeout: The cache timeout, in seconds.
+    override_min_tls: The minimum TLS version to require. If not provided, the
+      default is used.
+    override_max_tls: The maximum TLS version to require. If not provided, the
+      default is used.
+
+  Returns:
+    httplib2.Http with the specified options.
+  """
+    tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
+        GC_TLS_MIN_VERSION)
+    tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
+        GC_TLS_MAX_VERSION)
+    httpObj = httplib2.Http(ca_certs=GC_Values.get(GC_CA_FILE),
+                            tls_maximum_version=tls_maximum_version,
+                            tls_minimum_version=tls_minimum_version,
+                            cache=cache,
+                            timeout=timeout)
+    httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
+    return httpObj
+
+
+def create_request(http=None):
+    """Creates a uniform Request object with a default http, if not provided.
+
+  Args:
+    http: Optional httplib2.Http compatible object to be used with the request.
+      If not provided, a default HTTP will be used.
+
+  Returns:
+    Request: A google_auth_httplib2.Request compatible Request.
+  """
+    if not http:
+        http = create_http()
+    return Request(http)
+
+
+GAM_USER_AGENT = GAM_INFO
+
+
+def _force_user_agent(user_agent):
+    """Creates a decorator which can force a user agent in HTTP headers."""
+
+    def decorator(request_method):
+        """Wraps a request method to insert a user-agent in HTTP headers."""
+
+        def wrapped_request_method(*args, **kwargs):
+            """Modifies HTTP headers to include a specified user-agent."""
+            if kwargs.get('headers') is not None:
+                if kwargs['headers'].get('user-agent'):
+                    if user_agent not in kwargs['headers']['user-agent']:
+                        # Save the existing user-agent header and tack on our own.
+                        kwargs['headers']['user-agent'] = (
+                            f'{user_agent} '
+                            f'{kwargs["headers"]["user-agent"]}')
+                else:
+                    kwargs['headers']['user-agent'] = user_agent
+            else:
+                kwargs['headers'] = {'user-agent': user_agent}
+            return request_method(*args, **kwargs)
+
+        return wrapped_request_method
+
+    return decorator
+
+
+class Request(google_auth_httplib2.Request):
+    """A Request which forces a user agent."""
+
+    @_force_user_agent(GAM_USER_AGENT)
+    def __call__(self, *args, **kwargs):
+        """Inserts the GAM user-agent header in requests."""
+        return super(Request, self).__call__(*args, **kwargs)
+
+
+class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
+    """An AuthorizedHttp which forces a user agent during requests."""
+
+    @_force_user_agent(GAM_USER_AGENT)
+    def request(self, *args, **kwargs):
+        """Inserts the GAM user-agent header in requests."""
+        return super(AuthorizedHttp, self).request(*args, **kwargs)

src/gam/transport_test.py

@@ -0,0 +1,185 @@
+"""Tests for transport."""
+
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import google_auth_httplib2
+import httplib2
+
+from gam import transport
+
+
+class CreateHttpTest(unittest.TestCase):
+
+    def setUp(self):
+        SetGlobalVariables()
+        super(CreateHttpTest, self).setUp()
+
+    def test_create_http_sets_default_values_on_http(self):
+        http = transport.create_http()
+        self.assertIsNone(http.cache)
+        self.assertIsNone(http.timeout)
+        self.assertEqual(http.tls_minimum_version,
+                         transport.GC_Values[transport.GC_TLS_MIN_VERSION])
+        self.assertEqual(http.tls_maximum_version,
+                         transport.GC_Values[transport.GC_TLS_MAX_VERSION])
+        self.assertEqual(http.ca_certs,
+                         transport.GC_Values[transport.GC_CA_FILE])
+
+    def test_create_http_sets_tls_min_version(self):
+        http = transport.create_http(override_min_tls='TLSv1_1')
+        self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
+
+    def test_create_http_sets_tls_max_version(self):
+        http = transport.create_http(override_max_tls='TLSv1_3')
+        self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
+
+    def test_create_http_sets_cache(self):
+        fake_cache = {}
+        http = transport.create_http(cache=fake_cache)
+        self.assertEqual(http.cache, fake_cache)
+
+    def test_create_http_sets_cache_timeout(self):
+        http = transport.create_http(timeout=1234)
+        self.assertEqual(http.timeout, 1234)
+
+
+class TransportTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mock_http = MagicMock(spec=httplib2.Http)
+        self.mock_response = MagicMock(spec=httplib2.Response)
+        self.mock_content = MagicMock()
+        self.mock_http.request.return_value = (self.mock_response,
+                                               self.mock_content)
+        self.mock_credentials = MagicMock()
+        self.test_uri = 'http://example.com'
+        super(TransportTest, self).setUp()
+
+    @patch.object(transport, 'create_http')
+    def test_create_request_uses_default_http(self, mock_create_http):
+        request = transport.create_request()
+        self.assertEqual(request.http, mock_create_http.return_value)
+
+    def test_create_request_uses_provided_http(self):
+        request = transport.create_request(http=self.mock_http)
+        self.assertEqual(request.http, self.mock_http)
+
+    def test_create_request_returns_request_with_forced_user_agent(self):
+        request = transport.create_request()
+        self.assertIsInstance(request, transport.Request)
+
+    def test_request_is_google_auth_httplib2_compatible(self):
+        request = transport.create_request()
+        self.assertIsInstance(request, google_auth_httplib2.Request)
+
+    def test_request_call_returns_response_content(self):
+        request = transport.Request(self.mock_http)
+        response = request(self.test_uri)
+        self.assertEqual(self.mock_response.status, response.status)
+        self.assertEqual(self.mock_content, response.data)
+
+    def test_request_call_forces_user_agent_no_provided_headers(self):
+        request = transport.Request(self.mock_http)
+
+        request(self.test_uri)
+        headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', headers)
+        self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+    def test_request_call_forces_user_agent_no_agent_in_headers(self):
+        request = transport.Request(self.mock_http)
+        fake_request_headers = {
+            'some-header-thats-not-a-user-agent': 'someData'
+        }
+
+        request(self.test_uri, headers=fake_request_headers)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+        self.assertEqual('someData',
+                         final_headers['some-header-thats-not-a-user-agent'])
+
+    def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
+        request = transport.Request(self.mock_http)
+        headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+        request(self.test_uri, headers=headers_with_user_agent)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn('existing-user-agent', final_headers['user-agent'])
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+    def test_request_call_same_user_agent_already_in_headers(self):
+        request = transport.Request(self.mock_http)
+        same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+        request(self.test_uri, headers=same_user_agent_header)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        # Make sure the header wasn't duplicated
+        self.assertEqual(len(transport.GAM_USER_AGENT),
+                         len(final_headers['user-agent']))
+
+    def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
+        http = transport.AuthorizedHttp(self.mock_credentials)
+        self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
+
+    def test_authorizedhttp_request_returns_response_content(self):
+        http = transport.AuthorizedHttp(self.mock_credentials,
+                                        http=self.mock_http)
+        response, content = http.request(self.test_uri)
+        self.assertEqual(self.mock_response, response)
+        self.assertEqual(self.mock_content, content)
+
+    def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        authorized_http.request(self.test_uri)
+        headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', headers)
+        self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+    def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        fake_request_headers = {
+            'some-header-thats-not-a-user-agent': 'someData'
+        }
+
+        authorized_http.request(self.test_uri, headers=fake_request_headers)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+        self.assertEqual('someData',
+                         final_headers['some-header-thats-not-a-user-agent'])
+
+    def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
+        self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+        authorized_http.request(self.test_uri, headers=headers_with_user_agent)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn('existing-user-agent', final_headers['user-agent'])
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+    def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
+        authorized_http = transport.AuthorizedHttp(self.mock_credentials,
+                                                   http=self.mock_http)
+        same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+        authorized_http.request(self.test_uri, headers=same_user_agent_header)
+        final_headers = self.mock_http.request.call_args[1]['headers']
+        self.assertIn('user-agent', final_headers)
+        self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+        # Make sure the header wasn't duplicated
+        self.assertEqual(len(transport.GAM_USER_AGENT),
+                         len(final_headers['user-agent']))

src/gam/utils.py

@@ -0,0 +1,341 @@
+import datetime
+import re
+import sys
+import time
+from hashlib import md5
+from html.entities import name2codepoint
+from html.parser import HTMLParser
+import json
+import dateutil.parser
+
+from gam import controlflow
+from gam import fileutils
+from gam import transport
+from gam.var import *
+
+
+class _DeHTMLParser(HTMLParser):
+
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.__text = []
+
+    def handle_data(self, data):
+        self.__text.append(data)
+
+    def handle_charref(self, name):
+        self.__text.append(
+            chr(int(name[1:], 16)) if name.startswith('x') else chr(int(name)))
+
+    def handle_entityref(self, name):
+        cp = name2codepoint.get(name)
+        if cp:
+            self.__text.append(chr(cp))
+        else:
+            self.__text.append('&' + name)
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'p':
+            self.__text.append('\n\n')
+        elif tag == 'br':
+            self.__text.append('\n')
+        elif tag == 'a':
+            for attr in attrs:
+                if attr[0] == 'href':
+                    self.__text.append(f'({attr[1]}) ')
+                    break
+        elif tag == 'div':
+            if not attrs:
+                self.__text.append('\n')
+        elif tag in {'http:', 'https'}:
+            self.__text.append(f' ({tag}//{attrs[0][0]}) ')
+
+    def handle_startendtag(self, tag, attrs):
+        if tag == 'br':
+            self.__text.append('\n\n')
+
+    def text(self):
+        return re.sub(r'\n{2}\n+', '\n\n',
+                      re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
+
+
+def dehtml(text):
+    try:
+        parser = _DeHTMLParser()
+        parser.feed(str(text))
+        parser.close()
+        return parser.text()
+    except:
+        from traceback import print_exc
+        print_exc(file=sys.stderr)
+        return text
+
+
+def indentMultiLineText(message, n=0):
+    return message.replace('\n', '\n{0}'.format(' ' * n)).rstrip()
+
+
+def flatten_json(structure, key='', path='', flattened=None, listLimit=None):
+    if flattened is None:
+        flattened = {}
+    if not isinstance(structure, (dict, list)):
+        flattened[((path + '.') if path else '') + key] = structure
+    elif isinstance(structure, list):
+        for i, item in enumerate(structure):
+            if listLimit and (i >= listLimit):
+                break
+            flatten_json(item,
+                         f'{i}',
+                         '.'.join([item for item in [path, key] if item]),
+                         flattened=flattened,
+                         listLimit=listLimit)
+    else:
+        for new_key, value in list(structure.items()):
+            if new_key in ['kind', 'etag', '@type']:
+                continue
+            if value == NEVER_TIME:
+                value = 'Never'
+            flatten_json(value,
+                         new_key,
+                         '.'.join([item for item in [path, key] if item]),
+                         flattened=flattened,
+                         listLimit=listLimit)
+    return flattened
+
+
+def formatTimestampYMD(timestamp):
+    return datetime.datetime.fromtimestamp(int(timestamp) /
+                                           1000).strftime('%Y-%m-%d')
+
+
+def formatTimestampYMDHMS(timestamp):
+    return datetime.datetime.fromtimestamp(int(timestamp) /
+                                           1000).strftime('%Y-%m-%d %H:%M:%S')
+
+
+def formatTimestampYMDHMSF(timestamp):
+    return str(datetime.datetime.fromtimestamp(int(timestamp) / 1000))
+
+
+def formatFileSize(fileSize):
+    if fileSize == 0:
+        return '0kb'
+    if fileSize < ONE_KILO_BYTES:
+        return '1kb'
+    if fileSize < ONE_MEGA_BYTES:
+        return f'{fileSize // ONE_KILO_BYTES}kb'
+    if fileSize < ONE_GIGA_BYTES:
+        return f'{fileSize // ONE_MEGA_BYTES}mb'
+    return f'{fileSize // ONE_GIGA_BYTES}gb'
+
+
+def formatMilliSeconds(millis):
+    seconds, millis = divmod(millis, 1000)
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
+
+
+def integerLimits(minVal, maxVal, item='integer'):
+    if (minVal is not None) and (maxVal is not None):
+        return f'{item} {minVal}<=x<={maxVal}'
+    if minVal is not None:
+        return f'{item} x>={minVal}'
+    if maxVal is not None:
+        return f'{item} x<={maxVal}'
+    return f'{item} x'
+
+
+def get_string(i, item, optional=False, minLen=1, maxLen=None):
+    if i < len(sys.argv):
+        argstr = sys.argv[i]
+        if argstr:
+            if (len(argstr) >= minLen) and ((maxLen is None) or
+                                            (len(argstr) <= maxLen)):
+                return argstr
+            controlflow.system_error_exit(
+                2,
+                f'expected <{integerLimits(minLen, maxLen, "string length")} for {item}>'
+            )
+        if optional or (minLen == 0):
+            return ''
+        controlflow.system_error_exit(2, f'expected a Non-empty <{item}>')
+    elif optional:
+        return ''
+    controlflow.system_error_exit(2, f'expected a <{item}>')
+
+
+def get_delta(argstr, pattern):
+    tg = pattern.match(argstr.lower())
+    if tg is None:
+        return None
+    sign = tg.group(1)
+    delta = int(tg.group(2))
+    unit = tg.group(3)
+    if unit == 'y':
+        deltaTime = datetime.timedelta(days=delta * 365)
+    elif unit == 'w':
+        deltaTime = datetime.timedelta(weeks=delta)
+    elif unit == 'd':
+        deltaTime = datetime.timedelta(days=delta)
+    elif unit == 'h':
+        deltaTime = datetime.timedelta(hours=delta)
+    elif unit == 'm':
+        deltaTime = datetime.timedelta(minutes=delta)
+    if sign == '-':
+        return -deltaTime
+    return deltaTime
+
+
+def get_delta_date(argstr):
+    deltaDate = get_delta(argstr, DELTA_DATE_PATTERN)
+    if deltaDate is None:
+        controlflow.system_error_exit(
+            2, f'expected a <{DELTA_DATE_FORMAT_REQUIRED}>; got {argstr}')
+    return deltaDate
+
+
+def get_delta_time(argstr):
+    deltaTime = get_delta(argstr, DELTA_TIME_PATTERN)
+    if deltaTime is None:
+        controlflow.system_error_exit(
+            2, f'expected a <{DELTA_TIME_FORMAT_REQUIRED}>; got {argstr}')
+    return deltaTime
+
+
+def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
+    argstr = argstr.strip()
+    if argstr:
+        if argstr[0] in ['+', '-']:
+            today = datetime.date.today()
+            argstr = (datetime.datetime(today.year, today.month, today.day) +
+                      get_delta_date(argstr)).strftime(YYYYMMDD_FORMAT)
+        try:
+            dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
+            if returnTimeStamp:
+                return time.mktime(dateTime.timetuple()) * 1000
+            if returnDateTime:
+                return dateTime
+            return argstr
+        except ValueError:
+            controlflow.system_error_exit(
+                2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>; got {argstr}')
+    elif minLen == 0:
+        return ''
+    controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>')
+
+
+def get_time_or_delta_from_now(time_string):
+    """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+    time_string = time_string.strip().upper()
+    if time_string:
+        if time_string[0] not in ['+', '-']:
+            return time_string
+        return (datetime.datetime.utcnow() +
+                get_delta_time(time_string)).isoformat() + 'Z'
+    controlflow.system_error_exit(
+        2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+
+def get_row_filter_date_or_delta_from_now(date_string):
+    """Get an ISO 8601 date or a positive/negative delta applied to now.
+  Args:
+    date_string (string): The time or delta (e.g. '2017-09-01' or '-4y')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+    date_string = date_string.strip().upper()
+    if date_string:
+        if date_string[0] in ['+', '-']:
+            deltaDate = get_delta(date_string, DELTA_DATE_PATTERN)
+            if deltaDate is None:
+                return (False, DELTA_DATE_FORMAT_REQUIRED)
+            today = datetime.date.today()
+            return (True,
+                    (datetime.datetime(today.year, today.month, today.day) +
+                     deltaDate).isoformat() + 'Z')
+        try:
+            deltaDate = dateutil.parser.parse(date_string, ignoretz=True)
+            return (True,
+                    datetime.datetime(deltaDate.year, deltaDate.month,
+                                      deltaDate.day).isoformat() + 'Z')
+        except ValueError:
+            pass
+    return (False, YYYYMMDD_FORMAT_REQUIRED)
+
+
+def get_row_filter_time_or_delta_from_now(time_string):
+    """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  Exits:
+    2: Not a valid delta.
+  """
+    time_string = time_string.strip().upper()
+    if time_string:
+        if time_string[0] in ['+', '-']:
+            deltaTime = get_delta(time_string, DELTA_TIME_PATTERN)
+            if deltaTime is None:
+                return (False, DELTA_TIME_FORMAT_REQUIRED)
+            return (True,
+                    (datetime.datetime.utcnow() + deltaTime).isoformat() + 'Z')
+        try:
+            deltaTime = dateutil.parser.parse(time_string, ignoretz=True)
+            return (True, deltaTime.isoformat() + 'Z')
+        except ValueError:
+            pass
+    return (False, YYYYMMDDTHHMMSS_FORMAT_REQUIRED)
+
+
+def get_date_zero_time_or_full_time(time_string):
+    time_string = time_string.strip()
+    if time_string:
+        if YYYYMMDD_PATTERN.match(time_string):
+            return get_yyyymmdd(time_string) + 'T00:00:00.000Z'
+        return get_time_or_delta_from_now(time_string)
+    controlflow.system_error_exit(
+        2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+
+def md5_matches_file(local_file, expected_md5, exitOnError):
+    f = fileutils.open_file(local_file, 'rb')
+    hash_md5 = md5()
+    for chunk in iter(lambda: f.read(4096), b''):
+        hash_md5.update(chunk)
+    actual_hash = hash_md5.hexdigest()
+    if exitOnError and actual_hash != expected_md5:
+        controlflow.system_error_exit(
+            6, f'actual hash was {actual_hash}. Exiting on corrupt file.')
+    return actual_hash == expected_md5
+
+
+URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+
+def shorten_url(long_url, httpc=None):
+    if not httpc:
+        httpc = transport.create_http(timeout=10)
+    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
+    try:
+        payload = json.dumps({'long_url': long_url})
+        resp, content = httpc.request(URL_SHORTENER_ENDPOINT,
+                                      'POST',
+                                      payload,
+                                      headers=headers)
+    except:
+        return long_url
+    if resp.status != 200:
+        return long_url
+    try:
+        if isinstance(content, bytes):
+            content = content.decode()
+        return json.loads(content).get('short_url', long_url)
+    except:
+        return long_url

src/gapi/__init__.py

@@ -1,328 +0,0 @@
-"""Methods related to execution of GAPI requests."""
-
-import sys
-
-import googleapiclient.errors
-import google.auth.exceptions
-import httplib2
-
-import controlflow
-import display
-from gapi import errors
-import transport
-from var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
-                 GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
-                 MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
-                 MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
-
-
-def call(service,
-         function,
-         silent_errors=False,
-         soft_errors=False,
-         throw_reasons=None,
-         retry_reasons=None,
-         **kwargs):
-  """Executes a single request on a Google service function.
-
-  Args:
-    service: A Google service object for the desired API.
-    function: String, The name of a service request method to execute.
-    silent_errors: Bool, If True, error messages are suppressed when
-      encountered.
-    soft_errors: Bool, If True, writes non-fatal errors to stderr.
-    throw_reasons: A list of Google HTTP error reason strings indicating the
-      errors generated by this request should be re-thrown. All other HTTP
-      errors are consumed.
-    retry_reasons: A list of Google HTTP error reason strings indicating which
-      error should be retried, using exponential backoff techniques, when the
-      error reason is encountered.
-    **kwargs: Additional params to pass to the request method.
-
-  Returns:
-    A response object for the corresponding Google API call.
-  """
-  if throw_reasons is None:
-    throw_reasons = []
-  if retry_reasons is None:
-    retry_reasons = []
-
-  method = getattr(service, function)
-  retries = 10
-  parameters = dict(
-      list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
-  for n in range(1, retries + 1):
-    try:
-      return method(**parameters).execute()
-    except googleapiclient.errors.HttpError as e:
-      http_status, reason, message = errors.get_gapi_error_detail(
-          e,
-          soft_errors=soft_errors,
-          silent_errors=silent_errors,
-          retry_on_http_error=n < 3)
-      if http_status == -1:
-        # The error detail indicated that we should retry this request
-        # We'll refresh credentials and make another pass
-        service._http.credentials.refresh(transport.create_http())
-        continue
-      if http_status == 0:
-        return None
-
-      is_known_error_reason = reason in [r.value for r in errors.ErrorReason]
-      if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
-        if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
-          raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
-              message)
-        raise e
-      if (n != retries) and (is_known_error_reason and errors.ErrorReason(
-          reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
-        controlflow.wait_on_failure(n, retries, reason)
-        continue
-      if soft_errors:
-        display.print_error(f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}')
-        return None
-      controlflow.system_error_exit(
-          int(http_status), f'{http_status}: {message} - {reason}')
-    except google.auth.exceptions.RefreshError as e:
-      handle_oauth_token_error(
-          e, soft_errors or
-          errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
-      if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
-        raise errors.GapiServiceNotAvailableError(str(e))
-      display.print_error(f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
-      return None
-    except ValueError as e:
-      if hasattr(service._http, 'cache') and service._http.cache is not None:
-        service._http.cache = None
-        continue
-      controlflow.system_error_exit(4, str(e))
-    except (httplib2.ServerNotFoundError, RuntimeError) as e:
-      if n != retries:
-        service._http.connections = {}
-        controlflow.wait_on_failure(n, retries, str(e))
-        continue
-      controlflow.system_error_exit(4, str(e))
-    except TypeError as e:
-      controlflow.system_error_exit(4, str(e))
-
-
-def get_items(service,
-              function,
-              items='items',
-              throw_reasons=None,
-              retry_reasons=None,
-              **kwargs):
-  """Gets a single page of items from a Google service function that is paged.
-
-  Args:
-    service: A Google service object for the desired API.
-    function: String, The name of a service request method to execute.
-    items: String, the name of the resulting "items" field within the service
-      method's response object.
-    throw_reasons: A list of Google HTTP error reason strings indicating the
-      errors generated by this request should be re-thrown. All other HTTP
-      errors are consumed.
-    retry_reasons: A list of Google HTTP error reason strings indicating which
-      error should be retried, using exponential backoff techniques, when the
-      error reason is encountered.
-    **kwargs: Additional params to pass to the request method.
-
-  Returns:
-    The list of items in the first page of a response.
-  """
-  results = call(
-      service,
-      function,
-      throw_reasons=throw_reasons,
-      retry_reasons=retry_reasons,
-      **kwargs)
-  if results:
-    return results.get(items, [])
-  return []
-
-
-def _get_max_page_size_for_api_call(service, function, **kwargs):
-  """Gets the maximum number of results supported for a single API call.
-
-  Args:
-    service: A Google service object for the desired API.
-    function: String, The name of the service method to check for max page size.
-    **kwargs: Additional params that will be passed to the request method.
-
-  Returns:
-    Int, A value from discovery if it exists, otherwise value from
-        MAX_RESULTS_API_EXCEPTIONS, otherwise None
-  """
-  method = getattr(service, function)
-  api_id = method(**kwargs).methodId
-  for resource in service._rootDesc.get('resources', {}).values():
-    for a_method in resource.get('methods', {}).values():
-      if a_method.get('id') == api_id:
-        if not a_method.get('parameters') or a_method['parameters'].get(
-            'pageSize') or not a_method['parameters'].get('maxResults'):
-          # Make sure API call supports maxResults. For now we don't care to
-          # set pageSize since all known pageSize API calls have
-          # default pageSize == max pageSize.
-          return None
-        known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
-        max_results = a_method['parameters']['maxResults'].get(
-            'maximum', known_api_max)
-        return {'maxResults': max_results}
-
-  return None
-
-
-TOTAL_ITEMS_MARKER = '%%total_items%%'
-FIRST_ITEM_MARKER = '%%first_item%%'
-LAST_ITEM_MARKER = '%%last_item%%'
-
-def got_total_items_msg(items, eol):
-  """Format a page_message to be used by get_all_pages
-
-  The page message indicates the number of items returned
-
-  Args:
-    items: String, the description of the items being returned by get_all_pages
-    eol: String, the line terminator
-       Values used: '', '...', '\n', '...\n'
-
-  Returns:
-    The formatted page_message
-  """
-
-  return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
-
-def got_total_items_first_last_msg(items):
-  """Format a page_message to be used by get_all_pages
-
-  The page message indicates the number of items returned and the
-  value of the first and list items
-
-  Args:
-    items: String, the description of the items being returned by get_all_pages
-
-  Returns:
-    The formatted page_message
-  """
-
-  return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}'+'\n'
-
-def get_all_pages(service,
-                  function,
-                  items='items',
-                  page_message=None,
-                  message_attribute=None,
-                  soft_errors=False,
-                  throw_reasons=None,
-                  retry_reasons=None,
-                  **kwargs):
-  """Aggregates and returns all pages of a Google service function response.
-
-  All pages of items are aggregated and returned as a single list.
-
-  Args:
-    service: A Google service object for the desired API.
-    function: String, The name of a service request method to execute.
-    items: String, the name of the resulting "items" field within the method's
-      response object. The items in this field will be aggregated across all
-      pages and returned.
-    page_message: String, a message to be displayed to the user during paging.
-      Template strings allow for dynamic content to be inserted during paging.
-        Supported template strings:
-          TOTAL_ITEMS_MARKER : The current number of items discovered across all
-            pages.
-          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
-            display a unique property of the first item in the current page.
-          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
-            display a unique property of the last item in the current page.
-    message_attribute: String, the name of a signature field within a single
-      returned item which identifies that unique item. This field is used with
-      `page_message` to templatize a paging status message.
-    soft_errors: Bool, If True, writes non-fatal errors to stderr.
-    throw_reasons: A list of Google HTTP error reason strings indicating the
-      errors generated by this request should be re-thrown. All other HTTP
-      errors are consumed.
-    retry_reasons: A list of Google HTTP error reason strings indicating which
-      error should be retried, using exponential backoff techniques, when the
-      error reason is encountered.
-    **kwargs: Additional params to pass to the request method.
-
-  Returns:
-    A list of all items received from all paged responses.
-  """
-  if 'maxResults' not in kwargs and 'pageSize' not in kwargs:
-    page_key = _get_max_page_size_for_api_call(service, function, **kwargs)
-    if page_key:
-      kwargs.update(page_key)
-  all_items = []
-  page_token = None
-  total_items = 0
-  while True:
-    page = call(
-        service,
-        function,
-        soft_errors=soft_errors,
-        throw_reasons=throw_reasons,
-        retry_reasons=retry_reasons,
-        pageToken=page_token,
-        **kwargs)
-    if page:
-      page_token = page.get('nextPageToken')
-      page_items = page.get(items, [])
-      num_page_items = len(page_items)
-      total_items += num_page_items
-      all_items.extend(page_items)
-    else:
-      page_token = None
-      num_page_items = 0
-
-    # Show a paging message to the user that indicates paging progress
-    if page_message:
-      show_message = page_message.replace(TOTAL_ITEMS_MARKER, str(total_items))
-      if message_attribute:
-        first_item = page_items[0] if num_page_items > 0 else {}
-        last_item = page_items[-1] if num_page_items > 1 else first_item
-        show_message = show_message.replace(FIRST_ITEM_MARKER, str(first_item.get(message_attribute, '')))
-        show_message = show_message.replace(LAST_ITEM_MARKER, str(last_item.get(message_attribute, '')))
-      sys.stderr.write('\r')
-      sys.stderr.flush()
-      sys.stderr.write(show_message)
-
-    if not page_token:
-      # End the paging status message and return all items.
-      if page_message and (page_message[-1] != '\n'):
-        sys.stderr.write('\r\n')
-        sys.stderr.flush()
-      return all_items
-
-
-# TODO: Make this private once all execution related items that use this method
-# have been brought into this file
-def handle_oauth_token_error(e, soft_errors):
-  """On a token error, exits the application and writes a message to stderr.
-
-  Args:
-    e: google.auth.exceptions.RefreshError, The error to handle.
-    soft_errors: Boolean, if True, suppresses any applicable errors and instead
-      returns to the caller.
-  """
-  token_error = str(e).replace('.', '')
-  if token_error in errors.OAUTH2_TOKEN_ERRORS or e.startswith(
-      'Invalid response'):
-    if soft_errors:
-      return
-    if not GM_Globals[GM_CURRENT_API_USER]:
-      display.print_error(
-          MESSAGE_API_ACCESS_DENIED.format(
-              GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
-              ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
-      controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
-    else:
-      controlflow.system_error_exit(
-          19,
-          MESSAGE_SERVICE_NOT_APPLICABLE.format(
-              GM_Globals[GM_CURRENT_API_USER]))
-  controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
-
-def get_enum_values_minus_unspecified(values):
-  return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gapi/__init___test.py

@@ -1,502 +0,0 @@
-"""Tests for gapi."""
-
-import json
-import unittest
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-from gam import SetGlobalVariables
-import gapi
-from gapi import errors
-
-
-def create_http_error(status, reason, message):
-  """Creates a HttpError object similar to most Google API Errors.
-
-  Args:
-    status: Int, the error's HTTP response status number.
-    reason: String, a camelCase reason for the HttpError being given.
-    message: String, a general error message describing the error that occurred.
-
-  Returns:
-    googleapiclient.errors.HttpError
-  """
-  response = {
-      'status': status,
-      'content-type': 'application/json',
-  }
-  content = {
-      'error': {
-          'code': status,
-          'errors': [{
-              'reason': str(reason),
-              'message': message,
-          }]
-      }
-  }
-  content_bytes = json.dumps(content).encode('UTF-8')
-  return gapi.googleapiclient.errors.HttpError(response, content_bytes)
-
-
-class GapiTest(unittest.TestCase):
-
-  def setUp(self):
-    SetGlobalVariables()
-    self.mock_service = MagicMock()
-    self.mock_method_name = 'mock_method'
-    self.mock_method = getattr(self.mock_service, self.mock_method_name)
-
-    self.simple_3_page_response = [
-        {
-            'items': [{
-                'position': 'page1,item1'
-            }, {
-                'position': 'page1,item2'
-            }, {
-                'position': 'page1,item3'
-            }],
-            'nextPageToken': 'page2'
-        },
-        {
-            'items': [{
-                'position': 'page2,item1'
-            }, {
-                'position': 'page2,item2'
-            }, {
-                'position': 'page2,item3'
-            }],
-            'nextPageToken': 'page3'
-        },
-        {
-            'items': [{
-                'position': 'page3,item1'
-            }, {
-                'position': 'page3,item2'
-            }, {
-                'position': 'page3,item3'
-            }],
-        },
-    ]
-    self.empty_items_response = {'items': []}
-
-    super(GapiTest, self).setUp()
-
-  def test_call_returns_basic_200_response(self):
-    response = gapi.call(self.mock_service, self.mock_method_name)
-    self.assertEqual(response, self.mock_method().execute.return_value)
-
-  def test_call_passes_target_method_params(self):
-    gapi.call(
-        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
-    self.assertEqual(self.mock_method.call_count, 1)
-    method_kwargs = self.mock_method.call_args[1]
-    self.assertEqual(method_kwargs.get('my_param_1'), 1)
-    self.assertEqual(method_kwargs.get('my_param_2'), 2)
-
-  @patch.object(gapi.errors, 'get_gapi_error_detail')
-  def test_call_retries_with_soft_errors(self, mock_error_detail):
-    mock_error_detail.return_value = (-1, 'aReason', 'some message')
-
-    # Make the request fail first, then return the proper response on the retry.
-    fake_http_error = create_http_error(403, 'aReason', 'unused message')
-    fake_200_response = MagicMock()
-    self.mock_method.return_value.execute.side_effect = [
-        fake_http_error, fake_200_response
-    ]
-
-    response = gapi.call(
-        self.mock_service, self.mock_method_name, soft_errors=True)
-    self.assertEqual(response, fake_200_response)
-    self.assertEqual(
-        self.mock_service._http.credentials.refresh.call_count, 1)
-    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
-
-  def test_call_throws_for_provided_reason(self):
-    throw_reason = errors.ErrorReason.USER_NOT_FOUND
-    fake_http_error = create_http_error(404, throw_reason, 'forced throw')
-    self.mock_method.return_value.execute.side_effect = fake_http_error
-
-    gam_exception = errors.ERROR_REASON_TO_EXCEPTION[throw_reason]
-    with self.assertRaises(gam_exception):
-      gapi.call(
-          self.mock_service,
-          self.mock_method_name,
-          throw_reasons=[throw_reason])
-
-  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
-  # we're not actually testing over a network connection
-  @patch.object(gapi.controlflow, 'wait_on_failure')
-  def test_call_retries_request_for_default_retry_reasons(
-      self, mock_wait_on_failure):
-
-    # Test using one of the default retry reasons
-    default_throw_reason = errors.ErrorReason.BACKEND_ERROR
-    self.assertIn(default_throw_reason, errors.DEFAULT_RETRY_REASONS)
-
-    fake_http_error = create_http_error(404, default_throw_reason, 'message')
-    fake_200_response = MagicMock()
-    # Fail once, then succeed on retry
-    self.mock_method.return_value.execute.side_effect = [
-        fake_http_error, fake_200_response
-    ]
-
-    response = gapi.call(
-        self.mock_service, self.mock_method_name, retry_reasons=[])
-    self.assertEqual(response, fake_200_response)
-    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
-    # Make sure a backoff technique was used for retry.
-    self.assertEqual(mock_wait_on_failure.call_count, 1)
-
-  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
-  # we're not actually testing over a network connection
-  @patch.object(gapi.controlflow, 'wait_on_failure')
-  def test_call_retries_requests_for_provided_retry_reasons(
-      self, unused_mock_wait_on_failure):
-
-    retry_reason1 = errors.ErrorReason.INTERNAL_ERROR
-    fake_retrieable_error1 = create_http_error(400, retry_reason1,
-                                               'Forced Error 1')
-    retry_reason2 = errors.ErrorReason.SYSTEM_ERROR
-    fake_retrieable_error2 = create_http_error(400, retry_reason2,
-                                               'Forced Error 2')
-    non_retriable_reason = errors.ErrorReason.SERVICE_NOT_AVAILABLE
-    fake_non_retriable_error = create_http_error(
-        400, non_retriable_reason,
-        'This error should not cause the request to be retried')
-    # Fail once, then succeed on retry
-    self.mock_method.return_value.execute.side_effect = [
-        fake_retrieable_error1, fake_retrieable_error2, fake_non_retriable_error
-    ]
-
-    with self.assertRaises(SystemExit):
-      # The third call should raise the SystemExit when non_retriable_error is
-      # raised.
-      gapi.call(
-          self.mock_service,
-          self.mock_method_name,
-          retry_reasons=[retry_reason1, retry_reason2])
-
-    self.assertEqual(self.mock_method.return_value.execute.call_count, 3)
-
-  def test_call_exits_on_oauth_token_error(self):
-    # An error with any OAUTH2_TOKEN_ERROR
-    fake_token_error = gapi.google.auth.exceptions.RefreshError(
-        errors.OAUTH2_TOKEN_ERRORS[0])
-    self.mock_method.return_value.execute.side_effect = fake_token_error
-
-    with self.assertRaises(SystemExit):
-      gapi.call(self.mock_service, self.mock_method_name)
-
-  def test_call_exits_on_nonretriable_error(self):
-    error_reason = 'unknownReason'
-    fake_http_error = create_http_error(500, error_reason,
-                                        'Testing unretriable errors')
-    self.mock_method.return_value.execute.side_effect = fake_http_error
-
-    with self.assertRaises(SystemExit):
-      gapi.call(self.mock_service, self.mock_method_name)
-
-  def test_call_exits_on_request_valueerror(self):
-    self.mock_method.return_value.execute.side_effect = ValueError()
-
-    with self.assertRaises(SystemExit):
-      gapi.call(self.mock_service, self.mock_method_name)
-
-  def test_call_clears_bad_http_cache_on_request_failure(self):
-    self.mock_service._http.cache = 'something that is not None'
-    fake_200_response = MagicMock()
-    self.mock_method.return_value.execute.side_effect = [
-        ValueError(), fake_200_response
-    ]
-
-    self.assertIsNotNone(self.mock_service._http.cache)
-    response = gapi.call(self.mock_service, self.mock_method_name)
-    self.assertEqual(response, fake_200_response)
-    # Assert the cache was cleared
-    self.assertIsNone(self.mock_service._http.cache)
-
-  # Prevent wait_on_failure from performing actual backoff unnecessarily, since
-  # we're not actually testing over a network connection
-  @patch.object(gapi.controlflow, 'wait_on_failure')
-  def test_call_retries_requests_with_backoff_on_servernotfounderror(
-      self, mock_wait_on_failure):
-    fake_servernotfounderror = gapi.httplib2.ServerNotFoundError()
-    fake_200_response = MagicMock()
-    # Fail once, then succeed on retry
-    self.mock_method.return_value.execute.side_effect = [
-        fake_servernotfounderror, fake_200_response
-    ]
-
-    http_connections = self.mock_service._http.connections
-    response = gapi.call(self.mock_service, self.mock_method_name)
-    self.assertEqual(response, fake_200_response)
-    # HTTP cached connections should be cleared on receiving this error
-    self.assertNotEqual(http_connections, self.mock_service._http.connections)
-    self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
-    # Make sure a backoff technique was used for retry.
-    self.assertEqual(mock_wait_on_failure.call_count, 1)
-
-  def test_get_items_calls_correct_service_function(self):
-    gapi.get_items(self.mock_service, self.mock_method_name)
-    self.assertTrue(self.mock_method.called)
-
-  def test_get_items_returns_one_page(self):
-    fake_response = {'items': [{}, {}, {}]}
-    self.mock_method.return_value.execute.return_value = fake_response
-    page = gapi.get_items(self.mock_service, self.mock_method_name)
-    self.assertEqual(page, fake_response['items'])
-
-  def test_get_items_non_default_page_field_name(self):
-    field_name = 'things'
-    fake_response = {field_name: [{}, {}, {}]}
-    self.mock_method.return_value.execute.return_value = fake_response
-    page = gapi.get_items(
-        self.mock_service, self.mock_method_name, items=field_name)
-    self.assertEqual(page, fake_response[field_name])
-
-  def test_get_items_passes_additional_kwargs_to_service(self):
-    gapi.get_items(
-        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
-    self.assertEqual(self.mock_method.call_count, 1)
-    method_kwargs = self.mock_method.call_args[1]
-    self.assertEqual(1, method_kwargs.get('my_param_1'))
-    self.assertEqual(2, method_kwargs.get('my_param_2'))
-
-  def test_get_items_returns_empty_list_when_no_items_returned(self):
-    non_items_response = {'noItemsInThisResponse': {}}
-    self.mock_method.return_value.execute.return_value = non_items_response
-    page = gapi.get_items(self.mock_service, self.mock_method_name)
-    self.assertIsInstance(page, list)
-    self.assertEqual(0, len(page))
-
-  def test_get_all_pages_returns_all_items(self):
-    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': '2'}
-    page_2 = {'items': ['2-1', '2-2', '2-3'], 'nextPageToken': '3'}
-    page_3 = {'items': ['3-1', '3-2', '3-3']}
-    self.mock_method.return_value.execute.side_effect = [page_1, page_2, page_3]
-    response_items = gapi.get_all_pages(self.mock_service,
-                                        self.mock_method_name)
-    self.assertListEqual(response_items,
-                         page_1['items'] + page_2['items'] + page_3['items'])
-
-  def test_get_all_pages_includes_next_pagetoken_in_request(self):
-    page_1 = {'items': ['1-1', '1-2', '1-3'], 'nextPageToken': 'someToken'}
-    page_2 = {'items': ['2-1', '2-2', '2-3']}
-    self.mock_method.return_value.execute.side_effect = [page_1, page_2]
-
-    gapi.get_all_pages(self.mock_service, self.mock_method_name, pageSize=100)
-    self.assertEqual(self.mock_method.call_count, 2)
-    call_2_kwargs = self.mock_method.call_args_list[1][1]
-    self.assertIn('pageToken', call_2_kwargs)
-    self.assertEqual(call_2_kwargs['pageToken'], page_1['nextPageToken'])
-
-  def test_get_all_pages_uses_default_max_page_size(self):
-    sample_api_id = list(gapi.MAX_RESULTS_API_EXCEPTIONS.keys())[0]
-    sample_api_max_results = gapi.MAX_RESULTS_API_EXCEPTIONS[sample_api_id]
-    self.mock_method.return_value.methodId = sample_api_id
-    self.mock_service._rootDesc = {
-        'resources': {
-            'someResource': {
-                'methods': {
-                    'someMethod': {
-                        'id': sample_api_id,
-                        'parameters': {
-                            'maxResults': {
-                                'maximum': sample_api_max_results
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-    self.mock_method.return_value.execute.return_value = self.empty_items_response
-
-    gapi.get_all_pages(self.mock_service, self.mock_method_name)
-    request_method_kwargs = self.mock_method.call_args[1]
-    self.assertIn('maxResults', request_method_kwargs)
-    self.assertEqual(request_method_kwargs['maxResults'],
-                     gapi.MAX_RESULTS_API_EXCEPTIONS.get(sample_api_id))
-
-  def test_get_all_pages_max_page_size_overrided(self):
-    self.mock_method.return_value.execute.return_value = self.empty_items_response
-
-    gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, pageSize=123456)
-    request_method_kwargs = self.mock_method.call_args[1]
-    self.assertIn('pageSize', request_method_kwargs)
-    self.assertEqual(123456, request_method_kwargs['pageSize'])
-
-  def test_get_all_pages_prints_paging_message(self):
-    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
-
-    paging_message = 'A simple string displayed during paging'
-    with patch.object(gapi.sys.stderr, 'write') as mock_write:
-      gapi.get_all_pages(
-          self.mock_service, self.mock_method_name, page_message=paging_message)
-    messages_written = [
-        call_args[0][0] for call_args in mock_write.call_args_list
-    ]
-    self.assertIn(paging_message, messages_written)
-
-  def test_get_all_pages_prints_paging_message_inline(self):
-    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
-
-    paging_message = 'A simple string displayed during paging'
-    with patch.object(gapi.sys.stderr, 'write') as mock_write:
-      gapi.get_all_pages(
-          self.mock_service, self.mock_method_name, page_message=paging_message)
-    messages_written = [
-        call_args[0][0] for call_args in mock_write.call_args_list
-    ]
-
-    # Make sure a return carriage was written between two pages
-    paging_message_call_positions = [
-        i for i, message in enumerate(messages_written)
-        if message == paging_message
-    ]
-    self.assertGreater(len(paging_message_call_positions), 1)
-    printed_between_page_messages = messages_written[
-        paging_message_call_positions[0]:paging_message_call_positions[1]]
-    self.assertIn('\r', printed_between_page_messages)
-
-  def test_get_all_pages_ends_paging_message_with_newline(self):
-    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
-
-    paging_message = 'A simple string displayed during paging'
-    with patch.object(gapi.sys.stderr, 'write') as mock_write:
-      gapi.get_all_pages(
-          self.mock_service, self.mock_method_name, page_message=paging_message)
-    messages_written = [
-        call_args[0][0] for call_args in mock_write.call_args_list
-    ]
-    last_page_message_index = len(
-        messages_written) - messages_written[::-1].index(paging_message)
-    last_carriage_return_index = len(
-        messages_written) - messages_written[::-1].index('\r\n')
-    self.assertGreater(last_carriage_return_index, last_page_message_index)
-
-  def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
-    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
-
-    paging_message = 'Total number of items discovered: %%total_items%%'
-    with patch.object(gapi.sys.stderr, 'write') as mock_write:
-      gapi.get_all_pages(
-          self.mock_service, self.mock_method_name, page_message=paging_message)
-
-    messages_written = [
-        call_args[0][0] for call_args in mock_write.call_args_list
-    ]
-    page_1_item_count = len(self.simple_3_page_response[0]['items'])
-    page_1_message = paging_message.replace('%%total_items%%',
-                                            str(page_1_item_count))
-    self.assertIn(page_1_message, messages_written)
-
-    page_2_item_count = len(self.simple_3_page_response[1]['items'])
-    page_2_message = paging_message.replace(
-        '%%total_items%%', str(page_1_item_count + page_2_item_count))
-    self.assertIn(page_2_message, messages_written)
-
-    page_3_item_count = len(self.simple_3_page_response[2]['items'])
-    page_3_message = paging_message.replace(
-        '%%total_items%%',
-        str(page_1_item_count + page_2_item_count + page_3_item_count))
-    self.assertIn(page_3_message, messages_written)
-
-    # Assert that the template text is always replaced.
-    for message in messages_written:
-      self.assertNotIn('%%total_items', message)
-
-  def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
-    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
-
-    paging_message = 'First item in page: %%first_item%%'
-
-    with patch.object(gapi.sys.stderr, 'write') as mock_write:
-      gapi.get_all_pages(
-          self.mock_service,
-          self.mock_method_name,
-          page_message=paging_message,
-          message_attribute='position')
-
-    messages_written = [
-        call_args[0][0] for call_args in mock_write.call_args_list
-    ]
-    page_1_message = paging_message.replace(
-        '%%first_item%%',
-        self.simple_3_page_response[0]['items'][0]['position'])
-    self.assertIn(page_1_message, messages_written)
-
-    page_2_message = paging_message.replace(
-        '%%first_item%%',
-        self.simple_3_page_response[1]['items'][0]['position'])
-    self.assertIn(page_2_message, messages_written)
-
-    # Assert that the template text is always replaced.
-    for message in messages_written:
-      self.assertNotIn('%%first_item', message)
-
-  def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
-    self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
-
-    paging_message = 'Last item in page: %%last_item%%'
-    with patch.object(gapi.sys.stderr, 'write') as mock_write:
-      gapi.get_all_pages(
-          self.mock_service,
-          self.mock_method_name,
-          page_message=paging_message,
-          message_attribute='position')
-
-    messages_written = [
-        call_args[0][0] for call_args in mock_write.call_args_list
-    ]
-    page_1_message = paging_message.replace(
-        '%%last_item%%',
-        self.simple_3_page_response[0]['items'][-1]['position'])
-    self.assertIn(page_1_message, messages_written)
-
-    page_2_message = paging_message.replace(
-        '%%last_item%%',
-        self.simple_3_page_response[1]['items'][-1]['position'])
-    self.assertIn(page_2_message, messages_written)
-
-    # Assert that the template text is always replaced.
-    for message in messages_written:
-      self.assertNotIn('%%last_item', message)
-
-  def test_get_all_pages_prints_all_attributes_in_paging_message(self):
-    pass
-
-  def test_get_all_pages_passes_additional_kwargs_to_service_method(self):
-    self.mock_method.return_value.execute.return_value = self.empty_items_response
-    gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, my_param_1=1, my_param_2=2)
-    method_kwargs = self.mock_method.call_args[1]
-    self.assertEqual(method_kwargs.get('my_param_1'), 1)
-    self.assertEqual(method_kwargs.get('my_param_2'), 2)
-
-  @patch.object(gapi, 'call')
-  def test_get_all_pages_passes_throw_and_retry_reasons(self, mock_call):
-    throw_for = MagicMock()
-    retry_for = MagicMock()
-    mock_call.return_value = self.empty_items_response
-    gapi.get_all_pages(
-        self.mock_service,
-        self.mock_method_name,
-        throw_reasons=throw_for,
-        retry_reasons=retry_for)
-    method_kwargs = mock_call.call_args[1]
-    self.assertEqual(method_kwargs.get('throw_reasons'), throw_for)
-    self.assertEqual(method_kwargs.get('retry_reasons'), retry_for)
-
-  def test_get_all_pages_non_default_items_field_name(self):
-    field_name = 'things'
-    fake_response = {field_name: [{}, {}, {}]}
-    self.mock_method.return_value.execute.return_value = fake_response
-    page = gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, items=field_name)
-    self.assertEqual(page, fake_response[field_name])
-
-
-if __name__ == '__main__':
-  unittest.main()

src/gapi/directory/__init__.py

@@ -1,5 +0,0 @@
-import __main__
-
-
-def buildGAPIObject():
-    return __main__.buildGAPIObject('directory')

src/gapi/errors.py

@@ -1,362 +0,0 @@
-"""GAPI and OAuth Token related errors methods."""
-
-from enum import Enum
-import json
-
-import controlflow
-import display  # TODO: Change to relative import when gam is setup as a package
-from var import UTF8
-
-
-class GapiAbortedError(Exception):
-  pass
-
-
-class GapiAuthErrorError(Exception):
-  pass
-
-
-class GapiBadGatewayError(Exception):
-  pass
-
-
-class GapiBadRequestError(Exception):
-  pass
-
-
-class GapiConditionNotMetError(Exception):
-  pass
-
-
-class GapiCyclicMembershipsNotAllowedError(Exception):
-  pass
-
-
-class GapiDomainCannotUseApisError(Exception):
-  pass
-
-
-class GapiDomainNotFoundError(Exception):
-  pass
-
-
-class GapiDuplicateError(Exception):
-  pass
-
-
-class GapiFailedPreconditionError(Exception):
-  pass
-
-
-class GapiForbiddenError(Exception):
-  pass
-
-
-class GapiGatewayTimeoutError(Exception):
-  pass
-
-
-class GapiGroupNotFoundError(Exception):
-  pass
-
-
-class GapiInvalidError(Exception):
-  pass
-
-
-class GapiInvalidArgumentError(Exception):
-  pass
-
-
-class GapiInvalidMemberError(Exception):
-  pass
-
-
-class GapiMemberNotFoundError(Exception):
-  pass
-
-
-class GapiNotFoundError(Exception):
-  pass
-
-
-class GapiNotImplementedError(Exception):
-  pass
-
-
-class GapiPermissionDeniedError(Exception):
-  pass
-
-
-class GapiResourceNotFoundError(Exception):
-  pass
-
-
-class GapiServiceNotAvailableError(Exception):
-  pass
-
-
-class GapiUserNotFoundError(Exception):
-  pass
-
-
-# GAPI Error Reasons
-class ErrorReason(Enum):
-  """The reason why a non-200 HTTP response was returned from a GAPI."""
-  ABORTED = 'aborted'
-  AUTH_ERROR = 'authError'
-  BACKEND_ERROR = 'backendError'
-  BAD_GATEWAY = 'badGateway'
-  BAD_REQUEST = 'badRequest'
-  CONDITION_NOT_MET = 'conditionNotMet'
-  CYCLIC_MEMBERSHIPS_NOT_ALLOWED = 'cyclicMembershipsNotAllowed'
-  DOMAIN_CANNOT_USE_APIS = 'domainCannotUseApis'
-  DOMAIN_NOT_FOUND = 'domainNotFound'
-  DUPLICATE = 'duplicate'
-  FAILED_PRECONDITION = 'failedPrecondition'
-  FORBIDDEN = 'forbidden'
-  FOUR_O_THREE = '403'
-  GATEWAY_TIMEOUT = 'gatewayTimeout'
-  GROUP_NOT_FOUND = 'groupNotFound'
-  INTERNAL_ERROR = 'internalError'
-  INVALID = 'invalid'
-  INVALID_ARGUMENT = 'invalidArgument'
-  INVALID_MEMBER = 'invalidMember'
-  MEMBER_NOT_FOUND = 'memberNotFound'
-  NOT_FOUND = 'notFound'
-  NOT_IMPLEMENTED = 'notImplemented'
-  PERMISSION_DENIED = 'permissionDenied'
-  QUOTA_EXCEEDED = 'quotaExceeded'
-  RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
-  RESOURCE_NOT_FOUND = 'resourceNotFound'
-  SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
-  SERVICE_LIMIT = 'serviceLimit'
-  SYSTEM_ERROR = 'systemError'
-  USER_NOT_FOUND = 'userNotFound'
-  USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
-  FOUR_TWO_NINE = '429'
-  DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
-
-  def __str__(self):
-    return str(self.value)
-
-
-# Common sets of GAPI error reasons
-DEFAULT_RETRY_REASONS = [
-  ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
-  ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
-  ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT,
-  ErrorReason.INTERNAL_ERROR, ErrorReason.FOUR_TWO_NINE,
-  ]
-GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
-GROUP_GET_THROW_REASONS = [
-  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
-  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.FORBIDDEN,
-  ErrorReason.BAD_REQUEST
-  ]
-GROUP_GET_RETRY_REASONS = [ErrorReason.INVALID, ErrorReason.SYSTEM_ERROR]
-MEMBERS_THROW_REASONS = [
-  ErrorReason.GROUP_NOT_FOUND, ErrorReason.DOMAIN_NOT_FOUND,
-  ErrorReason.DOMAIN_CANNOT_USE_APIS, ErrorReason.INVALID,
-  ErrorReason.FORBIDDEN
-  ]
-MEMBERS_RETRY_REASONS = [ErrorReason.SYSTEM_ERROR]
-
-# A map of GAPI error reasons to the corresponding GAM Python Exception
-ERROR_REASON_TO_EXCEPTION = {
-  ErrorReason.ABORTED:
-    GapiAbortedError,
-  ErrorReason.AUTH_ERROR:
-    GapiAuthErrorError,
-  ErrorReason.BAD_GATEWAY:
-    GapiBadGatewayError,
-  ErrorReason.BAD_REQUEST:
-    GapiBadRequestError,
-  ErrorReason.CONDITION_NOT_MET:
-    GapiConditionNotMetError,
-  ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED:
-    GapiCyclicMembershipsNotAllowedError,
-  ErrorReason.DOMAIN_CANNOT_USE_APIS:
-    GapiDomainCannotUseApisError,
-  ErrorReason.DOMAIN_NOT_FOUND:
-    GapiDomainNotFoundError,
-  ErrorReason.DUPLICATE:
-    GapiDuplicateError,
-  ErrorReason.FAILED_PRECONDITION:
-    GapiFailedPreconditionError,
-  ErrorReason.FORBIDDEN:
-    GapiForbiddenError,
-  ErrorReason.GATEWAY_TIMEOUT:
-    GapiGatewayTimeoutError,
-  ErrorReason.GROUP_NOT_FOUND:
-    GapiGroupNotFoundError,
-  ErrorReason.INVALID:
-    GapiInvalidError,
-  ErrorReason.INVALID_ARGUMENT:
-    GapiInvalidArgumentError,
-  ErrorReason.INVALID_MEMBER:
-    GapiInvalidMemberError,
-  ErrorReason.MEMBER_NOT_FOUND:
-    GapiMemberNotFoundError,
-  ErrorReason.NOT_FOUND:
-    GapiNotFoundError,
-  ErrorReason.NOT_IMPLEMENTED:
-    GapiNotImplementedError,
-  ErrorReason.PERMISSION_DENIED:
-    GapiPermissionDeniedError,
-  ErrorReason.RESOURCE_NOT_FOUND:
-    GapiResourceNotFoundError,
-  ErrorReason.SERVICE_NOT_AVAILABLE:
-    GapiServiceNotAvailableError,
-  ErrorReason.USER_NOT_FOUND:
-    GapiUserNotFoundError,
-  }
-
-# OAuth Token Errors
-OAUTH2_TOKEN_ERRORS = [
-  'access_denied',
-  'access_denied: Requested client not authorized',
-  'internal_failure: Backend Error',
-  'internal_failure: None',
-  'invalid_grant',
-  'invalid_grant: Bad Request',
-  'invalid_grant: Invalid email or User ID',
-  'invalid_grant: Not a valid email',
-  'invalid_grant: Invalid JWT: No valid verifier found for issuer',
-  'invalid_request: Invalid impersonation prn email address',
-  'unauthorized_client: Client is unauthorized to retrieve access tokens '
-  'using this method',
-  'unauthorized_client: Client is unauthorized to retrieve access tokens '
-  'using this method, or client not authorized for any of the scopes '
-  'requested',
-  'unauthorized_client: Unauthorized client or scope in request',
-  ]
-
-
-def _create_http_error_dict(status_code, reason, message):
-  """Creates a basic error dict similar to most Google API Errors.
-
-  Args:
-    status_code: Int, the error's HTTP response status code.
-    reason: String, a camelCase reason for the HttpError being given.
-    message: String, a general error message describing the error that occurred.
-
-  Returns:
-    dict
-  """
-  return {
-    'error': {
-      'code': status_code,
-      'errors': [{
-        'reason': str(reason),
-        'message': message,
-        }]
-      }
-  }
-
-
-def get_gapi_error_detail(e,
-                          soft_errors=False,
-                          silent_errors=False,
-                          retry_on_http_error=False):
-  """Extracts error detail from a non-200 GAPI Response.
-
-  Args:
-    e: googleapiclient.HttpError, The HTTP Error received.
-    soft_errors: Boolean, If true, causes error messages to be surpressed,
-      rather than sending them to stderr.
-    silent_errors: Boolean, If true, suppresses and ignores any errors from
-      being displayed
-    retry_on_http_error: Boolean, If true, will return -1 as the HTTP Response
-      code, indicating that the request can be retried. TODO: Remove this param,
-        as it seems to be outside the scope of this method.
-
-  Returns:
-    A tuple containing the HTTP Response code, GAPI error reason, and error
-        message.
-  """
-  try:
-    error = json.loads(e.content.decode(UTF8))
-  except ValueError:
-    error_content = e.content.decode(UTF8) if isinstance(e.content,
-                                                         bytes) else e.content
-    if (e.resp['status'] == '503') and (
-        error_content == 'Quota exceeded for the current request'):
-      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
-    if (e.resp['status'] == '403') and (
-        error_content.startswith('Request rate higher than configured')):
-      return (e.resp['status'], ErrorReason.QUOTA_EXCEEDED.value, error_content)
-    if (e.resp['status'] == '502') and ('Bad Gateway' in error_content):
-      return (e.resp['status'], ErrorReason.BAD_GATEWAY.value, error_content)
-    if (e.resp['status'] == '504') and ('Gateway Timeout' in error_content):
-      return (e.resp['status'], ErrorReason.GATEWAY_TIMEOUT.value, error_content)
-    if (e.resp['status'] == '403') and ('Invalid domain.' in error_content):
-      error = _create_http_error_dict(403, ErrorReason.NOT_FOUND.value,
-                                      'Domain not found')
-    elif (e.resp['status'] == '400') and (
-        'InvalidSsoSigningKey' in error_content):
-      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
-                                      'InvalidSsoSigningKey')
-    elif (e.resp['status'] == '400') and ('UnknownError' in error_content):
-      error = _create_http_error_dict(400, ErrorReason.INVALID.value,
-                                      'UnknownError')
-    elif retry_on_http_error:
-      return (-1, None, None)
-    elif soft_errors:
-      if not silent_errors:
-        display.print_error(error_content)
-      return (0, None, None)
-    else:
-      controlflow.system_error_exit(5, error_content)
-    # END: ValueError catch
-
-  if 'error' in error:
-    http_status = error['error']['code']
-    try:
-      message = error['error']['errors'][0]['message']
-    except KeyError:
-      message = error['error']['message']
-  else:
-    if 'error_description' in error:
-      if error['error_description'] == 'Invalid Value':
-        message = error['error_description']
-        http_status = 400
-        error = _create_http_error_dict(400, ErrorReason.INVALID.value, message)
-      else:
-        controlflow.system_error_exit(4, str(error))
-    else:
-      controlflow.system_error_exit(4, str(error))
-
-  # Extract the error reason
-  try:
-    reason = error['error']['errors'][0]['reason']
-    if reason == 'notFound':
-      if 'userKey' in message:
-        reason = ErrorReason.USER_NOT_FOUND.value
-      elif 'groupKey' in message:
-        reason = ErrorReason.GROUP_NOT_FOUND.value
-      elif 'memberKey' in message:
-        reason = ErrorReason.MEMBER_NOT_FOUND.value
-      elif 'Domain not found' in message:
-        reason = ErrorReason.DOMAIN_NOT_FOUND.value
-      elif 'Resource Not Found' in message:
-        reason = ErrorReason.RESOURCE_NOT_FOUND.value
-    elif reason == 'invalid':
-      if 'userId' in message:
-        reason = ErrorReason.USER_NOT_FOUND.value
-      elif 'memberKey' in message:
-        reason = ErrorReason.INVALID_MEMBER.value
-    elif reason == 'failedPrecondition':
-      if 'Bad Request' in message:
-        reason = ErrorReason.BAD_REQUEST.value
-      elif 'Mail service not enabled' in message:
-        reason = ErrorReason.SERVICE_NOT_AVAILABLE.value
-    elif reason == 'required':
-      if 'memberKey' in message:
-        reason = ErrorReason.MEMBER_NOT_FOUND.value
-    elif reason == 'conditionNotMet':
-      if 'Cyclic memberships not allowed' in message:
-        reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
-  except KeyError:
-    reason = f'{http_status}'
-  return (http_status, reason, message)

src/gapi/errors_test.py

@@ -1,209 +0,0 @@
-"""Python unit tests for gapi.errors"""
-
-import json
-
-import unittest
-from unittest.mock import patch
-
-import googleapiclient.errors
-from gapi import errors
-
-
-def create_simple_http_error(status, reason, message):
-  content = errors._create_http_error_dict(status, reason, message)
-  return create_http_error(status, content)
-
-
-def create_http_error(status, content):
-  response = {
-      'status': status,
-      'content-type': 'application/json',
-  }
-  content_as_bytes = json.dumps(content).encode('UTF-8')
-  return googleapiclient.errors.HttpError(response, content_as_bytes)
-
-
-class ErrorsTest(unittest.TestCase):
-
-  def test_get_gapi_error_detail_quota_exceeded(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_detail_invalid_domain(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_detail_invalid_signing_key(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_detail_unknown_error(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_retry_http_error(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_prints_soft_errors(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_exits_on_unrecoverable_errors(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_quota_exceeded_for_current_request(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_quota_exceeded_high_request_rate(self):
-    # TODO: Add test logic once the opening ValueError exception case has a
-    # repro case (i.e. an Exception type/format that will cause it to raise).
-    pass
-
-  def test_get_gapi_error_extracts_user_not_found(self):
-    err = create_simple_http_error(404, 'notFound',
-                                   'Resource Not Found: userKey.')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 404)
-    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
-    self.assertEqual(message, 'Resource Not Found: userKey.')
-
-  def test_get_gapi_error_extracts_group_not_found(self):
-    err = create_simple_http_error(404, 'notFound',
-                                   'Resource Not Found: groupKey.')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 404)
-    self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
-    self.assertEqual(message, 'Resource Not Found: groupKey.')
-
-  def test_get_gapi_error_extracts_member_not_found(self):
-    err = create_simple_http_error(404, 'notFound',
-                                   'Resource Not Found: memberKey.')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 404)
-    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
-    self.assertEqual(message, 'Resource Not Found: memberKey.')
-
-  def test_get_gapi_error_extracts_domain_not_found(self):
-    err = create_simple_http_error(404, 'notFound', 'Domain not found.')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 404)
-    self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
-    self.assertEqual(message, 'Domain not found.')
-
-  def test_get_gapi_error_extracts_generic_resource_not_found(self):
-    err = create_simple_http_error(404, 'notFound',
-                                   'Resource Not Found: unknownResource.')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 404)
-    self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
-    self.assertEqual(message, 'Resource Not Found: unknownResource.')
-
-  def test_get_gapi_error_extracts_invalid_userid(self):
-    err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 400)
-    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
-    self.assertEqual(message, 'Invalid Input: userId')
-
-  def test_get_gapi_error_extracts_invalid_member(self):
-    err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 400)
-    self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
-    self.assertEqual(message, 'Invalid Input: memberKey')
-
-  def test_get_gapi_error_extracts_bad_request(self):
-    err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 400)
-    self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
-    self.assertEqual(message, 'Bad Request')
-
-  def test_get_gapi_error_extracts_service_not_available(self):
-    err = create_simple_http_error(400, 'failedPrecondition',
-                                   'Mail service not enabled')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 400)
-    self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
-    self.assertEqual(message, 'Mail service not enabled')
-
-  def test_get_gapi_error_extracts_required_member_not_found(self):
-    err = create_simple_http_error(400, 'required',
-                                   'Missing required field: memberKey')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 400)
-    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
-    self.assertEqual(message, 'Missing required field: memberKey')
-
-  def test_get_gapi_error_extracts_cyclic_memberships_error(self):
-    err = create_simple_http_error(400, 'conditionNotMet',
-                                   'Cyclic memberships not allowed')
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, 400)
-    self.assertEqual(reason,
-                     errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
-    self.assertEqual(message, 'Cyclic memberships not allowed')
-
-  def test_get_gapi_error_extracts_single_error_with_message(self):
-    status_code = 999
-    response = {'status': status_code}
-    # This error does not have an "errors" key describing each error.
-    content = {'error': {'code': status_code, 'message': 'unknown error'}}
-    content_as_bytes = json.dumps(content).encode('UTF-8')
-    err = googleapiclient.errors.HttpError(response, content_as_bytes)
-
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, status_code)
-    self.assertEqual(reason, str(status_code))
-    self.assertEqual(message, content['error']['message'])
-
-  def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
-      self):
-    status_code = 999
-    response = {'status': status_code}
-    # This error only has an error_description_field and an unknown description.
-    content = {'error_description': 'something errored'}
-    content_as_bytes = json.dumps(content).encode('UTF-8')
-    err = googleapiclient.errors.HttpError(response, content_as_bytes)
-
-    with self.assertRaises(SystemExit) as context:
-      errors.get_gapi_error_detail(err)
-    self.assertEqual(4, context.exception.code)
-
-  def test_get_gapi_error_exits_on_invalid_error_description(self):
-    status_code = 400
-    response = {'status': status_code}
-    content = {'error_description': 'Invalid Value'}
-    content_as_bytes = json.dumps(content).encode('UTF-8')
-    err = googleapiclient.errors.HttpError(response, content_as_bytes)
-
-    http_status, reason, message = errors.get_gapi_error_detail(err)
-    self.assertEqual(http_status, status_code)
-    self.assertEqual(reason, errors.ErrorReason.INVALID.value)
-    self.assertEqual(message, 'Invalid Value')
-
-  def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
-    status_code = 900
-    response = {'status': status_code}
-    content = {'notErrorContentThatIsExpected': 'foo'}
-    content_as_bytes = json.dumps(content).encode('UTF-8')
-    err = googleapiclient.errors.HttpError(response, content_as_bytes)
-
-    with self.assertRaises(SystemExit) as context:
-      errors.get_gapi_error_detail(err)
-    self.assertEqual(4, context.exception.code)
-
-
-if __name__ == '__main__':
-  unittest.main()

src/linux-build.sh

@@ -1,12 +0,0 @@
-rm -rf gam
-rm -rf build
-rm -rf dist
-rm -rf gam-$1-linux-$(arch).tar.xz
-
-export LD_LIBRARY_PATH=/usr/local/lib
-pyinstaller --clean -F --distpath=gam linux-gam.spec
-cp LICENSE gam
-cp whatsnew.txt gam
-cp GamCommands.txt gam
-
-tar cfJ gam-$1-linux-$(arch).tar.xz gam/ 

src/macos-build.sh

@@ -1,11 +0,0 @@
-rm -rf gam
-rm -rf build
-rm -rf dist
-rm -rf gam-$1-macos.tar.xz
-
-/Library/Frameworks/Python.framework/Versions/2.7/bin/pyinstaller --clean -F --distpath=gam macos-gam.spec
-cp LICENSE gam
-cp whatsnew.txt gam
-cp GamCommands.txt gam
-
-tar cfJ gam-$1-macos.tar.xz gam/ 

src/macos-gam.spec

@@ -1,32 +0,0 @@
-# -*- mode: python -*-
-import sys
-
-sys.modules['FixTk'] = None
-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )

src/requirements-dev.txt

@@ -0,0 +1,7 @@
+# This file contains all requirements needed for GAM development work
+
+# Include all build requirements
+-r requirements.txt
+
+# Dev-specific requirements
+pre-commit

src/requirements.txt

@@ -1,10 +1,10 @@
 cryptography
-python-dateutil
 distro; sys_platform == 'linux'
 filelock
 google-api-python-client>=1.7.10
-google-auth>=1.11.2
 google-auth-httplib2
 google-auth-oauthlib>=0.4.1
+google-auth>=1.11.2
 httplib2>=0.17.0
 passlib>=1.7.2; sys_platform == 'win32'
+python-dateutil

src/tools/a_atleast_b.py

@@ -9,7 +9,7 @@ b = sys.argv[2]
 #result = version.parse(a) >= version.parse(b)
 result = LooseVersion(a) >= LooseVersion(b)
 if result:
-  print('OK: %s is equal or newer than %s' % (a, b))
+    print('OK: %s is equal or newer than %s' % (a, b))
 else:
-  print('ERROR: %s is older than %s' % (a, b))
+    print('ERROR: %s is older than %s' % (a, b))
 sys.exit(not result)

src/transport.py

@@ -1,103 +0,0 @@
-"""Methods related to network transport."""
-
-import google_auth_httplib2
-import httplib2
-
-from var import GAM_INFO
-from var import GC_CA_FILE
-from var import GC_TLS_MAX_VERSION
-from var import GC_TLS_MIN_VERSION
-from var import GC_Values
-
-
-def create_http(cache=None,
-                timeout=None,
-                override_min_tls=None,
-                override_max_tls=None):
-  """Creates a uniform HTTP transport object.
-
-  Args:
-    cache: The HTTP cache to use.
-    timeout: The cache timeout, in seconds.
-    override_min_tls: The minimum TLS version to require. If not provided, the
-      default is used.
-    override_max_tls: The maximum TLS version to require. If not provided, the
-      default is used.
-
-  Returns:
-    httplib2.Http with the specified options.
-  """
-  tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
-      GC_TLS_MIN_VERSION)
-  tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
-      GC_TLS_MAX_VERSION)
-  httpObj = httplib2.Http(
-      ca_certs=GC_Values.get(GC_CA_FILE),
-      tls_maximum_version=tls_maximum_version,
-      tls_minimum_version=tls_minimum_version,
-      cache=cache,
-      timeout=timeout)
-  httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
-  return httpObj
-
-
-def create_request(http=None):
-  """Creates a uniform Request object with a default http, if not provided.
-
-  Args:
-    http: Optional httplib2.Http compatible object to be used with the request.
-      If not provided, a default HTTP will be used.
-
-  Returns:
-    Request: A google_auth_httplib2.Request compatible Request.
-  """
-  if not http:
-    http = create_http()
-  return Request(http)
-
-
-GAM_USER_AGENT = GAM_INFO
-
-
-def _force_user_agent(user_agent):
-  """Creates a decorator which can force a user agent in HTTP headers."""
-
-  def decorator(request_method):
-    """Wraps a request method to insert a user-agent in HTTP headers."""
-
-    def wrapped_request_method(*args, **kwargs):
-      """Modifies HTTP headers to include a specified user-agent."""
-      if kwargs.get('headers') is not None:
-        if kwargs['headers'].get('user-agent'):
-          if user_agent not in kwargs['headers']['user-agent']:
-            # Save the existing user-agent header and tack on our own.
-            kwargs['headers']['user-agent'] = (
-                f'{user_agent} '
-                f'{kwargs["headers"]["user-agent"]}')
-        else:
-          kwargs['headers']['user-agent'] = user_agent
-      else:
-        kwargs['headers'] = {'user-agent': user_agent}
-      return request_method(*args, **kwargs)
-
-    return wrapped_request_method
-
-  return decorator
-
-
-class Request(google_auth_httplib2.Request):
-  """A Request which forces a user agent."""
-
-  @_force_user_agent(GAM_USER_AGENT)
-  def __call__(self, *args, **kwargs):
-    """Inserts the GAM user-agent header in requests."""
-    return super(Request, self).__call__(*args, **kwargs)
-
-
-class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
-  """An AuthorizedHttp which forces a user agent during requests."""
-
-  @_force_user_agent(GAM_USER_AGENT)
-  def request(self, *args, **kwargs):
-    """Inserts the GAM user-agent header in requests."""
-    return super(AuthorizedHttp, self).request(*args, **kwargs)

src/transport_test.py

@@ -1,179 +0,0 @@
-"""Tests for transport."""
-
-import unittest
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-from gam import SetGlobalVariables
-import google_auth_httplib2
-import httplib2
-
-import transport
-
-
-class CreateHttpTest(unittest.TestCase):
-
-  def setUp(self):
-    SetGlobalVariables()
-    super(CreateHttpTest, self).setUp()
-
-  def test_create_http_sets_default_values_on_http(self):
-    http = transport.create_http()
-    self.assertIsNone(http.cache)
-    self.assertIsNone(http.timeout)
-    self.assertEqual(http.tls_minimum_version,
-                     transport.GC_Values[transport.GC_TLS_MIN_VERSION])
-    self.assertEqual(http.tls_maximum_version,
-                     transport.GC_Values[transport.GC_TLS_MAX_VERSION])
-    self.assertEqual(http.ca_certs, transport.GC_Values[transport.GC_CA_FILE])
-
-  def test_create_http_sets_tls_min_version(self):
-    http = transport.create_http(override_min_tls='TLSv1_1')
-    self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
-
-  def test_create_http_sets_tls_max_version(self):
-    http = transport.create_http(override_max_tls='TLSv1_3')
-    self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
-
-  def test_create_http_sets_cache(self):
-    fake_cache = {}
-    http = transport.create_http(cache=fake_cache)
-    self.assertEqual(http.cache, fake_cache)
-
-  def test_create_http_sets_cache_timeout(self):
-    http = transport.create_http(timeout=1234)
-    self.assertEqual(http.timeout, 1234)
-
-
-class TransportTest(unittest.TestCase):
-
-  def setUp(self):
-    self.mock_http = MagicMock(spec=httplib2.Http)
-    self.mock_response = MagicMock(spec=httplib2.Response)
-    self.mock_content = MagicMock()
-    self.mock_http.request.return_value = (self.mock_response,
-                                           self.mock_content)
-    self.mock_credentials = MagicMock()
-    self.test_uri = 'http://example.com'
-    super(TransportTest, self).setUp()
-
-  @patch.object(transport, 'create_http')
-  def test_create_request_uses_default_http(self, mock_create_http):
-    request = transport.create_request()
-    self.assertEqual(request.http, mock_create_http.return_value)
-
-  def test_create_request_uses_provided_http(self):
-    request = transport.create_request(http=self.mock_http)
-    self.assertEqual(request.http, self.mock_http)
-
-  def test_create_request_returns_request_with_forced_user_agent(self):
-    request = transport.create_request()
-    self.assertIsInstance(request, transport.Request)
-
-  def test_request_is_google_auth_httplib2_compatible(self):
-    request = transport.create_request()
-    self.assertIsInstance(request, google_auth_httplib2.Request)
-
-  def test_request_call_returns_response_content(self):
-    request = transport.Request(self.mock_http)
-    response = request(self.test_uri)
-    self.assertEqual(self.mock_response.status, response.status)
-    self.assertEqual(self.mock_content, response.data)
-
-  def test_request_call_forces_user_agent_no_provided_headers(self):
-    request = transport.Request(self.mock_http)
-
-    request(self.test_uri)
-    headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', headers)
-    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
-
-  def test_request_call_forces_user_agent_no_agent_in_headers(self):
-    request = transport.Request(self.mock_http)
-    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
-
-    request(self.test_uri, headers=fake_request_headers)
-    final_headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', final_headers)
-    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
-    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
-    self.assertEqual('someData',
-                     final_headers['some-header-thats-not-a-user-agent'])
-
-  def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
-    request = transport.Request(self.mock_http)
-    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
-
-    request(self.test_uri, headers=headers_with_user_agent)
-    final_headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', final_headers)
-    self.assertIn('existing-user-agent', final_headers['user-agent'])
-    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
-
-  def test_request_call_same_user_agent_already_in_headers(self):
-    request = transport.Request(self.mock_http)
-    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
-
-    request(self.test_uri, headers=same_user_agent_header)
-    final_headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', final_headers)
-    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
-    # Make sure the header wasn't duplicated
-    self.assertEqual(
-        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))
-
-  def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
-    http = transport.AuthorizedHttp(self.mock_credentials)
-    self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
-
-  def test_authorizedhttp_request_returns_response_content(self):
-    http = transport.AuthorizedHttp(self.mock_credentials, http=self.mock_http)
-    response, content = http.request(self.test_uri)
-    self.assertEqual(self.mock_response, response)
-    self.assertEqual(self.mock_content, content)
-
-  def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
-    authorized_http = transport.AuthorizedHttp(
-        self.mock_credentials, http=self.mock_http)
-    authorized_http.request(self.test_uri)
-    headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', headers)
-    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
-
-  def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
-    authorized_http = transport.AuthorizedHttp(
-        self.mock_credentials, http=self.mock_http)
-    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
-
-    authorized_http.request(self.test_uri, headers=fake_request_headers)
-    final_headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', final_headers)
-    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
-    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
-    self.assertEqual('someData',
-                     final_headers['some-header-thats-not-a-user-agent'])
-
-  def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
-      self):
-    authorized_http = transport.AuthorizedHttp(
-        self.mock_credentials, http=self.mock_http)
-    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
-
-    authorized_http.request(self.test_uri, headers=headers_with_user_agent)
-    final_headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', final_headers)
-    self.assertIn('existing-user-agent', final_headers['user-agent'])
-    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
-
-  def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
-    authorized_http = transport.AuthorizedHttp(
-        self.mock_credentials, http=self.mock_http)
-    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
-
-    authorized_http.request(self.test_uri, headers=same_user_agent_header)
-    final_headers = self.mock_http.request.call_args[1]['headers']
-    self.assertIn('user-agent', final_headers)
-    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
-    # Make sure the header wasn't duplicated
-    self.assertEqual(
-        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))

src/travis/linux-before-install.sh

@@ -104,10 +104,11 @@ else
     $pip install staticx
   fi
 
+  $pip install --upgrade git+git://github.com/pyinstaller/pyinstaller.git@$PYINSTALLER_COMMIT
+
   cd $whereibelong
 fi
 
 echo "Upgrading pip packages..."
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz

src/travis/linux-install.sh

@@ -1,33 +1,36 @@
 cd src
 if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
-  export gam="$python gam.py"
+  export gam="$python -m gam"
   export gampath=$(readlink -e .)
 else
-  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
-  export gam="gam/gam"
-  export gampath=$(readlink -e gam)
+  export gampath="dist/gam"
+  rm -rf $gampath
+  mkdir -p $gampath
+  export gampath=$(readlink -e $gampath)
+  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath $gampath gam.spec
+  export gam="${gampath}/gam"
   export GAMVERSION=`$gam version simple`
   cp LICENSE $gampath
-  cp whatsnew.txt $gampath
   cp GamCommands.txt $gampath
   this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
   GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
   rm $gampath/lastupdatecheck.txt
-  tar cfJ $GAM_ARCHIVE gam/
+  # tar will cd to dist and tar up gam/
+  tar -C dist/ --create --file $GAM_ARCHIVE --xz gam
   echo "PyInstaller GAM info:"
-  du -h gam/gam
+  du -h $gam
   time $gam version extended
-
-  if ([ "${TRAVIS_DIST}" == "trusty" ] || [ "${TRAVIS_DIST}" == "xenial" ]) && [ "${PLATFORM}" == "x86_64" ]; then
+  if [ "${TRAVIS_DIST}" == "xenial" ] && [ "${PLATFORM}" == "x86_64" ]; then
     GAM_LEGACY_ARCHIVE=gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-legacy.tar.xz
-    $python -OO -m staticx -l /lib/x86_64-linux-gnu/libresolv.so.2 -l /lib/x86_64-linux-gnu/libnss_dns.so.2 gam/gam gam/gam-staticx
-    strip gam/gam-staticx
-    rm gam/gam
-    mv gam/gam-staticx gam/gam
-    chmod 755 gam/gam
-    tar cvfJ $GAM_LEGACY_ARCHIVE gam/
+    $python -OO -m staticx -l /lib/x86_64-linux-gnu/libresolv.so.2 -l /lib/x86_64-linux-gnu/libnss_dns.so.2 $gam $gam-staticx
+    strip $gam-staticx
+    rm $gampath/gam
+    mv $gam-staticx $gam
+    chmod 755 $gam
+    rm $gampath/lastupdatecheck.txt
+    tar -C dist/ --create --file $GAM_LEGACY_ARCHIVE --xz gam
     echo "Legacy StaticX GAM info:"
-    du -h gam/gam
+    du -h $gam
     time $gam version extended
   fi
   echo "GAM packages:"

src/travis/osx-before-install.sh

@@ -9,7 +9,7 @@ echo "This device has $cpucount CPUs for compiling..."
 #brew upgrade
 
 # prefer standard GNU tools like date over MacOS defaults
-export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+export PATH="/usr/local/opt/coreutils/libexec/gnubin:$(brew --prefix)/opt/gnu-tar/libexec/gnubin:$PATH"
 
 cd ~
 
@@ -57,7 +57,7 @@ if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]
   rm -rf ssl
   mkdir python
   mkdir ssl
-  
+
   # Compile latest OpenSSL
   wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
   echo "Extracting OpenSSL..."
@@ -107,4 +107,5 @@ cd $whereibelong
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 
+$pip install --upgrade git+git://github.com/pyinstaller/pyinstaller.git@$PYINSTALLER_COMMIT
+ 

src/travis/osx-install.sh

@@ -1,15 +1,16 @@
 cd src
 echo "MacOS Version Info According to Python:"
 python -c "import platform; print(platform.mac_ver())"
-$python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=gam
+export gampath=dist/gam
+rm -rf $gampath
+$python -OO -m PyInstaller --clean --noupx --strip -F --distpath $gampath gam.spec
+export gam="$gampath/gam"
 $gam version extended
-export GAMVERSION=`gam/gam version simple`
-cp LICENSE gam
-cp whatsnew.txt gam
-cp GamCommands.txt gam
+export GAMVERSION=`$gam version simple`
+cp LICENSE $gampath
+cp GamCommands.txt $gampath
 MACOSVERSION=$(defaults read loginwindow SystemVersionStampAsString)
 GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-MacOS$MACOSVERSION.tar.xz
-rm gam/lastupdatecheck.txt
-tar cfJ $GAM_ARCHIVE gam/
+rm $gampath/lastupdatecheck.txt
+# tar will cd to dist/ and tar up gam/
+tar -C dist/ --create --file $GAM_ARCHIVE --xz gam

src/travis/svars-write.py

@@ -6,10 +6,7 @@ cfg = json.load(sys.stdin)
 cfg['client_secret'] = os.getenv('client_secret')
 jid = os.getenv('jid')
 cfg['refresh_token'] = os.getenv('refresh_%s' % jid)
-name = os.getenv('TRAVIS_JOB_NAME')
-if name.endswith('Testing'):
-    out_file = 'oauth2.txt'
-else:
-    out_file = 'gam/oauth2.txt'
+gampath = os.getenv('gampath')
+out_file = os.path.join(gampath, 'oauth2.txt')
 with open(out_file, 'w') as f:
-  json.dump(cfg, f)
+    json.dump(cfg, f)

src/travis/windows-before-install.sh

@@ -64,9 +64,10 @@ $pip install --upgrade -r src/requirements.txt
 # lots of malware uses PyInstaller default bootloader
 # https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
 echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
-tar xf develop.tar.gz
-cd pyinstaller-develop/bootloader
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/$PYINSTALLER_COMMIT.tar.gz
+tar xf $PYINSTALLER_COMMIT.tar.gz
+mv pyinstaller-$PYINSTALLER_COMMIT pyinstaller
+cd pyinstaller/bootloader
 echo "bootloader before:"
 md5sum ../PyInstaller/bootloader/Windows-${BITS}bit/*
 

src/travis/windows-install.sh

@@ -1,20 +1,25 @@
 cd src
 echo "compiling GAM with pyinstaller..."
-pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=$(readlink -e gam)
+export gampath="dist/gam"
+rm -rf $gampath
+mkdir -p $gampath
+export gampath=$(readlink -e $gampath)
+pyinstaller --clean --noupx -F --distpath $gampath gam.spec
+export gam="${gampath}/gam"
 echo "running compiled GAM..."
 $gam version
 export GAMVERSION=`$gam version simple`
-rm gam/lastupdatecheck.txt
-cp LICENSE gam
-cp GamCommands.txt gam
-cp whatsnew.txt gam
-cp gam-setup.bat gam
+rm $gampath/lastupdatecheck.txt
+cp LICENSE $gampath
+cp GamCommands.txt $gampath
+cp gam-setup.bat $gampath
 GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
-/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
-mkdir gam-64
-cp -rf gam/* gam-64/;
+/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE $gampath -xr!.svn
+
+echo "Running WIX candle $WIX_BITS..."
 /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch $WIX_BITS gam.wxs
+echo "Done with WIX candle..."
+echo "Running WIX light..."
 /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
+echo "Done with WIX light..."
 rm *.wixpdb

src/utils.py

@@ -1,291 +0,0 @@
-import datetime
-import re
-import sys
-import time
-from hashlib import md5
-from html.entities import name2codepoint
-from html.parser import HTMLParser
-import json
-import dateutil.parser
-
-import controlflow
-import fileutils
-import transport
-from var import *
-
-class _DeHTMLParser(HTMLParser):
-
-  def __init__(self):
-    HTMLParser.__init__(self)
-    self.__text = []
-
-  def handle_data(self, data):
-    self.__text.append(data)
-
-  def handle_charref(self, name):
-    self.__text.append(chr(int(name[1:], 16)) if name.startswith('x') else chr(int(name)))
-
-  def handle_entityref(self, name):
-    cp = name2codepoint.get(name)
-    if cp:
-      self.__text.append(chr(cp))
-    else:
-      self.__text.append('&'+name)
-
-  def handle_starttag(self, tag, attrs):
-    if tag == 'p':
-      self.__text.append('\n\n')
-    elif tag == 'br':
-      self.__text.append('\n')
-    elif tag == 'a':
-      for attr in attrs:
-        if attr[0] == 'href':
-          self.__text.append(f'({attr[1]}) ')
-          break
-    elif tag == 'div':
-      if not attrs:
-        self.__text.append('\n')
-    elif tag in {'http:', 'https'}:
-      self.__text.append(f' ({tag}//{attrs[0][0]}) ')
-
-  def handle_startendtag(self, tag, attrs):
-    if tag == 'br':
-      self.__text.append('\n\n')
-
-  def text(self):
-    return re.sub(r'\n{2}\n+', '\n\n', re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
-
-def dehtml(text):
-  try:
-    parser = _DeHTMLParser()
-    parser.feed(str(text))
-    parser.close()
-    return parser.text()
-  except:
-    from traceback import print_exc
-    print_exc(file=sys.stderr)
-    return text
-
-def indentMultiLineText(message, n=0):
-  return message.replace('\n', '\n{0}'.format(' ' * n)).rstrip()
-
-def flatten_json(structure, key='', path='', flattened=None, listLimit=None):
-  if flattened is None:
-    flattened = {}
-  if not isinstance(structure, (dict, list)):
-    flattened[((path + '.') if path else '') + key] = structure
-  elif isinstance(structure, list):
-    for i, item in enumerate(structure):
-      if listLimit and (i >= listLimit):
-        break
-      flatten_json(item, f'{i}', '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit)
-  else:
-    for new_key, value in list(structure.items()):
-      if new_key in ['kind', 'etag', '@type']:
-        continue
-      if value == NEVER_TIME:
-        value = 'Never'
-      flatten_json(value, new_key, '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit)
-  return flattened
-
-def formatTimestampYMD(timestamp):
-  return datetime.datetime.fromtimestamp(int(timestamp)/1000).strftime('%Y-%m-%d')
-
-def formatTimestampYMDHMS(timestamp):
-  return datetime.datetime.fromtimestamp(int(timestamp)/1000).strftime('%Y-%m-%d %H:%M:%S')
-
-def formatTimestampYMDHMSF(timestamp):
-  return str(datetime.datetime.fromtimestamp(int(timestamp)/1000))
-
-def formatFileSize(fileSize):
-  if fileSize == 0:
-    return '0kb'
-  if fileSize < ONE_KILO_BYTES:
-    return '1kb'
-  if fileSize < ONE_MEGA_BYTES:
-    return f'{fileSize // ONE_KILO_BYTES}kb'
-  if fileSize < ONE_GIGA_BYTES:
-    return f'{fileSize // ONE_MEGA_BYTES}mb'
-  return f'{fileSize // ONE_GIGA_BYTES}gb'
-
-def formatMilliSeconds(millis):
-  seconds, millis = divmod(millis, 1000)
-  minutes, seconds = divmod(seconds, 60)
-  hours, minutes = divmod(minutes, 60)
-  return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
-
-def integerLimits(minVal, maxVal, item='integer'):
-  if (minVal is not None) and (maxVal is not None):
-    return f'{item} {minVal}<=x<={maxVal}'
-  if minVal is not None:
-    return f'{item} x>={minVal}'
-  if maxVal is not None:
-    return f'{item} x<={maxVal}'
-  return f'{item} x'
-
-def get_string(i, item, optional=False, minLen=1, maxLen=None):
-  if i < len(sys.argv):
-    argstr = sys.argv[i]
-    if argstr:
-      if (len(argstr) >= minLen) and ((maxLen is None) or (len(argstr) <= maxLen)):
-        return argstr
-      controlflow.system_error_exit(2, f'expected <{integerLimits(minLen, maxLen, "string length")} for {item}>')
-    if optional or (minLen == 0):
-      return ''
-    controlflow.system_error_exit(2, f'expected a Non-empty <{item}>')
-  elif optional:
-    return ''
-  controlflow.system_error_exit(2, f'expected a <{item}>')
-
-def get_delta(argstr, pattern):
-  tg = pattern.match(argstr.lower())
-  if tg is None:
-    return None
-  sign = tg.group(1)
-  delta = int(tg.group(2))
-  unit = tg.group(3)
-  if unit == 'y':
-    deltaTime = datetime.timedelta(days=delta*365)
-  elif unit == 'w':
-    deltaTime = datetime.timedelta(weeks=delta)
-  elif unit == 'd':
-    deltaTime = datetime.timedelta(days=delta)
-  elif unit == 'h':
-    deltaTime = datetime.timedelta(hours=delta)
-  elif unit == 'm':
-    deltaTime = datetime.timedelta(minutes=delta)
-  if sign == '-':
-    return -deltaTime
-  return deltaTime
-
-def get_delta_date(argstr):
-  deltaDate = get_delta(argstr, DELTA_DATE_PATTERN)
-  if deltaDate is None:
-    controlflow.system_error_exit(2, f'expected a <{DELTA_DATE_FORMAT_REQUIRED}>; got {argstr}')
-  return deltaDate
-
-def get_delta_time(argstr):
-  deltaTime = get_delta(argstr, DELTA_TIME_PATTERN)
-  if deltaTime is None:
-    controlflow.system_error_exit(2, f'expected a <{DELTA_TIME_FORMAT_REQUIRED}>; got {argstr}')
-  return deltaTime
-
-def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
-  argstr = argstr.strip()
-  if argstr:
-    if argstr[0] in ['+', '-']:
-      today = datetime.date.today()
-      argstr = (datetime.datetime(today.year, today.month, today.day)+get_delta_date(argstr)).strftime(YYYYMMDD_FORMAT)
-    try:
-      dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
-      if returnTimeStamp:
-        return time.mktime(dateTime.timetuple())*1000
-      if returnDateTime:
-        return dateTime
-      return argstr
-    except ValueError:
-      controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>; got {argstr}')
-  elif minLen == 0:
-    return ''
-  controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>')
-
-def get_time_or_delta_from_now(time_string):
-  """Get an ISO 8601 time or a positive/negative delta applied to now.
-  Args:
-    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
-  Returns:
-    string: iso8601 formatted datetime in UTC.
-  """
-  time_string = time_string.strip().upper()
-  if time_string:
-    if time_string[0] not in ['+', '-']:
-      return time_string
-    return (datetime.datetime.utcnow() + get_delta_time(time_string)).isoformat() + 'Z'
-  controlflow.system_error_exit(2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
-
-def get_row_filter_date_or_delta_from_now(date_string):
-  """Get an ISO 8601 date or a positive/negative delta applied to now.
-  Args:
-    date_string (string): The time or delta (e.g. '2017-09-01' or '-4y')
-  Returns:
-    string: iso8601 formatted datetime in UTC.
-  """
-  date_string = date_string.strip().upper()
-  if date_string:
-    if date_string[0] in ['+', '-']:
-      deltaDate = get_delta(date_string, DELTA_DATE_PATTERN)
-      if deltaDate is None:
-        return (False, DELTA_DATE_FORMAT_REQUIRED)
-      today = datetime.date.today()
-      return (True, (datetime.datetime(today.year, today.month, today.day)+deltaDate).isoformat()+'Z')
-    try:
-      deltaDate = dateutil.parser.parse(date_string, ignoretz=True)
-      return (True, datetime.datetime(deltaDate.year, deltaDate.month, deltaDate.day).isoformat()+'Z')
-    except ValueError:
-      pass
-  return (False, YYYYMMDD_FORMAT_REQUIRED)
-
-def get_row_filter_time_or_delta_from_now(time_string):
-  """Get an ISO 8601 time or a positive/negative delta applied to now.
-  Args:
-    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
-  Returns:
-    string: iso8601 formatted datetime in UTC.
-  Exits:
-    2: Not a valid delta.
-  """
-  time_string = time_string.strip().upper()
-  if time_string:
-    if time_string[0] in ['+', '-']:
-      deltaTime = get_delta(time_string, DELTA_TIME_PATTERN)
-      if deltaTime is None:
-        return (False, DELTA_TIME_FORMAT_REQUIRED)
-      return (True, (datetime.datetime.utcnow()+deltaTime).isoformat()+'Z')
-    try:
-      deltaTime = dateutil.parser.parse(time_string, ignoretz=True)
-      return (True, deltaTime.isoformat()+'Z')
-    except ValueError:
-      pass
-  return (False, YYYYMMDDTHHMMSS_FORMAT_REQUIRED)
-
-def get_date_zero_time_or_full_time(time_string):
-  time_string = time_string.strip()
-  if time_string:
-    if YYYYMMDD_PATTERN.match(time_string):
-      return get_yyyymmdd(time_string)+'T00:00:00.000Z'
-    return get_time_or_delta_from_now(time_string)
-  controlflow.system_error_exit(2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
-
-def md5_matches_file(local_file, expected_md5, exitOnError):
-  f = fileutils.open_file(local_file, 'rb')
-  hash_md5 = md5()
-  for chunk in iter(lambda: f.read(4096), b""):
-    hash_md5.update(chunk)
-  actual_hash = hash_md5.hexdigest()
-  if exitOnError and actual_hash != expected_md5:
-    controlflow.system_error_exit(6, f'actual hash was {actual_hash}. Exiting on corrupt file.')
-  return actual_hash == expected_md5
-
-URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
-
-def shorten_url(long_url, httpc=None):
-    if not httpc:
-        httpc = transport.create_http(timeout=10)
-    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
-    try:
-        payload = json.dumps({'long_url': long_url})
-        resp, content = httpc.request(
-          URL_SHORTENER_ENDPOINT,
-          'POST',
-          payload,
-          headers=headers)
-    except:
-        return long_url
-    if resp.status != 200:
-        return long_url
-    try:
-        if isinstance(content, bytes):
-            content = content.decode()
-        return json.loads(content).get('short_url', long_url)
-    except:
-        return long_url

src/whatsnew.txt

@@ -1,681 +0,0 @@
-This file has been deprecated. For the latest features see https://git.io/gam-releases notes
-
-GAM 4.65
- - Manage email delivery settings for group members
- - Add operatingSystemType to user posix config (Roman)
- - Handle missing language setting (jalmeroth)
- - handle mixed case aliases (Ross)
- - contentmanager and fileorganizer roles in Team Drive (Ross)
- - various other fixes and optimizations (Jay and Ross)
-
-GAM 4.61
- - New Gmail delegation API
- - Remove "admin" command from user create/update to avoid accidental super admins. Still possible to give super admin rights via "gam create admin" command. (Ross)
- - Vault export fixes by Ross
- - minor fixes and improvements by Ross and Jay
-
-GAM 4.60
- - Google Vault Export API gam support
- - Ross - add textcolor and backgroundcolor options for Gmail labels
- - Ross - add owneremail option to print courses
- - GAM 4.50 had no bugs at all, nothing to fix.
- - Just kidding, lots of bug fixes and code cleanup. Thanks Ross.
-
-GAM 4.50
- - many cleanups, bugfixes and improvements by Ross and ejochman
- - multiple queries and more options for Chrome OS by Ross
- - per-API batch calls to solve global batch deprecation
- - handle new G Suite Enterprise for EDU and Cloud Identity SKUs
- - Library updates
-
-GAM 4.40
- - Team Drive Admin "asadmin" to give admins special access.
- - Manage Buildings, Features and Resource Calendars.
- - Download log files for kiosk Chrome devices.
- - TPM info and vulnerability status for Chrome devices.
- - New Google library for service account auth.
- - (Ross) increase cell count for "todrive" to 2M.
- - (Ross) improved course id handling.
- - (Ross) numerous cleanups and bug fixes
-
-GAM 4.32
- - Fixes and improvements by Ross
- - gam print courses now supports limiting results based on course state
- - handle issues adding members to a group who were in a pending status before
- - Data transfer commands now support transferring user Calendar events.
-
-GAM 4.31
- - Update course owners in Classroom
- - support for time deltas like -4h for some commands (danielx)
- - Set location, ssh keys and posix accounts details on user create/update
- - support batch move of Chrome OS devices for better performance
- - Huge amount of cleanup / performance improvements by Ross
-
-GAM 4.30
- - Google Vault Matters and Hold API support
- - Ross - "gam update group" now uses batch for better performance
- - "gam update project" enables new APIs for your project
- - Include root level OrgUnit in OrgUnit commands
- - Bulk move Chrome devices between OrgUnits
- - Usual fixes / cleanups / improvements by Ross and Jay
-
-GAM 4.22
- - Validate client id and secret from user input (reduces user errors on create project)
- - Ross - optimize "gam print printjobs"
- - Ross- countsonly option for gam print courses
-
-GAM 4.21
- - Drive v3 fixes/updates by Ross
- - "gam print crosactivty" command outputs active users and times
- - SMime and calendar ACL fixes by Ross
- - standardized cros info/print functionality by Ross
-
-GAM 4.2
- - Create, Update, Delete and List Team Drives
- - Start moving to Drive API v3
- - Disable GAM cache by default to prevent errors (Ross)
- - Use service accounts for all Calendar, Drive and Gmail operations to reduce scopes
- - Fix "Unknown" errors due to a scope issue (may require "gam oauth revoke" and re-authentication)
- - "gam info domain" shows basic user / license sums again
- - "gam report customer" now shows more browser usage stats
- - Fix project creation ToS error (Ross)
-
-GAM 4.12
- - Realtime 2SV user status in gam info user and gam print users. Thanks hajdbo!
- - Reseller API support. Create and manage customers and subscriptions.
- - Delete or modify Gmail threads. Improved message delete/modify performance.
- - Support for the new G Suite Enterprise license
- - Manage S/MIME certificates for G Suite Enterprise users
- - Many fixes and improvements by Ross.
-
-GAM 4.11
- - Allow newlines in calendar event descriptions (Ross)
- - All HTTP requests should now honor SSL verify setting, debug, etc
-
-GAM 4.1
- - Fix "gam create project"
- - project create cleanups by Ross
- - Various fixes by Ross
- - Improved batch command performance. Some commands will see 2-3x speedups.
- - Faster "gam info user" commands via batch license retrieval
-
-GAM 4.03
- - Minor fixes by Jay and Ross. Mostly to new install process.
-
-GAM 4.02
- - "gam create project" command simplifies creation of client_secrets.json and oauth2service.json project files.
- - Automated wizard simplifies GAM setup on Linux and MacOS (coming soon to Windows MSI).
- - "gam calendar <email> deleteevent" deletes events by ID or query
-
-GAM 3.8
-- Old GData APIs removed from GAM. Admin Settings and Email Audit commands are no longer included, keep a copy of GAM 3.72 around if you use them. All Email Settings commands now use new Gmail API.
-- Updated httplib2, google-api-client, uritemplate libraries
-- New update check code utilizes GitHub API to check for latest releases.
-- Many fixes and improvements by Ross
-
-GAM 3.72
-- Chrome OS device actions, disable, re-enable and deprovision devices.
-- (beta) MSI Windows build
-- (beta) binary Linux and MacOS builds
-- Numerous fixes and updates by Ross
-
-GAM 3.71
-- Fix update first / last name.
-- upgrade GAM versions of oauth2client, googleapiclient, RSA and six
-- Improved UTF-8 support for CSV commands (Ross)
-- Authorization flow improvements by Ross
-- Other minor cleanups and fixes
-
-GAM 3.7
-- Classroom Guardians API. Invite, list and delete guardians for a student.
-- Includes number of improvements from Ross in 3.66 (see below)
-
-To use this version, you must update the list of authorized scopes for your Gam OAuth2 Client and Service Account.
-Go here: https://github.com/jay0lee/GAM/wiki/CreatingClientSecretsFile
-Log on to the admin console as in steps 6.ii.c, d, e.
-In the list of Authorized API clients, locate your Gam OAuth2 Client, copy the Client ID and then remove the entry.
-Paste the Client ID into the Client name box as in step 6.ii.f, then do steps 6.ii.g and 6.ii.h.
-You'll notice that the API Scopes - OAuth2 list has additional entries, these are what is required in Gam 3.7.
-Skip down to step 6.iii.
-In the list of Authorized API clients, locate your Gam Service Account, copy the Client ID and then remove the entry.
-Paste the Client ID into the Client name box as in step 6.iii.c, then do steps 6.iii.d and 6.iii.e.
-You'll notice that the API Scopes - Service Account list has additional entries, these are what is required in Gam 3.7.
-
-GAM 3.66
-See GamCommands.txt for a complete syntax description.
-
-Added arguments to gam info group to suppress aliases listing and include groups of which this group is a member.
-gam info group <Group> ... [noaliases] [groups]
-
-Added argument to gam print cros to limit number of activeTimeRanges and recentUsers entries
-gam print cros ... [listlimit <Number>]
-
-Added argument to gam <UserTypeEntity> signature and gam <UserTypeEntity> vacation to allow specification of file character set so that extended characters can be read.
-Credit to Steve Main for suggesting the following enhancement.
-Added argument to gam <UserTypeEntity> signature and gam <UserTypeEntity> vacation to allow pattern substitution in the signature and vacation message.
-gam <UserTypeEntity> signature <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)*
-gam <UserTypeEntity> vacation <TrueValues> subject <String> (message <String>)|(file <FileName> [charset <CharSet>]) (replace <Tag> <String>)*
-	[contactsonly] [domainonly] [startdate <Date>] [enddate <Date>]
-Every instance of {<Tag>} in the signature/message will be replaced by <String>. Instances of the form {RT}...{Text}...{/RT} will be eliminated
-if there was no <Tag> specified that matches Text or if a <Tag> matching Text was specified but the matching <String> is empty.
-This is especially useful with CSV files.
-gam csv Users.csv gam user "~User" signature file SignatureTemplate.txt replace "#User#" "~User" replace "#Title#" "~Title"
-
-Added argument to gam <UserTypeEntity> show signature to format the signature.
-gam <UserTypeEntity> show signature [format]
-
-Added argument to gam add/update calendar to allow specification of event notifications
-gam <UserTypeEntity> add/update calendar <Calendar> notification email|sms eventcreation|eventchange|eventcancellation|eventresponce|agenda
-
-Added option to reminder and notification arguments of update calendar to allowing clearing of reminders/notifications.
-gam <UserTypeEntity> update calendar <Calendar> [reminder clear] [notification clear]
-
-Added arguments to gam print group-members to allow selecting subsets of groups.
-Added argument to gam print group-members to add member full name to output,
-Added argument to gam print group-members to allow output field selection.
-gam print group-members [todrive] ([domain <DomainName>] [member <UserItem>])|[group <GroupItem>] [membernames] [fields <MembersFieldNameList>]
-MembersFieldNameList is a comma separated list of field names: email | group | id | name | role | type
-
-Added argument to gam info user to specify SKUs for which license information is desired.
-gam info user [<UserItem>] ... [skus <SKUIDList>]
-
-Added argument to gam print printjobs and gam printjob <PrinterID> fetch to allow specifying the maximum number of print jobs to retrieve.
-gam printjob <PrinterID> fetch		... [limit <Number>]
-gam print printjobs  ... [limit <Number>]
-limit <Number> specifies the maximum number of print jobs to retrieve; defaults to 25, set limit to 0 to retrieve all print jobs.
-
-Credit to Seth Stein for the following enhancements.
-Added argument to gam <UserTypeEntity> get drivefile to allow downloading a specific revision of a drive file.
-gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(query <Query>) [format <FileFormatList>] [targetfolder <FilePath>] [revision <Number>]
-Added command to show drive file revisions.
-gam <UserTypeEntity> show filerevisions <DriveFileID>
-
-Added argument to gam print adminroles to allow uploading to Google Drive.
-gam print adminroles [todrive]
-
-Added group, groups, mobile arguments to gam report.
-gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token ...
-
-Added command to show user Google+ profile.
-gam <UserTypeEntity> show gplusprofile [todrive]
-To enable this command, visit: https://github.com/jay0lee/GAM/wiki/CreatingClientSecretsFile
-In step 3.v, enable the Google+ API
-In step 6.iii.d, add https://www.googleapis.com/auth/plus.me to the API scopes - Service Account list, then remove and re-add the authorization.
-
-Added argument to gam <UserTypeEntity> show labels to allow seeing message counts for each label.
-gam <UserTypeEntity> show labels|label [onlyuser] [showcounts]
-
-Added argument to gam <UserTypeEntity> show fileinfo to allow field selection.
-gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
-
-Added argument to gam update group to allow removing members by role.
-gam update group <Group> clear [members] [managers] [owners]
-If no option follows clear, all members will removed.
-
-Changed gam print admins to include 'id:' in OrgUnitID column as with other gam print commands.
-
-Fixed GAM to handle both future date error messages in gam report
-
-Fixed gam <UserTypeEntity> show delegates to handle Unicode characters in delagator name.
-
-Fixed gam <UserTypeEntity> get drivefile to properly handle file extension.
-
-Fixed gam create alias <Name> target <Group>.
-
-2016/07/29
-
-Added command to empty drive drive trash.
-gam <UserTypeEntity> empty drivetrash
-
-Added alternative command to add delegates and command to print delegates.
-gam <UserTypeEntity> add delegate|delegates <UserEntity>
-gam <UserTypeEntity> print delegates [todrive]
-
-Improved Gmail filter processing.
-gam <UserTypeEntity> [add] filter [from <EmailAddress>] [to <EmailAddress>] [subject <String>] [haswords|query <List>] [nowords|negatedquery <List>] [musthaveattachment|hasattachment] [excludechats] [size larger|smaller <ByteCount>]
-	[label <LabelID>] [important|notimportant] [star] [trash] [markread] [archive] [neverspam] [forward <EmailAddress>]
-gam <UserTypeEntity> delete filters <FilterIDEntity>
-gam <UserTypeEntity> show filters
-gam <UserTypeEntity> info filters <FilterIDEntity>
-gam <UserTypeEntity> print filters [todrive]
-
-Added commands to process Gmail forwarding addresses.
-gam <UserTypeEntity> add forwardingaddress|forwardingaddresses <EmailAddressEntity>
-gam <UserTypeEntity> delete forwardingaddress|forwardingaddresses <EmailAddressEntity>
-gam <UserTypeEntity> show forwardingaddress|forwardingaddresses
-gam <UserTypeEntity> info forwardingaddress|forwardingaddresses <EmailAddressEntity>
-gam <UserTypeEntity> print forwardingaddress|forwardingaddresses [todrive]
-
-Improved Gmail forward processing.
-gam <UserTypeEntity> forward <FalseValues>
-gam <UserTypeEntity> forward <TrueValues> keep|leaveininbox|archive|delete|trash|markread <EmailAddress>
-gam <UserTypeEntity> show forward
-gam <UserTypeEntity> print forward [todrive]
-
-Improved Gmail sendas processing.
-gam <UserTypeEntity> [add] sendas <EmailAddress> <Name> [replyto <EmailAddress>] [default] [treatasalias <Boolean>] [signature|sig <String>|(file <FileName> [charset <CharSet>]) (replace <REPattern> <String>)*]
-gam <UserTypeEntity> update sendas <EmailAddress> [name <Name>] [replyto <EmailAddress>] [default] [treatasalias <Boolean>] [signature|sig <String>|(file <FileName> [charset <CharSet>]) (replace <REPattern> <String>)*]
-gam <UserTypeEntity> delete sendas <EmailAddressEntity>
-gam <UserTypeEntity> show sendas [format]
-gam <UserTypeEntity> info sendas <EmailAddressEntity> [format]
-gam <UserTypeEntity> print sendas [todrive]
-
-Improved Gmail signature processing.
-gam <UserTypeEntity> signature|sig <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)* [name <String>] [replyto <EmailAddress>] 
-gam <UserTypeEntity> show signature|sig [format]
-
-Use Gmail API for POP/IMAP/Vacation processing.
-gam <UserTypeEntity> imap|imap4 <Boolean> [noautoexpunge] [expungebehavior archive|deleteforever|trash] [maxfoldersize 0|1000|2000|5000|10000]
-gam <UserTypeEntity> pop|pop3 <Boolean> [for allmail|newmail|mailfromnowon|fromnowown] [action keep|leaveininbox|archive|delete|trash|markread]
-gam <UserTypeEntity> vacation <FalseValues>
-gam <UserTypeEntity> vacation <TrueValues> subject <String> (message <String>)|(file <FileName> [charset <CharSet>]) (replace <Tag> <String>)* [html]
-	[contactsonly] [domainonly] [startdate <Date>] [enddate <Date>]
-gam <UserTypeEntity> show vacation [format]
-
-Added command toGet information about a specific calendar.
-gam <UserTypeEntity> info calendar <EmailAddress>
-
-Added command to print calendars to CSV file, dropped all arguments from gam show calendars.
-gam <UserTypeEntity> print calendars [todrive]
-gam <UserTypeEntity> show calendars
-
-Added command to print Gmail Profiles to CSV file, dropped all arguments from gam show gmailprofile.
-gam <UserTypeEntity> print gmailprofile [todrive]
-gam <UserTypeEntity> show gmailprofile
-
-Added command to print Gplus Profiles to CSV file, dropped all arguments from gam show gplusprofile.
-gam <UserTypeEntity> print gplusprofile [todrive]
-gam <UserTypeEntity> show gplusprofile
-
-Added command to print user schemas to CSV file, renamed command to display formatted user schemas to gam show schemas.
-gam print schemas [todrive]
-gam show schemas
-
-Added command to print user access tokens to CSV file.
-gam <UserTypeEntity> print tokens|token|3lo|oauth [todrive]
-
-Added arguments to gam info cros to allow specification of desired output fields.
-gam info cros <CrosDeviceEntity> [nolists] [listlimit <Number>]
-	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>]
-
-Added drivedir and targetfolder <FilePath> arguments to gam printjob fetch and gam get photo to
-allow specification of the destination folder for the file retrieved from Google. The default
-location for these commands is the current working directory, drivedir specifies the value of the environment variable GAMDRIVEDIR and
-targetfolder <FilePath> specifies a user-choosen path.
-gam printjob <PrinterID>|any fetch
-	[olderthan|newerthan <PrintJobAge>] [query <QueryPrintJob>]
-	[status <PrintJobStatus>]
-	[orderby <PrintJobOrderByFieldName> [ascending|descending]]
-	[owner|user <EmailAddress>]
-	[limit <Number>] [drivedir|(targetfolder <FilePath>)]
-gam <UserTypeEntity> get photo [drivedir|(targetfolder <FilePath>)]
-
-Added noshow argument to gam get photo to suppress displaying of photo data
-gam <UserTypeEntity> get photo [drivedir|(targetfolder <FilePath>)] [noshow]
-
-Changed gam print cros to match gam print users. Previously, gam print cros produced a full listing of CrOS devices
-and gam print users produced a listing of primaryEmail addresses. Now, gam print cros produces a listing of deviceIds.
-To get the previous behavior, say gam print cros full. See GamCommands.txt for a summary of CrOS and User printing.
-
-Commands that produce CSV file output have been changed to make the leftmost column(s) be the key fields.
-If you have scripts that process the CSV files as flat files, expecting the columns to be in a particular
-order, they will have to be updated. If your scripts process the CSV files by column header, no changes should be required.
-
-2016/07/31
-
-Changed gam get drivefile to take a list of file formats rather than a single format. The first format in the list that is available will be used.
-Added drivefilename argument to allow downloading file by name.
-<FileFormat> ::= csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
-<FileFormatList> ::= '<FileFormat>(,<FileFormat)*'
-gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>) [format <FileFormatList>] [targetfolder <FilePath>] [revision <Number>]
-
-2016/08/01
-
-Added delimiter <String> argument to gam print courses to allow choice of delimiter to separate aliases.
-gam print courses [todrive] [teacher] [student] [alias|aliases] [delimiter <String>]
-
-Added notsuspended argument to gam update group add/sync to prevent adding suspended users to a group
-when specifying all users, org <OrgUnitPath> or query <QueryUser> for <UserTypeEntity>.
-gam update group <GroupItem> add [owner|manager|member] [notsuspended] <UserTypeEntity>
-gam update group <GroupItem> sync [owner|manager|member] [notsuspended] <UserTypeEntity>
-
-Added basic|full argument to gam print mobile to allow specification of output detail desired.
-gam print mobile [todrive] [query <QueryMobile>] [basic|full] [orderby <MobileOrderByFieldName> [ascending|descending]]
-
-GAM 3.65
-  -fix vacation issues (Ross)
-  -fix Windows path issues (Ross)
-  -Add message undelete (Ross) and modify (Jay) commands
-  -Improve message delete performance
-  -Upgrade to new versions of oauth2client and googleapiclient
-
-GAM 3.63
-  -"gam update group ... sync users" now does batch add/remove of users for faster sync
-  -"gam file" can now use - to read list of users from stdin
-  -"gam csv file:column" can read CSV file of users and specify column to be used.
-
-GAM 3.62
-  -New Admin Roles API allows you to create, delete and print delegated and super admins
-  -New Resource Calendar API replaces old GData API and allows you to create, update, get info, delete and print all resource calendars.
-  -Major cleanups and design updates from Ross Scroggs @taers232c mean GAM is more reliable and stable for your needs. Huge thanks Ross!
-  -Guard additional fields with convertUTF8
-  -Correct gam create resource to replace resType with type
-  -Add type as an argument to gam print resources to make resource type visible
-  -Handle students/teachers with missing emails in gam course sync
-
-GAM 3.61
-  -Various fixes by Ross Scroggs for Domain API and Data Transfer commands
-  -Remove duplicate DoCreateDomain which broke "gam create domain"
-  
-GAM 3.6
-  -Change your primary domain!
-  -Transfer Google Drive and Google+ data between users.
-  -Domains API support includes ability to add, delete, update and print domains.
-
-GAM 3.51
-  -delete or trash messages in mailboxes based on a Gmail search.
-  -create and delete course aliases with Classroom API.
-  -HUGE number of bug fixes, large and small by Ross Scroggs. Many thanks Ross!
-
-GAM 3.5
-  -Support for the new Google Classroom API.
-   -create, update, info and delete courses
-   -add, remove and sync course teachers and students
-   -print courses and course participants
-  -Google CloudPrint API Support
-   -update, info, delete and report printers
-   -share, unshare and get ACLs for printers
-   -submit, cancel, report and delete print jobs
-  -Bug fixes and improvements to GAM batch commands
-
-GAM 3.45
-  -add six.py to solve compatability issues on OS X and Linux
-  -be conservative with password hashing to prevent timeouts
-  
-  If you see issues setting user passwords with GAM 3.44 or older, please upgrade to 3.45.
-
-GAM 3.44
-  -"gam update cros <id> assetid <asset>" allows updating of Chrome OS device Asset ID field. Thanks Erik Pitti!
-  -Windows versions of GAM now use pyinstaller instead of py2exe.
-  -upgraded versions of the googleapiclient and oauth2client libraries.
-  -"gam print cros" should produce a cleaner CSV now.
-
-GAM 3.43
-  -Fix crash when authenticating GAM related to Short URLs
-  -minor fixes to Drive and Calendar commands
-  -catch unauthorized service account errors and offer nice instructions.
-  -execute bit for gam.py (Thanks daethnir)
-
-GAM 3.42
-  -"gam <users> show driveactivity" displays Drive activity for user
-  -"gam report tokens" displays user OAuth activity report
-  -"gam mobile <id> action accountwipe" wipes account only on Android devices
-  -"gam <users> update labels" removes Inbox/ prefix from label names
-  -upgrades to oauth2client and googleapiclient libraries which GAM depends on
-  -"gam <users> show gmailprofile" shows Gmail mailbox details
-  -"gam license <sku>" commands to perform actions only on users with a given license
-  -bug fixes, lots of bug fixes
-
-GAM 3.41
-  -fix Google servers not returning file size on audit export download
-  -soft fail on license change errors
-  -fix for gam info domain errors
-
-GAM 3.4 "Oktoberfest"
-  -Support for creating and setting custom user schemas http://goo.gl/M9rQrI
-  -End user view of print users and user info commands.
-  -fix updating name/description for groups
-  -fix groups sync commands
-  -gam print groups members no longer fails on zero member groups
-  -make sure downloaded Drive file names are safe for OS filenames.
-  -gam info domain should no longer crash on getting customer ID
-  -other minor bug fixes
-
-GAM 3.32
-  -fix service account json files downloaded from new cloud console don't work with GAM
-  -use the new Gmail API for label command. Offers ability to rename labels and show/hide labels from the label and message list.
-  -copy Google drive files with commands like gam user <email> update drivefile id <fileid> copy
-  -print only one SKU for licenses with commands like gam print licenses sku vault
-  -short license names for Google Apps Message Security (gams), Google Apps Unlimtied (gau) and Google Vault Former Employee (vfe)
-
-GAM 3.31
-  -New command "gam user delete aliases" clears all aliases for user.
-  -"gam update user email vfe" will rename user to vfe.oldname.XXXXX@olddomain.com
-  -fix delete photo command
-  -fix password update/create for Windows builds
-
-GAM 3.3
-  -Major rewrites of "gam report" and "gam print users". Note that CSV headers have changed. Better performance and print users now supports "rich" fields like organization, phone, relations, etc.
-  -"gam report drive" now works for Google Apps Unlimited domains.
-  -"gam info user" now shows the user's licenses. Use "nolicenses" to prevent showing licenses.
-  -fix "gam print licenses" fails in large domains by reducing page size from 1000 to 100
-  -GAM now sends a sha-512 salted hashed password on user create and update for better security.
-  -cleanup unused features of old GData library.
-  -fix for Drive file downloading default format.
-  -upgrade to httplib v0.9 which may help with httplib.BadStatus errors.
-
-GAM 3.21
-  -Fix crash when attempting to perform Drive operations for users with Drive service disabled.
-  -Fix "gam info org" only prints first 100 users in org.
-  
-GAM 3.2
-  -New Google Drive GAM commands: http://goo.gl/l7F8hY
-  -New GAM batch and CSV commands allow bulk parallel GAM operations. Documentation coming at: http://goo.gl/2rsDnc
-
-GAM 3.04
-  -New domain verification commands "gam create verify <domain name>", "gam update verify <domain name> <verify type>" and "gam info verify".
-  -"gam update group sync" commands should now only sync changes to the specified role (member, manager, owner). Other roles should be left alone.
-  -Fix: setting or viewing email signatures and vacation messages with unicode characters caused GAM to crash.
-  -Fix: issues printing out ASPs and backup codes if user had none set.
-  -Fix: "gam print orgs parent" always fails.
-  
-GAM 3.03
-  -New GAM user security commands allow management of OAuth tokens, 2SV backup codes and application specific passwords
-  -Google Vault license commands now work
-  -"gam update user password random" now resets user password to a very long, random string
-  -fixed updating location for Chrome devices
-  -fixed "gam update org" commands broken
-
-GAM 3.02
-  -client_secrets.json is no longer shipped with GAM, you need to create your own with the instructions at http://goo.gl/QYaQ6R
-  -New "gam report logins" command to report on user login times and IP.
-  -Updated "gam report domain" command provides cleaner details of aggregate usage.
-  -"gam report admin" fixed.
-  -Fix "gam ou..." commands (they were hanging forever)
-  -Other minor cleanups and fixes.
-
-GAM 3.01
-  -Fix gamcache errors on Windows by keeping cache filenames much shorter.
-  -add (back) support for setting/updating calendar colors
-  -add support for bulk updating users specified on the command like (gam update users "user1@domain.com user2@domain.com user3@domain.com"... OR gam update users user1@domain.com,user2@domain.com,user3@domain.com...)
-  -fix setting "gal off" during user creation.
-  -rewrite "gam info domain" to use new API library (should help with Unicode/UTF-8 errors)
-  -fix "show pop" and signature commands
-  -handle out of memory errors more gracefully
-
-GAM 3.0
-  -Support for the Enterprise License API. Manage Drive storage and Google Coordinate licenses for users.
-  -Improved compatability with GAM commands from 2.55 and older.
-  -Fixed undelete user command.
-  -New "gam print group-members" command to print user membership of all groups.
-  -New "gam <some users> update user..." command to bulk modify settings for given users. For example, "gam all users update user changepassword on" will force password change for all users, 'gam group class-of-2013@acme.edu update user suspended on' will suspend all members of class-of-2013 group.
-  -Optimizations which should result in modest improvements to GAM startup time and performance.
-
-GAM 2.994
-  -Rewritten "gam reports" commands. gam report users, gam report domain, gam report docs, gam report admin
-  -If CSV file uploaded to drive on "gam report" and "gam print" commands with the todrive parameter are more than 400,000 cells or 256 columns, don't convert to GDoc Spreadsheet.
-  -Remove old Admin Audit API scope (replaced by Reports API).
-  -new command: gam all users prism off
-
-GAM 2.992
-  -Various minor fixes
-
-GAM 2.991
-  -gam print commands now support a "todrive" argument. When specified, instead of displaying the CSV output locally (or piping it to a file), GAM will upload the CSV data to a Google Docs Spreadsheet owned by the admin you've authenticated as. The spreadsheet will be opened automatically or, if you've created nobrowser.txt, a URL will be shown.
-  -GAM should handle non-English input characters better. Commands like: "gam.py update user rpinaya lastname Piñaya" should work without issue on Windows.
-  -Improved errors in various places (less "explosions" more meaningful instructions.
-  -gam undelete user is fixed to always use the user's id rather than email address. If an email address is supplied, GAM converts it to a id before attempting to delete. If more than 1 deleted user exists with that address, GAM displays the options.
-
-GAM 2.99
-  -Support for the newly announced Google Apps Admin SDK offering a richer feature set of management for your users, groups, aliases and other objects.
-  -Simplified OAuth 2.0 authentication
-  -Ability to manage Mobile and Chrome OS devices.
-  -Ability to add managers to groups
-  -Ability to manage group aliases
-  -Increased performance thanks to new Google API formats, caching, compression and partial update features.
-  -To many more features to list here! Download it now to see for yourself.
-
-GAM 2.55
-  -Fix change in Google APIs broke "gam whatis" command.
-  -Fix change in Google APIs broke "gam info domain" command on CNAME Verification Status message.
-
-GAM 2.54
-  -Fix a stupid bug that broke "gam print users" when used without additional attributes.
-  -Another fix for outbound gateway settings on "gam info domain"
-  -Get this whatsnew.txt doc up to date.
-
-GAM 2.53
-  -Two new group settings, spam_moderation_level and include_in_global_address_list allow further customization of your Google Groups.
-  -Error reporting for mailbox delegation has been further improved, GAM does a better job of pinpointing why a delegation failed.
-  -Fixed updating and deleting domain and default users for calendar ACLs
-  -proper error handling for adding and removing group members and owners
-  -Fixed error on gam info domain caused by failure to retrieve outbound gateway settings.
-  -An EXPERIMENTAL 64-bit build of GAM for Windows is now available. I do not expect it will be any bit faster for most GAM commands since most delay
-   is due to network I/O. However, some GAM commands like "gam print users", when run against large domains (10,000+ users), use a large amount of memory
-   and resources due to result size. In these scenarios, the x64 build MIGHT prove faster. If you try the x64 build, please report how it worked back to 
-   the mailing list. "It feels faster" is nice but hard numbers with details of what you did are better :-) Note that if you're using the Python source
-   build and your Python is 64-bit, you're already using 64-bit GAM :-)
-
-GAM 2.52
-  -It's a dud! Major bug caused me to pull this version 10 minutes after release :-)
-
-GAM 2.51
-  -New gam calendar wipe command allows clearing all data off a user's primary calendar
-  -create user and update user commands now support setting user's org.
-  -gam whatis command allows looking up an email address to determine if it's a user, alias or group.
-  -gam delete user no longer renames a user by default since undelete is now in CPanel. Added optional dorename parameter to force old renaming behavior.
-  -Fix issue that broke gam delete resource command
-  -GAM now offers to remember your client secret and key when entered the first time
-  -various other bug fixes
-
-GAM 2.5
-  -GAM now handles and retries errors consistently and provides nice error messages. Long running GAM processes
-   like "gam all users" should be much stabler now. Death to the 1000/Unknown errors!
-   This involved some major changes to the Google API calls so if you run into problems, try
-   downgrading to 2.3.1 and see if they go away. Be sure to submit bug reports!
-  -GAM checks for updates 
-  -New parameters for gam create user and gam update user
-  -New parameters for gam print group: owners, members and settings
-  -GAM now works for delegated admins with user read/create/update/delete API rights
-  -gam update group add owner now only adds the user as a group owner, not a member (Google Group member
-   and owner status are independant of each other)
-  -gam update group add member no longer revokes user's owner rights if they have them
-  -gam info group now shows owners who are not a member of the group
-  -gam now works around the group settings "Backend Error" by making an HTTP request to the groups website.
-   This workaround may cease to work if performed on more than a few hundred groups at a time.
-  -moving large numbers of users to an Organization is now more reliable and is performed 25 users at a time.
-  -gam print users aliases now makes only 1 API call to retrieve all user aliases
-  -New commands "gam oauth info" and "gam oauth revoke" allow further OAuth token management
-  -gam info domain now shows the unique customer id
-
-GAM 2.3.1
-  -Fixes to add calendar command
-  -Allow updating and removal of special Calendar ACL users domain and default
-  -pop commands now work without supplying all arguments (defaults to enable for all mail and keep)
-  -New "file" argument for signature and vacation commands allows specifying a file with message content.
-  -"gam create group" now only requires group name argument, rest are optional.
-  -special user * (everyone in domain) can now be added to a group via GAM
-  -print groups, print resources, print aliases and print orgs commands now output proper CSV
-  -Dito company information now displayed on OAuth token create
-  
-GAM 2.3
-
-  -GAM is now owned by Dito (www.ditoweb.com), the Google Apps Experts! See announcement and details at http://code.google.com/p/google-apps-manager
-  -New user profile photo management commands can update, get and delete user profile photos
-  -GAM now gracefully handles cross-domain mailbox delegations by using (or giving the delegate) an alias in the mailbox's domain.
-  -"gam user XXXX show delegates" now has optional argument "csv" to print existing delegations in CSV format
-  -GAM can now properly rename and delete long usernames by ensuring that the renamed user is max 64 characters in length
-  -"gam print groups" now has optional arguments nousermanagedgroups and onlyusermanagedgroups allowing user managed groups to be excluded from output or print user managed groups exclusively.
-
-GAM 2.2
-
-  -Update Calendar ACLs command, update user calendar settings command and ability to set calendar settings when subscribing user
-  -Delete Gmail labels command
-  -Fixes for *nix CSV formatting
-  -Fixes to make Windows and *nix generated oauth.txt files compatible
-  -"gam info user" now shows mailbox quota and user organization
-  -"gam update user" can now handle change of user's domain in renames. "gam multi" commands now fully deprecated.
-  -Fix reply_to and a few other group settings were never getting updated.
-  -"gam info group" now makes 3 efficient API calls rather than one per member/owner of the group greatly increasing performance with large groups
-  -GAM should do a better job of always printing out full email address instead of just username. If you see GAM reporting only the username and not the full email address, please report it as a bug.
-  -All OAuth scopes are now selected by default.
-
-GAM 2.1.1
-
-  -Fix to prevent unnecessary call to Groups Provisioning API when viewing detailed group settings
-  -should be show_in_group_directory not show_in_groups_directory.
-
-GAM 2.1
-
-  -New Reporting API Support allows you to pull 5 different daily reports: accounts, activity, disk_space, email_clients and summary.
-  -Fix for Adding calendars to a user's list of calendars. Bug in 2.0 meant calendar was always added to the calendar list of the admin who authorized GAM, not the target user.
-  -GAM now looks for an environment variable called OAUTHFILE. If it exists, GAM will use that file instead of oauth.txt for authentication. This allows admins of many Google Apps domains to switch quickly between domains.
-  -Fixes for many "gam print users" issues. Thanks to Craig Box for the patch.
-
-GAM 2.0
-
--Group Settings commands allow you to update Google Group settings
--Calendar commands allow you to grant access to calendars and modify user's list of calendars
--Update Admin Settings like the logo, outbound gateway, email migration and more
--OAuth is now the default authentication method. Support for username/password ClientLogin has been removed.
--Vacation/Away messages can now have a start and end date. They can also be limited to within the domain only.
--Further work to make all GAM commands multi-domain friendly.
--Lot's more bugfixes! look at the Wiki pages for details
-
-GAM 1.9.1
-
--"gam print postini" will print all of the Postini Batch commands necessary to "mirror" Google Apps email addresses
-  into a Postini standalone instance.
-
--"gam version" will print details about the version of GAM you are using.
-
-GAM 1.9 - "Baby Steps"
-
-GAM 1.9 is dedicated to David, my 13 month year old son. Whose just starting to step out into the world this week.
-
--whatnew.txt is new (is that an oxymoron?)
-
--Share or Hide users profile from autocomplete and contacts search.
-  "gam user jsmith show profile"
-  "gam user jsmith profile share"
-  "gam group asked-to-be-hidden profile unshare"
-  
-  Profile modifications only work with OAuth, not ClientLogin (username/password entered into GAM). 
-  Since the profile API uses a scope GAM was not previously making use of, you'll need to re-run
-  "gam oauth request" to include the Profile API scope.
-
--Numerous actions can now be performed for all users in a given Organizational
-  Unit just like they can be for a group or all users. i.e. "gam ou Students webclips off".
-
--Provisioning API OAuth scope has been subdivided into user, group, alias and ou scopes
-  offering finer granularity.
-
--"gam all users" will now include all users across primary and secondary domains instead of just primary domain users.
-
--"gam info user" will show all email aliases for a user, not just those in the primary domain.
-
--"gam print users" with any extra arguments would fail, this should be fixed now.
-
--"gam info group" and "gam print groups" should no longer fail for groups with custom permissions.
-
-GAM 1.8
-
--OAuth Support - GAM now supports OAuth Authentication. Instead of providing GAM your username and password, you grant GAM access to selected APIs from within your Google Account. This has a number of advantages:
-  -With OAuth GAM doesn't need to know your password.
-  -OAuth tokens don't expire, once you grant GAM OAuth access, GAM will have access until you revoke it within your Google account.
-  -OAuth has the concept of scopes, limiting the areas and services that access is granted to. This allows you to only provide GAM with the privileges it needs.
-  -More info about OAuth support is on it's way. But for now, you can try OAuth access by running "gam oauth request".
-
--The settings filter command http://code.google.com/p/google-apps-manager/wiki/ExamplesEmailSettings#Create_a_Filter now has more actions including forward, star, trash and never send to spam.
-
--Downloading Audit Exports now has partial resume support. GAM will not re-download files that already exist on the local drive. If a large export download fails you should delete the last file GAM was working on since it's incomplete and then restart the process, GAM will pick up with the last file.

src/windows-build.bat

@@ -1,31 +0,0 @@
-rmdir /q /s gam
-rmdir /q /s gam-64
-rmdir /q /s build
-rmdir /q /s dist
-del /q /f gam-%1-windows.zip
-del /q /f gam-%1-windows-x64.zip
-del /q /f gam-%1-windows-x64.msi
-del /q /f *.wixobj
-del /q /f *.wixpdb
-
-set WIXVERSION=3.11
-
-c:\python37-32\scripts\pyinstaller --clean -F --distpath=gam windows-gam.spec
-xcopy LICENSE gam\
-xcopy whatsnew.txt gam\
-xcopy gam-setup.bat gam\
-xcopy GamCommands.txt gam\
-del gam\w9xpopen.exe
-"%ProgramFiles%\7-Zip\7z.exe" a -tzip gam-%1-windows.zip gam\ -xr!.svn
-
-c:\python37-64\scripts\pyinstaller --clean -F --distpath=gam-64 windows-gam.spec
-xcopy LICENSE gam-64\
-xcopy whatsnew.txt gam-64\
-xcopy gam-setup.bat gam-64\
-xcopy GamCommands.txt gam-64\
-"%ProgramFiles%\7-Zip\7z.exe" a -tzip gam-%1-windows-x64.zip gam-64\ -xr!.svn
-
-set GAMVERSION=%1
-"%ProgramFiles(x86)%\WiX Toolset v%WIXVERSION%\bin\candle.exe" -arch x64 gam.wxs
-"%ProgramFiles(x86)%\WiX Toolset v%WIXVERSION%\bin\light.exe" -ext "%ProgramFiles(x86)%\WiX Toolset v%WIXVERSION%\bin\WixUIExtension.dll" gam.wixobj -o gam-%1-windows-x64.msi
-del /q /f gam-%1-windows-x64.wixpdb

src/windows-gam.spec

@@ -1,33 +0,0 @@
-# -*- mode: python -*-
-import sys
-
-sys.modules['FixTk'] = None
-
-a = Analysis(['gam.py'],
-             pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam.exe',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )