GAM 5.06

Add date range to skipdates

Make report name match dates; indicate if no data

If only enddate is specified change time period
current: now-1 month to enddate
improved: enddate-1 month to enddate

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - pinned
+  - enhancement
+  - help wanted
+  - security
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs.
+  
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.travis.yml

@@ -1,10 +1,15 @@
 if: tag IS blank
+os: linux
+language: python
+dist: xenial
 
 env:
   global:
-    - BUILD_PYTHON_VERSION=3.8.1
-    - BUILD_OPENSSL_VERSION=1.1.1d
-    - PATCHELF_VERSION=0.9
+    - BUILD_PYTHON_VERSION=3.8.2
+    - MIN_PYTHON_VERSION=3.8.2
+    - BUILD_OPENSSL_VERSION=1.1.1g
+    - MIN_OPENSSL_VERSION=1.1.1g
+    - PATCHELF_VERSION=0.10
     - PYINSTALLER_VERSION=3.5
     - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
     - secure: "J9380tGLOZWa7dSH1y5Il8T5JQpN6ad81gI6VR1HIU0svpRdjgikyDA7ca2MKYDUYYY9yVSkTV6gCl6iIU/9+SKaYugpP+tkvdGYkC2moJdcTgYM/WOnIK9ExQ3BPhN1neGxJjPTwKo1ft27mtZ2I5vuCiBwIcnKWLnKPyW3PD+mWpfqiLuEzkHoAh6G3jC4qbcCrZDeX/knE+PzqESUEi+8k1G8gYcSDWujba9ypSsqZ8T/MXagGla6l7y2Rz+/KZTJmFHwKAA10V+xPLVqxoiqi4ar66yUqy0BamwRXPcseI+ns3Q+4lUpMqVQ5GlRy7LF1xC8myjmcAexXk0F9hg+CMzewKI8UgmQH/ZJvQZEh8s6mW26+CqA4d3zMQkWaR0WtEtpiuH7AGHCflIqvEQ6UiG7ia3B8iZfW2wl0j/kqx4OuHkS3r0pWKVVIIvCj9Ow2BHP7SpiV1AcUGsVxzwbgTh67fitna3Z3c6Uj8ccQlNr7ZIt1az6Wf3w5njijkLOiBpQSLKunTTCTSge/JzBTKUcie3RE9vzirl58gUxAt36nDtPWnory+RttMZrOkBVbTeSxp+IUe8pNwLFPHABsafXsjkfzBOtFmm+0ZXWt2Rlog5NvlemJfQUWDlsL4g+BSakzN+4sIPKzSauWDHyaEeULY7Uprkil6c5zwo="
@@ -33,186 +38,94 @@ cache:
     - $HOME/python
     - $HOME/ssl
 
-matrix:
-  fast_finish: true
-  allow_failures:
-    - arch: arm64
+jobs:
   include:
     - os: linux
       name: "Linux 64-bit Bionic"
       dist: bionic
-      language: bash
-      env:
-         - GAMOS=linux
-         - PLATFORM=x86_64
-         - VMTYPE=build
+      language: shell
     - os: linux
       name: "Linux 64-bit Xenial"
       dist: xenial
-      language: bash
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
+      language: shell
     - os: linux
       dist: bionic
       arch: arm64
       name: "Linux ARM64 Bionic"
-      language: bash
+      language: shell
       filter_secrets: false
-      addons:
-        apt:
-          packages:
-            - ruby
-      env:
-        - GAMOS=linux
-        - PLATFORM=arm64
-        - VMTYPE=build
     - os: linux
       dist: xenial
       arch: arm64
       name: "Linux ARM64 Xenial"
-      language: bash
+      language: shell
       filter_secrets: false
-      addons:
-        apt:
-          packages:
-            - ruby
-      env:
-        - GAMOS=linux
-        - PLATFORM=arm64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Trusty"
-      dist: trusty
-      language: bash
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Precise"
-      dist: precise
-      language: bash
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: linux
-      name: "Python 3.5 Source Testing"
-      dist: xenial
-      language: python
-      python:
-        - "3.5"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
     - os: linux
       name: "Python 3.6 Source Testing"
-      dist: bionic
       language: python
-      python:
-        - "3.6"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
+      python: 3.6
     - os: linux
       name: "Python 3.7 Source Testing"
-      dist: bionic
       language: python
-      python:
-        - "3.7"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
+      python: 3.7
     - os: linux
       name: "Python nightly Source Testing"
-      dist: bionic
       language: python
-      python:
-        - "nightly"
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
+      python: nightly
     - os: linux
       name: "Python PyPi Source Testing"
-      dist: xenial
       language: python
-      python: pypy3.5
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: osx
-      name: "MacOS 10.12"
-      language: generic
-      osx_image: xcode9.2
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
+      python: pypy3
     - os: osx
       name: "MacOS 10.13"
       language: generic
       osx_image: xcode10.1
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: osx
       name: "MacOS 10.14"
       language: generic
       osx_image: xcode11.3
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: windows
       name: "Windows 64-bit"
       language: shell
-      filter_secrets: false
-      env:
-        - GAMOS=windows
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: windows
       name: "Windows 32-bit"
       language: shell
-      filter_secrets: false
-      env:
-        - GAMOS=windows
-        - PLATFORM=x86
-        - VMTYPE=build
-
-before_cache:
-- echo "this is before cache"
 
 before_install:
-- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-before-install.sh
+- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    export GAMOS="macos";
+  else
+    export GAMOS="${TRAVIS_OS_NAME}";
+  fi
+- if [ "${TRAVIS_JOB_NAME}" == "Windows 32-bit" ]; then
+    export PLATFORM="x86";
+  elif [ "${TRAVIS_CPU_ARCH}" == "amd64" ]; then
+    export PLATFORM="x86_64";
+  else
+    export PLATFORM="${TRAVIS_CPU_ARCH}";
+  fi
+- source src/travis/${TRAVIS_OS_NAME}-before-install.sh
 
 install:
-- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
+- source src/travis/${TRAVIS_OS_NAME}-install.sh
 
 script:
-# Discover and run all Python unit tests
-- $python -m unittest discover -s ./ -p "*_test.py"
+# Discover and run all Python unit tests. Buffer output so that it's not sent to the build log.
+- $python -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer
+- touch $gampath/nobrowser.txt
 - $gam version extended
 - $gam version | grep travis # travis should be part of the path (not /tmp or such)
 # determine which Python version GAM is built with and ensure it's at least build version from above.
-- if [ "$VMTYPE" == "build" ]; then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $BUILD_PYTHON_VERSION; fi
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]];  then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $MIN_PYTHON_VERSION; fi
 # determine which OpenSSL version GAM is built with and ensure it's at least build version from above.
-- if [ "$VMTYPE" == "build" ]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $BUILD_OPENSSL_VERSION; fi
-- if [ "$VMTYPE" == "build" ]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
-- if [ "$VMTYPE" == "build" ]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $MIN_OPENSSL_VERSION; fi
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
 - export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
 - if [ "$TRAVIS_EVENT_TYPE" != "pull_request" ]; then export e2e=true; fi
 - if [ "$e2e" = true ]; then export gam_user=gam-travis-$jid@pdl.jaylee.us; fi
 - if [ "$e2e" = true ]; then openssl aes-256-cbc -K $encrypted_ab10ec38326e_key -iv $encrypted_ab10ec38326e_iv -in travis/oauth2service.json.enc -out $gampath/oauth2service.json -d; fi
-- if [ "$e2e" = true ]; then cat travis/cfg_template.json | python travis/svars-write.py &> /dev/null; fi
+- if [ "$e2e" = true ]; then cat travis/cfg_template.json | $python travis/svars-write.py &> /dev/null; fi
 - if [ "$e2e" = true ]; then $gam info domain; fi
 - if [ "$e2e" = true ]; then $gam oauth info; fi
 - if [ "$e2e" = true ]; then $gam oauth refresh; fi
@@ -270,17 +183,14 @@ script:
 - if [ "$e2e" = true ]; then $gam calendar $gam_user add editor $newuser; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user showacl; fi
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete id ~id; fi
-- if [ "$e2e" = true ]; then $gam printer register; fi
-- if [ "$e2e" = true ]; then source travis/set_printer_csv_filter.sh; fi
-- if [ "$e2e" = true ]; then $gam print printers > printers.csv; fi
-- if [ "$e2e" = true ]; then unset GAM_CSV_ROW_FILTER; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam printer ~id add USER $newgroup; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam printjob ~id submit https://www.github.com/jay0lee/GAM; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam info printer ~id; fi
-- if [ "$e2e" = true ]; then $gam print printjobs; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam printjob ~id fetch; fi
-- if [ "$e2e" = true ]; then $gam print printjobs | $gam csv - gam printjob ~id delete; fi
-- if [ "$e2e" = true ]; then $gam csv printers.csv gam delete printer ~id; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user addevent summary "Travis test event" start $(date '+%FT%T.%N%:z' -d "now + 1 hour") end $(date '+%FT%T.%N%:z' -d "now + 2 hours") attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all; fi
+- if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
+- if [ "$e2e" = true ]; then matterid=uid:$($gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser | head -1 | cut -d ' ' -f 3); fi
+- if [ "$e2e" = true ]; then $gam create vaulthold matter $matterid name "Travis hold $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print vaultmatters matterstate open; fi
+- if [ "$e2e" = true ]; then $gam print vaultholds matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam create vaultexport matter $matterid name "Travis export $newbase" corpus mail accounts $newuser; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~; fi
 - if [ "$e2e" = true ]; then $gam csv sample.csv gam user ~email add calendar id:$newresource; fi
 - if [ "$e2e" = true ]; then $gam delete resource $newresource; fi
 - if [ "$e2e" = true ]; then $gam delete feature Whiteboard-$newbase; fi
@@ -290,11 +200,19 @@ script:
 - if [ "$e2e" = true ]; then $gam create alias $newalias user $newuser; fi
 - if [ "$e2e" = true ]; then $gam whatis $newuser; fi
 - if [ "$e2e" = true ]; then $gam user $gam_user show tokens; fi
+- if [ "$e2e" = true ]; then $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~; fi
+- if [ "$e2e" = true ]; then $gam delete hold "Travis hold $newbase" matter $matterid; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action close; fi
+- if [ "$e2e" = true ]; then $gam update matter $matterid action delete; fi
 - if [ "$e2e" = true ]; then $gam delete user $newuser; fi
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
 - if [ "$e2e" = true ]; then $gam print mobile; fi
 - if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
-- if [ "$TRAVIS_OS_NAME" != "windows" ]; then bash <(curl -s -S -L https://git.io/install-gam) -l; fi
+- if [ "$e2e" = true ]; then $gam report usageparameters customer; fi
+- if [ "$e2e" = true ]; then $gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins; fi
+- if [ "$e2e" = true ]; then $gam report customer todrive; fi
+- if [ "$e2e" = true ]; then $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
+- if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
 
 before_deploy:
 - export TRAVIS_TAG="preview"
@@ -302,14 +220,13 @@ before_deploy:
 
 deploy:
   provider: releases
-  api_key:
+  token:
     secure: bzambMcQwyv/o5c5GrKGCsZHgE5R85tg8sNFvPfpISz3+uosCjnBXas7wvCKzT75XUFi2ztfbYak6HdKf4sGnNHk0saEicB3slH+ghPyZbYzp76yvvduhFO2nWW3/F01tL+Yfqqt4/q8wFaWGjrC5km+6GLVyB4lWA/Uyu49qKnz02uSwyhBD/VFbO7DOQ65a1iWk9HngyMsu0Oi7HIbSjSLtxTHedNfOf3waW0NivTTxYXiYGX/MCu3GWhgIGj47a+H3A6FcQ/9QWvnKgnoixdgPBUz7kDb7ktsWwQsILPGStgH7iMuG49ZlXdEFmqwifBri2wvzmFEevBGZjHcupy1IGrNFRG+IUGKMotio+OkLHlLjuv7ZJtqCz/Vf5SNFgNyMSanx6jKEUJuYvndVg99IRXmYVwHFwPu5BAcJACpU6C0AfyGmmSqqwxCd46uXL62ynxNFpHuRfOqlDnmCTfZgjOciJSlDDpf+Xz9fF7+oCoeCi3mrcZVFjhd3tT6Oxw5HrsDtm0ZNld1cdLidaq8H6vOFgHMd0A9yNYZzTzXTvpmxzkXT4Zc7s+PYKN6z5fRZ+pJeckUjRXblvVEfs5HFSymavcOc5AkRwxpvOsTQMNmlnaJCBo5UNs0K/rVmRi5cFmaiwTcBCY0kTllOBJ4zWsfq8seiokWwNUNK2g=
   file_glob: true
   overwrite: true
   file: gam-$GAMVERSION-*
   skip_cleanup: true
   draft: true
-  all_branches: true
   on:
     repo: jay0lee/GAM
-    condition: $VMTYPE = build
+    condition: $TRAVIS_JOB_NAME != *"Testing"

src/GamCommands.txt

@@ -38,6 +38,7 @@ If an item contains spaces, it should be surrounded by ".
 	papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|
 	saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|
 	tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen
+<DayOfWeek> ::= mon|tue|wed|thu|fri|sat|sun
 <FileFormat> ::=
 	csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
 <LabelColorHex> ::=
@@ -880,6 +881,25 @@ gam update resoldsubscription <CustomerID> <SKUID>
 gam delete resoldsubscription <CustomerID> <SKUID> cancel|downgrade|transfer_to_direct
 gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 
+<ActivityApplicationName> ::=
+	access|accesstransparency|
+	admin|
+	calendar|calendars|
+	chat|
+	drive|doc|docs|
+	enterprisegroups|groupsenterprise|
+	gcp|
+	google+|gplus|
+	group|groups|
+	hangoutsmeet|meet|
+	jamboard|
+	login|logins|
+	mobile|
+	oauthtoken|token|tokens|
+	rules|
+	saml|
+	useraccounts
+
 <ReportsApp> ::=
 	accounts|
 	app_maker|
@@ -895,12 +915,28 @@ gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 	sites
 <ReportsAppList> ::= "<ReportsApp>(,<ReportsApp>)*"
 
-gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
-	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)] [filter|filters <String>] [fields|parameters <String>]
-gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
+gam report usageparameters customer|user [todrive]
+gam report usage user [todrive]
+	[<UserTypeItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
 	[fields|parameters <String>]
-gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token [todrive]
-	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
+gam report usage customer [todrive]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>[:<Date>](,<Date>[:<Date>])*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+
+gam report users|user [todrive]
+	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[filter|filters <String>] [fields|parameters <String>]
+gam report customers|customer|domain [todrive]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[fields|parameters <String>]
+gam report <ActivityApplicationName> [todrive]
+	[user all|<UserItem>]
+	[start <Time>] [end <Time>]
+	[filter|filters <String>] [event <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
 gam delete admin <RoleAssignmentId>
@@ -985,7 +1021,7 @@ The following attributes are equivalent:
 	(end (allday <Date>)|<Time>)|
 	guestscantinviteothers|
 	guestscantseeothers|
-	(id <String>)|
+	hangoutsmeet|
 	(location <String>)|
 	(noreminders| (reminder <Number> email|popup|sms))|
 	(optionalattendee <EmailAddress>)|
@@ -999,6 +1035,11 @@ The following attributes are equivalent:
 	(timezone <Timezone>)|
 	(visibility default|public|prvate)
 
+<EventUpdateAttributes> ::=
+	<EventAttributes>|
+	(removeattendee <EmailAddress>)|
+	(replacedescription <RegularExpression> <String>)
+
 <EventSelectProperty:> ::=
 	(after <Time>)|
 	(before <Time>)|
@@ -1010,9 +1051,10 @@ The following attributes are equivalent:
 <EventDisplayProperty> ::=
 	(timezone <TimeZone>)
 
-gam calendar <CalendarItem> addevent <EventAttributes>+ [<EventNotificationAttribute>]
+gam calendar <CalendarItem> addevent [id <String>] <EventAttributes>+ [<EventNotificationAttribute>]
 gam calendar <CalendarItem> deleteevent id|eventid <EventID> [doit] [<EventNotificationAttribute>]
 gam calendar <CalendarItem> moveevent id|eventid <EventID> [doit] [<EventNotificationAttribute>]
+gam calendar <CalendarItem> updateevent <EventID> <EventUpdateAttributes>+ [<EventNotificationAttribute>]
 gam calendar <CalendarItem> wipe
 gam calendar <CalendarItem> printevents <EventSelectProperty>* <EventDisplayProperty>* [todrive]
 
@@ -1079,7 +1121,7 @@ gam print mobile [todrive] [(query <QueryMobile>)|(queries <QueryMobileList>)] [
 	fields <MobileFieldNameList>] [delimiter <Character>] [appslimit <Number>] [listlimit <Number>]
 
 gam create group <EmailAddress> <GroupAttributes>*
-gam update group <GroupItem> [admincreated <Boolean>] [email <EmailAddress>] <GroupAttributes>*
+gam update group <GroupItem> [email <EmailAddress>] <GroupAttributes>*
 gam update group <GroupItem> add [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
 gam update group <GroupItem> delete|remove [owner|manager|member] <UserTypeEntity>
 gam update group <GroupItem> sync [owner|manager|member] [notsuspended|suspended] [allmail|daily|digest|none|nomail] <UserTypeEntity>
@@ -1228,7 +1270,7 @@ gam reopen vaultmatter|matter <MatterItem>
 gam delete vaultmatter|matter <MatterItem>
 gam undelete vaultmatter|matter <MatterItem>
 gam info vaultmatter|matter <MatterItem>
-gam print vaultmatters|matters [todrive] [basic|full]
+gam print vaultmatters|matters [todrive] [basic|full] [matterstate open|closed|deleted]
 
 gam <UserTypeEntity> delete|del asp|asps|applicationspecificpasswords all|<ASPIDList>
 gam <UserTypeEntity> show asps|asp|applicationspecificpasswords
@@ -1257,7 +1299,7 @@ gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
 gam <UserTypeEntity> show filerevisions <DriveFileID>
 gam <UserTypeEntity> show filetree [anyowner] (orderby <DriveOrderByFieldName> [ascending|descending])*
 
-gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive]
+gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive] [returnidonly]
 gam <UserTypeEntity> update drivefile (id <DriveFileID)|(drivefilename <DriveFileName>)|(query <QueryDriveFile) [copy] [newfilename <DriveFileName>] <DriveFileUpdateAttributes>*
 gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>)
 	[revision <Number>] [(format <FileFormatList>)|(csvsheet <String>)]

src/auth/__init__.py

@@ -0,0 +1,26 @@
+"""Authentication/Credentials general purpose and convenience methods."""
+
+import transport
+from var import _FN_OAUTH2_TXT
+from var import GC_OAUTH2_TXT
+from var import GC_Values
+from . import oauth
+# TODO: Move logic that determines file name into this module. We should be able
+# to discover the file location without accessing a private member or waiting
+# for a global initialization.
+DEFAULT_OAUTH_STORAGE_FILE = _FN_OAUTH2_TXT
+
+
+def get_admin_credentials_filename():
+    """Gets the name of the file that stores the admin account credentials."""
+    # If the environment globals are loaded, use the set global value. It may have
+    # some custom name in it. Otherwise, just use the default name.
+    if GC_Values[GC_OAUTH2_TXT]:
+        return GC_Values[GC_OAUTH2_TXT]
+    return DEFAULT_OAUTH_STORAGE_FILE
+
+
+def get_admin_credentials():
+    """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+    credential_file = get_admin_credentials_filename()
+    return oauth.Credentials.from_credentials_file(credential_file)

src/auth/oauth.py

@@ -0,0 +1,553 @@
+"""OAuth2.0 user credentials."""
+
+import datetime
+import json
+import os
+import re
+import threading
+from urllib.parse import urlencode
+
+from filelock import FileLock
+import google_auth_oauthlib.flow
+import google.oauth2.credentials
+import google.oauth2.id_token
+
+import fileutils
+import transport
+from var import GAM_INFO
+from var import GM_Globals
+from var import GM_WINDOWS
+import utils
+
+MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = ('\nGo to the following link in your '
+                                        'browser:\n\n\t{url}\n')
+MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
+MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = ('\nYour browser has been opened to'
+                                             ' visit:\n\n\t{url}\n\nIf your '
+                                             'browser is on a different machine'
+                                             ' then press CTRL+C and create a '
+                                             'file called nobrowser.txt in the '
+                                             'same folder as GAM.\n')
+MESSAGE_LOCAL_SERVER_SUCCESS = ('The authentication flow has completed. You may'
+                                ' close this browser window and return to GAM.')
+
+
+class CredentialsError(Exception):
+  """Base error class."""
+  pass
+
+
+class InvalidCredentialsFileError(CredentialsError):
+  """Error raised when a file cannot be opened into a credentials object."""
+  pass
+
+
+class EmptyCredentialsFileError(InvalidCredentialsFileError):
+  """Error raised when a credentials file contains no content."""
+  pass
+
+
+class InvalidClientSecretsFileFormatError(CredentialsError):
+  """Error raised when a client secrets file format is invalid."""
+  pass
+
+
+class InvalidClientSecretsFileError(CredentialsError):
+  """Error raised when client secrets file cannot be read."""
+  pass
+
+
+class Credentials(google.oauth2.credentials.Credentials):
+  """Google OAuth2.0 Credentials with GAM-specific properties and methods."""
+
+  DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+  def __init__(self,
+               token,
+               refresh_token=None,
+               id_token=None,
+               token_uri=None,
+               client_id=None,
+               client_secret=None,
+               scopes=None,
+               quota_project_id=None,
+               expiry=None,
+               id_token_data=None,
+               filename=None):
+    """A thread-safe OAuth2.0 credentials object.
+
+    Credentials adds additional utility properties and methods to a
+    standard OAuth2.0 credentials object. When used to store credentials on
+    disk, it implements a file lock to avoid collision during writes.
+
+    Args:
+      token: Optional String, The OAuth 2.0 access token. Can be None if refresh
+        information is provided.
+      refresh_token: String, The OAuth 2.0 refresh token. If specified,
+        credentials can be refreshed.
+      id_token: String, The Open ID Connect ID Token.
+      token_uri: String, The OAuth 2.0 authorization server's token endpoint
+        URI. Must be specified for refresh, can be left as None if the token can
+        not be refreshed.
+      client_id: String, The OAuth 2.0 client ID. Must be specified for refresh,
+        can be left as None if the token can not be refreshed.
+      client_secret: String, The OAuth 2.0 client secret. Must be specified for
+        refresh, can be left as None if the token can not be refreshed.
+      scopes: Sequence[str], The scopes used to obtain authorization.
+        This parameter is used by :meth:`has_scopes`. OAuth 2.0 credentials can
+          not request additional scopes after authorization. The scopes must be
+          derivable from the refresh token if refresh information is provided
+          (e.g. The refresh token scopes are a superset of this or contain a
+          wild card scope like
+            'https://www.googleapis.com/auth/any-api').
+      quota_project_id: String, The project ID used for quota and billing. This
+        project may be different from the project used to create the
+        credentials.
+      expiry: datetime.datetime, The time at which the provided token will
+        expire.
+      id_token_data: Oauth2.0 ID Token data which was previously fetched for
+        this access token against the google.oauth2.id_token library.
+      filename: String, Path to a file that will be used to store the
+        credentials. If provided, a lock file of the same name and a ".lock"
+          extension will be created for concurrency controls. Note: New
+            credentials are not saved to disk until write() or refresh() are
+            called.
+
+    Raises:
+      TypeError: If id_token_data is not the required dict type.
+    """
+    super(Credentials, self).__init__(
+        token=token,
+        refresh_token=refresh_token,
+        id_token=id_token,
+        token_uri=token_uri,
+        client_id=client_id,
+        client_secret=client_secret,
+        scopes=scopes,
+        quota_project_id=quota_project_id)
+
+    # Load data not restored by the super class
+    self.expiry = expiry
+    if id_token_data and not isinstance(id_token_data, dict):
+      raise TypeError(f'Expected type id_token_data dict but received '
+                      f'{type(id_token_data)}')
+    self._id_token_data = id_token_data.copy() if id_token_data else None
+
+    # If a filename is provided, use a lock file to control concurrent access
+    # to the resource. If no filename is provided, use a thread lock that has
+    # the same interface as FileLock in order to simplify the implementation.
+    if filename:
+      # Convert relative paths into absolute
+      self._filename = os.path.abspath(filename)
+      lock_file = os.path.abspath(f'{self._filename}.lock')
+      self._lock = FileLock(lock_file)
+    else:
+      self._filename = None
+      self._lock = _FileLikeThreadLock()
+
+  # Use a property to prevent external mutation of the filename.
+  @property
+  def filename(self):
+    return self._filename
+
+  @classmethod
+  def from_authorized_user_info(cls, info, filename=None):
+    """Generates Credentials from JSON containing authorized user info.
+
+    Args:
+      info: Dict, authorized user info in Google format.
+      filename: String, the filename used to store these credentials on disk. If
+        no filename is provided, the credentials will not be saved to disk.
+
+    Raises:
+      ValueError: If missing fields are detected in the info.
+    """
+    # We need all of these keys
+    keys_needed = set(('client_id', 'client_secret'))
+    # We need 1 or more of these keys
+    keys_need_one_of = set(('refresh_token', 'auth_token', 'token'))
+    missing = keys_needed.difference(info.keys())
+    has_one_of = set(info) & keys_need_one_of
+    if missing or not has_one_of:
+      raise ValueError(
+          'Authorized user info was not in the expected format, missing '
+          f'fields {", ".join(missing)} and one of '
+          f'{", ".join(keys_need_one_of)}.')
+
+    expiry = info.get('token_expiry')
+    if expiry:
+      # Convert the raw expiry to datetime
+      expiry = datetime.datetime.strptime(expiry, Credentials.DATETIME_FORMAT)
+    id_token_data = info.get('decoded_id_token')
+
+    # Provide backwards compatibility with field names when loading from JSON.
+    # Some field names may be different, depending on when/how the credentials
+    # were pickled.
+    return cls(
+        token=info.get('token', info.get('auth_token', '')),
+        refresh_token=info.get('refresh_token', ''),
+        id_token=info.get('id_token_jwt', info.get('id_token')),
+        token_uri=info.get('token_uri'),
+        client_id=info['client_id'],
+        client_secret=info['client_secret'],
+        scopes=info.get('scopes'),
+        quota_project_id=info.get('quota_project_id'),
+        expiry=expiry,
+        id_token_data=id_token_data,
+        filename=filename)
+
+  @classmethod
+  def from_google_oauth2_credentials(cls, credentials, filename=None):
+    """Generates Credentials from a google.oauth2.Credentials object."""
+    info = json.loads(credentials.to_json())
+    # Add properties which are not exported with the native to_json() output.
+    info['id_token'] = credentials.id_token
+    if credentials.expiry:
+      info['token_expiry'] = credentials.expiry.strftime(
+          Credentials.DATETIME_FORMAT)
+    info['quota_project_id'] = credentials.quota_project_id
+
+    return cls.from_authorized_user_info(info, filename=filename)
+
+  @classmethod
+  def from_credentials_file(cls, filename):
+    """Generates Credentials from a stored Credentials file.
+
+    The same file will be used to save the credentials when the access token is
+    refreshed.
+
+    Args:
+      filename: String, the name of a file containing JSON credentials to load.
+        The same filename will be used to save credentials back to disk.
+
+    Returns:
+      The credentials loaded from disk.
+
+    Raises:
+      InvalidCredentialsFileError: When the credentials file cannot be opened.
+      EmptyCredentialsFileError: When the provided file contains no credentials.
+    """
+    file_content = fileutils.read_file(
+        filename, continue_on_error=True, display_errors=False)
+    if file_content is None:
+      raise InvalidCredentialsFileError(f'File {filename} could not be opened')
+    info = json.loads(file_content)
+    if not info:
+      raise EmptyCredentialsFileError(
+          f'File {filename} contains no credential data')
+
+    try:
+      # We read the existing data from the passed in file, but we also want to
+      # save future data/tokens in the same place.
+      return cls.from_authorized_user_info(info, filename=filename)
+    except ValueError as e:
+      raise InvalidCredentialsFileError(str(e))
+
+  @classmethod
+  def from_client_secrets(cls,
+                          client_id,
+                          client_secret,
+                          scopes,
+                          access_type='offline',
+                          login_hint=None,
+                          filename=None,
+                          use_console_flow=False):
+    """Runs an OAuth Flow from client secrets to generate credentials.
+
+    Args:
+      client_id: String, The OAuth2.0 Client ID.
+      client_secret: String, The OAuth2.0 Client Secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      filename: String, the path to a file to use to save the credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Returns:
+      Credentials
+    """
+    client_config = {
+        'installed': {
+            'client_id': client_id,
+            'client_secret': client_secret,
+            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
+            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+            'token_uri': 'https://oauth2.googleapis.com/token',
+        }
+    }
+
+    flow = _ShortURLFlow.from_client_config(
+        client_config, scopes, autogenerate_code_verifier=True)
+    flow_kwargs = {'access_type': access_type}
+    if login_hint:
+      flow_kwargs['login_hint'] = login_hint
+
+    # TODO: Move code for browser detection somewhere in this file so that the
+    # messaging about `nobrowser.txt` is co-located with the logic that uses it.
+    if use_console_flow:
+      flow.run_console(
+          authorization_prompt_message=MESSAGE_CONSOLE_AUTHORIZATION_PROMPT,
+          authorization_code_message=MESSAGE_CONSOLE_AUTHORIZATION_CODE,
+          **flow_kwargs)
+    else:
+      flow.run_local_server(
+          authorization_prompt_message=MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT,
+          success_message=MESSAGE_LOCAL_SERVER_SUCCESS,
+          **flow_kwargs)
+    return cls.from_google_oauth2_credentials(
+        flow.credentials, filename=filename)
+
+  @classmethod
+  def from_client_secrets_file(cls,
+                               client_secrets_file,
+                               scopes,
+                               access_type='offline',
+                               login_hint=None,
+                               credentials_file=None,
+                               use_console_flow=False):
+    """Runs an OAuth Flow from secrets stored on disk to generate credentials.
+
+    Args:
+      client_secrets_file: String, path to a file containing a client ID and
+        secret.
+      scopes: Sequence[str], A list of scopes to include in the credentials.
+      access_type: String, 'offline' or 'online'.  Indicates whether your
+        application can refresh access tokens when the user is not present at
+        the browser. Valid parameter values are online, which is the default
+        value, and offline.  Set the value to offline if your application needs
+        to refresh access tokens when the user is not present at the browser.
+        This is the method of refreshing access tokens described later in this
+        document. This value instructs the Google authorization server to return
+        a refresh token and an access token the first time that your application
+        exchanges an authorization code for tokens.
+      login_hint: String, The email address that will be displayed on the Google
+        login page as a hint for the user to login to the correct account.
+      credentials_file: String, the path to a file to use to save the
+        credentials.
+      use_console_flow: Boolean, True if the authentication flow should be run
+        strictly from a console; False to launch a browser for authentication.
+
+    Raises:
+      InvalidClientSecretsFileError: If the client secrets file cannot be
+        opened.
+      InvalidClientSecretsFileFormatError: If the client secrets file does not
+        contain the required data or the data is malformed.
+
+    Returns:
+      Credentials
+    """
+    cs_data = fileutils.read_file(
+        client_secrets_file, continue_on_error=True, display_errors=False)
+    if not cs_data:
+      raise InvalidClientSecretsFileError(
+          f'File {client_secrets_file} could not be opened')
+    try:
+      cs_json = json.loads(cs_data)
+      client_id = cs_json['installed']['client_id']
+      # Chop off .apps.googleusercontent.com suffix as it's not needed
+      # and we need to keep things short for the Auth URL.
+      client_id = re.sub(r'\.apps\.googleusercontent\.com$', '', client_id)
+      client_secret = cs_json['installed']['client_secret']
+    except (ValueError, IndexError, KeyError):
+      raise InvalidClientSecretsFileFormatError(
+          f'Could not extract Client ID or Client Secret from file {client_secrets_file}'
+      )
+
+    return cls.from_client_secrets(
+        client_id,
+        client_secret,
+        scopes,
+        access_type=access_type,
+        login_hint=login_hint,
+        filename=credentials_file,
+        use_console_flow=use_console_flow)
+
+  def _fetch_id_token_data(self):
+    """Fetches verification details from Google for the OAuth2.0 token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Raises:
+      CredentialsError: If no id_token is present.
+    """
+    if not self.id_token:
+      raise CredentialsError('Failed to fetch token data. No id_token present.')
+
+    request = transport.create_request()
+    if self.expired:
+      # The id_token needs to be unexpired, in order to request data about it.
+      self.refresh(request)
+
+    self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
+        self.id_token, request)
+
+  def get_token_value(self, field):
+    """Retrieves data from the OAuth ID token.
+
+    See more: https://developers.google.com/identity/sign-in/web/backend-auth
+
+    Args:
+      field: The name of the key/field to fetch
+
+    Returns:
+      The value associated with the given key or 'Unknown' if the key data can
+      not be found in the access token data.
+    """
+    if not self._id_token_data:
+      self._fetch_id_token_data()
+    # Maintain legacy GAM behavior here to return "Unknown" if the field is
+    # otherwise unpopulated.
+    return self._id_token_data.get(field, 'Unknown')
+
+  def to_json(self, strip=None):
+    """Creates a JSON representation of a Credentials.
+
+    Args:
+        strip: Sequence[str], Optional list of members to exclude from the
+          generated JSON.
+
+    Returns:
+        str: A JSON representation of this instance, suitable to pass to
+             from_json().
+    """
+    expiry = self.expiry.strftime(
+        Credentials.DATETIME_FORMAT) if self.expiry else None
+    prep = {
+        'token': self.token,
+        'refresh_token': self.refresh_token,
+        'token_uri': self.token_uri,
+        'client_id': self.client_id,
+        'client_secret': self.client_secret,
+        'id_token': self.id_token,
+        # Google auth doesn't currently give us scopes back on refresh.
+        # 'scopes': sorted(self.scopes),
+        'token_expiry': expiry,
+        'decoded_id_token': self._id_token_data,
+    }
+
+    # Remove empty entries
+    prep = {k: v for k, v in prep.items() if v is not None}
+
+    # Remove entries that explicitly need to be removed
+    if strip is not None:
+      prep = {k: v for k, v in prep.items() if k not in strip}
+
+    return json.dumps(prep, indent=2, sort_keys=True)
+
+  def refresh(self, request=None):
+    """Refreshes the credential's access token.
+
+    Args:
+      request: google.auth.transport.Request, The object used to make HTTP
+        requests. If not provided, a default request will be used.
+
+    Raises:
+      google.auth.exceptions.RefreshError: If the credentials could not be
+      refreshed.
+    """
+    with self._lock:
+      if request is None:
+        request = transport.create_request()
+      self._locked_refresh(request)
+      # Save the new tokens back to disk, if these credentials are disk-backed.
+      if self._filename:
+        self._locked_write()
+
+  def _locked_refresh(self, request):
+    """Refreshes the credential's access token while the file lock is held."""
+    assert self._lock.is_locked
+    super(Credentials, self).refresh(request)
+
+  def write(self):
+    """Writes credentials to disk."""
+    with self._lock:
+      self._locked_write()
+
+  def _locked_write(self):
+    """Writes credentials to disk while the file lock is held."""
+    assert self._lock.is_locked
+    if not self.filename:
+      # If no filename was provided to the constructor, these credentials cannot
+      # be saved to disk.
+      raise CredentialsError(
+          'The credentials have no associated filename and cannot be saved '
+          'to disk.')
+    fileutils.write_file(self._filename, self.to_json())
+
+  def delete(self):
+    """Deletes all files on disk related to these credentials."""
+    with self._lock:
+      # Only attempt to remove the file if the lock we're using is a FileLock.
+      if isinstance(self._lock, FileLock):
+        os.remove(self._filename)
+        if self._lock.lock_file and not GM_Globals[GM_WINDOWS]:
+          os.remove(self._lock.lock_file)
+
+  _REVOKE_TOKEN_BASE_URI = 'https://accounts.google.com/o/oauth2/revoke'
+
+  def revoke(self, http=None):
+    """Revokes this credential's access token with the server.
+
+    Args:
+      http: httplib2.Http compatible object for use as a transport. If no http
+        is provided, a default will be used.
+    """
+    with self._lock:
+      if http is None:
+        http = transport.create_http()
+      params = urlencode({'token': self.refresh_token})
+      revoke_uri = f'{Credentials._REVOKE_TOKEN_BASE_URI}?{params}'
+      http.request(revoke_uri, 'GET')
+
+
+class _ShortURLFlow(google_auth_oauthlib.flow.InstalledAppFlow):
+  """InstalledAppFlow which utilizes a URL shortener for authorization URLs."""
+
+  URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+  def authorization_url(self, http=None, **kwargs):
+    """Gets a shortened authorization URL."""
+    long_url, state = super(_ShortURLFlow, self).authorization_url(**kwargs)
+    short_url = utils.shorten_url(long_url)
+    return short_url, state
+
+class _FileLikeThreadLock(object):
+  """A threading.lock which has the same interface as filelock.Filelock."""
+
+  def __init__(self):
+    """A shell object that holds a threading.Lock.
+
+    Since we cannot inherit from built-in classes such as threading.Lock, we
+    just use a shell object and maintain a lock inside of it.
+    """
+    self._lock = threading.Lock()
+
+  def __enter__(self, *args, **kwargs):
+    return self._lock.__enter__(*args, **kwargs)
+
+  def __exit__(self, *args, **kwargs):
+    return self._lock.__exit__(*args, **kwargs)
+
+  def acquire(self, **kwargs):
+    return self._lock.acquire(**kwargs)
+
+  def release(self):
+    return self._lock.release()
+
+  @property
+  def is_locked(self):
+    return self._lock.locked()
+
+  @property
+  def lock_file(self):
+    return None

src/auth/oauth_test.py

@@ -0,0 +1,705 @@
+"""Tests for oauth."""
+
+import datetime
+import json
+import os
+import platform
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import google.oauth2.credentials
+
+from auth import oauth
+
+
+class CredentialsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_token = 'fake_token'
+    self.fake_refresh_token = 'fake_refresh_token'
+    self.fake_id_token = 'fake_id_token'
+    self.fake_token_uri = 'https://fake.token.uri'
+    self.fake_client_id = 'fake_client_id'
+    self.fake_client_secret = 'fake_client_secret'
+    self.fake_scopes = [
+        'fake_api.readonly',
+        'fake_other_api.write',
+    ]
+    self.fake_quota_project_id = 'fake_quota_project_id'
+    self.fake_token_expiry = datetime.datetime(2020, 1, 1, 10)
+    self.fake_filename = 'fake_filename'
+    self.fake_token_data = {
+        'field': 'value',
+        'another-field': 'another-value',
+    }
+    self.info_with_only_required_fields = {
+        'refresh_token': self.fake_refresh_token,
+        'client_id': self.fake_client_id,
+        'client_secret': self.fake_client_secret,
+    }
+    super(CredentialsTest, self).setUp()
+
+  def tearDown(self):
+    # Remove any credential files that may have been created.
+    if os.path.exists(self.fake_filename):
+      os.remove(self.fake_filename)
+    if os.path.exists('%s.lock' % self.fake_filename):
+      os.remove('%s.lock' % self.fake_filename)
+    super(CredentialsTest, self).tearDown()
+
+  def test_from_authorized_user_info_only_required_info(self):
+    creds = oauth.Credentials.from_authorized_user_info(
+        self.info_with_only_required_fields)
+    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+    self.assertEqual(self.fake_client_id, creds.client_id)
+    self.assertEqual(self.fake_client_secret, creds.client_secret)
+    self.assertIsNone(creds.id_token)
+    self.assertIsNone(creds.expiry)
+    self.assertIsNone(creds.filename)
+
+  def test_from_authorized_user_info_all_info_provided(self):
+    info = {
+        'token':
+            self.fake_token,
+        'refresh_token':
+            self.fake_refresh_token,
+        'id_token':
+            self.fake_id_token,
+        'token_uri':
+            self.fake_token_uri,
+        'client_id':
+            self.fake_client_id,
+        'client_secret':
+            self.fake_client_secret,
+        'token_expiry':
+            self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+        'id_token_data':
+            self.fake_token_data,
+    }
+    creds = oauth.Credentials.from_authorized_user_info(info)
+    self.assertEqual(self.fake_refresh_token, creds.refresh_token)
+    self.assertEqual(self.fake_client_id, creds.client_id)
+    self.assertEqual(self.fake_client_secret, creds.client_secret)
+    self.assertEqual(self.fake_id_token, creds.id_token)
+    self.assertEqual(self.fake_token_uri, creds.token_uri)
+    self.assertEqual(self.fake_token_expiry, creds.expiry)
+    self.assertIsNone(creds.filename)
+
+  def test_from_authorized_user_info_missing_required_info(self):
+    info_with_missing_fields = {'token': self.fake_token}
+    with self.assertRaises(ValueError):
+      oauth.Credentials.from_authorized_user_info(info_with_missing_fields)
+
+  def test_from_authorized_user_info_no_expiry_in_info(self):
+    info_with_no_token_expiry = self.info_with_only_required_fields.copy()
+    self.assertIsNone(info_with_no_token_expiry.get('expiry'))
+    creds = oauth.Credentials.from_authorized_user_info(
+        info_with_no_token_expiry)
+    self.assertIsNone(creds.expiry)
+
+  def test_init_saves_filename(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+  def test_init_loads_decoded_id_token_data(self, mock_verify_token):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data)
+    self.assertEqual(
+        self.fake_token_data.get('field'), creds.get_token_value('field'))
+    # Verify the fetching method was not called, since the token
+    # data was supposed to be loaded from the passed in info.
+    self.assertEqual(mock_verify_token.call_count, 0)
+
+  def test_credentials_uses_file_lock_when_filename_provided(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    self.assertIsInstance(creds._lock, oauth.FileLock)
+    self.assertEqual(creds._lock.lock_file, '%s.lock' % creds.filename)
+
+  def test_credentials_uses_thread_lock_when_filename_not_provided(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=None)
+    self.assertIsInstance(creds._lock, oauth._FileLikeThreadLock)
+    self.assertIsNone(creds.filename)
+
+  def test_from_oauth2credentials(self):
+    google_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    creds = oauth.Credentials.from_google_oauth2_credentials(
+        google_creds, filename=self.fake_filename)
+    self.assertEqual(google_creds.token, creds.token)
+    self.assertEqual(google_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(google_creds.client_id, creds.client_id)
+    self.assertEqual(google_creds.client_secret, creds.client_secret)
+    self.assertEqual(google_creds.id_token, creds.id_token)
+    self.assertEqual(google_creds.expiry, creds.expiry)
+    self.assertEqual(google_creds.quota_project_id, creds.quota_project_id)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  def test_from_credentials_file_corrupt_or_missing_file_raises_error(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    with self.assertRaises(oauth.InvalidCredentialsFileError) as e:
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+    self.assertIn('could not be opened', str(e.exception))
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_no_serialized_data_in_file_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({})
+    with self.assertRaises(oauth.EmptyCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_missing_any_token_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({
+        # This data is missing a token key/value pair
+        'client_id': self.fake_client_id,
+        'client_secret': self.fake_client_secret,
+    })
+    with self.assertRaises(oauth.InvalidCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_credentials_file_missing_required_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({
+        # This data is missing a client_secret key/value pair
+        'client_id': self.fake_client_id,
+        'refresh_token': self.fake_refresh_token,
+    })
+    with self.assertRaises(oauth.InvalidCredentialsFileError):
+      oauth.Credentials.from_credentials_file(self.fake_filename)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_console_flow(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        use_console_flow=True)
+    self.assertTrue(mock_flow.return_value.run_console.called)
+    self.assertFalse(mock_flow.return_value.run_local_server.called)
+    self.assertEqual(flow_creds.token, creds.token)
+    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(flow_creds.client_id, creds.client_id)
+    self.assertEqual(flow_creds.client_secret, creds.client_secret)
+    self.assertEqual(flow_creds.id_token, creds.id_token)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_local_server_flow(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        use_console_flow=False)
+    self.assertFalse(mock_flow.return_value.run_console.called)
+    self.assertTrue(mock_flow.return_value.run_local_server.called)
+    self.assertEqual(flow_creds.token, creds.token)
+    self.assertEqual(flow_creds.refresh_token, creds.refresh_token)
+    self.assertEqual(flow_creds.client_id, creds.client_id)
+    self.assertEqual(flow_creds.client_secret, creds.client_secret)
+    self.assertEqual(flow_creds.id_token, creds.id_token)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_uses_login_hint(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        login_hint='someone@domain.com')
+
+    run_flow_args = mock_flow.return_value.run_local_server.call_args[1]
+    self.assertEqual('someone@domain.com', run_flow_args.get('login_hint'))
+
+  def test_from_client_secrets_uses_shortened_url_flow(self):
+    with patch.object(oauth._ShortURLFlow, 'from_client_config') as mock_flow:
+      flow_creds = google.oauth2.credentials.Credentials(
+          token=self.fake_token,
+          refresh_token=self.fake_refresh_token,
+          client_id=self.fake_client_id,
+          client_secret=self.fake_client_secret,
+          id_token=self.fake_id_token)
+      mock_flow.return_value.credentials = flow_creds
+      oauth.Credentials.from_client_secrets(self.fake_client_id,
+                                            self.fake_client_secret,
+                                            self.fake_scopes)
+    self.assertTrue(mock_flow.called)
+
+  @patch.object(oauth._ShortURLFlow, 'from_client_config')
+  def test_from_client_secrets_passes_credentials_filename(self, mock_flow):
+    flow_creds = google.oauth2.credentials.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token=self.fake_id_token)
+    mock_flow.return_value.credentials = flow_creds
+
+    creds = oauth.Credentials.from_client_secrets(
+        self.fake_client_id,
+        self.fake_client_secret,
+        self.fake_scopes,
+        filename=self.fake_filename)
+    self.assertEqual(os.path.abspath(self.fake_filename), creds.filename)
+
+  def test_from_client_secrets_file_corrupt_or_missing_file_raises_error(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    with self.assertRaises(oauth.InvalidClientSecretsFileError):
+      oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                 self.fake_scopes)
+
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_client_secrets_file_missing_required_json_raises_error(
+      self, mock_read_file):
+    mock_read_file.return_value = json.dumps({})
+    with self.assertRaises(oauth.InvalidClientSecretsFileFormatError) as e:
+      oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                                 self.fake_scopes)
+    self.assertIn('Could not extract Client ID or Client Secret',
+                  str(e.exception))
+
+  @patch.object(oauth.Credentials, 'from_client_secrets')
+  @patch.object(oauth.fileutils, 'read_file')
+  def test_from_client_secrets_file_strips_domain_from_client_id(
+      self, mock_read_file, mock_creds_from_client_secrets):
+    mock_read_file.return_value = json.dumps({
+        'installed': {
+            'client_id': self.fake_client_id + '.apps.googleusercontent.com',
+            'client_secret': self.fake_client_secret,
+        }
+    })
+
+    oauth.Credentials.from_client_secrets_file(self.fake_filename,
+                                               self.fake_scopes)
+    self.assertEqual(self.fake_client_id,
+                     mock_creds_from_client_secrets.call_args[0][0])
+
+  def test_get_token_value_known_token_field(self):
+    token_data = {'known-field': 'known-value'}
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token_data=token_data)
+    self.assertEqual('known-value', creds.get_token_value('known-field'))
+
+  def test_get_token_value_unknown_field_returns_unknown(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        id_token_data=self.fake_token_data)
+    self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
+
+  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+  def test_get_token_value_credentials_expired(self, mock_verify_oauth2_token):
+    mock_verify_oauth2_token.return_value = {'fetched-field': 'fetched-value'}
+    time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
+        minutes=5)
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        expiry=time_earlier_than_now,
+        id_token=self.fake_id_token,
+        id_token_data=None)
+    self.assertTrue(creds.expired)
+    creds.refresh = MagicMock()
+
+    token_value = creds.get_token_value('fetched-field')
+
+    self.assertEqual('fetched-value', token_value)
+    self.assertTrue(creds.refresh.called)
+
+  def test_to_json_contains_all_required_fields(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data,
+        token_uri=self.fake_token_uri,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        scopes=self.fake_scopes,
+        quota_project_id=self.fake_quota_project_id,
+        expiry=self.fake_token_expiry)
+    json_string = creds.to_json()
+    json_data = json.loads(json_string)
+    keys = json_data.keys()
+    self.assertIn('token', keys)
+    self.assertEqual(self.fake_token, json_data['token'])
+    self.assertIn('refresh_token', keys)
+    self.assertEqual(self.fake_refresh_token, json_data['refresh_token'])
+    self.assertIn('id_token', keys)
+    self.assertEqual(self.fake_id_token, json_data['id_token'])
+    self.assertIn('token_uri', keys)
+    self.assertEqual(self.fake_token_uri, json_data['token_uri'])
+    self.assertIn('client_id', keys)
+    self.assertEqual(self.fake_client_id, json_data['client_id'])
+    self.assertIn('client_secret', keys)
+    self.assertEqual(self.fake_client_secret, json_data['client_secret'])
+    self.assertNotIn('scopes', keys)  # Scopes are not currently saved
+    self.assertIn('token_expiry', keys)
+    self.assertEqual(
+        self.fake_token_expiry.strftime(oauth.Credentials.DATETIME_FORMAT),
+        json_data['token_expiry'])
+    self.assertIn('decoded_id_token', keys)
+    self.assertEqual(self.fake_token_data, json_data['decoded_id_token'])
+
+  def test_credentials_to_json_and_back(self):
+    original_creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        id_token=self.fake_id_token,
+        id_token_data=self.fake_token_data,
+        token_uri=self.fake_token_uri,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        scopes=self.fake_scopes,
+        quota_project_id=self.fake_quota_project_id,
+        expiry=self.fake_token_expiry)
+    pickled_creds = original_creds.to_json()
+    serialized_json = json.loads(pickled_creds)
+    unpickled_creds = oauth.Credentials.from_authorized_user_info(
+        serialized_json)
+    self.assertEqual(original_creds.token, unpickled_creds.token)
+    self.assertEqual(original_creds.refresh_token,
+                     unpickled_creds.refresh_token)
+    self.assertEqual(original_creds.id_token, unpickled_creds.id_token)
+    self.assertEqual(original_creds.token_uri, unpickled_creds.token_uri)
+    self.assertEqual(original_creds.client_id, unpickled_creds.client_id)
+    self.assertEqual(original_creds.client_secret,
+                     unpickled_creds.client_secret)
+    self.assertEqual(original_creds.expiry, unpickled_creds.expiry)
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  def test_refresh_calls_super_refresh(self, mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    request = MagicMock()
+
+    creds.refresh(request)
+    self.assertTrue(mock_super_refresh.called)
+    self.assertEqual(request, mock_super_refresh.call_args[0][0])
+
+  def test_refresh_locks_resource_during_refresh(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    lock = creds._lock
+
+    def check_lock_is_locked(*unused_args, **unused_kwargs):
+      self.assertTrue(lock.is_locked)
+
+    # We need to mock the superclass refresh so it doesn't actually try to
+    # refresh our fake token.
+    # At the same time, we'll make sure the lock is held during the refresh.
+    with patch.object(oauth.google.oauth2.credentials.Credentials,
+                      'refresh') as mock_refresh:
+      mock_refresh.side_effect = check_lock_is_locked
+      creds.refresh(request=MagicMock())
+
+    # Make sure our side effect was actually performed.
+    self.assertTrue(mock_refresh.called)
+    # The lock should be released after refresh
+    self.assertFalse(lock.is_locked)
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  @patch.object(oauth.fileutils, 'write_file')
+  def test_refresh_writes_new_credentials_to_disk_after_refresh(
+      self, mock_write_file, mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+
+    def update_access_token(unused_request):
+      creds.token = 'refreshed_access_token'
+
+    mock_super_refresh.side_effect = update_access_token
+
+    self.assertIsNone(creds.token)
+    creds.refresh(request=MagicMock())
+    self.assertEqual('refreshed_access_token', creds.token,
+                     'Access token was not refreshed')
+    text_written_to_file = mock_write_file.call_args[0][1]
+    self.assertIsNotNone(text_written_to_file, 'Nothing was written to file')
+    saved_json = json.loads(text_written_to_file)
+    self.assertEqual('refreshed_access_token', saved_json['token'],
+                     'Refreshed access token was not saved to disk')
+
+  def test_write_writes_credentials_to_disk(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+
+    self.assertFalse(os.path.exists(self.fake_filename))
+    creds.write()
+    self.assertTrue(os.path.exists(self.fake_filename))
+
+  def test_write_raises_error_when_no_credentials_file_is_set(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+
+    self.assertIsNone(creds.filename)
+    with self.assertRaises(oauth.CredentialsError):
+      creds.write()
+
+  @patch.object(oauth.google.oauth2.credentials.Credentials, 'refresh')
+  @patch.object(oauth.fileutils, 'write_file')
+  def test_write_locks_resource_during_write(self, mock_write_file,
+                                             unused_mock_super_refresh):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    lock = creds._lock
+
+    def check_lock_is_locked(*unused_args, **unused_kwargs):
+      self.assertTrue(creds._lock.is_locked)
+
+    mock_write_file.side_effect = check_lock_is_locked
+
+    self.assertFalse(lock.is_locked)
+    creds.refresh(request=MagicMock())
+    self.assertFalse(lock.is_locked)
+    self.assertTrue(mock_write_file.called)
+
+  def test_delete_removes_credentials_file(self):
+    self.assertFalse(os.path.exists(self.fake_filename))
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    creds.write()
+    self.assertTrue(os.path.exists(self.fake_filename))
+    creds.delete()
+    self.assertFalse(os.path.exists(self.fake_filename))
+
+  @unittest.skipIf(
+      platform.system() == 'Windows',
+      reason=('On Windows, Filelock deletes the lock file each time the lock '
+              'is released. Delete does not remove it.'))
+  def test_delete_removes_lock_file(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        filename=self.fake_filename)
+    lock_file = '%s.lock' % creds.filename
+    creds.write()
+    self.assertTrue(os.path.exists(lock_file))
+    creds.delete()
+    self.assertFalse(os.path.exists(lock_file))
+
+  def test_delete_is_noop_when_not_using_filelock(self):
+    creds = oauth.Credentials(
+        token=None,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    self.assertIsNone(creds.filename)
+    creds.delete()  # This should not raise an exception.
+
+  def test_revoke_requests_credential_revoke(self):
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        refresh_token=self.fake_refresh_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret)
+    mock_http = MagicMock()
+
+    creds.revoke(http=mock_http)
+
+    uri = mock_http.request.call_args[0][0]
+    self.assertRegex(uri, '^%s' % oauth.Credentials._REVOKE_TOKEN_BASE_URI)
+    params = uri[uri.index('?'):]
+    self.assertIn('token=%s' % creds.refresh_token, params)
+    self.assertEqual('GET', mock_http.request.call_args[0][1])
+
+
+class ShortUrlFlowTest(unittest.TestCase):
+
+  def setUp(self):
+    self.fake_client_id = 'fake_client_id'
+    self.fake_client_secret = 'fake_client_secret'
+    self.fake_scopes = [
+        'fake_api.readonly',
+        'fake_other_api.write',
+    ]
+    self.fake_client_config = {
+        'installed': {
+            'client_id': self.fake_client_id,
+            'client_secret': self.fake_client_secret,
+            'redirect_uris': ['http://localhost', 'urn:ietf:wg:oauth:2.0:oob'],
+            'auth_uri': 'https://accounts.google.com/o/oauth2/v2/auth',
+            'token_uri': 'https://oauth2.googleapis.com/token',
+        }
+    }
+    self.long_url = 'http://example.com/some/long/url'
+    self.short_url = 'http://ex.co/short'
+    super(ShortUrlFlowTest, self).setUp()
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  @unittest.skip('disable short url tests temporarily.')
+  def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = json.dumps({'short_url': self.short_url})
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.short_url, url)
+    self.assertEqual('fake_state', state)
+
+    # Verify request() was called with the expected arguments.
+    self.assertEqual(oauth._ShortURLFlow.URL_SHORTENER_ENDPOINT,
+                     mock_http.request.call_args[0][0])
+    self.assertEqual('POST', mock_http.request.call_args[0][1])
+    self.assertIn(self.long_url, mock_http.request.call_args[0][2])
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  @unittest.skip('disable short url tests temporarily.')
+  def test_shorturlflow_falls_back_to_long_url_on_request_error(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_http.request.side_effect = Exception()
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  @unittest.skip('disable short url tests temporarily.')
+  def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 404  # Use a status that is not 200
+    content = json.dumps({'short_url': self.short_url})
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  @unittest.skip('disable short url tests temporarily.')
+  def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = None
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+  @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
+                'authorization_url')
+  @unittest.skip('disable short url tests temporarily.')
+  def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
+      self, mock_super_auth_url):
+    url_flow = oauth._ShortURLFlow.from_client_config(
+        self.fake_client_config, scopes=self.fake_scopes)
+    mock_super_auth_url.return_value = (self.long_url, 'fake_state')
+
+    mock_http = MagicMock()
+    mock_response = MagicMock()
+    mock_response.status = 200
+    content = json.dumps({})  # This json content contains no "short-url" key
+    mock_http.request.return_value = (mock_response, content)
+
+    url, state = url_flow.authorization_url(http=mock_http)
+    self.assertEqual(self.long_url, url)
+    self.assertEqual('fake_state', state)
+
+
+if __name__ == '__main__':
+  unittest.main()

src/controlflow.py

@@ -20,6 +20,40 @@ def system_error_exit(return_code, message):
   sys.exit(return_code)
 
 
+def invalid_argument_exit(argument, command):
+  '''Indicate that the argument is not valid for the command.
+
+  Args:
+    argument: the invalid argument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'{argument} is not a valid argument for "{command}"')
+
+def missing_argument_exit(argument, command):
+  '''Indicate that the argument is missing for the command.
+
+  Args:
+    argument: the missingagrument
+    command: the base GAM command
+  '''
+  system_error_exit(
+      2,
+      f'missing argument {argument} for "{command}"')
+
+def expected_argument_exit(name, expected, argument):
+  '''Indicate that the argument does not have an expected value for the command.
+
+  Args:
+    name: the field name
+    expected: the expected values
+    argument: the invalid argument
+  '''
+  system_error_exit(
+      2,
+      f'{name} must be one of {expected}; got {argument}')
+
 def csv_field_error_exit(field_name, field_names):
   """Raises a system exit when a CSV field is malformed.
 
@@ -58,9 +92,8 @@ def wait_on_failure(current_attempt_num,
   wait_on_fail = min(2**current_attempt_num,
                      60) + float(random.randint(1, 1000)) / 1000
   if current_attempt_num > error_print_threshold:
-    sys.stderr.write(
-        'Temporary error: {0}, Backing off: {1} seconds, Retry: {2}/{3}\n'
-        .format(error_message, int(wait_on_fail), current_attempt_num,
-                total_num_retries))
+    sys.stderr.write((f'Temporary error: {error_message}, Backing off: '
+        f'{int(wait_on_fail)} seconds, Retry: '
+        f'{current_attempt_num}/{total_num_retries}\n'))
     sys.stderr.flush()
   time.sleep(wait_on_fail)

src/controlflow_test.py

@@ -86,23 +86,21 @@ class ControlFlowTest(unittest.TestCase):
 
   # Prevent the system from actually sleeping and thus slowing down the test.
   @patch.object(controlflow.time, 'sleep')
-  @patch.object(controlflow.sys.stderr, 'write')
-  def test_wait_on_failure_prints_errors(self, mock_stderr_write,
-                                         unused_mock_sleep):
+  def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
     message = 'An error message to display'
-    controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
     self.assertIn(message, mock_stderr_write.call_args[0][0])
 
   @patch.object(controlflow.time, 'sleep')
-  @patch.object(controlflow.sys.stderr, 'write')
-  def test_wait_on_failure_only_prints_after_threshold(self, mock_stderr_write,
-                                                       unused_mock_sleep):
+  def test_wait_on_failure_only_prints_after_threshold(self, unused_mock_sleep):
     total_attempts = 5
     threshold = 3
-    for attempt in range(1, total_attempts + 1):
-      controlflow.wait_on_failure(
-          attempt,
-          total_attempts,
-          'Attempt #%s' % attempt,
-          error_print_threshold=threshold)
+    with patch.object(controlflow.sys.stderr, 'write') as mock_stderr_write:
+      for attempt in range(1, total_attempts + 1):
+        controlflow.wait_on_failure(
+            attempt,
+            total_attempts,
+            'Attempt #%s' % attempt,
+            error_print_threshold=threshold)
     self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)

src/cros-aue-dates.json

@@ -1,256 +0,0 @@
-{
-  "acer ac700": "2016-08-01T00:00:00.000Z",
-  "acer c7 chromebook": "2017-10-01T00:00:00.000Z",
-  "acer c7 chromebook (c710)": "2017-10-01T00:00:00.000Z",
-  "acer c720 chromebook": "2019-06-01T00:00:00.000Z",
-  "acer c740 chromebook": "2019-06-01T00:00:00.000Z",
-  "acer chromebase": "2020-08-01T00:00:00.000Z",
-  "acer chromebase 24": "2021-06-01T00:00:00.000Z",
-  "acer chromebook 11 (c720, c720p)": "2019-06-01T00:00:00.000Z",
-  "acer chromebook 11 (c732, c732t, c732l, c732lt)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 11 (c740)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 11 (c771, c771t)": "2022-11-01T00:00:00.000Z",
-  "acer chromebook 11 (cb3-111, c730, c730e)": "2019-08-01T00:00:00.000Z",
-  "acer chromebook 11 (cb3-131, c735)": "2021-01-01T00:00:00.000Z",
-  "acer chromebook 11 (cb311-8h, cb311-8ht)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 11 n7 (c731, c731t)": "2022-01-01T00:00:00.000Z",
-  "acer chromebook 13 (cb5-311)": "2019-09-01T00:00:00.000Z",
-  "acer chromebook 13 (cb713-1w)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook 13(cb5-311, c810)": "2019-09-01T00:00:00.000Z",
-  "acer chromebook 14 (cb3-431)": "2021-06-01T00:00:00.000Z",
-  "acer chromebook 14 for work (cp5-471)": "2022-11-01T00:00:00.000Z",
-  "acer chromebook 15 (c910 / cb5-571)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 15 (cb3-531)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 15 (cb3-532)": "2021-08-01T00:00:00.000Z",
-  "acer chromebook 15 (cb315-1h,cb315-1ht)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 15 (cb5-571, c910)": "2020-06-01T00:00:00.000Z",
-  "acer chromebook 15 (cb515-1h,cb515-1ht)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 311": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 311 (c721, c733, c733u, c733t)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 315": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 315 (cb315-2h)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 512 (c851, c851t)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook 514": "2023-11-01T00:00:00.000Z",
-  "acer chromebook 714 (cb714-1w / cb714-1wt)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook 715 (cb715-1w / cb715-1wt)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook r11 (cb5-132t, c738t)": "2021-06-01T00:00:00.000Z",
-  "acer chromebook r13 (cb5-312t)": "2021-09-01T00:00:00.000Z",
-  "acer chromebook spin 11 (cp311-h1, cp311-1hn)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook spin 11 (r751t)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook spin 13 (cp713-1wn)": "2024-06-01T00:00:00.000Z",
-  "acer chromebook spin 15 (cp315)": "2023-11-01T00:00:00.000Z",
-  "acer chromebook spin 311 (r721t)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook spin 511": "2025-06-01T00:00:00.000Z",
-  "acer chromebook spin 511 (r752t, r752tn)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook spin 512 (r851tn)": "2025-06-01T00:00:00.000Z",
-  "acer chromebook tab 10": "2023-08-01T00:00:00.000Z",
-  "acer chromebox": "2019-09-01T00:00:00.000Z",
-  "acer chromebox cxi2": "2020-06-01T00:00:00.000Z",
-  "acer chromebox cxi2 / cxv2": "2020-06-01T00:00:00.000Z",
-  "acer chromebox cxi3": "2024-06-01T00:00:00.000Z",
-  "aopen chromebase commercial": "2020-09-01T00:00:00.000Z",
-  "aopen chromebase mini": "2022-02-01T00:00:00.000Z",
-  "aopen chromebox commercial": "2020-09-01T00:00:00.000Z",
-  "aopen chromebox commercial 2": "2024-06-01T00:00:00.000Z",
-  "aopen chromebox mini": "2022-02-01T00:00:00.000Z",
-  "asi chromebook": "2020-06-01T00:00:00.000Z",
-  "asus chromebit cs10": "2020-11-01T00:00:00.000Z",
-  "asus chromebook c200": "2019-06-01T00:00:00.000Z",
-  "asus chromebook c200ma": "2019-06-01T00:00:00.000Z",
-  "asus chromebook c201pa": "2020-06-01T00:00:00.000Z",
-  "asus chromebook c202sa": "2021-06-01T00:00:00.000Z",
-  "asus chromebook c204": "2025-06-01T00:00:00.000Z",
-  "asus chromebook c213na": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c223": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c300": "2019-08-01T00:00:00.000Z",
-  "asus chromebook c300ma": "2019-08-01T00:00:00.000Z",
-  "asus chromebook c300sa / c301sa": "2021-06-01T00:00:00.000Z",
-  "asus chromebook c403": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c423": "2023-11-01T00:00:00.000Z",
-  "asus chromebook c523": "2023-11-01T00:00:00.000Z",
-  "asus chromebook flip c100pa": "2020-07-01T00:00:00.000Z",
-  "asus chromebook flip c101pa": "2023-08-01T00:00:00.000Z",
-  "asus chromebook flip c213": "2023-11-01T00:00:00.000Z",
-  "asus chromebook flip c214": "2025-06-01T00:00:00.000Z",
-  "asus chromebook flip c302": "2022-11-01T00:00:00.000Z",
-  "asus chromebook flip c434": "2024-06-01T00:00:00.000Z",
-  "asus chromebook tablet ct100": "2023-08-01T00:00:00.000Z",
-  "asus chromebox (cn60)": "2019-09-01T00:00:00.000Z",
-  "asus chromebox 2 (cn62)": "2021-06-01T00:00:00.000Z",
-  "asus chromebox 3": "2024-06-01T00:00:00.000Z",
-  "asus chromebox 3 (cn65)": "2024-06-01T00:00:00.000Z",
-  "asus chromebox cn60": "2019-09-01T00:00:00.000Z",
-  "asus chromebox cn62": "2021-06-01T00:00:00.000Z",
-  "bobicus chromebook 11": "2020-06-01T00:00:00.000Z",
-  "chromebook 11 (c730 / cb3-111)": "2019-08-01T00:00:00.000Z",
-  "chromebook 11 (c735)": "2021-01-01T00:00:00.000Z",
-  "chromebook 15 (cb515 - 1ht / 1h)": "2023-11-01T00:00:00.000Z",
-  "chromebook 311 (c721)": "2025-06-01T00:00:00.000Z",
-  "chromebook pcm-116e": "2020-06-01T00:00:00.000Z",
-  "consumer chromebook": "2020-06-01T00:00:00.000Z",
-  "cr-48": "2015-12-01T00:00:00.000Z",
-  "crambo chromebook": "2020-06-01T00:00:00.000Z",
-  "ctl chromebook j41 / j41t": "2023-11-01T00:00:00.000Z",
-  "ctl chromebook nl7": "2023-11-01T00:00:00.000Z",
-  "ctl chromebook nl7t-360 / nl7tw-360": "2023-11-01T00:00:00.000Z",
-  "ctl chromebook tab tx1": "2023-08-01T00:00:00.000Z",
-  "ctl chromebook tablet tx1 for education": "2023-08-01T00:00:00.000Z",
-  "ctl chromebox cbx1": "2024-06-01T00:00:00.000Z",
-  "ctl j2 / j4 chromebook": "2020-06-01T00:00:00.000Z",
-  "ctl j5 chromebook": "2021-08-01T00:00:00.000Z",
-  "ctl n6 education chromebook": "2020-06-01T00:00:00.000Z",
-  "ctl nl61 chromebook": "2021-08-01T00:00:00.000Z",
-  "dell chromebook 11": "2019-06-01T00:00:00.000Z",
-  "dell chromebook 11 (3120)": "2020-06-01T00:00:00.000Z",
-  "dell chromebook 11 (3180)": "2022-05-01T00:00:00.000Z",
-  "dell chromebook 11 (5190)": "2023-11-01T00:00:00.000Z",
-  "dell chromebook 11 2-in-1 (3189)": "2022-05-01T00:00:00.000Z",
-  "dell chromebook 11 2-in-1 (5190)": "2023-11-01T00:00:00.000Z",
-  "dell chromebook 13 (3380)": "2022-11-01T00:00:00.000Z",
-  "dell chromebook 13 (7310)": "2020-09-01T00:00:00.000Z",
-  "dell chromebook 3100": "2025-06-01T00:00:00.000Z",
-  "dell chromebook 3100 2-in-1": "2025-06-01T00:00:00.000Z",
-  "dell chromebook 3400": "2025-06-01T00:00:00.000Z",
-  "dell chromebox": "2019-09-01T00:00:00.000Z",
-  "dell inspiron chromebook 14 2-in-1 (7486)": "2024-06-01T00:00:00.000Z",
-  "edugear chromebook k": "2020-06-01T00:00:00.000Z",
-  "edugear chromebook m": "2020-06-01T00:00:00.000Z",
-  "edugear chromebook r": "2020-06-01T00:00:00.000Z",
-  "edugear cmt chromebook": "2021-08-01T00:00:00.000Z",
-  "edxis chromebook": "2020-06-01T00:00:00.000Z",
-  "edxis education chromebook": "2020-06-01T00:00:00.000Z",
-  "epik 11.6\" chromebook  elb1101": "2020-06-01T00:00:00.000Z",
-  "google chromebook pixel": "2018-06-01T00:00:00.000Z",
-  "google chromebook pixel (2015)": "2020-06-01T00:00:00.000Z",
-  "google cr-48": "2015-12-01T00:00:00.000Z",
-  "google pixel slate": "2024-06-01T00:00:00.000Z",
-  "google pixelbook": "2024-06-01T00:00:00.000Z",
-  "haier chromebook 11": "2020-06-01T00:00:00.000Z",
-  "haier chromebook 11 c": "2021-08-01T00:00:00.000Z",
-  "haier chromebook 11 g2": "2020-09-01T00:00:00.000Z",
-  "haier chromebook 11e": "2020-06-01T00:00:00.000Z",
-  "hexa chromebook pi": "2020-06-01T00:00:00.000Z",
-  "hisense chromebook 11": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 1100-1199 / hp chromebook 11 g1": "2018-10-01T00:00:00.000Z",
-  "hp chromebook 11 2000-2099 / hp chromebook 11 g2": "2019-06-01T00:00:00.000Z",
-  "hp chromebook 11 2100-2199 / hp chromebook 11 g3": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 2200-2299 / hp chromebook 11 g4/g4 ee": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 g1": "2018-10-01T00:00:00.000Z",
-  "hp chromebook 11 g2": "2019-06-01T00:00:00.000Z",
-  "hp chromebook 11 g3": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 g4/g4 ee": "2020-06-01T00:00:00.000Z",
-  "hp chromebook 11 g5": "2021-07-01T00:00:00.000Z",
-  "hp chromebook 11 g5 / hp chromebook 11-vxxx": "2021-07-01T00:00:00.000Z",
-  "hp chromebook 11 g5 ee": "2022-01-01T00:00:00.000Z",
-  "hp chromebook 11 g6 ee": "2023-11-01T00:00:00.000Z",
-  "hp chromebook 11 g7 ee": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 11a g6 ee": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 13 g1": "2022-11-01T00:00:00.000Z",
-  "hp chromebook 14": "2019-06-01T00:00:00.000Z",
-  "hp chromebook 14 / hp chromebook 14 g5": "2023-11-01T00:00:00.000Z",
-  "hp chromebook 14 ak000-099 / hp chromebook 14 g4": "2021-09-01T00:00:00.000Z",
-  "hp chromebook 14 db0000-db0999": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 14 g3": "2019-10-01T00:00:00.000Z",
-  "hp chromebook 14 g4": "2021-09-01T00:00:00.000Z",
-  "hp chromebook 14 g5": "2023-11-01T00:00:00.000Z",
-  "hp chromebook 14 x000-x999 / hp chromebook 14 g3": "2019-10-01T00:00:00.000Z",
-  "hp chromebook 14a g5": "2025-06-01T00:00:00.000Z",
-  "hp chromebook 15 g1": "2024-06-01T00:00:00.000Z",
-  "hp chromebook x2 ": "2024-06-01T00:00:00.000Z",
-  "hp chromebook x360 11 g1 ee": "2023-11-01T00:00:00.000Z",
-  "hp chromebook x360 11 g2 ee": "2025-06-01T00:00:00.000Z",
-  "hp chromebook x360 14": "2024-06-01T00:00:00.000Z",
-  "hp chromebook x360 14 g1": "2024-06-01T00:00:00.000Z",
-  "hp chromebox cb1-(000-099) / hp chromebox g1/ hp chromebox for meetings": "2019-09-01T00:00:00.000Z",
-  "hp chromebox g1": "2019-09-01T00:00:00.000Z",
-  "hp chromebox g2": "2024-06-01T00:00:00.000Z",
-  "hp pavilion chromebook 14": "2018-02-01T00:00:00.000Z",
-  "jp sa couto chromebook": "2020-06-01T00:00:00.000Z",
-  "lava xolo chromebook": "2020-06-01T00:00:00.000Z",
-  "lenovo 100e chromebook": "2023-11-01T00:00:00.000Z",
-  "lenovo 100e chromebook 2nd gen": "2025-06-01T00:00:00.000Z",
-  "lenovo 100e chromebook 2nd gen mtk": "2025-06-01T00:00:00.000Z",
-  "lenovo 100s chromebook": "2020-09-01T00:00:00.000Z",
-  "lenovo 14e chromebook": "2025-06-01T00:00:00.000Z",
-  "lenovo 300e chromebook": "2025-06-01T00:00:00.000Z",
-  "lenovo 300e chromebook 2nd gen": "2025-06-01T00:00:00.000Z",
-  "lenovo 300e chromebook 2nd gen mtk": "2025-06-01T00:00:00.000Z",
-  "lenovo 500e chromebook": "2023-11-01T00:00:00.000Z",
-  "lenovo 500e chromebook 2nd gen": "2025-06-01T00:00:00.000Z",
-  "lenovo chromebook c330": "2022-06-01T00:00:00.000Z",
-  "lenovo chromebook s330": "2022-06-01T00:00:00.000Z",
-  "lenovo flex 11 chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo ideapad c330 chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo ideapad s330 chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo n20 chromebook": "2019-06-01T00:00:00.000Z",
-  "lenovo n21 chromebook": "2020-06-01T00:00:00.000Z",
-  "lenovo n22 chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo n23 chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo n23 yoga chromebook": "2022-06-01T00:00:00.000Z",
-  "lenovo n42 chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo thinkcentre chromebox": "2020-06-01T00:00:00.000Z",
-  "lenovo thinkpad 11e 3rd gen chromebook": "2021-06-01T00:00:00.000Z",
-  "lenovo thinkpad 11e 4th gen chromebook": "2023-11-01T00:00:00.000Z",
-  "lenovo thinkpad 11e chromebook": "2019-06-01T00:00:00.000Z",
-  "lenovo thinkpad 11e chromebook (4th gen)/lenovo thinkpad yoga 11e chromebook (4th gen)": "2023-11-01T00:00:00.000Z",
-  "lenovo thinkpad 13": "2022-11-01T00:00:00.000Z",
-  "lenovo thinkpad x131e chromebook": "2018-06-01T00:00:00.000Z",
-  "lenovo yoga c630 chromebook": "2024-06-01T00:00:00.000Z",
-  "lg chromebase (22cb25s)": "2020-06-01T00:00:00.000Z",
-  "lg chromebase (22cv241)": "2019-06-01T00:00:00.000Z",
-  "lumos education chromebook": "2020-06-01T00:00:00.000Z",
-  "m&a chromebook": "2020-06-01T00:00:00.000Z",
-  "mecer chromebook": "2020-06-01T00:00:00.000Z",
-  "mecer v2 chromebook": "2021-08-01T00:00:00.000Z",
-  "medion chromebook akoya s2013 ": "2020-06-01T00:00:00.000Z",
-  "medion chromebook s2015": "2020-06-01T00:00:00.000Z",
-  "multilaser chromebook m11c": "2021-08-01T00:00:00.000Z",
-  "ncomputing chromebook cx100": "2020-06-01T00:00:00.000Z",
-  "ncomputing chromebook cx110": "2020-06-01T00:00:00.000Z",
-  "nexian chromebook 11.6\"": "2020-06-01T00:00:00.000Z",
-  "pcmerge chromebook al116": "2023-11-01T00:00:00.000Z",
-  "pcmerge chromebookpcm-116e/pcm-116eb": "2020-06-01T00:00:00.000Z",
-  "pcmerge chromebookpcm-116t-432b": "2021-08-01T00:00:00.000Z",
-  "poin2 chromebook 11": "2020-06-01T00:00:00.000Z",
-  "poin2 chromebook 11c": "2022-11-01T00:00:00.000Z",
-  "poin2 chromebook 14": "2022-03-01T00:00:00.000Z",
-  "positivo chromebook c216b": "2021-08-01T00:00:00.000Z",
-  "positivo chromebook ch1190": "2020-06-01T00:00:00.000Z",
-  "promethean chromebox": "2024-06-01T00:00:00.000Z",
-  "prowise 11.6\" entry line chromebook": "2020-06-01T00:00:00.000Z",
-  "prowise chromebook eduline": "2023-11-01T00:00:00.000Z",
-  "prowise chromebook entryline": "2020-06-01T00:00:00.000Z",
-  "prowise chromebook proline": "2021-08-01T00:00:00.000Z",
-  "prowise proline chromebook": "2021-08-01T00:00:00.000Z",
-  "rgs education chromebook": "2020-06-01T00:00:00.000Z",
-  "samsung chromebook": "2018-07-01T00:00:00.000Z",
-  "samsung chromebook - xe303": "2018-07-01T00:00:00.000Z",
-  "samsung chromebook 2 11": "2019-06-01T00:00:00.000Z",
-  "samsung chromebook 2 11 - xe500c12": "2020-06-01T00:00:00.000Z",
-  "samsung chromebook 2 13": "2019-06-01T00:00:00.000Z",
-  "samsung chromebook 3": "2021-06-01T00:00:00.000Z",
-  "samsung chromebook plus": "2023-08-01T00:00:00.000Z",
-  "samsung chromebook plus (lte)": "2024-06-01T00:00:00.000Z",
-  "samsung chromebook plus (v2)": "2024-06-01T00:00:00.000Z",
-  "samsung chromebook pro": "2022-11-01T00:00:00.000Z",
-  "samsung chromebook series 5": "2016-06-01T00:00:00.000Z",
-  "samsung chromebook series 5 550": "2017-05-01T00:00:00.000Z",
-  "samsung chromebox series 3": "2018-03-01T00:00:00.000Z",
-  "sector 5 e1 rugged chromebook": "2020-06-01T00:00:00.000Z",
-  "sector 5 e3 chromebook": "2023-11-01T00:00:00.000Z",
-  "senkatel c1101 chromebook": "2020-06-01T00:00:00.000Z",
-  "thinkpad 11e chromebook 3rd gen (yoga/clamshell)": "2021-06-01T00:00:00.000Z",
-  "thinkpad 13 chromebook": "2022-11-01T00:00:00.000Z",
-  "toshiba chromebook": "2019-06-01T00:00:00.000Z",
-  "toshiba chromebook 2": "2020-06-01T00:00:00.000Z",
-  "toshiba chromebook 2 (2015 edition)": "2020-09-01T00:00:00.000Z",
-  "true idc chromebook": "2020-06-01T00:00:00.000Z",
-  "true idc chromebook 11": "2020-06-01T00:00:00.000Z",
-  "videonet chromebook": "2020-06-01T00:00:00.000Z",
-  "videonet chromebook bl10": "2020-06-01T00:00:00.000Z",
-  "viewsonic nmp660 chromebox": "2024-06-01T00:00:00.000Z",
-  "viglen chromebook 11": "2020-06-01T00:00:00.000Z",
-  "viglen chromebook 11c": "2023-11-01T00:00:00.000Z",
-  "viglen chromebook 360": "2021-08-01T00:00:00.000Z",
-  "xolo chromebook": "2020-06-01T00:00:00.000Z"
-}

src/display.py

@@ -1,18 +1,238 @@
 """Methods related to display of information to the user."""
 
+import csv
+import datetime
+import io
 import sys
-import utils
-from var import ERROR_PREFIX
-from var import WARNING_PREFIX
+import webbrowser
 
+import dateutil
+import googleapiclient.http
+
+#TODO: get rid of these hacks
+import __main__
+from var import *
+import controlflow
+import gapi
+
+
+def current_count(i, count):
+  return f' ({i}/{count})' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else ''
+
+def current_count_nl(i, count):
+  return f' ({i}/{count})\n' if (count > GC_Values[GC_SHOW_COUNTS_MIN]) else '\n'
+
+def add_field_to_fields_list(fieldName, fieldsChoiceMap, fieldsList):
+  fields = fieldsChoiceMap[fieldName.lower()]
+  if isinstance(fields, list):
+    fieldsList.extend(fields)
+  else:
+    fieldsList.append(fields)
+
+# Write a CSV file
+def add_titles_to_csv_file(addTitles, titles):
+  for title in addTitles:
+    if title not in titles:
+      titles.append(title)
+
+def add_row_titles_to_csv_file(row, csvRows, titles):
+  csvRows.append(row)
+  for title in row:
+    if title not in titles:
+      titles.append(title)
+
+# fieldName is command line argument
+# fieldNameMap maps fieldName to API field names; CSV file header will be API field name
+#ARGUMENT_TO_PROPERTY_MAP = {
+#  u'admincreated': [u'adminCreated'],
+#  u'aliases': [u'aliases', u'nonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_to_csv_file(fieldName, fieldNameMap, fieldsList, fieldsTitles, titles):
+  for ftList in fieldNameMap[fieldName]:
+    if ftList not in fieldsTitles:
+      fieldsList.append(ftList)
+      fieldsTitles[ftList] = ftList
+      add_titles_to_csv_file([ftList], titles)
+
+# fieldName is command line argument
+# fieldNameTitleMap maps fieldName to API field name and CSV file header
+#ARGUMENT_TO_PROPERTY_TITLE_MAP = {
+#  u'admincreated': [u'adminCreated', u'Admin_Created'],
+#  u'aliases': [u'aliases', u'Aliases', u'nonEditableAliases', u'NonEditableAliases'],
+#  }
+# fieldsList is the list of API fields
+# fieldsTitles maps the API field name to the CSV file header
+def add_field_title_to_csv_file(fieldName, fieldNameTitleMap, fieldsList, fieldsTitles, titles):
+  ftList = fieldNameTitleMap[fieldName]
+  for i in range(0, len(ftList), 2):
+    if ftList[i] not in fieldsTitles:
+      fieldsList.append(ftList[i])
+      fieldsTitles[ftList[i]] = ftList[i+1]
+      add_titles_to_csv_file([ftList[i+1]], titles)
+
+def sort_csv_titles(firstTitle, titles):
+  restoreTitles = []
+  for title in firstTitle:
+    if title in titles:
+      titles.remove(title)
+      restoreTitles.append(title)
+  titles.sort()
+  for title in restoreTitles[::-1]:
+    titles.insert(0, title)
+
+def QuotedArgumentList(items):
+  return ' '.join([item if item and (item.find(' ') == -1) and (item.find(',') == -1) else '"'+item+'"' for item in items])
+
+def write_csv_file(csvRows, titles, list_type, todrive):
+  def rowDateTimeFilterMatch(dateMode, rowDate, op, filterDate):
+    if not rowDate or not isinstance(rowDate, str):
+      return False
+    try:
+      rowTime = dateutil.parser.parse(rowDate, ignoretz=True)
+      if dateMode:
+        rowDate = datetime.datetime(rowTime.year, rowTime.month, rowTime.day).isoformat()+'Z'
+    except ValueError:
+      rowDate = NEVER_TIME
+    if op == '<':
+      return rowDate < filterDate
+    if op == '<=':
+      return rowDate <= filterDate
+    if op == '>':
+      return rowDate > filterDate
+    if op == '>=':
+      return rowDate >= filterDate
+    if op == '!=':
+      return rowDate != filterDate
+    return rowDate == filterDate
+
+  def rowCountFilterMatch(rowCount, op, filterCount):
+    if isinstance(rowCount, str):
+      if not rowCount.isdigit():
+        return False
+      rowCount = int(rowCount)
+    elif not isinstance(rowCount, int):
+      return False
+    if op == '<':
+      return rowCount < filterCount
+    if op == '<=':
+      return rowCount <= filterCount
+    if op == '>':
+      return rowCount > filterCount
+    if op == '>=':
+      return rowCount >= filterCount
+    if op == '!=':
+      return rowCount != filterCount
+    return rowCount == filterCount
+  def rowBooleanFilterMatch(rowBoolean, filterBoolean):
+    if not isinstance(rowBoolean, bool):
+      return False
+    return rowBoolean == filterBoolean
+
+  def headerFilterMatch(filters, title):
+    for filterStr in filters:
+      if filterStr.match(title):
+        return True
+    return False
+
+  if GC_Values[GC_CSV_ROW_FILTER]:
+    for column, filterVal in iter(GC_Values[GC_CSV_ROW_FILTER].items()):
+      if column not in titles:
+        sys.stderr.write(f'WARNING: Row filter column "{column}" is not in output columns\n')
+        continue
+      if filterVal[0] == 'regex':
+        csvRows = [row for row in csvRows if filterVal[1].search(str(row.get(column, '')))]
+      elif filterVal[0] == 'notregex':
+        csvRows = [row for row in csvRows if not filterVal[1].search(str(row.get(column, '')))]
+      elif filterVal[0] in ['date', 'time']:
+        csvRows = [row for row in csvRows if rowDateTimeFilterMatch(filterVal[0] == 'date', row.get(column, ''), filterVal[1], filterVal[2])]
+      elif filterVal[0] == 'count':
+        csvRows = [row for row in csvRows if rowCountFilterMatch(row.get(column, 0), filterVal[1], filterVal[2])]
+      else: #boolean
+        csvRows = [row for row in csvRows if rowBooleanFilterMatch(row.get(column, False), filterVal[1])]
+  if GC_Values[GC_CSV_HEADER_FILTER] or GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+    if GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+      titles = [t for t in titles if not headerFilterMatch(GC_Values[GC_CSV_HEADER_DROP_FILTER], t)]
+    if GC_Values[GC_CSV_HEADER_FILTER]:
+      titles = [t for t in titles if headerFilterMatch(GC_Values[GC_CSV_HEADER_FILTER], t)]
+    if not titles:
+      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER and GAM_CSV_HEADER_DROP_FILTER\n')
+      return
+  csv.register_dialect('nixstdout', lineterminator='\n')
+  if todrive:
+    write_to = io.StringIO()
+  else:
+    write_to = sys.stdout
+  writer = csv.DictWriter(write_to, fieldnames=titles, dialect='nixstdout', extrasaction='ignore', quoting=csv.QUOTE_MINIMAL)
+  try:
+    writer.writerow(dict((item, item) for item in writer.fieldnames))
+    writer.writerows(csvRows)
+  except IOError as e:
+    controlflow.system_error_exit(6, e)
+  if todrive:
+    admin_email = __main__._getValueFromOAuth('email')
+    _, drive = __main__.buildDrive3GAPIObject(admin_email)
+    if not drive:
+      print(f'''\nGAM is not authorized to create Drive files. Please run:
+gam user {admin_email} check serviceaccount
+and follow recommend steps to authorize GAM for Drive access.''')
+      sys.exit(5)
+    result = gapi.call(drive.about(), 'get', fields='maxImportSizes')
+    columns = len(titles)
+    rows = len(csvRows)
+    cell_count = rows * columns
+    data_size = len(write_to.getvalue())
+    max_sheet_bytes = int(result['maxImportSizes'][MIMETYPE_GA_SPREADSHEET])
+    if cell_count > MAX_GOOGLE_SHEET_CELLS or data_size > max_sheet_bytes:
+      print(f'{WARNING_PREFIX}{MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET}')
+      mimeType = 'text/csv'
+    else:
+      mimeType = MIMETYPE_GA_SPREADSHEET
+    body = {'description': QuotedArgumentList(sys.argv),
+            'name': f'{GC_Values[GC_DOMAIN]} - {list_type}',
+            'mimeType': mimeType}
+    result = gapi.call(drive.files(), 'create', fields='webViewLink',
+                       body=body,
+                       media_body=googleapiclient.http.MediaInMemoryUpload(write_to.getvalue().encode(),
+                                                                           mimetype='text/csv'))
+    file_url = result['webViewLink']
+    if GC_Values[GC_NO_BROWSER]:
+      msg_txt = f'Drive file uploaded to:\n {file_url}'
+      msg_subj = f'{GC_Values[GC_DOMAIN]} - {list_type}'
+      __main__.send_email(msg_subj, msg_txt)
+      print(msg_txt)
+    else:
+      webbrowser.open(file_url)
 
 def print_error(message):
   """Prints a one-line error message to stderr in a standard format."""
-  sys.stderr.write(
-      utils.convertUTF8('\n{0}{1}\n'.format(ERROR_PREFIX, message)))
+  sys.stderr.write('\n{0}{1}\n'.format(ERROR_PREFIX, message))
 
 
 def print_warning(message):
   """Prints a one-line warning message to stderr in a standard format."""
-  sys.stderr.write(
-      utils.convertUTF8('\n{0}{1}\n'.format(WARNING_PREFIX, message)))
+  sys.stderr.write('\n{0}{1}\n'.format(WARNING_PREFIX, message))
+
+def print_json(object_value, spacing=''):
+  """Prints Dict or Array to screen in clean human-readable format.."""
+  if isinstance(object_value, list):
+    if len(object_value) == 1 and isinstance(object_value[0], (str, int, bool)):
+      sys.stdout.write(f'{object_value[0]}\n')
+      return
+    if spacing:
+      sys.stdout.write('\n')
+    for i, a_value in enumerate(object_value):
+      if isinstance(a_value, (str, int, bool)):
+        sys.stdout.write(f' {spacing}{i+1}) {a_value}\n')
+      else:
+        sys.stdout.write(f' {spacing}{i+1}) ')
+        print_json(a_value, f' {spacing}')
+  elif isinstance(object_value, dict):
+    for key in ['kind', 'etag', 'etags']:
+      object_value.pop(key, None)
+    for another_object, another_value in object_value.items():
+      sys.stdout.write(f' {spacing}{another_object}: ')
+      print_json(another_value, f' {spacing}')
+  else:
+    sys.stdout.write(f'{object_value}\n')

src/display_test.py

@@ -10,50 +10,50 @@ from var import WARNING_PREFIX
 
 class DisplayTest(unittest.TestCase):
 
-  @patch.object(display.sys.stderr, 'write')
-  def test_print_error_prints_to_stderr(self, mock_write):
+  def test_print_error_prints_to_stderr(self):
     message = 'test error'
-    display.print_error(message)
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
     printed_message = mock_write.call_args[0][0]
     self.assertIn(message, printed_message)
 
-  @patch.object(display.sys.stderr, 'write')
-  def test_print_error_prints_error_prefix(self, mock_write):
+  def test_print_error_prints_error_prefix(self):
     message = 'test error'
-    display.print_error(message)
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
     printed_message = mock_write.call_args[0][0]
     self.assertLess(
         printed_message.find(ERROR_PREFIX), printed_message.find(message),
         'The error prefix does not appear before the error message')
 
-  @patch.object(display.sys.stderr, 'write')
-  def test_print_error_ends_message_with_newline(self, mock_write):
+  def test_print_error_ends_message_with_newline(self):
     message = 'test error'
-    display.print_error(message)
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
     printed_message = mock_write.call_args[0][0]
     self.assertRegex(printed_message, '\n$',
                      'The error message does not end in a newline.')
 
-  @patch.object(display.sys.stderr, 'write')
-  def test_print_warning_prints_to_stderr(self, mock_write):
+  def test_print_warning_prints_to_stderr(self):
     message = 'test warning'
-    display.print_warning(message)
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
     printed_message = mock_write.call_args[0][0]
     self.assertIn(message, printed_message)
 
-  @patch.object(display.sys.stderr, 'write')
-  def test_print_warning_prints_error_prefix(self, mock_write):
+  def test_print_warning_prints_error_prefix(self):
     message = 'test warning'
-    display.print_error(message)
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
     printed_message = mock_write.call_args[0][0]
     self.assertLess(
         printed_message.find(WARNING_PREFIX), printed_message.find(message),
         'The warning prefix does not appear before the error message')
 
-  @patch.object(display.sys.stderr, 'write')
-  def test_print_warning_ends_message_with_newline(self, mock_write):
+  def test_print_warning_ends_message_with_newline(self):
     message = 'test warning'
-    display.print_error(message)
+    with patch.object(display.sys.stderr, 'write') as mock_write:
+      display.print_error(message)
     printed_message = mock_write.call_args[0][0]
     self.assertRegex(printed_message, '\n$',
                      'The warning message does not end in a newline.')

src/fileutils.py

@@ -80,16 +80,21 @@ def open_file(filename,
     controlflow.system_error_exit(6, e)
 
 
-def close_file(f):
+def close_file(f, force_flush=False):
   """Closes a file.
 
   Args:
     f: The file to close
+    force_flush: Flush file to disk emptying Python and OS caches. See:
+       https://stackoverflow.com/a/13762137/1503886
 
   Returns:
     Boolean, True if the file was successfully closed. False if an error
         was encountered while closing.
   """
+  if force_flush:
+    f.flush()
+    os.fsync(f.fileno())
   try:
     f.close()
     return True

src/gam-install.sh

@@ -28,8 +28,8 @@ upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.27 2.23 2.19 2.15"
-gam_macos_vers="10.14.4 10.13.6 10.12.6"
+gam_glibc_vers="2.27 2.23"
+gam_macos_vers="10.14.6 10.13.6"
 
 while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
@@ -52,7 +52,7 @@ done
 target_dir=${target_dir%/}
 
 update_profile() {
-	[ -f "$1" ] || return 1
+	[ $2 -eq 1 ] || [ -f "$1" ] || return 1
 
 	grep -F "$alias_line" "$1" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then
@@ -234,6 +234,20 @@ else
   echo_green "Finished extracting GAM archive."
 fi
 
+# Update profile to add gam command
+if [ "$update_profile" = true ]; then
+  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
+  if [ "$gamos" == "linux" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0
+    update_profile "$HOME/.zshrc" 0
+  elif [ "$gamos" == "macos" ]; then
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.zshrc" 0
+  fi
+else
+  echo_yellow "skipping profile update."
+fi
+
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
   "$target_dir/gam/gam" version extended
@@ -247,18 +261,6 @@ if [ "$upgrade_only" = true ]; then
   exit
 fi
 
-# Update profile to add gam command
-if [ "$update_profile" = true ]; then
-  alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
-  if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile"
-  elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile"
-  fi
-else
-  echo_yellow "skipping profile update."
-fi
-
 while true; do
   read -p "Can you run a full browser on this machine? (usually Y for MacOS, N for Linux if you SSH into this machine) " yn
   case $yn in

src/gam.spec

@@ -2,23 +2,31 @@
 
 import sys
 
+import importlib
+from PyInstaller.utils.hooks import copy_metadata
+
 sys.modules['FixTk'] = None
 
+extra_files = [('cloudprint-v2.json', 'cloudprint-v2.json')]
+
+# dynamically determine where httplib2/cacerts.txt lives
+proot = os.path.dirname(importlib.import_module('httplib2').__file__)
+extra_files += [(os.path.join(proot, 'cacerts.txt'), 'httplib2')]
+
+extra_files += copy_metadata('google-api-python-client')
+
 a = Analysis(['gam.py'],
              hiddenimports=[],
              hookspath=None,
              excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+             datas=extra_files,
              runtime_hooks=None)
+
 for d in a.datas:
     if 'pyconfig' in d[0]:
         a.datas.remove(d)
         break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
 
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
 
 pyz = PYZ(a.pure)
 exe = EXE(pyz,

src/gapi/__init__.py

@@ -9,42 +9,13 @@ import httplib2
 import controlflow
 import display
 from gapi import errors
-from var import (GC_CA_FILE, GC_Values, GC_TLS_MIN_VERSION, GC_TLS_MAX_VERSION,
-                 GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
+import transport
+from var import (GM_Globals, GM_CURRENT_API_SCOPES, GM_CURRENT_API_USER,
                  GM_EXTRA_ARGS_DICT, GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID,
                  MAX_RESULTS_API_EXCEPTIONS, MESSAGE_API_ACCESS_CONFIG,
                  MESSAGE_API_ACCESS_DENIED, MESSAGE_SERVICE_NOT_APPLICABLE)
 
 
-def create_http(cache=None,
-                timeout=None,
-                override_min_tls=None,
-                override_max_tls=None):
-  """Creates a uniform HTTP transport object.
-
-  Args:
-    cache: The HTTP cache to use.
-    timeout: The cache timeout, in seconds.
-    override_min_tls: The minimum TLS version to require. If not provided, the
-      default is used.
-    override_max_tls: The maximum TLS version to require. If not provided, the
-      default is used.
-
-  Returns:
-    httplib2.Http with the specified options.
-  """
-  tls_minimum_version = override_min_tls if override_min_tls else GC_Values[
-    GC_TLS_MIN_VERSION]
-  tls_maximum_version = override_max_tls if override_max_tls else GC_Values[
-    GC_TLS_MAX_VERSION]
-  return httplib2.Http(
-    ca_certs=GC_Values[GC_CA_FILE],
-    tls_maximum_version=tls_maximum_version,
-    tls_minimum_version=tls_minimum_version,
-    cache=cache,
-    timeout=timeout)
-
-
 def call(service,
          function,
          silent_errors=False,
@@ -79,20 +50,20 @@ def call(service,
   method = getattr(service, function)
   retries = 10
   parameters = dict(
-    list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
+      list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
   for n in range(1, retries + 1):
     try:
       return method(**parameters).execute()
     except googleapiclient.errors.HttpError as e:
       http_status, reason, message = errors.get_gapi_error_detail(
-        e,
-        soft_errors=soft_errors,
-        silent_errors=silent_errors,
-        retry_on_http_error=n < 3)
+          e,
+          soft_errors=soft_errors,
+          silent_errors=silent_errors,
+          retry_on_http_error=n < 3)
       if http_status == -1:
         # The error detail indicated that we should retry this request
         # We'll refresh credentials and make another pass
-        service._http.request.credentials.refresh(create_http())
+        service._http.credentials.refresh(transport.create_http())
         continue
       if http_status == 0:
         return None
@@ -101,32 +72,27 @@ def call(service,
       if is_known_error_reason and errors.ErrorReason(reason) in throw_reasons:
         if errors.ErrorReason(reason) in errors.ERROR_REASON_TO_EXCEPTION:
           raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(reason)](
-            message)
+              message)
         raise e
       if (n != retries) and (is_known_error_reason and errors.ErrorReason(
           reason) in errors.DEFAULT_RETRY_REASONS + retry_reasons):
         controlflow.wait_on_failure(n, retries, reason)
         continue
       if soft_errors:
-        display.print_error('{0}: {1} - {2}{3}'.format(http_status, message,
-                                                       reason,
-                                                       ['',
-                                                        ': Giving up.'][n > 1]))
+        display.print_error(f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}')
         return None
       controlflow.system_error_exit(
-        int(http_status), '{0}: {1} - {2}'.format(http_status, message,
-                                                  reason))
+          int(http_status), f'{http_status}: {message} - {reason}')
     except google.auth.exceptions.RefreshError as e:
       handle_oauth_token_error(
-        e, soft_errors or
-        errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
+          e, soft_errors or
+          errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
       if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
         raise errors.GapiServiceNotAvailableError(str(e))
-      display.print_error('User {0}: {1}'.format(
-        GM_Globals[GM_CURRENT_API_USER], str(e)))
+      display.print_error(f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
       return None
     except ValueError as e:
-      if service._http.cache is not None:
+      if hasattr(service._http, 'cache') and service._http.cache is not None:
         service._http.cache = None
         continue
       controlflow.system_error_exit(4, str(e))
@@ -165,11 +131,11 @@ def get_items(service,
     The list of items in the first page of a response.
   """
   results = call(
-    service,
-    function,
-    throw_reasons=throw_reasons,
-    retry_reasons=retry_reasons,
-    **kwargs)
+      service,
+      function,
+      throw_reasons=throw_reasons,
+      retry_reasons=retry_reasons,
+      **kwargs)
   if results:
     return results.get(items, [])
   return []
@@ -200,12 +166,47 @@ def _get_max_page_size_for_api_call(service, function, **kwargs):
           return None
         known_api_max = MAX_RESULTS_API_EXCEPTIONS.get(api_id)
         max_results = a_method['parameters']['maxResults'].get(
-          'maximum', known_api_max)
+            'maximum', known_api_max)
         return {'maxResults': max_results}
 
   return None
 
 
+TOTAL_ITEMS_MARKER = '%%total_items%%'
+FIRST_ITEM_MARKER = '%%first_item%%'
+LAST_ITEM_MARKER = '%%last_item%%'
+
+def got_total_items_msg(items, eol):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+    eol: String, the line terminator
+       Values used: '', '...', '\n', '...\n'
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}{eol}'
+
+def got_total_items_first_last_msg(items):
+  """Format a page_message to be used by get_all_pages
+
+  The page message indicates the number of items returned and the
+  value of the first and list items
+
+  Args:
+    items: String, the description of the items being returned by get_all_pages
+
+  Returns:
+    The formatted page_message
+  """
+
+  return f'Got {TOTAL_ITEMS_MARKER} {items}: {FIRST_ITEM_MARKER} - {LAST_ITEM_MARKER}'+'\n'
+
 def get_all_pages(service,
                   function,
                   items='items',
@@ -228,11 +229,11 @@ def get_all_pages(service,
     page_message: String, a message to be displayed to the user during paging.
       Template strings allow for dynamic content to be inserted during paging.
         Supported template strings:
-          %%total_items%% : The current number of items discovered across all
+          TOTAL_ITEMS_MARKER : The current number of items discovered across all
             pages.
-          %%first_item%%  : In conjunction with `message_attribute` arg, will
+          FIRST_ITEM_MARKER  : In conjunction with `message_attribute` arg, will
             display a unique property of the first item in the current page.
-          %%last_item%%   : In conjunction with `message_attribute` arg, will
+          LAST_ITEM_MARKER   : In conjunction with `message_attribute` arg, will
             display a unique property of the last item in the current page.
     message_attribute: String, the name of a signature field within a single
       returned item which identifies that unique item. This field is used with
@@ -258,13 +259,13 @@ def get_all_pages(service,
   total_items = 0
   while True:
     page = call(
-      service,
-      function,
-      soft_errors=soft_errors,
-      throw_reasons=throw_reasons,
-      retry_reasons=retry_reasons,
-      pageToken=page_token,
-      **kwargs)
+        service,
+        function,
+        soft_errors=soft_errors,
+        throw_reasons=throw_reasons,
+        retry_reasons=retry_reasons,
+        pageToken=page_token,
+        **kwargs)
     if page:
       page_token = page.get('nextPageToken')
       page_items = page.get(items, [])
@@ -277,14 +278,12 @@ def get_all_pages(service,
 
     # Show a paging message to the user that indicates paging progress
     if page_message:
-      show_message = page_message.replace('%%total_items%%', str(total_items))
+      show_message = page_message.replace(TOTAL_ITEMS_MARKER, str(total_items))
       if message_attribute:
         first_item = page_items[0] if num_page_items > 0 else {}
         last_item = page_items[-1] if num_page_items > 1 else first_item
-        show_message = show_message.replace(
-          '%%first_item%%', str(first_item.get(message_attribute, '')))
-        show_message = show_message.replace(
-          '%%last_item%%', str(last_item.get(message_attribute, '')))
+        show_message = show_message.replace(FIRST_ITEM_MARKER, str(first_item.get(message_attribute, '')))
+        show_message = show_message.replace(LAST_ITEM_MARKER, str(last_item.get(message_attribute, '')))
       sys.stderr.write('\r')
       sys.stderr.flush()
       sys.stderr.write(show_message)
@@ -314,14 +313,16 @@ def handle_oauth_token_error(e, soft_errors):
       return
     if not GM_Globals[GM_CURRENT_API_USER]:
       display.print_error(
-        MESSAGE_API_ACCESS_DENIED.format(
-          GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
-          ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
+          MESSAGE_API_ACCESS_DENIED.format(
+              GM_Globals[GM_OAUTH2SERVICE_ACCOUNT_CLIENT_ID],
+              ','.join(GM_Globals[GM_CURRENT_API_SCOPES])))
       controlflow.system_error_exit(12, MESSAGE_API_ACCESS_CONFIG)
     else:
       controlflow.system_error_exit(
-        19,
-        MESSAGE_SERVICE_NOT_APPLICABLE.format(
-          GM_Globals[GM_CURRENT_API_USER]))
-  controlflow.system_error_exit(18,
-                                'Authentication Token Error - {0}'.format(e))
+          19,
+          MESSAGE_SERVICE_NOT_APPLICABLE.format(
+              GM_Globals[GM_CURRENT_API_USER]))
+  controlflow.system_error_exit(18, f'Authentication Token Error - {str(e)}')
+
+def get_enum_values_minus_unspecified(values):
+  return [a_type for a_type in values if '_UNSPECIFIED' not in a_type]

src/gapi/__init___test.py

@@ -38,40 +38,6 @@ def create_http_error(status, reason, message):
   return gapi.googleapiclient.errors.HttpError(response, content_bytes)
 
 
-class CreateHttpTest(unittest.TestCase):
-
-  def setUp(self):
-    SetGlobalVariables()
-    super(CreateHttpTest, self).setUp()
-
-  def test_create_http_sets_default_values_on_http(self):
-    http = gapi.create_http()
-    self.assertIsNone(http.cache)
-    self.assertIsNone(http.timeout)
-    self.assertEqual(http.tls_minimum_version,
-                     gapi.GC_Values[gapi.GC_TLS_MIN_VERSION])
-    self.assertEqual(http.tls_maximum_version,
-                     gapi.GC_Values[gapi.GC_TLS_MAX_VERSION])
-    self.assertEqual(http.ca_certs, gapi.GC_Values[gapi.GC_CA_FILE])
-
-  def test_create_http_sets_tls_min_version(self):
-    http = gapi.create_http(override_min_tls=1111)
-    self.assertEqual(http.tls_minimum_version, 1111)
-
-  def test_create_http_sets_tls_max_version(self):
-    http = gapi.create_http(override_max_tls=9999)
-    self.assertEqual(http.tls_maximum_version, 9999)
-
-  def test_create_http_sets_cache(self):
-    fake_cache = {}
-    http = gapi.create_http(cache=fake_cache)
-    self.assertEqual(http.cache, fake_cache)
-
-  def test_create_http_sets_cache_timeout(self):
-    http = gapi.create_http(timeout=1234)
-    self.assertEqual(http.timeout, 1234)
-
-
 class GapiTest(unittest.TestCase):
 
   def setUp(self):
@@ -142,7 +108,7 @@ class GapiTest(unittest.TestCase):
         self.mock_service, self.mock_method_name, soft_errors=True)
     self.assertEqual(response, fake_200_response)
     self.assertEqual(
-        self.mock_service._http.request.credentials.refresh.call_count, 1)
+        self.mock_service._http.credentials.refresh.call_count, 1)
     self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
 
   def test_call_throws_for_provided_reason(self):
@@ -361,25 +327,25 @@ class GapiTest(unittest.TestCase):
     self.assertIn('pageSize', request_method_kwargs)
     self.assertEqual(123456, request_method_kwargs['pageSize'])
 
-  @patch.object(gapi.sys.stderr, 'write')
-  def test_get_all_pages_prints_paging_message(self, mock_write):
+  def test_get_all_pages_prints_paging_message(self):
     self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
 
     paging_message = 'A simple string displayed during paging'
-    gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, page_message=paging_message)
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
     messages_written = [
         call_args[0][0] for call_args in mock_write.call_args_list
     ]
     self.assertIn(paging_message, messages_written)
 
-  @patch.object(gapi.sys.stderr, 'write')
-  def test_get_all_pages_prints_paging_message_inline(self, mock_write):
+  def test_get_all_pages_prints_paging_message_inline(self):
     self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
 
     paging_message = 'A simple string displayed during paging'
-    gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, page_message=paging_message)
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
     messages_written = [
         call_args[0][0] for call_args in mock_write.call_args_list
     ]
@@ -394,13 +360,13 @@ class GapiTest(unittest.TestCase):
         paging_message_call_positions[0]:paging_message_call_positions[1]]
     self.assertIn('\r', printed_between_page_messages)
 
-  @patch.object(gapi.sys.stderr, 'write')
-  def test_get_all_pages_ends_paging_message_with_newline(self, mock_write):
+  def test_get_all_pages_ends_paging_message_with_newline(self):
     self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
 
     paging_message = 'A simple string displayed during paging'
-    gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, page_message=paging_message)
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
     messages_written = [
         call_args[0][0] for call_args in mock_write.call_args_list
     ]
@@ -410,14 +376,13 @@ class GapiTest(unittest.TestCase):
         messages_written) - messages_written[::-1].index('\r\n')
     self.assertGreater(last_carriage_return_index, last_page_message_index)
 
-  @patch.object(gapi.sys.stderr, 'write')
-  def test_get_all_pages_prints_attribute_total_items_in_paging_message(
-      self, mock_write):
+  def test_get_all_pages_prints_attribute_total_items_in_paging_message(self):
     self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
 
     paging_message = 'Total number of items discovered: %%total_items%%'
-    gapi.get_all_pages(
-        self.mock_service, self.mock_method_name, page_message=paging_message)
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service, self.mock_method_name, page_message=paging_message)
 
     messages_written = [
         call_args[0][0] for call_args in mock_write.call_args_list
@@ -442,17 +407,17 @@ class GapiTest(unittest.TestCase):
     for message in messages_written:
       self.assertNotIn('%%total_items', message)
 
-  @patch.object(gapi.sys.stderr, 'write')
-  def test_get_all_pages_prints_attribute_first_item_in_paging_message(
-      self, mock_write):
+  def test_get_all_pages_prints_attribute_first_item_in_paging_message(self):
     self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
 
     paging_message = 'First item in page: %%first_item%%'
-    gapi.get_all_pages(
-        self.mock_service,
-        self.mock_method_name,
-        page_message=paging_message,
-        message_attribute='position')
+
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
 
     messages_written = [
         call_args[0][0] for call_args in mock_write.call_args_list
@@ -471,17 +436,16 @@ class GapiTest(unittest.TestCase):
     for message in messages_written:
       self.assertNotIn('%%first_item', message)
 
-  @patch.object(gapi.sys.stderr, 'write')
-  def test_get_all_pages_prints_attribute_last_item_in_paging_message(
-      self, mock_write):
+  def test_get_all_pages_prints_attribute_last_item_in_paging_message(self):
     self.mock_method.return_value.execute.side_effect = self.simple_3_page_response
 
     paging_message = 'Last item in page: %%last_item%%'
-    gapi.get_all_pages(
-        self.mock_service,
-        self.mock_method_name,
-        page_message=paging_message,
-        message_attribute='position')
+    with patch.object(gapi.sys.stderr, 'write') as mock_write:
+      gapi.get_all_pages(
+          self.mock_service,
+          self.mock_method_name,
+          page_message=paging_message,
+          message_attribute='position')
 
     messages_written = [
         call_args[0][0] for call_args in mock_write.call_args_list

src/gapi/calendar.py

@@ -0,0 +1,894 @@
+import csv
+import sys
+import uuid
+
+# TODO: get rid of these hacks
+import __main__
+from var import *
+
+import controlflow
+import display
+import fileutils
+import gapi
+import utils
+
+
+def normalizeCalendarId(calname, checkPrimary=False):
+    if checkPrimary and calname.lower() == 'primary':
+        return calname
+    if not GC_Values[GC_DOMAIN]:
+        GC_Values[GC_DOMAIN] = __main__._getValueFromOAuth('hd')
+    return __main__.convertUIDtoEmailAddress(calname,
+                                             email_types=['user', 'resource'])
+
+
+def buildCalendarGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+    return (calendarId, __main__.buildGAPIServiceObject('calendar',
+                                                        calendarId))
+
+
+def buildCalendarDataGAPIObject(calname):
+    calendarId = normalizeCalendarId(calname)
+
+    # Try to impersonate the calendar owner. If we fail, fall back to using
+    # admin for authentication. Resource calendars cannot be impersonated,
+    # so we need to access them as the admin.
+    cal = None
+    if not calname.endswith('.calendar.google.com'):
+        cal = __main__.buildGAPIServiceObject('calendar', calendarId, False)
+    if cal is None:
+        _, cal = buildCalendarGAPIObject(__main__._getValueFromOAuth('email'))
+    return (calendarId, cal)
+
+def printShowACLs(csvFormat):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    toDrive = False
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if csvFormat and myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            action = ['showacl', 'printacl'][csvFormat]
+            message = f"gam calendar <email> {action}"
+            controlflow.invalid_argument_exit(sys.argv[i], message)
+    acls = gapi.get_all_pages(
+        cal.acl(), 'list', 'items', calendarId=calendarId)
+    i = 0
+    if csvFormat:
+        titles = []
+        rows = []
+    else:
+        count = len(acls)
+    for rule in acls:
+        i += 1
+        if csvFormat:
+            row = utils.flatten_json(rule, None)
+            for key in row:
+                if key not in titles:
+                    titles.append(key)
+            rows.append(row)
+        else:
+            formatted_acl = formatACLRule(rule)
+            current_count = display.current_count(i, count)
+            print(f'Calendar: {calendarId}, ACL: {formatted_acl}{current_count}')
+    if csvFormat:
+        display.write_csv_file(
+            rows, titles, f'{calendarId} Calendar ACLs', toDrive)
+
+
+def _getCalendarACLScope(i, body):
+    body['scope'] = {}
+    myarg = sys.argv[i].lower()
+    body['scope']['type'] = myarg
+    i += 1
+    if myarg in ['user', 'group']:
+        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
+            sys.argv[i], noUid=True)
+        i += 1
+    elif myarg == 'domain':
+        if i < len(sys.argv) and \
+           sys.argv[i].lower().replace('_', '') != 'sendnotifications':
+            body['scope']['value'] = sys.argv[i].lower()
+            i += 1
+        else:
+            body['scope']['value'] = GC_Values[GC_DOMAIN]
+    elif myarg != 'default':
+        body['scope']['type'] = 'user'
+        body['scope']['value'] = __main__.normalizeEmailAddressOrUID(
+            myarg, noUid=True)
+    return i
+
+
+CALENDAR_ACL_ROLES_MAP = {
+    'editor': 'writer',
+    'freebusy': 'freeBusyReader',
+    'freebusyreader': 'freeBusyReader',
+    'owner': 'owner',
+    'read': 'reader',
+    'reader': 'reader',
+    'writer': 'writer',
+    'none': 'none',
+}
+
+
+def addACL(function):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    myarg = sys.argv[4].lower().replace('_', '')
+    if myarg not in CALENDAR_ACL_ROLES_MAP:
+        controlflow.expected_argument_exit(
+            "Role", ", ".join(CALENDAR_ACL_ROLES_MAP), myarg)
+    body = {'role': CALENDAR_ACL_ROLES_MAP[myarg]}
+    i = _getCalendarACLScope(5, body)
+    sendNotifications = True
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'sendnotifications':
+            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam calendar <email> {function.lower()}")
+    print(f'Calendar: {calendarId}, {function} ACL: {formatACLRule(body)}')
+    gapi.call(cal.acl(), 'insert', calendarId=calendarId,
+              body=body, sendNotifications=sendNotifications)
+
+
+def delACL():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    if sys.argv[4].lower() == 'id':
+        ruleId = sys.argv[5]
+        print(f'Removing rights for {ruleId} to {calendarId}')
+        gapi.call(cal.acl(), 'delete', calendarId=calendarId, ruleId=ruleId)
+    else:
+        body = {'role': 'none'}
+        _getCalendarACLScope(5, body)
+        print(f'Calendar: {calendarId}, Delete ACL: {formatACLScope(body)}')
+        gapi.call(cal.acl(), 'insert', calendarId=calendarId,
+                  body=body, sendNotifications=False)
+
+
+def wipeData():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    gapi.call(cal.calendars(), 'clear', calendarId=calendarId)
+
+
+def printEvents():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    q = showDeleted = showHiddenInvitations = timeMin = \
+        timeMax = timeZone = updatedMin = None
+    toDrive = False
+    titles = []
+    csvRows = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'query':
+            q = sys.argv[i+1]
+            i += 2
+        elif myarg == 'includedeleted':
+            showDeleted = True
+            i += 1
+        elif myarg == 'includehidden':
+            showHiddenInvitations = True
+            i += 1
+        elif myarg == 'after':
+            timeMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'before':
+            timeMax = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i+1]
+            i += 2
+        elif myarg == 'updated':
+            updatedMin = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            toDrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam calendar <email> printevents")
+    page_message = gapi.got_total_items_msg(f'Events for {calendarId}', '')
+    results = gapi.get_all_pages(cal.events(), 'list', 'items',
+                                 page_message=page_message,
+                                 calendarId=calendarId, q=q,
+                                 showDeleted=showDeleted,
+                                 showHiddenInvitations=showHiddenInvitations,
+                                 timeMin=timeMin, timeMax=timeMax,
+                                 timeZone=timeZone,
+                                 updatedMin=updatedMin)
+    for result in results:
+        row = {'calendarId': calendarId}
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(result, flattened=row), csvRows, titles)
+    display.sort_csv_titles(['calendarId', 'id', 'summary', 'status'], titles)
+    display.write_csv_file(csvRows, titles, 'Calendar Events', toDrive)
+
+
+def formatACLScope(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]})'
+    return f'(Scope: {rule["scope"]["type"]})'
+
+
+def formatACLRule(rule):
+    if rule['scope']['type'] != 'default':
+        return f'(Scope: {rule["scope"]["type"]}:{rule["scope"]["value"]}, ' \
+               f'Role: {rule["role"]})'
+    return f'(Scope: {rule["scope"]["type"]}, Role: {rule["role"]})'
+
+
+def getSendUpdates(myarg, i, cal):
+    if myarg == 'notifyattendees':
+        sendUpdates = 'all'
+        i += 1
+    elif myarg == 'sendnotifications':
+        sendUpdates = 'all' if __main__.getBoolean(sys.argv[i+1], myarg) else 'none'
+        i += 2
+    else:  # 'sendupdates':
+        sendUpdatesMap = {}
+        for val in cal._rootDesc['resources']['events']['methods']['delete'][
+                'parameters']['sendUpdates']['enum']:
+            sendUpdatesMap[val.lower()] = val
+        sendUpdates = sendUpdatesMap.get(sys.argv[i+1].lower(), False)
+        if not sendUpdates:
+            controlflow.expected_argument_exit(
+                "sendupdates", ", ".join(sendUpdatesMap), sys.argv[i+1])
+        i += 2
+    return (sendUpdates, i)
+
+
+def moveOrDeleteEvent(moveOrDelete):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    sendUpdates = None
+    doit = False
+    kwargs = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg in ['id', 'eventid']:
+            eventId = sys.argv[i+1]
+            i += 2
+        elif myarg in ['query', 'eventquery']:
+            controlflow.system_error_exit(
+                2, f'query is no longer supported for {moveOrDelete}event. ' \
+                   f'Use "gam calendar <email> printevents query <query> | ' \
+                   f'gam csv - gam {moveOrDelete}event id ~id" instead.')
+        elif myarg == 'doit':
+            doit = True
+            i += 1
+        elif moveOrDelete == 'move' and myarg == 'destination':
+            kwargs['destination'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam calendar <email> {moveOrDelete}event")
+    if doit:
+        print(f' going to {moveOrDelete} eventId {eventId}')
+        gapi.call(cal.events(), moveOrDelete, calendarId=calendarId,
+                  eventId=eventId, sendUpdates=sendUpdates, **kwargs)
+    else:
+        print(
+            f' would {moveOrDelete} eventId {eventId}. Add doit to command ' \
+            f'to actually {moveOrDelete} event')
+
+
+def infoEvent():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    eventId = sys.argv[4]
+    result = gapi.call(cal.events(), 'get',
+                       calendarId=calendarId, eventId=eventId)
+    display.print_json(result)
+
+
+def addOrUpdateEvent(action):
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    # only way for non-Google calendars to get updates is via email
+    kwargs = {}
+    body = {}
+    if action == 'add':
+        i = 4
+        func = 'insert'
+    else:
+        eventId = sys.argv[4]
+        kwargs = {'eventId': eventId}
+        i = 5
+        func = 'patch'
+        requires_full_update = ['attendee', 'optionalattendee',
+                                'removeattendee', 'replacedescription']
+        for arg in sys.argv[i:]:
+            if arg.replace('_', '').lower() in requires_full_update:
+                func = 'update'
+                body = gapi.call(cal.events(), 'get',
+                                 calendarId=calendarId, eventId=eventId)
+                break
+    sendUpdates, body = getEventAttributes(i, calendarId, cal, body, action)
+    result = gapi.call(cal.events(), func, conferenceDataVersion=1,
+                       supportsAttachments=True, calendarId=calendarId,
+                       sendUpdates=sendUpdates, body=body, fields='id',
+                       **kwargs)
+    print(f'Event {result["id"]} {action} finished')
+
+
+def _remove_attendee(attendees, remove_email):
+    return [attendee for attendee in attendees
+            if not attendee['email'].lower() == remove_email]
+
+
+def getEventAttributes(i, calendarId, cal, body, action):
+    # Default to external only so non-Google
+    # calendars are notified of changes
+    sendUpdates = 'externalOnly'
+    action = 'update' if body else 'add'
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['notifyattendees', 'sendnotifications', 'sendupdates']:
+            sendUpdates, i = getSendUpdates(myarg, i, cal)
+        elif myarg == 'attendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append({'email': sys.argv[i+1]})
+            i += 2
+        elif myarg == 'removeattendee' and action == 'update':
+            remove_email = sys.argv[i+1].lower()
+            if 'attendees' in body:
+                body['attendees'] = _remove_attendee(body['attendees'],
+                                                     remove_email)
+            i += 2
+        elif myarg == 'optionalattendee':
+            body.setdefault('attendees', [])
+            body['attendees'].append(
+                {'email': sys.argv[i+1], 'optional': True})
+            i += 2
+        elif myarg == 'anyonecanaddself':
+            body['anyoneCanAddSelf'] = True
+            i += 1
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'replacedescription' and action == 'update':
+            search = sys.argv[i+1]
+            replace = sys.argv[i+2]
+            if 'description' in body:
+                body['description'] = re.sub(search, replace, body['description'])
+            i += 3
+        elif myarg == 'start':
+            if sys.argv[i+1].lower() == 'allday':
+                body['start'] = {'date': utils.get_yyyymmdd(sys.argv[i+2])}
+                i += 3
+            else:
+                start_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                body['start'] = {'dateTime': start_time}
+                i += 2
+        elif myarg == 'end':
+            if sys.argv[i+1].lower() == 'allday':
+                body['end'] = {'date': utils.get_yyyymmdd(sys.argv[i+2])}
+                i += 3
+            else:
+                end_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
+                body['end'] = {'dateTime': end_time}
+                i += 2
+        elif myarg == 'guestscantinviteothers':
+            body['guestsCanInviteOthers'] = False
+            i += 1
+        elif myarg == 'guestscaninviteothers':
+            body['guestsCanInviteTohters'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscaninviteothers')
+            i += 2
+        elif myarg == 'guestscantseeothers':
+            body['guestsCanSeeOtherGuests'] = False
+            i += 1
+        elif myarg == 'guestscanseeothers':
+            body['guestsCanSeeOtherGuests'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscanseeothers')
+            i += 2
+        elif myarg == 'guestscanmodify':
+            body['guestsCanModify'] = __main__.getBoolean(
+                sys.argv[i+1], 'guestscanmodify')
+            i += 2
+        elif myarg == 'id':
+            if action == 'update':
+                controlflow.invalid_argument_exit(
+                    'id', 'gam calendar <calendar> updateevent')
+            body['id'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'available':
+            body['transparency'] = 'transparent'
+            i += 1
+        elif myarg == 'transparency':
+            validTransparency = ['opaque', 'transparent']
+            if sys.argv[i+1].lower() in validTransparency:
+                body['transparency'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    'transparency',
+                    ", ".join(validTransparency), sys.argv[i+1])
+            i += 2
+        elif myarg == 'visibility':
+            validVisibility = ['default', 'public', 'private']
+            if sys.argv[i+1].lower() in validVisibility:
+                body['visibility'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    "visibility", ", ".join(validVisibility), sys.argv[i+1])
+            i += 2
+        elif myarg == 'tentative':
+            body['status'] = 'tentative'
+            i += 1
+        elif myarg == 'status':
+            validStatus = ['confirmed', 'tentative', 'cancelled']
+            if sys.argv[i+1].lower() in validStatus:
+                body['status'] = sys.argv[i+1].lower()
+            else:
+                controlflow.expected_argument_exit(
+                    'visibility', ', '.join(validStatus), sys.argv[i+1])
+            i += 2
+        elif myarg == 'source':
+            body['source'] = {'title': sys.argv[i+1], 'url': sys.argv[i+2]}
+            i += 3
+        elif myarg == 'noreminders':
+            body['reminders'] = {'useDefault': False}
+            i += 1
+        elif myarg == 'reminder':
+            minutes = \
+            __main__.getInteger(sys.argv[i+1], myarg, minVal=0,
+                                maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+            reminder = {'minutes': minutes, 'method': sys.argv[i+2]}
+            body.setdefault(
+                'reminders', {'overrides': [], 'useDefault': False})
+            body['reminders']['overrides'].append(reminder)
+            i += 3
+        elif myarg == 'recurrence':
+            body.setdefault('recurrence', [])
+            body['recurrence'].append(sys.argv[i+1])
+            i += 2
+        elif myarg == 'timezone':
+            timeZone = sys.argv[i+1]
+            i += 2
+        elif myarg == 'privateproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['private'][sys.argv[i+1]] = sys.argv[i+2]
+            i += 3
+        elif myarg == 'sharedproperty':
+            if 'extendedProperties' not in body:
+                body['extendedProperties'] = {'private': {}, 'shared': {}}
+            body['extendedProperties']['shared'][sys.argv[i+1]] = sys.argv[i+2]
+            i += 3
+        elif myarg == 'colorindex':
+            body['colorId'] = __main__.getInteger(
+                sys.argv[i+1], myarg, CALENDAR_EVENT_MIN_COLOR_INDEX,
+                CALENDAR_EVENT_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'hangoutsmeet':
+            body['conferenceData'] = {'createRequest': {
+                'requestId': f'{str(uuid.uuid4())}'}}
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f'gam calendar <email> {action}event')
+    if ('recurrence' in body) and (('start' in body) or ('end' in body)):
+        if not timeZone:
+            timeZone = gapi.call(cal.calendars(), 'get',
+                                 calendarId=calendarId,
+                                 fields='timeZone')['timeZone']
+        if 'start' in body:
+            body['start']['timeZone'] = timeZone
+        if 'end' in body:
+            body['end']['timeZone'] = timeZone
+    return (sendUpdates, body)
+
+
+def modifySettings():
+    calendarId, cal = buildCalendarDataGAPIObject(sys.argv[2])
+    if not cal:
+        return
+    body = {}
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            body['location'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'summary':
+            body['summary'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'timezone':
+            body['timeZone'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam calendar <email> modify")
+    gapi.call(cal.calendars(), 'patch', calendarId=calendarId, body=body)
+
+
+def changeAttendees(users):
+    do_it = True
+    i = 5
+    allevents = False
+    start_date = end_date = None
+    while len(sys.argv) > i:
+        myarg = sys.argv[i].lower()
+        if myarg == 'csv':
+            csv_file = sys.argv[i+1]
+            i += 2
+        elif myarg == 'dryrun':
+            do_it = False
+            i += 1
+        elif myarg == 'start':
+            start_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'end':
+            end_date = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'allevents':
+            allevents = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam <users> update calattendees")
+    attendee_map = {}
+    f = fileutils.open_file(csv_file)
+    csvFile = csv.reader(f)
+    for row in csvFile:
+        attendee_map[row[0].lower()] = row[1].lower()
+    fileutils.close_file(f)
+    for user in users:
+        sys.stdout.write(f'Checking user {user}\n')
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        page_token = None
+        while True:
+            events_page = gapi.call(cal.events(), 'list', calendarId=user,
+                                    pageToken=page_token, timeMin=start_date,
+                                    timeMax=end_date, showDeleted=False,
+                                    showHiddenInvitations=False)
+            print(f'Got {len(events_page.get("items", []))}')
+            for event in events_page.get('items', []):
+                if event['status'] == 'cancelled':
+                    # print u' skipping cancelled event'
+                    continue
+                try:
+                    event_summary = event['summary']
+                except (KeyError, UnicodeEncodeError, UnicodeDecodeError):
+                    event_summary = event['id']
+                try:
+                    organizer = event['organizer']['email'].lower()
+                    if not allevents and organizer != user:
+                        #print(f' skipping not-my-event {event_summary}')
+                        continue
+                except KeyError:
+                    pass  # no email for organizer
+                needs_update = False
+                try:
+                    for attendee in event['attendees']:
+                        try:
+                            if attendee['email'].lower() in attendee_map:
+                                old_email = attendee['email'].lower()
+                                new_email = attendee_map[attendee['email'].lower(
+                                )]
+                                print(f' SWITCHING attendee {old_email} to ' \
+                                    f'{new_email} for {event_summary}')
+                                event['attendees'].remove(attendee)
+                                event['attendees'].append({'email': new_email})
+                                needs_update = True
+                        except KeyError:  # no email for that attendee
+                            pass
+                except KeyError:
+                    continue  # no attendees
+                if needs_update:
+                    body = {}
+                    body['attendees'] = event['attendees']
+                    print(f'UPDATING {event_summary}')
+                    if do_it:
+                        gapi.call(cal.events(), 'patch', calendarId=user,
+                                  eventId=event['id'],
+                                  sendNotifications=False, body=body)
+                    else:
+                        print(' not pulling the trigger.')
+                # else:
+                #  print(f' no update needed for {event_summary}')
+            try:
+                page_token = events_page['nextPageToken']
+            except KeyError:
+                break
+
+
+def deleteCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    for user in users:
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        gapi.call(cal.calendarList(), 'delete',
+                  soft_errors=True, calendarId=calendarId)
+
+
+CALENDAR_REMINDER_MAX_MINUTES = 40320
+
+CALENDAR_MIN_COLOR_INDEX = 1
+CALENDAR_MAX_COLOR_INDEX = 24
+
+CALENDAR_EVENT_MIN_COLOR_INDEX = 1
+CALENDAR_EVENT_MAX_COLOR_INDEX = 11
+
+
+def getCalendarAttributes(i, body, function):
+    colorRgbFormat = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'selected':
+            body['selected'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg == 'hidden':
+            body['hidden'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg == 'summary':
+            body['summaryOverride'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'colorindex':
+            body['colorId'] = __main__.getInteger(
+                sys.argv[i+1], myarg, minVal=CALENDAR_MIN_COLOR_INDEX,
+                maxVal=CALENDAR_MAX_COLOR_INDEX)
+            i += 2
+        elif myarg == 'backgroundcolor':
+            body['backgroundColor'] = __main__.getColor(sys.argv[i+1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'foregroundcolor':
+            body['foregroundColor'] = __main__.getColor(sys.argv[i+1])
+            colorRgbFormat = True
+            i += 2
+        elif myarg == 'reminder':
+            body.setdefault('defaultReminders', [])
+            method = sys.argv[i+1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_REMINDER_METHODS:
+                    controlflow.expected_argument_exit("Method", ", ".join(
+                        CALENDAR_REMINDER_METHODS+CLEAR_NONE_ARGUMENT), method)
+                minutes = __main__.getInteger(
+                    sys.argv[i+2], myarg, minVal=0,
+                    maxVal=CALENDAR_REMINDER_MAX_MINUTES)
+                body['defaultReminders'].append(
+                    {'method': method, 'minutes': minutes})
+                i += 3
+            else:
+                i += 2
+        elif myarg == 'notification':
+            body.setdefault('notificationSettings', {'notifications': []})
+            method = sys.argv[i+1].lower()
+            if method not in CLEAR_NONE_ARGUMENT:
+                if method not in CALENDAR_NOTIFICATION_METHODS:
+                    controlflow.expected_argument_exit("Method", ", ".join(
+                        CALENDAR_NOTIFICATION_METHODS+CLEAR_NONE_ARGUMENT), method)
+                eventType = sys.argv[i+2].lower()
+                if eventType not in CALENDAR_NOTIFICATION_TYPES_MAP:
+                    controlflow.expected_argument_exit("Event", ", ".join(
+                        CALENDAR_NOTIFICATION_TYPES_MAP), eventType)
+                notice = {'method': method,
+                          'type': CALENDAR_NOTIFICATION_TYPES_MAP[eventType]}
+                body['notificationSettings']['notifications'].append(notice)
+                i += 3
+            else:
+                i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], f"gam {function} calendar")
+    return colorRgbFormat
+
+
+def addCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5])
+    body = {'id': calendarId, 'selected': True, 'hidden': False}
+    colorRgbFormat = getCalendarAttributes(6, body, 'add')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f'Subscribing {user} to calendar {calendarId}{current_count}')
+        gapi.call(cal.calendarList(), 'insert', soft_errors=True,
+                  body=body, colorRgbFormat=colorRgbFormat)
+
+
+def updateCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    body = {}
+    colorRgbFormat = getCalendarAttributes(6, body, 'update')
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        current_count = display.current_count(i, count)
+        print(f"Updating {user}'s subscription to calendar ' \
+              f'{calendarId}{current_count}")
+        calId = calendarId if calendarId != 'primary' else user
+        gapi.call(cal.calendarList(), 'patch', soft_errors=True,
+                  calendarId=calId, body=body, colorRgbFormat=colorRgbFormat)
+
+
+def _showCalendar(userCalendar, j, jcount):
+    current_count = display.current_count(j, jcount)
+    summary = userCalendar.get("summaryOverride", userCalendar["summary"])
+    print(f'  Calendar: {userCalendar["id"]}{current_count}')
+    print(f'    Summary: {summary}')
+    print(f'    Description: {userCalendar.get("description", "")}')
+    print(f'    Access Level: {userCalendar["accessRole"]}')
+    print(f'    Timezone: {userCalendar["timeZone"]}')
+    print(f'    Location: {userCalendar.get("location", "")}')
+    print(f'    Hidden: {userCalendar.get("hidden", "False")}')
+    print(f'    Selected: {userCalendar.get("selected", "False")}')
+    print(f'    Color ID: {userCalendar["colorId"]}, ' \
+          f'Background Color: {userCalendar["backgroundColor"]}, ' \
+          f'Foreground Color: {userCalendar["foregroundColor"]}')
+    print(f'    Default Reminders:')
+    for reminder in userCalendar.get('defaultReminders', []):
+        print(f'      Method: {reminder["method"]}, ' \
+              f'Minutes: {reminder["minutes"]}')
+    print('    Notifications:')
+    if 'notificationSettings' in userCalendar:
+        notifications = userCalendar['notificationSettings'].get(
+            'notifications', [])
+        for notification in notifications:
+            print(f'      Method: {notification["method"]}, ' \
+                  f'Type: {notification["type"]}')
+
+
+def infoCalendar(users):
+    calendarId = normalizeCalendarId(sys.argv[5], checkPrimary=True)
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.call(cal.calendarList(), 'get',
+                           soft_errors=True,
+                           calendarId=calendarId)
+        if result:
+            print(f'User: {user}, Calendar:{display.current_count(i, count)}')
+            _showCalendar(result, 1, 1)
+
+
+def printShowCalendars(users, csvFormat):
+    if csvFormat:
+        todrive = False
+        titles = []
+        csvRows = []
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if csvFormat and myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, f"gam <users> {['show', 'print'][csvFormat]} calendars")
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        result = gapi.get_all_pages(
+            cal.calendarList(), 'list', 'items', soft_errors=True)
+        jcount = len(result)
+        if not csvFormat:
+            print(f'User: {user}, Calendars:{display.current_count(i, count)}')
+            if jcount == 0:
+                continue
+            j = 0
+            for userCalendar in result:
+                j += 1
+                _showCalendar(userCalendar, j, jcount)
+        else:
+            if jcount == 0:
+                continue
+            for userCalendar in result:
+                row = {'primaryEmail': user}
+                display.add_row_titles_to_csv_file(utils.flatten_json(
+                    userCalendar, flattened=row), csvRows, titles)
+    if csvFormat:
+        display.sort_csv_titles(['primaryEmail', 'id'], titles)
+        display.write_csv_file(csvRows, titles, 'Calendars', todrive)
+
+
+def showCalSettings(users):
+    i = 0
+    count = len(users)
+    for user in users:
+        i += 1
+        user, cal = buildCalendarGAPIObject(user)
+        if not cal:
+            continue
+        feed = gapi.get_all_pages(
+            cal.settings(), 'list', 'items', soft_errors=True)
+        if feed:
+            current_count = display.current_count(i, count)
+            print(f'User: {user}, Calendar Settings:{current_count}')
+            settings = {}
+            for setting in feed:
+                settings[setting['id']] = setting['value']
+            for attr, value in sorted(settings.items()):
+                print(f'  {attr}: {value}')
+
+
+def transferSecCals(users):
+    target_user = sys.argv[5]
+    remove_source_user = sendNotifications = True
+    i = 6
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'keepuser':
+            remove_source_user = False
+            i += 1
+        elif myarg == 'sendnotifications':
+            sendNotifications = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam <users> transfer seccals")
+    if remove_source_user:
+        target_user, target_cal = buildCalendarGAPIObject(target_user)
+        if not target_cal:
+            return
+    for user in users:
+        user, source_cal = buildCalendarGAPIObject(user)
+        if not source_cal:
+            continue
+        calendars = gapi.get_all_pages(source_cal.calendarList(), 'list',
+                                       'items', soft_errors=True,
+                                       minAccessRole='owner', showHidden=True,
+                                       fields='items(id),nextPageToken')
+        for calendar in calendars:
+            calendarId = calendar['id']
+            if calendarId.find('@group.calendar.google.com') != -1:
+                body = {'role': 'owner',
+                        'scope': {'type': 'user', 'value': target_user}}
+                gapi.call(source_cal.acl(), 'insert', calendarId=calendarId,
+                          body=body, sendNotifications=sendNotifications)
+                if remove_source_user:
+                    body = {'role': 'none',
+                            'scope': {'type': 'user', 'value': user}}
+                    gapi.call(target_cal.acl(), 'insert',
+                              calendarId=calendarId, body=body,
+                              sendNotifications=sendNotifications)

src/gapi/directory/__init__.py

@@ -0,0 +1,5 @@
+import __main__
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('directory')

src/gapi/directory/cros.py

@@ -0,0 +1,794 @@
+import datetime
+
+from var import *
+import __main__
+import controlflow
+import display
+import fileutils
+import gapi
+import gapi.directory
+import utils
+
+
+def doUpdateCros():
+    cd = gapi.directory.buildGAPIObject()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    update_body = {}
+    action_body = {}
+    orgUnitPath = None
+    ack_wipe = False
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'user':
+            update_body['annotatedUser'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'location':
+            update_body['annotatedLocation'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'notes':
+            update_body['notes'] = sys.argv[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg in ['tag', 'asset', 'assetid']:
+            update_body['annotatedAssetId'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['ou', 'org']:
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'action':
+            action = sys.argv[i+1].lower().replace('_', '').replace('-', '')
+            deprovisionReason = None
+            if action in ['deprovisionsamemodelreplace',
+                          'deprovisionsamemodelreplacement']:
+                action = 'deprovision'
+                deprovisionReason = 'same_model_replacement'
+            elif action in ['deprovisiondifferentmodelreplace',
+                            'deprovisiondifferentmodelreplacement']:
+                action = 'deprovision'
+                deprovisionReason = 'different_model_replacement'
+            elif action in ['deprovisionretiringdevice']:
+                action = 'deprovision'
+                deprovisionReason = 'retiring_device'
+            elif action not in ['disable', 'reenable']:
+                controlflow.system_error_exit(2, f'expected action of ' \
+                    f'deprovision_same_model_replace, ' \
+                    f'deprovision_different_model_replace, ' \
+                    f'deprovision_retiring_device, disable or reenable,'
+                    f' got {action}')
+            action_body = {'action': action}
+            if deprovisionReason:
+                action_body['deprovisionReason'] = deprovisionReason
+            i += 2
+        elif myarg == 'acknowledgedevicetouchrequirement':
+            ack_wipe = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam update cros")
+    i = 0
+    count = len(devices)
+    if action_body:
+        if action_body['action'] == 'deprovision' and not ack_wipe:
+            print(f'WARNING: Refusing to deprovision {count} devices because '
+                  'acknowledge_device_touch_requirement not specified. ' \
+                  'Deprovisioning a device means the device will have to ' \
+                  'be physically wiped and re-enrolled to be managed by ' \
+                  'your domain again. This requires physical access to ' \
+                  'the device and is very time consuming to perform for ' \
+                  'each device. Please add ' \
+                  '"acknowledge_device_touch_requirement" to the GAM ' \
+                  'command if you understand this and wish to proceed ' \
+                  'with the deprovision. Please also be aware that ' \
+                  'deprovisioning can have an effect on your device ' \
+                  'license count. See ' \
+                  'https://support.google.com/chrome/a/answer/3523633 '\
+                  'for full details.')
+            sys.exit(3)
+        for deviceId in devices:
+            i += 1
+            cur_count = __main__.currentCount(i, count)
+            print(f' performing action {action} for {deviceId}{cur_count}')
+            gapi.call(cd.chromeosdevices(), function='action',
+                      customerId=GC_Values[GC_CUSTOMER_ID],
+                      resourceId=deviceId, body=action_body)
+    else:
+        if update_body:
+            for deviceId in devices:
+                i += 1
+                current_count = __main__.currentCount(i, count)
+                print(f' updating {deviceId}{current_count}')
+                gapi.call(cd.chromeosdevices(), 'update',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          deviceId=deviceId, body=update_body)
+        if orgUnitPath:
+            # split moves into max 50 devices per batch
+            for l in range(0, len(devices), 50):
+                move_body = {'deviceIds': devices[l:l+50]}
+                print(f' moving {len(move_body["deviceIds"])} devices to ' \
+                      f'{orgUnitPath}')
+                gapi.call(cd.chromeosdevices(), 'moveDevicesToOu',
+                          customerId=GC_Values[GC_CUSTOMER_ID],
+                          orgUnitPath=orgUnitPath, body=move_body)
+
+
+def doGetCrosInfo():
+    cd = gapi.directory.buildGAPIObject()
+    i, devices = getCrOSDeviceEntity(3, cd)
+    downloadfile = None
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    projection = None
+    fieldsList = []
+    noLists = False
+    startDate = endDate = None
+    listLimit = 0
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'nolists':
+            noLists = True
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=-1)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            fieldsList = []
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[myarg])
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i+1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    fieldsList.extend(CROS_ARGUMENT_TO_PROPERTY_MAP[field])
+                    if field in CROS_ACTIVE_TIME_RANGES_ARGUMENTS + \
+                                CROS_DEVICE_FILES_ARGUMENTS + \
+                                CROS_RECENT_USERS_ARGUMENTS:
+                        projection = 'FULL'
+                        noLists = False
+                else:
+                    controlflow.invalid_argument_exit(
+                        field, "gam info cros fields")
+            i += 2
+        elif myarg == 'downloadfile':
+            downloadfile = sys.argv[i+1]
+            if downloadfile.lower() == 'latest':
+                downloadfile = downloadfile.lower()
+            i += 2
+        elif myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i+1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam info cros")
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = ','.join(set(fieldsList)).replace('.', '/')
+    else:
+        fields = None
+    i = 0
+    device_count = len(devices)
+    for deviceId in devices:
+        i += 1
+        cros = gapi.call(cd.chromeosdevices(), 'get',
+                         customerId=GC_Values[GC_CUSTOMER_ID],
+                         deviceId=deviceId, projection=projection,
+                         fields=fields)
+        print(f'CrOS Device: {deviceId} ({i} of {device_count})')
+        if 'notes' in cros:
+            cros['notes'] = cros['notes'].replace('\n', '\\n')
+        if 'autoUpdateExpiration' in cros:
+            cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                cros['autoUpdateExpiration'])
+        _checkTPMVulnerability(cros)
+        for up in CROS_SCALAR_PROPERTY_PRINT_ORDER:
+            if up in cros:
+                if isinstance(cros[up], str):
+                    print(f'  {up}: {cros[up]}')
+                else:
+                    sys.stdout.write(f'  {up}:')
+                    display.print_json(cros[up], '  ')
+        if not noLists:
+            activeTimeRanges = _filterTimeRanges(
+                cros.get('activeTimeRanges', []), startDate, endDate)
+            lenATR = len(activeTimeRanges)
+            if lenATR:
+                print('  activeTimeRanges')
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    active_date = activeTimeRange["date"]
+                    active_time = activeTimeRange["activeTime"]
+                    duration = utils.formatMilliSeconds(active_time)
+                    minutes = active_time // 60000
+                    print(f'    date: {active_date}')
+                    print(f'      activeTime: {active_time}')
+                    print(f'      duration: {duration}')
+                    print(f'      minutes: {minutes}')
+            recentUsers = cros.get('recentUsers', [])
+            lenRU = len(recentUsers)
+            if lenRU:
+                print('  recentUsers')
+                num_ranges = min(lenRU, listLimit or lenRU)
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get("email")
+                    if not useremail:
+                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    print(f'    type: {recentUser["type"]}')
+                    print(f'      email: {useremail}')
+            deviceFiles = _filterCreateReportTime(
+                cros.get('deviceFiles', []), 'createTime', startDate, endDate)
+            lenDF = len(deviceFiles)
+            if lenDF:
+                num_ranges = min(lenDF, listLimit or lenDF)
+                print('  deviceFiles')
+                for deviceFile in deviceFiles[:num_ranges]:
+                    device_type = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    print(f'    {device_type}: {create_time}')
+            if downloadfile:
+                deviceFiles = cros.get('deviceFiles', [])
+                lenDF = len(deviceFiles)
+                if lenDF:
+                    if downloadfile == 'latest':
+                        deviceFile = deviceFiles[-1]
+                    else:
+                        for deviceFile in deviceFiles:
+                            if deviceFile['createTime'] == downloadfile:
+                                break
+                        else:
+                            print(f'ERROR: file {downloadfile} not ' \
+                                  f'available to download.')
+                            deviceFile = None
+                    if deviceFile:
+                        created = deviceFile["createTime"]
+                        downloadfile = f'cros-logs-{deviceId}-{created}.zip'
+                        downloadfilename = os.path.join(targetFolder,
+                                                        downloadfile)
+                        dl_url = deviceFile['downloadUrl']
+                        _, content = cd._http.request(dl_url)
+                        fileutils.write_file(downloadfilename, content,
+                                             mode='wb',
+                                             continue_on_error=True)
+                        print(f'Downloaded: {downloadfilename}')
+                elif downloadfile:
+                    print('ERROR: no files to download.')
+            cpuStatusReports = _filterCreateReportTime(
+                cros.get('cpuStatusReports', []),
+                'reportTime',
+                startDate,
+                endDate)
+            lenCSR = len(cpuStatusReports)
+            if lenCSR:
+                print('  cpuStatusReports')
+                num_ranges = min(lenCSR, listLimit or lenCSR)
+                for cpuStatusReport in cpuStatusReports[:num_ranges]:
+                    print(f'    reportTime: {cpuStatusReport["reportTime"]}')
+                    print('      cpuTemperatureInfo')
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        temp_label = tempInfo['label'].strip()
+                        temperature = tempInfo['temperature']
+                        print(f'        {temp_label}: {temperature}')
+                    pct_info = cpuStatusReport["cpuUtilizationPercentageInfo"]
+                    util = ",".join([str(x) for x in pct_info])
+                    print(f'      cpuUtilizationPercentageInfo: {util}')
+            diskVolumeReports = cros.get('diskVolumeReports', [])
+            lenDVR = len(diskVolumeReports)
+            if lenDVR:
+                print('  diskVolumeReports')
+                print('    volumeInfo')
+                num_ranges = min(lenDVR, listLimit or lenDVR)
+                for diskVolumeReport in diskVolumeReports[:num_ranges]:
+                    volumeInfo = diskVolumeReport['volumeInfo']
+                    for volume in volumeInfo:
+                        vid = volume['volumeId']
+                        vstorage_free = volume['storageFree']
+                        vstorage_total = volume['storageTotal']
+                        print(f'      volumeId: {vid}')
+                        print(f'        storageFree: {vstorage_free}')
+                        print(f'        storageTotal: {vstorage_total}')
+            systemRamFreeReports = _filterCreateReportTime(
+                cros.get('systemRamFreeReports', []),
+                'reportTime', startDate, endDate)
+            lenSRFR = len(systemRamFreeReports)
+            if lenSRFR:
+                print('  systemRamFreeReports')
+                num_ranges = min(lenSRFR, listLimit or lenSRFR)
+                for systemRamFreeReport in systemRamFreeReports[:num_ranges]:
+                    report_time = systemRamFreeReport["reportTime"]
+                    free_info = systemRamFreeReport["systemRamFreeInfo"]
+                    free_ram = ",".join(free_info)
+                    print(f'    reportTime: {report_time}')
+                    print(f'      systemRamFreeInfo: {free_ram}')
+
+
+def doPrintCrosActivity():
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    titles = ['deviceId', 'annotatedAssetId',
+              'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    csvRows = []
+    fieldsList = ['deviceId', 'annotatedAssetId',
+                  'annotatedLocation', 'serialNumber', 'orgUnitPath']
+    startDate = endDate = None
+    selectActiveTimeRanges = selectDeviceFiles = selectRecentUsers = False
+    listLimit = 0
+    delimiter = ','
+    orgUnitPath = None
+    queries = [None]
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectActiveTimeRanges = True
+            i += 1
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectDeviceFiles = True
+            i += 1
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectRecentUsers = True
+            i += 1
+        elif myarg == 'both':
+            selectActiveTimeRanges = selectRecentUsers = True
+            i += 1
+        elif myarg == 'all':
+            selectActiveTimeRanges = selectDeviceFiles = True
+            selectRecentUsers = True
+            i += 1
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg == 'delimiter':
+            delimiter = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print crosactivity")
+    if not selectActiveTimeRanges and \
+       not selectDeviceFiles and \
+       not selectRecentUsers:
+        selectActiveTimeRanges = selectRecentUsers = True
+    if selectRecentUsers:
+        fieldsList.append('recentUsers')
+        display.add_titles_to_csv_file(['recentUsers.email', ], titles)
+    if selectActiveTimeRanges:
+        fieldsList.append('activeTimeRanges')
+        titles_to_add = ['activeTimeRanges.date',
+                         'activeTimeRanges.duration',
+                         'activeTimeRanges.minutes']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    if selectDeviceFiles:
+        fieldsList.append('deviceFiles')
+        titles_to_add = ['deviceFiles.type', 'deviceFiles.createTime']
+        display.add_titles_to_csv_file(titles_to_add, titles)
+    fields = f'nextPageToken,chromeosdevices({",".join(fieldsList)})'
+    for query in queries:
+        __main__.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                      'chromeosdevices',
+                                      page_message=page_message,
+                                      query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection='FULL',
+                                      fields=fields, orgUnitPath=orgUnitPath)
+        for cros in all_cros:
+            row = {}
+            skip_attribs = ['recentUsers', 'activeTimeRanges', 'deviceFiles']
+            for attrib in cros:
+                if attrib not in skip_attribs:
+                    row[attrib] = cros[attrib]
+            if selectActiveTimeRanges:
+                activeTimeRanges = _filterTimeRanges(
+                    cros.get('activeTimeRanges', []), startDate, endDate)
+                lenATR = len(activeTimeRanges)
+                num_ranges = min(lenATR, listLimit or lenATR)
+                for activeTimeRange in activeTimeRanges[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['activeTimeRanges.date'] = activeTimeRange['date']
+                    active_time = activeTimeRange['activeTime']
+                    newrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    newrow['activeTimeRanges.minutes'] = \
+                        activeTimeRange['activeTime']//60000
+                    csvRows.append(newrow)
+            if selectRecentUsers:
+                recentUsers = cros.get('recentUsers', [])
+                lenRU = len(recentUsers)
+                num_ranges = min(lenRU, listLimit or lenRU)
+                recent_users = []
+                for recentUser in recentUsers[:num_ranges]:
+                    useremail = recentUser.get("email")
+                    if not useremail:
+                        if recentUser["type"] == "USER_TYPE_UNMANAGED":
+                            useremail = 'UnmanagedUser'
+                        else:
+                            useremail = 'Unknown'
+                    recent_users.append(useremail)
+                row['recentUsers.email'] = delimiter.join(recent_users)
+                csvRows.append(row)
+            if selectDeviceFiles:
+                deviceFiles = _filterCreateReportTime(
+                    cros.get('deviceFiles', []),
+                    'createTime', startDate, endDate)
+                lenDF = len(deviceFiles)
+                num_ranges = min(lenDF, listLimit or lenDF)
+                for deviceFile in deviceFiles[:num_ranges]:
+                    newrow = row.copy()
+                    newrow['deviceFiles.type'] = deviceFile['type']
+                    create_time = deviceFile['createTime']
+                    newrow['deviceFiles.createTime'] = create_time
+                    csvRows.append(newrow)
+    display.write_csv_file(csvRows, titles, 'CrOS Activity', todrive)
+
+
+def _checkTPMVulnerability(cros):
+    if 'tpmVersionInfo' in cros and \
+       'firmwareVersion' in cros['tpmVersionInfo']:
+        firmware_version = cros['tpmVersionInfo']['firmwareVersion']
+        if firmware_version in CROS_TPM_VULN_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'VULNERABLE'
+        elif firmware_version in CROS_TPM_FIXED_VERSIONS:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'UPDATED'
+        else:
+            cros['tpmVersionInfo']['tpmVulnerability'] = 'NOT IMPACTED'
+
+
+def doPrintCrosDevices():
+    def _getSelectedLists(myarg):
+        if myarg in CROS_ACTIVE_TIME_RANGES_ARGUMENTS:
+            selectedLists['activeTimeRanges'] = True
+        elif myarg in CROS_RECENT_USERS_ARGUMENTS:
+            selectedLists['recentUsers'] = True
+        elif myarg in CROS_DEVICE_FILES_ARGUMENTS:
+            selectedLists['deviceFiles'] = True
+        elif myarg in CROS_CPU_STATUS_REPORTS_ARGUMENTS:
+            selectedLists['cpuStatusReports'] = True
+        elif myarg in CROS_DISK_VOLUME_REPORTS_ARGUMENTS:
+            selectedLists['diskVolumeReports'] = True
+        elif myarg in CROS_SYSTEM_RAM_FREE_REPORTS_ARGUMENTS:
+            selectedLists['systemRamFreeReports'] = True
+
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    display.add_field_to_csv_file(
+        'deviceid', CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList, fieldsTitles, titles)
+    projection = orderBy = sortOrder = orgUnitPath = None
+    queries = [None]
+    noLists = sortHeaders = False
+    selectedLists = {}
+    startDate = endDate = None
+    listLimit = 0
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ['query', 'queries']:
+            queries = __main__.getQueries(myarg, sys.argv[i+1])
+            i += 2
+        elif myarg == 'limittoou':
+            orgUnitPath = __main__.getOrgUnitItem(sys.argv[i+1])
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'nolists':
+            noLists = True
+            selectedLists = {}
+            i += 1
+        elif myarg == 'listlimit':
+            listLimit = __main__.getInteger(sys.argv[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg in CROS_START_ARGUMENTS:
+            startDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg in CROS_END_ARGUMENTS:
+            endDate = _getFilterDate(sys.argv[i+1])
+            i += 2
+        elif myarg == 'orderby':
+            orderBy = sys.argv[i+1].lower().replace('_', '')
+            validOrderBy = ['location', 'user', 'lastsync',
+                            'notes', 'serialnumber', 'status', 'supportenddate']
+            if orderBy not in validOrderBy:
+                controlflow.expected_argument_exit(
+                    "orderby", ", ".join(validOrderBy), orderBy)
+            if orderBy == 'location':
+                orderBy = 'annotatedLocation'
+            elif orderBy == 'user':
+                orderBy = 'annotatedUser'
+            elif orderBy == 'lastsync':
+                orderBy = 'lastSync'
+            elif orderBy == 'serialnumber':
+                orderBy = 'serialNumber'
+            elif orderBy == 'supportenddate':
+                orderBy = 'supportEndDate'
+            i += 2
+        elif myarg in SORTORDER_CHOICES_MAP:
+            sortOrder = SORTORDER_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            projection = PROJECTION_CHOICES_MAP[myarg]
+            sortHeaders = True
+            if projection == 'FULL':
+                fieldsList = []
+            else:
+                fieldsList = CROS_BASIC_FIELDS_LIST[:]
+            i += 1
+        elif myarg == 'allfields':
+            projection = 'FULL'
+            sortHeaders = True
+            fieldsList = []
+            i += 1
+        elif myarg == 'sortheaders':
+            sortHeaders = True
+            i += 1
+        elif myarg in CROS_LISTS_ARGUMENTS:
+            _getSelectedLists(myarg)
+            i += 1
+        elif myarg in CROS_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_fields_list(
+                myarg, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+            i += 1
+        elif myarg == 'fields':
+            fieldNameList = sys.argv[i+1]
+            for field in fieldNameList.lower().replace(',', ' ').split():
+                if field in CROS_LISTS_ARGUMENTS:
+                    _getSelectedLists(field)
+                elif field in CROS_ARGUMENT_TO_PROPERTY_MAP:
+                    display.add_field_to_fields_list(
+                        field, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+                else:
+                    controlflow.invalid_argument_exit(
+                        field, "gam print cros fields")
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam print cros")
+    if selectedLists:
+        noLists = False
+        projection = 'FULL'
+        for selectList in selectedLists:
+            display.add_field_to_fields_list(
+                selectList, CROS_ARGUMENT_TO_PROPERTY_MAP, fieldsList)
+    if fieldsList:
+        fieldsList.append('deviceId')
+        fields = f'nextPageToken,chromeosdevices({",".join(set(fieldsList))})'.replace(
+            '.', '/')
+    else:
+        fields = None
+    for query in queries:
+        __main__.printGettingAllItems('CrOS Devices', query)
+        page_message = gapi.got_total_items_msg('CrOS Devices', '...\n')
+        all_cros = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                      'chromeosdevices',
+                                      page_message=page_message, query=query,
+                                      customerId=GC_Values[GC_CUSTOMER_ID],
+                                      projection=projection,
+                                      orgUnitPath=orgUnitPath,
+                                      orderBy=orderBy, sortOrder=sortOrder,
+                                      fields=fields)
+        for cros in all_cros:
+            _checkTPMVulnerability(cros)
+        if not noLists and not selectedLists:
+            for cros in all_cros:
+                if 'notes' in cros:
+                    cros['notes'] = cros['notes'].replace('\n', '\\n')
+                if 'autoUpdateExpiration' in cros:
+                    cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                        cros['autoUpdateExpiration'])
+                for cpuStatusReport in cros.get('cpuStatusReports', []):
+                    tempInfos = cpuStatusReport.get('cpuTemperatureInfo', [])
+                    for tempInfo in tempInfos:
+                        tempInfo['label'] = tempInfo['label'].strip()
+                display.add_row_titles_to_csv_file(utils.flatten_json(
+                    cros, listLimit=listLimit), csvRows, titles)
+            continue
+        for cros in all_cros:
+            if 'notes' in cros:
+                cros['notes'] = cros['notes'].replace('\n', '\\n')
+            if 'autoUpdateExpiration' in cros:
+                cros['autoUpdateExpiration'] = utils.formatTimestampYMD(
+                    cros['autoUpdateExpiration'])
+            row = {}
+            for attrib in cros:
+                if attrib not in set(['kind', 'etag', 'tpmVersionInfo',
+                                      'recentUsers', 'activeTimeRanges',
+                                      'deviceFiles', 'cpuStatusReports',
+                                      'diskVolumeReports',
+                                      'systemRamFreeReports']):
+                    row[attrib] = cros[attrib]
+            if selectedLists.get('activeTimeRanges'):
+                timergs = cros.get('activeTimeRanges', [])
+            else:
+                timergs = []
+            activeTimeRanges = _filterTimeRanges(timergs, startDate, endDate)
+            if selectedLists.get('recentUsers'):
+                recentUsers = cros.get('recentUsers', [])
+            else:
+                recentUsers = []
+            if selectedLists.get('deviceFiles'):
+                device_files = cros.get('deviceFiles', [])
+            else:
+                device_files = []
+            deviceFiles = _filterCreateReportTime(device_files, 'createTime',
+                                                  startDate, endDate)
+            if selectedLists.get('cpuStatusReports'):
+                cpu_reports = cros.get('cpuStatusReports', [])
+            else:
+                cpu_reports = []
+            cpuStatusReports = _filterCreateReportTime(cpu_reports,
+                                                       'reportTime',
+                                                       startDate, endDate)
+            if selectedLists.get('diskVolumeReports'):
+                diskVolumeReports = cros.get('diskVolumeReports', [])
+            else:
+                diskVolumeReports = []
+            if selectedLists.get('systemRamFreeReports'):
+                ram_reports = cros.get('systemRamFreeReports', [])
+            else:
+                ram_reports = []
+            systemRamFreeReports = _filterCreateReportTime(ram_reports,
+                                                           'reportTime',
+                                                           startDate,
+                                                           endDate)
+            if noLists or (not activeTimeRanges and \
+                           not recentUsers and \
+                           not deviceFiles and \
+                           not cpuStatusReports and \
+                           not diskVolumeReports and \
+                           not systemRamFreeReports):
+                display.add_row_titles_to_csv_file(row, csvRows, titles)
+                continue
+            lenATR = len(activeTimeRanges)
+            lenRU = len(recentUsers)
+            lenDF = len(deviceFiles)
+            lenCSR = len(cpuStatusReports)
+            lenDVR = len(diskVolumeReports)
+            lenSRFR = len(systemRamFreeReports)
+            max_len = max(lenATR, lenRU, lenDF, lenCSR, lenDVR, lenSRFR)
+            for i in range(min(max_len, listLimit or max_len)):
+                nrow = row.copy()
+                if i < lenATR:
+                    nrow['activeTimeRanges.date'] = \
+                        activeTimeRanges[i]['date']
+                    nrow['activeTimeRanges.activeTime'] = \
+                        str(activeTimeRanges[i]['activeTime'])
+                    active_time = activeTimeRanges[i]['activeTime']
+                    nrow['activeTimeRanges.duration'] = \
+                        utils.formatMilliSeconds(active_time)
+                    nrow['activeTimeRanges.minutes'] = active_time // 60000
+                if i < lenRU:
+                    nrow['recentUsers.type'] = recentUsers[i]['type']
+                    nrow['recentUsers.email'] = recentUsers[i].get('email')
+                    if not nrow['recentUsers.email']:
+                        if nrow['recentUsers.type'] == 'USER_TYPE_UNMANAGED':
+                            nrow['recentUsers.email'] = 'UnmanagedUser'
+                        else:
+                            nrow['recentUsers.email'] = 'Unknown'
+                if i < lenDF:
+                    nrow['deviceFiles.type'] = deviceFiles[i]['type']
+                    nrow['deviceFiles.createTime'] = \
+                        deviceFiles[i]['createTime']
+                if i < lenCSR:
+                    nrow['cpuStatusReports.reportTime'] = \
+                        cpuStatusReports[i]['reportTime']
+                    tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo',
+                                                        [])
+                    for tempInfo in tempInfos:
+                        label = tempInfo["label"].strip()
+                        base = 'cpuStatusReports.cpuTemperatureInfo.'
+                        nrow[f'{base}{label}'] = tempInfo['temperature']
+                    cpu_field = 'cpuUtilizationPercentageInfo'
+                    cpu_reports = cpuStatusReports[i][cpu_field]
+                    cpu_pcts = [str(x) for x in cpu_reports]
+                    nrow[f'cpuStatusReports.{cpu_field}'] = ','.join(cpu_pcts)
+                if i < lenDVR:
+                    volumeInfo = diskVolumeReports[i]['volumeInfo']
+                    j = 0
+                    vfield = 'diskVolumeReports.volumeInfo.'
+                    for volume in volumeInfo:
+                        nrow[f'{vfield}{j}.volumeId'] = \
+                            volume['volumeId']
+                        nrow[f'{vfield}{j}.storageFree'] = \
+                            volume['storageFree']
+                        nrow[f'{vfield}{j}.storageTotal'] = \
+                            volume['storageTotal']
+                        j += 1
+                if i < lenSRFR:
+                    nrow['systemRamFreeReports.reportTime'] = \
+                        systemRamFreeReports[i]['reportTime']
+                    ram_reports = systemRamFreeReports[i]['systemRamFreeInfo']
+                    ram_info = [str(x) for x in ram_reports]
+                    nrow['systenRamFreeReports.systemRamFreeInfo'] = \
+                        ','.join(ram_info)
+                display.add_row_titles_to_csv_file(nrow, csvRows, titles)
+    if sortHeaders:
+        display.sort_csv_titles(['deviceId', ], titles)
+    display.write_csv_file(csvRows, titles, 'CrOS', todrive)
+
+
+def getCrOSDeviceEntity(i, cd):
+    myarg = sys.argv[i].lower()
+    if myarg == 'cros_sn':
+        return i+2, __main__.getUsersToModify('cros_sn', sys.argv[i+1])
+    if myarg == 'query':
+        return i+2, __main__.getUsersToModify('crosquery', sys.argv[i+1])
+    if myarg[:6] == 'query:':
+        query = sys.argv[i][6:]
+        if query[:12].lower() == 'orgunitpath:':
+            kwargs = {'orgUnitPath': query[12:]}
+        else:
+            kwargs = {'query': query}
+        fields = 'nextPageToken,chromeosdevices(deviceId)'
+        devices = gapi.get_all_pages(cd.chromeosdevices(), 'list',
+                                     'chromeosdevices',
+                                     customerId=GC_Values[GC_CUSTOMER_ID],
+                                     fields=fields, **kwargs)
+        return i+1, [device['deviceId'] for device in devices]
+    return i+1, sys.argv[i].replace(',', ' ').split()
+
+
+def _getFilterDate(dateStr):
+    return datetime.datetime.strptime(dateStr, YYYYMMDD_FORMAT)
+
+
+def _filterTimeRanges(activeTimeRanges, startDate, endDate):
+    if startDate is None and endDate is None:
+        return activeTimeRanges
+    filteredTimeRanges = []
+    for timeRange in activeTimeRanges:
+        activityDate = datetime.datetime.strptime(
+            timeRange['date'], YYYYMMDD_FORMAT)
+        if ((startDate is None) or \
+            (activityDate >= startDate)) and \
+           ((endDate is None) or \
+            (activityDate <= endDate)):
+            filteredTimeRanges.append(timeRange)
+    return filteredTimeRanges
+
+
+def _filterCreateReportTime(items, timeField, startTime, endTime):
+    if startTime is None and endTime is None:
+        return items
+    filteredItems = []
+    time_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    for item in items:
+        timeValue = datetime.datetime.strptime(item[timeField], time_format)
+        if ((startTime is None) or \
+            (timeValue >= startTime)) and \
+           ((endTime is None) or \
+            (timeValue <= endTime)):
+            filteredItems.append(item)
+    return filteredItems

src/gapi/directory/customer.py

@@ -0,0 +1,113 @@
+import datetime
+
+from var import *
+import controlflow
+import gapi
+import gapi.directory
+import gapi.reports
+
+
+def doGetCustomerInfo():
+    cd = gapi.directory.buildGAPIObject()
+    customer_info = gapi.call(cd.customers(), 'get',
+                              customerKey=GC_Values[GC_CUSTOMER_ID])
+    print(f'Customer ID: {customer_info["id"]}')
+    print(f'Primary Domain: {customer_info["customerDomain"]}')
+    result = gapi.call(cd.domains(), 'get', customer=customer_info['id'],
+                       domainName=customer_info['customerDomain'],
+                       fields='verified')
+    print(f'Primary Domain Verified: {result["verified"]}')
+    # If customer has changed primary domain customerCreationTime is date
+    # of current primary being added, not customer create date.
+    # We should also get all domains and use oldest date
+    customer_creation = customer_info['customerCreationTime']
+    date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    oldest = datetime.datetime.strptime(customer_creation, date_format)
+    domains = gapi.get_items(cd.domains(), 'list', 'domains',
+                             customer=GC_Values[GC_CUSTOMER_ID],
+                             fields='domains(creationTime)')
+    for domain in domains:
+        creation_timestamp = int(domain['creationTime'])/1000
+        domain_creation = datetime.datetime.fromtimestamp(creation_timestamp)
+        if domain_creation < oldest:
+            oldest = domain_creation
+    print(f'Customer Creation Time: {oldest.strftime(date_format)}')
+    customer_language = customer_info.get('language', 'Unset (defaults to en)')
+    print(f'Default Language: {customer_language}')
+    if 'postalAddress' in customer_info:
+        print('Address:')
+        for field in ADDRESS_FIELDS_PRINT_ORDER:
+            if field in customer_info['postalAddress']:
+                print(f' {field}: {customer_info["postalAddress"][field]}')
+    if 'phoneNumber' in customer_info:
+        print(f'Phone: {customer_info["phoneNumber"]}')
+    print(f'Admin Secondary Email: {customer_info["alternateEmail"]}')
+    user_counts_map = {
+        'accounts:num_users': 'Total Users',
+        'accounts:gsuite_basic_total_licenses': 'G Suite Basic Licenses',
+        'accounts:gsuite_basic_used_licenses': 'G Suite Basic Users',
+        'accounts:gsuite_enterprise_total_licenses': 'G Suite Enterprise ' \
+        'Licenses',
+        'accounts:gsuite_enterprise_used_licenses': 'G Suite Enterprise ' \
+        'Users',
+        'accounts:gsuite_unlimited_total_licenses': 'G Suite Business ' \
+        'Licenses',
+        'accounts:gsuite_unlimited_used_licenses': 'G Suite Business Users'
+    }
+    parameters = ','.join(list(user_counts_map))
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    rep = gapi.reports.buildGAPIObject()
+    usage = None
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    while True:
+        try:
+            usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                                       'usageReports',
+                                       throw_reasons=throw_reasons,
+                                       customerId=customerId, date=tryDate,
+                                       parameters=parameters)
+            break
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = gapi.reports._adjust_date(str(e))
+    if not usage:
+        print('No user count data available.')
+        return
+    print(f'User counts as of {tryDate}:')
+    for item in usage[0]['parameters']:
+        api_name = user_counts_map.get(item['name'])
+        api_value = int(item.get('intValue', 0))
+        if api_name and api_value:
+            print(f'  {api_name}: {api_value:,}')
+
+
+def doUpdateCustomer():
+    cd = gapi.directory.buildGAPIObject()
+    body = {}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg in ADDRESS_FIELDS_ARGUMENT_MAP:
+            body.setdefault('postalAddress', {})
+            arg = ADDRESS_FIELDS_ARGUMENT_MAP[myarg]
+            body['postalAddress'][arg] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['adminsecondaryemail', 'alternateemail']:
+            body['alternateEmail'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['phone', 'phonenumber']:
+            body['phoneNumber'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'language':
+            body['language'] = sys.argv[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam update customer")
+    if not body:
+        controlflow.system_error_exit(2, 'no arguments specified for "gam '
+                                         'update customer"')
+    gapi.call(cd.customers(), 'patch', customerKey=GC_Values[GC_CUSTOMER_ID],
+              body=body)
+    print('Updated customer')

src/gapi/directory/resource.py

@@ -0,0 +1,487 @@
+import sys
+import uuid
+
+import __main__
+from var import *
+import controlflow
+import display
+import gapi.directory
+import utils
+
+
+def printBuildings():
+    to_drive = False
+    cd = gapi.directory.buildGAPIObject()
+    titles = []
+    csvRows = []
+    fieldsList = ['buildingId']
+    # buildings.list() currently doesn't support paging
+    # but should soon, attempt to use it now so we
+    # won't break when it's turned on.
+    fields = 'nextPageToken,buildings(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Building']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        # Allows shorter arguments like "name" instead of "buildingname"
+        elif 'building'+myarg in possible_fields:
+            fieldsList.append(possible_fields['building'+myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print buildings")
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    for building in buildings:
+        building.pop('etags', None)
+        building.pop('etag', None)
+        building.pop('kind', None)
+        if 'buildingId' in building:
+            building['buildingId'] = f'id:{building["buildingId"]}'
+        if 'floorNames' in building:
+            building['floorNames'] = ','.join(building['floorNames'])
+        building = utils.flatten_json(building)
+        for item in building:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(building)
+    display.sort_csv_titles('buildingId', titles)
+    display.write_csv_file(csvRows, titles, 'Buildings', to_drive)
+
+
+def printResourceCalendars():
+    cd = gapi.directory.buildGAPIObject()
+    todrive = False
+    fieldsList = []
+    fieldsTitles = {}
+    titles = []
+    csvRows = []
+    query = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg == 'allfields':
+            fieldsList = []
+            fieldsTitles = {}
+            titles = []
+            for field in RESCAL_ALLFIELDS:
+                display.add_field_to_csv_file(field,
+                                              RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                              fieldsList, fieldsTitles,
+                                              titles)
+            i += 1
+        elif myarg in RESCAL_ARGUMENT_TO_PROPERTY_MAP:
+            display.add_field_to_csv_file(myarg,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print resources")
+    if not fieldsList:
+        for field in RESCAL_DFLTFIELDS:
+            display.add_field_to_csv_file(field,
+                                          RESCAL_ARGUMENT_TO_PROPERTY_MAP,
+                                          fieldsList, fieldsTitles, titles)
+    fields = f'nextPageToken,items({",".join(set(fieldsList))})'
+    if 'buildingId' in fieldsList:
+        display.add_field_to_csv_file('buildingName', {'buildingName': [
+            'buildingName', ]}, fieldsList, fieldsTitles, titles)
+    __main__.printGettingAllItems('Resource Calendars', None)
+    page_message = gapi.got_total_items_first_last_msg('Resource Calendars')
+    resources = gapi.get_all_pages(cd.resources().calendars(), 'list',
+                                   'items', page_message=page_message,
+                                   message_attribute='resourceId',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   query=query, fields=fields)
+    for resource in resources:
+        if 'featureInstances' in resource:
+            features = [a_feature['feature']['name'] for \
+                a_feature in resource['featureInstances']]
+            resource['featureInstances'] = ','.join(features)
+        if 'buildingId' in resource:
+            resource['buildingName'] = getBuildingNameById(
+                cd, resource['buildingId'])
+            resource['buildingId'] = f'id:{resource["buildingId"]}'
+        resUnit = {}
+        for field in fieldsList:
+            resUnit[fieldsTitles[field]] = resource.get(field, '')
+        csvRows.append(resUnit)
+    display.sort_csv_titles(
+        ['resourceId', 'resourceName', 'resourceEmail'], titles)
+    display.write_csv_file(csvRows, titles, 'Resources', todrive)
+
+
+RESCAL_DFLTFIELDS = ['id', 'name', 'email',]
+RESCAL_ALLFIELDS = ['id', 'name', 'email', 'description', 'type',
+                    'buildingid', 'category', 'capacity', 'features', 'floor',
+                    'floorsection', 'generatedresourcename',
+                    'uservisibledescription',]
+
+RESCAL_ARGUMENT_TO_PROPERTY_MAP = {
+    'description': ['resourceDescription'],
+    'building': ['buildingId', ],
+    'buildingid': ['buildingId', ],
+    'capacity': ['capacity', ],
+    'category': ['resourceCategory', ],
+    'email': ['resourceEmail'],
+    'feature': ['featureInstances', ],
+    'features': ['featureInstances', ],
+    'floor': ['floorName', ],
+    'floorname': ['floorName', ],
+    'floorsection': ['floorSection', ],
+    'generatedresourcename': ['generatedResourceName', ],
+    'id': ['resourceId'],
+    'name': ['resourceName'],
+    'type': ['resourceType'],
+    'userdescription': ['userVisibleDescription', ],
+    'uservisibledescription': ['userVisibleDescription', ],
+}
+
+
+def printFeatures():
+    to_drive = False
+    cd = gapi.directory.buildGAPIObject()
+    titles = []
+    csvRows = []
+    fieldsList = ['name']
+    fields = 'nextPageToken,features(%s)'
+    possible_fields = {}
+    for pfield in cd._rootDesc['schemas']['Feature']['properties']:
+        possible_fields[pfield.lower()] = pfield
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'todrive':
+            to_drive = True
+            i += 1
+        elif myarg == 'allfields':
+            fields = None
+            i += 1
+        elif myarg in possible_fields:
+            fieldsList.append(possible_fields[myarg])
+            i += 1
+        elif 'feature'+myarg in possible_fields:
+            fieldsList.append(possible_fields['feature'+myarg])
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam print features")
+    if fields:
+        fields = fields % ','.join(fieldsList)
+    features = gapi.get_all_pages(cd.resources().features(), 'list',
+                                  'features',
+                                  customer=GC_Values[GC_CUSTOMER_ID],
+                                  fields=fields)
+    for feature in features:
+        feature.pop('etags', None)
+        feature.pop('etag', None)
+        feature.pop('kind', None)
+        feature = utils.flatten_json(feature)
+        for item in feature:
+            if item not in titles:
+                titles.append(item)
+        csvRows.append(feature)
+    display.sort_csv_titles('name', titles)
+    display.write_csv_file(csvRows, titles, 'Features', to_drive)
+
+
+def _getBuildingAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'id':
+            body['buildingId'] = args[i+1]
+            i += 2
+        elif myarg == 'name':
+            body['buildingName'] = args[i+1]
+            i += 2
+        elif myarg in ['lat', 'latitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['latitude'] = args[i+1]
+            i += 2
+        elif myarg in ['long', 'lng', 'longitude']:
+            if 'coordinates' not in body:
+                body['coordinates'] = {}
+            body['coordinates']['longitude'] = args[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = args[i+1]
+            i += 2
+        elif myarg == 'floors':
+            body['floorNames'] = args[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, "gam create|update building")
+    return body
+
+
+def createBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    body = {'floorNames': ['1'],
+            'buildingId': str(uuid.uuid4()),
+            'buildingName': sys.argv[3]}
+    body = _getBuildingAttributes(sys.argv[4:], body)
+    print(f'Creating building {body["buildingId"]}...')
+    gapi.call(cd.resources().buildings(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def _makeBuildingIdNameMap(cd):
+    fields = 'nextPageToken,buildings(buildingId,buildingName)'
+    buildings = gapi.get_all_pages(cd.resources().buildings(), 'list',
+                                   'buildings',
+                                   customer=GC_Values[GC_CUSTOMER_ID],
+                                   fields=fields)
+    GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] = {}
+    GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] = {}
+    for building in buildings:
+        GM_Globals[GM_MAP_BUILDING_ID_TO_NAME][building['buildingId']
+                                               ] = building['buildingName']
+        GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][building['buildingName']
+                                               ] = building['buildingId']
+
+
+def getBuildingByNameOrId(cd, which_building, minLen=1):
+    if not which_building or \
+       (minLen == 0 and which_building in ['id:', 'uid:']):
+        if minLen == 0:
+            return ''
+        controlflow.system_error_exit(3, 'Building id/name is empty')
+    cg = UID_PATTERN.match(which_building)
+    if cg:
+        return cg.group(1)
+    if GM_Globals[GM_MAP_BUILDING_NAME_TO_ID] is None:
+        _makeBuildingIdNameMap(cd)
+    # Exact name match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID]:
+        return GM_Globals[GM_MAP_BUILDING_NAME_TO_ID][which_building]
+    # No exact name match, check for case insensitive name matches
+    which_building_lower = which_building.lower()
+    ci_matches = []
+    for buildingName, buildingId in GM_Globals[GM_MAP_BUILDING_NAME_TO_ID].items():
+        if buildingName.lower() == which_building_lower:
+            ci_matches.append(
+                {'buildingName': buildingName, 'buildingId': buildingId})
+    # One match, return ID
+    if len(ci_matches) == 1:
+        return ci_matches[0]['buildingId']
+    # No or multiple name matches, try ID
+    # Exact ID match, return ID
+    if which_building in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        return which_building
+    # No exact ID match, check for case insensitive id match
+    for buildingId in GM_Globals[GM_MAP_BUILDING_ID_TO_NAME]:
+        # Match, return ID
+        if buildingId.lower() == which_building_lower:
+            return buildingId
+    # Multiple name  matches
+    if len(ci_matches) > 1:
+        message = 'Multiple buildings with same name:\n'
+        for building in ci_matches:
+            message += f'  Name:{building["buildingName"]}  ' \
+                       f'id:{building["buildingId"]}\n'
+        message += '\nPlease specify building name by exact case or by id.'
+        controlflow.system_error_exit(3, message)
+    # No matches
+    else:
+        controlflow.system_error_exit(3, f'No such building {which_building}')
+
+
+def getBuildingNameById(cd, buildingId):
+    if GM_Globals[GM_MAP_BUILDING_ID_TO_NAME] is None:
+        _makeBuildingIdNameMap(cd)
+    return GM_Globals[GM_MAP_BUILDING_ID_TO_NAME].get(buildingId, 'UNKNOWN')
+
+
+def updateBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    body = _getBuildingAttributes(sys.argv[4:])
+    print(f'Updating building {buildingId}...')
+    gapi.call(cd.resources().buildings(), 'patch',
+              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId,
+              body=body)
+
+
+def getBuildingInfo():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    building = gapi.call(cd.resources().buildings(), 'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         buildingId=buildingId)
+    if 'buildingId' in building:
+        building['buildingId'] = f'id:{building["buildingId"]}'
+    if 'floorNames' in building:
+        building['floorNames'] = ','.join(building['floorNames'])
+    if 'buildingName' in building:
+        sys.stdout.write(building.pop('buildingName'))
+    display.print_json(building)
+
+
+def deleteBuilding():
+    cd = gapi.directory.buildGAPIObject()
+    buildingId = getBuildingByNameOrId(cd, sys.argv[3])
+    print(f'Deleting building {buildingId}...')
+    gapi.call(cd.resources().buildings(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], buildingId=buildingId)
+
+
+def _getFeatureAttributes(args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = args[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                myarg, "gam create|update feature")
+    return body
+
+
+def createFeature():
+    cd = gapi.directory.buildGAPIObject()
+    body = _getFeatureAttributes(sys.argv[3:])
+    print(f'Creating feature {body["name"]}...')
+    gapi.call(cd.resources().features(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def updateFeature():
+    # update does not work for name and name is only field to be updated
+    # if additional writable fields are added to feature in the future
+    # we'll add support for update as well as rename
+    cd = gapi.directory.buildGAPIObject()
+    oldName = sys.argv[3]
+    body = {'newName': sys.argv[5:]}
+    print(f'Updating feature {oldName}...')
+    gapi.call(cd.resources().features(), 'rename',
+              customer=GC_Values[GC_CUSTOMER_ID], oldName=oldName,
+              body=body)
+
+
+def deleteFeature():
+    cd = gapi.directory.buildGAPIObject()
+    featureKey = sys.argv[3]
+    print(f'Deleting feature {featureKey}...')
+    gapi.call(cd.resources().features(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], featureKey=featureKey)
+
+
+def _getResourceCalendarAttributes(cd, args, body={}):
+    i = 0
+    while i < len(args):
+        myarg = args[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['resourceName'] = args[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['resourceDescription'] = args[i+1].replace('\\n', '\n')
+            i += 2
+        elif myarg == 'type':
+            body['resourceType'] = args[i+1]
+            i += 2
+        elif myarg in ['building', 'buildingid']:
+            body['buildingId'] = getBuildingByNameOrId(
+                cd, args[i+1], minLen=0)
+            i += 2
+        elif myarg in ['capacity']:
+            body['capacity'] = __main__.getInteger(args[i+1], myarg, minVal=0)
+            i += 2
+        elif myarg in ['feature', 'features']:
+            features = args[i+1].split(',')
+            body['featureInstances'] = []
+            for feature in features:
+                instance = {'feature': {'name': feature}}
+                body['featureInstances'].append(instance)
+            i += 2
+        elif myarg in ['floor', 'floorname']:
+            body['floorName'] = args[i+1]
+            i += 2
+        elif myarg in ['floorsection']:
+            body['floorSection'] = args[i+1]
+            i += 2
+        elif myarg in ['category']:
+            body['resourceCategory'] = args[i+1].upper()
+            if body['resourceCategory'] == 'ROOM':
+                body['resourceCategory'] = 'CONFERENCE_ROOM'
+            i += 2
+        elif myarg in ['uservisibledescription', 'userdescription']:
+            body['userVisibleDescription'] = args[i+1]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(
+                args[i], "gam create|update resource")
+    return body
+
+
+def createResourceCalendar():
+    cd = gapi.directory.buildGAPIObject()
+    body = {'resourceId': sys.argv[3],
+            'resourceName': sys.argv[4]}
+    body = _getResourceCalendarAttributes(cd, sys.argv[5:], body)
+    print(f'Creating resource {body["resourceId"]}...')
+    gapi.call(cd.resources().calendars(), 'insert',
+              customer=GC_Values[GC_CUSTOMER_ID], body=body)
+
+
+def updateResourceCalendar():
+    cd = gapi.directory.buildGAPIObject()
+    resId = sys.argv[3]
+    body = _getResourceCalendarAttributes(cd, sys.argv[4:])
+    # Use patch since it seems to work better.
+    # update requires name to be set.
+    gapi.call(cd.resources().calendars(), 'patch',
+              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId,
+              body=body, fields='')
+    print(f'updated resource {resId}')
+
+
+def getResourceCalendarInfo():
+    cd = gapi.directory.buildGAPIObject()
+    resId = sys.argv[3]
+    resource = gapi.call(cd.resources().calendars(), 'get',
+                         customer=GC_Values[GC_CUSTOMER_ID],
+                         calendarResourceId=resId)
+    if 'featureInstances' in resource:
+        features = []
+        for a_feature in resource.pop('featureInstances'):
+            features.append(a_feature['feature']['name'])
+        resource['features'] = ', '.join(features)
+    if 'buildingId' in resource:
+        resource['buildingName'] = getBuildingNameById(
+            cd, resource['buildingId'])
+        resource['buildingId'] = f'id:{resource["buildingId"]}'
+    display.print_json(resource)
+
+
+def deleteResourceCalendar():
+    resId = sys.argv[3]
+    cd = gapi.directory.buildGAPIObject()
+    print(f'Deleting resource calendar {resId}')
+    gapi.call(cd.resources().calendars(), 'delete',
+              customer=GC_Values[GC_CUSTOMER_ID], calendarResourceId=resId)

src/gapi/errors.py

@@ -115,6 +115,7 @@ class ErrorReason(Enum):
   DUPLICATE = 'duplicate'
   FAILED_PRECONDITION = 'failedPrecondition'
   FORBIDDEN = 'forbidden'
+  FOUR_O_THREE = '403'
   GATEWAY_TIMEOUT = 'gatewayTimeout'
   GROUP_NOT_FOUND = 'groupNotFound'
   INTERNAL_ERROR = 'internalError'
@@ -129,9 +130,12 @@ class ErrorReason(Enum):
   RATE_LIMIT_EXCEEDED = 'rateLimitExceeded'
   RESOURCE_NOT_FOUND = 'resourceNotFound'
   SERVICE_NOT_AVAILABLE = 'serviceNotAvailable'
+  SERVICE_LIMIT = 'serviceLimit'
   SYSTEM_ERROR = 'systemError'
   USER_NOT_FOUND = 'userNotFound'
   USER_RATE_LIMIT_EXCEEDED = 'userRateLimitExceeded'
+  FOUR_TWO_NINE = '429'
+  DAILY_LIMIT_EXCEEDED = 'dailyLimitExceeded'
 
   def __str__(self):
     return str(self.value)
@@ -142,7 +146,7 @@ DEFAULT_RETRY_REASONS = [
   ErrorReason.QUOTA_EXCEEDED, ErrorReason.RATE_LIMIT_EXCEEDED,
   ErrorReason.USER_RATE_LIMIT_EXCEEDED, ErrorReason.BACKEND_ERROR,
   ErrorReason.BAD_GATEWAY, ErrorReason.GATEWAY_TIMEOUT,
-  ErrorReason.INTERNAL_ERROR
+  ErrorReason.INTERNAL_ERROR, ErrorReason.FOUR_TWO_NINE,
   ]
 GMAIL_THROW_REASONS = [ErrorReason.SERVICE_NOT_AVAILABLE]
 GROUP_GET_THROW_REASONS = [
@@ -354,5 +358,5 @@ def get_gapi_error_detail(e,
       if 'Cyclic memberships not allowed' in message:
         reason = ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value
   except KeyError:
-    reason = '{0}'.format(http_status)
+    reason = f'{http_status}'
   return (http_status, reason, message)

src/gapi/reports.py

@@ -0,0 +1,550 @@
+import calendar
+import datetime
+import re
+import sys
+
+from dateutil.relativedelta import relativedelta
+
+import __main__
+from var import *
+import controlflow
+import display
+import gapi
+import utils
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('reports')
+
+
+REPORT_CHOICE_MAP = {
+    'access': 'access_transparency',
+    'accesstransparency': 'access_transparency',
+    'calendars': 'calendar',
+    'customers': 'customer',
+    'doc': 'drive',
+    'docs': 'drive',
+    'domain': 'customer',
+    'enterprisegroups': 'groups_enterprise',
+    'google+': 'gplus',
+    'group': 'groups',
+    'groupsenterprise': 'groups_enterprise',
+    'hangoutsmeet': 'meet',
+    'logins': 'login',
+    'oauthtoken': 'token',
+    'tokens': 'token',
+    'usage': 'usage',
+    'usageparameters': 'usageparameters',
+    'users': 'user',
+    'useraccounts': 'user_accounts',
+}
+
+
+def showUsageParameters():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID,
+                     gapi.errors.ErrorReason.BAD_REQUEST]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit(
+            'user or customer', 'report usageparameters')
+    report = sys.argv[3].lower()
+    titles = ['parameter']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = {}
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = {'userKey': __main__._getValueFromOAuth('email')}
+    else:
+        controlflow.expected_argument_exit(
+            'usageparameters', ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    partial_apps = []
+    all_parameters = []
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam report usageparameters")
+    while True:
+        try:
+            response = gapi.call(endpoint, 'get',
+                                 throw_reasons=throw_reasons,
+                                 date=tryDate,
+                                 customerId=customerId,
+                                 **kwargs)
+            partial_on_thisday = []
+            for warning in response.get('warnings', []):
+                for data in warning.get('data', []):
+                    if data.get('key') == 'application':
+                        partial_on_thisday.append(data['value'])
+            if partial_apps:
+                partial_apps = [app for app in partial_apps if app in partial_on_thisday]
+            else:
+                partial_apps = partial_on_thisday
+            for parameter in response['usageReports'][0]['parameters']:
+                name = parameter.get('name')
+                if name and name not in all_parameters:
+                    all_parameters.append(name)
+            if not partial_apps:
+                break
+            tryDate = (utils.get_yyyymmdd(tryDate, returnDateTime=True) - \
+                    one_day).strftime(YYYYMMDD_FORMAT)
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = _adjust_date(str(e))
+    all_parameters.sort()
+    csvRows = []
+    for parameter in all_parameters:
+        csvRows.append({'parameter': parameter})
+    display.write_csv_file(
+        csvRows, titles, f'{report.capitalize()} Report Usage Parameters', todrive)
+
+REPORTS_PARAMETERS_SIMPLE_TYPES = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
+
+def showUsage():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID,
+                     gapi.errors.ErrorReason.BAD_REQUEST]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit(
+            'user or customer', 'report usage')
+    report = sys.argv[3].lower()
+    titles = ['date']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = [{}]
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = [{'userKey': 'all'}]
+        titles.append('user')
+    else:
+        controlflow.expected_argument_exit(
+            'usage', ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    parameters = []
+    start_date = end_date = orgUnitId = None
+    skip_day_numbers = []
+    skip_dates = set()
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'startdate':
+            start_date = utils.get_yyyymmdd(sys.argv[i+1], returnDateTime=True)
+            i += 2
+        elif myarg == 'enddate':
+            end_date = utils.get_yyyymmdd(sys.argv[i+1], returnDateTime=True)
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg == 'skipdates':
+            for skip in sys.argv[i+1].split(','):
+                if skip.find(':') == -1:
+                    skip_dates.add(utils.get_yyyymmdd(skip, returnDateTime=True))
+                else:
+                    skip_start, skip_end = skip.split(':', 1)
+                    skip_start = utils.get_yyyymmdd(skip_start, returnDateTime=True)
+                    skip_end = utils.get_yyyymmdd(skip_end, returnDateTime=True)
+                    while skip_start <= skip_end:
+                        skip_dates.add(skip_start)
+                        skip_start += one_day
+            i += 2
+        elif myarg == 'skipdaysofweek':
+            skipdaynames = sys.argv[i+1].split(',')
+            dow = [d.lower() for d in calendar.day_abbr]
+            skip_day_numbers = [dow.index(d) for d in skipdaynames if d in dow]
+            i += 2
+        elif report == 'user' and myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            i += 2
+        elif report == 'user' and myarg in usergroup_types:
+            users = __main__.getUsersToModify(myarg, sys.argv[i+1])
+            kwargs = [{'userKey': user} for user in users]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], f'gam report usage {report}')
+    if parameters:
+        titles.extend(parameters)
+        parameters = ','.join(parameters)
+    else:
+        parameters = None
+    if not end_date:
+        end_date = datetime.datetime.now()
+    if not start_date:
+        start_date = end_date + relativedelta(months=-1)
+    if orgUnitId:
+        for kw in kwargs:
+            kw['orgUnitID'] = orgUnitId
+    usage_on_date = start_date
+    start_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+    usage_end_date = end_date
+    end_date = end_date.strftime(YYYYMMDD_FORMAT)
+    start_use_date = end_use_date = None
+    csvRows = []
+    while usage_on_date <= usage_end_date:
+        if usage_on_date.weekday() in skip_day_numbers or \
+           usage_on_date in skip_dates:
+            usage_on_date += one_day
+            continue
+        use_date = usage_on_date.strftime(YYYYMMDD_FORMAT)
+        usage_on_date += one_day
+        try:
+            for kwarg in kwargs:
+                try:
+                    usage = gapi.get_all_pages(endpoint, 'get',
+                                               'usageReports',
+                                               throw_reasons=throw_reasons,
+                                               customerId=customerId,
+                                               date=use_date,
+                                               parameters=parameters,
+                                               **kwarg)
+                except gapi.errors.GapiBadRequestError:
+                    continue
+                for entity in usage:
+                    row = {'date': use_date}
+                    if 'userEmail' in entity['entity']:
+                        row['user'] = entity['entity']['userEmail']
+                    for item in entity.get('parameters', []):
+                        if 'name' not in item:
+                            continue
+                        name = item['name']
+                        if name == 'cros:device_version_distribution':
+                            for cros_ver in item['msgValue']:
+                                v = cros_ver['version_number']
+                                column_name = f'cros:num_devices_chrome_{v}'
+                                if column_name not in titles:
+                                    titles.append(column_name)
+                                row[column_name] = cros_ver['num_devices']
+                        else:
+                            if not name in titles:
+                                titles.append(name)
+                            for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                                if ptype in item:
+                                    row[name] = item[ptype]
+                                    break
+                            else:
+                                row[name] = ''
+                    if not start_use_date:
+                        start_use_date = use_date
+                    end_use_date = use_date
+                    csvRows.append(row)
+        except gapi.errors.GapiInvalidError as e:
+            display.print_warning(str(e))
+            break
+    if start_use_date:
+        report_name = f'{report.capitalize()} Usage Report - {start_use_date}:{end_use_date}'
+    else:
+        report_name = f'{report.capitalize()} Usage Report - {start_date}:{end_date} - No Data'
+    display.write_csv_file(
+        csvRows, titles, report_name, todrive)
+
+
+def showReport():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID]
+    report = sys.argv[2].lower()
+    report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    if report == 'usage':
+      showUsage()
+      return
+    if report == 'usageparameters':
+      showUsageParameters()
+      return
+    valid_apps = gapi.get_enum_values_minus_unspecified(
+        rep._rootDesc['resources']['activities']['methods']['list'][
+            'parameters']['applicationName']['enum'])+['customer', 'user']
+    if report not in valid_apps:
+        controlflow.expected_argument_exit(
+            "report", ", ".join(sorted(valid_apps)), report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    filters = parameters = actorIpAddress = startTime = endTime = eventName = orgUnitId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    to_drive = False
+    userKey = 'all'
+    fullDataRequired = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower()
+        if myarg == 'date':
+            tryDate = utils.get_yyyymmdd(sys.argv[i+1])
+            i += 2
+        elif myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            i += 2
+        elif myarg == 'fulldatarequired':
+            fullDataRequired = []
+            fdr = sys.argv[i+1].lower()
+            if fdr and fdr != 'all':
+                fullDataRequired = fdr.replace(',', ' ').split()
+            i += 2
+        elif myarg == 'start':
+            startTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'end':
+            endTime = utils.get_time_or_delta_from_now(sys.argv[i+1])
+            i += 2
+        elif myarg == 'event':
+            eventName = sys.argv[i+1]
+            i += 2
+        elif myarg == 'user':
+            userKey = __main__.normalizeEmailAddressOrUID(sys.argv[i+1])
+            i += 2
+        elif myarg in ['filter', 'filters']:
+            filters = sys.argv[i+1]
+            i += 2
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i+1]
+            i += 2
+        elif myarg == 'ip':
+            actorIpAddress = sys.argv[i+1]
+            i += 2
+        elif myarg == 'todrive':
+            to_drive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam report")
+    if report == 'user':
+        while True:
+            try:
+                if fullDataRequired is not None:
+                    warnings = gapi.get_items(rep.userUsageReport(), 'get',
+                                              'warnings',
+                                              throw_reasons=throw_reasons,
+                                              date=tryDate, userKey=userKey,
+                                              customerId=customerId,
+                                              orgUnitID=orgUnitId,
+                                              fields='warnings')
+                    fullData, tryDate = _check_full_data_available(
+                        warnings, tryDate, fullDataRequired)
+                    if fullData < 0:
+                        print('No user report available.')
+                        sys.exit(1)
+                    if fullData == 0:
+                        continue
+                page_message = gapi.got_total_items_msg('Users', '...\n')
+                usage = gapi.get_all_pages(rep.userUsageReport(), 'get',
+                                           'usageReports',
+                                           page_message=page_message,
+                                           throw_reasons=throw_reasons,
+                                           date=tryDate, userKey=userKey,
+                                           customerId=customerId,
+                                           orgUnitID=orgUnitId,
+                                           filters=filters,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No user report available.')
+            sys.exit(1)
+        titles = ['email', 'date']
+        csvRows = []
+        for user_report in usage:
+            if 'entity' not in user_report:
+                continue
+            row = {'email': user_report['entity']
+                            ['userEmail'], 'date': tryDate}
+            for item in user_report.get('parameters', []):
+                if 'name' not in item:
+                    continue
+                name = item['name']
+                if not name in titles:
+                    titles.append(name)
+                for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                    if ptype in item:
+                        row[name] = item[ptype]
+                        break
+                else:
+                    row[name] = ''
+            csvRows.append(row)
+        display.write_csv_file(
+            csvRows, titles, f'User Reports - {tryDate}', to_drive)
+    elif report == 'customer':
+        while True:
+            try:
+                if fullDataRequired is not None:
+                    warnings = gapi.get_items(rep.customerUsageReports(),
+                                              'get', 'warnings',
+                                              throw_reasons=throw_reasons,
+                                              customerId=customerId,
+                                              date=tryDate,
+                                              fields='warnings')
+                    fullData, tryDate = _check_full_data_available(
+                        warnings, tryDate, fullDataRequired)
+                    if fullData < 0:
+                        print('No customer report available.')
+                        sys.exit(1)
+                    if fullData == 0:
+                        continue
+                usage = gapi.get_all_pages(rep.customerUsageReports(), 'get',
+                                           'usageReports',
+                                           throw_reasons=throw_reasons,
+                                           customerId=customerId,
+                                           date=tryDate,
+                                           parameters=parameters)
+                break
+            except gapi.errors.GapiInvalidError as e:
+                tryDate = _adjust_date(str(e))
+        if not usage:
+            print('No customer report available.')
+            sys.exit(1)
+        titles = ['name', 'value', 'client_id']
+        csvRows = []
+        auth_apps = list()
+        for item in usage[0]['parameters']:
+            if 'name' not in item:
+                continue
+            name = item['name']
+            if 'intValue' in item:
+                value = item['intValue']
+            elif 'msgValue' in item:
+                if name == 'accounts:authorized_apps':
+                    for subitem in item['msgValue']:
+                        app = {}
+                        for an_item in subitem:
+                            if an_item == 'client_name':
+                                app['name'] = 'App: ' + \
+                                    subitem[an_item].replace('\n', '\\n')
+                            elif an_item == 'num_users':
+                                app['value'] = f'{subitem[an_item]} users'
+                            elif an_item == 'client_id':
+                                app['client_id'] = subitem[an_item]
+                        auth_apps.append(app)
+                    continue
+                values = []
+                for subitem in item['msgValue']:
+                    if 'count' in subitem:
+                        mycount = myvalue = None
+                        for key, value in list(subitem.items()):
+                            if key == 'count':
+                                mycount = value
+                            else:
+                                myvalue = value
+                            if mycount and myvalue:
+                                values.append(f'{myvalue}:{mycount}')
+                        value = ' '.join(values)
+                    elif 'version_number' in subitem \
+                         and 'num_devices' in subitem:
+                        values.append(
+                            f'{subitem["version_number"]}:'
+                            f'{subitem["num_devices"]}')
+                    else:
+                        continue
+                    value = ' '.join(sorted(values, reverse=True))
+            csvRows.append({'name': name, 'value': value})
+        for app in auth_apps:  # put apps at bottom
+            csvRows.append(app)
+        display.write_csv_file(
+            csvRows, titles, f'Customer Report - {tryDate}', todrive=to_drive)
+    else:
+        page_message = gapi.got_total_items_msg('Activities', '...\n')
+        activities = gapi.get_all_pages(rep.activities(), 'list', 'items',
+                                        page_message=page_message,
+                                        applicationName=report,
+                                        userKey=userKey,
+                                        customerId=customerId,
+                                        actorIpAddress=actorIpAddress,
+                                        startTime=startTime, endTime=endTime,
+                                        eventName=eventName, filters=filters,
+                                        orgUnitID=orgUnitId)
+        if activities:
+            titles = ['name']
+            csvRows = []
+            for activity in activities:
+                events = activity['events']
+                del activity['events']
+                activity_row = utils.flatten_json(activity)
+                purge_parameters = True
+                for event in events:
+                    for item in event.get('parameters', []):
+                        if set(item) == set(['value', 'name']):
+                            event[item['name']] = item['value']
+                        elif set(item) == set(['intValue', 'name']):
+                            if item['name'] in ['start_time', 'end_time']:
+                                val = item.get('intValue')
+                                if val is not None:
+                                    val = int(val)
+                                    if val >= 62135683200:
+                                        event[item['name']] = \
+                                            datetime.datetime.fromtimestamp(
+                                                val-62135683200).isoformat()
+                            else:
+                                event[item['name']] = item['intValue']
+                        elif set(item) == set(['boolValue', 'name']):
+                            event[item['name']] = item['boolValue']
+                        elif set(item) == set(['multiValue', 'name']):
+                            event[item['name']] = ' '.join(item['multiValue'])
+                        elif item['name'] == 'scope_data':
+                            parts = {}
+                            for message in item['multiMessageValue']:
+                                for mess in message['parameter']:
+                                    value = mess.get('value', ' '.join(
+                                        mess.get('multiValue', [])))
+                                    parts[mess['name']] = parts.get(
+                                        mess['name'], [])+[value]
+                            for part, v in parts.items():
+                                if part == 'scope_name':
+                                    part = 'scope'
+                                event[part] = ' '.join(v)
+                        else:
+                            purge_parameters = False
+                    if purge_parameters:
+                        event.pop('parameters', None)
+                    row = utils.flatten_json(event)
+                    row.update(activity_row)
+                    for item in row:
+                        if item not in titles:
+                            titles.append(item)
+                    csvRows.append(row)
+            display.sort_csv_titles(['name', ], titles)
+            display.write_csv_file(
+                csvRows, titles, f'{report.capitalize()} Activity Report',
+                to_drive)
+
+
+def _adjust_date(errMsg):
+    match_date = re.match('Data for dates later than (.*) is not yet '
+                          'available. Please check back later', errMsg)
+    if not match_date:
+        match_date = re.match('Start date can not be later than (.*)', errMsg)
+    if not match_date:
+        controlflow.system_error_exit(4, errMsg)
+    return str(match_date.group(1))
+
+
+def _check_full_data_available(warnings, tryDate, fullDataRequired):
+    one_day = datetime.timedelta(days=1)
+    for warning in warnings:
+        if warning['code'] == 'PARTIAL_DATA_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    tryDateTime = datetime.datetime.strptime(
+                        tryDate, YYYYMMDD_FORMAT)
+                    tryDateTime -= one_day
+                    return (0, tryDateTime.strftime(YYYYMMDD_FORMAT))
+        elif warning['code'] == 'DATA_NOT_AVAILABLE':
+            for app in warning['data']:
+                if app['key'] == 'application' and \
+                   app['value'] != 'docs' and \
+                   (not fullDataRequired or app['value'] in fullDataRequired):
+                    return (-1, tryDate)
+    return (1, tryDate)

src/gapi/storage.py

@@ -0,0 +1,73 @@
+import base64
+import os
+import re
+import sys
+
+import googleapiclient
+
+import __main__
+from var import *
+import controlflow
+import fileutils
+import gapi
+import utils
+
+
+def build_gapi():
+    return __main__.buildGAPIObject('storage')
+
+
+def get_cloud_storage_object(s, bucket, object_, local_file=None,
+                             expectedMd5=None):
+    if not local_file:
+        local_file = object_
+    if os.path.exists(local_file):
+        sys.stdout.write(' File already exists. ')
+        sys.stdout.flush()
+        if expectedMd5:
+            sys.stdout.write(f'Verifying {expectedMd5} hash...')
+            sys.stdout.flush()
+            if utils.md5_matches_file(local_file, expectedMd5, False):
+                print('VERIFIED')
+                return
+            print('not verified. Downloading again and over-writing...')
+        else:
+            return  # nothing to verify, just assume we're good.
+    print(f'saving to {local_file}')
+    request = s.objects().get_media(bucket=bucket, object=object_)
+    file_path = os.path.dirname(local_file)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    f = fileutils.open_file(local_file, 'wb')
+    downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+    done = False
+    while not done:
+        status, done = downloader.next_chunk()
+        sys.stdout.write(f' Downloaded: {status.progress():>7.2%}\r')
+        sys.stdout.flush()
+    sys.stdout.write('\n Download complete. Flushing to disk...\n')
+    fileutils.close_file(f, True)
+    if expectedMd5:
+        f = fileutils.open_file(local_file, 'rb')
+        sys.stdout.write(f' Verifying file hash is {expectedMd5}...')
+        sys.stdout.flush()
+        utils.md5_matches_file(local_file, expectedMd5, True)
+        print('VERIFIED')
+        fileutils.close_file(f)
+
+
+def download_bucket():
+    bucket = sys.argv[3]
+    s = build_gapi()
+    page_message = gapi.got_total_items_msg('Files', '...')
+    fields = 'nextPageToken,items(name,id,md5Hash)'
+    objects = gapi.get_all_pages(s.objects(), 'list', 'items',
+                                 page_message=page_message, bucket=bucket,
+                                 projection='noAcl', fields=fields)
+    i = 1
+    for object_ in objects:
+        print(f'{i}/{len(objects)}')
+        expectedMd5 = base64.b64decode(object_['md5Hash']).hex()
+        get_cloud_storage_object(
+            s, bucket, object_['name'], expectedMd5=expectedMd5)
+        i += 1

src/gapi/vault.py

@@ -0,0 +1,764 @@
+import datetime
+import json
+import sys
+
+import googleapiclient.http
+
+import __main__
+from var import *
+import controlflow
+import display
+import fileutils
+import gapi
+import gapi.storage
+import utils
+
+
+def buildGAPIObject():
+    return __main__.buildGAPIObject('vault')
+
+
+def validateCollaborators(collaboratorList, cd):
+    collaborators = []
+    for collaborator in collaboratorList.split(','):
+        collaborator_id = __main__.convertEmailAddressToUID(collaborator, cd)
+        if not collaborator_id:
+            controlflow.system_error_exit(4, f'failed to get a UID for '
+                                             f'{collaborator}. Please make '
+                                             f'sure this is a real user.')
+        collaborators.append({'email': collaborator, 'id': collaborator_id})
+    return collaborators
+
+
+def createMatter():
+    v = buildGAPIObject()
+    matter_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    body = {'name': f'New Matter - {matter_time}'}
+    collaborators = []
+    cd = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['collaborator', 'collaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create matter")
+    matterId = gapi.call(v.matters(), 'create', body=body,
+                         fields='matterId')['matterId']
+    print(f'Created matter {matterId}')
+    for collaborator in collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {'matterPermission': {
+            'role': 'COLLABORATOR',
+            'accountId': collaborator['id']}}
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+
+
+VAULT_SEARCH_METHODS_MAP = {
+    'account': 'ACCOUNT',
+    'accounts': 'ACCOUNT',
+    'entireorg': 'ENTIRE_ORG',
+    'everyone': 'ENTIRE_ORG',
+    'orgunit': 'ORG_UNIT',
+    'ou': 'ORG_UNIT',
+    'room': 'ROOM',
+    'rooms': 'ROOM',
+    'shareddrive': 'SHARED_DRIVE',
+    'shareddrives': 'SHARED_DRIVE',
+    'teamdrive': 'SHARED_DRIVE',
+    'teamdrives': 'SHARED_DRIVE',
+}
+VAULT_SEARCH_METHODS_LIST = ['accounts',
+                             'orgunit', 'shareddrives', 'rooms', 'everyone']
+
+
+def createExport():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['corpus']['enum'])
+    allowed_scopes = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Query']['properties']['dataScope']['enum'])
+    allowed_formats = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['MailExportOptions']['properties']
+        ['exportFormat']['enum'])
+    export_format = 'MBOX'
+    showConfidentialModeContent = None  # default to not even set
+    matterId = None
+    body = {'query': {'dataScope': 'ALL_DATA'}, 'exportOptions': {}}
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            body['matterId'] = matterId
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'corpus':
+            body['query']['corpus'] = sys.argv[i+1].upper()
+            if body['query']['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit(
+                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+            i += 2
+        elif myarg in VAULT_SEARCH_METHODS_MAP:
+            if body['query'].get('searchMethod'):
+                message = f'Multiple search methods ' \
+                          f'({", ".join(VAULT_SEARCH_METHODS_LIST)})' \
+                          f'specified, only one is allowed'
+                controlflow.system_error_exit(3, message)
+            searchMethod = VAULT_SEARCH_METHODS_MAP[myarg]
+            body['query']['searchMethod'] = searchMethod
+            if searchMethod == 'ACCOUNT':
+                body['query']['accountInfo'] = {
+                    'emails': sys.argv[i+1].split(',')}
+                i += 2
+            elif searchMethod == 'ORG_UNIT':
+                body['query']['orgUnitInfo'] = {
+                    'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+                i += 2
+            elif searchMethod == 'SHARED_DRIVE':
+                body['query']['sharedDriveInfo'] = {
+                    'sharedDriveIds': sys.argv[i+1].split(',')}
+                i += 2
+            elif searchMethod == 'ROOM':
+                body['query']['hangoutsChatInfo'] = {
+                    'roomId': sys.argv[i+1].split(',')}
+                i += 2
+            else:
+                i += 1
+        elif myarg == 'scope':
+            body['query']['dataScope'] = sys.argv[i+1].upper()
+            if body['query']['dataScope'] not in allowed_scopes:
+                controlflow.expected_argument_exit(
+                    "scope", ", ".join(allowed_scopes), sys.argv[i+1])
+            i += 2
+        elif myarg in ['terms']:
+            body['query']['terms'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            body['query']['startTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            body['query']['endTime'] = utils.get_date_zero_time_or_full_time(
+                sys.argv[i+1])
+            i += 2
+        elif myarg in ['timezone']:
+            body['query']['timeZone'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['excludedrafts']:
+            body['query']['mailOptions'] = {
+                'excludeDrafts': __main__.getBoolean(sys.argv[i+1], myarg)}
+            i += 2
+        elif myarg in ['driveversiondate']:
+            body['query'].setdefault('driveOptions', {})['versionDate'] = \
+                utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['includeshareddrives', 'includeteamdrives']:
+            body['query'].setdefault('driveOptions', {})[
+                'includeSharedDrives'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg in ['includerooms']:
+            body['query']['hangoutsChatOptions'] = {
+                'includeRooms': __main__.getBoolean(sys.argv[i+1], myarg)}
+            i += 2
+        elif myarg in ['format']:
+            export_format = sys.argv[i+1].upper()
+            if export_format not in allowed_formats:
+                controlflow.expected_argument_exit(
+                    "export format", ", ".join(allowed_formats), export_format)
+            i += 2
+        elif myarg in ['showconfidentialmodecontent']:
+            showConfidentialModeContent = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        elif myarg in ['region']:
+            allowed_regions = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['ExportOptions']['properties'][
+                    'region']['enum'])
+            body['exportOptions']['region'] = sys.argv[i+1].upper()
+            if body['exportOptions']['region'] not in allowed_regions:
+                controlflow.expected_argument_exit("region", ", ".join(
+                    allowed_regions), body['exportOptions']['region'])
+            i += 2
+        elif myarg in ['includeaccessinfo']:
+            body['exportOptions'].setdefault('driveOptions', {})[
+                'includeAccessInfo'] = __main__.getBoolean(sys.argv[i+1], myarg)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create export")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new export.')
+    if 'corpus' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a corpus for the ' \
+          f'new export. Choose one of {", ".join(allowed_corpuses)}')
+    if 'searchMethod' not in body['query']:
+        controlflow.system_error_exit(3, f'you must specify a search method ' \
+          'for the new export. Choose one of ' \
+          f'{", ".join(VAULT_SEARCH_METHODS_LIST)}')
+    if 'name' not in body:
+        corpus_name = body["query"]["corpus"]
+        corpus_date = datetime.datetime.now()
+        body['name'] = f'GAM {corpus_name} export - {corpus_date}'
+    options_field = None
+    if body['query']['corpus'] == 'MAIL':
+        options_field = 'mailOptions'
+    elif body['query']['corpus'] == 'GROUPS':
+        options_field = 'groupsOptions'
+    elif body['query']['corpus'] == 'HANGOUTS_CHAT':
+        options_field = 'hangoutsChatOptions'
+    if options_field:
+        body['exportOptions'].pop('driveOptions', None)
+        body['exportOptions'][options_field] = {'exportFormat': export_format}
+        if showConfidentialModeContent is not None:
+            body['exportOptions'][options_field][
+                'showConfidentialModeContent'] = showConfidentialModeContent
+    results = gapi.call(v.matters().exports(), 'create',
+                        matterId=matterId, body=body)
+    print(f'Created export {results["id"]}')
+    display.print_json(results)
+
+
+def deleteExport():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    print(f'Deleting export {sys.argv[4]} / {exportId}')
+    gapi.call(v.matters().exports(), 'delete',
+              matterId=matterId, exportId=exportId)
+
+
+def getExportInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    export = gapi.call(v.matters().exports(), 'get',
+                       matterId=matterId, exportId=exportId)
+    display.print_json(export)
+
+
+def createHold():
+    v = buildGAPIObject()
+    allowed_corpuses = gapi.get_enum_values_minus_unspecified(
+        v._rootDesc['schemas']['Hold']['properties']['corpus']['enum'])
+    body = {'query': {}}
+    i = 3
+    query = None
+    start_time = None
+    end_time = None
+    matterId = None
+    accounts = []
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg == 'corpus':
+            body['corpus'] = sys.argv[i+1].upper()
+            if body['corpus'] not in allowed_corpuses:
+                controlflow.expected_argument_exit(
+                    "corpus", ", ".join(allowed_corpuses), sys.argv[i+1])
+            i += 2
+        elif myarg in ['accounts', 'users', 'groups']:
+            accounts = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {
+                'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam create hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the new hold.')
+    if not body.get('name'):
+        controlflow.system_error_exit(
+            3, 'you must specify a name for the new hold.')
+    if not body.get('corpus'):
+        controlflow.system_error_exit(3, f'you must specify a corpus for ' \
+          f'the new hold. Choose one of {", ".join(allowed_corpuses)}')
+    if body['corpus'] == 'HANGOUTS_CHAT':
+        query_type = 'hangoutsChatQuery'
+    else:
+        query_type = f'{body["corpus"].lower()}Query'
+    body['query'][query_type] = {}
+    if body['corpus'] == 'DRIVE':
+        if query:
+            try:
+                body['query'][query_type] = json.loads(query)
+            except ValueError as e:
+                controlflow.system_error_exit(3, f'{str(e)}, query: {query}')
+    elif body['corpus'] in ['GROUPS', 'MAIL']:
+        if query:
+            body['query'][query_type] = {'terms': query}
+        if start_time:
+            body['query'][query_type]['startTime'] = start_time
+        if end_time:
+            body['query'][query_type]['endTime'] = end_time
+    if accounts:
+        body['accounts'] = []
+        cd = __main__.buildGAPIObject('directory')
+        account_type = 'group' if body['corpus'] == 'GROUPS' else 'user'
+        for account in accounts:
+            body['accounts'].append(
+                {'accountId': __main__.convertEmailAddressToUID(account,
+                                                                cd,
+                                                                account_type)}
+            )
+    holdId = gapi.call(v.matters().holds(), 'create',
+                       matterId=matterId, body=body, fields='holdId')
+    print(f'Created hold {holdId["holdId"]}')
+
+
+def deleteHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam delete hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    print(f'Deleting hold {hold} / {holdId}')
+    gapi.call(v.matters().holds(), 'delete', matterId=matterId, holdId=holdId)
+
+
+def getHoldInfo():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam info hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    results = gapi.call(v.matters().holds(), 'get',
+                        matterId=matterId, holdId=holdId)
+    cd = __main__.buildGAPIObject('directory')
+    if 'accounts' in results:
+        account_type = 'group' if results['corpus'] == 'GROUPS' else 'user'
+        for i in range(0, len(results['accounts'])):
+            uid = f'uid:{results["accounts"][i]["accountId"]}'
+            acct_email = __main__.convertUIDtoEmailAddress(
+                uid, cd, [account_type])
+            results['accounts'][i]['email'] = acct_email
+    if 'orgUnit' in results:
+        results['orgUnit']['orgUnitPath'] = __main__.doGetOrgInfo(
+            results['orgUnit']['orgUnitId'], return_attrib='orgUnitPath')
+    display.print_json(results)
+
+
+def convertExportNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'exports(id,name),nextPageToken'
+    exports = gapi.get_all_pages(v.matters().exports(
+    ), 'list', 'exports', matterId=matterId, fields=fields)
+    for export in exports:
+        if export['name'].lower() == nameOrID:
+            return export['id']
+    controlflow.system_error_exit(4, f'could not find export name {nameOrID} '
+                                     f'in matter {matterId}')
+
+
+def convertHoldNameToID(v, nameOrID, matterId):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'holds(holdId,name),nextPageToken'
+    holds = gapi.get_all_pages(v.matters().holds(
+    ), 'list', 'holds', matterId=matterId, fields=fields)
+    for hold in holds:
+        if hold['name'].lower() == nameOrID:
+            return hold['holdId']
+    controlflow.system_error_exit(4, f'could not find hold name {nameOrID} '
+                                     f'in matter {matterId}')
+
+
+def convertMatterNameToID(v, nameOrID):
+    nameOrID = nameOrID.lower()
+    cg = UID_PATTERN.match(nameOrID)
+    if cg:
+        return cg.group(1)
+    fields = 'matters(matterId,name),nextPageToken'
+    matters = gapi.get_all_pages(v.matters(
+    ), 'list', 'matters', view='BASIC', fields=fields)
+    for matter in matters:
+        if matter['name'].lower() == nameOrID:
+            return matter['matterId']
+    return None
+
+
+def getMatterItem(v, nameOrID):
+    matterId = convertMatterNameToID(v, nameOrID)
+    if not matterId:
+        controlflow.system_error_exit(4, f'could not find matter {nameOrID}')
+    return matterId
+
+
+def updateHold():
+    v = buildGAPIObject()
+    hold = sys.argv[3]
+    matterId = None
+    body = {}
+    query = None
+    add_accounts = []
+    del_accounts = []
+    start_time = None
+    end_time = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'matter':
+            matterId = getMatterItem(v, sys.argv[i+1])
+            holdId = convertHoldNameToID(v, hold, matterId)
+            i += 2
+        elif myarg == 'query':
+            query = sys.argv[i+1]
+            i += 2
+        elif myarg in ['orgunit', 'ou']:
+            body['orgUnit'] = {'orgUnitId': __main__.getOrgUnitId(sys.argv[i+1])[1]}
+            i += 2
+        elif myarg in ['start', 'starttime']:
+            start_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['end', 'endtime']:
+            end_time = utils.get_date_zero_time_or_full_time(sys.argv[i+1])
+            i += 2
+        elif myarg in ['addusers', 'addaccounts', 'addgroups']:
+            add_accounts = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg in ['removeusers', 'removeaccounts', 'removegroups']:
+            del_accounts = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam update hold")
+    if not matterId:
+        controlflow.system_error_exit(
+            3, 'you must specify a matter for the hold.')
+    if query or start_time or end_time or body.get('orgUnit'):
+        fields = 'corpus,query,orgUnit'
+        old_body = gapi.call(v.matters().holds(
+        ), 'get', matterId=matterId, holdId=holdId, fields=fields)
+        body['query'] = old_body['query']
+        body['corpus'] = old_body['corpus']
+        if 'orgUnit' in old_body and 'orgUnit' not in body:
+            # bah, API requires this to be sent
+            # on update even when it's not changing
+            body['orgUnit'] = old_body['orgUnit']
+        query_type = f'{body["corpus"].lower()}Query'
+        if body['corpus'] == 'DRIVE':
+            if query:
+                try:
+                    body['query'][query_type] = json.loads(query)
+                except ValueError as e:
+                    message = f'{str(e)}, query: {query}'
+                    controlflow.system_error_exit(3, message)
+        elif body['corpus'] in ['GROUPS', 'MAIL']:
+            if query:
+                body['query'][query_type]['terms'] = query
+            if start_time:
+                body['query'][query_type]['startTime'] = start_time
+            if end_time:
+                body['query'][query_type]['endTime'] = end_time
+    if body:
+        print(f'Updating hold {hold} / {holdId}')
+        gapi.call(v.matters().holds(), 'update',
+                  matterId=matterId, holdId=holdId, body=body)
+    if add_accounts or del_accounts:
+        cd = __main__.buildGAPIObject('directory')
+        for account in add_accounts:
+            print(f'adding {account} to hold.')
+            add_body = {'accountId': __main__.convertEmailAddressToUID(account, cd)}
+            gapi.call(v.matters().holds().accounts(), 'create',
+                      matterId=matterId, holdId=holdId, body=add_body)
+        for account in del_accounts:
+            print(f'removing {account} from hold.')
+            accountId = __main__.convertEmailAddressToUID(account, cd)
+            gapi.call(v.matters().holds().accounts(), 'delete',
+                      matterId=matterId, holdId=holdId, accountId=accountId)
+
+
+def updateMatter(action=None):
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    body = {}
+    action_kwargs = {'body': {}}
+    add_collaborators = []
+    remove_collaborators = []
+    cd = None
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'action':
+            action = sys.argv[i+1].lower()
+            if action not in VAULT_MATTER_ACTIONS:
+                controlflow.system_error_exit(3, f'allowed actions are ' \
+                    f'{", ".join(VAULT_MATTER_ACTIONS)}, got {action}')
+            i += 2
+        elif myarg == 'name':
+            body['name'] = sys.argv[i+1]
+            i += 2
+        elif myarg == 'description':
+            body['description'] = sys.argv[i+1]
+            i += 2
+        elif myarg in ['addcollaborator', 'addcollaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            add_collaborators.extend(validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        elif myarg in ['removecollaborator', 'removecollaborators']:
+            if not cd:
+                cd = __main__.buildGAPIObject('directory')
+            remove_collaborators.extend(
+                validateCollaborators(sys.argv[i+1], cd))
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam update matter")
+    if action == 'delete':
+        action_kwargs = {}
+    if body:
+        print(f'Updating matter {sys.argv[3]}...')
+        if 'name' not in body or 'description' not in body:
+            # bah, API requires name/description to be sent
+            # on update even when it's not changing
+            result = gapi.call(v.matters(), 'get',
+                               matterId=matterId, view='BASIC')
+            body.setdefault('name', result['name'])
+            body.setdefault('description', result.get('description'))
+        gapi.call(v.matters(), 'update', body=body, matterId=matterId)
+    if action:
+        print(f'Performing {action} on matter {sys.argv[3]}')
+        gapi.call(v.matters(), action, matterId=matterId, **action_kwargs)
+    for collaborator in add_collaborators:
+        print(f' adding collaborator {collaborator["email"]}')
+        body = {'matterPermission': {'role': 'COLLABORATOR',
+                                     'accountId': collaborator['id']}}
+        gapi.call(v.matters(), 'addPermissions', matterId=matterId, body=body)
+    for collaborator in remove_collaborators:
+        print(f' removing collaborator {collaborator["email"]}')
+        gapi.call(v.matters(), 'removePermissions', matterId=matterId,
+                  body={'accountId': collaborator['id']})
+
+
+def getMatterInfo():
+    v = buildGAPIObject()
+    matterId = getMatterItem(v, sys.argv[3])
+    result = gapi.call(v.matters(), 'get', matterId=matterId, view='FULL')
+    if 'matterPermissions' in result:
+        cd = __main__.buildGAPIObject('directory')
+        for i in range(0, len(result['matterPermissions'])):
+            uid = f'uid:{result["matterPermissions"][i]["accountId"]}'
+            user_email = __main__.convertUIDtoEmailAddress(uid, cd)
+            result['matterPermissions'][i]['email'] = user_email
+    display.print_json(result)
+
+
+def downloadExport():
+    verifyFiles = True
+    extractFiles = True
+    v = buildGAPIObject()
+    s = gapi.storage.build_gapi()
+    matterId = getMatterItem(v, sys.argv[3])
+    exportId = convertExportNameToID(v, sys.argv[4], matterId)
+    targetFolder = GC_Values[GC_DRIVE_DIR]
+    i = 5
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'targetfolder':
+            targetFolder = os.path.expanduser(sys.argv[i+1])
+            if not os.path.isdir(targetFolder):
+                os.makedirs(targetFolder)
+            i += 2
+        elif myarg == 'noverify':
+            verifyFiles = False
+            i += 1
+        elif myarg == 'noextract':
+            extractFiles = False
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(
+                sys.argv[i], "gam download export")
+    export = gapi.call(v.matters().exports(), 'get',
+                       matterId=matterId, exportId=exportId)
+    for s_file in export['cloudStorageSink']['files']:
+        bucket = s_file['bucketName']
+        s_object = s_file['objectName']
+        filename = os.path.join(targetFolder, s_object.replace('/', '-'))
+        print(f'saving to {filename}')
+        request = s.objects().get_media(bucket=bucket, object=s_object)
+        f = fileutils.open_file(filename, 'wb')
+        downloader = googleapiclient.http.MediaIoBaseDownload(f, request)
+        done = False
+        while not done:
+            status, done = downloader.next_chunk()
+            sys.stdout.write(
+                ' Downloaded: {0:>7.2%}\r'.format(status.progress()))
+            sys.stdout.flush()
+        sys.stdout.write('\n Download complete. Flushing to disk...\n')
+        fileutils.close_file(f, True)
+        if verifyFiles:
+            expected_hash = s_file['md5Hash']
+            sys.stdout.write(f' Verifying file hash is {expected_hash}...')
+            sys.stdout.flush()
+            utils.md5_matches_file(filename, expected_hash, True)
+            print('VERIFIED')
+        if extractFiles and re.search(r'\.zip$', filename):
+            __main__.extract_nested_zip(filename, targetFolder)
+
+
+def printMatters():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'name', 'description', 'state']
+    titles = initialTitles[:]
+    view = 'FULL'
+    state = None
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in PROJECTION_CHOICES_MAP:
+            view = PROJECTION_CHOICES_MAP[myarg]
+            i += 1
+        elif myarg == 'matterstate':
+            valid_states = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['Matter']['properties']['state'][
+                    'enum'])
+            state = sys.argv[i+1].upper()
+            if state not in valid_states:
+                controlflow.expected_argument_exit(
+                    'state', ', '.join(valid_states), state)
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print matters")
+    __main__.printGettingAllItems('Vault Matters', None)
+    page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
+    matters = gapi.get_all_pages(
+        v.matters(), 'list', 'matters', page_message=page_message, view=view,
+        state=state)
+    for matter in matters:
+        display.add_row_titles_to_csv_file(
+            utils.flatten_json(matter), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Matters', todrive)
+
+
+def printExports():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'id', 'name', 'createTime', 'status']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print exports")
+    if not matters:
+        fields = 'matters(matterId),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(
+        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        for matter in matters_results:
+            matterIds.append(matter['matterId'])
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving exports for matter {matterId}\n')
+        exports = gapi.get_all_pages(
+            v.matters().exports(), 'list', 'exports', matterId=matterId)
+        for export in exports:
+            display.add_row_titles_to_csv_file(utils.flatten_json(
+                export, flattened={'matterId': matterId}), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Exports', todrive)
+
+
+def printHolds():
+    v = buildGAPIObject()
+    todrive = False
+    csvRows = []
+    initialTitles = ['matterId', 'holdId', 'name', 'corpus', 'updateTime']
+    titles = initialTitles[:]
+    matters = []
+    matterIds = []
+    i = 3
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['matter', 'matters']:
+            matters = sys.argv[i+1].split(',')
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(myarg, "gam print holds")
+    if not matters:
+        fields = 'matters(matterId),nextPageToken'
+        matters_results = gapi.get_all_pages(v.matters(
+        ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
+        for matter in matters_results:
+            matterIds.append(matter['matterId'])
+    else:
+        for matter in matters:
+            matterIds.append(getMatterItem(v, matter))
+    for matterId in matterIds:
+        sys.stderr.write(f'Retrieving holds for matter {matterId}\n')
+        holds = gapi.get_all_pages(
+            v.matters().holds(), 'list', 'holds', matterId=matterId)
+        for hold in holds:
+            display.add_row_titles_to_csv_file(utils.flatten_json(
+                hold, flattened={'matterId': matterId}), csvRows, titles)
+    display.sort_csv_titles(initialTitles, titles)
+    display.write_csv_file(csvRows, titles, 'Vault Holds', todrive)

src/macos-gam.spec

@@ -1,32 +0,0 @@
-# -*- mode: python -*-
-import sys
-
-sys.modules['FixTk'] = None
-
-a = Analysis(['gam.py'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam',
-          debug=False,
-          strip=None,
-          upx=False,
-          console=True )

src/project-apis.txt

@@ -2,14 +2,16 @@ admin.googleapis.com
 alertcenter.googleapis.com
 appsactivity.googleapis.com
 calendar-json.googleapis.com
+chat.googleapis.com
 classroom.googleapis.com
+cloudidentity.googleapis.com
 contacts.googleapis.com
 drive.googleapis.com
+iap.googleapis.com
 gmail.googleapis.com
 groupssettings.googleapis.com
 iam.googleapis.com
 licensing.googleapis.com
-plus.googleapis.com
 reseller.googleapis.com
 sheets.googleapis.com
 siteverification.googleapis.com

src/requirements.txt

@@ -3,8 +3,8 @@ python-dateutil
 distro; sys_platform == 'linux'
 filelock
 google-api-python-client>=1.7.10
-google-auth
+google-auth>=1.11.2
 google-auth-httplib2
 google-auth-oauthlib>=0.4.1
-httplib2>=0.13.0
-passlib>=1.7.2
+httplib2>=0.17.0
+passlib>=1.7.2; sys_platform == 'win32'

src/tools/parse-aue.py

@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-from xml.etree import ElementTree as ET
-import requests
-from html.parser import HTMLParser
-import string
-import sys
-import json
-import dateutil.parser
-
-class MyHTMLParser(HTMLParser):
-    def handle_starttag(self, tag, attrs):
-        global next_data_is_oem, next_data_is_td
-        if tag == 'h2' and attrs == [('class', 'zippy')]:
-            next_data_is_oem = True
-        elif tag == 'td':
-            next_data_is_td = True
-
-    def handle_data(self, data):
-        global oem, next_data_is_oem, next_data_is_td, data_is_date, model, printable, output_rows
-        if next_data_is_oem:
-            oem = ''.join(filter(lambda x: x in printable, data))
-            next_data_is_oem = False
-        elif next_data_is_td:
-            if data_is_date:
-                if model.lower().startswith(oem.lower()):
-                    fullname = model.lower()
-                else:
-                    fullname = '%s %s' % (oem, model)
-                    fullname = fullname.lower()
-                date = dateutil.parser.parse(data).replace(day=1).strftime('%Y-%m-%dT00:00:00.000Z')
-                output_rows[fullname] = date
-                if fullname in exceptions:
-                    for value in exceptions[fullname]:
-                        output_rows[value] = date
-                data_is_date = False
-            else:
-                model = ''.join(filter(lambda x: x in printable, data))
-                data_is_date = True
-            next_data_is_td = False
-
-global oem, next_data_is_oem, next_data_is_td, data_is_date, model, printable, exceptions, output_rows
-output_rows = {}
-printable = set(string.printable)
-exceptions = {
-        # 'AUE OEM MODEL': ['API MODEL 1', ...]
-        'acer c7 chromebook': ['acer c7 chromebook (c710)'],
-        'acer chromebook 11 (c720, c720p)': ['acer c720 chromebook', 'acer c740 chromebook'],
-        'acer chromebook 11 (cb3-111, c730, c730e)': ['chromebook 11 (c730 / cb3-111)'],
-        'acer chromebook 11 (cb3-131, c735)': ['chromebook 11 (c735)'],
-        'acer chromebook 15 (cb515-1h,cb515-1ht)': ['chromebook 15 (cb515 - 1ht / 1h)'],
-        'acer chromebook 13(cb5-311, c810)': ['acer chromebook 13 (cb5-311)'],
-        'acer chromebook 15 (cb5-571, c910)': ['acer chromebook 15 (c910 / cb5-571)'],
-        'acer chromebook 311 (c721, c733, c733u, c733t)': ['acer chromebook 311', 'chromebook 311 (c721)'],
-        'acer chromebook 315 (cb315-2h)': ['acer chromebook 315'],
-        'acer chromebook spin 311 (r721t)': ['acer chromebook 311'],
-        'acer chromebook spin 511 (r752t, r752tn)': ['acer chromebook spin 511'],
-        'acer chromebox cxi2 / cxv2': ['acer chromebox cxi2'],
-        'asus chromebook c200': ['asus chromebook c200ma'],
-        'asus chromebook c201pa': ['asus chromebook c201pa'],
-        'asus chromebook c204': ['asus chromebook c204'],
-        'asus chromebook c300': ['asus chromebook c300ma'],
-        'asus chromebook flip c213': ['asus chromebook c213na'],
-        'asus chromebox 2 (cn62)': ['asus chromebox cn62'],
-        'asus chromebox 3 (cn65)': ['asus chromebox 3'],
-        'asus chromebox (cn60)': ['asus chromebox cn60'],
-        'ctl chromebook nl7 / nl7t-360 / nl7tw-360': ['ctl chromebook nl7'],
-        'ctl chromebook tablet tx1 for education': ['ctl chromebook tab tx1'],
-        'ctl nl61 chromebook': ['mecer v2 chromebook'],
-        'google cr-48': ['cr-48'],
-        'haier chromebook 11e': ['chromebook pcm-116e', 'lumos education chromebook'],
-        'haier chromebook 11': ['true idc chromebook 11', 'xolo chromebook'],
-        'hisense chromebook 11': ['epik 11.6" chromebook  elb1101', 'mecer chromebook', 'videonet chromebook bl10'],
-        'hp chromebook 11 g1': ['hp chromebook 11 1100-1199 / hp chromebook 11 g1'],
-        'hp chromebook 11 g2': ['hp chromebook 11 2000-2099 / hp chromebook 11 g2'],
-        'hp chromebook 11 g3': ['hp chromebook 11 2100-2199 / hp chromebook 11 g3'],
-        'hp chromebook 11 g4/g4 ee': ['hp chromebook 11 2200-2299 / hp chromebook 11 g4/g4 ee'],
-        'hp chromebook 11 g5': ['hp chromebook 11 g5 / hp chromebook 11-vxxx'],
-        'hp chromebook 14a g5': ['hp chromebook 14 db0000-db0999'],
-        'hp chromebook 14 g3': ['hp chromebook 14 x000-x999 / hp chromebook 14 g3'],
-        'hp chromebook 14 g4': ['hp chromebook 14 ak000-099 / hp chromebook 14 g4'],
-        'hp chromebook 14 g5': ['hp chromebook 14 / hp chromebook 14 g5'],
-        'hp chromebox g1': ['hp chromebox cb1-(000-099) / hp chromebox g1/ hp chromebox for meetings'],
-        'lenovo ideapad c330 chromebook': ['lenovo chromebook c330'],
-        'lenovo ideapad s330 chromebook': ['lenovo chromebook s330'],
-        'lenovo n21 chromebook': ['asi chromebook', 'crambo chromebook', 'jp sa couto chromebook', 'rgs education chromebook', 'true idc chromebook', 'videonet chromebook', 'consumer chromebook'],
-        'lenovo thinkpad 11e 3rd gen chromebook': ['thinkpad 11e chromebook 3rd gen (yoga/clamshell)'],
-        'lenovo thinkpad 11e 4th gen chromebook': ['lenovo thinkpad 11e chromebook (4th gen)/lenovo thinkpad yoga 11e chromebook (4th gen)'],
-        'lenovo thinkpad 13': ['thinkpad 13 chromebook'],
-        'poin2 chromebook 14': ['poin2 chromebook 11c'],
-        'prowise chromebook eduline': ['viglen chromebook 11c'],
-        'prowise chromebook entryline': ['prowise 11.6\" entry line chromebook'],
-        'prowise chromebook proline': ['prowise proline chromebook'],
-        'samsung chromebook - xe303': ['samsung chromebook'],
-        }
-next_data_is_oem = False
-next_data_is_td = False
-data_is_date = False
-auepage = requests.get('https://support.google.com/chrome/a/answer/6220366?hl=en')
-parser = MyHTMLParser()
-parser.feed(auepage.content.decode('utf-8'))
-print(json.dumps(output_rows, indent=2, sort_keys=True))

src/transport.py

@@ -0,0 +1,103 @@
+"""Methods related to network transport."""
+
+import google_auth_httplib2
+import httplib2
+
+from var import GAM_INFO
+from var import GC_CA_FILE
+from var import GC_TLS_MAX_VERSION
+from var import GC_TLS_MIN_VERSION
+from var import GC_Values
+
+
+def create_http(cache=None,
+                timeout=None,
+                override_min_tls=None,
+                override_max_tls=None):
+  """Creates a uniform HTTP transport object.
+
+  Args:
+    cache: The HTTP cache to use.
+    timeout: The cache timeout, in seconds.
+    override_min_tls: The minimum TLS version to require. If not provided, the
+      default is used.
+    override_max_tls: The maximum TLS version to require. If not provided, the
+      default is used.
+
+  Returns:
+    httplib2.Http with the specified options.
+  """
+  tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
+      GC_TLS_MIN_VERSION)
+  tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
+      GC_TLS_MAX_VERSION)
+  httpObj = httplib2.Http(
+      ca_certs=GC_Values.get(GC_CA_FILE),
+      tls_maximum_version=tls_maximum_version,
+      tls_minimum_version=tls_minimum_version,
+      cache=cache,
+      timeout=timeout)
+  httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
+  return httpObj
+
+
+def create_request(http=None):
+  """Creates a uniform Request object with a default http, if not provided.
+
+  Args:
+    http: Optional httplib2.Http compatible object to be used with the request.
+      If not provided, a default HTTP will be used.
+
+  Returns:
+    Request: A google_auth_httplib2.Request compatible Request.
+  """
+  if not http:
+    http = create_http()
+  return Request(http)
+
+
+GAM_USER_AGENT = GAM_INFO
+
+
+def _force_user_agent(user_agent):
+  """Creates a decorator which can force a user agent in HTTP headers."""
+
+  def decorator(request_method):
+    """Wraps a request method to insert a user-agent in HTTP headers."""
+
+    def wrapped_request_method(*args, **kwargs):
+      """Modifies HTTP headers to include a specified user-agent."""
+      if kwargs.get('headers') is not None:
+        if kwargs['headers'].get('user-agent'):
+          if user_agent not in kwargs['headers']['user-agent']:
+            # Save the existing user-agent header and tack on our own.
+            kwargs['headers']['user-agent'] = (
+                f'{user_agent} '
+                f'{kwargs["headers"]["user-agent"]}')
+        else:
+          kwargs['headers']['user-agent'] = user_agent
+      else:
+        kwargs['headers'] = {'user-agent': user_agent}
+      return request_method(*args, **kwargs)
+
+    return wrapped_request_method
+
+  return decorator
+
+
+class Request(google_auth_httplib2.Request):
+  """A Request which forces a user agent."""
+
+  @_force_user_agent(GAM_USER_AGENT)
+  def __call__(self, *args, **kwargs):
+    """Inserts the GAM user-agent header in requests."""
+    return super(Request, self).__call__(*args, **kwargs)
+
+
+class AuthorizedHttp(google_auth_httplib2.AuthorizedHttp):
+  """An AuthorizedHttp which forces a user agent during requests."""
+
+  @_force_user_agent(GAM_USER_AGENT)
+  def request(self, *args, **kwargs):
+    """Inserts the GAM user-agent header in requests."""
+    return super(AuthorizedHttp, self).request(*args, **kwargs)

src/transport_test.py

@@ -0,0 +1,179 @@
+"""Tests for transport."""
+
+import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from gam import SetGlobalVariables
+import google_auth_httplib2
+import httplib2
+
+import transport
+
+
+class CreateHttpTest(unittest.TestCase):
+
+  def setUp(self):
+    SetGlobalVariables()
+    super(CreateHttpTest, self).setUp()
+
+  def test_create_http_sets_default_values_on_http(self):
+    http = transport.create_http()
+    self.assertIsNone(http.cache)
+    self.assertIsNone(http.timeout)
+    self.assertEqual(http.tls_minimum_version,
+                     transport.GC_Values[transport.GC_TLS_MIN_VERSION])
+    self.assertEqual(http.tls_maximum_version,
+                     transport.GC_Values[transport.GC_TLS_MAX_VERSION])
+    self.assertEqual(http.ca_certs, transport.GC_Values[transport.GC_CA_FILE])
+
+  def test_create_http_sets_tls_min_version(self):
+    http = transport.create_http(override_min_tls='TLSv1_1')
+    self.assertEqual(http.tls_minimum_version, 'TLSv1_1')
+
+  def test_create_http_sets_tls_max_version(self):
+    http = transport.create_http(override_max_tls='TLSv1_3')
+    self.assertEqual(http.tls_maximum_version, 'TLSv1_3')
+
+  def test_create_http_sets_cache(self):
+    fake_cache = {}
+    http = transport.create_http(cache=fake_cache)
+    self.assertEqual(http.cache, fake_cache)
+
+  def test_create_http_sets_cache_timeout(self):
+    http = transport.create_http(timeout=1234)
+    self.assertEqual(http.timeout, 1234)
+
+
+class TransportTest(unittest.TestCase):
+
+  def setUp(self):
+    self.mock_http = MagicMock(spec=httplib2.Http)
+    self.mock_response = MagicMock(spec=httplib2.Response)
+    self.mock_content = MagicMock()
+    self.mock_http.request.return_value = (self.mock_response,
+                                           self.mock_content)
+    self.mock_credentials = MagicMock()
+    self.test_uri = 'http://example.com'
+    super(TransportTest, self).setUp()
+
+  @patch.object(transport, 'create_http')
+  def test_create_request_uses_default_http(self, mock_create_http):
+    request = transport.create_request()
+    self.assertEqual(request.http, mock_create_http.return_value)
+
+  def test_create_request_uses_provided_http(self):
+    request = transport.create_request(http=self.mock_http)
+    self.assertEqual(request.http, self.mock_http)
+
+  def test_create_request_returns_request_with_forced_user_agent(self):
+    request = transport.create_request()
+    self.assertIsInstance(request, transport.Request)
+
+  def test_request_is_google_auth_httplib2_compatible(self):
+    request = transport.create_request()
+    self.assertIsInstance(request, google_auth_httplib2.Request)
+
+  def test_request_call_returns_response_content(self):
+    request = transport.Request(self.mock_http)
+    response = request(self.test_uri)
+    self.assertEqual(self.mock_response.status, response.status)
+    self.assertEqual(self.mock_content, response.data)
+
+  def test_request_call_forces_user_agent_no_provided_headers(self):
+    request = transport.Request(self.mock_http)
+
+    request(self.test_uri)
+    headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', headers)
+    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+  def test_request_call_forces_user_agent_no_agent_in_headers(self):
+    request = transport.Request(self.mock_http)
+    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
+
+    request(self.test_uri, headers=fake_request_headers)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+    self.assertEqual('someData',
+                     final_headers['some-header-thats-not-a-user-agent'])
+
+  def test_request_call_forces_user_agent_with_another_agent_in_headers(self):
+    request = transport.Request(self.mock_http)
+    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+    request(self.test_uri, headers=headers_with_user_agent)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn('existing-user-agent', final_headers['user-agent'])
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+  def test_request_call_same_user_agent_already_in_headers(self):
+    request = transport.Request(self.mock_http)
+    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+    request(self.test_uri, headers=same_user_agent_header)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    # Make sure the header wasn't duplicated
+    self.assertEqual(
+        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))
+
+  def test_authorizedhttp_is_google_auth_httplib2_compatible(self):
+    http = transport.AuthorizedHttp(self.mock_credentials)
+    self.assertIsInstance(http, google_auth_httplib2.AuthorizedHttp)
+
+  def test_authorizedhttp_request_returns_response_content(self):
+    http = transport.AuthorizedHttp(self.mock_credentials, http=self.mock_http)
+    response, content = http.request(self.test_uri)
+    self.assertEqual(self.mock_response, response)
+    self.assertEqual(self.mock_content, content)
+
+  def test_authorizedhttp_request_forces_user_agent_no_provided_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    authorized_http.request(self.test_uri)
+    headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', headers)
+    self.assertIn(transport.GAM_USER_AGENT, headers['user-agent'])
+
+  def test_authorizedhttp_request_forces_user_agent_no_agent_in_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    fake_request_headers = {'some-header-thats-not-a-user-agent': 'someData'}
+
+    authorized_http.request(self.test_uri, headers=fake_request_headers)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    self.assertIn('some-header-thats-not-a-user-agent', final_headers)
+    self.assertEqual('someData',
+                     final_headers['some-header-thats-not-a-user-agent'])
+
+  def test_authorizedhttp_request_forces_user_agent_with_another_agent_in_headers(
+      self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    headers_with_user_agent = {'user-agent': 'existing-user-agent'}
+
+    authorized_http.request(self.test_uri, headers=headers_with_user_agent)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn('existing-user-agent', final_headers['user-agent'])
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+
+  def test_authorizedhttp_request_same_user_agent_already_in_headers(self):
+    authorized_http = transport.AuthorizedHttp(
+        self.mock_credentials, http=self.mock_http)
+    same_user_agent_header = {'user-agent': transport.GAM_USER_AGENT}
+
+    authorized_http.request(self.test_uri, headers=same_user_agent_header)
+    final_headers = self.mock_http.request.call_args[1]['headers']
+    self.assertIn('user-agent', final_headers)
+    self.assertIn(transport.GAM_USER_AGENT, final_headers['user-agent'])
+    # Make sure the header wasn't duplicated
+    self.assertEqual(
+        len(transport.GAM_USER_AGENT), len(final_headers['user-agent']))

src/travis/linux-arm64-before-install.sh

@@ -1,73 +0,0 @@
-export whereibelong=$(pwd)
-export dist=$(lsb_release --codename --short)
-echo "We are running on Ubuntu $dist"
-export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
-cpucount=$(nproc --all)
-echo "This device has $cpucount CPUs for compiling..."
-sudo apt-get -qq --yes update > /dev/null
-sudo apt-get -qq --yes install xz-utils > /dev/null
-SSLVER=$(~/ssl/bin/openssl version)
-SSLRESULT=$?
-PYVER=$(~/python/bin/python3 -V)
-PYRESULT=$?
-if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
-  echo "RUNNING: apt dist-upgrade..."
-  sudo apt-get -qq --yes dist-upgrade > /dev/null
-  echo "Installing build tools..."
-  sudo apt-get -qq --yes install build-essential
-
-  echo "Installing deps for python3"
-  sudo cp -v /etc/apt/sources.list /tmp
-  sudo chmod a+rwx /tmp/sources.list
-  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
-  sudo cp -v /tmp/sources.list /etc/apt
-  sudo apt-get -qq --yes update > /dev/null
-  sudo apt-get -qq --yes build-dep python3 > /dev/null
-  sudo apt-get -qq --yes install zlib1g-dev > /dev/null
-  sudo apt-get -qq --yes install libffi-dev > /dev/null
-
-  # Compile latest OpenSSL
-  echo "Downloading OpenSSL..."
-  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  echo "Extracting OpenSSL..."
-  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  cd openssl-$BUILD_OPENSSL_VERSION
-  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-  ./config shared --prefix=$HOME/ssl
-  echo "Running make for OpenSSL..."
-  make -j$cpucount -s
-  echo "Running make install for OpenSSL..."
-  make install > /dev/null
-  cd ~
-
-  # Compile latest Python
-  echo "Downloading Python $BUILD_PYTHON_VERSION..."
-  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
-  echo "Extracting Python..."
-  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
-  cd Python-$BUILD_PYTHON_VERSION
-  echo "Compiling Python $BUILD_PYTHON_VERSION..."
-  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
-  unsafe_flags="--enable-optimizations --with-lto"  
-  echo "running configure with safe and unsafe"
-  ./configure $safe_flags $unsafe_flags > /dev/null
-  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
-  echo "Installing Python..."
-  make install > /dev/null
-fi
-
-python=~/python/bin/python3
-pip=~/python/bin/pip3
-
-$python -V
-
-cd $whereibelong
-
-echo "Upgrading pip packages..."
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
-
-mkdir ~/.ruby
-export GEM_HOME=~/.ruby
-export PATH=$PATH:~/.ruby/bin

src/travis/linux-before-install.sh

@@ -1,12 +1,11 @@
-if [ "$VMTYPE" == "test" ]; then
+if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
   export python="python"
   export pip="pip"
   echo "Travis setup Python $TRAVIS_PYTHON_VERSION"
   echo "running tests with this version"
 else
   export whereibelong=$(pwd)
-  export dist=$(lsb_release --codename --short)
-  echo "We are running on Ubuntu $dist"
+  echo "We are running on Ubuntu $TRAVIS_DIST $PLATFORM"
   export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   cpucount=$(nproc --all)
   echo "This device has $cpucount CPUs for compiling..."
@@ -42,7 +41,7 @@ else
     echo "Installing deps for python3"
     sudo cp -v /etc/apt/sources.list /tmp
     sudo chmod a+rwx /tmp/sources.list
-    echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+    echo "deb-src http://archive.ubuntu.com/ubuntu/ $TRAVIS_DIST main" >> /tmp/sources.list
     sudo cp -v /tmp/sources.list /etc/apt
     sudo apt-get -qq --yes update > /dev/null
     sudo apt-get -qq --yes build-dep python3 > /dev/null
@@ -91,9 +90,8 @@ else
   python=~/python/bin/python3
   pip=~/python/bin/pip3
 
-  if [[ "$dist" == "precise" ]]; then
+  if ([ "${TRAVIS_DIST}" == "trusty" ] || [ "${TRAVIS_DIST}" == "xenial" ]) && [ "${PLATFORM}" == "x86_64" ]; then
     echo "Installing deps for StaticX..."
-    sudo apt-get install --yes scons
     if [ ! -d patchelf-$PATCHELF_VERSION ]; then
       echo "Downloading PatchELF $PATCHELF_VERSION"
       wget https://nixos.org/releases/patchelf/patchelf-$PATCHELF_VERSION/patchelf-$PATCHELF_VERSION.tar.bz2
@@ -103,13 +101,14 @@ else
       make
       sudo make install
     fi
-    $pip install git+https://github.com/JonathonReinhart/staticx.git@master
+    $pip install staticx
   fi
 
+  $pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+
   cd $whereibelong
 fi
 
 echo "Upgrading pip packages..."
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz

src/travis/linux-install.sh

@@ -1,9 +1,9 @@
 cd src
-if [ "$VMTYPE" == "test" ]; then
+if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
   export gam="$python gam.py"
   export gampath=$(readlink -e .)
 else
-  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam gam.spec
   export gam="gam/gam"
   export gampath=$(readlink -e gam)
   export GAMVERSION=`$gam version simple`
@@ -18,16 +18,18 @@ else
   du -h gam/gam
   time $gam version extended
 
-  if [[ "$dist" == "precise" ]]; then
-    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
-    $python -OO -m staticx gam/gam gam/gam-staticx
+  if ([ "${TRAVIS_DIST}" == "trusty" ] || [ "${TRAVIS_DIST}" == "xenial" ]) && [ "${PLATFORM}" == "x86_64" ]; then
+    GAM_LEGACY_ARCHIVE=gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-legacy.tar.xz
+    $python -OO -m staticx -l /lib/x86_64-linux-gnu/libresolv.so.2 -l /lib/x86_64-linux-gnu/libnss_dns.so.2 gam/gam gam/gam-staticx
     strip gam/gam-staticx
     rm gam/gam
     mv gam/gam-staticx gam/gam
-    tar cfJ $GAM_LEGACY_ARCHIVE gam/
+    chmod 755 gam/gam
+    tar cvfJ $GAM_LEGACY_ARCHIVE gam/
     echo "Legacy StaticX GAM info:"
     du -h gam/gam
     time $gam version extended
   fi
-
+  echo "GAM packages:"
+  ls -l gam-*.tar.xz
 fi

src/travis/linux-x86_64-install.sh

@@ -1,32 +0,0 @@
-cd src
-if [ "$VMTYPE" == "test" ]; then
-  export gam="$python gam.py"
-  export gampath=$(readlink -e .)
-else
-  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
-  export gam="gam/gam"
-  export gampath=$(readlink -e gam)
-  export GAMVERSION=`$gam version simple`
-  cp LICENSE $gampath
-  cp whatsnew.txt $gampath
-  cp GamCommands.txt $gampath
-  this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
-  GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
-  rm $gampath/lastupdatecheck.txt
-  tar cfJ $GAM_ARCHIVE gam/
-  echo "PyInstaller GAM info:"
-  du -h gam/gam
-  time $gam version extended
-
-  if [[ "$dist" == "precise" ]]; then
-    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
-    $python -OO -m staticx gam/gam gam/gam-staticx
-    strip gam/gam-staticx
-    rm gam/gam
-    mv gam/gam-staticx gam/gam
-    tar cfJ $GAM_LEGACY_ARCHIVE gam/
-    echo "Legacy StaticX GAM info:"
-    du -h gam/gam
-    time $gam version extended
-  fi
-fi

src/travis/osx-before-install.sh

@@ -0,0 +1,110 @@
+mypath=$HOME
+whereibelong=$(pwd)
+cpucount=$(sysctl -n hw.ncpu)
+echo "This device has $cpucount CPUs for compiling..."
+
+#echo "Brew installing xz..."
+#brew install xz > /dev/null
+
+#brew upgrade
+
+# prefer standard GNU tools like date over MacOS defaults
+export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+
+cd ~
+
+#if [ ! -f python-$MIN_PYTHON_VERSION-macosx10.9.pkg ]; then
+#  wget --quiet https://www.python.org/ftp/python/$MIN_PYTHON_VERSION/python-$MIN_PYTHON_VERSION-macosx10.9.pkg
+#fi
+#sudo installer -pkg python-$MIN_PYTHON_VERSION-macosx10.9.pkg -target /
+
+#brew install openssl@1.1
+#brew upgrade python
+
+#export python=python3
+#export pip=pip3
+
+#echo "Python location:"
+#which $python
+
+cd ~
+
+export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+export openssl=~/ssl/bin/openssl
+export python=~/python/bin/python3
+export pip=~/python/bin/pip3
+SSLVER=$($openssl version)
+SSLRESULT=$?
+PYVER=$($python -V)
+PYRESULT=$?
+if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
+  echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+  if [ $SSLRESULT -ne 0 ]; then
+    echo "sslresult -ne 0"
+  fi
+  if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+    echo "sslver not equal to..."
+  fi
+  if [ $PYRESULT -ne 0 ]; then
+    echo "pyresult -ne 0"
+  fi
+  if [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION" ]]; then
+    echo "pyver not equal to..."
+  fi
+
+  # Start clean
+  rm -rf python
+  rm -rf ssl
+  mkdir python
+  mkdir ssl
+  
+  # Compile latest OpenSSL
+  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  echo "Extracting OpenSSL..."
+  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  cd openssl-$BUILD_OPENSSL_VERSION
+  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+  ./config shared --prefix=$HOME/ssl
+  echo "Running make for OpenSSL..."
+  make -j$cpucount -s
+  echo "Running make install for OpenSSL..."
+  make install > /dev/null
+  cd ~
+
+  # Compile latest Python
+  echo "Downloading Python $BUILD_PYTHON_VERSION..."
+  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+  echo "Extracting Python..."
+  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+  cd Python-$BUILD_PYTHON_VERSION
+  echo "Compiling Python $BUILD_PYTHON_VERSION..."
+  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+  unsafe_flags="--enable-optimizations --with-lto"
+  if [ ! -e Makefile ]; then
+    echo "running configure with safe and unsafe"
+    ./configure $safe_flags $unsafe_flags > /dev/null
+  fi
+  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+  RESULT=$?
+  echo "First make exited with $RESULT"
+  if [ $RESULT != 0 ]; then
+    echo "Trying Python compile again without unsafe flags..."
+    make clean
+    ./configure $safe_flags > /dev/null
+    make -j$cpucount -s
+    echo "Sticking with safe Python for now..."
+  fi
+  echo "Installing Python..."
+  make install > /dev/null
+  cd ~
+fi
+
+$python -V
+
+cd $whereibelong
+
+#export PATH=/usr/local/opt/python/libexec/bin:$PATH
+$pip install --upgrade pip
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 

src/travis/osx-install.sh

@@ -1,7 +1,7 @@
 cd src
 echo "MacOS Version Info According to Python:"
 python -c "import platform; print(platform.mac_ver())"
-$python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+$python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam gam.spec
 export gam="gam/gam"
 export gampath=gam
 $gam version extended

src/travis/osx-x86_64-before-install.sh

@@ -1,78 +0,0 @@
-mypath=$HOME
-whereibelong=$(pwd)
-#echo "Brew installing xz..."
-#brew install xz > /dev/null
-
-#brew upgrade
-
-cd ~
-
-if [ ! -f python-$BUILD_PYTHON_VERSION-macosx10.9.pkg ]; then
-  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
-fi
-sudo installer -pkg python-$BUILD_PYTHON_VERSION-macosx10.9.pkg -target /
-
-#brew install openssl@1.1
-#brew upgrade python
-
-export python=python3
-export pip=pip3
-
-echo "Python location:"
-which $python
-
-# Compile latest OpenSSL
-#if [ ! -d openssl-$BUILD_OPENSSL_VERSION ]; then
-#  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
-#  echo "Extracting OpenSSL..."
-#  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-#fi
-#cd openssl-$BUILD_OPENSSL_VERSION
-#echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-#./config shared --prefix=$mypath/ssl
-#echo "Running make for OpenSSL..."
-#make -j$cpucount -s
-#echo "Running make install for OpenSSL..."
-#make install > /dev/null
-#export LD_LIBRARY_PATH=~/ssl/lib
-#cd ~
-
-# Compile latest Python
-#if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
-#  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
-#  echo "Extracting Python..."
-#  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
-#fi
-#cd Python-$BUILD_PYTHON_VERSION
-#echo "Compiling Python $BUILD_PYTHON_VERSION..."
-#safe_flags="--with-openssl=$mypath/ssl --enable-shared --prefix=$mypath/python --with-ensurepip=upgrade"
-#unsafe_flags="--enable-optimizations --with-lto"
-#if [ ! -e Makefile ]; then
-#  ./configure $safe_flags $unsafe_flags > /dev/null
-#fi
-#make -j$cpucount -s
-#RESULT=$?
-#echo "Make Python exited with $RESULT"
-#if [ $RESULT != 0 ]; then
-#  echo "Trying Python make again without unsafe flags..."
-#  make clean
-#  ./configure $safe_flags > /dev/null
-#  make -j$cpucount -s
-#fi
-#echo "Installing Python..."
-#make install > /dev/null
-#cd ~
-
-#export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
-#python=~/python/bin/python3
-#pip=~/python/bin/pip3
-
-$python -V
-
-cd $whereibelong
-
-export PATH=/usr/local/opt/python/libexec/bin:$PATH
-$pip install --upgrade pip
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 

src/travis/set_printer_csv_filter.sh

@@ -1 +0,0 @@
-export GAM_CSV_ROW_FILTER='{"type": "regex:^GOOGLE"}'

src/travis/svars-write.py

@@ -6,8 +6,8 @@ cfg = json.load(sys.stdin)
 cfg['client_secret'] = os.getenv('client_secret')
 jid = os.getenv('jid')
 cfg['refresh_token'] = os.getenv('refresh_%s' % jid)
-vmtype = os.getenv('VMTYPE')
-if vmtype == 'test':
+name = os.getenv('TRAVIS_JOB_NAME')
+if name.endswith('Testing'):
     out_file = 'oauth2.txt'
 else:
     out_file = 'gam/oauth2.txt'

src/travis/windows-before-install.sh

@@ -0,0 +1,81 @@
+if [[ "$PLATFORM" == "x86_64" ]]; then
+  export BITS="64"
+  export PYTHONFILE_BITS="-amd64"
+  export OPENSSL_BITS="-x64"
+  export WIX_BITS="x64"
+elif [[ "$PLATFORM" == "x86" ]]; then
+  export BITS="32"
+  export PYTHONFILE_BITS=""
+  export OPENSSL_BITS=""
+  export WIX_BITS="x86"
+fi
+echo "This is a ${BITS}-bit build for ${PLATFORM}"
+
+export mypath=$(pwd)
+cd ~
+
+# .NET Core
+echo "Installing Net-Framework-Core..."
+until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying .net again..."; done
+
+# VS 2015
+echo "Installing Visual Studio 2015.."
+until choco install vcbuildtools; do echo "Trying Visual Studio again..."; done
+
+# Python
+echo "Installing Python..."
+export python_file=python-${BUILD_PYTHON_VERSION}${PYTHONFILE_BITS}.exe
+if [ ! -e $python_file ]; then
+  echo "Downloading $python_file..."
+  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
+fi
+until powershell ".\\${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python"; do echo "trying python again..."; done
+export python=/c/python/python.exe
+export pip=/c/python/scripts/pip.exe
+until [ -f $python ]; do sleep 1; done
+export PATH=$PATH:/c/python/scripts
+
+# OpenSSL
+echo "Installing OpenSSL..."
+export exefile=Win${BITS}OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
+if [ ! -e $exefile ]; then
+  echo "Downloading $exefile..."
+  wget --quiet https://slproweb.com/download/$exefile
+fi
+until powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"; do echo "trying openssl again..."; done
+until cp -v /c/ssl/libcrypto-1_1${OPENSSL_BITS}.dll /c/python/DLLs/; do echo "trying libcrypto copy again..."; sleep 3; done
+until cp -v /c/ssl/libssl-1_1${OPENSSL_BITS}.dll /c/python/DLLs/; do echo "trying libssl copy again..."; done
+if [[ "$PLATFORM" == "x86_64" ]]; then
+  cp -v /c/python/DLLs/libssl-1_1-x64.dll /c/python/DLLs/libssl-1_1.dll
+  cp -v /c/python/DLLs/libcrypto-1_1-x64.dll /c/python/DLLs/libcrypto-1_1.dll
+fi
+
+# WIX Toolset
+until cinst -y wixtoolset; do echo "trying wix install again..."; done
+
+cd $mypath
+
+$pip install --upgrade pip
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+#$pip install --upgrade pyinstaller
+# Install PyInstaller from source and build bootloader
+# to try and avoid getting flagged as malware since
+# lots of malware uses PyInstaller default bootloader
+# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
+echo "Downloading PyInstaller..."
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+tar xf develop.tar.gz
+cd pyinstaller-develop/bootloader
+echo "bootloader before:"
+md5sum ../PyInstaller/bootloader/Windows-${BITS}bit/*
+
+$python ./waf all --target-arch=${BITS}bit --msvc_version "msvc 14.0"
+
+echo "bootloader after:"
+md5sum ../PyInstaller/bootloader/Windows-${BITS}bit/*
+echo "PATH: $PATH"
+cd ..
+$python setup.py install
+echo "cd to $mypath"
+cd $mypath

src/travis/windows-install.sh

@@ -1,6 +1,6 @@
 cd src
 echo "compiling GAM with pyinstaller..."
-pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
+pyinstaller --clean --noupx -F --distpath=gam gam.spec
 export gam="gam/gam"
 export gampath=$(readlink -e gam)
 echo "running compiled GAM..."
@@ -15,6 +15,6 @@ GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
 /c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
 mkdir gam-64
 cp -rf gam/* gam-64/;
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch x64 gam.wxs
+/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch $WIX_BITS gam.wxs
 /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
 rm *.wixpdb

src/travis/windows-x86-before-install.sh

@@ -1,34 +0,0 @@
-echo "Installing Net-Framework-Core..."
-export mypath=$(pwd)
-cd ~
-until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cinst -y --forcex86 python3
-until cinst -y wixtoolset; do echo "trying again..."; done
-export PATH=$PATH:/c/Python38/scripts
-cd $mypath
-export python=/c/Python38/python.exe
-export pip=/c/Python38/scripts/pip.exe
-
-$pip install --upgrade pip
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-
-#$pip install --upgrade pyinstaller
-# Install PyInstaller from source and build bootloader
-# to try and avoid getting flagged as malware since
-# lots of malware uses PyInstaller default bootloader
-# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
-echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 
-tar xf develop.tar.gz
-cd pyinstaller-develop/bootloader
-echo "bootloader before:"
-md5sum ../PyInstaller/bootloader/Windows-32bit/*
-$python ./waf all --target-arch=32bit
-echo "bootloader after:"
-md5sum ../PyInstaller/bootloader/Windows-32bit/*
-echo "PATH: $PATH"
-cd ..
-$python setup.py install
-echo "cd to $mypath"
-cd $mypath

src/travis/windows-x86-install.sh

@@ -1,18 +0,0 @@
-cd src
-pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=$(readlink -e gam)
-$gam version extended
-export GAMVERSION=`$gam version simple`
-rm gam/lastupdatecheck.txt
-cp LICENSE gam
-cp GamCommands.txt gam
-cp whatsnew.txt gam
-cp gam-setup.bat gam
-GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
-/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
-mkdir gam-64
-cp -rf gam/* gam-64/;
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch x86 gam.wxs
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
-rm *.wixpdb

src/travis/windows-x86_64-before-install.sh

@@ -1,46 +0,0 @@
-echo "Installing Net-Framework-Core..."
-export mypath=$(pwd)
-until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~
-#export exefile=Win64OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
-#if [ ! -e $exefile ]; then
-#  echo "Downloading $exefile..."
-#  wget --quiet https://slproweb.com/download/$exefile
-#fi
-#echo "Installing $exefile..."
-#powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
-cinst -y python3
-until cinst -y wixtoolset; do echo "trying again..."; done
-#until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/Python37/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
-#until cp -v /c/ssl/libssl-1_1-x64.dll /c/Python37/DLLs/libssl-1_1.dll; do echo "trying again..."; done
-export PATH=$PATH:/c/Python38/scripts
-cd $mypath
-export python=/c/Python38/python.exe
-export pip=/c/Python38/scripts/pip.exe
-
-$pip install --upgrade pip
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-#$pip install --upgrade pyinstaller
-# Install PyInstaller from source and build bootloader
-# to try and avoid getting flagged as malware since
-# lots of malware uses PyInstaller default bootloader
-# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
-echo "Downloading PyInstaller..."
-#wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
-wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
-#tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
-tar xf develop.tar.gz
-#cd PyInstaller-$PYINSTALLER_VERSION/bootloader
-cd pyinstaller-develop/bootloader
-echo "bootloader before:"
-md5sum ../PyInstaller/bootloader/Windows-64bit/*
-$python ./waf all --target-arch=64bit
-echo "bootloader after:"
-md5sum ../PyInstaller/bootloader/Windows-64bit/*
-echo "PATH: $PATH"
-cd ..
-$python setup.py install
-echo "cd to $mypath..."
-#until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
-cd $mypath

src/utils.py

@@ -1,17 +1,17 @@
 import datetime
 import re
 import sys
+import time
+from hashlib import md5
 from html.entities import name2codepoint
 from html.parser import HTMLParser
+import json
+import dateutil.parser
 
-ONE_KILO_BYTES = 1000
-ONE_MEGA_BYTES = 1000000
-ONE_GIGA_BYTES = 1000000000
-
-
-def convertUTF8(data):
-  return data
-
+import controlflow
+import fileutils
+import transport
+from var import *
 
 class _DeHTMLParser(HTMLParser):
 
@@ -30,7 +30,7 @@ class _DeHTMLParser(HTMLParser):
     if cp:
       self.__text.append(chr(cp))
     else:
-      self.__text.append('&' + name)
+      self.__text.append('&'+name)
 
   def handle_starttag(self, tag, attrs):
     if tag == 'p':
@@ -40,22 +40,20 @@ class _DeHTMLParser(HTMLParser):
     elif tag == 'a':
       for attr in attrs:
         if attr[0] == 'href':
-          self.__text.append('({0}) '.format(attr[1]))
+          self.__text.append(f'({attr[1]}) ')
           break
     elif tag == 'div':
       if not attrs:
         self.__text.append('\n')
-    elif tag in ['http:', 'https']:
-      self.__text.append(' ({0}//{1}) '.format(tag, attrs[0][0]))
+    elif tag in {'http:', 'https'}:
+      self.__text.append(f' ({tag}//{attrs[0][0]}) ')
 
   def handle_startendtag(self, tag, attrs):
     if tag == 'br':
       self.__text.append('\n\n')
 
   def text(self):
-    return re.sub(r'\n{2}\n+', '\n\n',
-                  re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
-
+    return re.sub(r'\n{2}\n+', '\n\n', re.sub(r'\n +', '\n', ''.join(self.__text))).strip()
 
 def dehtml(text):
   try:
@@ -68,10 +66,27 @@ def dehtml(text):
     print_exc(file=sys.stderr)
     return text
 
-
 def indentMultiLineText(message, n=0):
   return message.replace('\n', '\n{0}'.format(' ' * n)).rstrip()
 
+def flatten_json(structure, key='', path='', flattened=None, listLimit=None):
+  if flattened is None:
+    flattened = {}
+  if not isinstance(structure, (dict, list)):
+    flattened[((path + '.') if path else '') + key] = structure
+  elif isinstance(structure, list):
+    for i, item in enumerate(structure):
+      if listLimit and (i >= listLimit):
+        break
+      flatten_json(item, f'{i}', '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit)
+  else:
+    for new_key, value in list(structure.items()):
+      if new_key in ['kind', 'etag', '@type']:
+        continue
+      if value == NEVER_TIME:
+        value = 'Never'
+      flatten_json(value, new_key, '.'.join([item for item in [path, key] if item]), flattened=flattened, listLimit=listLimit)
+  return flattened
 
 def formatTimestampYMD(timestamp):
   return datetime.datetime.fromtimestamp(int(timestamp)/1000).strftime('%Y-%m-%d')
@@ -82,21 +97,195 @@ def formatTimestampYMDHMS(timestamp):
 def formatTimestampYMDHMSF(timestamp):
   return str(datetime.datetime.fromtimestamp(int(timestamp)/1000))
 
-
 def formatFileSize(fileSize):
   if fileSize == 0:
     return '0kb'
   if fileSize < ONE_KILO_BYTES:
     return '1kb'
   if fileSize < ONE_MEGA_BYTES:
-    return '{0}kb'.format(fileSize // ONE_KILO_BYTES)
+    return f'{fileSize // ONE_KILO_BYTES}kb'
   if fileSize < ONE_GIGA_BYTES:
-    return '{0}mb'.format(fileSize // ONE_MEGA_BYTES)
-  return '{0}gb'.format(fileSize // ONE_GIGA_BYTES)
-
+    return f'{fileSize // ONE_MEGA_BYTES}mb'
+  return f'{fileSize // ONE_GIGA_BYTES}gb'
 
 def formatMilliSeconds(millis):
   seconds, millis = divmod(millis, 1000)
   minutes, seconds = divmod(seconds, 60)
   hours, minutes = divmod(minutes, 60)
-  return '%02d:%02d:%02d' % (hours, minutes, seconds)
+  return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
+
+def integerLimits(minVal, maxVal, item='integer'):
+  if (minVal is not None) and (maxVal is not None):
+    return f'{item} {minVal}<=x<={maxVal}'
+  if minVal is not None:
+    return f'{item} x>={minVal}'
+  if maxVal is not None:
+    return f'{item} x<={maxVal}'
+  return f'{item} x'
+
+def get_string(i, item, optional=False, minLen=1, maxLen=None):
+  if i < len(sys.argv):
+    argstr = sys.argv[i]
+    if argstr:
+      if (len(argstr) >= minLen) and ((maxLen is None) or (len(argstr) <= maxLen)):
+        return argstr
+      controlflow.system_error_exit(2, f'expected <{integerLimits(minLen, maxLen, "string length")} for {item}>')
+    if optional or (minLen == 0):
+      return ''
+    controlflow.system_error_exit(2, f'expected a Non-empty <{item}>')
+  elif optional:
+    return ''
+  controlflow.system_error_exit(2, f'expected a <{item}>')
+
+def get_delta(argstr, pattern):
+  tg = pattern.match(argstr.lower())
+  if tg is None:
+    return None
+  sign = tg.group(1)
+  delta = int(tg.group(2))
+  unit = tg.group(3)
+  if unit == 'y':
+    deltaTime = datetime.timedelta(days=delta*365)
+  elif unit == 'w':
+    deltaTime = datetime.timedelta(weeks=delta)
+  elif unit == 'd':
+    deltaTime = datetime.timedelta(days=delta)
+  elif unit == 'h':
+    deltaTime = datetime.timedelta(hours=delta)
+  elif unit == 'm':
+    deltaTime = datetime.timedelta(minutes=delta)
+  if sign == '-':
+    return -deltaTime
+  return deltaTime
+
+def get_delta_date(argstr):
+  deltaDate = get_delta(argstr, DELTA_DATE_PATTERN)
+  if deltaDate is None:
+    controlflow.system_error_exit(2, f'expected a <{DELTA_DATE_FORMAT_REQUIRED}>; got {argstr}')
+  return deltaDate
+
+def get_delta_time(argstr):
+  deltaTime = get_delta(argstr, DELTA_TIME_PATTERN)
+  if deltaTime is None:
+    controlflow.system_error_exit(2, f'expected a <{DELTA_TIME_FORMAT_REQUIRED}>; got {argstr}')
+  return deltaTime
+
+def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
+  argstr = argstr.strip()
+  if argstr:
+    if argstr[0] in ['+', '-']:
+      today = datetime.date.today()
+      argstr = (datetime.datetime(today.year, today.month, today.day)+get_delta_date(argstr)).strftime(YYYYMMDD_FORMAT)
+    try:
+      dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
+      if returnTimeStamp:
+        return time.mktime(dateTime.timetuple())*1000
+      if returnDateTime:
+        return dateTime
+      return argstr
+    except ValueError:
+      controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>; got {argstr}')
+  elif minLen == 0:
+    return ''
+  controlflow.system_error_exit(2, f'expected a <{YYYYMMDD_FORMAT_REQUIRED}>')
+
+def get_time_or_delta_from_now(time_string):
+  """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+  time_string = time_string.strip().upper()
+  if time_string:
+    if time_string[0] not in ['+', '-']:
+      return time_string
+    return (datetime.datetime.utcnow() + get_delta_time(time_string)).isoformat() + 'Z'
+  controlflow.system_error_exit(2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+def get_row_filter_date_or_delta_from_now(date_string):
+  """Get an ISO 8601 date or a positive/negative delta applied to now.
+  Args:
+    date_string (string): The time or delta (e.g. '2017-09-01' or '-4y')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  """
+  date_string = date_string.strip().upper()
+  if date_string:
+    if date_string[0] in ['+', '-']:
+      deltaDate = get_delta(date_string, DELTA_DATE_PATTERN)
+      if deltaDate is None:
+        return (False, DELTA_DATE_FORMAT_REQUIRED)
+      today = datetime.date.today()
+      return (True, (datetime.datetime(today.year, today.month, today.day)+deltaDate).isoformat()+'Z')
+    try:
+      deltaDate = dateutil.parser.parse(date_string, ignoretz=True)
+      return (True, datetime.datetime(deltaDate.year, deltaDate.month, deltaDate.day).isoformat()+'Z')
+    except ValueError:
+      pass
+  return (False, YYYYMMDD_FORMAT_REQUIRED)
+
+def get_row_filter_time_or_delta_from_now(time_string):
+  """Get an ISO 8601 time or a positive/negative delta applied to now.
+  Args:
+    time_string (string): The time or delta (e.g. '2017-09-01T12:34:56Z' or '-4h')
+  Returns:
+    string: iso8601 formatted datetime in UTC.
+  Exits:
+    2: Not a valid delta.
+  """
+  time_string = time_string.strip().upper()
+  if time_string:
+    if time_string[0] in ['+', '-']:
+      deltaTime = get_delta(time_string, DELTA_TIME_PATTERN)
+      if deltaTime is None:
+        return (False, DELTA_TIME_FORMAT_REQUIRED)
+      return (True, (datetime.datetime.utcnow()+deltaTime).isoformat()+'Z')
+    try:
+      deltaTime = dateutil.parser.parse(time_string, ignoretz=True)
+      return (True, deltaTime.isoformat()+'Z')
+    except ValueError:
+      pass
+  return (False, YYYYMMDDTHHMMSS_FORMAT_REQUIRED)
+
+def get_date_zero_time_or_full_time(time_string):
+  time_string = time_string.strip()
+  if time_string:
+    if YYYYMMDD_PATTERN.match(time_string):
+      return get_yyyymmdd(time_string)+'T00:00:00.000Z'
+    return get_time_or_delta_from_now(time_string)
+  controlflow.system_error_exit(2, f'expected a <{YYYYMMDDTHHMMSS_FORMAT_REQUIRED}>')
+
+def md5_matches_file(local_file, expected_md5, exitOnError):
+  f = fileutils.open_file(local_file, 'rb')
+  hash_md5 = md5()
+  for chunk in iter(lambda: f.read(4096), b""):
+    hash_md5.update(chunk)
+  actual_hash = hash_md5.hexdigest()
+  if exitOnError and actual_hash != expected_md5:
+    controlflow.system_error_exit(6, f'actual hash was {actual_hash}. Exiting on corrupt file.')
+  return actual_hash == expected_md5
+
+URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
+
+def shorten_url(long_url, httpc=None):
+    if not httpc:
+        httpc = transport.create_http(timeout=10)
+    headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}
+    try:
+        payload = json.dumps({'long_url': long_url})
+        resp, content = httpc.request(
+          URL_SHORTENER_ENDPOINT,
+          'POST',
+          payload,
+          headers=headers)
+    except:
+        return long_url
+    if resp.status != 200:
+        return long_url
+    try:
+        if isinstance(content, bytes):
+            content = content.decode()
+        return json.loads(content).get('short_url', long_url)
+    except:
+        return long_url

src/var.py

@@ -6,21 +6,19 @@ import platform
 import re
 
 gam_author = 'Jay Lee <jay0lee@gmail.com>'
-gam_version = '4.97'
+gam_version = '5.06'
 gam_license = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
-GAM_INFO = 'GAM {0} - {1} / {2} / Python {3}.{4}.{5} {6} / {7} {8} /'.format(gam_version, GAM_URL,
-                                                                             gam_author,
-                                                                             sys.version_info[0], sys.version_info[1],
-                                                                             sys.version_info[2], sys.version_info[3],
-                                                                             platform.platform(), platform.machine())
+GAM_INFO = (f'GAM {gam_version} - {GAM_URL} / {gam_author} / '
+            f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
+            f'{platform.platform()} {platform.machine()}')
 
 GAM_RELEASES = 'https://github.com/jay0lee/GAM/releases'
 GAM_WIKI = 'https://github.com/jay0lee/GAM/wiki'
 GAM_ALL_RELEASES = 'https://api.github.com/repos/jay0lee/GAM/releases'
 GAM_LATEST_RELEASE = GAM_ALL_RELEASES+'/latest'
-GAM_PROJECT_FILEPATH = 'https://raw.githubusercontent.com/jay0lee/GAM/master/'
+GAM_PROJECT_FILEPATH = 'https://raw.githubusercontent.com/jay0lee/GAM/master/src/'
 
 true_values = ['on', 'yes', 'enabled', 'true', '1']
 false_values = ['off', 'no', 'disabled', 'false', '0']
@@ -49,7 +47,7 @@ SKUS = {
   '1010310002': {
     'product': '101031', 'aliases': ['gsefe', 'e4e', 'gsuiteenterpriseeducation'], 'displayName': 'G Suite Enterprise for Education'},
   '1010310003': {
-    'product': '101031', 'aliases': ['gsefes', 'e4es', 'gsuiteenterpriseeducationstudent'], 'displayName': 'G Suite Enterprise for Education Student'},
+    'product': '101031', 'aliases': ['gsefes', 'e4es', 'gsuiteenterpriseeducationstudent'], 'displayName': 'G Suite Enterprise for Education (Student)'},
   '1010330003': {
     'product': '101033', 'aliases': ['gvstarter', 'voicestarter', 'googlevoicestarter'], 'displayName': 'Google Voice Starter'},
   '1010330004': {
@@ -107,7 +105,6 @@ SKUS = {
 PRODUCTID_NAME_MAPPINGS = {
   '101001': 'Cloud Identity Free',
   '101005': 'Cloud Identity Premium',
-  '101006': 'Drive Enterprise',
   '101031': 'G Suite Enterprise for Education',
   '101033': 'Google Voice',
   '101034': 'G Suite Archived',
@@ -139,6 +136,8 @@ API_VER_MAPPING = {
   'calendar': 'v3',
   'classroom': 'v1',
   'cloudprint': 'v2',
+  'cloudresourcemanager': 'v2',
+  'cloudresourcemanagerv1': 'v1',
   'datatransfer': 'datatransfer_v1',
   'directory': 'directory_v1',
   'drive': 'v2',
@@ -146,11 +145,13 @@ API_VER_MAPPING = {
   'gmail': 'v1',
   'groupssettings': 'v1',
   'iam': 'v1',
+  'iap': 'v1',
   'licensing': 'v1',
   'oauth2': 'v2',
   'pubsub': 'v1',
   'reports': 'reports_v1',
   'reseller': 'v1',
+  'servicemanagement': 'v1',
   'sheets': 'v4',
   'siteVerification': 'v1',
   'storage': 'v1',
@@ -626,8 +627,6 @@ CROS_SCALAR_PROPERTY_PRINT_ORDER = [
   'manufactureDate',
   'supportEndDate',
   'autoUpdateExpiration',
-  'guessedAUEDate',
-  'guessedAUEModel',
   'tpmVersionInfo',
   'willAutoRenew',
   ]
@@ -726,6 +725,8 @@ GROUP_SETTINGS_BOOLEAN_ATTRIBUTES = set([
 GM_SYSEXITRC = 'sxrc'
 # Path to gam
 GM_GAM_PATH = 'gpth'
+# Python source, PyInstaller or StaticX?
+GM_GAM_TYPE = 'gtyp'
 # Are we on Windows?
 GM_WINDOWS = 'wndo'
 # Encodings
@@ -769,6 +770,7 @@ _FN_OAUTH2_TXT = 'oauth2.txt'
 GM_Globals = {
   GM_SYSEXITRC: 0,
   GM_GAM_PATH: None,
+  GM_GAM_TYPE: None,
   GM_WINDOWS: os.name == 'nt',
   GM_SYS_ENCODING: _DEFAULT_CHARSET,
   GM_EXTRA_ARGS_DICT:  {'prettyPrint': False},
@@ -841,6 +843,8 @@ GC_SHOW_GETTINGS = 'show_gettings'
 GC_SITE_DIR = 'site_dir'
 # CSV Columns GAM should show on CSV output
 GC_CSV_HEADER_FILTER = 'csv_header_filter'
+# CSV Columns GAM should not show on CSV output
+GC_CSV_HEADER_DROP_FILTER = 'csv_header_drop_filter'
 # CSV Rows GAM should filter
 GC_CSV_ROW_FILTER = 'csv_row_filter'
 # Minimum TLS Version required for HTTPS connections
@@ -876,6 +880,7 @@ GC_Defaults = {
   GC_SHOW_GETTINGS: True,
   GC_SITE_DIR: '',
   GC_CSV_HEADER_FILTER: '',
+  GC_CSV_HEADER_DROP_FILTER: '',
   GC_CSV_ROW_FILTER: '',
   GC_TLS_MIN_VERSION: tls_min,
   GC_TLS_MAX_VERSION: None,
@@ -923,6 +928,7 @@ GC_VAR_INFO = {
   GC_SHOW_GETTINGS: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_SITE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
   GC_CSV_HEADER_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
+  GC_CSV_HEADER_DROP_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
   GC_CSV_ROW_FILTER: {GC_VAR_TYPE: GC_TYPE_ROWFILTER},
   GC_TLS_MIN_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_TLS_MAX_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},
@@ -941,22 +947,18 @@ CLEAR_NONE_ARGUMENT = ['clear', 'none',]
 #
 MESSAGE_API_ACCESS_CONFIG = 'API access is configured in your Control Panel under: Security-Show more-Advanced settings-Manage API client access'
 MESSAGE_API_ACCESS_DENIED = 'API access Denied.\n\nPlease make sure the Client ID: {0} is authorized for the API Scope(s): {1}'
-MESSAGE_CONSOLE_AUTHORIZATION_PROMPT = '\nGo to the following link in your browser:\n\n\t{url}\n'
-MESSAGE_CONSOLE_AUTHORIZATION_CODE = 'Enter verification code: '
 MESSAGE_GAM_EXITING_FOR_UPDATE = 'GAM is now exiting so that you can overwrite this old version with the latest release'
 MESSAGE_GAM_OUT_OF_MEMORY = 'GAM has run out of memory. If this is a large G Suite instance, you should use a 64-bit version of GAM on Windows or a 64-bit version of Python on other systems.'
 MESSAGE_HEADER_NOT_FOUND_IN_CSV_HEADERS = 'Header "{0}" not found in CSV headers of "{1}".'
 MESSAGE_HIT_CONTROL_C_TO_UPDATE = '\n\nHit CTRL+C to visit the GAM website and download the latest release or wait 15 seconds continue with this boring old version. GAM won\'t bother you with this announcement for 1 week or you can create a file named noupdatecheck.txt in the same location as gam.py or gam.exe and GAM won\'t ever check for updates.'
 MESSAGE_INVALID_JSON = 'The file {0} has an invalid format.'
-MESSAGE_LOCAL_SERVER_AUTHORIZATION_PROMPT = '\nYour browser has been opened to visit:\n\n\t{url}\n\nIf your browser is on a different machine then press CTRL+C and create a file called nobrowser.txt in the same folder as GAM.\n'
-MESSAGE_LOCAL_SERVER_SUCCESS = 'The authentication flow has completed. You may close this browser window and return to GAM.'
 MESSAGE_NO_DISCOVERY_INFORMATION = 'No online discovery doc and {0} does not exist locally'
 MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = 'Cowardly refusing to perform migration due to lack of target drive space. Source size: {0}mb Target Free: {1}mb'
 MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = 'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.'
 MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}. Please make sure service is enabled for user and run\n\ngam user <user> check serviceaccount\n\nfor further instructions'
 MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON = 'Please run\n\ngam create project\ngam user <user> check serviceaccount\n\nto create and configure a service account.'
 MESSAGE_UPDATE_GAM_TO_64BIT = "You're running a 32-bit version of GAM on a 64-bit version of Windows, upgrade to a windows-x86_64 version of GAM"
-MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from Google by %s'
+MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from %s by %s'
 
 USER_ADDRESS_TYPES = ['home', 'work', 'other']
 USER_EMAIL_TYPES = ['home', 'work', 'other']
@@ -1183,3 +1185,22 @@ MAX_RESULTS_API_EXCEPTIONS = {
   'directory.chromeosdevices.list': 200,
   'drive.files.list': 1000,
   }
+
+ONE_KILO_BYTES = 1000
+ONE_MEGA_BYTES = 1000000
+ONE_GIGA_BYTES = 1000000000
+
+DELTA_DATE_PATTERN = re.compile(r'^([+-])(\d+)([dwy])$')
+DELTA_DATE_FORMAT_REQUIRED = '(+|-)<Number>(d|w|y)'
+
+DELTA_TIME_PATTERN = re.compile(r'^([+-])(\d+)([mhdwy])$')
+DELTA_TIME_FORMAT_REQUIRED = '(+|-)<Number>(m|h|d|w|y)'
+
+YYYYMMDD_FORMAT = '%Y-%m-%d'
+YYYYMMDD_FORMAT_REQUIRED = 'yyyy-mm-dd'
+
+YYYYMMDDTHHMMSS_FORMAT_REQUIRED = 'yyyy-mm-ddThh:mm:ss[.fff](Z|(+|-(hh:mm)))'
+
+YYYYMMDD_PATTERN = re.compile(r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$')
+
+UID_PATTERN = re.compile(r'u?id: ?(.+)', re.IGNORECASE)

src/windows-gam.spec

@@ -1,33 +0,0 @@
-# -*- mode: python -*-
-import sys
-
-sys.modules['FixTk'] = None
-
-a = Analysis(['gam.py'],
-             pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
-             hiddenimports=[],
-             hookspath=None,
-             excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]:
-        a.datas.remove(d)
-        break
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-
-# dynamically determine where httplib2/cacerts.txt lives
-import importlib
-proot = os.path.dirname(importlib.import_module('httplib2').__file__)
-a.datas += [('httplib2/cacerts.txt', os.path.join(proot, 'cacerts.txt'), 'DATA')]
-
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam.exe',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )